Structural boundaries, exact reassembly, and source offsets
The policy implementation follows.
ts
TextBlocks = { split: splitTextBlocks } as const
export function splitTextBlocks(text: string, options: TextBlockOptions): TextBlock[] {
breakWhile(
typeof text !== 'string' ||
!options ||
!Number.isSafeInteger(options.targetChars) ||
options.targetChars < 1,
'Text block splitting requires text and a positive integer target size',
);
const targetChars = options.targetChars;
const blocks: TextBlock[] = [];
let buffer = '';
let bufferStart = 0;
let cursor = 0;
const flush = () => {
if (!buffer) return;
const previous = blocks.at(-1);
if (!buffer.trim() && previous) {
// Whitespace-only blocks carry no section, so they stay attached to the previous block
// instead of becoming a block the reader has to page through.
blocks[blocks.length - 1] = {
...previous,
text: previous.text + buffer,
end: previous.end + buffer.length,
};
} else {
blocks.push({ number: blocks.length + 1, text: buffer, start: bufferStart, end: cursor });
}
buffer = '';
};
// Keeping a line's trailing newline makes every block reassemble into the original text,
// including CRLF input.
for (const line of text.match(/[^\n]*\n|[^\n]+$/g) ?? []) {
if (line.length > targetChars) {
flush();
blocks.push({
number: blocks.length + 1,
text: line,
start: cursor,
end: cursor + line.length,
});
cursor += line.length;
continue;
}
if (headingLine(line)) flush();
if (buffer.length + line.length > targetChars) flush();
if (!buffer) bufferStart = cursor;
buffer += line;
cursor += line.length;
if ((!line.trim() || headingLine(line)) && buffer.length >= targetChars / 2) flush();
else if (buffer.length >= targetChars) flush();
}
flush();
return blocks;
}