33 lines
947 B
JavaScript
33 lines
947 B
JavaScript
const CHUNK_SIZE = 2000;
|
|
const CHUNK_OVERLAP = 200;
|
|
|
|
export function chunkText(text) {
|
|
const cleaned = String(text || "").replace(/\r\n/g, "\n").trim();
|
|
if (!cleaned) return [];
|
|
if (cleaned.length <= CHUNK_SIZE) return [cleaned];
|
|
|
|
const chunks = [];
|
|
let start = 0;
|
|
|
|
while (start < cleaned.length) {
|
|
let end = Math.min(start + CHUNK_SIZE, cleaned.length);
|
|
|
|
if (end < cleaned.length) {
|
|
const slice = cleaned.slice(start, end);
|
|
const lastBreak = Math.max(slice.lastIndexOf("\n"), slice.lastIndexOf(" "));
|
|
if (lastBreak > CHUNK_SIZE * 0.5) {
|
|
end = start + lastBreak;
|
|
}
|
|
}
|
|
|
|
const chunk = cleaned.slice(start, end).trim();
|
|
if (chunk) chunks.push(chunk);
|
|
if (end >= cleaned.length) break;
|
|
|
|
const nextStart = end - CHUNK_OVERLAP;
|
|
start = nextStart <= start ? end : nextStart;
|
|
}
|
|
|
|
return chunks;
|
|
}
|