Add shared library for APIs/chunking
This commit is contained in:
27
lib/chunk.test.js
Normal file
27
lib/chunk.test.js
Normal file
@@ -0,0 +1,27 @@
|
||||
import { describe, it } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { chunkText } from "./chunk.js";
|
||||
|
||||
describe("chunkText", () => {
|
||||
it("returns empty array for blank input", () => {
|
||||
assert.deepEqual(chunkText(""), []);
|
||||
assert.deepEqual(chunkText(" "), []);
|
||||
});
|
||||
|
||||
it("keeps short text as a single chunk", () => {
|
||||
const text = "A short paragraph.";
|
||||
assert.deepEqual(chunkText(text), [text]);
|
||||
});
|
||||
|
||||
it("splits long text into overlapping chunks under 2000 chars", () => {
|
||||
const word = "word ";
|
||||
const text = word.repeat(900);
|
||||
const chunks = chunkText(text);
|
||||
assert.ok(chunks.length > 1);
|
||||
for (const chunk of chunks) {
|
||||
assert.ok(chunk.length <= 2000);
|
||||
}
|
||||
const overlap = chunks[0].slice(-50);
|
||||
assert.ok(chunks[1].includes(overlap.trim().slice(0, 20)));
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user