This commit is contained in:
KS Jannette
2026-08-01 01:42:43 -04:00
parent 62477d009c
commit 7b47e46852
3 changed files with 181 additions and 184 deletions

View File

@@ -4,10 +4,9 @@ import { Readable } from "node:stream";
import { batch } from "../lib/streams.js";
const client = new VoyageAIClient({
apiKey: process.env.VOYAGEAI_API_KEY,
apiKey: process.env.VOYAGEAI_API_KEY,
});
// Well under Voyage's per-request input and token ceilings.
const EMBED_BATCH_SIZE = 128;
/**
@@ -15,28 +14,28 @@ const EMBED_BATCH_SIZE = 128;
* @returns {Promise<Map<string, number[]>>} noteId → embedding vector
*/
export const embedNotes = async (notes) => {
const embeddingMap = new Map();
const embeddingMap = new Map();
if (!notes || notes.length === 0) {
return embeddingMap;
}
await pipeline(
Readable.from(notes, { objectMode: true }),
batch(EMBED_BATCH_SIZE),
async (batches) => {
for await (const chunk of batches) {
const response = await client.embed({
input: chunk.map((n) => n.text),
model: "voyage-3",
});
response.data.forEach((item, i) => {
embeddingMap.set(chunk[i].id, item.embedding);
});
}
if (!notes || notes.length === 0) {
return embeddingMap;
}
);
return embeddingMap;
await pipeline(
Readable.from(notes, { objectMode: true }),
batch(EMBED_BATCH_SIZE),
async (batches) => {
for await (const chunk of batches) {
const response = await client.embed({
input: chunk.map((n) => n.text),
model: "voyage-3",
});
response.data.forEach((item, i) => {
embeddingMap.set(chunk[i].id, item.embedding);
});
}
}
);
return embeddingMap;
};