Add shared library for APIs/chunking

ingest.js

@@ -1,10 +1,11 @@
 import { S3Client, GetObjectCommand } from "@aws-sdk/client-s3";
-import { BedrockRuntimeClient, InvokeModelCommand } from "@aws-sdk/client-bedrock-runtime";
 import { Pinecone } from "@pinecone-database/pinecone";
+import { chunkText } from "./lib/chunk.js";
+import { embedTexts } from "./lib/voyage.js";
 
 const s3 = new S3Client({});
-const bedrock = new BedrockRuntimeClient({});
 const pc = new Pinecone({ apiKey: process.env.PINECONE_API_KEY });
+const UPSERT_BATCH_SIZE = 100;
 
 export const handler = async (event) => {
     const bucket = event.Records[0].s3.bucket.name;
@@ -13,28 +14,22 @@ export const handler = async (event) => {
     const s3Response = await s3.send(new GetObjectCommand({ Bucket: bucket, Key: key }));
     const rawText = await s3Response.Body.transformToString();
 
-    const chunks = rawText.match(/[\s\S]{1,500}/g) || [];
+    const chunks = chunkText(rawText);
+    if (chunks.length === 0) {
+        return { status: "Success", processedChunks: 0 };
+    }
 
+    const embeddings = await embedTexts(chunks, "document");
     const index = pc.index(process.env.PINECONE_INDEX_NAME);
 
-    for (let i = 0; i < chunks.length; i++) {
-        const chunk = chunks[i];
+    const vectors = chunks.map((chunk, i) => ({
+        id: `${key}_chunk_${i}`,
+        values: embeddings[i],
+        metadata: { text: chunk, source: key, chunkIndex: i },
+    }));
 
-        const bedrockResponse = await bedrock.send(new InvokeModelCommand({
-            modelId: "amazon.titan-embed-text-v1",
-            contentType: "application/json",
-            accept: "application/json",
-            body: JSON.stringify({ inputText: chunk })
-        }));
-
-        const { embedding } = JSON.parse(new TextDecoder().decode(bedrockResponse.body));
-
-        // 4. Upsert into Vector Database
-        await index.upsert([{
-            id: `${key}_chunk_${i}`,
-            values: embedding,
-            metadata: { text: chunk, source: key }
-        }]);
+    for (let i = 0; i < vectors.length; i += UPSERT_BATCH_SIZE) {
+        await index.upsert(vectors.slice(i, i + UPSERT_BATCH_SIZE));
     }
 
     return { status: "Success", processedChunks: chunks.length };