Reorganized library of lambdas for serverless cloud RAG pipeline

ingest.js

@@ -0,0 +1,41 @@
+import { S3Client, GetObjectCommand } from "@aws-sdk/client-s3";
+import { BedrockRuntimeClient, InvokeModelCommand } from "@aws-sdk/client-bedrock-runtime";
+import { Pinecone } from "@pinecone-database/pinecone";
+
+const s3 = new S3Client({});
+const bedrock = new BedrockRuntimeClient({});
+const pc = new Pinecone({ apiKey: process.env.PINECONE_API_KEY });
+
+export const handler = async (event) => {
+    const bucket = event.Records[0].s3.bucket.name;
+    const key = decodeURIComponent(event.Records[0].s3.object.key.replace(/\+/g, " "));
+
+    const s3Response = await s3.send(new GetObjectCommand({ Bucket: bucket, Key: key }));
+    const rawText = await s3Response.Body.transformToString();
+
+    const chunks = rawText.match(/[\s\S]{1,500}/g) || [];
+
+    const index = pc.index(process.env.PINECONE_INDEX_NAME);
+
+    for (let i = 0; i < chunks.length; i++) {
+        const chunk = chunks[i];
+
+        const bedrockResponse = await bedrock.send(new InvokeModelCommand({
+            modelId: "amazon.titan-embed-text-v1",
+            contentType: "application/json",
+            accept: "application/json",
+            body: JSON.stringify({ inputText: chunk })
+        }));
+
+        const { embedding } = JSON.parse(new TextDecoder().decode(bedrockResponse.body));
+
+        // 4. Upsert into Vector Database
+        await index.upsert([{
+            id: `${key}_chunk_${i}`,
+            values: embedding,
+            metadata: { text: chunk, source: key }
+        }]);
+    }
+
+    return { status: "Success", processedChunks: chunks.length };
+};

query.js

@@ -0,0 +1,52 @@
+import { BedrockRuntimeClient, InvokeModelCommand } from "@aws-sdk/client-bedrock-runtime";
+import { Pinecone } from "@pinecone-database/pinecone";
+
+const bedrock = new BedrockRuntimeClient({});
+const pc = new Pinecone({ apiKey: process.env.PINECONE_API_KEY });
+
+export const handler = async (event) => {
+    const { question } = JSON.parse(event.body);
+
+    // 1. Embed user query
+    const embedResponse = await bedrock.send(new InvokeModelCommand({
+        modelId: "amazon.titan-embed-text-v1",
+        contentType: "application/json",
+        accept: "application/json",
+        body: JSON.stringify({ inputText: question })
+    }));
+    const { embedding } = JSON.parse(new TextDecoder().decode(embedResponse.body));
+
+    // 2. Query Vector DB for relevant context
+    const index = pc.index(process.env.PINECONE_INDEX_NAME);
+    const queryResponse = await index.query({
+        vector: embedding,
+        topK: 3,
+        includeMetadata: true
+    });
+
+    const context = queryResponse.matches.map(match => match.metadata.text).join("\n\n");
+
+    const systemPrompt = `Use the following context to answer the question. If you don't know, say you don't know.\n\nContext:\n${context}`;
+
+    const llmResponse = await bedrock.send(new InvokeModelCommand({
+        modelId: "anthropic.claude-3-haiku-20240307-v1:0", // Fast & Cost-effective
+        contentType: "application/json",
+        accept: "application/json",
+        body: JSON.stringify({
+            anthropic_version: "bedrock-2023-05-31",
+            max_tokens: 500,
+            messages: [
+                { role: "user", content: `${systemPrompt}\n\nQuestion: ${question}` }
+            ]
+        })
+    }));
+
+    const result = JSON.parse(new TextDecoder().decode(llmResponse.body));
+    const answer = result.content[0].text;
+
+    return {
+        statusCode: 200,
+        headers: { "Content-Type": "application/json" },
+        body: JSON.stringify({ answer, contextUsed: queryResponse.matches.length })
+    };
+};