42 lines
1.5 KiB
JavaScript
42 lines
1.5 KiB
JavaScript
import { S3Client, GetObjectCommand } from "@aws-sdk/client-s3";
|
|
import { BedrockRuntimeClient, InvokeModelCommand } from "@aws-sdk/client-bedrock-runtime";
|
|
import { Pinecone } from "@pinecone-database/pinecone";
|
|
|
|
const s3 = new S3Client({});
|
|
const bedrock = new BedrockRuntimeClient({});
|
|
const pc = new Pinecone({ apiKey: process.env.PINECONE_API_KEY });
|
|
|
|
export const handler = async (event) => {
|
|
const bucket = event.Records[0].s3.bucket.name;
|
|
const key = decodeURIComponent(event.Records[0].s3.object.key.replace(/\+/g, " "));
|
|
|
|
const s3Response = await s3.send(new GetObjectCommand({ Bucket: bucket, Key: key }));
|
|
const rawText = await s3Response.Body.transformToString();
|
|
|
|
const chunks = rawText.match(/[\s\S]{1,500}/g) || [];
|
|
|
|
const index = pc.index(process.env.PINECONE_INDEX_NAME);
|
|
|
|
for (let i = 0; i < chunks.length; i++) {
|
|
const chunk = chunks[i];
|
|
|
|
const bedrockResponse = await bedrock.send(new InvokeModelCommand({
|
|
modelId: "amazon.titan-embed-text-v1",
|
|
contentType: "application/json",
|
|
accept: "application/json",
|
|
body: JSON.stringify({ inputText: chunk })
|
|
}));
|
|
|
|
const { embedding } = JSON.parse(new TextDecoder().decode(bedrockResponse.body));
|
|
|
|
// 4. Upsert into Vector Database
|
|
await index.upsert([{
|
|
id: `${key}_chunk_${i}`,
|
|
values: embedding,
|
|
metadata: { text: chunk, source: key }
|
|
}]);
|
|
}
|
|
|
|
return { status: "Success", processedChunks: chunks.length };
|
|
};
|