Add nonblocking/asyn I/O operations

backend/services/clustering.service.js

@@ -1,4 +1,6 @@
 import Anthropic from "@anthropic-ai/sdk";
+import { pipeline } from "node:stream/promises";
+import { Transform, Writable } from "node:stream";
 import { embedNotes } from "./embedding.service.js";
 import { validateStructure, computeCohesionScore } from "./validation.service.js";
 
@@ -33,31 +35,71 @@ Here are the notes:
 ${notesJson}`;
 };
 
-const requestClusters = async (notes) => {
-  const response = await client.messages.create({
-    model: "claude-sonnet-4-20250514",
-    max_tokens: 4096,
-    messages: [
-      { role: "user", content: buildPrompt(notes) },
-    ],
-  });
+const textDeltas = () => new Transform({
+  objectMode: true,
+  transform(event, _encoding, callback) {
+    if (event?.type === 'content_block_delta' && event.delta?.type === 'text_delta') {
+      callback(null, event.delta.text);
+      return;
+    }
+    callback();
+  },
+});
 
-  const textBlock = response?.content?.[0];
+// Rejects as soon as the first non-whitespace character proves the response
+// is not the JSON array we asked for, rather than after the full generation.
+const collectClusterJson = (sink) => new Writable({
+  objectMode: true,
+  write(text, _encoding, callback) {
+    if (!sink.sawOpeningBracket) {
+      const leading = (sink.parts.join('') + text).trimStart();
+      if (leading.length > 0) {
+        if (!leading.startsWith('[')) {
+          callback(new Error('LLM API returned non-JSON response'));
+          return;
+        }
+        sink.sawOpeningBracket = true;
+      }
+    }
+    sink.parts.push(text);
+    callback();
+  },
+});
 
-  if (!textBlock || textBlock.type !== 'text' || typeof textBlock.text !== 'string') {
+const requestClusters = async (notes, signal) => {
+  const sink = { parts: [], sawOpeningBracket: false };
+
+  const options = signal ? [{ signal }] : [];
+
+  const events = client.messages.stream(
+    {
+      model: "claude-sonnet-4-20250514",
+      max_tokens: 4096,
+      messages: [
+        { role: "user", content: buildPrompt(notes) },
+      ],
+    },
+    ...options
+  );
+
+  await pipeline(events, textDeltas(), collectClusterJson(sink), ...options);
+
+  const text = sink.parts.join('');
+
+  if (text.trim().length === 0) {
     throw new Error('Unexpected response from LLM API: no text content returned');
   }
 
   try {
-    return JSON.parse(textBlock.text);
+    return JSON.parse(text);
   } catch {
     throw new Error('LLM API returned non-JSON response');
   }
 };
 
-export const clusterNotes = async (notes) => {
+export const clusterNotes = async (notes, { signal } = {}) => {
   const [clusters, embeddingMap] = await Promise.all([
-    requestClusters(notes),
+    requestClusters(notes, signal),
     embedNotes(notes),
   ]);
 

backend/services/embedding.service.js

@@ -1,25 +1,42 @@
 import { VoyageAIClient } from "voyageai";
+import { pipeline } from "node:stream/promises";
+import { Readable } from "node:stream";
+import { batch } from "../lib/streams.js";
 
 const client = new VoyageAIClient({
   apiKey: process.env.VOYAGEAI_API_KEY,
 });
 
+// Well under Voyage's per-request input and token ceilings.
+const EMBED_BATCH_SIZE = 128;
+
 /**
  * @param {Array<{id: string, text: string}>} notes
  * @returns {Promise<Map<string, number[]>>} noteId → embedding vector
  */
 export const embedNotes = async (notes) => {
-  const texts = notes.map((n) => n.text);
-
-  const response = await client.embed({
-    input: texts,
-    model: "voyage-3",
-  });
-
   const embeddingMap = new Map();
-  response.data.forEach((item, i) => {
-    embeddingMap.set(notes[i].id, item.embedding);
-  });
+
+  if (!notes || notes.length === 0) {
+    return embeddingMap;
+  }
+
+  await pipeline(
+    Readable.from(notes, { objectMode: true }),
+    batch(EMBED_BATCH_SIZE),
+    async (batches) => {
+      for await (const chunk of batches) {
+        const response = await client.embed({
+          input: chunk.map((n) => n.text),
+          model: "voyage-3",
+        });
+
+        response.data.forEach((item, i) => {
+          embeddingMap.set(chunk[i].id, item.embedding);
+        });
+      }
+    }
+  );
 
   return embeddingMap;
 };