Infrastructure build to support third-party app integrations

backend/services/clustering.service.ts

@@ -0,0 +1,145 @@
+import Anthropic from "@anthropic-ai/sdk";
+import { pipeline } from "node:stream/promises";
+import { Transform, Writable, type TransformCallback } from "node:stream";
+import { embedNotes } from "./embedding.service.js";
+import { validateStructure, computeCohesionScore } from "./validation.service.js";
+import { isClusterArray, type Cluster, type ClusterResponse } from "../types/domain.js";
+
+const client = new Anthropic({
+  apiKey: process.env.ANTHROPIC_API_KEY,
+});
+
+type ClusterableNote = { id: string; text: string };
+
+type ClusterOptions = { signal?: AbortSignal };
+
+type JsonSink = { parts: string[]; sawOpeningBracket: boolean };
+
+const buildPrompt = (notes: ClusterableNote[]): string => {
+  const notesJson = JSON.stringify(notes, null, 2);
+
+  return `You are an expert at analyzing text for semantic similarity and thematic patterns.
+
+Below is a JSON array of sticky notes. Each note has an "id" and a "text" field. Analyze the "text" field of every note and group them into meaningful thematic clusters.
+
+For each cluster, return ONLY a valid JSON array with the below exact structure — no markdown, no explanation, no extra text - where the value for the "label" key is a name you create to describe the cluster's theme and the value for the "noteIds" key is an array containing the Ids of the notes that fit into that cluster theme.
+
+[
+  {
+    "label": "Short descriptive theme name for cluster",
+    "noteIds": ["note_001", "note_002"]
+  }
+]
+
+Rules:
+- Every note must appear in exactly one cluster
+- Each cluster must have a concise, descriptive label
+- Group by semantic meaning, not by keywords
+- Aim for the most natural number of groups given the data
+
+Here are the notes:
+
+${notesJson}`;
+};
+
+const isTextDelta = (event: unknown): event is { delta: { text: string } } => {
+  if (typeof event !== 'object' || event === null) return false;
+  const candidate = event as { type?: unknown; delta?: { type?: unknown; text?: unknown } };
+  return (
+    candidate.type === 'content_block_delta' &&
+    candidate.delta?.type === 'text_delta' &&
+    typeof candidate.delta.text === 'string'
+  );
+};
+
+const textDeltas = (): Transform => new Transform({
+  objectMode: true,
+  transform(event: unknown, _encoding: BufferEncoding, callback: TransformCallback) {
+    if (isTextDelta(event)) {
+      callback(null, event.delta.text);
+      return;
+    }
+    callback();
+  },
+});
+
+// Rejects as soon as the first non-whitespace character proves the response
+// is not the JSON array we asked for, rather than after the full generation.
+const collectClusterJson = (sink: JsonSink): Writable => new Writable({
+  objectMode: true,
+  write(text: string, _encoding: BufferEncoding, callback: (error?: Error | null) => void) {
+    if (!sink.sawOpeningBracket) {
+      const leading = (sink.parts.join('') + text).trimStart();
+      if (leading.length > 0) {
+        if (!leading.startsWith('[')) {
+          callback(new Error('LLM API returned non-JSON response'));
+          return;
+        }
+        sink.sawOpeningBracket = true;
+      }
+    }
+    sink.parts.push(text);
+    callback();
+  },
+});
+
+const requestClusters = async (
+  notes: ClusterableNote[],
+  signal?: AbortSignal
+): Promise<Cluster[]> => {
+  const sink: JsonSink = { parts: [], sawOpeningBracket: false };
+
+  const options = signal ? [{ signal }] : [];
+
+  const events = client.messages.stream(
+    {
+      model: "claude-sonnet-5",
+      max_tokens: 4096,
+      messages: [
+        { role: "user", content: buildPrompt(notes) },
+      ],
+    },
+    ...options
+  );
+
+  await pipeline(events, textDeltas(), collectClusterJson(sink), ...options);
+
+  const text = sink.parts.join('');
+
+  if (text.trim().length === 0) {
+    throw new Error('Unexpected response from LLM API: no text content returned');
+  }
+
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(text);
+  } catch {
+    throw new Error('LLM API returned non-JSON response');
+  }
+
+  if (!isClusterArray(parsed)) {
+    throw new Error('LLM API returned clusters in an unexpected shape');
+  }
+
+  return parsed;
+};
+
+export const clusterNotes = async (
+  notes: ClusterableNote[],
+  { signal }: ClusterOptions = {}
+): Promise<ClusterResponse> => {
+  const [clusters, embeddingMap] = await Promise.all([
+    requestClusters(notes, signal),
+    embedNotes(notes),
+  ]);
+
+  const noteIds = notes.map((n) => n.id);
+  const { valid, reasons } = validateStructure(clusters, noteIds);
+  if (!valid) {
+    throw new Error(`Cluster validation failed: ${reasons.join('; ')}`);
+  }
+
+  const score = computeCohesionScore(clusters, embeddingMap);
+
+  return { clusters, score: Math.round(score * 100) / 100 };
+};