Infrastructure build to support third-party app integrations

backend/services/clustering.service.ts

@@ -1,14 +1,21 @@
 import Anthropic from "@anthropic-ai/sdk";
 import { pipeline } from "node:stream/promises";
-import { Transform, Writable } from "node:stream";
+import { Transform, Writable, type TransformCallback } from "node:stream";
 import { embedNotes } from "./embedding.service.js";
 import { validateStructure, computeCohesionScore } from "./validation.service.js";
+import { isClusterArray, type Cluster, type ClusterResponse } from "../types/domain.js";
 
 const client = new Anthropic({
   apiKey: process.env.ANTHROPIC_API_KEY,
 });
 
-const buildPrompt = (notes) => {
+type ClusterableNote = { id: string; text: string };
+
+type ClusterOptions = { signal?: AbortSignal };
+
+type JsonSink = { parts: string[]; sawOpeningBracket: boolean };
+
+const buildPrompt = (notes: ClusterableNote[]): string => {
   const notesJson = JSON.stringify(notes, null, 2);
 
   return `You are an expert at analyzing text for semantic similarity and thematic patterns.
@@ -35,10 +42,20 @@ Here are the notes:
 ${notesJson}`;
 };
 
-const textDeltas = () => new Transform({
+const isTextDelta = (event: unknown): event is { delta: { text: string } } => {
+  if (typeof event !== 'object' || event === null) return false;
+  const candidate = event as { type?: unknown; delta?: { type?: unknown; text?: unknown } };
+  return (
+    candidate.type === 'content_block_delta' &&
+    candidate.delta?.type === 'text_delta' &&
+    typeof candidate.delta.text === 'string'
+  );
+};
+
+const textDeltas = (): Transform => new Transform({
   objectMode: true,
-  transform(event, _encoding, callback) {
-    if (event?.type === 'content_block_delta' && event.delta?.type === 'text_delta') {
+  transform(event: unknown, _encoding: BufferEncoding, callback: TransformCallback) {
+    if (isTextDelta(event)) {
       callback(null, event.delta.text);
       return;
     }
@@ -48,9 +65,9 @@ const textDeltas = () => new Transform({
 
 // Rejects as soon as the first non-whitespace character proves the response
 // is not the JSON array we asked for, rather than after the full generation.
-const collectClusterJson = (sink) => new Writable({
+const collectClusterJson = (sink: JsonSink): Writable => new Writable({
   objectMode: true,
-  write(text, _encoding, callback) {
+  write(text: string, _encoding: BufferEncoding, callback: (error?: Error | null) => void) {
     if (!sink.sawOpeningBracket) {
       const leading = (sink.parts.join('') + text).trimStart();
       if (leading.length > 0) {
@@ -66,8 +83,11 @@ const collectClusterJson = (sink) => new Writable({
   },
 });
 
-const requestClusters = async (notes, signal) => {
-  const sink = { parts: [], sawOpeningBracket: false };
+const requestClusters = async (
+  notes: ClusterableNote[],
+  signal?: AbortSignal
+): Promise<Cluster[]> => {
+  const sink: JsonSink = { parts: [], sawOpeningBracket: false };
 
   const options = signal ? [{ signal }] : [];
 
@@ -90,14 +110,24 @@ const requestClusters = async (notes, signal) => {
     throw new Error('Unexpected response from LLM API: no text content returned');
   }
 
+  let parsed: unknown;
   try {
-    return JSON.parse(text);
+    parsed = JSON.parse(text);
   } catch {
     throw new Error('LLM API returned non-JSON response');
   }
+
+  if (!isClusterArray(parsed)) {
+    throw new Error('LLM API returned clusters in an unexpected shape');
+  }
+
+  return parsed;
 };
 
-export const clusterNotes = async (notes, { signal } = {}) => {
+export const clusterNotes = async (
+  notes: ClusterableNote[],
+  { signal }: ClusterOptions = {}
+): Promise<ClusterResponse> => {
   const [clusters, embeddingMap] = await Promise.all([
     requestClusters(notes, signal),
     embedNotes(notes),

backend/services/embedding.service.ts

@@ -2,6 +2,7 @@ import { VoyageAIClient } from "voyageai";
 import { pipeline } from "node:stream/promises";
 import { Readable } from "node:stream";
 import { batch } from "../lib/streams.js";
+import type { EmbeddingMap } from "../types/domain.js";
 
 const client = new VoyageAIClient({
     apiKey: process.env.VOYAGEAI_API_KEY,
@@ -9,12 +10,13 @@ const client = new VoyageAIClient({
 
 const EMBED_BATCH_SIZE = 128;
 
+type EmbeddableNote = { id: string; text: string };
+
 /**
- * @param {Array<{id: string, text: string}>} notes
- * @returns {Promise<Map<string, number[]>>} noteId → embedding vector
+ * @returns noteId to embedding vector
  */
-export const embedNotes = async (notes) => {
-    const embeddingMap = new Map();
+export const embedNotes = async (notes: EmbeddableNote[]): Promise<EmbeddingMap> => {
+    const embeddingMap: EmbeddingMap = new Map();
 
     if (!notes || notes.length === 0) {
         return embeddingMap;
@@ -22,16 +24,18 @@ export const embedNotes = async (notes) => {
 
     await pipeline(
         Readable.from(notes, { objectMode: true }),
-        batch(EMBED_BATCH_SIZE),
-        async (batches) => {
+        batch<EmbeddableNote>(EMBED_BATCH_SIZE),
+        async (batches: AsyncIterable<EmbeddableNote[]>) => {
             for await (const chunk of batches) {
                 const response = await client.embed({
                     input: chunk.map((n) => n.text),
                     model: "voyage-3.5",
                 });
 
-                response.data.forEach((item, i) => {
-                    embeddingMap.set(chunk[i].id, item.embedding);
+                response.data?.forEach((item, i) => {
+                    if (item.embedding) {
+                        embeddingMap.set(chunk[i].id, item.embedding);
+                    }
                 });
             }
         }

backend/services/normalize.service.ts

@@ -0,0 +1,23 @@
+import { getProvider } from '../config/providers.js';
+import { NormalizationError } from '../config/normalizers.js';
+import type { NormalizedDelivery, ProviderSlug } from '../types/integration.js';
+
+export { NormalizationError };
+
+export const normalize = (
+  provider: ProviderSlug,
+  payload: unknown
+): NormalizedDelivery => {
+  const normalizer = getProvider(provider).normalize;
+
+  if (!normalizer) {
+    throw new NormalizationError(
+      `No normalizer registered for provider "${provider}"`
+    );
+  }
+
+  return normalizer(payload);
+};
+
+export const hasNormalizer = (provider: ProviderSlug): boolean =>
+  getProvider(provider).normalize !== undefined;

backend/services/oauth.service.ts

@@ -0,0 +1,182 @@
+import { requestJson } from '../lib/httpClient.js';
+import { getProvider } from '../config/providers.js';
+import {
+  getAccessToken,
+  getRefreshToken,
+  findByProviderWorkspace,
+  updateTokens,
+} from '../db/integrations.dao.js';
+import type { IntegrationRow, ProviderSlug } from '../types/integration.js';
+
+/** Refresh this far ahead of expiry so an in-flight call cannot straddle it. */
+const REFRESH_MARGIN_MS = 60_000;
+
+export type TokenSet = {
+  accessToken: string;
+  refreshToken?: string;
+  expiresAt?: Date;
+  scopes: string[];
+};
+
+type TokenResponse = {
+  access_token: string;
+  refresh_token?: string;
+  expires_in?: number;
+  scope?: string;
+};
+
+export class OAuthError extends Error {
+  constructor(message: string) {
+    super(message);
+    this.name = 'OAuthError';
+  }
+}
+
+const isTokenResponse = (value: unknown): value is TokenResponse => {
+  if (typeof value !== 'object' || value === null) return false;
+  const body = value as Record<string, unknown>;
+  if (typeof body.access_token !== 'string' || body.access_token.length === 0) return false;
+  if (body.refresh_token !== undefined && typeof body.refresh_token !== 'string') return false;
+  if (body.expires_in !== undefined && typeof body.expires_in !== 'number') return false;
+  if (body.scope !== undefined && typeof body.scope !== 'string') return false;
+  return true;
+};
+
+const credentials = (provider: ProviderSlug): { id: string; secret: string } => {
+  const prefix = provider.toUpperCase();
+  const id = process.env[`${prefix}_CLIENT_ID`];
+  const secret = process.env[`${prefix}_CLIENT_SECRET`];
+
+  if (!id || !secret) {
+    throw new OAuthError(
+      `Missing ${prefix}_CLIENT_ID or ${prefix}_CLIENT_SECRET`
+    );
+  }
+
+  return { id, secret };
+};
+
+const oauthConfig = (provider: ProviderSlug) => {
+  const config = getProvider(provider).oauth;
+  if (!config) {
+    throw new OAuthError(`Provider "${provider}" does not support OAuth`);
+  }
+  return config;
+};
+
+const toTokenSet = (body: TokenResponse): TokenSet => ({
+  accessToken: body.access_token,
+  refreshToken: body.refresh_token,
+  expiresAt: body.expires_in
+    ? new Date(Date.now() + body.expires_in * 1000)
+    : undefined,
+  scopes: body.scope ? body.scope.split(/[\s,]+/).filter(Boolean) : [],
+});
+
+export const buildAuthorizeUrl = (
+  provider: ProviderSlug,
+  input: { redirectUri: string; state: string }
+): string => {
+  const config = oauthConfig(provider);
+  const url = new URL(config.authorizeUrl);
+
+  url.searchParams.set('client_id', credentials(provider).id);
+  url.searchParams.set('redirect_uri', input.redirectUri);
+  url.searchParams.set('response_type', 'code');
+  url.searchParams.set('state', input.state);
+  url.searchParams.set('scope', config.scopes.join(' '));
+
+  return url.toString();
+};
+
+const postForm = async (
+  url: string,
+  form: Record<string, string>
+): Promise<TokenSet> => {
+  const body = await requestJson<unknown>(url, {
+    method: 'POST',
+    headers: {
+      'content-type': 'application/x-www-form-urlencoded',
+      accept: 'application/json',
+    },
+    body: new URLSearchParams(form).toString(),
+  });
+
+  if (!isTokenResponse(body)) {
+    throw new OAuthError('Token endpoint returned an unexpected payload');
+  }
+
+  return toTokenSet(body);
+};
+
+export const exchangeCode = async (
+  provider: ProviderSlug,
+  input: { code: string; redirectUri: string }
+): Promise<TokenSet> => {
+  const { id, secret } = credentials(provider);
+
+  return postForm(oauthConfig(provider).tokenUrl, {
+    grant_type: 'authorization_code',
+    code: input.code,
+    redirect_uri: input.redirectUri,
+    client_id: id,
+    client_secret: secret,
+  });
+};
+
+export const refreshAccessToken = async (
+  provider: ProviderSlug,
+  refreshToken: string
+): Promise<TokenSet> => {
+  const { id, secret } = credentials(provider);
+
+  return postForm(oauthConfig(provider).tokenUrl, {
+    grant_type: 'refresh_token',
+    refresh_token: refreshToken,
+    client_id: id,
+    client_secret: secret,
+  });
+};
+
+const needsRefresh = (integration: IntegrationRow): boolean => {
+  if (!integration.tokenExpiresAt) return false;
+  return integration.tokenExpiresAt.getTime() - Date.now() <= REFRESH_MARGIN_MS;
+};
+
+/**
+ * Resolves a usable access token, refreshing first when the stored one is at
+ * or near expiry. Throws rather than returning null so a caller cannot make an
+ * unauthenticated request by forgetting a null check.
+ */
+export const getValidAccessToken = async (
+  provider: ProviderSlug,
+  externalWorkspaceId: string
+): Promise<string> => {
+  const integration = await findByProviderWorkspace(provider, externalWorkspaceId);
+  if (!integration) {
+    throw new OAuthError(`No ${provider} integration for workspace ${externalWorkspaceId}`);
+  }
+
+  if (needsRefresh(integration)) {
+    const refreshToken = await getRefreshToken(integration.id);
+    if (!refreshToken) {
+      throw new OAuthError(`${provider} token expired and no refresh token is stored`);
+    }
+
+    const tokens = await refreshAccessToken(provider, refreshToken);
+    await updateTokens({
+      id: integration.id,
+      accessToken: tokens.accessToken,
+      refreshToken: tokens.refreshToken,
+      expiresAt: tokens.expiresAt,
+    });
+    return tokens.accessToken;
+  }
+
+  const accessToken = await getAccessToken(integration.id);
+  if (!accessToken) {
+    throw new OAuthError(`No access token stored for ${provider}`);
+  }
+
+  return accessToken;
+};

backend/services/validation.service.ts

@@ -1,18 +1,22 @@
+import type { Cluster, EmbeddingMap, ValidationResult } from '../types/domain.js';
+
 /**
  * Structural validation for LLM output
  *
- * @param {Array<{label: string, noteIds: string[]}>} clusters
- * @param {string[]} inputNoteIds - the original note IDs that were sent to the LLM
- * @returns {{valid: boolean, reasons: string[]}}
+ * @param clusters
+ * @param inputNoteIds - the original note IDs that were sent to the LLM
  */
-export const validateStructure = (clusters, inputNoteIds) => {
-    const reasons = [];
+export const validateStructure = (
+    clusters: Cluster[],
+    inputNoteIds: string[]
+): ValidationResult => {
+    const reasons: string[] = [];
 
     if (!Array.isArray(clusters) || clusters.length === 0) {
         return { valid: false, reasons: ['Response is not a non-empty array'] };
     }
 
-    const assignedIds = [];
+    const assignedIds: string[] = [];
     for (const cluster of clusters) {
         if (!cluster.label || typeof cluster.label !== 'string') {
             reasons.push(`Cluster missing a valid label`);
@@ -47,7 +51,7 @@ export const validateStructure = (clusters, inputNoteIds) => {
     return { valid: reasons.length === 0, reasons };
 };
 
-const cosineSimilarity = (a, b) => {
+const cosineSimilarity = (a: number[], b: number[]): number => {
     let dot = 0;
     let magA = 0;
     let magB = 0;
@@ -67,14 +71,17 @@ const cosineSimilarity = (a, b) => {
  * versus the nearest neighboring cluster. Returns a score in [-1, 1]
  * where higher is better.
  *
- * @param {Array<{label: string, noteIds: string[]}>} clusters
- * @param {Map<string, number[]>} embeddingMap - noteId → vector
- * @returns {number} average silhouette score
+ * @param clusters
+ * @param embeddingMap - noteId to vector
+ * @returns average silhouette score
  */
-export const computeCohesionScore = (clusters, embeddingMap) => {
+export const computeCohesionScore = (
+    clusters: Cluster[],
+    embeddingMap: EmbeddingMap
+): number => {
     if (clusters.length <= 1) return 1.0;
 
-    const scores = [];
+    const scores: number[] = [];
 
     for (let ci = 0; ci < clusters.length; ci++) {
         const clusterIds = clusters[ci].noteIds;