Infrastructure build to support third-party app integrations

backend/routes/ingest.routes.ts

@@ -0,0 +1,48 @@
+import { Router, type Request, type Response } from 'express';
+import { requireApiKey } from '../middleware/apiKey.js';
+import { createNotes } from '../db/notes.dao.js';
+import { isNoteInputArray } from '../config/normalizers.js';
+import type { NoteInput } from '../types/domain.js';
+
+const router = Router();
+
+const MAX_BATCH = 5000;
+
+type IngestBody = { notes: NoteInput[] };
+
+const isIngestBody = (value: unknown): value is IngestBody => {
+  if (typeof value !== 'object' || value === null) return false;
+  const body = value as Record<string, unknown>;
+  return isNoteInputArray(body.notes);
+};
+
+router.post('/', requireApiKey, async (req: Request, res: Response) => {
+  try {
+    if (!isIngestBody(req.body)) {
+      res.status(400).json({
+        error: 'Body must be { notes: [{ id, text, author, ... }] }',
+      });
+      return;
+    }
+
+    const { notes } = req.body;
+
+    if (notes.length === 0) {
+      res.status(400).json({ error: 'notes must not be empty' });
+      return;
+    }
+
+    if (notes.length > MAX_BATCH) {
+      res.status(400).json({ error: `notes exceeds the ${MAX_BATCH} per-request limit` });
+      return;
+    }
+
+    const inserted = await createNotes(notes);
+    res.status(201).json({ inserted: inserted.length, notes: inserted });
+  } catch (err) {
+    console.error(`Ingest failed: ${err}`);
+    res.status(500).json({ error: 'Ingest failed' });
+  }
+});
+
+export default router;

backend/routes/notes.routes.ts

@@ -1,4 +1,4 @@
-import { Router } from 'express';
+import { Router, type Request, type Response } from 'express';
 import { pipeline } from 'node:stream/promises';
 import { getAllNotes, streamAllNotes } from '../db/notes.dao.js';
 import { clusterNotes } from '../services/clustering.service.js';
@@ -6,7 +6,7 @@ import { jsonArray } from '../lib/streams.js';
 
 const router = Router();
 
-router.get('/', async (req, res) => {
+router.get('/', async (_req: Request, res: Response) => {
   try {
     const rows = await streamAllNotes();
     res.type('application/json');
@@ -14,14 +14,14 @@ router.get('/', async (req, res) => {
   } catch (err) {
     console.error(`Error loading notes: ${err}`);
     if (res.headersSent) {
-      res.destroy(err);
+      res.destroy(err as Error);
       return;
     }
     res.status(500).json({ error: 'Failed to load notes' });
   }
 });
 
-router.post('/cluster', async (req, res) => {
+router.post('/cluster', async (_req: Request, res: Response) => {
   const controller = new AbortController();
   res.on('close', () => {
     if (!res.writableEnded) controller.abort();

backend/routes/webhooks.routes.ts

@@ -0,0 +1,113 @@
+import { Router, type NextFunction, type Request, type Response } from 'express';
+import { getProvider } from '../config/providers.js';
+import { verifySignature } from '../lib/signatures.js';
+import { enqueue } from '../lib/queue.js';
+import { normalize, NormalizationError } from '../services/normalize.service.js';
+import {
+  markDone,
+  markFailed,
+  markProcessing,
+  recordDelivery,
+} from '../db/ingest_events.dao.js';
+import { createNotes } from '../db/notes.dao.js';
+import { isProviderSlug, type NormalizedDelivery } from '../types/integration.js';
+
+const router = Router();
+
+const parseJson = (raw: Buffer): unknown => {
+  if (raw.length === 0) return {};
+  return JSON.parse(raw.toString('utf8'));
+};
+
+router.post('/:provider', async (req: Request, res: Response, next: NextFunction) => {
+  try {
+    const slug = req.params.provider;
+
+    if (!isProviderSlug(slug)) {
+      res.status(404).json({ error: `Unknown provider "${slug}"` });
+      return;
+    }
+
+    const config = getProvider(slug);
+    const rawBody = Buffer.isBuffer(req.body) ? req.body : Buffer.alloc(0);
+
+    let payload: unknown;
+    try {
+      payload = parseJson(rawBody);
+    } catch {
+      res.status(400).json({ error: 'Body is not valid JSON' });
+      return;
+    }
+
+    const challenge = config.challenge?.(payload);
+    if (challenge) {
+      res.status(challenge.status).json(challenge.body);
+      return;
+    }
+
+    if (config.signatureScheme !== 'none') {
+      const secret = config.secretEnvVar ? process.env[config.secretEnvVar] : undefined;
+      if (!secret) {
+        console.error(`No signing secret configured for provider "${slug}"`);
+        res.status(500).json({ error: 'Provider is not configured' });
+        return;
+      }
+
+      const result = verifySignature({
+        provider: slug,
+        rawBody,
+        headers: req.headers,
+        secret,
+      });
+
+      if (!result.ok) {
+        res.status(401).json({ error: 'Invalid signature' });
+        return;
+      }
+    }
+
+    if (!config.normalize) {
+      res.status(501).json({ error: `No ingestion mapping for provider "${slug}"` });
+      return;
+    }
+
+    // Normalizing before the ack costs nothing (it is pure) and lets a malformed
+    // payload fail loudly here rather than silently in a background job.
+    let delivery: NormalizedDelivery;
+    try {
+      delivery = normalize(slug, payload);
+    } catch (err) {
+      if (!(err instanceof NormalizationError)) throw err;
+      res.status(400).json({ error: err.message });
+      return;
+    }
+
+    const eventId = await recordDelivery({
+      provider: slug,
+      externalId: delivery.externalId,
+    });
+
+    if (eventId === null) {
+      res.status(200).json({ duplicate: true });
+      return;
+    }
+
+    const { notes } = delivery;
+    enqueue(`${slug}:${delivery.externalId}`, async () => {
+      await markProcessing(eventId);
+      try {
+        await createNotes(notes);
+        await markDone(eventId);
+      } catch (err) {
+        await markFailed(eventId, err instanceof Error ? err.message : String(err));
+        throw err;
+      }
+    });
+
+    res.status(200).json({ accepted: true, notes: notes.length });
+  } catch (err) {
+    next(err);
+  }
+});
+
+export default router;