kjannette/kongruity
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Infrastructure build to support third-party app integrations

Browse Source
This commit is contained in:
KS Jannette
2026-08-01 07:35:01 -04:00
parent b4666c5439
commit 15af3465e2
53 changed files with 5864 additions and 659 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
backend/app.js
View File

@@ -1,15 +0,0 @@
import express from 'express';
import cors from 'cors';
import router from './routes/notes.routes.js';
const app = express();
app.use(cors());
app.use(express.json());
app.use('/v1/notes', router);
app.use((req, res) => {
res.status(404).json({ error: `Requested path is invalid or does not exist: ${req.method} ${req.originalUrl}` });
});
export default app;

30
backend/app.ts Normal file
View File

@@ -0,0 +1,30 @@
import express, { type NextFunction, type Request, type Response } from 'express';
import cors from 'cors';
import notesRouter from './routes/notes.routes.js';
import ingestRouter from './routes/ingest.routes.js';
import webhooksRouter from './routes/webhooks.routes.js';
const app = express();
app.use(cors());
// Order matters: the raw parser must claim webhook bodies before express.json
// consumes them, because signature verification needs the exact bytes sent.
// Moving this below express.json silently breaks every signature check.
app.use('/v1/webhooks', express.raw({ type: '*/*', limit: '2mb' }), webhooksRouter);
app.use(express.json({ limit: '2mb' }));
app.use('/v1/notes', ingestRouter);
app.use('/v1/notes', notesRouter);
app.use((req: Request, res: Response) => {
res.status(404).json({ error: `Requested path is invalid or does not exist: ${req.method} ${req.originalUrl}` });
});
app.use((err: Error, _req: Request, res: Response, _next: NextFunction) => {
console.error(`Unhandled error: ${err.stack ?? err.message}`);
if (res.headersSent) return;
res.status(500).json({ error: 'Internal server error' });
});
export default app;

113
backend/config/normalizers.ts Normal file
View File

@@ -0,0 +1,113 @@
import type { NormalizedDelivery, NoteProvenance } from '../types/integration.js';
import type { NoteInput } from '../types/domain.js';
export class NormalizationError extends Error {
constructor(message: string) {
super(message);
this.name = 'NormalizationError';
}
}
const asRecord = (value: unknown): Record<string, unknown> => {
if (typeof value !== 'object' || value === null || Array.isArray(value)) {
throw new NormalizationError('Payload is not an object');
}
return value as Record<string, unknown>;
};
const asString = (value: unknown): string | undefined =>
typeof value === 'string' && value.length > 0 ? value : undefined;
/** notes.id is VARCHAR(64), so composite keys have to be trimmed to fit. */
const noteId = (provider: string, externalId: string): string =>
`${provider}_${externalId}`.slice(0, 64);
const provenance = (
meta: Omit<NoteProvenance, 'receivedAt'>
): Record<string, unknown> => ({
...meta,
receivedAt: new Date().toISOString(),
});
const isNoteInput = (value: unknown): value is NoteInput => {
if (typeof value !== 'object' || value === null) return false;
const note = value as Record<string, unknown>;
if (typeof note.id !== 'string' || note.id.length === 0) return false;
if (typeof note.text !== 'string' || note.text.length === 0) return false;
if (typeof note.author !== 'string' || note.author.length === 0) return false;
if (note.x !== undefined && typeof note.x !== 'number') return false;
if (note.y !== undefined && typeof note.y !== 'number') return false;
if (note.color !== undefined && typeof note.color !== 'string') return false;
return true;
};
export const isNoteInputArray = (value: unknown): value is NoteInput[] =>
Array.isArray(value) && value.every(isNoteInput);
/**
* Direct push: the caller already speaks our note shape, so normalization is
* a validation step rather than a translation.
*/
export const normalizeRest = (payload: unknown): NormalizedDelivery => {
const body = asRecord(payload);
if (!isNoteInputArray(body.notes)) {
throw new NormalizationError('Expected a "notes" array of note objects');
}
const externalId = asString(body.batchId) ?? randomBatchId();
return {
externalId,
notes: body.notes.map((note) => ({
...note,
sourceMeta: provenance({ provider: 'rest', externalId }),
})),
};
};
const randomBatchId = (): string =>
`batch_${Date.now()}_${Math.random().toString(36).slice(2, 10)}`;
/**
* Linear comment webhook. Kept minimal on purpose: this proves the webhook
* pipeline end to end. Label filtering and issue-thread context belong to the
* Linear integration proper, not to this infrastructure.
*/
export const normalizeLinear = (payload: unknown): NormalizedDelivery => {
const body = asRecord(payload);
const data = asRecord(body.data);
const externalId = asString(data.id);
if (!externalId) {
throw new NormalizationError('Linear payload is missing data.id');
}
const text = asString(data.body);
if (!text) {
return { externalId, notes: [] };
}
const user = typeof data.user === 'object' && data.user !== null
? (data.user as Record<string, unknown>)
: {};
return {
externalId,
notes: [
{
id: noteId('linear', externalId),
text,
author: asString(user.name) ?? 'unknown',
sourceMeta: provenance({
provider: 'linear',
externalId,
permalink: asString(data.url),
authorHandle: asString(user.name),
}),
},
],
};
};

95
backend/config/providers.ts Normal file
View File

@@ -0,0 +1,95 @@
import type {
NormalizedDelivery,
ProviderSlug,
SignatureScheme,
} from '../types/integration.js';
import { normalizeLinear, normalizeRest } from './normalizers.js';
export type ChallengeResponse = { status: number; body: unknown };
export type ProviderConfig = {
slug: ProviderSlug;
signatureScheme: SignatureScheme;
/** Header carrying the signature. Empty only when the scheme is 'none'. */
signatureHeader: string;
timestampHeader?: string;
/** Environment variable holding the shared secret when the install has none. */
secretEnvVar?: string;
/** Returns a response when the request is a handshake rather than an event. */
challenge?: (body: unknown) => ChallengeResponse | null;
/**
* Absent until that provider's integration is built. The transport above is
* infrastructure; the payload mapping belongs to the integration itself.
*/
normalize?: (payload: unknown) => NormalizedDelivery;
oauth?: {
authorizeUrl: string;
tokenUrl: string;
scopes: string[];
};
};
const slackChallenge = (body: unknown): ChallengeResponse | null => {
if (typeof body !== 'object' || body === null) return null;
const candidate = body as { type?: unknown; challenge?: unknown };
if (candidate.type !== 'url_verification') return null;
if (typeof candidate.challenge !== 'string') return null;
return { status: 200, body: { challenge: candidate.challenge } };
};
export const providers: Record<ProviderSlug, ProviderConfig> = {
rest: {
slug: 'rest',
signatureScheme: 'none',
signatureHeader: '',
normalize: normalizeRest,
},
linear: {
slug: 'linear',
signatureScheme: 'linear-sha256',
signatureHeader: 'linear-signature',
secretEnvVar: 'LINEAR_SIGNING_SECRET',
normalize: normalizeLinear,
oauth: {
authorizeUrl: 'https://linear.app/oauth/authorize',
tokenUrl: 'https://api.linear.app/oauth/token',
scopes: ['read'],
},
},
slack: {
slug: 'slack',
signatureScheme: 'slack-v0',
signatureHeader: 'x-slack-signature',
timestampHeader: 'x-slack-request-timestamp',
secretEnvVar: 'SLACK_SIGNING_SECRET',
challenge: slackChallenge,
oauth: {
authorizeUrl: 'https://slack.com/oauth/v2/authorize',
tokenUrl: 'https://slack.com/api/oauth.v2.access',
scopes: ['channels:history', 'reactions:read', 'users:read'],
},
},
github: {
slug: 'github',
signatureScheme: 'github-sha256',
signatureHeader: 'x-hub-signature-256',
secretEnvVar: 'GITHUB_WEBHOOK_SECRET',
oauth: {
authorizeUrl: 'https://github.com/login/oauth/authorize',
tokenUrl: 'https://github.com/login/oauth/access_token',
scopes: ['repo', 'read:discussion'],
},
},
jira: {
slug: 'jira',
signatureScheme: 'none',
signatureHeader: '',
oauth: {
authorizeUrl: 'https://auth.atlassian.com/authorize',
tokenUrl: 'https://auth.atlassian.com/oauth/token',
scopes: ['read:jira-work', 'offline_access'],
},
},
};
export const getProvider = (slug: ProviderSlug): ProviderConfig => providers[slug];

13
backend/db/index.js
View File

@@ -1,13 +0,0 @@
import pg from 'pg';
const pool = new pg.Pool({
connectionString: process.env.DATABASE_URL,
});
export const query = (text, params) => pool.query(text, params);
export const getPool = () => pool;
export const close = () => pool.end();
export default pool;

17
backend/db/index.ts Normal file
View File

@@ -0,0 +1,17 @@
import pg from 'pg';
import type { QueryResult, QueryResultRow } from 'pg';
const pool = new pg.Pool({
connectionString: process.env.DATABASE_URL,
});
export const query = <R extends QueryResultRow = QueryResultRow>(
text: string,
params?: unknown[]
): Promise<QueryResult<R>> => pool.query<R>(text, params);
export const getPool = (): pg.Pool => pool;
export const close = (): Promise<void> => pool.end();
export default pool;

64
backend/db/ingest_events.dao.ts Normal file
View File

@@ -0,0 +1,64 @@
import { query } from './index.js';
import type { ProviderSlug } from '../types/integration.js';
/**
* Returns the new row id, or null when this delivery was already recorded.
* Providers deliver at least once, so the unique constraint firing is the
* expected path for a redelivery rather than an error.
*/
export const recordDelivery = async (input: {
provider: ProviderSlug;
externalId: string;
integrationId?: number;
}): Promise<number | null> => {
const { rows } = await query<{ id: number }>(
`INSERT INTO ingest_events (provider, external_id, integration_id)
VALUES ($1, $2, $3)
ON CONFLICT (provider, external_id) DO NOTHING
RETURNING id`,
[input.provider, input.externalId, input.integrationId ?? null]
);
return rows[0]?.id ?? null;
};
export const markProcessing = async (id: number): Promise<void> => {
await query(
`UPDATE ingest_events
SET status = 'processing', attempts = attempts + 1, updated_at = NOW()
WHERE id = $1`,
[id]
);
};
export const markDone = async (id: number): Promise<void> => {
await query(
`UPDATE ingest_events
SET status = 'done', last_error = NULL, updated_at = NOW()
WHERE id = $1`,
[id]
);
};
export const markFailed = async (id: number, error: string): Promise<void> => {
await query(
`UPDATE ingest_events
SET status = 'failed', last_error = $2, updated_at = NOW()
WHERE id = $1`,
[id, error.slice(0, 2000)]
);
};
/**
* The queue lives in process memory, so a restart strands anything that was
* mid-flight. Returns rows to 'pending' so they can be picked up again.
*/
export const resetStaleProcessing = async (olderThanMs: number): Promise<number> => {
const { rowCount } = await query(
`UPDATE ingest_events
SET status = 'pending', updated_at = NOW()
WHERE status = 'processing'
AND updated_at < NOW() - ($1 || ' milliseconds')::interval`,
[String(olderThanMs)]
);
return rowCount ?? 0;
};

132
backend/db/integrations.dao.ts Normal file
View File

@@ -0,0 +1,132 @@
import { query } from './index.js';
import { decryptSecret, encryptSecret } from '../lib/crypto.js';
import type { IntegrationRow, ProviderSlug } from '../types/integration.js';
type IntegrationRecord = {
id: number;
provider: ProviderSlug;
external_workspace_id: string | null;
display_name: string | null;
api_key_hash: string | null;
access_token_ciphertext: string | null;
refresh_token_ciphertext: string | null;
signing_secret_ciphertext: string | null;
token_expires_at: Date | null;
scopes: string[] | null;
};
const PUBLIC_COLUMNS = `
id, provider, external_workspace_id, display_name, token_expires_at, scopes
`;
/**
* Ciphertext columns are dropped here rather than in the query so that every
* caller path converges on a value that structurally cannot carry a secret.
*/
const toPublicRow = (record: IntegrationRecord): IntegrationRow => ({
id: record.id,
provider: record.provider,
externalWorkspaceId: record.external_workspace_id,
displayName: record.display_name,
scopes: record.scopes ?? [],
tokenExpiresAt: record.token_expires_at,
});
export const findByApiKeyHash = async (
hash: string
): Promise<IntegrationRow | null> => {
const { rows } = await query<IntegrationRecord>(
`SELECT ${PUBLIC_COLUMNS} FROM integrations WHERE api_key_hash = $1`,
[hash]
);
return rows[0] ? toPublicRow(rows[0]) : null;
};
export const findByProviderWorkspace = async (
provider: ProviderSlug,
externalWorkspaceId: string
): Promise<IntegrationRow | null> => {
const { rows } = await query<IntegrationRecord>(
`SELECT ${PUBLIC_COLUMNS} FROM integrations
WHERE provider = $1 AND external_workspace_id = $2`,
[provider, externalWorkspaceId]
);
return rows[0] ? toPublicRow(rows[0]) : null;
};
export const upsertInstall = async (input: {
provider: ProviderSlug;
externalWorkspaceId: string;
displayName?: string;
apiKeyHash?: string;
signingSecret?: string;
scopes?: string[];
}): Promise<IntegrationRow> => {
const { rows } = await query<IntegrationRecord>(
`INSERT INTO integrations
(provider, external_workspace_id, display_name, api_key_hash,
signing_secret_ciphertext, scopes)
VALUES ($1, $2, $3, $4, $5, $6)
ON CONFLICT (provider, external_workspace_id) DO UPDATE SET
display_name = EXCLUDED.display_name,
api_key_hash = COALESCE(EXCLUDED.api_key_hash, integrations.api_key_hash),
signing_secret_ciphertext = COALESCE(
EXCLUDED.signing_secret_ciphertext, integrations.signing_secret_ciphertext
),
scopes = EXCLUDED.scopes,
updated_at = NOW()
RETURNING ${PUBLIC_COLUMNS}`,
[
input.provider,
input.externalWorkspaceId,
input.displayName ?? null,
input.apiKeyHash ?? null,
input.signingSecret ? encryptSecret(input.signingSecret) : null,
input.scopes ?? [],
]
);
return toPublicRow(rows[0]);
};
export const updateTokens = async (input: {
id: number;
accessToken: string;
refreshToken?: string;
expiresAt?: Date;
}): Promise<void> => {
await query(
`UPDATE integrations SET
access_token_ciphertext = $2,
refresh_token_ciphertext = COALESCE($3, refresh_token_ciphertext),
token_expires_at = $4,
updated_at = NOW()
WHERE id = $1`,
[
input.id,
encryptSecret(input.accessToken),
input.refreshToken ? encryptSecret(input.refreshToken) : null,
input.expiresAt ?? null,
]
);
};
const readSecret = async (
id: number,
column: 'access_token_ciphertext' | 'refresh_token_ciphertext' | 'signing_secret_ciphertext'
): Promise<string | null> => {
const { rows } = await query<Record<string, string | null>>(
`SELECT ${column} AS value FROM integrations WHERE id = $1`,
[id]
);
const value = rows[0]?.value;
return value ? decryptSecret(value) : null;
};
export const getAccessToken = (id: number): Promise<string | null> =>
readSecret(id, 'access_token_ciphertext');
export const getRefreshToken = (id: number): Promise<string | null> =>
readSecret(id, 'refresh_token_ciphertext');
export const getSigningSecret = (id: number): Promise<string | null> =>
readSecret(id, 'signing_secret_ciphertext');

30
backend/db/migrate.js
View File

@@ -1,30 +0,0 @@
import 'dotenv/config';
import { query, close } from './index.js';
const up = `
CREATE TABLE IF NOT EXISTS notes (
id VARCHAR(64) PRIMARY KEY,
text TEXT NOT NULL,
x INTEGER DEFAULT 0,
y INTEGER DEFAULT 0,
author VARCHAR(128),
color VARCHAR(32) DEFAULT 'yellow',
source_meta JSONB DEFAULT '{}',
created_at TIMESTAMPTZ DEFAULT NOW(),
updated_at TIMESTAMPTZ DEFAULT NOW()
);
`;
const run = async () => {
try {
await query(up);
console.log('Migration complete — notes table ready.');
} catch (err) {
console.error('Migration failed:', err.message);
process.exit(1);
} finally {
await close();
}
};
run();

67
backend/db/migrate.ts Normal file
View File

@@ -0,0 +1,67 @@
import 'dotenv/config';
import { query, close } from './index.js';
const up = `
CREATE TABLE IF NOT EXISTS notes (
id VARCHAR(64) PRIMARY KEY,
text TEXT NOT NULL,
x INTEGER DEFAULT 0,
y INTEGER DEFAULT 0,
author VARCHAR(128),
color VARCHAR(32) DEFAULT 'yellow',
source_meta JSONB DEFAULT '{}',
created_at TIMESTAMPTZ DEFAULT NOW(),
updated_at TIMESTAMPTZ DEFAULT NOW()
);
CREATE TABLE IF NOT EXISTS integrations (
id SERIAL PRIMARY KEY,
provider VARCHAR(32) NOT NULL,
external_workspace_id VARCHAR(128),
display_name VARCHAR(255),
api_key_hash CHAR(64),
access_token_ciphertext TEXT,
refresh_token_ciphertext TEXT,
signing_secret_ciphertext TEXT,
token_expires_at TIMESTAMPTZ,
scopes TEXT[] DEFAULT '{}',
created_at TIMESTAMPTZ DEFAULT NOW(),
updated_at TIMESTAMPTZ DEFAULT NOW(),
UNIQUE (provider, external_workspace_id)
);
CREATE INDEX IF NOT EXISTS integrations_api_key_hash_idx
ON integrations (api_key_hash);
-- The unique constraint is the deduplication mechanism: providers deliver
-- at least once, so a redelivery must collide rather than insert.
CREATE TABLE IF NOT EXISTS ingest_events (
id SERIAL PRIMARY KEY,
provider VARCHAR(32) NOT NULL,
external_id VARCHAR(255) NOT NULL,
integration_id INTEGER REFERENCES integrations(id) ON DELETE SET NULL,
status VARCHAR(16) NOT NULL DEFAULT 'pending',
attempts INTEGER NOT NULL DEFAULT 0,
last_error TEXT,
received_at TIMESTAMPTZ DEFAULT NOW(),
updated_at TIMESTAMPTZ DEFAULT NOW(),
UNIQUE (provider, external_id)
);
CREATE INDEX IF NOT EXISTS ingest_events_status_idx
ON ingest_events (status, updated_at);
`;
const run = async (): Promise<void> => {
try {
await query(up);
console.log('Migration complete — notes, integrations, ingest_events ready.');
} catch (err) {
console.error('Migration failed:', err instanceof Error ? err.message : err);
process.exit(1);
} finally {
await close();
}
};
run();

110
backend/db/notes.dao.js
View File

@@ -1,110 +0,0 @@
import QueryStream from 'pg-query-stream';
import { pipeline } from 'node:stream/promises';
import { Readable } from 'node:stream';
import { query, getPool } from './index.js';
import { batch } from '../lib/streams.js';
const SELECT_NOTES = 'SELECT id, text, x, y, author, color FROM notes ORDER BY id';
// Postgres caps a statement at 65535 bind parameters; six columns per note
// leaves 10922 as the hard ceiling.
const INSERT_BATCH_SIZE = 1000;
export const getAllNotes = async () => {
const { rows } = await query(SELECT_NOTES);
return rows;
};
/**
* Streams every note as an object-mode Readable. The pooled client is released on end, error, or
* destruction by consumer.
*
* @returns {Promise<import('node:stream').Readable>}
*/
export const streamAllNotes = async () => {
const client = await getPool().connect();
let released = false;
const release = () => {
if (released) return;
released = true;
client.release();
};
try {
const rows = client.query(new QueryStream(SELECT_NOTES));
rows.once('end', release);
rows.once('error', release);
rows.once('close', release);
return rows;
} catch (err) {
release();
throw err;
}
};
export const getNoteById = async (id) => {
const { rows } = await query(
'SELECT id, text, x, y, author, color FROM notes WHERE id = $1',
[id]
);
return rows[0] || null;
};
export const createNote = async (note) => {
const { rows } = await query(
`INSERT INTO notes (id, text, x, y, author, color)
VALUES ($1, $2, $3, $4, $5, $6)
RETURNING id, text, x, y, author, color`,
[note.id, note.text, note.x ?? 0, note.y ?? 0, note.author, note.color ?? 'yellow']
);
return rows[0];
};
const insertNoteBatch = async (notes) => {
const values = [];
const placeholders = [];
notes.forEach((note, i) => {
const offset = i * 6;
placeholders.push(
`($${offset + 1}, $${offset + 2}, $${offset + 3}, $${offset + 4}, $${offset + 5}, $${offset + 6})`
);
values.push(
note.id,
note.text,
note.x ?? 0,
note.y ?? 0,
note.author,
note.color ?? 'yellow'
);
});
const { rows } = await query(
`INSERT INTO notes (id, text, x, y, author, color)
VALUES ${placeholders.join(', ')}
RETURNING id, text, x, y, author, color`,
values
);
return rows;
};
export const createNotes = async (notes) => {
if (!notes || notes.length === 0) {
return [];
}
const inserted = [];
await pipeline(
Readable.from(notes, { objectMode: true }),
batch(INSERT_BATCH_SIZE),
async (batches) => {
for await (const chunk of batches) {
inserted.push(...await insertNoteBatch(chunk));
}
}
);
return inserted;
};

125
backend/db/notes.dao.ts Normal file
View File

@@ -0,0 +1,125 @@
import QueryStream from 'pg-query-stream';
import { pipeline } from 'node:stream/promises';
import { Readable } from 'node:stream';
import { query, getPool } from './index.js';
import { batch } from '../lib/streams.js';
import type { Note, NoteInput } from '../types/domain.js';
const SELECT_NOTES = 'SELECT id, text, x, y, author, color FROM notes ORDER BY id';
// Postgres caps a statement at 65535 bind parameters; seven columns per note
// leaves 9362 as the hard ceiling.
const INSERT_BATCH_SIZE = 1000;
const INSERT_COLUMNS = 7;
export const getAllNotes = async (): Promise<Note[]> => {
const { rows } = await query<Note>(SELECT_NOTES);
return rows;
};
/**
* Streams every note as an object-mode Readable. The pooled client is released on end, error, or
* destruction by consumer.
*/
export const streamAllNotes = async (): Promise<Readable> => {
const client = await getPool().connect();
let released = false;
const release = () => {
if (released) return;
released = true;
client.release();
};
try {
const rows = client.query(new QueryStream(SELECT_NOTES)) as unknown as Readable;
rows.once('end', release);
rows.once('error', release);
rows.once('close', release);
return rows;
} catch (err) {
release();
throw err;
}
};
export const getNoteById = async (id: string): Promise<Note | null> => {
const { rows } = await query<Note>(
'SELECT id, text, x, y, author, color FROM notes WHERE id = $1',
[id]
);
return rows[0] || null;
};
export const createNote = async (note: NoteInput): Promise<Note> => {
const { rows } = await query<Note>(
`INSERT INTO notes (id, text, x, y, author, color, source_meta)
VALUES ($1, $2, $3, $4, $5, $6, $7)
RETURNING id, text, x, y, author, color`,
[
note.id,
note.text,
note.x ?? 0,
note.y ?? 0,
note.author,
note.color ?? 'yellow',
JSON.stringify(note.sourceMeta ?? {}),
]
);
return rows[0];
};
const insertNoteBatch = async (notes: NoteInput[]): Promise<Note[]> => {
const values: unknown[] = [];
const placeholders: string[] = [];
notes.forEach((note, i) => {
const offset = i * INSERT_COLUMNS;
const slots = Array.from(
{ length: INSERT_COLUMNS },
(_, col) => `$${offset + col + 1}`
);
placeholders.push(`(${slots.join(', ')})`);
values.push(
note.id,
note.text,
note.x ?? 0,
note.y ?? 0,
note.author,
note.color ?? 'yellow',
JSON.stringify(note.sourceMeta ?? {})
);
});
// Redelivered webhooks can carry a note that already landed; skipping the
// conflict keeps ingestion idempotent at the row level as well.
const { rows } = await query<Note>(
`INSERT INTO notes (id, text, x, y, author, color, source_meta)
VALUES ${placeholders.join(', ')}
ON CONFLICT (id) DO NOTHING
RETURNING id, text, x, y, author, color`,
values
);
return rows;
};
export const createNotes = async (notes: NoteInput[]): Promise<Note[]> => {
if (!notes || notes.length === 0) {
return [];
}
const inserted: Note[] = [];
await pipeline(
Readable.from(notes, { objectMode: true }),
batch<NoteInput>(INSERT_BATCH_SIZE),
async (batches: AsyncIterable<NoteInput[]>) => {
for await (const chunk of batches) {
inserted.push(...await insertNoteBatch(chunk));
}
}
);
return inserted;
};

24
backend/db/seed.js → backend/db/seed.ts
View File

@@ -2,33 +2,35 @@ import 'dotenv/config';
import split2 from 'split2';
import { from as copyFrom } from 'pg-copy-streams';
import { createReadStream } from 'node:fs';
import { Transform } from 'node:stream';
import { Transform, type TransformCallback } from 'node:stream';
import { pipeline } from 'node:stream/promises';
import { fileURLToPath } from 'node:url';
import { getPool, close } from './index.js';
const FIXTURE = fileURLToPath(new URL('./fixtures/notes.jsonl', import.meta.url));
const COLUMNS = ['id', 'text', 'x', 'y', 'author', 'color'];
const COLUMNS = ['id', 'text', 'x', 'y', 'author', 'color'] as const;
const DEFAULTS = { x: 0, y: 0, color: 'yellow' };
type Column = typeof COLUMNS[number];
const csvField = (value) => {
const DEFAULTS: Partial<Record<Column, unknown>> = { x: 0, y: 0, color: 'yellow' };
const csvField = (value: unknown): string => {
if (value === null || value === undefined) return '';
return `"${String(value).replaceAll('"', '""')}"`;
};
const toCsvRows = () => new Transform({
const toCsvRows = (): Transform => new Transform({
writableObjectMode: true,
transform(line, _encoding, callback) {
transform(line: string, _encoding: BufferEncoding, callback: TransformCallback) {
if (line.trim().length === 0) {
callback();
return;
}
let note;
let note: Record<string, unknown>;
try {
note = JSON.parse(line);
note = JSON.parse(line) as Record<string, unknown>;
} catch {
callback(new Error(`Fixture contains a malformed JSON line: ${line.slice(0, 80)}`));
return;
@@ -39,7 +41,7 @@ const toCsvRows = () => new Transform({
},
});
const run = async () => {
const run = async (): Promise<void> => {
const client = await getPool().connect();
try {
@@ -66,10 +68,10 @@ const run = async () => {
await client.query('COMMIT');
console.log(`Seed complete — ${inserted} notes inserted (${staged - inserted} already existed).`);
console.log(`Seed complete — ${inserted} notes inserted (${(staged ?? 0) - (inserted ?? 0)} already existed).`);
} catch (err) {
await client.query('ROLLBACK').catch(() => {});
console.error('Seed failed:', err.message);
console.error('Seed failed:', err instanceof Error ? err.message : err);
process.exitCode = 1;
} finally {
client.release();

77
backend/lib/crypto.ts Normal file
View File

@@ -0,0 +1,77 @@
import {
createCipheriv,
createDecipheriv,
createHash,
randomBytes,
timingSafeEqual,
} from 'node:crypto';
const ALGORITHM = 'aes-256-gcm';
const IV_BYTES = 12;
const KEY_BYTES = 32;
const loadKey = (): Buffer => {
const raw = process.env.TOKEN_ENCRYPTION_KEY;
if (!raw) {
throw new Error('TOKEN_ENCRYPTION_KEY is not set');
}
const key = Buffer.from(raw, 'base64');
if (key.length !== KEY_BYTES) {
throw new Error(
`TOKEN_ENCRYPTION_KEY must decode to ${KEY_BYTES} bytes, got ${key.length}`
);
}
return key;
};
/** Serialized as base64(iv):base64(authTag):base64(ciphertext). */
export const encryptSecret = (plaintext: string): string => {
const iv = randomBytes(IV_BYTES);
const cipher = createCipheriv(ALGORITHM, loadKey(), iv);
const payload = Buffer.concat([
cipher.update(plaintext, 'utf8'),
cipher.final(),
]);
return [
iv.toString('base64'),
cipher.getAuthTag().toString('base64'),
payload.toString('base64'),
].join(':');
};
export const decryptSecret = (ciphertext: string): string => {
const parts = ciphertext.split(':');
if (parts.length !== 3) {
throw new Error('Ciphertext is not in the expected iv:tag:payload form');
}
const [iv, tag, payload] = parts;
const decipher = createDecipheriv(
ALGORITHM,
loadKey(),
Buffer.from(iv, 'base64')
);
decipher.setAuthTag(Buffer.from(tag, 'base64'));
return Buffer.concat([
decipher.update(Buffer.from(payload, 'base64')),
decipher.final(),
]).toString('utf8');
};
export const hashApiKey = (key: string): string =>
createHash('sha256').update(key, 'utf8').digest('hex');
/**
* Constant-time string comparison. Both sides are hashed first so that
* unequal lengths cannot short-circuit the comparison or throw.
*/
export const safeEquals = (a: string, b: string): boolean => {
const digestA = createHash('sha256').update(a, 'utf8').digest();
const digestB = createHash('sha256').update(b, 'utf8').digest();
return timingSafeEqual(digestA, digestB);
};

234
backend/lib/httpClient.ts Normal file
View File

@@ -0,0 +1,234 @@
const DEFAULT_TIMEOUT_MS = 10_000;
const DEFAULT_MAX_ATTEMPTS = 3;
const DEFAULT_BASE_DELAY_MS = 250;
const DEFAULT_RETRY_AFTER_CAP_MS = 60_000;
export type HttpFailureKind = 'status' | 'timeout' | 'network' | 'aborted' | 'invalid-body';
export type RequestJsonInit = RequestInit & {
timeoutMs?: number;
maxAttempts?: number;
/** First-retry backoff, doubled per attempt. Exposed so tests need not wait on real backoff. */
baseDelayMs?: number;
/** Upper bound on a honored `Retry-After`, so a hostile header cannot park the process. */
retryAfterCapMs?: number;
};
export class HttpRequestError extends Error {
readonly url: string;
readonly status: number | undefined;
readonly attempts: number;
readonly kind: HttpFailureKind;
constructor(
url: string,
kind: HttpFailureKind,
attempts: number,
status?: number,
detail?: string
) {
const statusPart = status === undefined ? '' : ` status ${status}`;
const detailPart = detail === undefined ? '' : `: ${detail}`;
super(
`HTTP request to ${url} failed after ${attempts} attempt(s) (${kind}${statusPart})${detailPart}`
);
this.name = 'HttpRequestError';
this.url = url;
this.status = status;
this.attempts = attempts;
this.kind = kind;
}
}
export const isHttpRequestError = (err: unknown): err is HttpRequestError =>
err instanceof HttpRequestError;
/**
* Returns the delay a `Retry-After` header asks for, clamped to `capMs`, or null when the
* header is absent or unintelligible. Supports both delay-seconds and HTTP-date forms.
*/
export const parseRetryAfter = (
headerValue: string | null,
capMs: number = DEFAULT_RETRY_AFTER_CAP_MS
): number | null => {
if (headerValue === null) {
return null;
}
const raw = headerValue.trim();
if (raw === '') {
return null;
}
if (/^\d+$/.test(raw)) {
return Math.min(Number(raw) * 1000, capMs);
}
const deadline = Date.parse(raw);
if (Number.isNaN(deadline)) {
return null;
}
return Math.min(Math.max(deadline - Date.now(), 0), capMs);
};
const backoffDelay = (attempt: number, baseDelayMs: number, capMs: number): number => {
const ceiling = Math.min(baseDelayMs * 2 ** (attempt - 1), capMs);
// Half fixed, half jittered, so concurrent callers do not resynchronize on the same tick.
return Math.round(ceiling / 2 + Math.random() * (ceiling / 2));
};
/** Settles early when `signal` fires; the caller re-checks the signal before retrying. */
const sleep = (ms: number, signal: AbortSignal | undefined): Promise<void> =>
new Promise<void>((resolve) => {
if (signal?.aborted === true) {
resolve();
return;
}
let timer: ReturnType<typeof setTimeout>;
const onAbort = (): void => {
clearTimeout(timer);
resolve();
};
timer = setTimeout(() => {
signal?.removeEventListener('abort', onAbort);
resolve();
}, ms);
signal?.addEventListener('abort', onAbort, { once: true });
});
const isRetryableStatus = (status: number): boolean => status === 429 || status >= 500;
/** Frees the socket for reuse; a body we never read would otherwise stay pending. */
const discardBody = async (response: Response): Promise<void> => {
try {
await response.body?.cancel();
} catch {
// A already-consumed or errored body is irrelevant to the retry decision.
}
};
const readJson = async <T>(response: Response, url: string, attempts: number): Promise<T> => {
// 204/205 are defined as bodiless, so absence of JSON is the correct outcome, not a failure.
if (response.status === 204 || response.status === 205) {
return undefined as T;
}
let text: string;
try {
text = await response.text();
} catch (err) {
throw new HttpRequestError(
url,
'invalid-body',
attempts,
response.status,
err instanceof Error ? err.message : 'could not read response body'
);
}
try {
return JSON.parse(text) as T;
} catch {
throw new HttpRequestError(
url,
'invalid-body',
attempts,
response.status,
'response body was not valid JSON'
);
}
};
/**
* Performs a JSON request, retrying only failures a retry can plausibly fix: 429, 5xx, timeouts
* and transport errors. Any other non-2xx fails on the first attempt, since re-sending a 400 or
* 401 only spends rate limit against an answer that will not change.
*
* Resolves `undefined` for bodiless 204/205 responses; any other 2xx whose body is not valid
* JSON rejects rather than resolving `undefined` silently.
*/
export const requestJson = async <T>(url: string, init?: RequestJsonInit): Promise<T> => {
const {
timeoutMs = DEFAULT_TIMEOUT_MS,
maxAttempts = DEFAULT_MAX_ATTEMPTS,
baseDelayMs = DEFAULT_BASE_DELAY_MS,
retryAfterCapMs = DEFAULT_RETRY_AFTER_CAP_MS,
signal,
...requestInit
} = init ?? {};
if (!Number.isInteger(maxAttempts) || maxAttempts < 1) {
throw new TypeError('requestJson(init.maxAttempts) requires a positive integer');
}
const callerSignal = signal ?? undefined;
// Read through a call so narrowing never freezes this at its first observed value.
const callerAborted = (): boolean => callerSignal !== undefined && callerSignal.aborted;
let lastFailure: HttpRequestError | undefined;
for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
if (callerAborted()) {
throw new HttpRequestError(url, 'aborted', attempt - 1, undefined, 'caller aborted');
}
const timeoutSignal = AbortSignal.timeout(timeoutMs);
const attemptSignal =
callerSignal === undefined
? timeoutSignal
: AbortSignal.any([callerSignal, timeoutSignal]);
let response: Response;
try {
response = await fetch(url, { ...requestInit, signal: attemptSignal });
} catch (err) {
if (callerAborted()) {
throw new HttpRequestError(url, 'aborted', attempt, undefined, 'caller aborted');
}
const kind: HttpFailureKind = timeoutSignal.aborted ? 'timeout' : 'network';
const detail = timeoutSignal.aborted
? `attempt exceeded ${timeoutMs}ms`
: err instanceof Error
? err.message
: 'transport error';
lastFailure = new HttpRequestError(url, kind, attempt, undefined, detail);
if (attempt === maxAttempts) {
throw lastFailure;
}
await sleep(backoffDelay(attempt, baseDelayMs, retryAfterCapMs), callerSignal);
continue;
}
if (response.ok) {
return await readJson<T>(response, url, attempt);
}
await discardBody(response);
if (!isRetryableStatus(response.status)) {
throw new HttpRequestError(url, 'status', attempt, response.status);
}
lastFailure = new HttpRequestError(url, 'status', attempt, response.status);
if (attempt === maxAttempts) {
throw lastFailure;
}
const retryAfter = parseRetryAfter(response.headers.get('retry-after'), retryAfterCapMs);
await sleep(
retryAfter ?? backoffDelay(attempt, baseDelayMs, retryAfterCapMs),
callerSignal
);
}
// Unreachable while maxAttempts >= 1; the loop either returns or throws.
throw lastFailure ?? new HttpRequestError(url, 'network', maxAttempts);
};

91
backend/lib/queue.ts Normal file
View File

@@ -0,0 +1,91 @@
export type Job = () => Promise<void>;
type QueueEntry = { name: string; job: Job };
type QueueSettings = { maxAttempts: number; baseDelayMs: number };
const settings: QueueSettings = { maxAttempts: 3, baseDelayMs: 100 };
const pending: QueueEntry[] = [];
const idleWaiters: Array<() => void> = [];
let active = false;
const sleep = (ms: number): Promise<void> =>
new Promise((resolve) => {
setTimeout(resolve, ms);
});
// Delay for attempt n is drawn from [base * 2^(n-1), base * 2^n), so successive
// waits always grow while jitter keeps retries from synchronizing across jobs.
const backoffDelay = (attempt: number): number => {
const window = settings.baseDelayMs * 2 ** (attempt - 1);
return window + Math.random() * window;
};
const runEntry = async (entry: QueueEntry): Promise<void> => {
for (let attempt = 1; ; attempt += 1) {
try {
await entry.job();
return;
} catch (err) {
if (attempt >= settings.maxAttempts) {
console.error(
`[queue] job "${entry.name}" abandoned after ${attempt} attempt(s)`,
err
);
return;
}
await sleep(backoffDelay(attempt));
}
}
};
const runLoop = async (): Promise<void> => {
try {
for (;;) {
const entry = pending.shift();
if (!entry) return;
await runEntry(entry);
}
} finally {
active = false;
for (const resolve of idleWaiters.splice(0)) resolve();
}
};
export const enqueue = (name: string, job: Job): void => {
pending.push({ name, job });
if (active) return;
active = true;
// Deferred to a microtask so enqueue() returns to its caller — typically a
// request handler that has already responded — before any job body runs.
void Promise.resolve()
.then(runLoop)
.catch((err: unknown) => {
active = false;
console.error('[queue] queue loop stopped unexpectedly', err);
});
};
export const size = (): number => pending.length;
export const drain = (): Promise<void> => {
if (!active && pending.length === 0) return Promise.resolve();
return new Promise<void>((resolve) => {
idleWaiters.push(resolve);
});
};
export const configureQueue = (opts: {
maxAttempts?: number;
baseDelayMs?: number;
}): void => {
if (opts.maxAttempts !== undefined) {
settings.maxAttempts = Math.max(1, Math.floor(opts.maxAttempts));
}
if (opts.baseDelayMs !== undefined) {
settings.baseDelayMs = Math.max(0, opts.baseDelayMs);
}
};

81
backend/lib/signatures.ts Normal file
View File

@@ -0,0 +1,81 @@
import { createHmac, timingSafeEqual } from 'node:crypto';
import type { IncomingHttpHeaders } from 'node:http';
import { getProvider } from '../config/providers.js';
import type { ProviderSlug, SignatureResult } from '../types/integration.js';
const DEFAULT_TOLERANCE_SECONDS = 300;
const OK: SignatureResult = { ok: true };
const fail = (reason: Exclude<SignatureResult, { ok: true }>['reason']): SignatureResult => ({
ok: false,
reason,
});
const header = (headers: IncomingHttpHeaders, name: string): string | undefined => {
const value = headers[name.toLowerCase()];
if (Array.isArray(value)) return value[0];
return value;
};
const hmacHex = (secret: string, payload: string | Buffer): string =>
createHmac('sha256', secret).update(payload).digest('hex');
const constantTimeEquals = (a: string, b: string): boolean => {
const bufA = Buffer.from(a, 'utf8');
const bufB = Buffer.from(b, 'utf8');
if (bufA.length !== bufB.length) return false;
return timingSafeEqual(bufA, bufB);
};
const withinTolerance = (timestamp: string, toleranceSeconds: number): boolean => {
const sent = Number(timestamp);
if (!Number.isFinite(sent)) return false;
const nowSeconds = Math.floor(Date.now() / 1000);
return Math.abs(nowSeconds - sent) <= toleranceSeconds;
};
/**
* Never throws. A malformed header from an anonymous caller must be an
* ordinary negative result, not an exception reachable from the edge.
*/
export const verifySignature = (input: {
provider: ProviderSlug;
rawBody: Buffer;
headers: IncomingHttpHeaders;
secret: string;
toleranceSeconds?: number;
}): SignatureResult => {
const config = getProvider(input.provider);
const tolerance = input.toleranceSeconds ?? DEFAULT_TOLERANCE_SECONDS;
if (config.signatureScheme === 'none') return OK;
const presented = header(input.headers, config.signatureHeader);
if (!presented) return fail('missing');
if (config.signatureScheme === 'slack-v0') {
const timestamp = config.timestampHeader
? header(input.headers, config.timestampHeader)
: undefined;
if (!timestamp) return fail('missing');
if (!withinTolerance(timestamp, tolerance)) return fail('stale');
if (!presented.startsWith('v0=')) return fail('malformed');
const base = `v0:${timestamp}:${input.rawBody.toString('utf8')}`;
const expected = `v0=${hmacHex(input.secret, base)}`;
return constantTimeEquals(presented, expected) ? OK : fail('mismatch');
}
if (config.signatureScheme === 'github-sha256') {
if (!presented.startsWith('sha256=')) return fail('malformed');
const expected = `sha256=${hmacHex(input.secret, input.rawBody)}`;
return constantTimeEquals(presented, expected) ? OK : fail('mismatch');
}
// linear-sha256: bare lowercase hex digest of the raw body.
if (!/^[0-9a-f]{64}$/.test(presented)) return fail('malformed');
const expected = hmacHex(input.secret, input.rawBody);
return constantTimeEquals(presented, expected) ? OK : fail('mismatch');
};

26
backend/lib/streams.js → backend/lib/streams.ts
View File

@@ -1,21 +1,20 @@
import { Transform } from 'node:stream';
import { Transform, type TransformCallback } from 'node:stream';
/**
* Backpressure on readable side is limits how many batches are in flight.
*
* @param {number} size - maximum items per emitted batch
* @returns {Transform}
* @param size - maximum items per emitted batch
*/
export const batch = (size) => {
export const batch = <T>(size: number): Transform => {
if (!Number.isInteger(size) || size < 1) {
throw new TypeError('batch(size) requires a positive integer size');
}
let pending = [];
let pending: T[] = [];
return new Transform({
objectMode: true,
transform(item, _encoding, callback) {
transform(item: T, _encoding: BufferEncoding, callback: TransformCallback) {
pending.push(item);
if (pending.length < size) {
@@ -27,7 +26,7 @@ export const batch = (size) => {
pending = [];
callback(null, full);
},
flush(callback) {
flush(callback: TransformCallback) {
if (pending.length === 0) {
callback();
return;
@@ -40,20 +39,17 @@ export const batch = (size) => {
});
};
/**
* @returns {Transform}
*/
export const jsonArray = () => {
export const jsonArray = (): Transform => {
let wroteFirst = false;
return new Transform({
writableObjectMode: true,
transform(item, _encoding, callback) {
let serialized;
transform(item: unknown, _encoding: BufferEncoding, callback: TransformCallback) {
let serialized: string;
try {
serialized = JSON.stringify(item);
} catch (err) {
callback(err);
callback(err as Error);
return;
}
@@ -61,7 +57,7 @@ export const jsonArray = () => {
wroteFirst = true;
callback(null, prefix + serialized);
},
flush(callback) {
flush(callback: TransformCallback) {
callback(null, wroteFirst ? ']' : '[]');
},
});

41
backend/middleware/apiKey.ts Normal file
View File

@@ -0,0 +1,41 @@
import type { NextFunction, Request, RequestHandler, Response } from 'express';
import { hashApiKey } from '../lib/crypto.js';
import { findByApiKeyHash } from '../db/integrations.dao.js';
import type { IntegrationRow } from '../types/integration.js';
declare module 'express-serve-static-core' {
interface Request {
integration?: IntegrationRow;
}
}
const BEARER = /^Bearer (.+)$/;
/**
* Every rejection returns the same body. Distinguishing "no such key" from
* "wrong key" would let a caller enumerate valid keys.
*/
export const requireApiKey: RequestHandler = async (
req: Request,
res: Response,
next: NextFunction
) => {
try {
const match = BEARER.exec(req.get('authorization') ?? '');
if (!match) {
res.status(401).json({ error: 'Unauthorized' });
return;
}
const integration = await findByApiKeyHash(hashApiKey(match[1]));
if (!integration) {
res.status(401).json({ error: 'Unauthorized' });
return;
}
req.integration = integration;
next();
} catch (err) {
next(err);
}
};

1447
backend/package-lock.json generated
View File

File diff suppressed because it is too large Load Diff

23
backend/package.json
View File

@@ -3,13 +3,15 @@
"version": "0.1.0",
"description": "kongruity AI stick note clustering feature - backend for ProjectPilot",
"type": "module",
"main": "server.js",
"main": "dist/server.js",
"scripts": {
"start": "node server.js",
"dev": "nodemon server.js",
"build": "tsc -p tsconfig.build.json",
"start": "node dist/server.js",
"dev": "tsx watch server.ts",
"type-check": "tsc --noEmit",
"test": "vitest run",
"db:migrate": "node db/migrate.js",
"db:seed": "node db/seed.js"
"db:migrate": "tsx db/migrate.ts",
"db:seed": "tsx db/seed.ts"
},
"dependencies": {
"@anthropic-ai/sdk": "^0.74.0",
@@ -19,11 +21,20 @@
"pg": "^8.18.0",
"pg-copy-streams": "^7.0.0",
"pg-query-stream": "^4.16.0",
"split2": "^4.2.0",
"voyageai": "^0.1.0"
},
"devDependencies": {
"nodemon": "^3.1.9",
"@types/cors": "^2.8.19",
"@types/express": "^4.17.25",
"@types/node": "^26.1.2",
"@types/pg": "^8.20.3",
"@types/pg-copy-streams": "^1.2.5",
"@types/split2": "^4.2.3",
"@types/supertest": "^7.2.1",
"supertest": "^7.2.2",
"tsx": "^4.23.1",
"typescript": "^7.0.2",
"vitest": "^4.0.18"
}
}

48
backend/routes/ingest.routes.ts Normal file
View File

@@ -0,0 +1,48 @@
import { Router, type Request, type Response } from 'express';
import { requireApiKey } from '../middleware/apiKey.js';
import { createNotes } from '../db/notes.dao.js';
import { isNoteInputArray } from '../config/normalizers.js';
import type { NoteInput } from '../types/domain.js';
const router = Router();
const MAX_BATCH = 5000;
type IngestBody = { notes: NoteInput[] };
const isIngestBody = (value: unknown): value is IngestBody => {
if (typeof value !== 'object' || value === null) return false;
const body = value as Record<string, unknown>;
return isNoteInputArray(body.notes);
};
router.post('/', requireApiKey, async (req: Request, res: Response) => {
try {
if (!isIngestBody(req.body)) {
res.status(400).json({
error: 'Body must be { notes: [{ id, text, author, ... }] }',
});
return;
}
const { notes } = req.body;
if (notes.length === 0) {
res.status(400).json({ error: 'notes must not be empty' });
return;
}
if (notes.length > MAX_BATCH) {
res.status(400).json({ error: `notes exceeds the ${MAX_BATCH} per-request limit` });
return;
}
const inserted = await createNotes(notes);
res.status(201).json({ inserted: inserted.length, notes: inserted });
} catch (err) {
console.error(`Ingest failed: ${err}`);
res.status(500).json({ error: 'Ingest failed' });
}
});
export default router;

8
backend/routes/notes.routes.js → backend/routes/notes.routes.ts
View File

@@ -1,4 +1,4 @@
import { Router } from 'express';
import { Router, type Request, type Response } from 'express';
import { pipeline } from 'node:stream/promises';
import { getAllNotes, streamAllNotes } from '../db/notes.dao.js';
import { clusterNotes } from '../services/clustering.service.js';
@@ -6,7 +6,7 @@ import { jsonArray } from '../lib/streams.js';
const router = Router();
router.get('/', async (req, res) => {
router.get('/', async (_req: Request, res: Response) => {
try {
const rows = await streamAllNotes();
res.type('application/json');
@@ -14,14 +14,14 @@ router.get('/', async (req, res) => {
} catch (err) {
console.error(`Error loading notes: ${err}`);
if (res.headersSent) {
res.destroy(err);
res.destroy(err as Error);
return;
}
res.status(500).json({ error: 'Failed to load notes' });
}
});
router.post('/cluster', async (req, res) => {
router.post('/cluster', async (_req: Request, res: Response) => {
const controller = new AbortController();
res.on('close', () => {
if (!res.writableEnded) controller.abort();

113
backend/routes/webhooks.routes.ts Normal file
View File

@@ -0,0 +1,113 @@
import { Router, type NextFunction, type Request, type Response } from 'express';
import { getProvider } from '../config/providers.js';
import { verifySignature } from '../lib/signatures.js';
import { enqueue } from '../lib/queue.js';
import { normalize, NormalizationError } from '../services/normalize.service.js';
import {
markDone,
markFailed,
markProcessing,
recordDelivery,
} from '../db/ingest_events.dao.js';
import { createNotes } from '../db/notes.dao.js';
import { isProviderSlug, type NormalizedDelivery } from '../types/integration.js';
const router = Router();
const parseJson = (raw: Buffer): unknown => {
if (raw.length === 0) return {};
return JSON.parse(raw.toString('utf8'));
};
router.post('/:provider', async (req: Request, res: Response, next: NextFunction) => {
try {
const slug = req.params.provider;
if (!isProviderSlug(slug)) {
res.status(404).json({ error: `Unknown provider "${slug}"` });
return;
}
const config = getProvider(slug);
const rawBody = Buffer.isBuffer(req.body) ? req.body : Buffer.alloc(0);
let payload: unknown;
try {
payload = parseJson(rawBody);
} catch {
res.status(400).json({ error: 'Body is not valid JSON' });
return;
}
const challenge = config.challenge?.(payload);
if (challenge) {
res.status(challenge.status).json(challenge.body);
return;
}
if (config.signatureScheme !== 'none') {
const secret = config.secretEnvVar ? process.env[config.secretEnvVar] : undefined;
if (!secret) {
console.error(`No signing secret configured for provider "${slug}"`);
res.status(500).json({ error: 'Provider is not configured' });
return;
}
const result = verifySignature({
provider: slug,
rawBody,
headers: req.headers,
secret,
});
if (!result.ok) {
res.status(401).json({ error: 'Invalid signature' });
return;
}
}
if (!config.normalize) {
res.status(501).json({ error: `No ingestion mapping for provider "${slug}"` });
return;
}
// Normalizing before the ack costs nothing (it is pure) and lets a malformed
// payload fail loudly here rather than silently in a background job.
let delivery: NormalizedDelivery;
try {
delivery = normalize(slug, payload);
} catch (err) {
if (!(err instanceof NormalizationError)) throw err;
res.status(400).json({ error: err.message });
return;
}
const eventId = await recordDelivery({
provider: slug,
externalId: delivery.externalId,
});
if (eventId === null) {
res.status(200).json({ duplicate: true });
return;
}
const { notes } = delivery;
enqueue(`${slug}:${delivery.externalId}`, async () => {
await markProcessing(eventId);
try {
await createNotes(notes);
await markDone(eventId);
} catch (err) {
await markFailed(eventId, err instanceof Error ? err.message : String(err));
throw err;
}
});
res.status(200).json({ accepted: true, notes: notes.length });
} catch (err) {
next(err);
}
});
export default router;

2
backend/server.js → backend/server.ts
View File

@@ -5,4 +5,4 @@ const PORT = process.env.PORT || 3001;
app.listen(PORT, () => {
console.log(`Backend running on port ${PORT}`);
});
});

52
backend/services/clustering.service.js → backend/services/clustering.service.ts
View File

@@ -1,14 +1,21 @@
import Anthropic from "@anthropic-ai/sdk";
import { pipeline } from "node:stream/promises";
import { Transform, Writable } from "node:stream";
import { Transform, Writable, type TransformCallback } from "node:stream";
import { embedNotes } from "./embedding.service.js";
import { validateStructure, computeCohesionScore } from "./validation.service.js";
import { isClusterArray, type Cluster, type ClusterResponse } from "../types/domain.js";
const client = new Anthropic({
apiKey: process.env.ANTHROPIC_API_KEY,
});
const buildPrompt = (notes) => {
type ClusterableNote = { id: string; text: string };
type ClusterOptions = { signal?: AbortSignal };
type JsonSink = { parts: string[]; sawOpeningBracket: boolean };
const buildPrompt = (notes: ClusterableNote[]): string => {
const notesJson = JSON.stringify(notes, null, 2);
return `You are an expert at analyzing text for semantic similarity and thematic patterns.
@@ -35,10 +42,20 @@ Here are the notes:
${notesJson}`;
};
const textDeltas = () => new Transform({
const isTextDelta = (event: unknown): event is { delta: { text: string } } => {
if (typeof event !== 'object' || event === null) return false;
const candidate = event as { type?: unknown; delta?: { type?: unknown; text?: unknown } };
return (
candidate.type === 'content_block_delta' &&
candidate.delta?.type === 'text_delta' &&
typeof candidate.delta.text === 'string'
);
};
const textDeltas = (): Transform => new Transform({
objectMode: true,
transform(event, _encoding, callback) {
if (event?.type === 'content_block_delta' && event.delta?.type === 'text_delta') {
transform(event: unknown, _encoding: BufferEncoding, callback: TransformCallback) {
if (isTextDelta(event)) {
callback(null, event.delta.text);
return;
}
@@ -48,9 +65,9 @@ const textDeltas = () => new Transform({
// Rejects as soon as the first non-whitespace character proves the response
// is not the JSON array we asked for, rather than after the full generation.
const collectClusterJson = (sink) => new Writable({
const collectClusterJson = (sink: JsonSink): Writable => new Writable({
objectMode: true,
write(text, _encoding, callback) {
write(text: string, _encoding: BufferEncoding, callback: (error?: Error | null) => void) {
if (!sink.sawOpeningBracket) {
const leading = (sink.parts.join('') + text).trimStart();
if (leading.length > 0) {
@@ -66,8 +83,11 @@ const collectClusterJson = (sink) => new Writable({
},
});
const requestClusters = async (notes, signal) => {
const sink = { parts: [], sawOpeningBracket: false };
const requestClusters = async (
notes: ClusterableNote[],
signal?: AbortSignal
): Promise<Cluster[]> => {
const sink: JsonSink = { parts: [], sawOpeningBracket: false };
const options = signal ? [{ signal }] : [];
@@ -90,14 +110,24 @@ const requestClusters = async (notes, signal) => {
throw new Error('Unexpected response from LLM API: no text content returned');
}
let parsed: unknown;
try {
return JSON.parse(text);
parsed = JSON.parse(text);
} catch {
throw new Error('LLM API returned non-JSON response');
}
if (!isClusterArray(parsed)) {
throw new Error('LLM API returned clusters in an unexpected shape');
}
return parsed;
};
export const clusterNotes = async (notes, { signal } = {}) => {
export const clusterNotes = async (
notes: ClusterableNote[],
{ signal }: ClusterOptions = {}
): Promise<ClusterResponse> => {
const [clusters, embeddingMap] = await Promise.all([
requestClusters(notes, signal),
embedNotes(notes),

20
backend/services/embedding.service.js → backend/services/embedding.service.ts
View File

@@ -2,6 +2,7 @@ import { VoyageAIClient } from "voyageai";
import { pipeline } from "node:stream/promises";
import { Readable } from "node:stream";
import { batch } from "../lib/streams.js";
import type { EmbeddingMap } from "../types/domain.js";
const client = new VoyageAIClient({
apiKey: process.env.VOYAGEAI_API_KEY,
@@ -9,12 +10,13 @@ const client = new VoyageAIClient({
const EMBED_BATCH_SIZE = 128;
type EmbeddableNote = { id: string; text: string };
/**
* @param {Array<{id: string, text: string}>} notes
* @returns {Promise<Map<string, number[]>>} noteId → embedding vector
* @returns noteId to embedding vector
*/
export const embedNotes = async (notes) => {
const embeddingMap = new Map();
export const embedNotes = async (notes: EmbeddableNote[]): Promise<EmbeddingMap> => {
const embeddingMap: EmbeddingMap = new Map();
if (!notes || notes.length === 0) {
return embeddingMap;
@@ -22,16 +24,18 @@ export const embedNotes = async (notes) => {
await pipeline(
Readable.from(notes, { objectMode: true }),
batch(EMBED_BATCH_SIZE),
async (batches) => {
batch<EmbeddableNote>(EMBED_BATCH_SIZE),
async (batches: AsyncIterable<EmbeddableNote[]>) => {
for await (const chunk of batches) {
const response = await client.embed({
input: chunk.map((n) => n.text),
model: "voyage-3.5",
});
response.data.forEach((item, i) => {
embeddingMap.set(chunk[i].id, item.embedding);
response.data?.forEach((item, i) => {
if (item.embedding) {
embeddingMap.set(chunk[i].id, item.embedding);
}
});
}
}

23
backend/services/normalize.service.ts Normal file
View File

@@ -0,0 +1,23 @@
import { getProvider } from '../config/providers.js';
import { NormalizationError } from '../config/normalizers.js';
import type { NormalizedDelivery, ProviderSlug } from '../types/integration.js';
export { NormalizationError };
export const normalize = (
provider: ProviderSlug,
payload: unknown
): NormalizedDelivery => {
const normalizer = getProvider(provider).normalize;
if (!normalizer) {
throw new NormalizationError(
`No normalizer registered for provider "${provider}"`
);
}
return normalizer(payload);
};
export const hasNormalizer = (provider: ProviderSlug): boolean =>
getProvider(provider).normalize !== undefined;

182
backend/services/oauth.service.ts Normal file
View File

@@ -0,0 +1,182 @@
import { requestJson } from '../lib/httpClient.js';
import { getProvider } from '../config/providers.js';
import {
getAccessToken,
getRefreshToken,
findByProviderWorkspace,
updateTokens,
} from '../db/integrations.dao.js';
import type { IntegrationRow, ProviderSlug } from '../types/integration.js';
/** Refresh this far ahead of expiry so an in-flight call cannot straddle it. */
const REFRESH_MARGIN_MS = 60_000;
export type TokenSet = {
accessToken: string;
refreshToken?: string;
expiresAt?: Date;
scopes: string[];
};
type TokenResponse = {
access_token: string;
refresh_token?: string;
expires_in?: number;
scope?: string;
};
export class OAuthError extends Error {
constructor(message: string) {
super(message);
this.name = 'OAuthError';
}
}
const isTokenResponse = (value: unknown): value is TokenResponse => {
if (typeof value !== 'object' || value === null) return false;
const body = value as Record<string, unknown>;
if (typeof body.access_token !== 'string' || body.access_token.length === 0) return false;
if (body.refresh_token !== undefined && typeof body.refresh_token !== 'string') return false;
if (body.expires_in !== undefined && typeof body.expires_in !== 'number') return false;
if (body.scope !== undefined && typeof body.scope !== 'string') return false;
return true;
};
const credentials = (provider: ProviderSlug): { id: string; secret: string } => {
const prefix = provider.toUpperCase();
const id = process.env[`${prefix}_CLIENT_ID`];
const secret = process.env[`${prefix}_CLIENT_SECRET`];
if (!id || !secret) {
throw new OAuthError(
`Missing ${prefix}_CLIENT_ID or ${prefix}_CLIENT_SECRET`
);
}
return { id, secret };
};
const oauthConfig = (provider: ProviderSlug) => {
const config = getProvider(provider).oauth;
if (!config) {
throw new OAuthError(`Provider "${provider}" does not support OAuth`);
}
return config;
};
const toTokenSet = (body: TokenResponse): TokenSet => ({
accessToken: body.access_token,
refreshToken: body.refresh_token,
expiresAt: body.expires_in
? new Date(Date.now() + body.expires_in * 1000)
: undefined,
scopes: body.scope ? body.scope.split(/[\s,]+/).filter(Boolean) : [],
});
export const buildAuthorizeUrl = (
provider: ProviderSlug,
input: { redirectUri: string; state: string }
): string => {
const config = oauthConfig(provider);
const url = new URL(config.authorizeUrl);
url.searchParams.set('client_id', credentials(provider).id);
url.searchParams.set('redirect_uri', input.redirectUri);
url.searchParams.set('response_type', 'code');
url.searchParams.set('state', input.state);
url.searchParams.set('scope', config.scopes.join(' '));
return url.toString();
};
const postForm = async (
url: string,
form: Record<string, string>
): Promise<TokenSet> => {
const body = await requestJson<unknown>(url, {
method: 'POST',
headers: {
'content-type': 'application/x-www-form-urlencoded',
accept: 'application/json',
},
body: new URLSearchParams(form).toString(),
});
if (!isTokenResponse(body)) {
throw new OAuthError('Token endpoint returned an unexpected payload');
}
return toTokenSet(body);
};
export const exchangeCode = async (
provider: ProviderSlug,
input: { code: string; redirectUri: string }
): Promise<TokenSet> => {
const { id, secret } = credentials(provider);
return postForm(oauthConfig(provider).tokenUrl, {
grant_type: 'authorization_code',
code: input.code,
redirect_uri: input.redirectUri,
client_id: id,
client_secret: secret,
});
};
export const refreshAccessToken = async (
provider: ProviderSlug,
refreshToken: string
): Promise<TokenSet> => {
const { id, secret } = credentials(provider);
return postForm(oauthConfig(provider).tokenUrl, {
grant_type: 'refresh_token',
refresh_token: refreshToken,
client_id: id,
client_secret: secret,
});
};
const needsRefresh = (integration: IntegrationRow): boolean => {
if (!integration.tokenExpiresAt) return false;
return integration.tokenExpiresAt.getTime() - Date.now() <= REFRESH_MARGIN_MS;
};
/**
* Resolves a usable access token, refreshing first when the stored one is at
* or near expiry. Throws rather than returning null so a caller cannot make an
* unauthenticated request by forgetting a null check.
*/
export const getValidAccessToken = async (
provider: ProviderSlug,
externalWorkspaceId: string
): Promise<string> => {
const integration = await findByProviderWorkspace(provider, externalWorkspaceId);
if (!integration) {
throw new OAuthError(`No ${provider} integration for workspace ${externalWorkspaceId}`);
}
if (needsRefresh(integration)) {
const refreshToken = await getRefreshToken(integration.id);
if (!refreshToken) {
throw new OAuthError(`${provider} token expired and no refresh token is stored`);
}
const tokens = await refreshAccessToken(provider, refreshToken);
await updateTokens({
id: integration.id,
accessToken: tokens.accessToken,
refreshToken: tokens.refreshToken,
expiresAt: tokens.expiresAt,
});
return tokens.accessToken;
}
const accessToken = await getAccessToken(integration.id);
if (!accessToken) {
throw new OAuthError(`No access token stored for ${provider}`);
}
return accessToken;
};

31
backend/services/validation.service.js → backend/services/validation.service.ts
View File

@@ -1,18 +1,22 @@
import type { Cluster, EmbeddingMap, ValidationResult } from '../types/domain.js';
/**
* Structural validation for LLM output
*
* @param {Array<{label: string, noteIds: string[]}>} clusters
* @param {string[]} inputNoteIds - the original note IDs that were sent to the LLM
* @returns {{valid: boolean, reasons: string[]}}
* @param clusters
* @param inputNoteIds - the original note IDs that were sent to the LLM
*/
export const validateStructure = (clusters, inputNoteIds) => {
const reasons = [];
export const validateStructure = (
clusters: Cluster[],
inputNoteIds: string[]
): ValidationResult => {
const reasons: string[] = [];
if (!Array.isArray(clusters) || clusters.length === 0) {
return { valid: false, reasons: ['Response is not a non-empty array'] };
}
const assignedIds = [];
const assignedIds: string[] = [];
for (const cluster of clusters) {
if (!cluster.label || typeof cluster.label !== 'string') {
reasons.push(`Cluster missing a valid label`);
@@ -47,7 +51,7 @@ export const validateStructure = (clusters, inputNoteIds) => {
return { valid: reasons.length === 0, reasons };
};
const cosineSimilarity = (a, b) => {
const cosineSimilarity = (a: number[], b: number[]): number => {
let dot = 0;
let magA = 0;
let magB = 0;
@@ -67,14 +71,17 @@ const cosineSimilarity = (a, b) => {
* versus the nearest neighboring cluster. Returns a score in [-1, 1]
* where higher is better.
*
* @param {Array<{label: string, noteIds: string[]}>} clusters
* @param {Map<string, number[]>} embeddingMap - noteId → vector
* @returns {number} average silhouette score
* @param clusters
* @param embeddingMap - noteId to vector
* @returns average silhouette score
*/
export const computeCohesionScore = (clusters, embeddingMap) => {
export const computeCohesionScore = (
clusters: Cluster[],
embeddingMap: EmbeddingMap
): number => {
if (clusters.length <= 1) return 1.0;
const scores = [];
const scores: number[] = [];
for (let ci = 0; ci < clusters.length; ci++) {
const clusterIds = clusters[ci].noteIds;

34
backend/tests/Clustering.service.test.js → backend/tests/Clustering.service.test.ts
View File

@@ -1,19 +1,23 @@
import { describe, it, expect, vi, beforeEach } from 'vitest';
import type { Cluster, EmbeddingMap } from '../types/domain.js';
type StreamEvent = {
type: string;
delta?: { type: string; text: string };
};
const { streamMock, mockEmbeddings } = vi.hoisted(() => {
const embeddings = new Map([
const embeddings = new Map<string, number[]>([
['note_001', [1.0, 0.0, 0.0]],
['note_002', [0.0, 1.0, 0.0]],
]);
return { streamMock: vi.fn(), mockEmbeddings: embeddings };
return { streamMock: vi.fn(), mockEmbeddings: embeddings as EmbeddingMap };
});
vi.mock('@anthropic-ai/sdk', () => {
return {
default: class MockAnthropic {
constructor() {
this.messages = { stream: streamMock };
}
messages = { stream: streamMock };
},
};
});
@@ -29,16 +33,16 @@ const MOCK_NOTES = [
{ id: 'note_002', text: 'Export fails' },
];
const MOCK_CLUSTERS = [
const MOCK_CLUSTERS: Cluster[] = [
{ label: 'Auth Issues', noteIds: ['note_001'] },
{ label: 'Export Issues', noteIds: ['note_002'] },
];
// Splits text into several text_delta events so the service is exercised
// against a genuinely incremental stream rather than one whole payload.
const textEvents = (text, pieces = 4) => {
const textEvents = (text: string, pieces = 4): StreamEvent[] => {
const size = Math.max(1, Math.ceil(text.length / pieces));
const events = [];
const events: StreamEvent[] = [];
for (let i = 0; i < text.length; i += size) {
events.push({
type: 'content_block_delta',
@@ -48,8 +52,8 @@ const textEvents = (text, pieces = 4) => {
return events;
};
const mockStreamOf = (text) => {
const events = [
const mockStreamOf = (text: string): void => {
const events: StreamEvent[] = [
{ type: 'message_start' },
...textEvents(text),
{ type: 'message_stop' },
@@ -61,7 +65,7 @@ const mockStreamOf = (text) => {
}));
};
const mockStreamThrowing = (err) => {
const mockStreamThrowing = (err: Error): void => {
streamMock.mockImplementation(() => ({
async *[Symbol.asyncIterator]() {
throw err;
@@ -152,11 +156,17 @@ describe('clusterNotes service', () => {
});
it('should throw a validation error when a note is missing from clusters', async () => {
const incompleteClusters = [
const incompleteClusters: Cluster[] = [
{ label: 'Auth Issues', noteIds: ['note_001'] },
];
mockStreamOf(JSON.stringify(incompleteClusters));
await expect(clusterNotes(MOCK_NOTES)).rejects.toThrow('Cluster validation failed');
});
it('should throw when the API returns a JSON array of the wrong shape', async () => {
mockStreamOf(JSON.stringify([{ name: 'Auth Issues', ids: ['note_001'] }]));
await expect(clusterNotes(MOCK_NOTES)).rejects.toThrow('unexpected shape');
});
});

59
backend/tests/Notes.test.js → backend/tests/Notes.test.ts
View File

@@ -2,6 +2,7 @@ import { describe, it, expect, vi, beforeEach } from 'vitest';
import request from 'supertest';
import { Readable } from 'node:stream';
import app from '../app.js';
import type { Cluster, ClusterResponse, Note } from '../types/domain.js';
vi.mock('../db/notes.dao.js', () => ({
getAllNotes: vi.fn(),
@@ -15,18 +16,24 @@ vi.mock('../services/clustering.service.js', () => ({
import { getAllNotes, streamAllNotes } from '../db/notes.dao.js';
import { clusterNotes } from '../services/clustering.service.js';
const MOCK_NOTES = [
const mockGetAllNotes = vi.mocked(getAllNotes);
const mockStreamAllNotes = vi.mocked(streamAllNotes);
const mockClusterNotes = vi.mocked(clusterNotes);
const MOCK_NOTES: Note[] = [
{ id: 'note_001', text: 'Login flow feels confusing', x: 193, y: 191, author: 'user_5', color: 'yellow' },
{ id: 'note_002', text: 'Login flow is broken on mobile', x: 214, y: 281, author: 'user_9', color: 'yellow' },
{ id: 'note_003', text: 'Export takes too long', x: 798, y: 211, author: 'user_2', color: 'green' },
];
const MOCK_CLUSTERS = [
const MOCK_CLUSTERS: Cluster[] = [
{ label: 'Login Issues', noteIds: ['note_001', 'note_002'] },
{ label: 'Export Problems', noteIds: ['note_003'] },
];
const rowStream = (rows) => Readable.from(rows, { objectMode: true });
const MOCK_RESULT: ClusterResponse = { clusters: MOCK_CLUSTERS, score: 0.09 };
const rowStream = (rows: Note[]): Readable => Readable.from(rows, { objectMode: true });
describe('GET /v1/notes', () => {
@@ -35,7 +42,7 @@ describe('GET /v1/notes', () => {
});
it('should return 200 and an array of notes', async () => {
streamAllNotes.mockResolvedValue(rowStream(MOCK_NOTES));
mockStreamAllNotes.mockResolvedValue(rowStream(MOCK_NOTES));
const res = await request(app).get('/v1/notes');
@@ -45,7 +52,7 @@ describe('GET /v1/notes', () => {
});
it('should send JSON incrementally rather than buffering the row set', async () => {
streamAllNotes.mockResolvedValue(rowStream(MOCK_NOTES));
mockStreamAllNotes.mockResolvedValue(rowStream(MOCK_NOTES));
const res = await request(app).get('/v1/notes');
@@ -54,7 +61,7 @@ describe('GET /v1/notes', () => {
});
it('should return an empty array when there are no notes', async () => {
streamAllNotes.mockResolvedValue(rowStream([]));
mockStreamAllNotes.mockResolvedValue(rowStream([]));
const res = await request(app).get('/v1/notes');
@@ -63,7 +70,7 @@ describe('GET /v1/notes', () => {
});
it('should return notes with expected properties', async () => {
streamAllNotes.mockResolvedValue(rowStream(MOCK_NOTES));
mockStreamAllNotes.mockResolvedValue(rowStream(MOCK_NOTES));
const res = await request(app).get('/v1/notes');
const note = res.body[0];
@@ -77,7 +84,7 @@ describe('GET /v1/notes', () => {
});
it('should return 500 when the database query fails', async () => {
streamAllNotes.mockRejectedValue(new Error('connection refused'));
mockStreamAllNotes.mockRejectedValue(new Error('connection refused'));
const res = await request(app).get('/v1/notes');
@@ -94,7 +101,7 @@ describe('GET /v1/notes', () => {
this.destroy(new Error('connection lost'));
},
});
streamAllNotes.mockResolvedValue(failing);
mockStreamAllNotes.mockResolvedValue(failing);
await expect(request(app).get('/v1/notes')).rejects.toThrow();
});
@@ -107,21 +114,21 @@ describe('POST /v1/notes/cluster', () => {
});
it('should return 200 and clustered results', async () => {
getAllNotes.mockResolvedValue(MOCK_NOTES);
clusterNotes.mockResolvedValue(MOCK_CLUSTERS);
mockGetAllNotes.mockResolvedValue(MOCK_NOTES);
mockClusterNotes.mockResolvedValue(MOCK_RESULT);
const res = await request(app).post('/v1/notes/cluster');
expect(res.status).toBe(200);
expect(res.body).toEqual(MOCK_CLUSTERS);
expect(res.body).toEqual(MOCK_RESULT);
});
it('should return clusters with the expected shape (label, noteIds)', async () => {
getAllNotes.mockResolvedValue(MOCK_NOTES);
clusterNotes.mockResolvedValue(MOCK_CLUSTERS);
mockGetAllNotes.mockResolvedValue(MOCK_NOTES);
mockClusterNotes.mockResolvedValue(MOCK_RESULT);
const res = await request(app).post('/v1/notes/cluster');
const cluster = res.body[0];
const cluster = res.body.clusters[0];
expect(cluster).toHaveProperty('label');
expect(cluster).toHaveProperty('noteIds');
@@ -130,23 +137,23 @@ describe('POST /v1/notes/cluster', () => {
});
it('should pass the loaded notes to clusterNotes', async () => {
getAllNotes.mockResolvedValue(MOCK_NOTES);
clusterNotes.mockResolvedValue(MOCK_CLUSTERS);
mockGetAllNotes.mockResolvedValue(MOCK_NOTES);
mockClusterNotes.mockResolvedValue(MOCK_RESULT);
await request(app).post('/v1/notes/cluster');
expect(clusterNotes).toHaveBeenCalledOnce();
expect(clusterNotes).toHaveBeenCalledWith(
expect(mockClusterNotes).toHaveBeenCalledOnce();
expect(mockClusterNotes).toHaveBeenCalledWith(
MOCK_NOTES,
expect.objectContaining({ signal: expect.any(AbortSignal) })
);
});
it('should pass a signal that is not aborted while the request is open', async () => {
getAllNotes.mockResolvedValue(MOCK_NOTES);
clusterNotes.mockImplementation(async (_notes, { signal }) => {
expect(signal.aborted).toBe(false);
return MOCK_CLUSTERS;
mockGetAllNotes.mockResolvedValue(MOCK_NOTES);
mockClusterNotes.mockImplementation(async (_notes, options = {}) => {
expect(options.signal?.aborted).toBe(false);
return MOCK_RESULT;
});
const res = await request(app).post('/v1/notes/cluster');
@@ -155,8 +162,8 @@ describe('POST /v1/notes/cluster', () => {
});
it('should return 500 when clusterNotes (API call) fails', async () => {
getAllNotes.mockResolvedValue(MOCK_NOTES);
clusterNotes.mockRejectedValue(new Error('LLM API error'));
mockGetAllNotes.mockResolvedValue(MOCK_NOTES);
mockClusterNotes.mockRejectedValue(new Error('LLM API error'));
const res = await request(app).post('/v1/notes/cluster');
@@ -166,7 +173,7 @@ describe('POST /v1/notes/cluster', () => {
});
it('should return 500 when the database query fails', async () => {
getAllNotes.mockRejectedValue(new Error('connection refused'));
mockGetAllNotes.mockRejectedValue(new Error('connection refused'));
const res = await request(app).post('/v1/notes/cluster');

170
backend/tests/crypto.test.ts Normal file
View File

@@ -0,0 +1,170 @@
import { describe, it, expect, beforeEach, afterEach } from 'vitest';
import { randomBytes } from 'node:crypto';
import {
encryptSecret,
decryptSecret,
hashApiKey,
safeEquals,
} from '../lib/crypto.js';
const base64Key = (bytes: number): string => randomBytes(bytes).toString('base64');
const VALID_KEY = base64Key(32);
/** Rewrites one `iv:tag:payload` segment, flipping every bit of its first byte. */
const corruptSegment = (ciphertext: string, index: number): string => {
const parts = ciphertext.split(':');
const bytes = Buffer.from(parts[index], 'base64');
bytes[0] = bytes[0] ^ 0xff;
parts[index] = bytes.toString('base64');
return parts.join(':');
};
describe('encryptSecret / decryptSecret', () => {
const originalKey = process.env.TOKEN_ENCRYPTION_KEY;
beforeEach(() => {
process.env.TOKEN_ENCRYPTION_KEY = VALID_KEY;
});
afterEach(() => {
if (originalKey === undefined) {
delete process.env.TOKEN_ENCRYPTION_KEY;
} else {
process.env.TOKEN_ENCRYPTION_KEY = originalKey;
}
});
it('should round-trip a plaintext secret', () => {
const plaintext = 'lin_oauth_abc123';
expect(decryptSecret(encryptSecret(plaintext))).toBe(plaintext);
});
it('should round-trip an empty string and multi-byte characters', () => {
expect(decryptSecret(encryptSecret(''))).toBe('');
expect(decryptSecret(encryptSecret('clé—😀'))).toBe('clé—😀');
});
it('should emit three base64 segments', () => {
const parts = encryptSecret('token').split(':');
expect(parts).toHaveLength(3);
expect(Buffer.from(parts[0], 'base64')).toHaveLength(12);
expect(Buffer.from(parts[1], 'base64')).toHaveLength(16);
});
it('should produce different ciphertext for the same plaintext each time', () => {
const first = encryptSecret('same-secret');
const second = encryptSecret('same-secret');
expect(first).not.toBe(second);
expect(first.split(':')[0]).not.toBe(second.split(':')[0]);
expect(decryptSecret(first)).toBe('same-secret');
expect(decryptSecret(second)).toBe('same-secret');
});
it('should throw when the ciphertext payload is tampered with', () => {
const tampered = corruptSegment(encryptSecret('payload-under-attack'), 2);
expect(() => decryptSecret(tampered)).toThrow();
});
it('should throw when the auth tag is tampered with', () => {
const tampered = corruptSegment(encryptSecret('tag-under-attack'), 1);
expect(() => decryptSecret(tampered)).toThrow();
});
it('should throw when the iv is tampered with', () => {
const tampered = corruptSegment(encryptSecret('iv-under-attack'), 0);
expect(() => decryptSecret(tampered)).toThrow();
});
it('should throw for a ciphertext that is not in iv:tag:payload form', () => {
expect(() => decryptSecret('not-a-ciphertext')).toThrow(
/iv:tag:payload/
);
expect(() => decryptSecret('only:two')).toThrow(/iv:tag:payload/);
expect(() => decryptSecret('a:b:c:d')).toThrow(/iv:tag:payload/);
expect(() => decryptSecret('')).toThrow(/iv:tag:payload/);
});
it('should throw when decrypting under a different key', () => {
const ciphertext = encryptSecret('bound-to-one-key');
process.env.TOKEN_ENCRYPTION_KEY = base64Key(32);
expect(() => decryptSecret(ciphertext)).toThrow();
});
it('should throw when TOKEN_ENCRYPTION_KEY is missing', () => {
delete process.env.TOKEN_ENCRYPTION_KEY;
expect(() => encryptSecret('anything')).toThrow(/not set/);
expect(() => decryptSecret('a:b:c')).toThrow(/not set/);
});
it('should throw when TOKEN_ENCRYPTION_KEY is empty', () => {
process.env.TOKEN_ENCRYPTION_KEY = '';
expect(() => encryptSecret('anything')).toThrow(/not set/);
});
it('should throw when the key decodes to the wrong byte length', () => {
process.env.TOKEN_ENCRYPTION_KEY = base64Key(16);
expect(() => encryptSecret('anything')).toThrow(/32 bytes, got 16/);
process.env.TOKEN_ENCRYPTION_KEY = base64Key(48);
expect(() => encryptSecret('anything')).toThrow(/32 bytes, got 48/);
});
it('should read the key at call time rather than at import time', () => {
process.env.TOKEN_ENCRYPTION_KEY = base64Key(31);
expect(() => encryptSecret('anything')).toThrow(/got 31/);
process.env.TOKEN_ENCRYPTION_KEY = VALID_KEY;
expect(decryptSecret(encryptSecret('recovered'))).toBe('recovered');
});
});
describe('hashApiKey', () => {
it('should be stable for the same input', () => {
expect(hashApiKey('kg_live_abc')).toBe(hashApiKey('kg_live_abc'));
});
it('should differ for different inputs', () => {
expect(hashApiKey('kg_live_abc')).not.toBe(hashApiKey('kg_live_abd'));
expect(hashApiKey('')).not.toBe(hashApiKey(' '));
});
it('should return a 64-character lowercase hex digest', () => {
expect(hashApiKey('kg_live_abc')).toMatch(/^[0-9a-f]{64}$/);
});
it('should match the plain sha256 digest of the input', () => {
expect(hashApiKey('')).toBe(
'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855'
);
});
});
describe('safeEquals', () => {
it('should return true for identical strings', () => {
expect(safeEquals('kg_live_abc', 'kg_live_abc')).toBe(true);
expect(safeEquals('', '')).toBe(true);
});
it('should return false for different strings of equal length', () => {
expect(safeEquals('kg_live_abc', 'kg_live_abd')).toBe(false);
});
it('should return false for strings of differing lengths without throwing', () => {
expect(safeEquals('short', 'a-much-longer-value')).toBe(false);
expect(safeEquals('', 'x')).toBe(false);
});
it('should be case sensitive', () => {
expect(safeEquals('Secret', 'secret')).toBe(false);
});
});

241
backend/tests/httpClient.test.ts Normal file
View File

@@ -0,0 +1,241 @@
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
import { requestJson, isHttpRequestError, parseRetryAfter } from '../lib/httpClient.js';
const URL_UNDER_TEST = 'https://api.example.com/v1/token';
const jsonResponse = (
body: unknown,
status = 200,
headers: Record<string, string> = {}
): Response =>
new Response(JSON.stringify(body), {
status,
headers: { 'content-type': 'application/json', ...headers },
});
/** Never settles on its own; only the per-attempt timeout or the caller's signal ends it. */
const hangUntilAborted = (init?: RequestInit): Promise<Response> =>
new Promise<Response>((_resolve, reject) => {
init?.signal?.addEventListener('abort', () => {
reject(init.signal?.reason ?? new Error('aborted'));
});
});
// Backoff is kept at a single millisecond so retries are asserted by call count, never by clock.
const fast = { maxAttempts: 3, baseDelayMs: 1 } as const;
describe('requestJson', () => {
let fetchMock: ReturnType<typeof vi.fn>;
beforeEach(() => {
fetchMock = vi.fn();
vi.stubGlobal('fetch', fetchMock);
});
afterEach(() => {
vi.unstubAllGlobals();
});
it('should resolve and parse a 200 JSON body', async () => {
fetchMock.mockResolvedValueOnce(jsonResponse({ access_token: 'abc', expires_in: 3600 }));
const result = await requestJson<{ access_token: string; expires_in: number }>(
URL_UNDER_TEST,
fast
);
expect(result).toEqual({ access_token: 'abc', expires_in: 3600 });
expect(fetchMock).toHaveBeenCalledTimes(1);
});
it('should retry a 429 with a small Retry-After and then succeed', async () => {
fetchMock
.mockResolvedValueOnce(jsonResponse({ error: 'slow down' }, 429, { 'retry-after': '0' }))
.mockResolvedValueOnce(jsonResponse({ ok: true }));
const result = await requestJson<{ ok: boolean }>(URL_UNDER_TEST, fast);
expect(result).toEqual({ ok: true });
expect(fetchMock).toHaveBeenCalledTimes(2);
});
it('should honor an HTTP-date Retry-After on a 429', async () => {
const httpDate = new Date(Date.now() + 500).toUTCString();
fetchMock
.mockResolvedValueOnce(jsonResponse({}, 429, { 'retry-after': httpDate }))
.mockResolvedValueOnce(jsonResponse({ ok: true }));
const result = await requestJson<{ ok: boolean }>(URL_UNDER_TEST, fast);
expect(result).toEqual({ ok: true });
expect(fetchMock).toHaveBeenCalledTimes(2);
});
it('should retry a 500 up to maxAttempts and then throw', async () => {
fetchMock.mockImplementation(() => Promise.resolve(jsonResponse({ error: 'boom' }, 500)));
await expect(requestJson(URL_UNDER_TEST, fast)).rejects.toThrow(/status 500/);
expect(fetchMock).toHaveBeenCalledTimes(3);
});
it('should throw immediately on a 400 without retrying', async () => {
fetchMock.mockResolvedValueOnce(jsonResponse({ error: 'invalid_grant' }, 400));
await expect(requestJson(URL_UNDER_TEST, fast)).rejects.toThrow();
expect(fetchMock).toHaveBeenCalledTimes(1);
});
it('should throw immediately on a 401 without retrying', async () => {
fetchMock.mockResolvedValueOnce(jsonResponse({ error: 'unauthorized' }, 401));
await expect(requestJson(URL_UNDER_TEST, fast)).rejects.toThrow();
expect(fetchMock).toHaveBeenCalledTimes(1);
});
it('should treat a per-attempt timeout as retryable', async () => {
fetchMock
.mockImplementationOnce((_url: string, init?: RequestInit) => hangUntilAborted(init))
.mockResolvedValueOnce(jsonResponse({ ok: true }));
const result = await requestJson<{ ok: boolean }>(URL_UNDER_TEST, {
...fast,
timeoutMs: 1,
});
expect(result).toEqual({ ok: true });
expect(fetchMock).toHaveBeenCalledTimes(2);
});
it('should exhaust attempts when every attempt times out', async () => {
fetchMock.mockImplementation((_url: string, init?: RequestInit) => hangUntilAborted(init));
const error = await requestJson(URL_UNDER_TEST, { ...fast, maxAttempts: 2, timeoutMs: 1 })
.then(() => null)
.catch((err: unknown) => err);
expect(isHttpRequestError(error)).toBe(true);
expect(isHttpRequestError(error) && error.kind).toBe('timeout');
expect(fetchMock).toHaveBeenCalledTimes(2);
});
it('should identify the url, status and attempt count in the thrown message', async () => {
fetchMock.mockResolvedValueOnce(jsonResponse({ error: 'invalid_grant' }, 400));
const error = await requestJson(URL_UNDER_TEST, fast)
.then(() => null)
.catch((err: unknown) => err);
expect(isHttpRequestError(error)).toBe(true);
const message = error instanceof Error ? error.message : '';
expect(message).toContain(URL_UNDER_TEST);
expect(message).toContain('status 400');
expect(message).toContain('1 attempt(s)');
expect(isHttpRequestError(error) && error.status).toBe(400);
});
it('should distinguish an HTTP failure from a programming error', async () => {
fetchMock.mockResolvedValueOnce(jsonResponse({}, 400));
const httpError = await requestJson(URL_UNDER_TEST, fast)
.then(() => null)
.catch((err: unknown) => err);
expect(isHttpRequestError(httpError)).toBe(true);
expect(isHttpRequestError(new TypeError('bad call'))).toBe(false);
});
it('should reject a 2xx whose body is not valid JSON', async () => {
fetchMock.mockResolvedValueOnce(new Response('<html>maintenance</html>', { status: 200 }));
const error = await requestJson(URL_UNDER_TEST, fast)
.then(() => null)
.catch((err: unknown) => err);
expect(isHttpRequestError(error) && error.kind).toBe('invalid-body');
expect(fetchMock).toHaveBeenCalledTimes(1);
});
it('should resolve undefined for a bodiless 204', async () => {
fetchMock.mockResolvedValueOnce(new Response(null, { status: 204 }));
await expect(requestJson(URL_UNDER_TEST, fast)).resolves.toBeUndefined();
});
it('should retry a transport error and then succeed', async () => {
fetchMock
.mockRejectedValueOnce(new TypeError('fetch failed'))
.mockResolvedValueOnce(jsonResponse({ ok: true }));
const result = await requestJson<{ ok: boolean }>(URL_UNDER_TEST, fast);
expect(result).toEqual({ ok: true });
expect(fetchMock).toHaveBeenCalledTimes(2);
});
it('should not call fetch when the caller signal is already aborted', async () => {
fetchMock.mockResolvedValue(jsonResponse({ ok: true }));
const error = await requestJson(URL_UNDER_TEST, { ...fast, signal: AbortSignal.abort() })
.then(() => null)
.catch((err: unknown) => err);
expect(isHttpRequestError(error) && error.kind).toBe('aborted');
expect(fetchMock).not.toHaveBeenCalled();
});
it('should abort without retrying when the caller signal fires mid-flight', async () => {
const controller = new AbortController();
fetchMock.mockImplementation((_url: string, init?: RequestInit) => {
queueMicrotask(() => controller.abort());
return hangUntilAborted(init);
});
const error = await requestJson(URL_UNDER_TEST, { ...fast, signal: controller.signal })
.then(() => null)
.catch((err: unknown) => err);
expect(isHttpRequestError(error) && error.kind).toBe('aborted');
expect(fetchMock).toHaveBeenCalledTimes(1);
});
it('should reject a non-positive maxAttempts', async () => {
await expect(requestJson(URL_UNDER_TEST, { maxAttempts: 0 })).rejects.toThrow(TypeError);
expect(fetchMock).not.toHaveBeenCalled();
});
});
describe('parseRetryAfter', () => {
it('should read the delay-seconds form', () => {
expect(parseRetryAfter('30')).toBe(30_000);
expect(parseRetryAfter('0')).toBe(0);
});
it('should read the HTTP-date form as a delay from now', () => {
const delay = parseRetryAfter(new Date(Date.now() + 30_000).toUTCString());
expect(delay).not.toBeNull();
expect(delay).toBeGreaterThan(25_000);
expect(delay).toBeLessThanOrEqual(30_000);
});
it('should clamp a past HTTP-date to zero', () => {
expect(parseRetryAfter(new Date(Date.now() - 60_000).toUTCString())).toBe(0);
});
it('should cap a hostile delay-seconds value', () => {
expect(parseRetryAfter('86400')).toBe(60_000);
});
it('should cap a hostile HTTP-date value', () => {
expect(parseRetryAfter(new Date(Date.now() + 86_400_000).toUTCString())).toBe(60_000);
});
it('should return null for a missing or unintelligible header', () => {
expect(parseRetryAfter(null)).toBeNull();
expect(parseRetryAfter(' ')).toBeNull();
expect(parseRetryAfter('soon')).toBeNull();
});
});

158
backend/tests/ingest.routes.test.ts Normal file
View File

@@ -0,0 +1,158 @@
import { describe, it, expect, vi, beforeEach } from 'vitest';
import request from 'supertest';
import app from '../app.js';
import type { IntegrationRow } from '../types/integration.js';
import type { Note, NoteInput } from '../types/domain.js';
vi.mock('../db/integrations.dao.js', () => ({
findByApiKeyHash: vi.fn(),
}));
vi.mock('../db/notes.dao.js', () => ({
createNotes: vi.fn(),
getAllNotes: vi.fn(),
streamAllNotes: vi.fn(),
}));
import { findByApiKeyHash } from '../db/integrations.dao.js';
import { createNotes } from '../db/notes.dao.js';
const mockFindByApiKeyHash = vi.mocked(findByApiKeyHash);
const mockCreateNotes = vi.mocked(createNotes);
const INTEGRATION: IntegrationRow = {
id: 7,
provider: 'rest',
externalWorkspaceId: 'acme',
displayName: 'Acme',
scopes: [],
tokenExpiresAt: null,
};
const INPUT: NoteInput[] = [
{ id: 'note_100', text: 'Retro: deploys are scary', author: 'user_1' },
];
const STORED: Note[] = [
{ id: 'note_100', text: 'Retro: deploys are scary', x: 0, y: 0, author: 'user_1', color: 'yellow' },
];
const post = (body: object, key = 'secret-key') =>
request(app).post('/v1/notes').set('Authorization', `Bearer ${key}`).send(body);
describe('POST /v1/notes', () => {
beforeEach(() => {
vi.clearAllMocks();
mockFindByApiKeyHash.mockResolvedValue(INTEGRATION);
mockCreateNotes.mockResolvedValue(STORED);
});
it('should insert notes and return 201 with the stored rows', async () => {
const res = await post({ notes: INPUT });
expect(res.status).toBe(201);
expect(res.body).toEqual({ inserted: 1, notes: STORED });
expect(mockCreateNotes).toHaveBeenCalledWith(INPUT);
});
it('should report the inserted count from the database, not the request', async () => {
mockCreateNotes.mockResolvedValue([]);
const res = await post({ notes: INPUT });
expect(res.body.inserted).toBe(0);
});
it('should reject a body with no notes key', async () => {
const res = await post({});
expect(res.status).toBe(400);
expect(mockCreateNotes).not.toHaveBeenCalled();
});
it('should reject notes that is not an array', async () => {
const res = await post({ notes: 'nope' });
expect(res.status).toBe(400);
});
it('should reject an empty notes array', async () => {
const res = await post({ notes: [] });
expect(res.status).toBe(400);
expect(mockCreateNotes).not.toHaveBeenCalled();
});
it('should reject a note missing text', async () => {
const res = await post({ notes: [{ id: 'a', author: 'user_1' }] });
expect(res.status).toBe(400);
});
it('should reject a note whose x is not a number', async () => {
const res = await post({ notes: [{ ...INPUT[0], x: 'left' }] });
expect(res.status).toBe(400);
});
it('should reject a batch over the per-request limit', async () => {
const many = Array.from({ length: 5001 }, (_, i) => ({
id: `note_${i}`,
text: 'text',
author: 'user_1',
}));
const res = await post({ notes: many });
expect(res.status).toBe(400);
expect(mockCreateNotes).not.toHaveBeenCalled();
});
it('should return 401 when no Authorization header is sent', async () => {
const res = await request(app).post('/v1/notes').send({ notes: INPUT });
expect(res.status).toBe(401);
expect(mockCreateNotes).not.toHaveBeenCalled();
});
it('should return 401 for a non-bearer scheme', async () => {
const res = await request(app)
.post('/v1/notes')
.set('Authorization', 'Basic abc123')
.send({ notes: INPUT });
expect(res.status).toBe(401);
});
it('should return 401 for an unknown key without revealing why', async () => {
mockFindByApiKeyHash.mockResolvedValue(null);
const res = await post({ notes: INPUT });
expect(res.status).toBe(401);
expect(res.body).toEqual({ error: 'Unauthorized' });
});
it('should not send the raw key to the database', async () => {
await post({ notes: INPUT }, 'plaintext-key');
const [hash] = mockFindByApiKeyHash.mock.calls[0];
expect(hash).not.toContain('plaintext-key');
expect(hash).toMatch(/^[0-9a-f]{64}$/);
});
it('should return 500 when the insert fails', async () => {
mockCreateNotes.mockRejectedValue(new Error('connection refused'));
const res = await post({ notes: INPUT });
expect(res.status).toBe(500);
});
it('should leave the existing GET /v1/notes route reachable', async () => {
const res = await request(app).get('/v1/notes');
expect(res.status).not.toBe(404);
});
});

142
backend/tests/ingest_events.dao.test.ts Normal file
View File

@@ -0,0 +1,142 @@
import { describe, it, expect, vi, beforeEach } from 'vitest';
const { mockQuery, mockConnect } = vi.hoisted(() => ({
mockQuery: vi.fn(),
mockConnect: vi.fn(),
}));
vi.mock('../db/index.js', () => ({
query: mockQuery,
getPool: () => ({ connect: mockConnect }),
}));
import {
recordDelivery,
markProcessing,
markDone,
markFailed,
resetStaleProcessing,
} from '../db/ingest_events.dao.js';
describe('ingest_events.dao', () => {
beforeEach(() => {
vi.clearAllMocks();
});
describe('recordDelivery', () => {
it('should return the new row id for a delivery that has not been seen', async () => {
mockQuery.mockResolvedValue({ rows: [{ id: 42 }] });
const result = await recordDelivery({ provider: 'slack', externalId: 'evt_1' });
expect(result).toBe(42);
});
it('should return null when the delivery conflicts with an existing row', async () => {
mockQuery.mockResolvedValue({ rows: [] });
const result = await recordDelivery({ provider: 'slack', externalId: 'evt_1' });
expect(result).toBeNull();
});
it('should insert with ON CONFLICT DO NOTHING on the provider and external id', async () => {
mockQuery.mockResolvedValue({ rows: [{ id: 1 }] });
await recordDelivery({ provider: 'linear', externalId: 'evt_2' });
const [sql] = mockQuery.mock.calls[0] as [string, unknown[]];
expect(sql).toContain('INSERT INTO ingest_events');
expect(sql).toContain('ON CONFLICT (provider, external_id) DO NOTHING');
expect(sql).toContain('RETURNING id');
});
it('should bind the provider, external id and integration id', async () => {
mockQuery.mockResolvedValue({ rows: [{ id: 7 }] });
await recordDelivery({ provider: 'github', externalId: 'evt_3', integrationId: 12 });
const [, params] = mockQuery.mock.calls[0] as [string, unknown[]];
expect(params).toEqual(['github', 'evt_3', 12]);
});
it('should bind null when no integration id is supplied', async () => {
mockQuery.mockResolvedValue({ rows: [{ id: 8 }] });
await recordDelivery({ provider: 'rest', externalId: 'evt_4' });
const [, params] = mockQuery.mock.calls[0] as [string, unknown[]];
expect(params[2]).toBeNull();
});
});
describe('markProcessing', () => {
it('should set the status to processing and increment attempts', async () => {
mockQuery.mockResolvedValue({ rows: [] });
await markProcessing(5);
const [sql, params] = mockQuery.mock.calls[0] as [string, unknown[]];
expect(sql).toContain("status = 'processing'");
expect(sql).toContain('attempts = attempts + 1');
expect(params).toEqual([5]);
});
});
describe('markDone', () => {
it('should set the status to done and clear the last error', async () => {
mockQuery.mockResolvedValue({ rows: [] });
await markDone(9);
const [sql, params] = mockQuery.mock.calls[0] as [string, unknown[]];
expect(sql).toContain("status = 'done'");
expect(sql).toContain('last_error = NULL');
expect(params).toEqual([9]);
});
});
describe('markFailed', () => {
it('should store the failure message against the row', async () => {
mockQuery.mockResolvedValue({ rows: [] });
await markFailed(3, 'normalizer threw');
const [sql, params] = mockQuery.mock.calls[0] as [string, unknown[]];
expect(sql).toContain("status = 'failed'");
expect(sql).toContain('last_error = $2');
expect(params).toEqual([3, 'normalizer threw']);
});
it('should truncate an over-long error to 2000 characters', async () => {
mockQuery.mockResolvedValue({ rows: [] });
await markFailed(3, 'x'.repeat(5000));
const [, params] = mockQuery.mock.calls[0] as [string, unknown[]];
expect(params[1]).toBe('x'.repeat(2000));
});
});
describe('resetStaleProcessing', () => {
it('should return the number of rows returned to pending', async () => {
mockQuery.mockResolvedValue({ rows: [], rowCount: 4 });
const result = await resetStaleProcessing(30_000);
expect(result).toBe(4);
const [sql, params] = mockQuery.mock.calls[0] as [string, unknown[]];
expect(sql).toContain("SET status = 'pending'");
expect(sql).toContain("status = 'processing'");
expect(params).toEqual(['30000']);
});
it('should return 0 when the driver reports a null row count', async () => {
mockQuery.mockResolvedValue({ rows: [], rowCount: null });
const result = await resetStaleProcessing(30_000);
expect(result).toBe(0);
});
});
});

271
backend/tests/integrations.dao.test.ts Normal file
View File

@@ -0,0 +1,271 @@
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
const { mockQuery, mockConnect } = vi.hoisted(() => ({
mockQuery: vi.fn(),
mockConnect: vi.fn(),
}));
vi.mock('../db/index.js', () => ({
query: mockQuery,
getPool: () => ({ connect: mockConnect }),
}));
import {
findByApiKeyHash,
findByProviderWorkspace,
upsertInstall,
updateTokens,
getAccessToken,
getRefreshToken,
getSigningSecret,
} from '../db/integrations.dao.js';
import { encryptSecret } from '../lib/crypto.js';
const TEST_KEY = Buffer.alloc(32, 0x2b).toString('base64');
const RAW_ROW = {
id: 3,
provider: 'slack',
external_workspace_id: 'T123',
display_name: 'Acme',
api_key_hash: 'abc123',
access_token_ciphertext: 'iv:tag:payload',
refresh_token_ciphertext: 'iv:tag:payload',
signing_secret_ciphertext: 'iv:tag:payload',
token_expires_at: new Date('2030-01-01T00:00:00.000Z'),
scopes: ['channels:history'],
};
describe('integrations.dao', () => {
let originalKey: string | undefined;
beforeEach(() => {
vi.clearAllMocks();
originalKey = process.env.TOKEN_ENCRYPTION_KEY;
process.env.TOKEN_ENCRYPTION_KEY = TEST_KEY;
});
afterEach(() => {
if (originalKey === undefined) {
delete process.env.TOKEN_ENCRYPTION_KEY;
} else {
process.env.TOKEN_ENCRYPTION_KEY = originalKey;
}
});
describe('findByApiKeyHash', () => {
it('should return the integration mapped to camelCase fields', async () => {
mockQuery.mockResolvedValue({ rows: [RAW_ROW] });
const result = await findByApiKeyHash('abc123');
expect(result).toEqual({
id: 3,
provider: 'slack',
externalWorkspaceId: 'T123',
displayName: 'Acme',
scopes: ['channels:history'],
tokenExpiresAt: new Date('2030-01-01T00:00:00.000Z'),
});
const [sql, params] = mockQuery.mock.calls[0] as [string, unknown[]];
expect(sql).toContain('WHERE api_key_hash = $1');
expect(params).toEqual(['abc123']);
});
it('should not expose any ciphertext or token field on the returned row', async () => {
mockQuery.mockResolvedValue({ rows: [RAW_ROW] });
const result = await findByApiKeyHash('abc123');
expect(result).not.toBeNull();
expect(result).not.toHaveProperty('access_token_ciphertext');
expect(result).not.toHaveProperty('refresh_token_ciphertext');
expect(result).not.toHaveProperty('signing_secret_ciphertext');
expect(result).not.toHaveProperty('accessToken');
expect(result).not.toHaveProperty('api_key_hash');
expect(Object.keys(result ?? {}).sort()).toEqual([
'displayName',
'externalWorkspaceId',
'id',
'provider',
'scopes',
'tokenExpiresAt',
]);
});
it('should default scopes to an empty array when the column is null', async () => {
mockQuery.mockResolvedValue({ rows: [{ ...RAW_ROW, scopes: null }] });
const result = await findByApiKeyHash('abc123');
expect(result?.scopes).toEqual([]);
});
it('should return null when no integration matches the hash', async () => {
mockQuery.mockResolvedValue({ rows: [] });
const result = await findByApiKeyHash('nope');
expect(result).toBeNull();
});
});
describe('findByProviderWorkspace', () => {
it('should bind the provider and external workspace id', async () => {
mockQuery.mockResolvedValue({ rows: [RAW_ROW] });
const result = await findByProviderWorkspace('slack', 'T123');
expect(result?.id).toBe(3);
const [sql, params] = mockQuery.mock.calls[0] as [string, unknown[]];
expect(sql).toContain('WHERE provider = $1 AND external_workspace_id = $2');
expect(params).toEqual(['slack', 'T123']);
});
it('should return null when the workspace has no integration', async () => {
mockQuery.mockResolvedValue({ rows: [] });
const result = await findByProviderWorkspace('slack', 'T999');
expect(result).toBeNull();
});
});
describe('upsertInstall', () => {
it('should encrypt the signing secret before binding it', async () => {
mockQuery.mockResolvedValue({ rows: [RAW_ROW] });
await upsertInstall({
provider: 'slack',
externalWorkspaceId: 'T123',
displayName: 'Acme',
signingSecret: 'super-secret',
});
const [sql, params] = mockQuery.mock.calls[0] as [string, unknown[]];
expect(sql).toContain('INSERT INTO integrations');
expect(sql).toContain('ON CONFLICT (provider, external_workspace_id) DO UPDATE SET');
const stored = params[4];
expect(typeof stored).toBe('string');
expect(stored).not.toBe('super-secret');
expect(String(stored).split(':')).toHaveLength(3);
});
it('should bind null when no signing secret is supplied', async () => {
mockQuery.mockResolvedValue({ rows: [RAW_ROW] });
await upsertInstall({ provider: 'slack', externalWorkspaceId: 'T123' });
const [, params] = mockQuery.mock.calls[0] as [string, unknown[]];
expect(params[4]).toBeNull();
expect(params[2]).toBeNull();
expect(params[3]).toBeNull();
expect(params[5]).toEqual([]);
});
it('should return the public row for the upserted integration', async () => {
mockQuery.mockResolvedValue({ rows: [RAW_ROW] });
const result = await upsertInstall({ provider: 'slack', externalWorkspaceId: 'T123' });
expect(result.externalWorkspaceId).toBe('T123');
expect(result).not.toHaveProperty('signing_secret_ciphertext');
});
});
describe('updateTokens', () => {
it('should encrypt the access token rather than storing it in plaintext', async () => {
mockQuery.mockResolvedValue({ rows: [] });
await updateTokens({ id: 3, accessToken: 'at-plain', refreshToken: 'rt-plain' });
const [sql, params] = mockQuery.mock.calls[0] as [string, unknown[]];
expect(sql).toContain('access_token_ciphertext = $2');
expect(params[0]).toBe(3);
expect(params[1]).not.toBe('at-plain');
expect(String(params[1]).split(':')).toHaveLength(3);
expect(params[2]).not.toBe('rt-plain');
});
it('should bind null for an absent refresh token and expiry', async () => {
mockQuery.mockResolvedValue({ rows: [] });
await updateTokens({ id: 3, accessToken: 'at-plain' });
const [, params] = mockQuery.mock.calls[0] as [string, unknown[]];
expect(params[2]).toBeNull();
expect(params[3]).toBeNull();
});
it('should bind the supplied expiry date', async () => {
mockQuery.mockResolvedValue({ rows: [] });
const expiresAt = new Date('2031-05-05T10:00:00.000Z');
await updateTokens({ id: 3, accessToken: 'at-plain', expiresAt });
const [, params] = mockQuery.mock.calls[0] as [string, unknown[]];
expect(params[3]).toEqual(expiresAt);
});
});
describe('getAccessToken', () => {
it('should decrypt the stored ciphertext back to the original token', async () => {
mockQuery.mockResolvedValue({ rows: [{ value: encryptSecret('at-plain') }] });
const result = await getAccessToken(3);
expect(result).toBe('at-plain');
const [sql, params] = mockQuery.mock.calls[0] as [string, unknown[]];
expect(sql).toContain('SELECT access_token_ciphertext AS value');
expect(params).toEqual([3]);
});
it('should return null when no access token is stored', async () => {
mockQuery.mockResolvedValue({ rows: [{ value: null }] });
const result = await getAccessToken(3);
expect(result).toBeNull();
});
it('should return null when the integration does not exist', async () => {
mockQuery.mockResolvedValue({ rows: [] });
const result = await getAccessToken(404);
expect(result).toBeNull();
});
});
describe('getRefreshToken', () => {
it('should decrypt the refresh token column', async () => {
mockQuery.mockResolvedValue({ rows: [{ value: encryptSecret('rt-plain') }] });
const result = await getRefreshToken(3);
expect(result).toBe('rt-plain');
const [sql] = mockQuery.mock.calls[0] as [string, unknown[]];
expect(sql).toContain('SELECT refresh_token_ciphertext AS value');
});
});
describe('getSigningSecret', () => {
it('should decrypt the signing secret column', async () => {
mockQuery.mockResolvedValue({ rows: [{ value: encryptSecret('shh') }] });
const result = await getSigningSecret(3);
expect(result).toBe('shh');
const [sql] = mockQuery.mock.calls[0] as [string, unknown[]];
expect(sql).toContain('SELECT signing_secret_ciphertext AS value');
});
it('should return null when no signing secret is stored', async () => {
mockQuery.mockResolvedValue({ rows: [{ value: null }] });
const result = await getSigningSecret(3);
expect(result).toBeNull();
});
});
});

340
backend/tests/normalizers.test.ts Normal file
View File

@@ -0,0 +1,340 @@
import { describe, it, expect } from 'vitest';
import {
NormalizationError,
isNoteInputArray,
normalizeLinear,
normalizeRest,
} from '../config/normalizers.js';
import { normalize, hasNormalizer } from '../services/normalize.service.js';
import type { NoteInput } from '../types/domain.js';
const note = (overrides: Record<string, unknown> = {}): Record<string, unknown> => ({
id: 'n1',
text: 'Deploys are scary',
author: 'kim',
...overrides,
});
const sourceMeta = (input: NoteInput): Record<string, unknown> => {
expect(input.sourceMeta).toBeDefined();
return input.sourceMeta as Record<string, unknown>;
};
const linearPayload = (
data: Record<string, unknown> = {}
): Record<string, unknown> => ({
action: 'create',
type: 'Comment',
data: {
id: 'cmt_9f2',
body: 'Retros keep surfacing the same deploy pain',
url: 'https://linear.app/acme/issue/ENG-42#comment-cmt_9f2',
user: { name: 'dana' },
...data,
},
});
describe('isNoteInputArray', () => {
it('should accept an empty array', () => {
expect(isNoteInputArray([])).toBe(true);
});
it('should accept an array of well-formed notes', () => {
expect(isNoteInputArray([note(), note({ id: 'n2', x: 1, y: 2, color: '#fff' })])).toBe(true);
});
it('should reject a non-array', () => {
expect(isNoteInputArray(undefined)).toBe(false);
expect(isNoteInputArray(null)).toBe(false);
expect(isNoteInputArray('notes')).toBe(false);
expect(isNoteInputArray({ 0: note(), length: 1 })).toBe(false);
});
it('should reject an element that is not an object', () => {
expect(isNoteInputArray([note(), 'nope'])).toBe(false);
expect(isNoteInputArray([null])).toBe(false);
});
it('should reject an element missing a required field', () => {
expect(isNoteInputArray([note({ id: undefined })])).toBe(false);
expect(isNoteInputArray([note({ text: undefined })])).toBe(false);
expect(isNoteInputArray([note({ author: undefined })])).toBe(false);
});
it('should reject an element whose required field has the wrong type', () => {
expect(isNoteInputArray([note({ id: 7 })])).toBe(false);
expect(isNoteInputArray([note({ text: { body: 'hi' } })])).toBe(false);
expect(isNoteInputArray([note({ author: ['kim'] })])).toBe(false);
});
it('should reject an element with empty strings', () => {
expect(isNoteInputArray([note({ id: '' })])).toBe(false);
expect(isNoteInputArray([note({ text: '' })])).toBe(false);
expect(isNoteInputArray([note({ author: '' })])).toBe(false);
});
it('should reject an element whose optional field has the wrong type', () => {
expect(isNoteInputArray([note({ x: '10' })])).toBe(false);
expect(isNoteInputArray([note({ y: null })])).toBe(false);
expect(isNoteInputArray([note({ color: 0xffffff })])).toBe(false);
});
it('should accept an element whose optional fields are undefined', () => {
expect(isNoteInputArray([note({ x: undefined, y: undefined, color: undefined })])).toBe(true);
});
});
describe('normalizeRest', () => {
it('should return the supplied notes with provenance attached', () => {
const result = normalizeRest({
batchId: 'batch_7',
notes: [note(), note({ id: 'n2', text: 'Standups run long' })],
});
expect(result.externalId).toBe('batch_7');
expect(result.notes).toHaveLength(2);
expect(result.notes[0].id).toBe('n1');
expect(result.notes[0].text).toBe('Deploys are scary');
expect(result.notes[0].author).toBe('kim');
expect(sourceMeta(result.notes[0])).toMatchObject({
provider: 'rest',
externalId: 'batch_7',
});
expect(sourceMeta(result.notes[1]).externalId).toBe('batch_7');
});
it('should record an ISO receivedAt timestamp in provenance', () => {
const result = normalizeRest({ notes: [note()] });
const receivedAt = String(sourceMeta(result.notes[0]).receivedAt);
expect(new Date(receivedAt).toISOString()).toBe(receivedAt);
});
it('should preserve optional positional and color fields', () => {
const result = normalizeRest({
notes: [note({ x: 12, y: -3, color: '#ffcc00' })],
});
expect(result.notes[0]).toMatchObject({ x: 12, y: -3, color: '#ffcc00' });
});
it('should generate a non-empty externalId when batchId is absent', () => {
const result = normalizeRest({ notes: [note()] });
expect(result.externalId.length).toBeGreaterThan(0);
expect(sourceMeta(result.notes[0]).externalId).toBe(result.externalId);
});
it('should generate a distinct externalId per call', () => {
const first = normalizeRest({ notes: [note()] });
const second = normalizeRest({ notes: [note()] });
expect(first.externalId).not.toBe(second.externalId);
});
it('should ignore an empty batchId in favour of a generated one', () => {
const result = normalizeRest({ batchId: '', notes: [note()] });
expect(result.externalId.length).toBeGreaterThan(0);
});
it('should ignore a non-string batchId in favour of a generated one', () => {
const result = normalizeRest({ batchId: 42, notes: [note()] });
expect(result.externalId).not.toBe('42');
expect(result.externalId.length).toBeGreaterThan(0);
});
it('should accept an empty notes array', () => {
const result = normalizeRest({ batchId: 'batch_empty', notes: [] });
expect(result).toEqual({ externalId: 'batch_empty', notes: [] });
});
it('should throw when notes is missing', () => {
expect(() => normalizeRest({ batchId: 'batch_7' })).toThrow(NormalizationError);
expect(() => normalizeRest({})).toThrow(/"notes" array/);
});
it('should throw when notes is not an array', () => {
expect(() => normalizeRest({ notes: 'one note' })).toThrow(NormalizationError);
expect(() => normalizeRest({ notes: { id: 'n1' } })).toThrow(NormalizationError);
});
it('should throw when a note is missing text', () => {
expect(() => normalizeRest({ notes: [note({ text: undefined })] })).toThrow(
NormalizationError
);
});
it('should throw when a note id is not a string', () => {
expect(() => normalizeRest({ notes: [note({ id: 99 })] })).toThrow(NormalizationError);
});
it('should throw when a note carries empty strings', () => {
expect(() => normalizeRest({ notes: [note({ text: '' })] })).toThrow(NormalizationError);
expect(() => normalizeRest({ notes: [note({ id: '' })] })).toThrow(NormalizationError);
expect(() => normalizeRest({ notes: [note({ author: '' })] })).toThrow(NormalizationError);
});
it('should throw when one note in an otherwise valid batch is invalid', () => {
expect(() => normalizeRest({ notes: [note(), note({ id: 'n2', author: '' })] })).toThrow(
NormalizationError
);
});
it('should throw for a non-object payload', () => {
expect(() => normalizeRest(null)).toThrow(/not an object/);
expect(() => normalizeRest(undefined)).toThrow(NormalizationError);
expect(() => normalizeRest('notes')).toThrow(NormalizationError);
expect(() => normalizeRest([note()])).toThrow(/not an object/);
});
it('should overwrite any caller-supplied sourceMeta with real provenance', () => {
const result = normalizeRest({
batchId: 'batch_7',
notes: [note({ sourceMeta: { provider: 'slack', externalId: 'spoofed' } })],
});
expect(sourceMeta(result.notes[0])).toMatchObject({
provider: 'rest',
externalId: 'batch_7',
});
});
});
describe('normalizeLinear', () => {
it('should map a comment webhook to a single note', () => {
const result = normalizeLinear(linearPayload());
expect(result.externalId).toBe('cmt_9f2');
expect(result.notes).toHaveLength(1);
expect(result.notes[0].id).toBe('linear_cmt_9f2');
expect(result.notes[0].text).toBe('Retros keep surfacing the same deploy pain');
expect(result.notes[0].author).toBe('dana');
});
it('should attach provenance including provider, externalId and permalink', () => {
const meta = sourceMeta(normalizeLinear(linearPayload()).notes[0]);
expect(meta).toMatchObject({
provider: 'linear',
externalId: 'cmt_9f2',
permalink: 'https://linear.app/acme/issue/ENG-42#comment-cmt_9f2',
authorHandle: 'dana',
});
expect(typeof meta.receivedAt).toBe('string');
});
it('should fall back to an unknown author when the user is missing or unnamed', () => {
expect(normalizeLinear(linearPayload({ user: undefined })).notes[0].author).toBe('unknown');
expect(normalizeLinear(linearPayload({ user: null })).notes[0].author).toBe('unknown');
expect(normalizeLinear(linearPayload({ user: {} })).notes[0].author).toBe('unknown');
expect(normalizeLinear(linearPayload({ user: { name: '' } })).notes[0].author).toBe('unknown');
expect(normalizeLinear(linearPayload({ user: 'dana' })).notes[0].author).toBe('unknown');
});
it('should omit the permalink when url is absent', () => {
const meta = sourceMeta(normalizeLinear(linearPayload({ url: undefined })).notes[0]);
expect(meta.permalink).toBeUndefined();
});
it('should throw when data.id is missing', () => {
expect(() => normalizeLinear(linearPayload({ id: undefined }))).toThrow(NormalizationError);
expect(() => normalizeLinear(linearPayload({ id: '' }))).toThrow(/missing data.id/);
expect(() => normalizeLinear(linearPayload({ id: 42 }))).toThrow(NormalizationError);
});
it('should throw when data itself is missing or not an object', () => {
expect(() => normalizeLinear({ action: 'create', type: 'Comment' })).toThrow(
/not an object/
);
expect(() => normalizeLinear({ data: 'cmt_9f2' })).toThrow(NormalizationError);
expect(() => normalizeLinear({ data: [] })).toThrow(NormalizationError);
});
it('should return zero notes but keep the externalId when the body is empty', () => {
expect(normalizeLinear(linearPayload({ body: '' }))).toEqual({
externalId: 'cmt_9f2',
notes: [],
});
});
it('should return zero notes but keep the externalId when the body is absent', () => {
expect(normalizeLinear(linearPayload({ body: undefined }))).toEqual({
externalId: 'cmt_9f2',
notes: [],
});
expect(normalizeLinear(linearPayload({ body: null }))).toEqual({
externalId: 'cmt_9f2',
notes: [],
});
});
it('should throw for a non-object payload', () => {
expect(() => normalizeLinear(null)).toThrow(/not an object/);
expect(() => normalizeLinear('cmt_9f2')).toThrow(NormalizationError);
expect(() => normalizeLinear(7)).toThrow(NormalizationError);
expect(() => normalizeLinear([linearPayload()])).toThrow(/not an object/);
});
it('should truncate a generated note id to 64 characters', () => {
const longId = 'x'.repeat(400);
const result = normalizeLinear(linearPayload({ id: longId }));
expect(result.externalId).toBe(longId);
expect(result.notes[0].id).toHaveLength(64);
expect(result.notes[0].id).toBe(`linear_${longId}`.slice(0, 64));
expect(sourceMeta(result.notes[0]).externalId).toBe(longId);
});
it('should produce notes accepted by the note input guard', () => {
expect(isNoteInputArray(normalizeLinear(linearPayload()).notes)).toBe(true);
});
});
describe('normalize', () => {
it('should delegate rest deliveries to the rest normalizer', () => {
const result = normalize('rest', { batchId: 'batch_7', notes: [note()] });
expect(result.externalId).toBe('batch_7');
expect(result.notes[0]).toMatchObject({ id: 'n1', author: 'kim' });
expect(sourceMeta(result.notes[0])).toMatchObject({
provider: 'rest',
externalId: 'batch_7',
});
});
it('should delegate linear deliveries to the linear normalizer', () => {
const result = normalize('linear', linearPayload());
expect(result.externalId).toBe('cmt_9f2');
expect(result.notes[0].id).toBe('linear_cmt_9f2');
});
it('should propagate normalizer failures unchanged', () => {
expect(() => normalize('rest', {})).toThrow(NormalizationError);
expect(() => normalize('linear', {})).toThrow(NormalizationError);
});
it('should throw for a provider with no registered normalizer', () => {
expect(() => normalize('github', {})).toThrow(NormalizationError);
expect(() => normalize('github', {})).toThrow(/No normalizer registered/);
expect(() => normalize('jira', {})).toThrow(/No normalizer registered/);
expect(() => normalize('slack', {})).toThrow(/No normalizer registered/);
});
});
describe('hasNormalizer', () => {
it('should be true for providers with a normalizer', () => {
expect(hasNormalizer('rest')).toBe(true);
expect(hasNormalizer('linear')).toBe(true);
});
it('should be false for providers awaiting an integration', () => {
expect(hasNormalizer('github')).toBe(false);
expect(hasNormalizer('jira')).toBe(false);
expect(hasNormalizer('slack')).toBe(false);
});
});

49
backend/tests/notes.dao.test.js → backend/tests/notes.dao.test.ts
View File

@@ -1,5 +1,6 @@
import { describe, it, expect, vi, beforeEach } from 'vitest';
import { Readable } from 'node:stream';
import type { Note, NoteInput } from '../types/domain.js';
const { mockQuery, mockConnect } = vi.hoisted(() => ({
mockQuery: vi.fn(),
@@ -19,7 +20,7 @@ import {
createNotes,
} from '../db/notes.dao.js';
const MOCK_ROWS = [
const MOCK_ROWS: Note[] = [
{ id: 'note_001', text: 'Login flow feels confusing', x: 193, y: 191, author: 'user_5', color: 'yellow' },
{ id: 'note_002', text: 'Login flow is broken on mobile', x: 214, y: 281, author: 'user_9', color: 'yellow' },
];
@@ -58,7 +59,7 @@ describe('notes.dao', () => {
});
describe('streamAllNotes', () => {
const mockClient = (rows) => {
const mockClient = (rows: Note[]) => {
const release = vi.fn();
const client = {
release,
@@ -72,8 +73,8 @@ describe('notes.dao', () => {
mockClient(MOCK_ROWS);
const stream = await streamAllNotes();
const received = [];
for await (const row of stream) received.push(row);
const received: Note[] = [];
for await (const row of stream) received.push(row as Note);
expect(received).toEqual(MOCK_ROWS);
});
@@ -82,7 +83,7 @@ describe('notes.dao', () => {
const { release } = mockClient(MOCK_ROWS);
const stream = await streamAllNotes();
for await (const _row of stream) { /* drain */ }
for await (const row of stream) void row;
expect(release).toHaveBeenCalledOnce();
});
@@ -133,25 +134,40 @@ describe('notes.dao', () => {
describe('createNote', () => {
it('should insert a note and return it', async () => {
const input = { id: 'note_003', text: 'Export fails', x: 100, y: 200, author: 'user_1', color: 'blue' };
const input: NoteInput = { id: 'note_003', text: 'Export fails', x: 100, y: 200, author: 'user_1', color: 'blue' };
mockQuery.mockResolvedValue({ rows: [input] });
const result = await createNote(input);
expect(result).toEqual(input);
expect(mockQuery).toHaveBeenCalledOnce();
const [sql, params] = mockQuery.mock.calls[0];
const [sql, params] = mockQuery.mock.calls[0] as [string, unknown[]];
expect(sql).toContain('INSERT INTO notes');
expect(params).toEqual(['note_003', 'Export fails', 100, 200, 'user_1', 'blue']);
expect(params).toEqual(['note_003', 'Export fails', 100, 200, 'user_1', 'blue', '{}']);
});
it('should serialize provenance into source_meta', async () => {
const input: NoteInput = {
id: 'note_005',
text: 'From Slack',
author: 'user_3',
sourceMeta: { provider: 'slack', externalId: 'msg_1' },
};
mockQuery.mockResolvedValue({ rows: [input] });
await createNote(input);
const params = mockQuery.mock.calls[0][1] as unknown[];
expect(params[6]).toBe('{"provider":"slack","externalId":"msg_1"}');
});
it('should use defaults for missing x, y, and color', async () => {
const input = { id: 'note_004', text: 'Needs fixing', author: 'user_2' };
const input: NoteInput = { id: 'note_004', text: 'Needs fixing', author: 'user_2' };
mockQuery.mockResolvedValue({ rows: [{ ...input, x: 0, y: 0, color: 'yellow' }] });
await createNote(input);
const params = mockQuery.mock.calls[0][1];
const params = mockQuery.mock.calls[0][1] as unknown[];
expect(params[2]).toBe(0);
expect(params[3]).toBe(0);
expect(params[5]).toBe('yellow');
@@ -166,9 +182,10 @@ describe('notes.dao', () => {
expect(result).toEqual(MOCK_ROWS);
expect(mockQuery).toHaveBeenCalledOnce();
const [sql, params] = mockQuery.mock.calls[0];
const [sql, params] = mockQuery.mock.calls[0] as [string, unknown[]];
expect(sql).toContain('INSERT INTO notes');
expect(params).toHaveLength(12);
expect(sql).toContain('ON CONFLICT (id) DO NOTHING');
expect(params).toHaveLength(14);
});
it('should return an empty array without querying when given no notes', async () => {
@@ -179,20 +196,20 @@ describe('notes.dao', () => {
});
it('should split large inputs into multiple statements under the bind-parameter limit', async () => {
const many = Array.from({ length: 2500 }, (_, i) => ({
const many: NoteInput[] = Array.from({ length: 2500 }, (_, i) => ({
id: `note_${i}`,
text: `text ${i}`,
author: 'user_1',
}));
mockQuery.mockImplementation(async (_sql, params) => ({
rows: new Array(params.length / 6).fill(null).map((_, i) => ({ i })),
mockQuery.mockImplementation(async (_sql: string, params: unknown[]) => ({
rows: new Array(params.length / 7).fill(null).map((_, i) => ({ i })),
}));
const result = await createNotes(many);
expect(mockQuery).toHaveBeenCalledTimes(3);
expect(result).toHaveLength(2500);
for (const [, params] of mockQuery.mock.calls) {
for (const [, params] of mockQuery.mock.calls as [string, unknown[]][]) {
expect(params.length).toBeLessThan(65535);
}
});

284
backend/tests/oauth.service.test.ts Normal file
View File

@@ -0,0 +1,284 @@
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
const {
mockRequestJson,
mockFindByProviderWorkspace,
mockGetAccessToken,
mockGetRefreshToken,
mockUpdateTokens,
} = vi.hoisted(() => ({
mockRequestJson: vi.fn(),
mockFindByProviderWorkspace: vi.fn(),
mockGetAccessToken: vi.fn(),
mockGetRefreshToken: vi.fn(),
mockUpdateTokens: vi.fn(),
}));
vi.mock('../lib/httpClient.js', () => ({
requestJson: mockRequestJson,
}));
vi.mock('../db/integrations.dao.js', () => ({
findByProviderWorkspace: mockFindByProviderWorkspace,
getAccessToken: mockGetAccessToken,
getRefreshToken: mockGetRefreshToken,
updateTokens: mockUpdateTokens,
}));
import {
buildAuthorizeUrl,
exchangeCode,
refreshAccessToken,
getValidAccessToken,
OAuthError,
} from '../services/oauth.service.js';
import type { IntegrationRow } from '../types/integration.js';
const ENV_KEYS = [
'LINEAR_CLIENT_ID',
'LINEAR_CLIENT_SECRET',
'SLACK_CLIENT_ID',
'SLACK_CLIENT_SECRET',
] as const;
const integration = (overrides: Partial<IntegrationRow> = {}): IntegrationRow => ({
id: 11,
provider: 'linear',
externalWorkspaceId: 'ws_1',
displayName: 'Acme',
scopes: ['read'],
tokenExpiresAt: null,
...overrides,
});
describe('oauth.service', () => {
const originalEnv = new Map<string, string | undefined>();
beforeEach(() => {
vi.clearAllMocks();
for (const key of ENV_KEYS) originalEnv.set(key, process.env[key]);
process.env.LINEAR_CLIENT_ID = 'client-id-1';
process.env.LINEAR_CLIENT_SECRET = 'client-secret-1';
process.env.SLACK_CLIENT_ID = 'slack-id';
process.env.SLACK_CLIENT_SECRET = 'slack-secret';
});
afterEach(() => {
for (const key of ENV_KEYS) {
const value = originalEnv.get(key);
if (value === undefined) delete process.env[key];
else process.env[key] = value;
}
});
describe('buildAuthorizeUrl', () => {
it('should build an authorize url against the provider endpoint with all required params', () => {
const url = new URL(
buildAuthorizeUrl('linear', { redirectUri: 'https://app.test/cb', state: 'st_1' })
);
expect(`${url.origin}${url.pathname}`).toBe('https://linear.app/oauth/authorize');
expect(url.searchParams.get('client_id')).toBe('client-id-1');
expect(url.searchParams.get('redirect_uri')).toBe('https://app.test/cb');
expect(url.searchParams.get('response_type')).toBe('code');
expect(url.searchParams.get('state')).toBe('st_1');
expect(url.searchParams.get('scope')).toBe('read');
});
it('should join multiple configured scopes with spaces', () => {
const url = new URL(
buildAuthorizeUrl('slack', { redirectUri: 'https://app.test/cb', state: 'st_2' })
);
expect(url.searchParams.get('scope')).toBe('channels:history reactions:read users:read');
});
it('should throw an OAuthError when client credentials are not configured', () => {
delete process.env.LINEAR_CLIENT_SECRET;
expect(() =>
buildAuthorizeUrl('linear', { redirectUri: 'https://app.test/cb', state: 'st_1' })
).toThrow(OAuthError);
});
it('should throw an OAuthError for a provider without OAuth support', () => {
expect(() =>
buildAuthorizeUrl('rest', { redirectUri: 'https://app.test/cb', state: 'st_1' })
).toThrow(/does not support OAuth/);
});
});
describe('exchangeCode', () => {
it('should post a form-encoded authorization_code grant to the token endpoint', async () => {
mockRequestJson.mockResolvedValue({ access_token: 'at_1' });
await exchangeCode('linear', { code: 'code_1', redirectUri: 'https://app.test/cb' });
const [url, init] = mockRequestJson.mock.calls[0] as [string, RequestInit];
expect(url).toBe('https://api.linear.app/oauth/token');
expect(init.method).toBe('POST');
expect(init.headers).toMatchObject({
'content-type': 'application/x-www-form-urlencoded',
});
const form = new URLSearchParams(String(init.body));
expect(form.get('grant_type')).toBe('authorization_code');
expect(form.get('code')).toBe('code_1');
expect(form.get('redirect_uri')).toBe('https://app.test/cb');
expect(form.get('client_id')).toBe('client-id-1');
expect(form.get('client_secret')).toBe('client-secret-1');
});
it('should map the token response into a TokenSet', async () => {
mockRequestJson.mockResolvedValue({
access_token: 'at_1',
refresh_token: 'rt_1',
expires_in: 3600,
scope: 'read write',
});
const before = Date.now();
const tokens = await exchangeCode('linear', {
code: 'code_1',
redirectUri: 'https://app.test/cb',
});
expect(tokens.accessToken).toBe('at_1');
expect(tokens.refreshToken).toBe('rt_1');
expect(tokens.scopes).toEqual(['read', 'write']);
expect(tokens.expiresAt).toBeInstanceOf(Date);
expect(tokens.expiresAt?.getTime()).toBeGreaterThanOrEqual(before + 3_600_000);
expect(tokens.expiresAt?.getTime()).toBeLessThanOrEqual(Date.now() + 3_600_000);
});
it('should leave expiresAt undefined and scopes empty when the response omits them', async () => {
mockRequestJson.mockResolvedValue({ access_token: 'at_1' });
const tokens = await exchangeCode('linear', {
code: 'code_1',
redirectUri: 'https://app.test/cb',
});
expect(tokens.expiresAt).toBeUndefined();
expect(tokens.scopes).toEqual([]);
});
it('should throw an OAuthError when the payload carries no usable access token', async () => {
mockRequestJson.mockResolvedValue({ token_type: 'bearer' });
await expect(
exchangeCode('linear', { code: 'code_1', redirectUri: 'https://app.test/cb' })
).rejects.toThrow(OAuthError);
});
it('should throw an OAuthError when credentials are missing', async () => {
delete process.env.LINEAR_CLIENT_ID;
await expect(
exchangeCode('linear', { code: 'code_1', redirectUri: 'https://app.test/cb' })
).rejects.toThrow(/LINEAR_CLIENT_ID/);
expect(mockRequestJson).not.toHaveBeenCalled();
});
});
describe('refreshAccessToken', () => {
it('should post a refresh_token grant carrying the stored refresh token', async () => {
mockRequestJson.mockResolvedValue({ access_token: 'at_2' });
const tokens = await refreshAccessToken('linear', 'rt_1');
expect(tokens.accessToken).toBe('at_2');
const [, init] = mockRequestJson.mock.calls[0] as [string, RequestInit];
const form = new URLSearchParams(String(init.body));
expect(form.get('grant_type')).toBe('refresh_token');
expect(form.get('refresh_token')).toBe('rt_1');
});
});
describe('getValidAccessToken', () => {
it('should return the stored access token when it is not near expiry', async () => {
mockFindByProviderWorkspace.mockResolvedValue(
integration({ tokenExpiresAt: new Date(Date.now() + 3_600_000) })
);
mockGetAccessToken.mockResolvedValue('at_stored');
const result = await getValidAccessToken('linear', 'ws_1');
expect(result).toBe('at_stored');
expect(mockRequestJson).not.toHaveBeenCalled();
expect(mockUpdateTokens).not.toHaveBeenCalled();
});
it('should return the stored access token when no expiry is recorded', async () => {
mockFindByProviderWorkspace.mockResolvedValue(integration());
mockGetAccessToken.mockResolvedValue('at_stored');
const result = await getValidAccessToken('linear', 'ws_1');
expect(result).toBe('at_stored');
expect(mockGetAccessToken).toHaveBeenCalledWith(11);
});
it('should refresh and persist the new token when expiry is within the refresh margin', async () => {
mockFindByProviderWorkspace.mockResolvedValue(
integration({ tokenExpiresAt: new Date(Date.now() + 30_000) })
);
mockGetRefreshToken.mockResolvedValue('rt_1');
mockRequestJson.mockResolvedValue({
access_token: 'at_refreshed',
refresh_token: 'rt_2',
expires_in: 3600,
});
const result = await getValidAccessToken('linear', 'ws_1');
expect(result).toBe('at_refreshed');
expect(mockUpdateTokens).toHaveBeenCalledWith(
expect.objectContaining({ id: 11, accessToken: 'at_refreshed', refreshToken: 'rt_2' })
);
expect(mockGetAccessToken).not.toHaveBeenCalled();
});
it('should refresh when the token has already expired', async () => {
mockFindByProviderWorkspace.mockResolvedValue(
integration({ tokenExpiresAt: new Date(Date.now() - 1_000) })
);
mockGetRefreshToken.mockResolvedValue('rt_1');
mockRequestJson.mockResolvedValue({ access_token: 'at_refreshed' });
const result = await getValidAccessToken('linear', 'ws_1');
expect(result).toBe('at_refreshed');
});
it('should throw an OAuthError when the workspace has no integration', async () => {
mockFindByProviderWorkspace.mockResolvedValue(null);
await expect(getValidAccessToken('linear', 'ws_missing')).rejects.toThrow(OAuthError);
await expect(getValidAccessToken('linear', 'ws_missing')).rejects.toThrow(
/No linear integration for workspace ws_missing/
);
});
it('should throw an OAuthError when a refresh is needed but no refresh token is stored', async () => {
mockFindByProviderWorkspace.mockResolvedValue(
integration({ tokenExpiresAt: new Date(Date.now() + 1_000) })
);
mockGetRefreshToken.mockResolvedValue(null);
await expect(getValidAccessToken('linear', 'ws_1')).rejects.toThrow(
/no refresh token is stored/
);
expect(mockUpdateTokens).not.toHaveBeenCalled();
});
it('should throw an OAuthError when no access token is stored', async () => {
mockFindByProviderWorkspace.mockResolvedValue(integration());
mockGetAccessToken.mockResolvedValue(null);
await expect(getValidAccessToken('linear', 'ws_1')).rejects.toThrow(
/No access token stored for linear/
);
});
});
});

215
backend/tests/queue.test.ts Normal file
View File

@@ -0,0 +1,215 @@
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
import { enqueue, size, drain, configureQueue } from '../lib/queue.js';
type Deferred = { promise: Promise<void>; resolve: () => void };
const deferred = (): Deferred => {
let resolve!: () => void;
const promise = new Promise<void>((res) => {
resolve = res;
});
return { promise, resolve };
};
// A macrotask boundary: enough for the queue to pick up work it deferred.
const flush = (): Promise<void> =>
new Promise((resolve) => {
setTimeout(resolve, 0);
});
describe('queue', () => {
let errors: ReturnType<typeof vi.spyOn>;
beforeEach(() => {
configureQueue({ maxAttempts: 3, baseDelayMs: 0 });
errors = vi.spyOn(console, 'error').mockImplementation(() => {});
});
afterEach(async () => {
await drain();
vi.useRealTimers();
vi.restoreAllMocks();
});
it('should run jobs one at a time in FIFO order', async () => {
const order: string[] = [];
const record = (id: string) => async (): Promise<void> => {
order.push(`${id}:start`);
await Promise.resolve();
order.push(`${id}:end`);
};
enqueue('a', record('a'));
enqueue('b', record('b'));
enqueue('c', record('c'));
await drain();
expect(order).toEqual([
'a:start', 'a:end',
'b:start', 'b:end',
'c:start', 'c:end',
]);
});
it('should not throw synchronously when a job throws synchronously', async () => {
const thrower = (): Promise<void> => {
throw new Error('sync boom');
};
expect(() => enqueue('sync-thrower', thrower)).not.toThrow();
await drain();
expect(errors).toHaveBeenCalled();
});
it('should retry a failing job up to the configured cap and then give up', async () => {
configureQueue({ maxAttempts: 4 });
let attempts = 0;
enqueue('always-failing', async () => {
attempts += 1;
throw new Error('boom');
});
await drain();
expect(attempts).toBe(4);
expect(errors).toHaveBeenCalledOnce();
expect(String(errors.mock.calls[0]?.[0])).toContain('always-failing');
});
it('should stop retrying as soon as an attempt succeeds', async () => {
let attempts = 0;
enqueue('flaky', async () => {
attempts += 1;
if (attempts < 2) throw new Error('transient');
});
await drain();
expect(attempts).toBe(2);
expect(errors).not.toHaveBeenCalled();
});
it('should keep running later jobs after one fails permanently', async () => {
const completed: string[] = [];
enqueue('doomed', async () => {
throw new Error('boom');
});
enqueue('survivor', async () => {
completed.push('survivor');
});
await drain();
expect(completed).toEqual(['survivor']);
});
it('should grow the backoff delay between attempts', async () => {
vi.useFakeTimers();
vi.spyOn(Math, 'random').mockReturnValue(0);
configureQueue({ maxAttempts: 4, baseDelayMs: 100 });
let attempts = 0;
enqueue('retrying', async () => {
attempts += 1;
throw new Error('boom');
});
await vi.advanceTimersByTimeAsync(0);
expect(attempts).toBe(1);
await vi.advanceTimersByTimeAsync(99);
expect(attempts).toBe(1);
await vi.advanceTimersByTimeAsync(2);
expect(attempts).toBe(2);
await vi.advanceTimersByTimeAsync(198);
expect(attempts).toBe(2);
await vi.advanceTimersByTimeAsync(2);
expect(attempts).toBe(3);
await vi.advanceTimersByTimeAsync(398);
expect(attempts).toBe(3);
await vi.advanceTimersByTimeAsync(2);
expect(attempts).toBe(4);
});
it('should apply jitter within the backoff window', async () => {
vi.useFakeTimers();
vi.spyOn(Math, 'random').mockReturnValue(0.75);
configureQueue({ maxAttempts: 2, baseDelayMs: 100 });
let attempts = 0;
enqueue('jittered', async () => {
attempts += 1;
throw new Error('boom');
});
await vi.advanceTimersByTimeAsync(174);
expect(attempts).toBe(1);
await vi.advanceTimersByTimeAsync(2);
expect(attempts).toBe(2);
});
it('should resolve drain() only once in-flight work has finished', async () => {
const gate = deferred();
let finished = false;
enqueue('blocked', async () => {
await gate.promise;
finished = true;
});
await flush();
expect(finished).toBe(false);
gate.resolve();
await drain();
expect(finished).toBe(true);
});
it('should resolve every concurrent drain() caller', async () => {
const gate = deferred();
enqueue('blocked', () => gate.promise);
const waiters = Promise.all([drain(), drain(), drain()]);
gate.resolve();
await expect(waiters).resolves.toEqual([undefined, undefined, undefined]);
});
it('should resolve drain() immediately when the queue is idle', async () => {
const winner = await Promise.race([
drain().then(() => 'drained'),
new Promise<string>((resolve) => {
setTimeout(() => resolve('timer'), 0);
}),
]);
expect(winner).toBe('drained');
});
it('should report the pending count excluding the job in flight', async () => {
const gate = deferred();
enqueue('blocked', () => gate.promise);
enqueue('second', async () => {});
enqueue('third', async () => {});
expect(size()).toBe(3);
await flush();
expect(size()).toBe(2);
gate.resolve();
await drain();
expect(size()).toBe(0);
});
});

304
backend/tests/signatures.test.ts Normal file
View File

@@ -0,0 +1,304 @@
import { describe, it, expect, afterEach, vi } from 'vitest';
import { createHmac } from 'node:crypto';
import type { IncomingHttpHeaders } from 'node:http';
import { verifySignature } from '../lib/signatures.js';
import { providers } from '../config/providers.js';
import type { ProviderSlug } from '../types/integration.js';
const SECRET = 'top-secret-signing-key';
const RAW_BODY = Buffer.from(JSON.stringify({ action: 'create', id: 'c1' }), 'utf8');
const OTHER_BODY = Buffer.from(JSON.stringify({ action: 'remove', id: 'c1' }), 'utf8');
const hmacHex = (payload: string | Buffer, secret = SECRET): string =>
createHmac('sha256', secret).update(payload).digest('hex');
const verify = (
provider: ProviderSlug,
headers: IncomingHttpHeaders,
rawBody: Buffer = RAW_BODY,
toleranceSeconds?: number
) => verifySignature({ provider, rawBody, headers, secret: SECRET, toleranceSeconds });
const nowSeconds = (): string => Math.floor(Date.now() / 1000).toString();
const slackHeaders = (rawBody: Buffer, timestamp: string): IncomingHttpHeaders => ({
'x-slack-request-timestamp': timestamp,
'x-slack-signature': `v0=${hmacHex(`v0:${timestamp}:${rawBody.toString('utf8')}`)}`,
});
describe('verifySignature: linear-sha256', () => {
it('should accept a correct bare hex digest of the raw body', () => {
expect(verify('linear', { 'linear-signature': hmacHex(RAW_BODY) })).toEqual({ ok: true });
});
it('should read the first value when the header arrives as an array', () => {
expect(verify('linear', { 'linear-signature': [hmacHex(RAW_BODY)] })).toEqual({ ok: true });
});
it('should reject a signature computed over a different body', () => {
expect(verify('linear', { 'linear-signature': hmacHex(OTHER_BODY) })).toEqual({
ok: false,
reason: 'mismatch',
});
});
it('should reject a signature computed with a different secret', () => {
expect(verify('linear', { 'linear-signature': hmacHex(RAW_BODY, 'wrong') })).toEqual({
ok: false,
reason: 'mismatch',
});
});
it('should report a missing header', () => {
expect(verify('linear', {})).toEqual({ ok: false, reason: 'missing' });
expect(verify('linear', { 'linear-signature': '' })).toEqual({
ok: false,
reason: 'missing',
});
});
it('should report a malformed header', () => {
expect(verify('linear', { 'linear-signature': `sha256=${hmacHex(RAW_BODY)}` })).toEqual({
ok: false,
reason: 'malformed',
});
expect(verify('linear', { 'linear-signature': 'deadbeef' })).toEqual({
ok: false,
reason: 'malformed',
});
expect(verify('linear', { 'linear-signature': hmacHex(RAW_BODY).toUpperCase() })).toEqual({
ok: false,
reason: 'malformed',
});
});
it('should be case sensitive about the header name only', () => {
expect(verify('linear', { 'LINEAR-SIGNATURE': hmacHex(RAW_BODY) })).toEqual({
ok: false,
reason: 'missing',
});
});
});
describe('verifySignature: slack-v0', () => {
afterEach(() => {
vi.useRealTimers();
});
it('should accept a correct v0 signature over the timestamped base string', () => {
const timestamp = nowSeconds();
expect(verify('slack', slackHeaders(RAW_BODY, timestamp))).toEqual({ ok: true });
});
it('should accept a signature generated against a frozen clock', () => {
vi.useFakeTimers();
vi.setSystemTime(new Date('2026-08-01T12:00:00.000Z'));
const headers = slackHeaders(RAW_BODY, nowSeconds());
expect(verify('slack', headers)).toEqual({ ok: true });
});
it('should reject a signature computed over a different body', () => {
const timestamp = nowSeconds();
expect(verify('slack', slackHeaders(OTHER_BODY, timestamp))).toEqual({
ok: false,
reason: 'mismatch',
});
});
it('should reject a signature bound to a different timestamp', () => {
const timestamp = nowSeconds();
const headers = {
...slackHeaders(RAW_BODY, timestamp),
'x-slack-request-timestamp': (Number(timestamp) - 1).toString(),
};
expect(verify('slack', headers)).toEqual({ ok: false, reason: 'mismatch' });
});
it('should report a missing signature header', () => {
expect(verify('slack', { 'x-slack-request-timestamp': nowSeconds() })).toEqual({
ok: false,
reason: 'missing',
});
});
it('should report a missing timestamp header', () => {
const { 'x-slack-signature': signature } = slackHeaders(RAW_BODY, nowSeconds());
expect(verify('slack', { 'x-slack-signature': signature })).toEqual({
ok: false,
reason: 'missing',
});
});
it('should report a malformed signature header', () => {
const timestamp = nowSeconds();
expect(
verify('slack', { ...slackHeaders(RAW_BODY, timestamp), 'x-slack-signature': 'v1=abc' })
).toEqual({ ok: false, reason: 'malformed' });
expect(
verify('slack', {
...slackHeaders(RAW_BODY, timestamp),
'x-slack-signature': hmacHex(RAW_BODY),
})
).toEqual({ ok: false, reason: 'malformed' });
});
it('should reject a timestamp older than the tolerance window', () => {
const stale = (Math.floor(Date.now() / 1000) - 301).toString();
expect(verify('slack', slackHeaders(RAW_BODY, stale))).toEqual({
ok: false,
reason: 'stale',
});
});
it('should reject a timestamp too far in the future', () => {
const future = (Math.floor(Date.now() / 1000) + 301).toString();
expect(verify('slack', slackHeaders(RAW_BODY, future))).toEqual({
ok: false,
reason: 'stale',
});
});
it('should reject a request that goes stale while the clock advances', () => {
vi.useFakeTimers();
vi.setSystemTime(new Date('2026-08-01T12:00:00.000Z'));
const headers = slackHeaders(RAW_BODY, nowSeconds());
expect(verify('slack', headers)).toEqual({ ok: true });
vi.advanceTimersByTime(301_000);
expect(verify('slack', headers)).toEqual({ ok: false, reason: 'stale' });
});
it('should honor an explicit tolerance', () => {
const old = (Math.floor(Date.now() / 1000) - 600).toString();
expect(verify('slack', slackHeaders(RAW_BODY, old), RAW_BODY, 900)).toEqual({ ok: true });
expect(verify('slack', slackHeaders(RAW_BODY, old), RAW_BODY, 60)).toEqual({
ok: false,
reason: 'stale',
});
});
it('should reject a non-numeric timestamp as stale', () => {
expect(
verify('slack', {
'x-slack-request-timestamp': 'yesterday',
'x-slack-signature': `v0=${hmacHex('v0:yesterday:x')}`,
})
).toEqual({ ok: false, reason: 'stale' });
});
});
describe('verifySignature: github-sha256', () => {
it('should accept a correct sha256-prefixed signature', () => {
expect(verify('github', { 'x-hub-signature-256': `sha256=${hmacHex(RAW_BODY)}` })).toEqual({
ok: true,
});
});
it('should reject a signature computed over a different body', () => {
expect(verify('github', { 'x-hub-signature-256': `sha256=${hmacHex(OTHER_BODY)}` })).toEqual({
ok: false,
reason: 'mismatch',
});
});
it('should report a missing header', () => {
expect(verify('github', {})).toEqual({ ok: false, reason: 'missing' });
});
it('should report a malformed header', () => {
expect(verify('github', { 'x-hub-signature-256': hmacHex(RAW_BODY) })).toEqual({
ok: false,
reason: 'malformed',
});
expect(verify('github', { 'x-hub-signature-256': `sha1=${hmacHex(RAW_BODY)}` })).toEqual({
ok: false,
reason: 'malformed',
});
});
it('should reject a truncated but correctly prefixed signature', () => {
expect(
verify('github', { 'x-hub-signature-256': `sha256=${hmacHex(RAW_BODY).slice(0, 32)}` })
).toEqual({ ok: false, reason: 'mismatch' });
});
});
describe('verifySignature: none', () => {
it('should accept rest and jira without any header', () => {
expect(verify('rest', {})).toEqual({ ok: true });
expect(verify('jira', {})).toEqual({ ok: true });
});
it('should accept the none scheme even when a garbage header is present', () => {
expect(verify('rest', { 'linear-signature': 'nonsense' })).toEqual({ ok: true });
});
it('should cover every configured provider with a known scheme', () => {
const schemes = Object.values(providers).map((config) => config.signatureScheme);
expect(new Set(schemes)).toEqual(
new Set(['none', 'linear-sha256', 'slack-v0', 'github-sha256'])
);
});
});
describe('verifySignature: hostile input', () => {
const garbage: readonly string[] = [
'',
' ',
'v0=',
'sha256=',
':::',
'v0=zzzz',
'%%%%',
'0'.repeat(10_000),
'\u0000\u0000',
'null',
];
it('should never throw for any provider and any garbage header value', () => {
const slugs = Object.keys(providers) as ProviderSlug[];
for (const slug of slugs) {
for (const value of garbage) {
const headers: IncomingHttpHeaders = {
'linear-signature': value,
'x-hub-signature-256': value,
'x-slack-signature': value,
'x-slack-request-timestamp': value,
};
expect(() => verify(slug, headers)).not.toThrow();
expect(typeof verify(slug, headers).ok).toBe('boolean');
}
}
});
it('should never throw for an empty body or empty header set', () => {
const slugs = Object.keys(providers) as ProviderSlug[];
for (const slug of slugs) {
expect(() => verify(slug, {}, Buffer.alloc(0))).not.toThrow();
}
});
it('should tolerate array-valued and duplicated headers', () => {
expect(() =>
verify('slack', {
'x-slack-signature': ['v0=abc', 'v0=def'],
'x-slack-request-timestamp': [nowSeconds(), 'garbage'],
})
).not.toThrow();
});
});

20
backend/tests/streams.test.js → backend/tests/streams.test.ts
View File

@@ -1,11 +1,11 @@
import { describe, it, expect } from 'vitest';
import { Readable } from 'node:stream';
import { Readable, type Transform } from 'node:stream';
import { pipeline } from 'node:stream/promises';
import { batch, jsonArray } from '../lib/streams.js';
const collect = async (source, transform) => {
const out = [];
await pipeline(source, transform, async (results) => {
const collect = async <T>(source: Readable, transform: Transform): Promise<T[]> => {
const out: T[] = [];
await pipeline(source, transform, async (results: AsyncIterable<T>) => {
for await (const item of results) out.push(item);
});
return out;
@@ -15,7 +15,7 @@ describe('batch', () => {
it('should group items into fixed-size arrays', async () => {
const source = Readable.from([1, 2, 3, 4], { objectMode: true });
const result = await collect(source, batch(2));
const result = await collect<number[]>(source, batch<number>(2));
expect(result).toEqual([[1, 2], [3, 4]]);
});
@@ -23,7 +23,7 @@ describe('batch', () => {
it('should flush a partial trailing batch', async () => {
const source = Readable.from([1, 2, 3, 4, 5], { objectMode: true });
const result = await collect(source, batch(2));
const result = await collect<number[]>(source, batch<number>(2));
expect(result).toEqual([[1, 2], [3, 4], [5]]);
});
@@ -31,7 +31,7 @@ describe('batch', () => {
it('should emit nothing for an empty source', async () => {
const source = Readable.from([], { objectMode: true });
const result = await collect(source, batch(3));
const result = await collect<number[]>(source, batch<number>(3));
expect(result).toEqual([]);
});
@@ -43,8 +43,8 @@ describe('batch', () => {
});
describe('jsonArray', () => {
const serialize = async (items) => {
const chunks = await collect(
const serialize = async (items: unknown[]): Promise<string> => {
const chunks = await collect<string>(
Readable.from(items, { objectMode: true }),
jsonArray()
);
@@ -74,7 +74,7 @@ describe('jsonArray', () => {
});
it('should propagate serialization errors', async () => {
const circular = {};
const circular: Record<string, unknown> = {};
circular.self = circular;
await expect(serialize([circular])).rejects.toThrow();

253
backend/tests/webhooks.routes.test.ts Normal file
View File

@@ -0,0 +1,253 @@
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
import request from 'supertest';
import { createHmac } from 'node:crypto';
import app from '../app.js';
import { configureQueue, drain } from '../lib/queue.js';
vi.mock('../db/ingest_events.dao.js', () => ({
recordDelivery: vi.fn(),
markProcessing: vi.fn(),
markDone: vi.fn(),
markFailed: vi.fn(),
resetStaleProcessing: vi.fn(),
}));
vi.mock('../db/notes.dao.js', () => ({
createNotes: vi.fn(),
getAllNotes: vi.fn(),
streamAllNotes: vi.fn(),
}));
import {
recordDelivery,
markDone,
markFailed,
markProcessing,
} from '../db/ingest_events.dao.js';
import { createNotes } from '../db/notes.dao.js';
const mockRecordDelivery = vi.mocked(recordDelivery);
const mockCreateNotes = vi.mocked(createNotes);
const mockMarkDone = vi.mocked(markDone);
const mockMarkFailed = vi.mocked(markFailed);
const mockMarkProcessing = vi.mocked(markProcessing);
const SECRET = 'linear-test-secret';
// Deliberately irregular spacing: if any middleware parsed and re-serialized
// this body, the signature computed over these exact bytes would not verify.
const RAW_BODY = '{"action":"create", "type":"Comment","data":{"id":"cmt_1","body":"Deploys are scary","url":"https://linear.app/c/1","user":{"name":"jane"}} }';
const sign = (body: string, secret = SECRET): string =>
createHmac('sha256', secret).update(body).digest('hex');
const postLinear = (body: string, signature: string) =>
request(app)
.post('/v1/webhooks/linear')
.set('Content-Type', 'application/json')
.set('linear-signature', signature)
.send(body);
describe('POST /v1/webhooks/:provider', () => {
const originalSecret = process.env.LINEAR_SIGNING_SECRET;
beforeEach(() => {
vi.clearAllMocks();
process.env.LINEAR_SIGNING_SECRET = SECRET;
configureQueue({ baseDelayMs: 0, maxAttempts: 2 });
mockRecordDelivery.mockResolvedValue(42);
mockCreateNotes.mockResolvedValue([]);
});
afterEach(async () => {
await drain();
if (originalSecret === undefined) {
delete process.env.LINEAR_SIGNING_SECRET;
} else {
process.env.LINEAR_SIGNING_SECRET = originalSecret;
}
});
it('should accept a correctly signed delivery', async () => {
const res = await postLinear(RAW_BODY, sign(RAW_BODY));
expect(res.status).toBe(200);
expect(res.body).toEqual({ accepted: true, notes: 1 });
});
it('should preserve the exact request bytes for signature verification', async () => {
// The signature is over RAW_BODY verbatim. This passing is the regression
// test for express.raw being mounted ahead of express.json in app.ts.
const res = await postLinear(RAW_BODY, sign(RAW_BODY));
expect(res.status).toBe(200);
});
it('should insert the normalized note after responding', async () => {
await postLinear(RAW_BODY, sign(RAW_BODY));
await drain();
expect(mockCreateNotes).toHaveBeenCalledOnce();
const [notes] = mockCreateNotes.mock.calls[0];
expect(notes).toHaveLength(1);
expect(notes[0].text).toBe('Deploys are scary');
expect(notes[0].author).toBe('jane');
expect(notes[0].id).toBe('linear_cmt_1');
});
it('should record provenance on the ingested note', async () => {
await postLinear(RAW_BODY, sign(RAW_BODY));
await drain();
const [notes] = mockCreateNotes.mock.calls[0];
expect(notes[0].sourceMeta).toMatchObject({
provider: 'linear',
externalId: 'cmt_1',
permalink: 'https://linear.app/c/1',
});
});
it('should mark the delivery done once the insert succeeds', async () => {
await postLinear(RAW_BODY, sign(RAW_BODY));
await drain();
expect(mockMarkProcessing).toHaveBeenCalledWith(42);
expect(mockMarkDone).toHaveBeenCalledWith(42);
expect(mockMarkFailed).not.toHaveBeenCalled();
});
it('should mark the delivery failed when the insert keeps failing', async () => {
mockCreateNotes.mockRejectedValue(new Error('connection refused'));
await postLinear(RAW_BODY, sign(RAW_BODY));
await drain();
expect(mockMarkFailed).toHaveBeenCalledWith(42, 'connection refused');
expect(mockMarkDone).not.toHaveBeenCalled();
});
it('should respond without waiting for the insert to finish', async () => {
let finishInsert: () => void = () => {};
mockCreateNotes.mockImplementation(
() => new Promise((resolve) => {
finishInsert = () => resolve([]);
})
);
const res = await postLinear(RAW_BODY, sign(RAW_BODY));
expect(res.status).toBe(200);
expect(mockMarkDone).not.toHaveBeenCalled();
finishInsert();
await drain();
expect(mockMarkDone).toHaveBeenCalledWith(42);
});
it('should drop a redelivery without enqueueing work', async () => {
mockRecordDelivery.mockResolvedValue(null);
const res = await postLinear(RAW_BODY, sign(RAW_BODY));
await drain();
expect(res.status).toBe(200);
expect(res.body).toEqual({ duplicate: true });
expect(mockCreateNotes).not.toHaveBeenCalled();
});
it('should reject a signature computed with the wrong secret', async () => {
const res = await postLinear(RAW_BODY, sign(RAW_BODY, 'wrong-secret'));
expect(res.status).toBe(401);
expect(mockRecordDelivery).not.toHaveBeenCalled();
});
it('should reject a delivery whose body was altered after signing', async () => {
const signature = sign(RAW_BODY);
const tampered = RAW_BODY.replace('Deploys are scary', 'Deploys are fine');
const res = await postLinear(tampered, signature);
expect(res.status).toBe(401);
});
it('should reject a delivery with no signature header', async () => {
const res = await request(app)
.post('/v1/webhooks/linear')
.set('Content-Type', 'application/json')
.send(RAW_BODY);
expect(res.status).toBe(401);
});
it('should return 500 when no signing secret is configured', async () => {
delete process.env.LINEAR_SIGNING_SECRET;
const res = await postLinear(RAW_BODY, sign(RAW_BODY));
expect(res.status).toBe(500);
expect(mockRecordDelivery).not.toHaveBeenCalled();
});
it('should return 500 when recording the delivery fails', async () => {
mockRecordDelivery.mockRejectedValue(new Error('connection refused'));
const res = await postLinear(RAW_BODY, sign(RAW_BODY));
expect(res.status).toBe(500);
expect(mockCreateNotes).not.toHaveBeenCalled();
});
it('should return 404 for an unregistered provider', async () => {
const res = await request(app)
.post('/v1/webhooks/notion')
.set('Content-Type', 'application/json')
.send('{}');
expect(res.status).toBe(404);
});
it('should return 501 for a provider with no ingestion mapping yet', async () => {
process.env.GITHUB_WEBHOOK_SECRET = 'gh-secret';
const body = '{"action":"created"}';
const signature = `sha256=${createHmac('sha256', 'gh-secret').update(body).digest('hex')}`;
const res = await request(app)
.post('/v1/webhooks/github')
.set('Content-Type', 'application/json')
.set('x-hub-signature-256', signature)
.send(body);
delete process.env.GITHUB_WEBHOOK_SECRET;
expect(res.status).toBe(501);
});
it('should return 400 for a body that is not valid JSON', async () => {
const body = 'not json at all';
const res = await postLinear(body, sign(body));
expect(res.status).toBe(400);
});
it('should return 400 when the payload is missing the fields the provider guarantees', async () => {
const body = '{"action":"create","data":{}}';
const res = await postLinear(body, sign(body));
expect(res.status).toBe(400);
expect(mockRecordDelivery).not.toHaveBeenCalled();
});
it('should answer a Slack url_verification handshake without a signature', async () => {
const res = await request(app)
.post('/v1/webhooks/slack')
.set('Content-Type', 'application/json')
.send('{"type":"url_verification","challenge":"abc123"}');
expect(res.status).toBe(200);
expect(res.body).toEqual({ challenge: 'abc123' });
});
});

7
backend/tsconfig.build.json Normal file
View File

@@ -0,0 +1,7 @@
{
"extends": "./tsconfig.json",
"compilerOptions": {
"noEmit": false
},
"exclude": ["node_modules", "dist", "tests"]
}

26
backend/tsconfig.json Normal file
View File

@@ -0,0 +1,26 @@
{
"compilerOptions": {
"target": "ES2022",
"lib": ["ES2022"],
"module": "NodeNext",
"moduleResolution": "NodeNext",
"types": ["node"],
"rootDir": ".",
"outDir": "dist",
"esModuleInterop": true,
"resolveJsonModule": true,
"skipLibCheck": true,
"sourceMap": true,
"noEmit": true,
/* Linting */
"strict": true,
"noUnusedLocals": true,
"noUnusedParameters": true,
"noImplicitOverride": true,
"noFallthroughCasesInSwitch": true,
"noUncheckedSideEffectImports": true
},
"include": ["**/*.ts"],
"exclude": ["node_modules", "dist"]
}

55
backend/types/domain.ts Normal file
View File

@@ -0,0 +1,55 @@
export type Note = {
id: string;
text: string;
x: number;
y: number;
author: string;
color: string;
};
/** Shape accepted when writing a note; positional and color fields fall back to defaults. */
export type NoteInput = {
id: string;
text: string;
author: string;
x?: number;
y?: number;
color?: string;
sourceMeta?: Record<string, unknown>;
};
export type Cluster = {
label: string;
noteIds: string[];
};
export type ClusterResponse = {
clusters: Cluster[];
score: number;
};
export type ValidationResult = {
valid: boolean;
reasons: string[];
};
/** noteId to embedding vector. */
export type EmbeddingMap = Map<string, number[]>;
const isStringArray = (value: unknown): value is string[] =>
Array.isArray(value) && value.every((entry) => typeof entry === 'string');
/**
* Runtime guard for LLM output, which arrives as parsed JSON of unknown shape.
* Structural correctness beyond this (complete coverage, no duplicates) is the
* job of validateStructure.
*/
export const isClusterArray = (value: unknown): value is Cluster[] => {
if (!Array.isArray(value)) return false;
return value.every((entry) => {
if (typeof entry !== 'object' || entry === null) return false;
const candidate = entry as Record<string, unknown>;
return typeof candidate.label === 'string' && isStringArray(candidate.noteIds);
});
};

43
backend/types/integration.ts Normal file
View File

@@ -0,0 +1,43 @@
import type { NoteInput } from './domain.js';
export type ProviderSlug = 'slack' | 'jira' | 'linear' | 'github' | 'rest';
export type SignatureScheme = 'slack-v0' | 'github-sha256' | 'linear-sha256' | 'none';
export type DeliveryStatus = 'pending' | 'processing' | 'done' | 'failed';
/**
* Integration as exposed to application code. Ciphertext columns are
* deliberately absent so a value of this type can never leak a secret.
*/
export type IntegrationRow = {
id: number;
provider: ProviderSlug;
externalWorkspaceId: string | null;
displayName: string | null;
scopes: string[];
tokenExpiresAt: Date | null;
};
export type SignatureResult =
| { ok: true }
| { ok: false; reason: 'missing' | 'malformed' | 'mismatch' | 'stale' };
export type NormalizedDelivery = {
externalId: string;
notes: NoteInput[];
};
/** Provenance recorded on each ingested note's source_meta column. */
export type NoteProvenance = {
provider: ProviderSlug;
externalId: string;
permalink?: string;
authorHandle?: string;
receivedAt: string;
};
const SLUGS: readonly string[] = ['slack', 'jira', 'linear', 'github', 'rest'];
export const isProviderSlug = (value: string): value is ProviderSlug =>
SLUGS.includes(value);