Cleanup
This commit is contained in:
KS Jannette
@@ -7,105 +7,104 @@ import { batch } from '../lib/streams.js';
|
|||||||
const SELECT_NOTES = 'SELECT id, text, x, y, author, color FROM notes ORDER BY id';
|
const SELECT_NOTES = 'SELECT id, text, x, y, author, color FROM notes ORDER BY id';
|
||||||
|
|
||||||
// Postgres caps a statement at 65535 bind parameters; six columns per note
|
// Postgres caps a statement at 65535 bind parameters; six columns per note
|
||||||
// leaves 10922 as the hard ceiling, so stay well under it.
|
// leaves 10922 as the hard ceiling.
|
||||||
const INSERT_BATCH_SIZE = 1000;
|
const INSERT_BATCH_SIZE = 1000;
|
||||||
|
|
||||||
export const getAllNotes = async () => {
|
export const getAllNotes = async () => {
|
||||||
const { rows } = await query(SELECT_NOTES);
|
const { rows } = await query(SELECT_NOTES);
|
||||||
return rows;
|
return rows;
|
||||||
};
|
};
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Streams every note as an object-mode Readable. The pooled client is checked
|
* Streams every note as an object-mode Readable. The pooled client is released on end, error, or
|
||||||
* out for the life of the stream and released once it ends, errors, or is
|
* destruction by consumer.
|
||||||
* destroyed early by a consumer.
|
|
||||||
*
|
*
|
||||||
* @returns {Promise<import('node:stream').Readable>}
|
* @returns {Promise<import('node:stream').Readable>}
|
||||||
*/
|
*/
|
||||||
export const streamAllNotes = async () => {
|
export const streamAllNotes = async () => {
|
||||||
const client = await getPool().connect();
|
const client = await getPool().connect();
|
||||||
|
|
||||||
let released = false;
|
let released = false;
|
||||||
const release = () => {
|
const release = () => {
|
||||||
if (released) return;
|
if (released) return;
|
||||||
released = true;
|
released = true;
|
||||||
client.release();
|
client.release();
|
||||||
};
|
};
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const rows = client.query(new QueryStream(SELECT_NOTES));
|
const rows = client.query(new QueryStream(SELECT_NOTES));
|
||||||
rows.once('end', release);
|
rows.once('end', release);
|
||||||
rows.once('error', release);
|
rows.once('error', release);
|
||||||
rows.once('close', release);
|
rows.once('close', release);
|
||||||
return rows;
|
return rows;
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
release();
|
release();
|
||||||
throw err;
|
throw err;
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
export const getNoteById = async (id) => {
|
export const getNoteById = async (id) => {
|
||||||
const { rows } = await query(
|
const { rows } = await query(
|
||||||
'SELECT id, text, x, y, author, color FROM notes WHERE id = $1',
|
'SELECT id, text, x, y, author, color FROM notes WHERE id = $1',
|
||||||
[id]
|
[id]
|
||||||
);
|
);
|
||||||
return rows[0] || null;
|
return rows[0] || null;
|
||||||
};
|
};
|
||||||
|
|
||||||
export const createNote = async (note) => {
|
export const createNote = async (note) => {
|
||||||
const { rows } = await query(
|
const { rows } = await query(
|
||||||
`INSERT INTO notes (id, text, x, y, author, color)
|
`INSERT INTO notes (id, text, x, y, author, color)
|
||||||
VALUES ($1, $2, $3, $4, $5, $6)
|
VALUES ($1, $2, $3, $4, $5, $6)
|
||||||
RETURNING id, text, x, y, author, color`,
|
RETURNING id, text, x, y, author, color`,
|
||||||
[note.id, note.text, note.x ?? 0, note.y ?? 0, note.author, note.color ?? 'yellow']
|
[note.id, note.text, note.x ?? 0, note.y ?? 0, note.author, note.color ?? 'yellow']
|
||||||
);
|
);
|
||||||
return rows[0];
|
return rows[0];
|
||||||
};
|
};
|
||||||
|
|
||||||
const insertNoteBatch = async (notes) => {
|
const insertNoteBatch = async (notes) => {
|
||||||
const values = [];
|
const values = [];
|
||||||
const placeholders = [];
|
const placeholders = [];
|
||||||
|
|
||||||
notes.forEach((note, i) => {
|
notes.forEach((note, i) => {
|
||||||
const offset = i * 6;
|
const offset = i * 6;
|
||||||
placeholders.push(
|
placeholders.push(
|
||||||
`($${offset + 1}, $${offset + 2}, $${offset + 3}, $${offset + 4}, $${offset + 5}, $${offset + 6})`
|
`($${offset + 1}, $${offset + 2}, $${offset + 3}, $${offset + 4}, $${offset + 5}, $${offset + 6})`
|
||||||
);
|
);
|
||||||
values.push(
|
values.push(
|
||||||
note.id,
|
note.id,
|
||||||
note.text,
|
note.text,
|
||||||
note.x ?? 0,
|
note.x ?? 0,
|
||||||
note.y ?? 0,
|
note.y ?? 0,
|
||||||
note.author,
|
note.author,
|
||||||
note.color ?? 'yellow'
|
note.color ?? 'yellow'
|
||||||
);
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
const { rows } = await query(
|
const { rows } = await query(
|
||||||
`INSERT INTO notes (id, text, x, y, author, color)
|
`INSERT INTO notes (id, text, x, y, author, color)
|
||||||
VALUES ${placeholders.join(', ')}
|
VALUES ${placeholders.join(', ')}
|
||||||
RETURNING id, text, x, y, author, color`,
|
RETURNING id, text, x, y, author, color`,
|
||||||
values
|
values
|
||||||
);
|
);
|
||||||
return rows;
|
return rows;
|
||||||
};
|
};
|
||||||
|
|
||||||
export const createNotes = async (notes) => {
|
export const createNotes = async (notes) => {
|
||||||
if (!notes || notes.length === 0) {
|
if (!notes || notes.length === 0) {
|
||||||
return [];
|
return [];
|
||||||
}
|
|
||||||
|
|
||||||
const inserted = [];
|
|
||||||
|
|
||||||
await pipeline(
|
|
||||||
Readable.from(notes, { objectMode: true }),
|
|
||||||
batch(INSERT_BATCH_SIZE),
|
|
||||||
async (batches) => {
|
|
||||||
for await (const chunk of batches) {
|
|
||||||
inserted.push(...await insertNoteBatch(chunk));
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
);
|
|
||||||
|
|
||||||
return inserted;
|
const inserted = [];
|
||||||
|
|
||||||
|
await pipeline(
|
||||||
|
Readable.from(notes, { objectMode: true }),
|
||||||
|
batch(INSERT_BATCH_SIZE),
|
||||||
|
async (batches) => {
|
||||||
|
for await (const chunk of batches) {
|
||||||
|
inserted.push(...await insertNoteBatch(chunk));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
);
|
||||||
|
|
||||||
|
return inserted;
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -4,10 +4,9 @@ import { Readable } from "node:stream";
|
|||||||
import { batch } from "../lib/streams.js";
|
import { batch } from "../lib/streams.js";
|
||||||
|
|
||||||
const client = new VoyageAIClient({
|
const client = new VoyageAIClient({
|
||||||
apiKey: process.env.VOYAGEAI_API_KEY,
|
apiKey: process.env.VOYAGEAI_API_KEY,
|
||||||
});
|
});
|
||||||
|
|
||||||
// Well under Voyage's per-request input and token ceilings.
|
|
||||||
const EMBED_BATCH_SIZE = 128;
|
const EMBED_BATCH_SIZE = 128;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -15,28 +14,28 @@ const EMBED_BATCH_SIZE = 128;
|
|||||||
* @returns {Promise<Map<string, number[]>>} noteId → embedding vector
|
* @returns {Promise<Map<string, number[]>>} noteId → embedding vector
|
||||||
*/
|
*/
|
||||||
export const embedNotes = async (notes) => {
|
export const embedNotes = async (notes) => {
|
||||||
const embeddingMap = new Map();
|
const embeddingMap = new Map();
|
||||||
|
|
||||||
if (!notes || notes.length === 0) {
|
if (!notes || notes.length === 0) {
|
||||||
return embeddingMap;
|
return embeddingMap;
|
||||||
}
|
|
||||||
|
|
||||||
await pipeline(
|
|
||||||
Readable.from(notes, { objectMode: true }),
|
|
||||||
batch(EMBED_BATCH_SIZE),
|
|
||||||
async (batches) => {
|
|
||||||
for await (const chunk of batches) {
|
|
||||||
const response = await client.embed({
|
|
||||||
input: chunk.map((n) => n.text),
|
|
||||||
model: "voyage-3",
|
|
||||||
});
|
|
||||||
|
|
||||||
response.data.forEach((item, i) => {
|
|
||||||
embeddingMap.set(chunk[i].id, item.embedding);
|
|
||||||
});
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
);
|
|
||||||
|
|
||||||
return embeddingMap;
|
await pipeline(
|
||||||
|
Readable.from(notes, { objectMode: true }),
|
||||||
|
batch(EMBED_BATCH_SIZE),
|
||||||
|
async (batches) => {
|
||||||
|
for await (const chunk of batches) {
|
||||||
|
const response = await client.embed({
|
||||||
|
input: chunk.map((n) => n.text),
|
||||||
|
model: "voyage-3",
|
||||||
|
});
|
||||||
|
|
||||||
|
response.data.forEach((item, i) => {
|
||||||
|
embeddingMap.set(chunk[i].id, item.embedding);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
);
|
||||||
|
|
||||||
|
return embeddingMap;
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -1,68 +1,67 @@
|
|||||||
/**
|
/**
|
||||||
* Structural validation: confirms the LLM output is well-formed
|
* Structural validation for LLM output
|
||||||
* before it reaches the frontend.
|
|
||||||
*
|
*
|
||||||
* @param {Array<{label: string, noteIds: string[]}>} clusters
|
* @param {Array<{label: string, noteIds: string[]}>} clusters
|
||||||
* @param {string[]} inputNoteIds - the original note IDs that were sent to the LLM
|
* @param {string[]} inputNoteIds - the original note IDs that were sent to the LLM
|
||||||
* @returns {{valid: boolean, reasons: string[]}}
|
* @returns {{valid: boolean, reasons: string[]}}
|
||||||
*/
|
*/
|
||||||
export const validateStructure = (clusters, inputNoteIds) => {
|
export const validateStructure = (clusters, inputNoteIds) => {
|
||||||
const reasons = [];
|
const reasons = [];
|
||||||
|
|
||||||
if (!Array.isArray(clusters) || clusters.length === 0) {
|
if (!Array.isArray(clusters) || clusters.length === 0) {
|
||||||
return { valid: false, reasons: ['Response is not a non-empty array'] };
|
return { valid: false, reasons: ['Response is not a non-empty array'] };
|
||||||
}
|
|
||||||
|
|
||||||
const assignedIds = [];
|
|
||||||
for (const cluster of clusters) {
|
|
||||||
if (!cluster.label || typeof cluster.label !== 'string') {
|
|
||||||
reasons.push(`Cluster missing a valid label`);
|
|
||||||
}
|
}
|
||||||
if (!Array.isArray(cluster.noteIds) || cluster.noteIds.length === 0) {
|
|
||||||
reasons.push(`Cluster "${cluster.label ?? '(unlabeled)'}" has no noteIds`);
|
const assignedIds = [];
|
||||||
|
for (const cluster of clusters) {
|
||||||
|
if (!cluster.label || typeof cluster.label !== 'string') {
|
||||||
|
reasons.push(`Cluster missing a valid label`);
|
||||||
|
}
|
||||||
|
if (!Array.isArray(cluster.noteIds) || cluster.noteIds.length === 0) {
|
||||||
|
reasons.push(`Cluster "${cluster.label ?? '(unlabeled)'}" has no noteIds`);
|
||||||
|
}
|
||||||
|
assignedIds.push(...(cluster.noteIds ?? []));
|
||||||
}
|
}
|
||||||
assignedIds.push(...(cluster.noteIds ?? []));
|
|
||||||
}
|
|
||||||
|
|
||||||
const inputSet = new Set(inputNoteIds);
|
const inputSet = new Set(inputNoteIds);
|
||||||
const assignedSet = new Set(assignedIds);
|
const assignedSet = new Set(assignedIds);
|
||||||
|
|
||||||
if (assignedIds.length !== assignedSet.size) {
|
if (assignedIds.length !== assignedSet.size) {
|
||||||
reasons.push('One or more notes appear in multiple clusters');
|
reasons.push('One or more notes appear in multiple clusters');
|
||||||
}
|
}
|
||||||
|
|
||||||
const missing = inputNoteIds.filter((id) => !assignedSet.has(id));
|
const missing = inputNoteIds.filter((id) => !assignedSet.has(id));
|
||||||
if (missing.length > 0) {
|
if (missing.length > 0) {
|
||||||
reasons.push(`Notes missing from clusters: ${missing.join(', ')}`);
|
reasons.push(`Notes missing from clusters: ${missing.join(', ')}`);
|
||||||
}
|
}
|
||||||
|
|
||||||
const extra = assignedIds.filter((id) => !inputSet.has(id));
|
const extra = assignedIds.filter((id) => !inputSet.has(id));
|
||||||
if (extra.length > 0) {
|
if (extra.length > 0) {
|
||||||
reasons.push(`Unknown noteIds in clusters: ${[...new Set(extra)].join(', ')}`);
|
reasons.push(`Unknown noteIds in clusters: ${[...new Set(extra)].join(', ')}`);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (clusters.length > inputNoteIds.length) {
|
if (clusters.length > inputNoteIds.length) {
|
||||||
reasons.push(`More clusters (${clusters.length}) than notes (${inputNoteIds.length})`);
|
reasons.push(`More clusters (${clusters.length}) than notes (${inputNoteIds.length})`);
|
||||||
}
|
}
|
||||||
|
|
||||||
return { valid: reasons.length === 0, reasons };
|
return { valid: reasons.length === 0, reasons };
|
||||||
};
|
};
|
||||||
|
|
||||||
const cosineSimilarity = (a, b) => {
|
const cosineSimilarity = (a, b) => {
|
||||||
let dot = 0;
|
let dot = 0;
|
||||||
let magA = 0;
|
let magA = 0;
|
||||||
let magB = 0;
|
let magB = 0;
|
||||||
for (let i = 0; i < a.length; i++) {
|
for (let i = 0; i < a.length; i++) {
|
||||||
dot += a[i] * b[i];
|
dot += a[i] * b[i];
|
||||||
magA += a[i] * a[i];
|
magA += a[i] * a[i];
|
||||||
magB += b[i] * b[i];
|
magB += b[i] * b[i];
|
||||||
}
|
}
|
||||||
const denom = Math.sqrt(magA) * Math.sqrt(magB);
|
const denom = Math.sqrt(magA) * Math.sqrt(magB);
|
||||||
return denom === 0 ? 0 : dot / denom;
|
return denom === 0 ? 0 : dot / denom;
|
||||||
};
|
};
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Computes a silhouette-style cohesion score for the clustering.
|
* Computes silhouette-style cohesion score for the clustering.
|
||||||
*
|
*
|
||||||
* For each note, measures how much more similar it is to its own cluster
|
* For each note, measures how much more similar it is to its own cluster
|
||||||
* versus the nearest neighboring cluster. Returns a score in [-1, 1]
|
* versus the nearest neighboring cluster. Returns a score in [-1, 1]
|
||||||
@@ -73,57 +72,57 @@ const cosineSimilarity = (a, b) => {
|
|||||||
* @returns {number} average silhouette score
|
* @returns {number} average silhouette score
|
||||||
*/
|
*/
|
||||||
export const computeCohesionScore = (clusters, embeddingMap) => {
|
export const computeCohesionScore = (clusters, embeddingMap) => {
|
||||||
if (clusters.length <= 1) return 1.0;
|
if (clusters.length <= 1) return 1.0;
|
||||||
|
|
||||||
const scores = [];
|
const scores = [];
|
||||||
|
|
||||||
for (let ci = 0; ci < clusters.length; ci++) {
|
for (let ci = 0; ci < clusters.length; ci++) {
|
||||||
const clusterIds = clusters[ci].noteIds;
|
const clusterIds = clusters[ci].noteIds;
|
||||||
if (clusterIds.length <= 1) {
|
if (clusterIds.length <= 1) {
|
||||||
scores.push(0);
|
scores.push(0);
|
||||||
continue;
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
for (const noteId of clusterIds) {
|
||||||
|
const vec = embeddingMap.get(noteId);
|
||||||
|
if (!vec) continue;
|
||||||
|
|
||||||
|
// a(i): avg distance to other notes in same cluster
|
||||||
|
let intraSum = 0;
|
||||||
|
let intraCount = 0;
|
||||||
|
for (const otherId of clusterIds) {
|
||||||
|
if (otherId === noteId) continue;
|
||||||
|
const otherVec = embeddingMap.get(otherId);
|
||||||
|
if (!otherVec) continue;
|
||||||
|
intraSum += 1 - cosineSimilarity(vec, otherVec);
|
||||||
|
intraCount++;
|
||||||
|
}
|
||||||
|
const a = intraCount > 0 ? intraSum / intraCount : 0;
|
||||||
|
|
||||||
|
// b(i): min avg distance to notes in any other cluster
|
||||||
|
let b = Infinity;
|
||||||
|
for (let oi = 0; oi < clusters.length; oi++) {
|
||||||
|
if (oi === ci) continue;
|
||||||
|
const otherClusterIds = clusters[oi].noteIds;
|
||||||
|
let interSum = 0;
|
||||||
|
let interCount = 0;
|
||||||
|
for (const otherId of otherClusterIds) {
|
||||||
|
const otherVec = embeddingMap.get(otherId);
|
||||||
|
if (!otherVec) continue;
|
||||||
|
interSum += 1 - cosineSimilarity(vec, otherVec);
|
||||||
|
interCount++;
|
||||||
|
}
|
||||||
|
if (interCount > 0) {
|
||||||
|
b = Math.min(b, interSum / interCount);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (b === Infinity) b = 0;
|
||||||
|
|
||||||
|
const max = Math.max(a, b);
|
||||||
|
scores.push(max === 0 ? 0 : (b - a) / max);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
for (const noteId of clusterIds) {
|
if (scores.length === 0) return 0;
|
||||||
const vec = embeddingMap.get(noteId);
|
return scores.reduce((sum, s) => sum + s, 0) / scores.length;
|
||||||
if (!vec) continue;
|
|
||||||
|
|
||||||
// a(i): avg distance to other notes in same cluster
|
|
||||||
let intraSum = 0;
|
|
||||||
let intraCount = 0;
|
|
||||||
for (const otherId of clusterIds) {
|
|
||||||
if (otherId === noteId) continue;
|
|
||||||
const otherVec = embeddingMap.get(otherId);
|
|
||||||
if (!otherVec) continue;
|
|
||||||
intraSum += 1 - cosineSimilarity(vec, otherVec);
|
|
||||||
intraCount++;
|
|
||||||
}
|
|
||||||
const a = intraCount > 0 ? intraSum / intraCount : 0;
|
|
||||||
|
|
||||||
// b(i): min avg distance to notes in any other cluster
|
|
||||||
let b = Infinity;
|
|
||||||
for (let oi = 0; oi < clusters.length; oi++) {
|
|
||||||
if (oi === ci) continue;
|
|
||||||
const otherClusterIds = clusters[oi].noteIds;
|
|
||||||
let interSum = 0;
|
|
||||||
let interCount = 0;
|
|
||||||
for (const otherId of otherClusterIds) {
|
|
||||||
const otherVec = embeddingMap.get(otherId);
|
|
||||||
if (!otherVec) continue;
|
|
||||||
interSum += 1 - cosineSimilarity(vec, otherVec);
|
|
||||||
interCount++;
|
|
||||||
}
|
|
||||||
if (interCount > 0) {
|
|
||||||
b = Math.min(b, interSum / interCount);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if (b === Infinity) b = 0;
|
|
||||||
|
|
||||||
const max = Math.max(a, b);
|
|
||||||
scores.push(max === 0 ? 0 : (b - a) / max);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (scores.length === 0) return 0;
|
|
||||||
return scores.reduce((sum, s) => sum + s, 0) / scores.length;
|
|
||||||
};
|
};
|
||||||
|
|||||||
Reference in New Issue
Block a user