first commit of v.02 application
This commit is contained in:
KS Jannette
66
server/src/index.js
Normal file
66
server/src/index.js
Normal file
@@ -0,0 +1,66 @@
|
||||
'use strict';
|
||||
|
||||
import 'dotenv/config';
|
||||
import express from 'express';
|
||||
import cors from 'cors';
|
||||
import logger from './logger.js';
|
||||
import notebookRoutes from './routes/notebooks.js';
|
||||
import sourceRoutes from './routes/sources.js';
|
||||
import queryRoutes from './routes/query.js';
|
||||
import citationDetailRoutes from './routes/citationDetail.js';
|
||||
import documentRoutes from './routes/documents.js';
|
||||
|
||||
const PORT = process.env.PORT || 8080;
|
||||
|
||||
const REQUIRED_KEYS = ['ANTHROPIC_API_KEY', 'VOYAGE_API_KEY', 'OPENAI_API_KEY'];
|
||||
for (const key of REQUIRED_KEYS) {
|
||||
if (!process.env[key]) {
|
||||
logger.warn(`${key} is not set — API calls that need it will fail`);
|
||||
}
|
||||
}
|
||||
|
||||
const app = express();
|
||||
|
||||
// CORS: wide-open for local dev. In production, lock this to specific origins.
|
||||
app.use(cors());
|
||||
app.use(express.json());
|
||||
|
||||
app.use((req, res, next) => {
|
||||
const start = Date.now();
|
||||
res.on('finish', () => {
|
||||
logger.info({
|
||||
method: req.method,
|
||||
url: req.originalUrl,
|
||||
status: res.statusCode,
|
||||
ms: Date.now() - start,
|
||||
});
|
||||
});
|
||||
next();
|
||||
});
|
||||
|
||||
app.get('/.well-known/healthcheck', (req, res) => {
|
||||
res.json({
|
||||
service: 'notebook-clone',
|
||||
status: 'ok',
|
||||
uptime: process.uptime(),
|
||||
});
|
||||
});
|
||||
|
||||
app.use('/api/notebooks', notebookRoutes);
|
||||
app.use('/api/sources', sourceRoutes);
|
||||
app.use('/api/query', queryRoutes);
|
||||
app.use('/api/citation-detail', citationDetailRoutes);
|
||||
app.use('/api/documents', documentRoutes);
|
||||
|
||||
app.use((err, req, res, _next) => {
|
||||
logger.error({ err, method: req.method, url: req.originalUrl });
|
||||
res.status(err.status || 500).json({
|
||||
error: err.message || 'internal server error',
|
||||
});
|
||||
});
|
||||
|
||||
app.listen(PORT, () => {
|
||||
logger.info({ port: PORT }, 'server listening');
|
||||
});
|
||||
|
||||
export default app;
|
||||
14
server/src/logger.js
Normal file
14
server/src/logger.js
Normal file
@@ -0,0 +1,14 @@
|
||||
'use strict';
|
||||
|
||||
import pino from 'pino';
|
||||
|
||||
const DEBUG = !!process.env.DEBUG;
|
||||
|
||||
const logger = pino({
|
||||
level: DEBUG ? 'debug' : 'info',
|
||||
transport: process.stdout.isTTY
|
||||
? { target: 'pino-pretty' }
|
||||
: undefined,
|
||||
});
|
||||
|
||||
export default logger;
|
||||
44
server/src/middleware/validation.js
Normal file
44
server/src/middleware/validation.js
Normal file
@@ -0,0 +1,44 @@
|
||||
'use strict';
|
||||
|
||||
import * as notebookStore from '../stores/notebookStore.js';
|
||||
|
||||
/**
|
||||
* Ensures notebookId is present in req.body or req.query.
|
||||
* Normalises the value onto req.notebookId for downstream handlers.
|
||||
*/
|
||||
export function requireNotebookId(req, res, next) {
|
||||
const notebookId = req.body?.notebookId ?? req.query?.notebookId;
|
||||
if (!notebookId) {
|
||||
return res.status(400).json({ error: 'notebookId is required' });
|
||||
}
|
||||
req.notebookId = notebookId;
|
||||
next();
|
||||
}
|
||||
|
||||
/**
|
||||
* Must run after requireNotebookId.
|
||||
* Loads the notebook from the store and attaches it as req.notebook.
|
||||
* Returns 404 if the notebook doesn't exist.
|
||||
*/
|
||||
export function requireNotebook(req, res, next) {
|
||||
const notebook = notebookStore.getNotebook(req.notebookId);
|
||||
if (!notebook) {
|
||||
return res.status(404).json({ error: 'notebook not found' });
|
||||
}
|
||||
req.notebook = notebook;
|
||||
next();
|
||||
}
|
||||
|
||||
export function requireFile(req, res, next) {
|
||||
if (!req.file) {
|
||||
return res.status(400).json({ error: 'file is required' });
|
||||
}
|
||||
next();
|
||||
}
|
||||
|
||||
export function requireUrl(req, res, next) {
|
||||
if (!req.body?.url) {
|
||||
return res.status(400).json({ error: 'url is required' });
|
||||
}
|
||||
next();
|
||||
}
|
||||
34
server/src/routes/citationDetail.js
Normal file
34
server/src/routes/citationDetail.js
Normal file
@@ -0,0 +1,34 @@
|
||||
'use strict';
|
||||
|
||||
import { Router } from 'express';
|
||||
import logger from '../logger.js';
|
||||
import { generateCitationDetail } from '../services/generationService.js';
|
||||
|
||||
const router = Router();
|
||||
|
||||
router.post('/', async (req, res, next) => {
|
||||
try {
|
||||
const { chunkTexts, sourceName, answer, citationIndex } = req.body;
|
||||
|
||||
if (!chunkTexts?.length || !answer || citationIndex == null) {
|
||||
return res.status(400).json({
|
||||
error: 'chunkTexts, answer, and citationIndex are required',
|
||||
});
|
||||
}
|
||||
|
||||
logger.info({ citationIndex, sourceName }, 'citation detail requested');
|
||||
|
||||
const detail = await generateCitationDetail({
|
||||
chunkTexts,
|
||||
sourceName: sourceName || 'Unknown',
|
||||
answer,
|
||||
citationIndex,
|
||||
});
|
||||
|
||||
res.json(detail);
|
||||
} catch (err) {
|
||||
next(err);
|
||||
}
|
||||
});
|
||||
|
||||
export default router;
|
||||
63
server/src/routes/documents.js
Normal file
63
server/src/routes/documents.js
Normal file
@@ -0,0 +1,63 @@
|
||||
'use strict';
|
||||
|
||||
import { Router } from 'express';
|
||||
import logger from '../logger.js';
|
||||
import * as notebookStore from '../stores/notebookStore.js';
|
||||
import * as documentCacheStore from '../stores/documentCacheStore.js';
|
||||
import {
|
||||
generateStudyGuide,
|
||||
generateFaq,
|
||||
generateExecutiveBrief,
|
||||
} from '../services/documentService.js';
|
||||
|
||||
const GENERATORS = {
|
||||
'study-guide': generateStudyGuide,
|
||||
'faq': generateFaq,
|
||||
'executive-brief': generateExecutiveBrief,
|
||||
};
|
||||
|
||||
const router = Router();
|
||||
|
||||
router.post('/generate', async (req, res, next) => {
|
||||
try {
|
||||
const { notebookId, type } = req.body;
|
||||
|
||||
if (!notebookId || !type) {
|
||||
return res.status(400).json({ error: 'notebookId and type are required' });
|
||||
}
|
||||
|
||||
const generator = GENERATORS[type];
|
||||
if (!generator) {
|
||||
return res.status(400).json({
|
||||
error: `Invalid type. Must be one of: ${Object.keys(GENERATORS).join(', ')}`,
|
||||
});
|
||||
}
|
||||
|
||||
const chunks = notebookStore.getChunksForNotebook(notebookId);
|
||||
if (chunks.length === 0) {
|
||||
return res.status(422).json({ error: 'No source material available in this notebook' });
|
||||
}
|
||||
|
||||
const sources = notebookStore.getSources(notebookId);
|
||||
const cached = documentCacheStore.getCachedDocument(notebookId, type);
|
||||
if (cached && documentCacheStore.isFresh(notebookId, type, sources)) {
|
||||
logger.info({ notebookId, type }, 'serving cached document');
|
||||
return res.json({ type, document: cached.document });
|
||||
}
|
||||
|
||||
const sourceGroups = notebookStore.buildSourceGroups(notebookId, chunks);
|
||||
|
||||
logger.info({ notebookId, type, sourceCount: sourceGroups.length, chunkCount: chunks.length }, 'document generation started');
|
||||
|
||||
const document = await generator(sourceGroups);
|
||||
|
||||
documentCacheStore.setCachedDocument(notebookId, type, document, sources);
|
||||
logger.info({ notebookId, type }, 'document generation complete');
|
||||
|
||||
res.json({ type, document });
|
||||
} catch (err) {
|
||||
next(err);
|
||||
}
|
||||
});
|
||||
|
||||
export default router;
|
||||
42
server/src/routes/notebooks.js
Normal file
42
server/src/routes/notebooks.js
Normal file
@@ -0,0 +1,42 @@
|
||||
'use strict';
|
||||
|
||||
import { Router } from 'express';
|
||||
import { v4 as uuidv4 } from 'uuid';
|
||||
import * as notebookStore from '../stores/notebookStore.js';
|
||||
import { deleteVectorsForChunks } from '../stores/vectorStore.js';
|
||||
|
||||
const router = Router();
|
||||
|
||||
router.get('/', (req, res) => {
|
||||
res.json(notebookStore.getAllNotebooks());
|
||||
});
|
||||
|
||||
router.post('/', (req, res) => {
|
||||
const { name } = req.body;
|
||||
if (!name || !name.trim()) {
|
||||
return res.status(400).json({ error: 'name is required' });
|
||||
}
|
||||
|
||||
const notebook = notebookStore.createNotebook({
|
||||
id: uuidv4(),
|
||||
name: name.trim(),
|
||||
createdAt: new Date().toISOString(),
|
||||
});
|
||||
|
||||
res.status(201).json(notebook);
|
||||
});
|
||||
|
||||
router.delete('/:id', (req, res) => {
|
||||
const chunks = notebookStore.getChunksForNotebook(req.params.id);
|
||||
const deleted = notebookStore.deleteNotebook(req.params.id);
|
||||
if (!deleted) {
|
||||
return res.status(404).json({ error: 'notebook not found' });
|
||||
}
|
||||
if (chunks.length) {
|
||||
deleteVectorsForChunks(chunks.map((c) => c.id));
|
||||
}
|
||||
res.json({ ok: true });
|
||||
});
|
||||
|
||||
export default router;
|
||||
|
||||
88
server/src/routes/query.js
Normal file
88
server/src/routes/query.js
Normal file
@@ -0,0 +1,88 @@
|
||||
'use strict';
|
||||
|
||||
import { Router } from 'express';
|
||||
import logger from '../logger.js';
|
||||
import { embedTexts, search, rerank } from '../services/retrievalService.js';
|
||||
import { generate } from '../services/generationService.js';
|
||||
import { computeGroundedness } from '../services/scoringService.js';
|
||||
import * as notebookStore from '../stores/notebookStore.js';
|
||||
|
||||
const TOP_K_SEARCH = 20;
|
||||
const TOP_K_RERANK = 5;
|
||||
|
||||
const router = Router();
|
||||
|
||||
router.post('/', async (req, res, next) => {
|
||||
try {
|
||||
const { notebookId, question } = req.body;
|
||||
if (!notebookId || !question) {
|
||||
return res.status(400).json({ error: 'notebookId and question are required' });
|
||||
}
|
||||
|
||||
logger.info({ notebookId, question }, 'query received');
|
||||
|
||||
const [queryEmbedding] = await embedTexts([question], 'query');
|
||||
|
||||
const searchResults = search(queryEmbedding, notebookId, TOP_K_SEARCH);
|
||||
if (searchResults.length === 0) {
|
||||
return res.json({
|
||||
answer: 'No sources found for this notebook. Upload some documents first.',
|
||||
citations: [],
|
||||
groundednessScore: null,
|
||||
followUpQuestions: [],
|
||||
});
|
||||
}
|
||||
|
||||
const candidateChunks = searchResults.map((r) => r.chunk);
|
||||
const reranked = await rerank(question, candidateChunks);
|
||||
const topChunks = reranked.slice(0, TOP_K_RERANK).map((r) => r.chunk);
|
||||
|
||||
logger.debug({
|
||||
searchHits: searchResults.length,
|
||||
rerankedTop: topChunks.length,
|
||||
}, 'retrieval complete');
|
||||
|
||||
const sourceGroups = notebookStore.buildSourceGroups(notebookId, topChunks);
|
||||
|
||||
const { answer, citedSourceIndices, followUpQuestions } = await generate(
|
||||
question,
|
||||
sourceGroups,
|
||||
);
|
||||
|
||||
const citedChunkIds = [];
|
||||
for (const idx of citedSourceIndices) {
|
||||
const group = sourceGroups.find((g) => g.docIndex === idx);
|
||||
if (group) {
|
||||
citedChunkIds.push(...group.chunks.map((c) => c.id));
|
||||
}
|
||||
}
|
||||
|
||||
const groundednessScore = await computeGroundedness(answer, citedChunkIds);
|
||||
|
||||
logger.info({ groundednessScore: groundednessScore.toFixed(3) }, 'query complete');
|
||||
|
||||
const citations = citedSourceIndices.map((idx) => {
|
||||
const group = sourceGroups.find((g) => g.docIndex === idx);
|
||||
return group
|
||||
? {
|
||||
sourceIndex: idx,
|
||||
sourceId: group.sourceId,
|
||||
name: group.name,
|
||||
chunkTexts: group.chunks.map((c) => c.text),
|
||||
}
|
||||
: { sourceIndex: idx, sourceId: null, name: null, chunkTexts: [] };
|
||||
});
|
||||
|
||||
res.json({
|
||||
answer,
|
||||
citations,
|
||||
groundednessScore,
|
||||
followUpQuestions,
|
||||
});
|
||||
} catch (err) {
|
||||
next(err);
|
||||
}
|
||||
});
|
||||
|
||||
export default router;
|
||||
|
||||
138
server/src/routes/sources.js
Normal file
138
server/src/routes/sources.js
Normal file
@@ -0,0 +1,138 @@
|
||||
'use strict';
|
||||
|
||||
import { Router } from 'express';
|
||||
import multer from 'multer';
|
||||
import { v4 as uuidv4 } from 'uuid';
|
||||
import logger from '../logger.js';
|
||||
import * as notebookStore from '../stores/notebookStore.js';
|
||||
import {
|
||||
parseFile,
|
||||
chunkText,
|
||||
parseUrl,
|
||||
parseYoutubeUrl,
|
||||
isYoutubeUrl,
|
||||
} from '../services/sourceService.js';
|
||||
import { embedTexts, storeChunkEmbeddings } from '../services/retrievalService.js';
|
||||
import { triggerPreGeneration } from '../services/preGenerationService.js';
|
||||
import {
|
||||
requireNotebookId,
|
||||
requireNotebook,
|
||||
requireFile,
|
||||
requireUrl,
|
||||
} from '../middleware/validation.js';
|
||||
|
||||
const upload = multer({ storage: multer.memoryStorage(), limits: { fileSize: 50 * 1024 * 1024 } });
|
||||
const router = Router();
|
||||
|
||||
const TIMESTAMP_RE = /\s+\d{1,2}\.\d{2}\.\d{2}\s*[\u2018\u2019''\s]?\s*[AP]M(?=\.\w+$)/i;
|
||||
function cleanFilename(name) {
|
||||
return name.replace(TIMESTAMP_RE, '');
|
||||
}
|
||||
|
||||
router.get('/', requireNotebookId, (req, res) => {
|
||||
res.json(notebookStore.getSources(req.notebookId));
|
||||
});
|
||||
|
||||
|
||||
function multerUpload(req, res, next) {
|
||||
upload.single('file')(req, res, (err) => {
|
||||
if (err) {
|
||||
if (err.code === 'LIMIT_FILE_SIZE') {
|
||||
return res.status(413).json({ error: 'File too large. Maximum size is 50 MB.' });
|
||||
}
|
||||
return next(err);
|
||||
}
|
||||
next();
|
||||
});
|
||||
}
|
||||
|
||||
router.post(
|
||||
'/',
|
||||
multerUpload,
|
||||
requireNotebookId,
|
||||
requireNotebook,
|
||||
requireFile,
|
||||
async (req, res, next) => {
|
||||
try {
|
||||
const sourceId = uuidv4();
|
||||
const rawName = Buffer.from(req.file.originalname, 'latin1').toString('utf-8');
|
||||
const displayName = cleanFilename(rawName);
|
||||
const text = await parseFile(req.file.buffer, req.file.mimetype, displayName);
|
||||
const chunks = chunkText(text, sourceId);
|
||||
|
||||
logger.info({
|
||||
sourceId,
|
||||
filename: displayName,
|
||||
chunkCount: chunks.length,
|
||||
}, 'parsed and chunked source');
|
||||
|
||||
notebookStore.addChunksToNotebook(req.notebookId, chunks);
|
||||
|
||||
const texts = chunks.map((c) => c.text);
|
||||
const embeddings = await embedTexts(texts, 'document');
|
||||
storeChunkEmbeddings(chunks, embeddings);
|
||||
|
||||
const source = notebookStore.addSource(req.notebookId, {
|
||||
id: sourceId,
|
||||
name: displayName,
|
||||
mimetype: req.file.mimetype,
|
||||
chunkCount: chunks.length,
|
||||
uploadedAt: new Date().toISOString(),
|
||||
});
|
||||
|
||||
triggerPreGeneration(req.notebookId);
|
||||
|
||||
res.status(201).json(source);
|
||||
} catch (err) {
|
||||
next(err);
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
router.post(
|
||||
'/url',
|
||||
requireNotebookId,
|
||||
requireNotebook,
|
||||
requireUrl,
|
||||
async (req, res, next) => {
|
||||
try {
|
||||
const { url } = req.body;
|
||||
const sourceId = uuidv4();
|
||||
const isYT = isYoutubeUrl(url);
|
||||
const displayName = isYT
|
||||
? `YouTube: ${url}`
|
||||
: url.replace(/^https?:\/\//, '').slice(0, 60);
|
||||
|
||||
logger.info({ sourceId, url, isYT }, 'processing URL source');
|
||||
|
||||
const text = isYT ? await parseYoutubeUrl(url) : await parseUrl(url);
|
||||
const chunks = chunkText(text, sourceId);
|
||||
|
||||
if (chunks.length === 0) {
|
||||
return res.status(422).json({ error: 'No usable text could be extracted from the URL' });
|
||||
}
|
||||
|
||||
notebookStore.addChunksToNotebook(req.notebookId, chunks);
|
||||
|
||||
const texts = chunks.map((c) => c.text);
|
||||
const embeddings = await embedTexts(texts, 'document');
|
||||
storeChunkEmbeddings(chunks, embeddings);
|
||||
|
||||
const source = notebookStore.addSource(req.notebookId, {
|
||||
id: sourceId,
|
||||
name: displayName,
|
||||
mimetype: isYT ? 'video/youtube' : 'text/html',
|
||||
chunkCount: chunks.length,
|
||||
uploadedAt: new Date().toISOString(),
|
||||
});
|
||||
|
||||
triggerPreGeneration(req.notebookId);
|
||||
|
||||
res.status(201).json(source);
|
||||
} catch (err) {
|
||||
next(err);
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
export default router;
|
||||
199
server/src/services/documentService.js
Normal file
199
server/src/services/documentService.js
Normal file
@@ -0,0 +1,199 @@
|
||||
'use strict';
|
||||
|
||||
import Anthropic from '@anthropic-ai/sdk';
|
||||
import logger from '../logger.js';
|
||||
|
||||
const MODEL = 'claude-sonnet-4-5-20250929';
|
||||
|
||||
function getClient() {
|
||||
const key = process.env.ANTHROPIC_API_KEY;
|
||||
if (!key) throw new Error('ANTHROPIC_API_KEY is not set');
|
||||
return new Anthropic({ apiKey: key });
|
||||
}
|
||||
|
||||
const MAX_SOURCE_CHARS = 30000;
|
||||
|
||||
function buildSourceBlock(sourceGroups) {
|
||||
const perSourceBudget = Math.floor(MAX_SOURCE_CHARS / (sourceGroups.length || 1));
|
||||
|
||||
return sourceGroups
|
||||
.map((g) => {
|
||||
const combined = g.chunks.map((c) => c.text).join('\n\n');
|
||||
const trimmed =
|
||||
combined.length > perSourceBudget
|
||||
? combined.slice(0, perSourceBudget) + '\n[...truncated]'
|
||||
: combined;
|
||||
return `[Source ${g.docIndex}] (${g.name})\n${trimmed}`;
|
||||
})
|
||||
.join('\n\n---\n\n');
|
||||
}
|
||||
|
||||
const STUDY_GUIDE_SCHEMA = {
|
||||
type: 'object',
|
||||
properties: {
|
||||
title: { type: 'string', description: 'A concise title for the study guide' },
|
||||
sections: {
|
||||
type: 'array',
|
||||
items: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
heading: { type: 'string' },
|
||||
bullets: { type: 'array', items: { type: 'string' }, description: 'Key points as bullet items' },
|
||||
keyTerms: {
|
||||
type: 'array',
|
||||
items: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
term: { type: 'string' },
|
||||
definition: { type: 'string' },
|
||||
},
|
||||
required: ['term', 'definition'],
|
||||
additionalProperties: false,
|
||||
},
|
||||
},
|
||||
reviewQuestions: { type: 'array', items: { type: 'string' }, description: 'Self-test questions for this section' },
|
||||
},
|
||||
required: ['heading', 'bullets', 'keyTerms', 'reviewQuestions'],
|
||||
additionalProperties: false,
|
||||
},
|
||||
},
|
||||
mnemonics: {
|
||||
type: 'array',
|
||||
items: { type: 'string' },
|
||||
description: 'Memory aids, simplified restatements, or mnemonic devices for the hardest concepts',
|
||||
},
|
||||
},
|
||||
required: ['title', 'sections', 'mnemonics'],
|
||||
additionalProperties: false,
|
||||
};
|
||||
|
||||
export async function generateStudyGuide(sourceGroups) {
|
||||
const client = getClient();
|
||||
const sources = buildSourceBlock(sourceGroups);
|
||||
const start = Date.now();
|
||||
|
||||
const prompt = `You are an expert educator. Given the source documents below, produce a comprehensive study guide. Follow these rules:
|
||||
|
||||
1. Create a structured outline of the main ideas organized into logical sections.
|
||||
2. For each section, provide: bullet-point key concepts, key terms with definitions, and 2-3 self-review questions.
|
||||
3. At the end, provide mnemonic devices or simplified restatements to help memorize the hardest concepts.
|
||||
4. Be concise but thorough. Use simple, clear language.
|
||||
|
||||
--- SOURCE DOCUMENTS ---
|
||||
|
||||
${sources}`;
|
||||
|
||||
const message = await client.messages.create({
|
||||
model: MODEL,
|
||||
max_tokens: 4096,
|
||||
messages: [{ role: 'user', content: prompt }],
|
||||
output_config: { format: { type: 'json_schema', schema: STUDY_GUIDE_SCHEMA } },
|
||||
});
|
||||
|
||||
const parsed = JSON.parse(message.content[0]?.text || '{}');
|
||||
const elapsed = ((Date.now() - start) / 1000).toFixed(1);
|
||||
logger.info({ sections: parsed.sections?.length, elapsedSec: elapsed }, 'study guide generated');
|
||||
return parsed;
|
||||
}
|
||||
|
||||
const FAQ_SCHEMA = {
|
||||
type: 'object',
|
||||
properties: {
|
||||
subject: { type: 'string', description: 'The identified common subject/theme across all sources' },
|
||||
faqPairs: {
|
||||
type: 'array',
|
||||
items: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
question: { type: 'string' },
|
||||
answer: { type: 'string' },
|
||||
},
|
||||
required: ['question', 'answer'],
|
||||
additionalProperties: false,
|
||||
},
|
||||
description: '8-15 frequently asked questions with concise answers',
|
||||
},
|
||||
},
|
||||
required: ['subject', 'faqPairs'],
|
||||
additionalProperties: false,
|
||||
};
|
||||
|
||||
export async function generateFaq(sourceGroups) {
|
||||
const client = getClient();
|
||||
const sources = buildSourceBlock(sourceGroups);
|
||||
const start = Date.now();
|
||||
|
||||
const prompt = `You are a subject-matter expert. Given the source documents below, perform two tasks:
|
||||
|
||||
1. IDENTIFY THE SUBJECT: Analyze all sources and determine the common subject, theme, or topic they share. Do NOT ask the user — infer it from the semantic overlap across the documents.
|
||||
2. GENERATE FAQ: Produce 8-15 frequently asked questions that someone studying or researching this subject would likely ask, along with concise, accurate answers grounded in the source material.
|
||||
|
||||
Make the questions range from foundational ("What is...") to more advanced/nuanced. Answers should be 2-4 sentences each.
|
||||
|
||||
--- SOURCE DOCUMENTS ---
|
||||
|
||||
${sources}`;
|
||||
|
||||
const message = await client.messages.create({
|
||||
model: MODEL,
|
||||
max_tokens: 4096,
|
||||
messages: [{ role: 'user', content: prompt }],
|
||||
output_config: { format: { type: 'json_schema', schema: FAQ_SCHEMA } },
|
||||
});
|
||||
|
||||
const parsed = JSON.parse(message.content[0]?.text || '{}');
|
||||
const elapsed = ((Date.now() - start) / 1000).toFixed(1);
|
||||
logger.info({ subject: parsed.subject, pairs: parsed.faqPairs?.length, elapsedSec: elapsed }, 'FAQ generated');
|
||||
return parsed;
|
||||
}
|
||||
|
||||
const EXEC_BRIEF_SCHEMA = {
|
||||
type: 'object',
|
||||
properties: {
|
||||
title: { type: 'string', description: 'The main subject header for the brief' },
|
||||
sections: {
|
||||
type: 'array',
|
||||
items: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
subhead: { type: 'string' },
|
||||
prose: { type: 'string', description: 'Well-written prose paragraph(s) for this section' },
|
||||
},
|
||||
required: ['subhead', 'prose'],
|
||||
additionalProperties: false,
|
||||
},
|
||||
},
|
||||
},
|
||||
required: ['title', 'sections'],
|
||||
additionalProperties: false,
|
||||
};
|
||||
|
||||
export async function generateExecutiveBrief(sourceGroups) {
|
||||
const client = getClient();
|
||||
const sources = buildSourceBlock(sourceGroups);
|
||||
const start = Date.now();
|
||||
|
||||
const prompt = `You are a senior analyst writing an executive brief. Given the source documents below, produce a highly organized, prose-form summary. Follow these rules:
|
||||
|
||||
1. Identify the overarching subject and use it as the main title/header.
|
||||
2. Organize the information into logical sections, each with a clear subheading.
|
||||
3. Write in polished prose — NO bullet points, NO lists. Use well-structured paragraphs.
|
||||
4. Condense the source material to the equivalent of approximately 3 pages. Prioritize the most important findings, arguments, and conclusions.
|
||||
5. Maintain an authoritative, professional tone suitable for executive-level readers.
|
||||
|
||||
--- SOURCE DOCUMENTS ---
|
||||
|
||||
${sources}`;
|
||||
|
||||
const message = await client.messages.create({
|
||||
model: MODEL,
|
||||
max_tokens: 4096,
|
||||
messages: [{ role: 'user', content: prompt }],
|
||||
output_config: { format: { type: 'json_schema', schema: EXEC_BRIEF_SCHEMA } },
|
||||
});
|
||||
|
||||
const parsed = JSON.parse(message.content[0]?.text || '{}');
|
||||
const elapsed = ((Date.now() - start) / 1000).toFixed(1);
|
||||
logger.info({ title: parsed.title, sections: parsed.sections?.length, elapsedSec: elapsed }, 'executive brief generated');
|
||||
return parsed;
|
||||
}
|
||||
156
server/src/services/documentService.test.js
Normal file
156
server/src/services/documentService.test.js
Normal file
@@ -0,0 +1,156 @@
|
||||
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
|
||||
|
||||
const { mockCreate } = vi.hoisted(() => ({ mockCreate: vi.fn() }));
|
||||
|
||||
vi.mock('@anthropic-ai/sdk', () => ({
|
||||
default: vi.fn(() => ({ messages: { create: mockCreate } })),
|
||||
}));
|
||||
|
||||
vi.mock('../logger.js', () => ({
|
||||
default: { debug: vi.fn(), info: vi.fn(), warn: vi.fn(), error: vi.fn() },
|
||||
}));
|
||||
|
||||
import {
|
||||
generateStudyGuide,
|
||||
generateFaq,
|
||||
generateExecutiveBrief,
|
||||
} from './documentService.js';
|
||||
|
||||
const SOURCE_GROUPS = [
|
||||
{ docIndex: 1, name: 'Doc A', chunks: [{ text: 'Alpha content.' }] },
|
||||
{ docIndex: 2, name: 'Doc B', chunks: [{ text: 'Beta first.' }, { text: 'Beta second.' }] },
|
||||
];
|
||||
|
||||
function apiResponse(obj) {
|
||||
return { content: [{ text: JSON.stringify(obj) }] };
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
process.env.ANTHROPIC_API_KEY = 'test-key';
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
delete process.env.ANTHROPIC_API_KEY;
|
||||
});
|
||||
|
||||
describe('generateStudyGuide', () => {
|
||||
const STUDY_GUIDE = {
|
||||
title: 'Test Guide',
|
||||
sections: [{ heading: 'Intro', bullets: ['b1'], keyTerms: [], reviewQuestions: [] }],
|
||||
mnemonics: ['ABC'],
|
||||
};
|
||||
|
||||
it('returns the parsed API response', async () => {
|
||||
mockCreate.mockResolvedValue(apiResponse(STUDY_GUIDE));
|
||||
const result = await generateStudyGuide(SOURCE_GROUPS);
|
||||
expect(result).toEqual(STUDY_GUIDE);
|
||||
});
|
||||
|
||||
it('sends source content in the prompt', async () => {
|
||||
mockCreate.mockResolvedValue(apiResponse(STUDY_GUIDE));
|
||||
await generateStudyGuide(SOURCE_GROUPS);
|
||||
|
||||
const prompt = mockCreate.mock.calls[0][0].messages[0].content;
|
||||
expect(prompt).toContain('[Source 1] (Doc A)');
|
||||
expect(prompt).toContain('Alpha content.');
|
||||
expect(prompt).toContain('[Source 2] (Doc B)');
|
||||
expect(prompt).toContain('Beta first.');
|
||||
expect(prompt).toContain('Beta second.');
|
||||
});
|
||||
|
||||
it('separates source groups with delimiters', async () => {
|
||||
mockCreate.mockResolvedValue(apiResponse(STUDY_GUIDE));
|
||||
await generateStudyGuide(SOURCE_GROUPS);
|
||||
|
||||
const prompt = mockCreate.mock.calls[0][0].messages[0].content;
|
||||
expect(prompt).toContain('---');
|
||||
});
|
||||
});
|
||||
|
||||
describe('generateFaq', () => {
|
||||
const FAQ = {
|
||||
subject: 'Testing',
|
||||
faqPairs: [{ question: 'Q?', answer: 'A.' }],
|
||||
};
|
||||
|
||||
it('returns the parsed API response', async () => {
|
||||
mockCreate.mockResolvedValue(apiResponse(FAQ));
|
||||
const result = await generateFaq(SOURCE_GROUPS);
|
||||
expect(result).toEqual(FAQ);
|
||||
});
|
||||
|
||||
it('sends source content in the prompt', async () => {
|
||||
mockCreate.mockResolvedValue(apiResponse(FAQ));
|
||||
await generateFaq(SOURCE_GROUPS);
|
||||
|
||||
const prompt = mockCreate.mock.calls[0][0].messages[0].content;
|
||||
expect(prompt).toContain('[Source 1] (Doc A)');
|
||||
expect(prompt).toContain('[Source 2] (Doc B)');
|
||||
});
|
||||
});
|
||||
|
||||
describe('generateExecutiveBrief', () => {
|
||||
const BRIEF = {
|
||||
title: 'Exec Brief',
|
||||
sections: [{ subhead: 'Overview', prose: 'Some prose.' }],
|
||||
};
|
||||
|
||||
it('returns the parsed API response', async () => {
|
||||
mockCreate.mockResolvedValue(apiResponse(BRIEF));
|
||||
const result = await generateExecutiveBrief(SOURCE_GROUPS);
|
||||
expect(result).toEqual(BRIEF);
|
||||
});
|
||||
|
||||
it('sends source content in the prompt', async () => {
|
||||
mockCreate.mockResolvedValue(apiResponse(BRIEF));
|
||||
await generateExecutiveBrief(SOURCE_GROUPS);
|
||||
|
||||
const prompt = mockCreate.mock.calls[0][0].messages[0].content;
|
||||
expect(prompt).toContain('[Source 1] (Doc A)');
|
||||
expect(prompt).toContain('[Source 2] (Doc B)');
|
||||
});
|
||||
});
|
||||
|
||||
describe('shared behaviour', () => {
|
||||
it('throws when ANTHROPIC_API_KEY is not set', async () => {
|
||||
delete process.env.ANTHROPIC_API_KEY;
|
||||
await expect(generateStudyGuide(SOURCE_GROUPS)).rejects.toThrow('ANTHROPIC_API_KEY is not set');
|
||||
});
|
||||
|
||||
it('returns empty object when the API returns empty content', async () => {
|
||||
mockCreate.mockResolvedValue({ content: [{ text: '' }] });
|
||||
const result = await generateStudyGuide(SOURCE_GROUPS);
|
||||
expect(result).toEqual({});
|
||||
});
|
||||
|
||||
it('truncates sources that exceed the per-source character budget', async () => {
|
||||
const longText = 'x'.repeat(35_000);
|
||||
const bigGroups = [
|
||||
{ docIndex: 1, name: 'Big', chunks: [{ text: longText }] },
|
||||
];
|
||||
mockCreate.mockResolvedValue(
|
||||
apiResponse({ title: 't', sections: [], mnemonics: [] }),
|
||||
);
|
||||
|
||||
await generateStudyGuide(bigGroups);
|
||||
|
||||
const prompt = mockCreate.mock.calls[0][0].messages[0].content;
|
||||
expect(prompt).toContain('[...truncated]');
|
||||
expect(prompt.length).toBeLessThan(longText.length);
|
||||
});
|
||||
|
||||
it('combines multiple chunks within a source with double newlines', async () => {
|
||||
const groups = [
|
||||
{ docIndex: 1, name: 'Multi', chunks: [{ text: 'AAA' }, { text: 'BBB' }] },
|
||||
];
|
||||
mockCreate.mockResolvedValue(
|
||||
apiResponse({ title: 't', sections: [], mnemonics: [] }),
|
||||
);
|
||||
|
||||
await generateStudyGuide(groups);
|
||||
|
||||
const prompt = mockCreate.mock.calls[0][0].messages[0].content;
|
||||
expect(prompt).toContain('AAA\n\nBBB');
|
||||
});
|
||||
});
|
||||
173
server/src/services/generationService.js
Normal file
173
server/src/services/generationService.js
Normal file
@@ -0,0 +1,173 @@
|
||||
'use strict';
|
||||
|
||||
import Anthropic from '@anthropic-ai/sdk';
|
||||
import logger from '../logger.js';
|
||||
|
||||
const MODEL = 'claude-opus-4-6';
|
||||
|
||||
const RESPONSE_SCHEMA = {
|
||||
type: 'object',
|
||||
properties: {
|
||||
answer: {
|
||||
type: 'string',
|
||||
description:
|
||||
'The answer with inline numeric citations like [1], [2] matching the source document numbers provided',
|
||||
},
|
||||
citedSourceIndices: {
|
||||
type: 'array',
|
||||
items: { type: 'integer' },
|
||||
description: 'The document numbers cited in the answer',
|
||||
},
|
||||
followUpQuestions: {
|
||||
type: 'array',
|
||||
items: { type: 'string' },
|
||||
description: 'Exactly 3 follow-up questions the user might ask',
|
||||
},
|
||||
},
|
||||
required: ['answer', 'citedSourceIndices', 'followUpQuestions'],
|
||||
additionalProperties: false,
|
||||
};
|
||||
|
||||
function getClient() {
|
||||
const key = process.env.ANTHROPIC_API_KEY;
|
||||
if (!key) throw new Error('ANTHROPIC_API_KEY is not set');
|
||||
return new Anthropic({ apiKey: key });
|
||||
}
|
||||
|
||||
export function buildPrompt(query, sourceGroups) {
|
||||
const today = new Date().toLocaleDateString('en-US', {
|
||||
weekday: 'long',
|
||||
year: 'numeric',
|
||||
month: 'long',
|
||||
day: 'numeric',
|
||||
});
|
||||
|
||||
const sources = sourceGroups
|
||||
.map((g) => {
|
||||
const combined = g.chunks.map((c) => c.text).join('\n\n');
|
||||
return `[Source ${g.docIndex}] (${g.name})\n${combined}`;
|
||||
})
|
||||
.join('\n\n---\n\n');
|
||||
|
||||
return `You are a research assistant. Today's date is ${today}. Answer the user's question using ONLY the source documents provided below. Follow these rules strictly:
|
||||
|
||||
1. Ground every claim in a specific source document. Cite sources inline using numeric notation like [1], [2], etc., matching the source document numbers below.
|
||||
2. If the sources do not contain enough information to answer, say so honestly.
|
||||
3. After your answer, suggest exactly 3 follow-up questions the user might ask based on the sources.
|
||||
|
||||
--- SOURCE DOCUMENTS ---
|
||||
|
||||
${sources}
|
||||
|
||||
--- USER QUESTION ---
|
||||
|
||||
${query}`;
|
||||
}
|
||||
|
||||
export async function generate(query, sourceGroups) {
|
||||
const client = getClient();
|
||||
|
||||
const message = await client.messages.create({
|
||||
model: MODEL,
|
||||
max_tokens: 2048,
|
||||
messages: [
|
||||
{ role: 'user', content: buildPrompt(query, sourceGroups) },
|
||||
],
|
||||
output_config: {
|
||||
format: {
|
||||
type: 'json_schema',
|
||||
schema: RESPONSE_SCHEMA,
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const raw = message.content[0]?.text || '';
|
||||
logger.debug({ rawLength: raw.length }, 'claude response received');
|
||||
|
||||
const parsed = JSON.parse(raw);
|
||||
|
||||
const validIndices = new Set(sourceGroups.map((g) => g.docIndex));
|
||||
const citedSourceIndices = [
|
||||
...new Set(
|
||||
(parsed.citedSourceIndices || []).filter((idx) => validIndices.has(idx)),
|
||||
),
|
||||
];
|
||||
|
||||
return {
|
||||
answer: parsed.answer || '',
|
||||
citedSourceIndices,
|
||||
followUpQuestions: parsed.followUpQuestions || [],
|
||||
};
|
||||
}
|
||||
|
||||
const CITATION_DETAIL_SCHEMA = {
|
||||
type: 'object',
|
||||
properties: {
|
||||
citedSentence: {
|
||||
type: 'string',
|
||||
description: 'The exact sentence or passage from the source material that the citation refers to',
|
||||
},
|
||||
topicSummary: {
|
||||
type: 'string',
|
||||
description: 'A concise summary of the broader topic/context surrounding the cited passage',
|
||||
},
|
||||
additionalInsight: {
|
||||
type: 'string',
|
||||
description: 'Additional expert insight, analysis, or context on the subject beyond what the source states',
|
||||
},
|
||||
relevantLinks: {
|
||||
type: 'array',
|
||||
items: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
title: { type: 'string', description: 'Descriptive link title' },
|
||||
url: { type: 'string', description: 'Full URL to the resource' },
|
||||
},
|
||||
required: ['title', 'url'],
|
||||
additionalProperties: false,
|
||||
},
|
||||
description: 'Up to 3 relevant web resources for further reading on the topic',
|
||||
},
|
||||
},
|
||||
required: ['citedSentence', 'topicSummary', 'additionalInsight', 'relevantLinks'],
|
||||
additionalProperties: false,
|
||||
};
|
||||
|
||||
export async function generateCitationDetail({ chunkTexts, sourceName, answer, citationIndex }) {
|
||||
const client = getClient();
|
||||
|
||||
const sourceText = chunkTexts.join('\n\n');
|
||||
|
||||
const prompt = `You are a research assistant. A user is reading a research response and wants to dive deeper into a specific citation. Below is the full answer they are reading, followed by the source material for citation [${citationIndex}] from "${sourceName}".
|
||||
|
||||
Your task:
|
||||
1. Identify the specific sentence or passage from the source material that citation [${citationIndex}] refers to in the answer.
|
||||
2. Summarize the broader topic/context surrounding that passage in the source material.
|
||||
3. Provide additional expert insight or analysis on this subject that would help the reader understand it more deeply.
|
||||
4. Suggest up to 3 relevant, real web resources (articles, papers, official sites) where the reader can learn more about this specific topic. Only include links you are highly confident are real and accessible.
|
||||
|
||||
--- ANSWER THE USER WAS READING ---
|
||||
|
||||
${answer}
|
||||
|
||||
--- SOURCE MATERIAL FOR [${citationIndex}] (${sourceName}) ---
|
||||
|
||||
${sourceText}`;
|
||||
|
||||
const message = await client.messages.create({
|
||||
model: MODEL,
|
||||
max_tokens: 1536,
|
||||
messages: [{ role: 'user', content: prompt }],
|
||||
output_config: {
|
||||
format: {
|
||||
type: 'json_schema',
|
||||
schema: CITATION_DETAIL_SCHEMA,
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const raw = message.content[0]?.text || '';
|
||||
logger.debug({ citationIndex, rawLength: raw.length }, 'citation detail response received');
|
||||
|
||||
return JSON.parse(raw);
|
||||
}
|
||||
66
server/src/services/generationService.test.js
Normal file
66
server/src/services/generationService.test.js
Normal file
@@ -0,0 +1,66 @@
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import { buildPrompt } from './generationService.js';
|
||||
|
||||
describe('buildPrompt', () => {
|
||||
const sourceGroups = [
|
||||
{
|
||||
docIndex: 1,
|
||||
name: 'Weather Facts',
|
||||
chunks: [{ text: 'The sky is blue.' }],
|
||||
},
|
||||
{
|
||||
docIndex: 2,
|
||||
name: 'Water Science',
|
||||
chunks: [
|
||||
{ text: 'Water is wet.' },
|
||||
{ text: 'Water boils at 100°C.' },
|
||||
],
|
||||
},
|
||||
];
|
||||
|
||||
it('includes the user query', () => {
|
||||
const prompt = buildPrompt('Why is the sky blue?', sourceGroups);
|
||||
expect(prompt).toContain('Why is the sky blue?');
|
||||
});
|
||||
|
||||
it('labels each source with [Source N] and its name', () => {
|
||||
const prompt = buildPrompt('test', sourceGroups);
|
||||
expect(prompt).toContain('[Source 1] (Weather Facts)');
|
||||
expect(prompt).toContain('[Source 2] (Water Science)');
|
||||
});
|
||||
|
||||
it('includes all chunk texts within their source groups', () => {
|
||||
const prompt = buildPrompt('test', sourceGroups);
|
||||
expect(prompt).toContain('The sky is blue.');
|
||||
expect(prompt).toContain('Water is wet.');
|
||||
expect(prompt).toContain('Water boils at 100°C.');
|
||||
});
|
||||
|
||||
it('instructs citation format using numeric source references', () => {
|
||||
const prompt = buildPrompt('test', sourceGroups);
|
||||
expect(prompt).toContain('[1]');
|
||||
expect(prompt).toContain('[2]');
|
||||
});
|
||||
|
||||
it('separates source groups with delimiters', () => {
|
||||
const prompt = buildPrompt('test', sourceGroups);
|
||||
expect(prompt).toContain('---');
|
||||
});
|
||||
|
||||
it('works with a single source group', () => {
|
||||
const prompt = buildPrompt('test', [sourceGroups[0]]);
|
||||
expect(prompt).toContain('[Source 1] (Weather Facts)');
|
||||
expect(prompt).not.toContain('[Source 2]');
|
||||
});
|
||||
|
||||
it('combines multiple chunks within a single source group', () => {
|
||||
const prompt = buildPrompt('test', [sourceGroups[1]]);
|
||||
expect(prompt).toContain('Water is wet.');
|
||||
expect(prompt).toContain('Water boils at 100°C.');
|
||||
});
|
||||
|
||||
it('includes today date', () => {
|
||||
const prompt = buildPrompt('test', sourceGroups);
|
||||
expect(prompt).toMatch(/Today's date is .+\d{4}/);
|
||||
});
|
||||
});
|
||||
91
server/src/services/preGenerationService.js
Normal file
91
server/src/services/preGenerationService.js
Normal file
@@ -0,0 +1,91 @@
|
||||
'use strict';
|
||||
|
||||
import logger from '../logger.js';
|
||||
import * as notebookStore from '../stores/notebookStore.js';
|
||||
import * as documentCacheStore from '../stores/documentCacheStore.js';
|
||||
import {
|
||||
generateStudyGuide,
|
||||
generateFaq,
|
||||
generateExecutiveBrief,
|
||||
} from './documentService.js';
|
||||
|
||||
const MIN_SOURCES = 2;
|
||||
|
||||
const GENERATORS = {
|
||||
'study-guide': generateStudyGuide,
|
||||
'faq': generateFaq,
|
||||
'executive-brief': generateExecutiveBrief,
|
||||
};
|
||||
|
||||
const DEBOUNCE_MS = 5000;
|
||||
|
||||
const pendingReGen = new Set();
|
||||
const debounceTimers = new Map();
|
||||
|
||||
export function triggerPreGeneration(notebookId) {
|
||||
const notebook = notebookStore.getNotebook(notebookId);
|
||||
if (!notebook) return;
|
||||
|
||||
const sources = notebookStore.getSources(notebookId);
|
||||
if (sources.length < MIN_SOURCES) return;
|
||||
|
||||
if (documentCacheStore.isGenerating(notebookId)) {
|
||||
pendingReGen.add(notebookId);
|
||||
documentCacheStore.invalidate(notebookId);
|
||||
logger.debug({ notebookId }, 'pre-generation in progress, queued re-generation');
|
||||
return;
|
||||
}
|
||||
|
||||
clearTimeout(debounceTimers.get(notebookId));
|
||||
debounceTimers.set(
|
||||
notebookId,
|
||||
setTimeout(() => {
|
||||
debounceTimers.delete(notebookId);
|
||||
runPreGeneration(notebookId);
|
||||
}, DEBOUNCE_MS),
|
||||
);
|
||||
logger.debug({ notebookId, debounceMs: DEBOUNCE_MS }, 'pre-generation debounced');
|
||||
}
|
||||
|
||||
function runPreGeneration(notebookId) {
|
||||
try {
|
||||
const sources = notebookStore.getSources(notebookId);
|
||||
const chunks = notebookStore.getChunksForNotebook(notebookId);
|
||||
const sourceGroups = notebookStore.buildSourceGroups(notebookId, chunks);
|
||||
|
||||
documentCacheStore.invalidate(notebookId);
|
||||
documentCacheStore.markGenerating(notebookId);
|
||||
|
||||
logger.info(
|
||||
{ notebookId, sourceCount: sources.length, chunkCount: chunks.length },
|
||||
'background pre-generation started',
|
||||
);
|
||||
|
||||
const jobs = Object.entries(GENERATORS).map(async ([type, generator]) => {
|
||||
try {
|
||||
const document = await generator(sourceGroups);
|
||||
documentCacheStore.setCachedDocument(notebookId, type, document, sources);
|
||||
logger.info({ notebookId, type }, 'background pre-generation complete for type');
|
||||
} catch (err) {
|
||||
logger.error({ notebookId, type, err: err.message }, 'background pre-generation failed for type');
|
||||
}
|
||||
});
|
||||
|
||||
Promise.all(jobs)
|
||||
.then(() => {
|
||||
logger.info({ notebookId }, 'all background pre-generation complete');
|
||||
})
|
||||
.finally(() => {
|
||||
documentCacheStore.clearGenerating(notebookId);
|
||||
|
||||
if (pendingReGen.has(notebookId)) {
|
||||
pendingReGen.delete(notebookId);
|
||||
logger.info({ notebookId }, 're-triggering pre-generation for updated sources');
|
||||
runPreGeneration(notebookId);
|
||||
}
|
||||
});
|
||||
} catch (err) {
|
||||
logger.error({ notebookId, err: err.message }, 'pre-generation setup failed');
|
||||
documentCacheStore.clearGenerating(notebookId);
|
||||
}
|
||||
}
|
||||
216
server/src/services/preGenerationService.test.js
Normal file
216
server/src/services/preGenerationService.test.js
Normal file
@@ -0,0 +1,216 @@
|
||||
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
|
||||
|
||||
vi.mock('../stores/notebookStore.js', () => ({
|
||||
getNotebook: vi.fn(),
|
||||
getSources: vi.fn(),
|
||||
getChunksForNotebook: vi.fn(),
|
||||
buildSourceGroups: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock('../stores/documentCacheStore.js', () => ({
|
||||
isGenerating: vi.fn(),
|
||||
invalidate: vi.fn(),
|
||||
markGenerating: vi.fn(),
|
||||
clearGenerating: vi.fn(),
|
||||
setCachedDocument: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock('./documentService.js', () => ({
|
||||
generateStudyGuide: vi.fn(),
|
||||
generateFaq: vi.fn(),
|
||||
generateExecutiveBrief: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock('../logger.js', () => ({
|
||||
default: { debug: vi.fn(), info: vi.fn(), warn: vi.fn(), error: vi.fn() },
|
||||
}));
|
||||
|
||||
import { triggerPreGeneration } from './preGenerationService.js';
|
||||
import * as notebookStore from '../stores/notebookStore.js';
|
||||
import * as documentCacheStore from '../stores/documentCacheStore.js';
|
||||
import {
|
||||
generateStudyGuide,
|
||||
generateFaq,
|
||||
generateExecutiveBrief,
|
||||
} from './documentService.js';
|
||||
|
||||
function setupNotebook(id) {
|
||||
notebookStore.getNotebook.mockReturnValue({ id });
|
||||
notebookStore.getSources.mockReturnValue([{ id: 's1' }, { id: 's2' }]);
|
||||
notebookStore.getChunksForNotebook.mockReturnValue([{ id: 'c1', text: 'hello' }]);
|
||||
notebookStore.buildSourceGroups.mockReturnValue([
|
||||
{ docIndex: 1, name: 'Doc', chunks: [{ text: 'hello' }] },
|
||||
]);
|
||||
}
|
||||
|
||||
function stubGeneratorsOk() {
|
||||
generateStudyGuide.mockResolvedValue({ title: 'guide' });
|
||||
generateFaq.mockResolvedValue({ subject: 'topic' });
|
||||
generateExecutiveBrief.mockResolvedValue({ title: 'brief' });
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
vi.useFakeTimers();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.useRealTimers();
|
||||
});
|
||||
|
||||
describe('triggerPreGeneration guards', () => {
|
||||
it('does nothing if the notebook does not exist', () => {
|
||||
notebookStore.getNotebook.mockReturnValue(null);
|
||||
triggerPreGeneration('nb-missing');
|
||||
expect(notebookStore.getSources).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('does nothing if fewer than 2 sources', () => {
|
||||
notebookStore.getNotebook.mockReturnValue({ id: 'nb-few' });
|
||||
notebookStore.getSources.mockReturnValue([{ id: 's1' }]);
|
||||
triggerPreGeneration('nb-few');
|
||||
expect(documentCacheStore.markGenerating).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe('queuing when already generating', () => {
|
||||
it('invalidates cache and skips runPreGeneration', () => {
|
||||
setupNotebook('nb-q');
|
||||
documentCacheStore.isGenerating.mockReturnValue(true);
|
||||
|
||||
triggerPreGeneration('nb-q');
|
||||
|
||||
expect(documentCacheStore.invalidate).toHaveBeenCalledWith('nb-q');
|
||||
expect(documentCacheStore.markGenerating).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe('debounce', () => {
|
||||
it('does not run generation immediately', () => {
|
||||
setupNotebook('nb-debounce');
|
||||
documentCacheStore.isGenerating.mockReturnValue(false);
|
||||
stubGeneratorsOk();
|
||||
|
||||
triggerPreGeneration('nb-debounce');
|
||||
|
||||
expect(documentCacheStore.markGenerating).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('runs generation after the debounce delay', async () => {
|
||||
setupNotebook('nb-delay');
|
||||
documentCacheStore.isGenerating.mockReturnValue(false);
|
||||
stubGeneratorsOk();
|
||||
|
||||
triggerPreGeneration('nb-delay');
|
||||
await vi.advanceTimersByTimeAsync(5000);
|
||||
|
||||
await vi.waitFor(() => {
|
||||
expect(documentCacheStore.clearGenerating).toHaveBeenCalledWith('nb-delay');
|
||||
});
|
||||
|
||||
expect(documentCacheStore.markGenerating).toHaveBeenCalledWith('nb-delay');
|
||||
});
|
||||
|
||||
it('resets the timer on repeated calls, running generation only once', async () => {
|
||||
setupNotebook('nb-batch');
|
||||
documentCacheStore.isGenerating.mockReturnValue(false);
|
||||
stubGeneratorsOk();
|
||||
|
||||
triggerPreGeneration('nb-batch');
|
||||
await vi.advanceTimersByTimeAsync(3000);
|
||||
triggerPreGeneration('nb-batch');
|
||||
await vi.advanceTimersByTimeAsync(3000);
|
||||
triggerPreGeneration('nb-batch');
|
||||
await vi.advanceTimersByTimeAsync(5000);
|
||||
|
||||
await vi.waitFor(() => {
|
||||
expect(documentCacheStore.clearGenerating).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
expect(documentCacheStore.markGenerating).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe('runPreGeneration (via triggerPreGeneration)', () => {
|
||||
it('runs all three generators and caches results', async () => {
|
||||
setupNotebook('nb-happy');
|
||||
documentCacheStore.isGenerating.mockReturnValue(false);
|
||||
stubGeneratorsOk();
|
||||
|
||||
triggerPreGeneration('nb-happy');
|
||||
await vi.advanceTimersByTimeAsync(5000);
|
||||
|
||||
await vi.waitFor(() => {
|
||||
expect(documentCacheStore.clearGenerating).toHaveBeenCalledWith('nb-happy');
|
||||
});
|
||||
|
||||
expect(generateStudyGuide).toHaveBeenCalled();
|
||||
expect(generateFaq).toHaveBeenCalled();
|
||||
expect(generateExecutiveBrief).toHaveBeenCalled();
|
||||
expect(documentCacheStore.setCachedDocument).toHaveBeenCalledTimes(3);
|
||||
});
|
||||
|
||||
it('marks generating before starting and clears it after', async () => {
|
||||
setupNotebook('nb-flag');
|
||||
documentCacheStore.isGenerating.mockReturnValue(false);
|
||||
stubGeneratorsOk();
|
||||
|
||||
triggerPreGeneration('nb-flag');
|
||||
await vi.advanceTimersByTimeAsync(5000);
|
||||
|
||||
expect(documentCacheStore.markGenerating).toHaveBeenCalledWith('nb-flag');
|
||||
|
||||
await vi.waitFor(() => {
|
||||
expect(documentCacheStore.clearGenerating).toHaveBeenCalledWith('nb-flag');
|
||||
});
|
||||
});
|
||||
|
||||
it('still clears generating flag when a generator fails', async () => {
|
||||
setupNotebook('nb-partial');
|
||||
documentCacheStore.isGenerating.mockReturnValue(false);
|
||||
generateStudyGuide.mockRejectedValue(new Error('boom'));
|
||||
generateFaq.mockResolvedValue({ subject: 'ok' });
|
||||
generateExecutiveBrief.mockResolvedValue({ title: 'ok' });
|
||||
|
||||
triggerPreGeneration('nb-partial');
|
||||
await vi.advanceTimersByTimeAsync(5000);
|
||||
|
||||
await vi.waitFor(() => {
|
||||
expect(documentCacheStore.clearGenerating).toHaveBeenCalledWith('nb-partial');
|
||||
});
|
||||
|
||||
expect(documentCacheStore.setCachedDocument).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
});
|
||||
|
||||
describe('re-trigger for queued notebooks', () => {
|
||||
it('runs a second generation cycle after the first finishes', async () => {
|
||||
setupNotebook('nb-re');
|
||||
documentCacheStore.isGenerating
|
||||
.mockReturnValueOnce(false) // first trigger → debounced
|
||||
.mockReturnValueOnce(true); // second trigger → queues
|
||||
|
||||
let resolveFirst;
|
||||
generateStudyGuide
|
||||
.mockImplementationOnce(() => new Promise((r) => { resolveFirst = r; }))
|
||||
.mockResolvedValue({ title: 'guide' });
|
||||
generateFaq.mockResolvedValue({ subject: 'topic' });
|
||||
generateExecutiveBrief.mockResolvedValue({ title: 'brief' });
|
||||
|
||||
triggerPreGeneration('nb-re');
|
||||
await vi.advanceTimersByTimeAsync(5000);
|
||||
// First run is in-flight, blocked on generateStudyGuide
|
||||
|
||||
triggerPreGeneration('nb-re');
|
||||
// Queued via pendingReGen since isGenerating returns true
|
||||
|
||||
resolveFirst({ title: 'guide' });
|
||||
|
||||
await vi.waitFor(() => {
|
||||
expect(documentCacheStore.markGenerating).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
expect(documentCacheStore.clearGenerating).toHaveBeenCalledTimes(2);
|
||||
expect(documentCacheStore.setCachedDocument).toHaveBeenCalledTimes(6);
|
||||
});
|
||||
});
|
||||
138
server/src/services/retrievalService.js
Normal file
138
server/src/services/retrievalService.js
Normal file
@@ -0,0 +1,138 @@
|
||||
'use strict';
|
||||
|
||||
import logger from '../logger.js';
|
||||
import * as vectorStore from '../stores/vectorStore.js';
|
||||
import * as notebookStore from '../stores/notebookStore.js';
|
||||
|
||||
const VOYAGE_API_URL = 'https://api.voyageai.com/v1';
|
||||
const EMBED_MODEL = 'voyage-3';
|
||||
const RERANK_MODEL = 'rerank-2';
|
||||
|
||||
function getApiKey() {
|
||||
const key = process.env.VOYAGE_API_KEY;
|
||||
if (!key) throw new Error('VOYAGE_API_KEY is not set');
|
||||
return key;
|
||||
}
|
||||
|
||||
export async function embedTexts(texts, inputType = 'document') {
|
||||
const res = await fetch(`${VOYAGE_API_URL}/embeddings`, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
Authorization: `Bearer ${getApiKey()}`,
|
||||
},
|
||||
body: JSON.stringify({
|
||||
model: EMBED_MODEL,
|
||||
input: texts,
|
||||
input_type: inputType,
|
||||
}),
|
||||
});
|
||||
|
||||
if (!res.ok) {
|
||||
const body = await res.text();
|
||||
throw new Error(`Voyage embed failed (${res.status}): ${body}`);
|
||||
}
|
||||
|
||||
const json = await res.json();
|
||||
return json.data.map((d) => d.embedding);
|
||||
}
|
||||
|
||||
export function storeChunkEmbeddings(chunks, embeddings) {
|
||||
for (let i = 0; i < chunks.length; i++) {
|
||||
vectorStore.storeVector(chunks[i].id, embeddings[i]);
|
||||
}
|
||||
logger.debug({ count: chunks.length }, 'stored chunk embeddings');
|
||||
}
|
||||
|
||||
export function search(queryEmbedding, notebookId, topK = 10) {
|
||||
const chunks = notebookStore.getChunksForNotebook(notebookId);
|
||||
if (chunks.length === 0) return [];
|
||||
|
||||
const scored = [];
|
||||
for (const chunk of chunks) {
|
||||
const vec = vectorStore.getVector(chunk.id);
|
||||
if (!vec) continue;
|
||||
scored.push({ chunk, score: cosineSimilarity(queryEmbedding, vec) });
|
||||
}
|
||||
|
||||
scored.sort((a, b) => b.score - a.score);
|
||||
return scored.slice(0, topK);
|
||||
}
|
||||
|
||||
export async function rerank(query, chunks) {
|
||||
if (chunks.length === 0) return [];
|
||||
|
||||
const res = await fetch(`${VOYAGE_API_URL}/rerank`, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
Authorization: `Bearer ${getApiKey()}`,
|
||||
},
|
||||
body: JSON.stringify({
|
||||
model: RERANK_MODEL,
|
||||
query,
|
||||
documents: chunks.map((c) => c.text),
|
||||
}),
|
||||
});
|
||||
|
||||
if (!res.ok) {
|
||||
const body = await res.text();
|
||||
throw new Error(`Voyage rerank failed (${res.status}): ${body}`);
|
||||
}
|
||||
|
||||
const json = await res.json();
|
||||
return json.data.map((item) => ({
|
||||
chunk: chunks[item.index],
|
||||
relevanceScore: item.relevance_score,
|
||||
}));
|
||||
}
|
||||
|
||||
// Computes cosine similarity between two vectors `a` and `b`.
|
||||
// Cosine similarity measures how similar two vectors' *directions* are,
|
||||
// ignoring their magnitudes. It returns a value from -1 (opposite) to 1 (identical direction).
|
||||
// Formula: cos(θ) = (a · b) / (||a|| * ||b||)
|
||||
function cosineSimilarity(a, b) {
|
||||
// Accumulator for the dot product (a · b): the sum of element-wise products.
|
||||
// The dot product captures how much the two vectors "agree" — it grows
|
||||
// when corresponding elements point the same way and shrinks when they oppose.
|
||||
let dot = 0;
|
||||
|
||||
// Accumulator for the squared magnitude of vector `a` (sum of a[i]²).
|
||||
// After the loop, Math.sqrt(normA) will give ||a||, the Euclidean length of `a`.
|
||||
// This is needed for the denominator, which normalizes out each vector's magnitude
|
||||
// so the result reflects only directional similarity, not scale.
|
||||
let normA = 0;
|
||||
|
||||
// Same as above, but for vector `b`. Together with normA, these two values
|
||||
// will form the denominator ||a|| * ||b|| that scales the dot product into
|
||||
// the -1..1 cosine similarity range.
|
||||
let normB = 0;
|
||||
|
||||
// Walk through every dimension of the two vectors in lockstep.
|
||||
for (let i = 0; i < a.length; i++) {
|
||||
// Multiply the i-th elements of `a` and `b` and add to the running dot product.
|
||||
// Each term a[i]*b[i] contributes positively when both components share the same
|
||||
// sign (both positive or both negative) and negatively when they differ.
|
||||
dot += a[i] * b[i];
|
||||
|
||||
// Square the i-th element of `a` and accumulate it toward `a`'s squared magnitude.
|
||||
// Squaring ensures every component contributes positively regardless of sign.
|
||||
normA += a[i] * a[i];
|
||||
|
||||
// Same for vector `b` — accumulate toward `b`'s squared magnitude.
|
||||
normB += b[i] * b[i];
|
||||
}
|
||||
|
||||
// Compute the denominator: the product of the two vectors' Euclidean lengths.
|
||||
// Taking the square root of each accumulated sum-of-squares converts them from
|
||||
// squared magnitudes back into actual magnitudes (lengths).
|
||||
const denom = Math.sqrt(normA) * Math.sqrt(normB);
|
||||
|
||||
// If either vector has zero magnitude (all zeros), the denominator is 0 and
|
||||
// division would produce NaN, so we return 0 (no meaningful similarity).
|
||||
// Otherwise, dividing the dot product by the combined magnitudes yields the
|
||||
// cosine of the angle between the vectors — our similarity score.
|
||||
return denom === 0 ? 0 : dot / denom;
|
||||
}
|
||||
|
||||
export { cosineSimilarity };
|
||||
34
server/src/services/retrievalService.test.js
Normal file
34
server/src/services/retrievalService.test.js
Normal file
@@ -0,0 +1,34 @@
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import { cosineSimilarity } from './retrievalService.js';
|
||||
|
||||
describe('cosineSimilarity', () => {
|
||||
it('returns 1 for identical vectors', () => {
|
||||
expect(cosineSimilarity([1, 2, 3], [1, 2, 3])).toBeCloseTo(1.0);
|
||||
});
|
||||
|
||||
it('returns 0 for orthogonal vectors', () => {
|
||||
expect(cosineSimilarity([1, 0, 0], [0, 1, 0])).toBeCloseTo(0.0);
|
||||
});
|
||||
|
||||
it('returns -1 for opposite vectors', () => {
|
||||
expect(cosineSimilarity([1, 0], [-1, 0])).toBeCloseTo(-1.0);
|
||||
});
|
||||
|
||||
it('returns 0 when a vector is all zeros', () => {
|
||||
expect(cosineSimilarity([0, 0, 0], [1, 2, 3])).toBe(0);
|
||||
});
|
||||
|
||||
it('handles high-dimensional vectors', () => {
|
||||
const a = Array.from({ length: 1024 }, (_, i) => Math.sin(i));
|
||||
const b = Array.from({ length: 1024 }, (_, i) => Math.cos(i));
|
||||
const sim = cosineSimilarity(a, b);
|
||||
expect(sim).toBeGreaterThanOrEqual(-1);
|
||||
expect(sim).toBeLessThanOrEqual(1);
|
||||
});
|
||||
|
||||
it('is symmetric', () => {
|
||||
const a = [0.5, 0.3, 0.8];
|
||||
const b = [0.1, 0.9, 0.4];
|
||||
expect(cosineSimilarity(a, b)).toBeCloseTo(cosineSimilarity(b, a));
|
||||
});
|
||||
});
|
||||
76
server/src/services/scoringService.js
Normal file
76
server/src/services/scoringService.js
Normal file
@@ -0,0 +1,76 @@
|
||||
'use strict';
|
||||
|
||||
import { embedTexts, cosineSimilarity } from './retrievalService.js';
|
||||
import * as vectorStore from '../stores/vectorStore.js';
|
||||
import logger from '../logger.js';
|
||||
|
||||
// Calibration bounds for voyage-3 cosine similarity.
|
||||
// Empirically, near-direct-quote sentences top out around 0.68-0.72
|
||||
// raw cosine similarity; unrelated text falls below ~0.35.
|
||||
const SIM_FLOOR = 0.35;
|
||||
const SIM_CEILING = 0.65;
|
||||
const MIN_SENTENCE_LENGTH = 20;
|
||||
|
||||
function splitIntoSentences(text) {
|
||||
const cleaned = text.replace(/\[\d+\]/g, '').trim();
|
||||
const raw = cleaned.split(/(?<=[.!?])\s+/);
|
||||
return raw
|
||||
.map((s) => s.trim())
|
||||
.filter((s) => s.length >= MIN_SENTENCE_LENGTH);
|
||||
}
|
||||
|
||||
function calibrate(rawSim) {
|
||||
const scaled =
|
||||
(rawSim - SIM_FLOOR) / (SIM_CEILING - SIM_FLOOR);
|
||||
return Math.max(0, Math.min(1, scaled));
|
||||
}
|
||||
|
||||
export async function computeGroundedness(
|
||||
answerText,
|
||||
citedChunkIds
|
||||
) {
|
||||
if (!citedChunkIds || citedChunkIds.length === 0) return 0;
|
||||
|
||||
const sentences = splitIntoSentences(answerText);
|
||||
if (sentences.length === 0) return 0;
|
||||
|
||||
const sentenceEmbeddings = await embedTexts(
|
||||
sentences,
|
||||
'document'
|
||||
);
|
||||
|
||||
const chunkVectors = [];
|
||||
for (const chunkId of citedChunkIds) {
|
||||
const vec = vectorStore.getVector(chunkId);
|
||||
if (vec) {
|
||||
chunkVectors.push(vec);
|
||||
} else {
|
||||
logger.debug({ chunkId }, 'no embedding found for cited chunk');
|
||||
}
|
||||
}
|
||||
|
||||
if (chunkVectors.length === 0) return 0;
|
||||
|
||||
let totalCalibrated = 0;
|
||||
|
||||
for (let i = 0; i < sentenceEmbeddings.length; i++) {
|
||||
let bestSim = -1;
|
||||
for (const chunkVec of chunkVectors) {
|
||||
const sim = cosineSimilarity(sentenceEmbeddings[i], chunkVec);
|
||||
if (sim > bestSim) bestSim = sim;
|
||||
}
|
||||
const scored = calibrate(bestSim);
|
||||
logger.debug(
|
||||
{
|
||||
sentence: sentences[i].slice(0, 60),
|
||||
rawSim: bestSim.toFixed(3),
|
||||
calibrated: scored.toFixed(3),
|
||||
},
|
||||
'sentence groundedness'
|
||||
);
|
||||
totalCalibrated += scored;
|
||||
}
|
||||
|
||||
return totalCalibrated / sentenceEmbeddings.length;
|
||||
}
|
||||
|
||||
124
server/src/services/scoringService.test.js
Normal file
124
server/src/services/scoringService.test.js
Normal file
@@ -0,0 +1,124 @@
|
||||
import { describe, it, expect, vi, beforeEach } from 'vitest';
|
||||
|
||||
vi.mock('./retrievalService.js', () => ({
|
||||
embedTexts: vi.fn(),
|
||||
cosineSimilarity: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock('../stores/vectorStore.js', () => ({
|
||||
getVector: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock('../logger.js', () => ({
|
||||
default: { debug: vi.fn(), info: vi.fn(), warn: vi.fn() },
|
||||
}));
|
||||
|
||||
import { computeGroundedness } from './scoringService.js';
|
||||
import { embedTexts, cosineSimilarity } from './retrievalService.js';
|
||||
import * as vectorStore from '../stores/vectorStore.js';
|
||||
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
});
|
||||
|
||||
describe('computeGroundedness', () => {
|
||||
it('returns 0 when citedChunkIds is empty', async () => {
|
||||
expect(await computeGroundedness('Some answer text here.', [])).toBe(0);
|
||||
});
|
||||
|
||||
it('returns 0 when citedChunkIds is null/undefined', async () => {
|
||||
expect(await computeGroundedness('Some answer text.', null)).toBe(0);
|
||||
expect(await computeGroundedness('Some answer text.', undefined)).toBe(0);
|
||||
});
|
||||
|
||||
it('returns 0 when answer has no sentences long enough', async () => {
|
||||
expect(await computeGroundedness('Short.', ['chunk-1'])).toBe(0);
|
||||
});
|
||||
|
||||
it('returns 0 when no chunk vectors are found', async () => {
|
||||
vectorStore.getVector.mockReturnValue(null);
|
||||
const answer = 'This is a sentence that is definitely long enough to pass the filter.';
|
||||
expect(await computeGroundedness(answer, ['chunk-1'])).toBe(0);
|
||||
});
|
||||
|
||||
it('returns 1.0 when all sentences have similarity at or above the ceiling', async () => {
|
||||
const fakeVec = new Float32Array([1, 0, 0]);
|
||||
vectorStore.getVector.mockReturnValue(fakeVec);
|
||||
embedTexts.mockResolvedValue([new Float32Array([1, 0, 0])]);
|
||||
cosineSimilarity.mockReturnValue(0.70);
|
||||
|
||||
const answer = 'This sentence is long enough to be included in the analysis.';
|
||||
const score = await computeGroundedness(answer, ['chunk-1']);
|
||||
expect(score).toBe(1);
|
||||
});
|
||||
|
||||
it('returns 0 when all sentences have similarity at or below the floor', async () => {
|
||||
const fakeVec = new Float32Array([1, 0, 0]);
|
||||
vectorStore.getVector.mockReturnValue(fakeVec);
|
||||
embedTexts.mockResolvedValue([new Float32Array([0, 1, 0])]);
|
||||
cosineSimilarity.mockReturnValue(0.20);
|
||||
|
||||
const answer = 'This sentence is long enough to be included in the analysis.';
|
||||
const score = await computeGroundedness(answer, ['chunk-1']);
|
||||
expect(score).toBe(0);
|
||||
});
|
||||
|
||||
it('returns a value between 0 and 1 for mid-range similarity', async () => {
|
||||
const fakeVec = new Float32Array([1, 0, 0]);
|
||||
vectorStore.getVector.mockReturnValue(fakeVec);
|
||||
embedTexts.mockResolvedValue([new Float32Array([1, 0, 0])]);
|
||||
cosineSimilarity.mockReturnValue(0.50);
|
||||
|
||||
const answer = 'This sentence is long enough to be included in the analysis.';
|
||||
const score = await computeGroundedness(answer, ['chunk-1']);
|
||||
expect(score).toBeGreaterThan(0);
|
||||
expect(score).toBeLessThan(1);
|
||||
expect(score).toBeCloseTo(0.5, 1);
|
||||
});
|
||||
|
||||
it('averages across multiple sentences', async () => {
|
||||
const fakeVec = new Float32Array([1, 0, 0]);
|
||||
vectorStore.getVector.mockReturnValue(fakeVec);
|
||||
embedTexts.mockResolvedValue([
|
||||
new Float32Array([1, 0, 0]),
|
||||
new Float32Array([0, 1, 0]),
|
||||
]);
|
||||
cosineSimilarity
|
||||
.mockReturnValueOnce(0.65)
|
||||
.mockReturnValueOnce(0.35);
|
||||
|
||||
const answer =
|
||||
'This is the first sentence that is long enough. This is the second sentence that is also long enough.';
|
||||
const score = await computeGroundedness(answer, ['chunk-1']);
|
||||
expect(score).toBeCloseTo(0.5, 1);
|
||||
});
|
||||
|
||||
it('picks the best similarity across multiple chunk vectors', async () => {
|
||||
vectorStore.getVector
|
||||
.mockReturnValueOnce(new Float32Array([1, 0, 0]))
|
||||
.mockReturnValueOnce(new Float32Array([0, 1, 0]));
|
||||
embedTexts.mockResolvedValue([new Float32Array([1, 0, 0])]);
|
||||
cosineSimilarity
|
||||
.mockReturnValueOnce(0.40)
|
||||
.mockReturnValueOnce(0.65);
|
||||
|
||||
const answer = 'This sentence is long enough to be included in the analysis.';
|
||||
const score = await computeGroundedness(answer, ['chunk-1', 'chunk-2']);
|
||||
expect(score).toBe(1);
|
||||
});
|
||||
|
||||
it('strips citation markers [1] [2] before splitting sentences', async () => {
|
||||
const fakeVec = new Float32Array([1, 0, 0]);
|
||||
vectorStore.getVector.mockReturnValue(fakeVec);
|
||||
embedTexts.mockResolvedValue([new Float32Array([1, 0, 0])]);
|
||||
cosineSimilarity.mockReturnValue(0.50);
|
||||
|
||||
const answer = 'The sky is blue according to science [1]. Water covers most of the earth [2].';
|
||||
await computeGroundedness(answer, ['chunk-1']);
|
||||
|
||||
const embeddedTexts = embedTexts.mock.calls[0][0];
|
||||
for (const text of embeddedTexts) {
|
||||
expect(text).not.toMatch(/\[\d+\]/);
|
||||
}
|
||||
});
|
||||
});
|
||||
275
server/src/services/sourceService.js
Normal file
275
server/src/services/sourceService.js
Normal file
@@ -0,0 +1,275 @@
|
||||
'use strict';
|
||||
|
||||
import pdfParse from 'pdf-parse';
|
||||
import mammoth from 'mammoth';
|
||||
import * as cheerio from 'cheerio';
|
||||
import OpenAI from 'openai';
|
||||
import { execFile } from 'node:child_process';
|
||||
import { promisify } from 'node:util';
|
||||
import { mkdtempSync, readdirSync, readFileSync, rmSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { v4 as uuidv4 } from 'uuid';
|
||||
import dns from 'node:dns/promises';
|
||||
import { isIP } from 'node:net';
|
||||
import logger from '../logger.js';
|
||||
|
||||
const execFileAsync = promisify(execFile);
|
||||
|
||||
const CHARS_PER_CHUNK = 2000;
|
||||
const OVERLAP_CHARS = 200;
|
||||
|
||||
const AUDIO_MIMETYPES = new Set([
|
||||
'audio/mpeg',
|
||||
'audio/mp3',
|
||||
'audio/wav',
|
||||
'audio/x-wav',
|
||||
'audio/mp4',
|
||||
'audio/x-m4a',
|
||||
'audio/ogg',
|
||||
'audio/flac',
|
||||
'audio/webm',
|
||||
]);
|
||||
|
||||
export function isAudioMimetype(mimetype) {
|
||||
return AUDIO_MIMETYPES.has(mimetype);
|
||||
}
|
||||
|
||||
export async function parseFile(buffer, mimetype, filename) {
|
||||
if (mimetype === 'application/pdf') {
|
||||
const result = await pdfParse(buffer);
|
||||
return result.text;
|
||||
}
|
||||
|
||||
if (
|
||||
mimetype === 'application/vnd.openxmlformats-officedocument.wordprocessingml.document' ||
|
||||
filename?.toLowerCase().endsWith('.docx')
|
||||
) {
|
||||
const result = await mammoth.extractRawText({ buffer });
|
||||
return result.value;
|
||||
}
|
||||
|
||||
if (isAudioMimetype(mimetype)) {
|
||||
return transcribeAudio(buffer, filename);
|
||||
}
|
||||
|
||||
return buffer.toString('utf-8');
|
||||
}
|
||||
|
||||
const WHISPER_MAX_BYTES = 25 * 1024 * 1024;
|
||||
|
||||
async function transcribeAudio(buffer, filename) {
|
||||
const key = process.env.OPENAI_API_KEY;
|
||||
if (!key) throw new Error('OPENAI_API_KEY is required for audio transcription');
|
||||
|
||||
if (buffer.length > WHISPER_MAX_BYTES) {
|
||||
throw new Error('Audio file exceeds 25 MB Whisper API limit');
|
||||
}
|
||||
|
||||
const client = new OpenAI({ apiKey: key });
|
||||
|
||||
const ext = filename?.split('.').pop()?.toLowerCase() || 'mp3';
|
||||
const file = new File([buffer], `audio.${ext}`, { type: `audio/${ext}` });
|
||||
|
||||
logger.info({ filename, bytes: buffer.length }, 'transcribing audio with Whisper');
|
||||
|
||||
const response = await client.audio.transcriptions.create({
|
||||
model: 'whisper-1',
|
||||
file,
|
||||
});
|
||||
|
||||
return response.text;
|
||||
}
|
||||
|
||||
const BLOCKED_IP_RANGES = [
|
||||
/^127\./,
|
||||
/^10\./,
|
||||
/^172\.(1[6-9]|2\d|3[01])\./,
|
||||
/^192\.168\./,
|
||||
/^169\.254\./,
|
||||
/^0\./,
|
||||
/^::1$/,
|
||||
/^fc00:/i,
|
||||
/^fe80:/i,
|
||||
/^fd/i,
|
||||
];
|
||||
|
||||
function isBlockedIp(ip) {
|
||||
return BLOCKED_IP_RANGES.some((re) => re.test(ip));
|
||||
}
|
||||
|
||||
async function validateUrl(raw) {
|
||||
let parsed;
|
||||
try {
|
||||
parsed = new URL(raw);
|
||||
} catch {
|
||||
throw new Error('Invalid URL');
|
||||
}
|
||||
|
||||
if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') {
|
||||
throw new Error('Only http and https URLs are allowed');
|
||||
}
|
||||
|
||||
const hostname = parsed.hostname;
|
||||
|
||||
if (isIP(hostname)) {
|
||||
if (isBlockedIp(hostname)) {
|
||||
throw new Error('URLs pointing to private/internal networks are not allowed');
|
||||
}
|
||||
} else {
|
||||
const { address } = await dns.lookup(hostname);
|
||||
if (isBlockedIp(address)) {
|
||||
throw new Error('URLs pointing to private/internal networks are not allowed');
|
||||
}
|
||||
}
|
||||
|
||||
return parsed.href;
|
||||
}
|
||||
|
||||
export async function parseUrl(url) {
|
||||
const safeUrl = await validateUrl(url);
|
||||
logger.info({ url: safeUrl }, 'fetching URL content');
|
||||
|
||||
const res = await fetch(safeUrl, {
|
||||
headers: {
|
||||
'User-Agent': 'Mozilla/5.0 (compatible; NotebookClone/1.0)',
|
||||
'Accept': 'text/html,application/xhtml+xml,text/plain',
|
||||
},
|
||||
signal: AbortSignal.timeout(15000),
|
||||
});
|
||||
|
||||
if (!res.ok) {
|
||||
throw new Error(`Failed to fetch URL: ${res.status} ${res.statusText}`);
|
||||
}
|
||||
|
||||
const contentType = res.headers.get('content-type') || '';
|
||||
const body = await res.text();
|
||||
|
||||
if (contentType.includes('text/plain')) {
|
||||
return body;
|
||||
}
|
||||
|
||||
const $ = cheerio.load(body);
|
||||
$('script, style, nav, footer, header, iframe, noscript').remove();
|
||||
|
||||
const article = $('article').length ? $('article') : $('main').length ? $('main') : $('body');
|
||||
const text = article.text().replace(/\s+/g, ' ').trim();
|
||||
|
||||
if (!text) {
|
||||
throw new Error('Could not extract meaningful text from URL');
|
||||
}
|
||||
|
||||
return text;
|
||||
}
|
||||
|
||||
const YT_URL_RE = /(?:youtube\.com\/watch\?v=|youtu\.be\/|youtube\.com\/embed\/)([a-zA-Z0-9_-]{11})/;
|
||||
|
||||
export function isYoutubeUrl(url) {
|
||||
return YT_URL_RE.test(url);
|
||||
}
|
||||
|
||||
export function extractVideoId(url) {
|
||||
const match = url.match(YT_URL_RE);
|
||||
return match ? match[1] : null;
|
||||
}
|
||||
|
||||
export async function parseYoutubeUrl(url) {
|
||||
const videoId = extractVideoId(url);
|
||||
if (!videoId) {
|
||||
throw new Error('Could not extract YouTube video ID from URL');
|
||||
}
|
||||
|
||||
logger.info({ videoId }, 'fetching YouTube subtitles via yt-dlp');
|
||||
|
||||
const tmpDir = mkdtempSync(join(tmpdir(), 'yt-'));
|
||||
try {
|
||||
const { stderr } = await execFileAsync('yt-dlp', [
|
||||
'--skip-download',
|
||||
'--write-auto-sub',
|
||||
'--write-sub',
|
||||
'--sub-lang', 'en',
|
||||
'--sub-format', 'json3',
|
||||
'--no-warnings',
|
||||
'-o', join(tmpDir, '%(id)s'),
|
||||
`https://www.youtube.com/watch?v=${videoId}`,
|
||||
], { timeout: 20000 });
|
||||
|
||||
if (stderr) {
|
||||
logger.warn({ videoId, stderr: stderr.slice(0, 500) }, 'yt-dlp stderr output');
|
||||
}
|
||||
|
||||
const files = readdirSync(tmpDir);
|
||||
const subFile = files.find((f) => f.endsWith('.json3'));
|
||||
if (!subFile) {
|
||||
throw new Error('No English subtitles available for this YouTube video');
|
||||
}
|
||||
|
||||
const data = JSON.parse(readFileSync(join(tmpDir, subFile), 'utf-8'));
|
||||
const events = (data.events || []).filter((e) => e.segs);
|
||||
const text = events
|
||||
.map((e) => e.segs.map((s) => s.utf8 || '').join(''))
|
||||
.join(' ')
|
||||
.replace(/\n/g, ' ')
|
||||
.replace(/\s+/g, ' ')
|
||||
.trim();
|
||||
|
||||
if (!text) {
|
||||
throw new Error('Subtitles were empty');
|
||||
}
|
||||
|
||||
logger.info({ videoId, textLength: text.length }, 'YouTube subtitles extracted');
|
||||
return text;
|
||||
} catch (err) {
|
||||
if (err.code === 'ENOENT') {
|
||||
throw new Error('YouTube support requires yt-dlp to be installed (brew install yt-dlp)');
|
||||
}
|
||||
if (err.killed) {
|
||||
throw new Error('yt-dlp timed out fetching subtitles');
|
||||
}
|
||||
if (err.message.startsWith('No English') || err.message.startsWith('Subtitles')) {
|
||||
throw err;
|
||||
}
|
||||
const detail = err.stderr?.slice(0, 300) || err.message;
|
||||
throw new Error(`Failed to extract YouTube subtitles: ${detail}`);
|
||||
} finally {
|
||||
rmSync(tmpDir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
export function chunkText(text, sourceId) {
|
||||
const cleaned = text.replace(/\r\n/g, '\n').trim();
|
||||
if (!cleaned) return [];
|
||||
|
||||
const chunks = [];
|
||||
let start = 0;
|
||||
let index = 0;
|
||||
|
||||
while (start < cleaned.length) {
|
||||
const end = Math.min(start + CHARS_PER_CHUNK, cleaned.length);
|
||||
let sliceEnd = end;
|
||||
|
||||
if (end < cleaned.length) {
|
||||
const lastNewline = cleaned.lastIndexOf('\n', end);
|
||||
const lastPeriod = cleaned.lastIndexOf('. ', end);
|
||||
const breakAt = Math.max(lastNewline, lastPeriod);
|
||||
if (breakAt > start + CHARS_PER_CHUNK * 0.5) {
|
||||
sliceEnd = breakAt + 1;
|
||||
}
|
||||
}
|
||||
|
||||
chunks.push({
|
||||
id: uuidv4(),
|
||||
text: cleaned.slice(start, sliceEnd),
|
||||
sourceId,
|
||||
index,
|
||||
});
|
||||
|
||||
index++;
|
||||
if (sliceEnd >= cleaned.length) break;
|
||||
start = sliceEnd - OVERLAP_CHARS;
|
||||
if (start < 0) start = 0;
|
||||
if (start >= sliceEnd) break;
|
||||
}
|
||||
|
||||
return chunks;
|
||||
}
|
||||
158
server/src/services/sourceService.test.js
Normal file
158
server/src/services/sourceService.test.js
Normal file
@@ -0,0 +1,158 @@
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import { chunkText, parseFile, isYoutubeUrl, extractVideoId, parseUrl } from './sourceService.js';
|
||||
|
||||
describe('chunkText', () => {
|
||||
it('returns a single chunk for short text', () => {
|
||||
const chunks = chunkText('Hello world.', 'src-1');
|
||||
expect(chunks).toHaveLength(1);
|
||||
expect(chunks[0].text).toBe('Hello world.');
|
||||
expect(chunks[0].sourceId).toBe('src-1');
|
||||
expect(chunks[0].index).toBe(0);
|
||||
expect(chunks[0].id).toBeTruthy();
|
||||
});
|
||||
|
||||
it('returns an empty array for empty text', () => {
|
||||
expect(chunkText('', 'src-1')).toEqual([]);
|
||||
expect(chunkText(' ', 'src-1')).toEqual([]);
|
||||
});
|
||||
|
||||
it('splits long text into multiple chunks', () => {
|
||||
const text = 'A'.repeat(5000);
|
||||
const chunks = chunkText(text, 'src-2');
|
||||
expect(chunks.length).toBeGreaterThan(1);
|
||||
});
|
||||
|
||||
it('assigns sequential indices', () => {
|
||||
const text = 'word '.repeat(2000);
|
||||
const chunks = chunkText(text, 'src-3');
|
||||
chunks.forEach((c, i) => {
|
||||
expect(c.index).toBe(i);
|
||||
});
|
||||
});
|
||||
|
||||
it('generates unique ids per chunk', () => {
|
||||
const text = 'word '.repeat(2000);
|
||||
const chunks = chunkText(text, 'src-4');
|
||||
const ids = new Set(chunks.map((c) => c.id));
|
||||
expect(ids.size).toBe(chunks.length);
|
||||
});
|
||||
|
||||
it('includes overlap between consecutive chunks', () => {
|
||||
const text = 'word '.repeat(2000);
|
||||
const chunks = chunkText(text, 'src-5');
|
||||
if (chunks.length >= 2) {
|
||||
const end0 = chunks[0].text.slice(-100);
|
||||
const start1 = chunks[1].text.slice(0, 100);
|
||||
const hasOverlap = chunks[1].text.includes(end0.slice(-50));
|
||||
expect(hasOverlap).toBe(true);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('parseFile', () => {
|
||||
it('passes through plain text from a buffer', async () => {
|
||||
const buf = Buffer.from('Hello, plain text.');
|
||||
const result = await parseFile(buf, 'text/plain');
|
||||
expect(result).toBe('Hello, plain text.');
|
||||
});
|
||||
|
||||
it('passes through markdown from a buffer', async () => {
|
||||
const buf = Buffer.from('# Heading\n\nSome markdown.');
|
||||
const result = await parseFile(buf, 'text/markdown');
|
||||
expect(result).toBe('# Heading\n\nSome markdown.');
|
||||
});
|
||||
});
|
||||
|
||||
describe('isYoutubeUrl', () => {
|
||||
it('recognizes standard watch URLs', () => {
|
||||
expect(isYoutubeUrl('https://www.youtube.com/watch?v=dQw4w9WgXcQ')).toBe(true);
|
||||
});
|
||||
|
||||
it('recognizes short youtu.be URLs', () => {
|
||||
expect(isYoutubeUrl('https://youtu.be/dQw4w9WgXcQ')).toBe(true);
|
||||
});
|
||||
|
||||
it('recognizes embed URLs', () => {
|
||||
expect(isYoutubeUrl('https://www.youtube.com/embed/dQw4w9WgXcQ')).toBe(true);
|
||||
});
|
||||
|
||||
it('rejects non-YouTube URLs', () => {
|
||||
expect(isYoutubeUrl('https://example.com/watch?v=abc')).toBe(false);
|
||||
});
|
||||
|
||||
it('rejects URLs with wrong video ID length', () => {
|
||||
expect(isYoutubeUrl('https://youtube.com/watch?v=short')).toBe(false);
|
||||
});
|
||||
|
||||
it('rejects plain text that is not a URL', () => {
|
||||
expect(isYoutubeUrl('not a url')).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('extractVideoId', () => {
|
||||
it('extracts ID from standard watch URL', () => {
|
||||
expect(extractVideoId('https://www.youtube.com/watch?v=dQw4w9WgXcQ')).toBe('dQw4w9WgXcQ');
|
||||
});
|
||||
|
||||
it('extracts ID from short URL', () => {
|
||||
expect(extractVideoId('https://youtu.be/dQw4w9WgXcQ')).toBe('dQw4w9WgXcQ');
|
||||
});
|
||||
|
||||
it('extracts ID from embed URL', () => {
|
||||
expect(extractVideoId('https://www.youtube.com/embed/dQw4w9WgXcQ')).toBe('dQw4w9WgXcQ');
|
||||
});
|
||||
|
||||
it('extracts ID with extra query params', () => {
|
||||
expect(extractVideoId('https://www.youtube.com/watch?v=dQw4w9WgXcQ&t=120')).toBe('dQw4w9WgXcQ');
|
||||
});
|
||||
|
||||
it('returns null for non-YouTube URL', () => {
|
||||
expect(extractVideoId('https://example.com/page')).toBeNull();
|
||||
});
|
||||
|
||||
it('returns null for empty string', () => {
|
||||
expect(extractVideoId('')).toBeNull();
|
||||
});
|
||||
|
||||
it('handles IDs with hyphens and underscores', () => {
|
||||
expect(extractVideoId('https://youtu.be/Ab-_cd12EfG')).toBe('Ab-_cd12EfG');
|
||||
});
|
||||
});
|
||||
|
||||
describe('parseUrl – validation & SSRF prevention', () => {
|
||||
it('rejects non-URL strings', async () => {
|
||||
await expect(parseUrl('not a url')).rejects.toThrow('Invalid URL');
|
||||
});
|
||||
|
||||
it('rejects ftp:// scheme', async () => {
|
||||
await expect(parseUrl('ftp://example.com/file.txt')).rejects.toThrow('Only http and https');
|
||||
});
|
||||
|
||||
it('rejects file:// scheme', async () => {
|
||||
await expect(parseUrl('file:///etc/passwd')).rejects.toThrow('Only http and https');
|
||||
});
|
||||
|
||||
it('rejects data: scheme', async () => {
|
||||
await expect(parseUrl('data:text/html,<h1>hi</h1>')).rejects.toThrow('Only http and https');
|
||||
});
|
||||
|
||||
it('rejects localhost IP (127.0.0.1)', async () => {
|
||||
await expect(parseUrl('http://127.0.0.1/admin')).rejects.toThrow('private/internal');
|
||||
});
|
||||
|
||||
it('rejects private 10.x range', async () => {
|
||||
await expect(parseUrl('http://10.0.0.1/secrets')).rejects.toThrow('private/internal');
|
||||
});
|
||||
|
||||
it('rejects private 192.168.x range', async () => {
|
||||
await expect(parseUrl('http://192.168.1.1/')).rejects.toThrow('private/internal');
|
||||
});
|
||||
|
||||
it('rejects link-local 169.254.x range', async () => {
|
||||
await expect(parseUrl('http://169.254.169.254/metadata')).rejects.toThrow('private/internal');
|
||||
});
|
||||
|
||||
it('rejects 0.0.0.0', async () => {
|
||||
await expect(parseUrl('http://0.0.0.0/')).rejects.toThrow('private/internal');
|
||||
});
|
||||
});
|
||||
52
server/src/stores/documentCacheStore.js
Normal file
52
server/src/stores/documentCacheStore.js
Normal file
@@ -0,0 +1,52 @@
|
||||
'use strict';
|
||||
|
||||
const cache = new Map();
|
||||
|
||||
function buildSourceHash(sources) {
|
||||
return sources
|
||||
.map((s) => s.id)
|
||||
.sort()
|
||||
.join('|');
|
||||
}
|
||||
|
||||
export function getCachedDocument(notebookId, type) {
|
||||
const entry = cache.get(`${notebookId}:${type}`);
|
||||
if (!entry) return null;
|
||||
return entry;
|
||||
}
|
||||
|
||||
export function setCachedDocument(notebookId, type, document, sources) {
|
||||
const key = `${notebookId}:${type}`;
|
||||
cache.set(key, {
|
||||
document,
|
||||
sourceHash: buildSourceHash(sources),
|
||||
createdAt: Date.now(),
|
||||
});
|
||||
}
|
||||
|
||||
export function isFresh(notebookId, type, currentSources) {
|
||||
const entry = cache.get(`${notebookId}:${type}`);
|
||||
if (!entry) return false;
|
||||
return entry.sourceHash === buildSourceHash(currentSources);
|
||||
}
|
||||
|
||||
export function invalidate(notebookId) {
|
||||
for (const key of cache.keys()) {
|
||||
if (key.startsWith(`${notebookId}:`)) {
|
||||
cache.delete(key);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export function markGenerating(notebookId) {
|
||||
const key = `${notebookId}:__generating`;
|
||||
cache.set(key, true);
|
||||
}
|
||||
|
||||
export function clearGenerating(notebookId) {
|
||||
cache.delete(`${notebookId}:__generating`);
|
||||
}
|
||||
|
||||
export function isGenerating(notebookId) {
|
||||
return cache.get(`${notebookId}:__generating`) === true;
|
||||
}
|
||||
96
server/src/stores/documentCacheStore.test.js
Normal file
96
server/src/stores/documentCacheStore.test.js
Normal file
@@ -0,0 +1,96 @@
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import {
|
||||
getCachedDocument,
|
||||
setCachedDocument,
|
||||
isFresh,
|
||||
invalidate,
|
||||
markGenerating,
|
||||
isGenerating,
|
||||
clearGenerating,
|
||||
} from './documentCacheStore.js';
|
||||
|
||||
describe('documentCacheStore', () => {
|
||||
|
||||
describe('getCachedDocument / setCachedDocument', () => {
|
||||
it('returns null for an uncached entry', () => {
|
||||
expect(getCachedDocument('miss-1', 'faq')).toBeNull();
|
||||
});
|
||||
|
||||
it('round-trips a cached document', () => {
|
||||
const doc = { title: 'Guide' };
|
||||
const sources = [{ id: 's1' }, { id: 's2' }];
|
||||
setCachedDocument('rt-1', 'study-guide', doc, sources);
|
||||
|
||||
const entry = getCachedDocument('rt-1', 'study-guide');
|
||||
expect(entry.document).toEqual(doc);
|
||||
expect(entry.sourceHash).toBe('s1|s2');
|
||||
expect(entry.createdAt).toBeTypeOf('number');
|
||||
});
|
||||
|
||||
it('overwrites an existing entry for the same key', () => {
|
||||
setCachedDocument('ow-1', 'faq', { v: 1 }, [{ id: 'a' }]);
|
||||
setCachedDocument('ow-1', 'faq', { v: 2 }, [{ id: 'b' }]);
|
||||
expect(getCachedDocument('ow-1', 'faq').document).toEqual({ v: 2 });
|
||||
});
|
||||
|
||||
it('keeps entries for different types separate', () => {
|
||||
setCachedDocument('sep-1', 'faq', { kind: 'faq' }, [{ id: 'x' }]);
|
||||
setCachedDocument('sep-1', 'study-guide', { kind: 'sg' }, [{ id: 'x' }]);
|
||||
expect(getCachedDocument('sep-1', 'faq').document.kind).toBe('faq');
|
||||
expect(getCachedDocument('sep-1', 'study-guide').document.kind).toBe('sg');
|
||||
});
|
||||
});
|
||||
|
||||
describe('isFresh', () => {
|
||||
it('returns false when nothing is cached', () => {
|
||||
expect(isFresh('fresh-miss', 'faq', [{ id: 'a' }])).toBe(false);
|
||||
});
|
||||
|
||||
it('returns true when source IDs match regardless of order', () => {
|
||||
setCachedDocument('fresh-match', 'faq', {}, [{ id: 'b' }, { id: 'a' }]);
|
||||
expect(isFresh('fresh-match', 'faq', [{ id: 'a' }, { id: 'b' }])).toBe(true);
|
||||
});
|
||||
|
||||
it('returns false when source IDs differ', () => {
|
||||
setCachedDocument('fresh-diff', 'faq', {}, [{ id: 'a' }]);
|
||||
expect(isFresh('fresh-diff', 'faq', [{ id: 'a' }, { id: 'b' }])).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('invalidate', () => {
|
||||
it('removes all entries for a notebook', () => {
|
||||
setCachedDocument('inv-1', 'faq', { a: 1 }, [{ id: 'x' }]);
|
||||
setCachedDocument('inv-1', 'study-guide', { b: 2 }, [{ id: 'x' }]);
|
||||
invalidate('inv-1');
|
||||
expect(getCachedDocument('inv-1', 'faq')).toBeNull();
|
||||
expect(getCachedDocument('inv-1', 'study-guide')).toBeNull();
|
||||
});
|
||||
|
||||
it('does not affect other notebooks', () => {
|
||||
setCachedDocument('inv-a', 'faq', { a: 1 }, [{ id: 'x' }]);
|
||||
setCachedDocument('inv-b', 'faq', { b: 2 }, [{ id: 'x' }]);
|
||||
invalidate('inv-a');
|
||||
expect(getCachedDocument('inv-a', 'faq')).toBeNull();
|
||||
expect(getCachedDocument('inv-b', 'faq').document).toEqual({ b: 2 });
|
||||
});
|
||||
});
|
||||
|
||||
describe('generating flag', () => {
|
||||
it('defaults to not generating', () => {
|
||||
expect(isGenerating('gen-default')).toBe(false);
|
||||
});
|
||||
|
||||
it('can be set and cleared', () => {
|
||||
markGenerating('gen-cycle');
|
||||
expect(isGenerating('gen-cycle')).toBe(true);
|
||||
clearGenerating('gen-cycle');
|
||||
expect(isGenerating('gen-cycle')).toBe(false);
|
||||
});
|
||||
|
||||
it('is cleared when the notebook is invalidated', () => {
|
||||
markGenerating('gen-inv');
|
||||
invalidate('gen-inv');
|
||||
expect(isGenerating('gen-inv')).toBe(false);
|
||||
});
|
||||
});
|
||||
});
|
||||
88
server/src/stores/notebookStore.js
Normal file
88
server/src/stores/notebookStore.js
Normal file
@@ -0,0 +1,88 @@
|
||||
'use strict';
|
||||
|
||||
export class NotebookStore {
|
||||
constructor() {
|
||||
this.notebooks = new Map();
|
||||
}
|
||||
|
||||
getAllNotebooks() {
|
||||
return Array.from(this.notebooks.values()).map(({ chunks, ...meta }) => meta);
|
||||
}
|
||||
|
||||
getNotebook(id) {
|
||||
return this.notebooks.get(id) || null;
|
||||
}
|
||||
|
||||
createNotebook(notebook) {
|
||||
const record = { ...notebook, sources: [], chunks: [] };
|
||||
this.notebooks.set(record.id, record);
|
||||
return { id: record.id, name: record.name, createdAt: record.createdAt };
|
||||
}
|
||||
|
||||
deleteNotebook(id) {
|
||||
return this.notebooks.delete(id);
|
||||
}
|
||||
|
||||
addSource(notebookId, source) {
|
||||
const notebook = this.notebooks.get(notebookId);
|
||||
if (!notebook) return null;
|
||||
notebook.sources.push(source);
|
||||
return source;
|
||||
}
|
||||
|
||||
getSources(notebookId) {
|
||||
const notebook = this.notebooks.get(notebookId);
|
||||
if (!notebook) return [];
|
||||
return notebook.sources;
|
||||
}
|
||||
|
||||
addChunksToNotebook(notebookId, chunks) {
|
||||
const notebook = this.notebooks.get(notebookId);
|
||||
if (!notebook) return null;
|
||||
notebook.chunks = notebook.chunks.concat(chunks);
|
||||
return notebook;
|
||||
}
|
||||
|
||||
getChunksForNotebook(notebookId) {
|
||||
const notebook = this.notebooks.get(notebookId);
|
||||
if (!notebook) return [];
|
||||
return notebook.chunks;
|
||||
}
|
||||
|
||||
buildSourceGroups(notebookId, chunks) {
|
||||
const sources = this.getSources(notebookId);
|
||||
const sourceIndexMap = new Map();
|
||||
sources.forEach((src, i) => {
|
||||
sourceIndexMap.set(src.id, i + 1);
|
||||
});
|
||||
|
||||
const groupMap = new Map();
|
||||
for (const chunk of chunks) {
|
||||
const docIndex = sourceIndexMap.get(chunk.sourceId) || 0;
|
||||
if (!groupMap.has(chunk.sourceId)) {
|
||||
const src = sources.find((s) => s.id === chunk.sourceId);
|
||||
groupMap.set(chunk.sourceId, {
|
||||
docIndex,
|
||||
sourceId: chunk.sourceId,
|
||||
name: src?.name || 'Unknown',
|
||||
chunks: [],
|
||||
});
|
||||
}
|
||||
groupMap.get(chunk.sourceId).chunks.push(chunk);
|
||||
}
|
||||
|
||||
return Array.from(groupMap.values()).sort((a, b) => a.docIndex - b.docIndex);
|
||||
}
|
||||
}
|
||||
|
||||
const defaultStore = new NotebookStore();
|
||||
|
||||
export const getAllNotebooks = defaultStore.getAllNotebooks.bind(defaultStore);
|
||||
export const getNotebook = defaultStore.getNotebook.bind(defaultStore);
|
||||
export const createNotebook = defaultStore.createNotebook.bind(defaultStore);
|
||||
export const deleteNotebook = defaultStore.deleteNotebook.bind(defaultStore);
|
||||
export const addSource = defaultStore.addSource.bind(defaultStore);
|
||||
export const getSources = defaultStore.getSources.bind(defaultStore);
|
||||
export const addChunksToNotebook = defaultStore.addChunksToNotebook.bind(defaultStore);
|
||||
export const getChunksForNotebook = defaultStore.getChunksForNotebook.bind(defaultStore);
|
||||
export const buildSourceGroups = defaultStore.buildSourceGroups.bind(defaultStore);
|
||||
65
server/src/stores/notebookStore.test.js
Normal file
65
server/src/stores/notebookStore.test.js
Normal file
@@ -0,0 +1,65 @@
|
||||
import { describe, it, expect, beforeEach } from 'vitest';
|
||||
import { NotebookStore } from './notebookStore.js';
|
||||
|
||||
describe('notebookStore', () => {
|
||||
let store;
|
||||
|
||||
beforeEach(() => {
|
||||
store = new NotebookStore();
|
||||
});
|
||||
|
||||
it('creates and retrieves a notebook', () => {
|
||||
const nb = store.createNotebook({ id: 'nb-1', name: 'Test', createdAt: '2026-01-01' });
|
||||
expect(nb.id).toBe('nb-1');
|
||||
expect(nb.name).toBe('Test');
|
||||
|
||||
const found = store.getNotebook('nb-1');
|
||||
expect(found).not.toBeNull();
|
||||
expect(found.name).toBe('Test');
|
||||
});
|
||||
|
||||
it('lists all notebooks without chunks', () => {
|
||||
store.createNotebook({ id: 'nb-1', name: 'A', createdAt: '2026-01-01' });
|
||||
store.createNotebook({ id: 'nb-2', name: 'B', createdAt: '2026-01-02' });
|
||||
const all = store.getAllNotebooks();
|
||||
expect(all).toHaveLength(2);
|
||||
expect(all[0]).not.toHaveProperty('chunks');
|
||||
});
|
||||
|
||||
it('deletes a notebook', () => {
|
||||
store.createNotebook({ id: 'nb-1', name: 'A', createdAt: '2026-01-01' });
|
||||
expect(store.deleteNotebook('nb-1')).toBe(true);
|
||||
expect(store.getNotebook('nb-1')).toBeNull();
|
||||
});
|
||||
|
||||
it('returns false when deleting a non-existent notebook', () => {
|
||||
expect(store.deleteNotebook('nope')).toBe(false);
|
||||
});
|
||||
|
||||
it('adds and retrieves sources', () => {
|
||||
store.createNotebook({ id: 'nb-1', name: 'A', createdAt: '2026-01-01' });
|
||||
store.addSource('nb-1', { id: 's-1', name: 'doc.pdf' });
|
||||
const sources = store.getSources('nb-1');
|
||||
expect(sources).toHaveLength(1);
|
||||
expect(sources[0].name).toBe('doc.pdf');
|
||||
});
|
||||
|
||||
it('returns empty sources for non-existent notebook', () => {
|
||||
expect(store.getSources('nope')).toEqual([]);
|
||||
});
|
||||
|
||||
it('adds and retrieves chunks', () => {
|
||||
store.createNotebook({ id: 'nb-1', name: 'A', createdAt: '2026-01-01' });
|
||||
store.addChunksToNotebook('nb-1', [
|
||||
{ id: 'c-1', text: 'chunk one' },
|
||||
{ id: 'c-2', text: 'chunk two' },
|
||||
]);
|
||||
const chunks = store.getChunksForNotebook('nb-1');
|
||||
expect(chunks).toHaveLength(2);
|
||||
expect(chunks[0].text).toBe('chunk one');
|
||||
});
|
||||
|
||||
it('returns empty chunks for non-existent notebook', () => {
|
||||
expect(store.getChunksForNotebook('nope')).toEqual([]);
|
||||
});
|
||||
});
|
||||
21
server/src/stores/vectorStore.js
Normal file
21
server/src/stores/vectorStore.js
Normal file
@@ -0,0 +1,21 @@
|
||||
'use strict';
|
||||
|
||||
const vectors = new Map();
|
||||
|
||||
export function storeVector(chunkId, embedding) {
|
||||
vectors.set(chunkId, new Float32Array(embedding));
|
||||
}
|
||||
|
||||
export function getVector(chunkId) {
|
||||
return vectors.get(chunkId) || null;
|
||||
}
|
||||
|
||||
export function getAllVectors() {
|
||||
return vectors;
|
||||
}
|
||||
|
||||
export function deleteVectorsForChunks(chunkIds) {
|
||||
for (const id of chunkIds) {
|
||||
vectors.delete(id);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user