first commit of v.02 application

server/src/routes/citationDetail.js

@@ -0,0 +1,34 @@
+'use strict';
+
+import { Router } from 'express';
+import logger from '../logger.js';
+import { generateCitationDetail } from '../services/generationService.js';
+
+const router = Router();
+
+router.post('/', async (req, res, next) => {
+  try {
+    const { chunkTexts, sourceName, answer, citationIndex } = req.body;
+
+    if (!chunkTexts?.length || !answer || citationIndex == null) {
+      return res.status(400).json({
+        error: 'chunkTexts, answer, and citationIndex are required',
+      });
+    }
+
+    logger.info({ citationIndex, sourceName }, 'citation detail requested');
+
+    const detail = await generateCitationDetail({
+      chunkTexts,
+      sourceName: sourceName || 'Unknown',
+      answer,
+      citationIndex,
+    });
+
+    res.json(detail);
+  } catch (err) {
+    next(err);
+  }
+});
+
+export default router;

server/src/routes/documents.js

@@ -0,0 +1,63 @@
+'use strict';
+
+import { Router } from 'express';
+import logger from '../logger.js';
+import * as notebookStore from '../stores/notebookStore.js';
+import * as documentCacheStore from '../stores/documentCacheStore.js';
+import {
+  generateStudyGuide,
+  generateFaq,
+  generateExecutiveBrief,
+} from '../services/documentService.js';
+
+const GENERATORS = {
+  'study-guide': generateStudyGuide,
+  'faq': generateFaq,
+  'executive-brief': generateExecutiveBrief,
+};
+
+const router = Router();
+
+router.post('/generate', async (req, res, next) => {
+  try {
+    const { notebookId, type } = req.body;
+
+    if (!notebookId || !type) {
+      return res.status(400).json({ error: 'notebookId and type are required' });
+    }
+
+    const generator = GENERATORS[type];
+    if (!generator) {
+      return res.status(400).json({
+        error: `Invalid type. Must be one of: ${Object.keys(GENERATORS).join(', ')}`,
+      });
+    }
+
+    const chunks = notebookStore.getChunksForNotebook(notebookId);
+    if (chunks.length === 0) {
+      return res.status(422).json({ error: 'No source material available in this notebook' });
+    }
+
+    const sources = notebookStore.getSources(notebookId);
+    const cached = documentCacheStore.getCachedDocument(notebookId, type);
+    if (cached && documentCacheStore.isFresh(notebookId, type, sources)) {
+      logger.info({ notebookId, type }, 'serving cached document');
+      return res.json({ type, document: cached.document });
+    }
+
+    const sourceGroups = notebookStore.buildSourceGroups(notebookId, chunks);
+
+    logger.info({ notebookId, type, sourceCount: sourceGroups.length, chunkCount: chunks.length }, 'document generation started');
+
+    const document = await generator(sourceGroups);
+
+    documentCacheStore.setCachedDocument(notebookId, type, document, sources);
+    logger.info({ notebookId, type }, 'document generation complete');
+
+    res.json({ type, document });
+  } catch (err) {
+    next(err);
+  }
+});
+
+export default router;

server/src/routes/notebooks.js

@@ -0,0 +1,42 @@
+'use strict';
+
+import { Router } from 'express';
+import { v4 as uuidv4 } from 'uuid';
+import * as notebookStore from '../stores/notebookStore.js';
+import { deleteVectorsForChunks } from '../stores/vectorStore.js';
+
+const router = Router();
+
+router.get('/', (req, res) => {
+  res.json(notebookStore.getAllNotebooks());
+});
+
+router.post('/', (req, res) => {
+  const { name } = req.body;
+  if (!name || !name.trim()) {
+    return res.status(400).json({ error: 'name is required' });
+  }
+
+  const notebook = notebookStore.createNotebook({
+    id: uuidv4(),
+    name: name.trim(),
+    createdAt: new Date().toISOString(),
+  });
+
+  res.status(201).json(notebook);
+});
+
+router.delete('/:id', (req, res) => {
+  const chunks = notebookStore.getChunksForNotebook(req.params.id);
+  const deleted = notebookStore.deleteNotebook(req.params.id);
+  if (!deleted) {
+    return res.status(404).json({ error: 'notebook not found' });
+  }
+  if (chunks.length) {
+    deleteVectorsForChunks(chunks.map((c) => c.id));
+  }
+  res.json({ ok: true });
+});
+
+export default router;
+

server/src/routes/query.js

@@ -0,0 +1,88 @@
+'use strict';
+
+import { Router } from 'express';
+import logger from '../logger.js';
+import { embedTexts, search, rerank } from '../services/retrievalService.js';
+import { generate } from '../services/generationService.js';
+import { computeGroundedness } from '../services/scoringService.js';
+import * as notebookStore from '../stores/notebookStore.js';
+
+const TOP_K_SEARCH = 20;
+const TOP_K_RERANK = 5;
+
+const router = Router();
+
+router.post('/', async (req, res, next) => {
+  try {
+    const { notebookId, question } = req.body;
+    if (!notebookId || !question) {
+      return res.status(400).json({ error: 'notebookId and question are required' });
+    }
+
+    logger.info({ notebookId, question }, 'query received');
+
+    const [queryEmbedding] = await embedTexts([question], 'query');
+
+    const searchResults = search(queryEmbedding, notebookId, TOP_K_SEARCH);
+    if (searchResults.length === 0) {
+      return res.json({
+        answer: 'No sources found for this notebook. Upload some documents first.',
+        citations: [],
+        groundednessScore: null,
+        followUpQuestions: [],
+      });
+    }
+
+    const candidateChunks = searchResults.map((r) => r.chunk);
+    const reranked = await rerank(question, candidateChunks);
+    const topChunks = reranked.slice(0, TOP_K_RERANK).map((r) => r.chunk);
+
+    logger.debug({
+      searchHits: searchResults.length,
+      rerankedTop: topChunks.length,
+    }, 'retrieval complete');
+
+    const sourceGroups = notebookStore.buildSourceGroups(notebookId, topChunks);
+
+    const { answer, citedSourceIndices, followUpQuestions } = await generate(
+      question,
+      sourceGroups,
+    );
+
+    const citedChunkIds = [];
+    for (const idx of citedSourceIndices) {
+      const group = sourceGroups.find((g) => g.docIndex === idx);
+      if (group) {
+        citedChunkIds.push(...group.chunks.map((c) => c.id));
+      }
+    }
+
+    const groundednessScore = await computeGroundedness(answer, citedChunkIds);
+
+    logger.info({ groundednessScore: groundednessScore.toFixed(3) }, 'query complete');
+
+    const citations = citedSourceIndices.map((idx) => {
+      const group = sourceGroups.find((g) => g.docIndex === idx);
+      return group
+        ? {
+          sourceIndex: idx,
+          sourceId: group.sourceId,
+          name: group.name,
+          chunkTexts: group.chunks.map((c) => c.text),
+        }
+        : { sourceIndex: idx, sourceId: null, name: null, chunkTexts: [] };
+    });
+
+    res.json({
+      answer,
+      citations,
+      groundednessScore,
+      followUpQuestions,
+    });
+  } catch (err) {
+    next(err);
+  }
+});
+
+export default router;
+

server/src/routes/sources.js

@@ -0,0 +1,138 @@
+'use strict';
+
+import { Router } from 'express';
+import multer from 'multer';
+import { v4 as uuidv4 } from 'uuid';
+import logger from '../logger.js';
+import * as notebookStore from '../stores/notebookStore.js';
+import {
+  parseFile,
+  chunkText,
+  parseUrl,
+  parseYoutubeUrl,
+  isYoutubeUrl,
+} from '../services/sourceService.js';
+import { embedTexts, storeChunkEmbeddings } from '../services/retrievalService.js';
+import { triggerPreGeneration } from '../services/preGenerationService.js';
+import {
+  requireNotebookId,
+  requireNotebook,
+  requireFile,
+  requireUrl,
+} from '../middleware/validation.js';
+
+const upload = multer({ storage: multer.memoryStorage(), limits: { fileSize: 50 * 1024 * 1024 } });
+const router = Router();
+
+const TIMESTAMP_RE = /\s+\d{1,2}\.\d{2}\.\d{2}\s*[\u2018\u2019''\s]?\s*[AP]M(?=\.\w+$)/i;
+function cleanFilename(name) {
+  return name.replace(TIMESTAMP_RE, '');
+}
+
+router.get('/', requireNotebookId, (req, res) => {
+  res.json(notebookStore.getSources(req.notebookId));
+});
+
+
+function multerUpload(req, res, next) {
+  upload.single('file')(req, res, (err) => {
+    if (err) {
+      if (err.code === 'LIMIT_FILE_SIZE') {
+        return res.status(413).json({ error: 'File too large. Maximum size is 50 MB.' });
+      }
+      return next(err);
+    }
+    next();
+  });
+}
+
+router.post(
+  '/',
+  multerUpload,
+  requireNotebookId,
+  requireNotebook,
+  requireFile,
+  async (req, res, next) => {
+    try {
+      const sourceId = uuidv4();
+      const rawName = Buffer.from(req.file.originalname, 'latin1').toString('utf-8');
+      const displayName = cleanFilename(rawName);
+      const text = await parseFile(req.file.buffer, req.file.mimetype, displayName);
+      const chunks = chunkText(text, sourceId);
+
+      logger.info({
+        sourceId,
+        filename: displayName,
+        chunkCount: chunks.length,
+      }, 'parsed and chunked source');
+
+      notebookStore.addChunksToNotebook(req.notebookId, chunks);
+
+      const texts = chunks.map((c) => c.text);
+      const embeddings = await embedTexts(texts, 'document');
+      storeChunkEmbeddings(chunks, embeddings);
+
+      const source = notebookStore.addSource(req.notebookId, {
+        id: sourceId,
+        name: displayName,
+        mimetype: req.file.mimetype,
+        chunkCount: chunks.length,
+        uploadedAt: new Date().toISOString(),
+      });
+
+      triggerPreGeneration(req.notebookId);
+
+      res.status(201).json(source);
+    } catch (err) {
+      next(err);
+    }
+  },
+);
+
+router.post(
+  '/url',
+  requireNotebookId,
+  requireNotebook,
+  requireUrl,
+  async (req, res, next) => {
+    try {
+      const { url } = req.body;
+      const sourceId = uuidv4();
+      const isYT = isYoutubeUrl(url);
+      const displayName = isYT
+        ? `YouTube: ${url}`
+        : url.replace(/^https?:\/\//, '').slice(0, 60);
+
+      logger.info({ sourceId, url, isYT }, 'processing URL source');
+
+      const text = isYT ? await parseYoutubeUrl(url) : await parseUrl(url);
+      const chunks = chunkText(text, sourceId);
+
+      if (chunks.length === 0) {
+        return res.status(422).json({ error: 'No usable text could be extracted from the URL' });
+      }
+
+      notebookStore.addChunksToNotebook(req.notebookId, chunks);
+
+      const texts = chunks.map((c) => c.text);
+      const embeddings = await embedTexts(texts, 'document');
+      storeChunkEmbeddings(chunks, embeddings);
+
+      const source = notebookStore.addSource(req.notebookId, {
+        id: sourceId,
+        name: displayName,
+        mimetype: isYT ? 'video/youtube' : 'text/html',
+        chunkCount: chunks.length,
+        uploadedAt: new Date().toISOString(),
+      });
+
+      triggerPreGeneration(req.notebookId);
+
+      res.status(201).json(source);
+    } catch (err) {
+      next(err);
+    }
+  },
+);
+
+export default router;