Syntax changes in front and back end dirs. These do not affect functionality and are purely intended to better capture and describe the app's functionality and purpose

This commit is contained in:
KS Jannette
2026-08-03 00:07:58 -04:00
parent 4a5e5d6612
commit 0352bdf516
42 changed files with 812 additions and 812 deletions

View File

@@ -1,11 +1,11 @@
{
"name": "notebook-clone-server",
"name": "source-sentinel-server",
"version": "0.1.0",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "notebook-clone-server",
"name": "source-sentinel-server",
"version": "0.1.0",
"dependencies": {
"@anthropic-ai/sdk": "^0.78.0",

View File

@@ -1,5 +1,5 @@
{
"name": "notebook-clone-server",
"name": "source-sentinel-server",
"version": "0.1.0",
"private": true,
"engines": {

View File

@@ -4,7 +4,7 @@ import 'dotenv/config';
import express, { type Request, type Response, type NextFunction } from 'express';
import cors from 'cors';
import logger from './logger.js';
import notebookRoutes from './routes/notebooks.js';
import sourceCorpusRoutes from './routes/sourceCorpus.js';
import sourceRoutes from './routes/sources.js';
import queryRoutes from './routes/query.js';
import citationDetailRoutes from './routes/citationDetail.js';
@@ -39,13 +39,13 @@ app.use((req: Request, res: Response, next: NextFunction) => {
app.get('/.well-known/healthcheck', (_req: Request, res: Response) => {
res.json({
service: 'notebook-clone',
service: 'source-sentinel',
status: 'ok',
uptime: process.uptime(),
});
});
app.use('/api/notebooks', notebookRoutes);
app.use('/api/source-corpus', sourceCorpusRoutes);
app.use('/api/sources', sourceRoutes);
app.use('/api/query', queryRoutes);
app.use('/api/citation-detail', citationDetailRoutes);

View File

@@ -1,31 +1,31 @@
'use strict';
import type { Request, Response, NextFunction } from 'express';
import * as notebookStore from '../stores/notebookStore.js';
import type { Notebook } from '../stores/notebookStore.js';
import * as sourceCorpusStore from '../stores/sourceCorpusStore.js';
import type { SourceCorpus } from '../stores/sourceCorpusStore.js';
export interface NotebookRequest extends Request {
notebookId?: string;
notebook?: Notebook;
export interface SourceCorpusRequest extends Request {
sourceCorpusId?: string;
sourceCorpus?: SourceCorpus;
}
export function requireNotebookId(req: NotebookRequest, res: Response, next: NextFunction): void {
const notebookId = (req.body?.notebookId ?? req.query?.notebookId) as string | undefined;
if (!notebookId) {
res.status(400).json({ error: 'notebookId is required' });
export function requireSourceCorpusId(req: SourceCorpusRequest, res: Response, next: NextFunction): void {
const sourceCorpusId = (req.body?.sourceCorpusId ?? req.query?.sourceCorpusId) as string | undefined;
if (!sourceCorpusId) {
res.status(400).json({ error: 'sourceCorpusId is required' });
return;
}
req.notebookId = notebookId;
req.sourceCorpusId = sourceCorpusId;
next();
}
export function requireNotebook(req: NotebookRequest, res: Response, next: NextFunction): void {
const notebook = notebookStore.getNotebook(req.notebookId!);
if (!notebook) {
res.status(404).json({ error: 'notebook not found' });
export function requireSourceCorpus(req: SourceCorpusRequest, res: Response, next: NextFunction): void {
const sourceCorpus = sourceCorpusStore.getSourceCorpus(req.sourceCorpusId!);
if (!sourceCorpus) {
res.status(404).json({ error: 'source corpus not found' });
return;
}
req.notebook = notebook;
req.sourceCorpus = sourceCorpus;
next();
}

View File

@@ -2,8 +2,8 @@
import { Router, type Request, type Response, type NextFunction } from 'express';
import logger from '../logger.js';
import * as notebookStore from '../stores/notebookStore.js';
import type { TextChunk, Source, SourceGroup } from '../stores/notebookStore.js';
import * as sourceCorpusStore from '../stores/sourceCorpusStore.js';
import type { TextChunk, Source, SourceGroup } from '../stores/sourceCorpusStore.js';
import * as documentCacheStore from '../stores/documentCacheStore.js';
import type { CacheEntry } from '../stores/documentCacheStore.js';
import {
@@ -27,16 +27,16 @@ const GENERATORS: Record<DocumentType, GeneratorFn> = {
const router = Router();
interface GenerateBody {
notebookId?: string;
sourceCorpusId?: string;
type?: string;
}
router.post('/generate', async (req: Request<unknown, unknown, GenerateBody>, res: Response, next: NextFunction) => {
try {
const { notebookId, type } = req.body;
const { sourceCorpusId, type } = req.body;
if (!notebookId || !type) {
res.status(400).json({ error: 'notebookId and type are required' });
if (!sourceCorpusId || !type) {
res.status(400).json({ error: 'sourceCorpusId and type are required' });
return;
}
@@ -48,31 +48,31 @@ router.post('/generate', async (req: Request<unknown, unknown, GenerateBody>, re
return;
}
const chunks: TextChunk[] = notebookStore.getChunksForNotebook(notebookId);
const chunks: TextChunk[] = sourceCorpusStore.getChunksForSourceCorpus(sourceCorpusId);
if (chunks.length === 0) {
res.status(422).json({ error: 'No source material available in this notebook' });
res.status(422).json({ error: 'No source material available in this source corpus' });
return;
}
const sources: Source[] = notebookStore.getSources(notebookId);
const cached: CacheEntry | null = documentCacheStore.getCachedDocument(notebookId, type);
if (cached && documentCacheStore.isFresh(notebookId, type, sources)) {
logger.info({ notebookId, type }, 'serving cached document');
const sources: Source[] = sourceCorpusStore.getSources(sourceCorpusId);
const cached: CacheEntry | null = documentCacheStore.getCachedDocument(sourceCorpusId, type);
if (cached && documentCacheStore.isFresh(sourceCorpusId, type, sources)) {
logger.info({ sourceCorpusId, type }, 'serving cached document');
res.json({ type, document: cached.document });
return;
}
const sourceGroups: SourceGroup[] = notebookStore.buildSourceGroups(notebookId, chunks);
const sourceGroups: SourceGroup[] = sourceCorpusStore.buildSourceGroups(sourceCorpusId, chunks);
logger.info(
{ notebookId, type, sourceCount: sourceGroups.length, chunkCount: chunks.length },
{ sourceCorpusId, type, sourceCount: sourceGroups.length, chunkCount: chunks.length },
'document generation started'
);
const document: StudyGuide | Faq | ExecutiveBrief = await generator(sourceGroups);
documentCacheStore.setCachedDocument(notebookId, type, document, sources);
logger.info({ notebookId, type }, 'document generation complete');
documentCacheStore.setCachedDocument(sourceCorpusId, type, document, sources);
logger.info({ sourceCorpusId, type }, 'document generation complete');
res.json({ type, document });
} catch (err) {

View File

@@ -7,8 +7,8 @@ import type { ScoredChunk, RankedChunk, TextChunk } from '../services/retrievalS
import { generate } from '../services/generationService.js';
import type { GenerationResult } from '../services/generationService.js';
import { computeGroundedness } from '../services/scoringService.js';
import * as notebookStore from '../stores/notebookStore.js';
import type { SourceGroup } from '../stores/notebookStore.js';
import * as sourceCorpusStore from '../stores/sourceCorpusStore.js';
import type { SourceGroup } from '../stores/sourceCorpusStore.js';
const TOP_K_SEARCH = 20;
const TOP_K_RERANK = 5;
@@ -16,7 +16,7 @@ const TOP_K_RERANK = 5;
const router = Router();
interface QueryBody {
notebookId?: string;
sourceCorpusId?: string;
question?: string;
}
@@ -36,20 +36,20 @@ interface QueryResponse {
router.post('/', async (req: Request<unknown, unknown, QueryBody>, res: Response, next: NextFunction) => {
try {
const { notebookId, question } = req.body;
if (!notebookId || !question) {
res.status(400).json({ error: 'notebookId and question are required' });
const { sourceCorpusId, question } = req.body;
if (!sourceCorpusId || !question) {
res.status(400).json({ error: 'sourceCorpusId and question are required' });
return;
}
logger.info({ notebookId, question }, 'query received');
logger.info({ sourceCorpusId, question }, 'query received');
const [queryEmbedding]: number[][] = await embedTexts([question], 'query');
const searchResults: ScoredChunk[] = search(queryEmbedding, notebookId, TOP_K_SEARCH);
const searchResults: ScoredChunk[] = search(queryEmbedding, sourceCorpusId, TOP_K_SEARCH);
if (searchResults.length === 0) {
res.json({
answer: 'No sources found for this notebook. Upload some documents first.',
answer: 'No sources found for this source corpus. Upload some documents first.',
citations: [],
groundednessScore: null,
followUpQuestions: [],
@@ -69,7 +69,7 @@ router.post('/', async (req: Request<unknown, unknown, QueryBody>, res: Response
'retrieval complete'
);
const sourceGroups: SourceGroup[] = notebookStore.buildSourceGroups(notebookId, topChunks);
const sourceGroups: SourceGroup[] = sourceCorpusStore.buildSourceGroups(sourceCorpusId, topChunks);
const { answer, citedSourceIndices, followUpQuestions }: GenerationResult = await generate(question, sourceGroups);

View File

@@ -2,33 +2,33 @@
import { Router, type Request, type Response } from 'express';
import { v4 as uuidv4 } from 'uuid';
import * as notebookStore from '../stores/notebookStore.js';
import * as sourceCorpusStore from '../stores/sourceCorpusStore.js';
import { deleteVectorsForChunks } from '../stores/vectorStore.js';
const router = Router();
router.get('/', (_req: Request, res: Response) => {
res.json(notebookStore.getAllNotebooks());
res.json(sourceCorpusStore.getAllSourceCorpora());
});
interface CreateNotebookBody {
interface CreateSourceCorpusBody {
name?: string;
}
router.post('/', (req: Request<unknown, unknown, CreateNotebookBody>, res: Response) => {
router.post('/', (req: Request<unknown, unknown, CreateSourceCorpusBody>, res: Response) => {
const { name } = req.body;
if (!name || !name.trim()) {
res.status(400).json({ error: 'name is required' });
return;
}
const notebook = notebookStore.createNotebook({
const sourceCorpus = sourceCorpusStore.createSourceCorpus({
id: uuidv4(),
name: name.trim(),
createdAt: new Date().toISOString(),
});
res.status(201).json(notebook);
res.status(201).json(sourceCorpus);
});
interface DeleteParams {
@@ -36,10 +36,10 @@ interface DeleteParams {
}
router.delete('/:id', (req: Request<DeleteParams>, res: Response) => {
const chunks = notebookStore.getChunksForNotebook(req.params.id);
const deleted = notebookStore.deleteNotebook(req.params.id);
const chunks = sourceCorpusStore.getChunksForSourceCorpus(req.params.id);
const deleted = sourceCorpusStore.deleteSourceCorpus(req.params.id);
if (!deleted) {
res.status(404).json({ error: 'notebook not found' });
res.status(404).json({ error: 'source corpus not found' });
return;
}
if (chunks.length) {

View File

@@ -4,18 +4,18 @@ import { Router, type Request, type Response, type NextFunction } from 'express'
import multer from 'multer';
import { v4 as uuidv4 } from 'uuid';
import logger from '../logger.js';
import * as notebookStore from '../stores/notebookStore.js';
import type { Source } from '../stores/notebookStore.js';
import * as sourceCorpusStore from '../stores/sourceCorpusStore.js';
import type { Source } from '../stores/sourceCorpusStore.js';
import { parseFile, chunkText, parseUrl, parseYoutubeUrl, isYoutubeUrl } from '../services/sourceService.js';
import type { TextChunk } from '../services/sourceService.js';
import { embedTexts, storeChunkEmbeddings } from '../services/retrievalService.js';
import { triggerPreGeneration } from '../services/preGenerationService.js';
import {
requireNotebookId,
requireNotebook,
requireSourceCorpusId,
requireSourceCorpus,
requireFile,
requireUrl,
type NotebookRequest,
type SourceCorpusRequest,
} from '../middleware/validation.js';
const upload = multer({ storage: multer.memoryStorage(), limits: { fileSize: 50 * 1024 * 1024 } });
@@ -26,8 +26,8 @@ function cleanFilename(name: string): string {
return name.replace(TIMESTAMP_RE, '');
}
router.get('/', requireNotebookId, (req: NotebookRequest, res: Response) => {
res.json(notebookStore.getSources(req.notebookId!));
router.get('/', requireSourceCorpusId, (req: SourceCorpusRequest, res: Response) => {
res.json(sourceCorpusStore.getSources(req.sourceCorpusId!));
});
function multerUpload(req: Request, res: Response, next: NextFunction): void {
@@ -47,10 +47,10 @@ function multerUpload(req: Request, res: Response, next: NextFunction): void {
router.post(
'/',
multerUpload,
requireNotebookId,
requireNotebook,
requireSourceCorpusId,
requireSourceCorpus,
requireFile,
async (req: NotebookRequest, res: Response, next: NextFunction) => {
async (req: SourceCorpusRequest, res: Response, next: NextFunction) => {
try {
const sourceId: string = uuidv4();
const rawName: string = Buffer.from(req.file!.originalname, 'latin1').toString('utf-8');
@@ -67,13 +67,13 @@ router.post(
'parsed and chunked source'
);
notebookStore.addChunksToNotebook(req.notebookId!, chunks);
sourceCorpusStore.addChunksToSourceCorpus(req.sourceCorpusId!, chunks);
const texts: string[] = chunks.map((c) => c.text);
const embeddings: number[][] = await embedTexts(texts, 'document');
storeChunkEmbeddings(chunks, embeddings);
const source: Source | null = notebookStore.addSource(req.notebookId!, {
const source: Source | null = sourceCorpusStore.addSource(req.sourceCorpusId!, {
id: sourceId,
name: displayName,
mimetype: req.file!.mimetype,
@@ -81,7 +81,7 @@ router.post(
uploadedAt: new Date().toISOString(),
});
triggerPreGeneration(req.notebookId!);
triggerPreGeneration(req.sourceCorpusId!);
res.status(201).json(source);
} catch (err) {
@@ -96,10 +96,10 @@ interface UrlBody {
router.post(
'/url',
requireNotebookId,
requireNotebook,
requireSourceCorpusId,
requireSourceCorpus,
requireUrl,
async (req: NotebookRequest & Request<unknown, unknown, UrlBody>, res: Response, next: NextFunction) => {
async (req: SourceCorpusRequest & Request<unknown, unknown, UrlBody>, res: Response, next: NextFunction) => {
try {
const { url } = req.body;
const sourceId: string = uuidv4();
@@ -116,13 +116,13 @@ router.post(
return;
}
notebookStore.addChunksToNotebook(req.notebookId!, chunks);
sourceCorpusStore.addChunksToSourceCorpus(req.sourceCorpusId!, chunks);
const texts: string[] = chunks.map((c) => c.text);
const embeddings: number[][] = await embedTexts(texts, 'document');
storeChunkEmbeddings(chunks, embeddings);
const source: Source | null = notebookStore.addSource(req.notebookId!, {
const source: Source | null = sourceCorpusStore.addSource(req.sourceCorpusId!, {
id: sourceId,
name: displayName,
mimetype: isYT ? 'video/youtube' : 'text/html',
@@ -130,7 +130,7 @@ router.post(
uploadedAt: new Date().toISOString(),
});
triggerPreGeneration(req.notebookId!);
triggerPreGeneration(req.sourceCorpusId!);
res.status(201).json(source);
} catch (err) {

View File

@@ -1,9 +1,9 @@
import { describe, it, expect, vi, beforeEach, afterEach, type Mock } from 'vitest';
vi.mock('../stores/notebookStore.js', () => ({
getNotebook: vi.fn(),
vi.mock('../stores/sourceCorpusStore.js', () => ({
getSourceCorpus: vi.fn(),
getSources: vi.fn(),
getChunksForNotebook: vi.fn(),
getChunksForSourceCorpus: vi.fn(),
buildSourceGroups: vi.fn(),
}));
@@ -26,14 +26,14 @@ vi.mock('../logger.js', () => ({
}));
import { triggerPreGeneration } from './preGenerationService.js';
import * as notebookStore from '../stores/notebookStore.js';
import * as sourceCorpusStore from '../stores/sourceCorpusStore.js';
import * as documentCacheStore from '../stores/documentCacheStore.js';
import { generateStudyGuide, generateFaq, generateExecutiveBrief } from './documentService.js';
const mockedGetNotebook = notebookStore.getNotebook as Mock;
const mockedGetSources = notebookStore.getSources as Mock;
const mockedGetChunksForNotebook = notebookStore.getChunksForNotebook as Mock;
const mockedBuildSourceGroups = notebookStore.buildSourceGroups as Mock;
const mockedGetSourceCorpus = sourceCorpusStore.getSourceCorpus as Mock;
const mockedGetSources = sourceCorpusStore.getSources as Mock;
const mockedGetChunksForSourceCorpus = sourceCorpusStore.getChunksForSourceCorpus as Mock;
const mockedBuildSourceGroups = sourceCorpusStore.buildSourceGroups as Mock;
const mockedIsGenerating = documentCacheStore.isGenerating as Mock;
const mockedInvalidate = documentCacheStore.invalidate as Mock;
const mockedMarkGenerating = documentCacheStore.markGenerating as Mock;
@@ -43,10 +43,10 @@ const mockedGenerateStudyGuide = generateStudyGuide as Mock;
const mockedGenerateFaq = generateFaq as Mock;
const mockedGenerateExecutiveBrief = generateExecutiveBrief as Mock;
function setupNotebook(id: string): void {
mockedGetNotebook.mockReturnValue({ id });
function setupSourceCorpus(id: string): void {
mockedGetSourceCorpus.mockReturnValue({ id });
mockedGetSources.mockReturnValue([{ id: 's1' }, { id: 's2' }]);
mockedGetChunksForNotebook.mockReturnValue([{ id: 'c1', text: 'hello' }]);
mockedGetChunksForSourceCorpus.mockReturnValue([{ id: 'c1', text: 'hello' }]);
mockedBuildSourceGroups.mockReturnValue([{ docIndex: 1, name: 'Doc', chunks: [{ text: 'hello' }] }]);
}
@@ -66,14 +66,14 @@ afterEach(() => {
});
describe('triggerPreGeneration guards', () => {
it('does nothing if the notebook does not exist', () => {
mockedGetNotebook.mockReturnValue(null);
it('does nothing if the sourceCorpus does not exist', () => {
mockedGetSourceCorpus.mockReturnValue(null);
triggerPreGeneration('nb-missing');
expect(mockedGetSources).not.toHaveBeenCalled();
});
it('does nothing if fewer than 2 sources', () => {
mockedGetNotebook.mockReturnValue({ id: 'nb-few' });
mockedGetSourceCorpus.mockReturnValue({ id: 'nb-few' });
mockedGetSources.mockReturnValue([{ id: 's1' }]);
triggerPreGeneration('nb-few');
expect(mockedMarkGenerating).not.toHaveBeenCalled();
@@ -82,7 +82,7 @@ describe('triggerPreGeneration guards', () => {
describe('queuing when already generating', () => {
it('invalidates cache and skips runPreGeneration', () => {
setupNotebook('nb-q');
setupSourceCorpus('nb-q');
mockedIsGenerating.mockReturnValue(true);
triggerPreGeneration('nb-q');
@@ -94,7 +94,7 @@ describe('queuing when already generating', () => {
describe('debounce', () => {
it('does not run generation immediately', () => {
setupNotebook('nb-debounce');
setupSourceCorpus('nb-debounce');
mockedIsGenerating.mockReturnValue(false);
stubGeneratorsOk();
@@ -104,7 +104,7 @@ describe('debounce', () => {
});
it('runs generation after the debounce delay', async () => {
setupNotebook('nb-delay');
setupSourceCorpus('nb-delay');
mockedIsGenerating.mockReturnValue(false);
stubGeneratorsOk();
@@ -119,7 +119,7 @@ describe('debounce', () => {
});
it('resets the timer on repeated calls, running generation only once', async () => {
setupNotebook('nb-batch');
setupSourceCorpus('nb-batch');
mockedIsGenerating.mockReturnValue(false);
stubGeneratorsOk();
@@ -140,7 +140,7 @@ describe('debounce', () => {
describe('runPreGeneration (via triggerPreGeneration)', () => {
it('runs all three generators and caches results', async () => {
setupNotebook('nb-happy');
setupSourceCorpus('nb-happy');
mockedIsGenerating.mockReturnValue(false);
stubGeneratorsOk();
@@ -158,7 +158,7 @@ describe('runPreGeneration (via triggerPreGeneration)', () => {
});
it('marks generating before starting and clears it after', async () => {
setupNotebook('nb-flag');
setupSourceCorpus('nb-flag');
mockedIsGenerating.mockReturnValue(false);
stubGeneratorsOk();
@@ -173,7 +173,7 @@ describe('runPreGeneration (via triggerPreGeneration)', () => {
});
it('still clears generating flag when a generator fails', async () => {
setupNotebook('nb-partial');
setupSourceCorpus('nb-partial');
mockedIsGenerating.mockReturnValue(false);
mockedGenerateStudyGuide.mockRejectedValue(new Error('boom'));
mockedGenerateFaq.mockResolvedValue({ subject: 'ok' });
@@ -190,9 +190,9 @@ describe('runPreGeneration (via triggerPreGeneration)', () => {
});
});
describe('re-trigger for queued notebooks', () => {
describe('re-trigger for queued sourceCorpora', () => {
it('runs a second generation cycle after the first finishes', async () => {
setupNotebook('nb-re');
setupSourceCorpus('nb-re');
mockedIsGenerating.mockReturnValueOnce(false).mockReturnValueOnce(true);
let resolveFirst: (value: { title: string }) => void;

View File

@@ -1,7 +1,7 @@
'use strict';
import logger from '../logger.js';
import * as notebookStore from '../stores/notebookStore.js';
import * as sourceCorpusStore from '../stores/sourceCorpusStore.js';
import * as documentCacheStore from '../stores/documentCacheStore.js';
import {
generateStudyGuide,
@@ -29,45 +29,45 @@ const DEBOUNCE_MS = 5000;
const pendingReGen = new Set<string>();
const debounceTimers = new Map<string, ReturnType<typeof setTimeout>>();
export function triggerPreGeneration(notebookId: string): void {
const notebook = notebookStore.getNotebook(notebookId);
if (!notebook) return;
export function triggerPreGeneration(sourceCorpusId: string): void {
const sourceCorpus = sourceCorpusStore.getSourceCorpus(sourceCorpusId);
if (!sourceCorpus) return;
const sources = notebookStore.getSources(notebookId);
const sources = sourceCorpusStore.getSources(sourceCorpusId);
if (sources.length < MIN_SOURCES) return;
if (documentCacheStore.isGenerating(notebookId)) {
pendingReGen.add(notebookId);
documentCacheStore.invalidate(notebookId);
logger.debug({ notebookId }, 'pre-generation in progress, queued re-generation');
if (documentCacheStore.isGenerating(sourceCorpusId)) {
pendingReGen.add(sourceCorpusId);
documentCacheStore.invalidate(sourceCorpusId);
logger.debug({ sourceCorpusId }, 'pre-generation in progress, queued re-generation');
return;
}
const existingTimer = debounceTimers.get(notebookId);
const existingTimer = debounceTimers.get(sourceCorpusId);
if (existingTimer !== undefined) {
clearTimeout(existingTimer);
}
debounceTimers.set(
notebookId,
sourceCorpusId,
setTimeout(() => {
debounceTimers.delete(notebookId);
runPreGeneration(notebookId);
debounceTimers.delete(sourceCorpusId);
runPreGeneration(sourceCorpusId);
}, DEBOUNCE_MS)
);
logger.debug({ notebookId, debounceMs: DEBOUNCE_MS }, 'pre-generation debounced');
logger.debug({ sourceCorpusId, debounceMs: DEBOUNCE_MS }, 'pre-generation debounced');
}
function runPreGeneration(notebookId: string): void {
function runPreGeneration(sourceCorpusId: string): void {
try {
const sources = notebookStore.getSources(notebookId);
const chunks = notebookStore.getChunksForNotebook(notebookId);
const sourceGroups = notebookStore.buildSourceGroups(notebookId, chunks) as SourceGroup[];
const sources = sourceCorpusStore.getSources(sourceCorpusId);
const chunks = sourceCorpusStore.getChunksForSourceCorpus(sourceCorpusId);
const sourceGroups = sourceCorpusStore.buildSourceGroups(sourceCorpusId, chunks) as SourceGroup[];
documentCacheStore.invalidate(notebookId);
documentCacheStore.markGenerating(notebookId);
documentCacheStore.invalidate(sourceCorpusId);
documentCacheStore.markGenerating(sourceCorpusId);
logger.info(
{ notebookId, sourceCount: sources.length, chunkCount: chunks.length },
{ sourceCorpusId, sourceCount: sources.length, chunkCount: chunks.length },
'background pre-generation started'
);
@@ -75,31 +75,31 @@ function runPreGeneration(notebookId: string): void {
async ([type, generator]) => {
try {
const document = await generator(sourceGroups);
documentCacheStore.setCachedDocument(notebookId, type, document, sources);
logger.info({ notebookId, type }, 'background pre-generation complete for type');
documentCacheStore.setCachedDocument(sourceCorpusId, type, document, sources);
logger.info({ sourceCorpusId, type }, 'background pre-generation complete for type');
} catch (err) {
const error = err as Error;
logger.error({ notebookId, type, err: error.message }, 'background pre-generation failed for type');
logger.error({ sourceCorpusId, type, err: error.message }, 'background pre-generation failed for type');
}
}
);
Promise.all(jobs)
.then(() => {
logger.info({ notebookId }, 'all background pre-generation complete');
logger.info({ sourceCorpusId }, 'all background pre-generation complete');
})
.finally(() => {
documentCacheStore.clearGenerating(notebookId);
documentCacheStore.clearGenerating(sourceCorpusId);
if (pendingReGen.has(notebookId)) {
pendingReGen.delete(notebookId);
logger.info({ notebookId }, 're-triggering pre-generation for updated sources');
runPreGeneration(notebookId);
if (pendingReGen.has(sourceCorpusId)) {
pendingReGen.delete(sourceCorpusId);
logger.info({ sourceCorpusId }, 're-triggering pre-generation for updated sources');
runPreGeneration(sourceCorpusId);
}
});
} catch (err) {
const error = err as Error;
logger.error({ notebookId, err: error.message }, 'pre-generation setup failed');
documentCacheStore.clearGenerating(notebookId);
logger.error({ sourceCorpusId, err: error.message }, 'pre-generation setup failed');
documentCacheStore.clearGenerating(sourceCorpusId);
}
}

View File

@@ -2,7 +2,7 @@
import logger from '../logger.js';
import * as vectorStore from '../stores/vectorStore.js';
import * as notebookStore from '../stores/notebookStore.js';
import * as sourceCorpusStore from '../stores/sourceCorpusStore.js';
const VOYAGE_API_URL = 'https://api.voyageai.com/v1';
const EMBED_MODEL = 'voyage-3';
@@ -72,8 +72,8 @@ export function storeChunkEmbeddings(chunks: TextChunk[], embeddings: number[][]
logger.debug({ count: chunks.length }, 'stored chunk embeddings');
}
export function search(queryEmbedding: number[], notebookId: string, topK = 10): ScoredChunk[] {
const chunks = notebookStore.getChunksForNotebook(notebookId) as TextChunk[];
export function search(queryEmbedding: number[], sourceCorpusId: string, topK = 10): ScoredChunk[] {
const chunks = sourceCorpusStore.getChunksForSourceCorpus(sourceCorpusId) as TextChunk[];
if (chunks.length === 0) return [];
const scored: ScoredChunk[] = [];

View File

@@ -136,7 +136,7 @@ export async function parseUrl(url: string): Promise<string> {
const res = await fetch(safeUrl, {
headers: {
'User-Agent': 'Mozilla/5.0 (compatible; NotebookClone/1.0)',
'User-Agent': 'Mozilla/5.0 (compatible; SourceSentinel/1.0)',
Accept: 'text/html,application/xhtml+xml,text/plain',
},
signal: AbortSignal.timeout(15000),

View File

@@ -57,7 +57,7 @@ describe('documentCacheStore', () => {
});
describe('invalidate', () => {
it('removes all entries for a notebook', () => {
it('removes all entries for a sourceCorpus', () => {
setCachedDocument('inv-1', 'faq', { a: 1 }, [{ id: 'x' }]);
setCachedDocument('inv-1', 'study-guide', { b: 2 }, [{ id: 'x' }]);
invalidate('inv-1');
@@ -65,7 +65,7 @@ describe('documentCacheStore', () => {
expect(getCachedDocument('inv-1', 'study-guide')).toBeNull();
});
it('does not affect other notebooks', () => {
it('does not affect other sourceCorpora', () => {
setCachedDocument('inv-a', 'faq', { a: 1 }, [{ id: 'x' }]);
setCachedDocument('inv-b', 'faq', { b: 2 }, [{ id: 'x' }]);
invalidate('inv-a');
@@ -86,7 +86,7 @@ describe('documentCacheStore', () => {
expect(isGenerating('gen-cycle')).toBe(false);
});
it('is cleared when the notebook is invalidated', () => {
it('is cleared when the sourceCorpus is invalidated', () => {
markGenerating('gen-inv');
invalidate('gen-inv');
expect(isGenerating('gen-inv')).toBe(false);

View File

@@ -19,19 +19,19 @@ function buildSourceHash(sources: CacheSource[]): string {
.join('|');
}
export function getCachedDocument(notebookId: string, type: string): CacheEntry | null {
const entry = cache.get(`${notebookId}:${type}`);
export function getCachedDocument(sourceCorpusId: string, type: string): CacheEntry | null {
const entry = cache.get(`${sourceCorpusId}:${type}`);
if (!entry || typeof entry === 'boolean') return null;
return entry;
}
export function setCachedDocument(
notebookId: string,
sourceCorpusId: string,
type: string,
document: unknown,
sources: CacheSource[]
): void {
const key = `${notebookId}:${type}`;
const key = `${sourceCorpusId}:${type}`;
cache.set(key, {
document,
sourceHash: buildSourceHash(sources),
@@ -39,29 +39,29 @@ export function setCachedDocument(
});
}
export function isFresh(notebookId: string, type: string, currentSources: CacheSource[]): boolean {
const entry = cache.get(`${notebookId}:${type}`);
export function isFresh(sourceCorpusId: string, type: string, currentSources: CacheSource[]): boolean {
const entry = cache.get(`${sourceCorpusId}:${type}`);
if (!entry || typeof entry === 'boolean') return false;
return entry.sourceHash === buildSourceHash(currentSources);
}
export function invalidate(notebookId: string): void {
export function invalidate(sourceCorpusId: string): void {
for (const key of cache.keys()) {
if (key.startsWith(`${notebookId}:`)) {
if (key.startsWith(`${sourceCorpusId}:`)) {
cache.delete(key);
}
}
}
export function markGenerating(notebookId: string): void {
const key = `${notebookId}:__generating`;
export function markGenerating(sourceCorpusId: string): void {
const key = `${sourceCorpusId}:__generating`;
cache.set(key, true);
}
export function clearGenerating(notebookId: string): void {
cache.delete(`${notebookId}:__generating`);
export function clearGenerating(sourceCorpusId: string): void {
cache.delete(`${sourceCorpusId}:__generating`);
}
export function isGenerating(notebookId: string): boolean {
return cache.get(`${notebookId}:__generating`) === true;
export function isGenerating(sourceCorpusId: string): boolean {
return cache.get(`${sourceCorpusId}:__generating`) === true;
}

View File

@@ -1,65 +0,0 @@
import { describe, it, expect, beforeEach } from 'vitest';
import { NotebookStore } from './notebookStore.js';
describe('notebookStore', () => {
let store: NotebookStore;
beforeEach(() => {
store = new NotebookStore();
});
it('creates and retrieves a notebook', () => {
const nb = store.createNotebook({ id: 'nb-1', name: 'Test', createdAt: '2026-01-01' });
expect(nb.id).toBe('nb-1');
expect(nb.name).toBe('Test');
const found = store.getNotebook('nb-1');
expect(found).not.toBeNull();
expect(found!.name).toBe('Test');
});
it('lists all notebooks without chunks', () => {
store.createNotebook({ id: 'nb-1', name: 'A', createdAt: '2026-01-01' });
store.createNotebook({ id: 'nb-2', name: 'B', createdAt: '2026-01-02' });
const all = store.getAllNotebooks();
expect(all).toHaveLength(2);
expect(all[0]).not.toHaveProperty('chunks');
});
it('deletes a notebook', () => {
store.createNotebook({ id: 'nb-1', name: 'A', createdAt: '2026-01-01' });
expect(store.deleteNotebook('nb-1')).toBe(true);
expect(store.getNotebook('nb-1')).toBeNull();
});
it('returns false when deleting a non-existent notebook', () => {
expect(store.deleteNotebook('nope')).toBe(false);
});
it('adds and retrieves sources', () => {
store.createNotebook({ id: 'nb-1', name: 'A', createdAt: '2026-01-01' });
store.addSource('nb-1', { id: 's-1', name: 'doc.pdf' });
const sources = store.getSources('nb-1');
expect(sources).toHaveLength(1);
expect(sources[0].name).toBe('doc.pdf');
});
it('returns empty sources for non-existent notebook', () => {
expect(store.getSources('nope')).toEqual([]);
});
it('adds and retrieves chunks', () => {
store.createNotebook({ id: 'nb-1', name: 'A', createdAt: '2026-01-01' });
store.addChunksToNotebook('nb-1', [
{ id: 'c-1', text: 'chunk one', sourceId: 's-1', index: 0 },
{ id: 'c-2', text: 'chunk two', sourceId: 's-1', index: 1 },
]);
const chunks = store.getChunksForNotebook('nb-1');
expect(chunks).toHaveLength(2);
expect(chunks[0].text).toBe('chunk one');
});
it('returns empty chunks for non-existent notebook', () => {
expect(store.getChunksForNotebook('nope')).toEqual([]);
});
});

View File

@@ -1,126 +0,0 @@
'use strict';
export interface TextChunk {
id: string;
text: string;
sourceId: string;
index: number;
}
export interface Source {
id: string;
name: string;
mimetype?: string;
chunkCount?: number;
uploadedAt?: string;
}
export interface Notebook {
id: string;
name: string;
createdAt: string;
sources: Source[];
chunks: TextChunk[];
}
export interface NotebookMeta {
id: string;
name: string;
createdAt: string;
}
export interface SourceGroup {
docIndex: number;
sourceId: string;
name: string;
chunks: TextChunk[];
}
export class NotebookStore {
private notebooks = new Map<string, Notebook>();
getAllNotebooks(): NotebookMeta[] {
return Array.from(this.notebooks.values()).map(({ id, name, createdAt }) => ({
id,
name,
createdAt,
}));
}
getNotebook(id: string): Notebook | null {
return this.notebooks.get(id) || null;
}
createNotebook(notebook: { id: string; name: string; createdAt: string }): NotebookMeta {
const record: Notebook = { ...notebook, sources: [], chunks: [] };
this.notebooks.set(record.id, record);
return { id: record.id, name: record.name, createdAt: record.createdAt };
}
deleteNotebook(id: string): boolean {
return this.notebooks.delete(id);
}
addSource(notebookId: string, source: Source): Source | null {
const notebook = this.notebooks.get(notebookId);
if (!notebook) return null;
notebook.sources.push(source);
return source;
}
getSources(notebookId: string): Source[] {
const notebook = this.notebooks.get(notebookId);
if (!notebook) return [];
return notebook.sources;
}
addChunksToNotebook(notebookId: string, chunks: TextChunk[]): Notebook | null {
const notebook = this.notebooks.get(notebookId);
if (!notebook) return null;
notebook.chunks = notebook.chunks.concat(chunks);
return notebook;
}
getChunksForNotebook(notebookId: string): TextChunk[] {
const notebook = this.notebooks.get(notebookId);
if (!notebook) return [];
return notebook.chunks;
}
buildSourceGroups(notebookId: string, chunks: TextChunk[]): SourceGroup[] {
const sources = this.getSources(notebookId);
const sourceIndexMap = new Map<string, number>();
sources.forEach((src, i) => {
sourceIndexMap.set(src.id, i + 1);
});
const groupMap = new Map<string, SourceGroup>();
for (const chunk of chunks) {
const docIndex = sourceIndexMap.get(chunk.sourceId) || 0;
if (!groupMap.has(chunk.sourceId)) {
const src = sources.find((s) => s.id === chunk.sourceId);
groupMap.set(chunk.sourceId, {
docIndex,
sourceId: chunk.sourceId,
name: src?.name || 'Unknown',
chunks: [],
});
}
groupMap.get(chunk.sourceId)!.chunks.push(chunk);
}
return Array.from(groupMap.values()).sort((a, b) => a.docIndex - b.docIndex);
}
}
const defaultStore = new NotebookStore();
export const getAllNotebooks = defaultStore.getAllNotebooks.bind(defaultStore);
export const getNotebook = defaultStore.getNotebook.bind(defaultStore);
export const createNotebook = defaultStore.createNotebook.bind(defaultStore);
export const deleteNotebook = defaultStore.deleteNotebook.bind(defaultStore);
export const addSource = defaultStore.addSource.bind(defaultStore);
export const getSources = defaultStore.getSources.bind(defaultStore);
export const addChunksToNotebook = defaultStore.addChunksToNotebook.bind(defaultStore);
export const getChunksForNotebook = defaultStore.getChunksForNotebook.bind(defaultStore);
export const buildSourceGroups = defaultStore.buildSourceGroups.bind(defaultStore);

View File

@@ -0,0 +1,65 @@
import { describe, it, expect, beforeEach } from 'vitest';
import { SourceCorpusStore } from './sourceCorpusStore.js';
describe('sourceCorpusStore', () => {
let store: SourceCorpusStore;
beforeEach(() => {
store = new SourceCorpusStore();
});
it('creates and retrieves a sourceCorpus', () => {
const corpus = store.createSourceCorpus({ id: 'nb-1', name: 'Test', createdAt: '2026-01-01' });
expect(corpus.id).toBe('nb-1');
expect(corpus.name).toBe('Test');
const found = store.getSourceCorpus('nb-1');
expect(found).not.toBeNull();
expect(found!.name).toBe('Test');
});
it('lists all sourceCorpora without chunks', () => {
store.createSourceCorpus({ id: 'nb-1', name: 'A', createdAt: '2026-01-01' });
store.createSourceCorpus({ id: 'nb-2', name: 'B', createdAt: '2026-01-02' });
const all = store.getAllSourceCorpora();
expect(all).toHaveLength(2);
expect(all[0]).not.toHaveProperty('chunks');
});
it('deletes a sourceCorpus', () => {
store.createSourceCorpus({ id: 'nb-1', name: 'A', createdAt: '2026-01-01' });
expect(store.deleteSourceCorpus('nb-1')).toBe(true);
expect(store.getSourceCorpus('nb-1')).toBeNull();
});
it('returns false when deleting a non-existent sourceCorpus', () => {
expect(store.deleteSourceCorpus('nope')).toBe(false);
});
it('adds and retrieves sources', () => {
store.createSourceCorpus({ id: 'nb-1', name: 'A', createdAt: '2026-01-01' });
store.addSource('nb-1', { id: 's-1', name: 'doc.pdf' });
const sources = store.getSources('nb-1');
expect(sources).toHaveLength(1);
expect(sources[0].name).toBe('doc.pdf');
});
it('returns empty sources for non-existent sourceCorpus', () => {
expect(store.getSources('nope')).toEqual([]);
});
it('adds and retrieves chunks', () => {
store.createSourceCorpus({ id: 'nb-1', name: 'A', createdAt: '2026-01-01' });
store.addChunksToSourceCorpus('nb-1', [
{ id: 'c-1', text: 'chunk one', sourceId: 's-1', index: 0 },
{ id: 'c-2', text: 'chunk two', sourceId: 's-1', index: 1 },
]);
const chunks = store.getChunksForSourceCorpus('nb-1');
expect(chunks).toHaveLength(2);
expect(chunks[0].text).toBe('chunk one');
});
it('returns empty chunks for non-existent sourceCorpus', () => {
expect(store.getChunksForSourceCorpus('nope')).toEqual([]);
});
});

View File

@@ -0,0 +1,126 @@
'use strict';
export interface TextChunk {
id: string;
text: string;
sourceId: string;
index: number;
}
export interface Source {
id: string;
name: string;
mimetype?: string;
chunkCount?: number;
uploadedAt?: string;
}
export interface SourceCorpus {
id: string;
name: string;
createdAt: string;
sources: Source[];
chunks: TextChunk[];
}
export interface SourceCorpusMeta {
id: string;
name: string;
createdAt: string;
}
export interface SourceGroup {
docIndex: number;
sourceId: string;
name: string;
chunks: TextChunk[];
}
export class SourceCorpusStore {
private sourceCorpora = new Map<string, SourceCorpus>();
getAllSourceCorpora(): SourceCorpusMeta[] {
return Array.from(this.sourceCorpora.values()).map(({ id, name, createdAt }) => ({
id,
name,
createdAt,
}));
}
getSourceCorpus(id: string): SourceCorpus | null {
return this.sourceCorpora.get(id) || null;
}
createSourceCorpus(sourceCorpus: { id: string; name: string; createdAt: string }): SourceCorpusMeta {
const record: SourceCorpus = { ...sourceCorpus, sources: [], chunks: [] };
this.sourceCorpora.set(record.id, record);
return { id: record.id, name: record.name, createdAt: record.createdAt };
}
deleteSourceCorpus(id: string): boolean {
return this.sourceCorpora.delete(id);
}
addSource(sourceCorpusId: string, source: Source): Source | null {
const sourceCorpus = this.sourceCorpora.get(sourceCorpusId);
if (!sourceCorpus) return null;
sourceCorpus.sources.push(source);
return source;
}
getSources(sourceCorpusId: string): Source[] {
const sourceCorpus = this.sourceCorpora.get(sourceCorpusId);
if (!sourceCorpus) return [];
return sourceCorpus.sources;
}
addChunksToSourceCorpus(sourceCorpusId: string, chunks: TextChunk[]): SourceCorpus | null {
const sourceCorpus = this.sourceCorpora.get(sourceCorpusId);
if (!sourceCorpus) return null;
sourceCorpus.chunks = sourceCorpus.chunks.concat(chunks);
return sourceCorpus;
}
getChunksForSourceCorpus(sourceCorpusId: string): TextChunk[] {
const sourceCorpus = this.sourceCorpora.get(sourceCorpusId);
if (!sourceCorpus) return [];
return sourceCorpus.chunks;
}
buildSourceGroups(sourceCorpusId: string, chunks: TextChunk[]): SourceGroup[] {
const sources = this.getSources(sourceCorpusId);
const sourceIndexMap = new Map<string, number>();
sources.forEach((src, i) => {
sourceIndexMap.set(src.id, i + 1);
});
const groupMap = new Map<string, SourceGroup>();
for (const chunk of chunks) {
const docIndex = sourceIndexMap.get(chunk.sourceId) || 0;
if (!groupMap.has(chunk.sourceId)) {
const src = sources.find((s) => s.id === chunk.sourceId);
groupMap.set(chunk.sourceId, {
docIndex,
sourceId: chunk.sourceId,
name: src?.name || 'Unknown',
chunks: [],
});
}
groupMap.get(chunk.sourceId)!.chunks.push(chunk);
}
return Array.from(groupMap.values()).sort((a, b) => a.docIndex - b.docIndex);
}
}
const defaultStore = new SourceCorpusStore();
export const getAllSourceCorpora = defaultStore.getAllSourceCorpora.bind(defaultStore);
export const getSourceCorpus = defaultStore.getSourceCorpus.bind(defaultStore);
export const createSourceCorpus = defaultStore.createSourceCorpus.bind(defaultStore);
export const deleteSourceCorpus = defaultStore.deleteSourceCorpus.bind(defaultStore);
export const addSource = defaultStore.addSource.bind(defaultStore);
export const getSources = defaultStore.getSources.bind(defaultStore);
export const addChunksToSourceCorpus = defaultStore.addChunksToSourceCorpus.bind(defaultStore);
export const getChunksForSourceCorpus = defaultStore.getChunksForSourceCorpus.bind(defaultStore);
export const buildSourceGroups = defaultStore.buildSourceGroups.bind(defaultStore);