From d918b41bbaab349193b1cfb9ed2a004982fb342a Mon Sep 17 00:00:00 2001 From: KS Jannette Date: Sat, 9 May 2026 17:17:10 -0400 Subject: [PATCH] addiitonal ts implementation --- server/src/routes/documents.ts | 13 +++---- server/src/routes/query.ts | 47 +++++++++++++++++-------- server/src/routes/sources.ts | 34 +++++++++--------- server/src/stores/documentCacheStore.ts | 10 +++--- 4 files changed, 63 insertions(+), 41 deletions(-) diff --git a/server/src/routes/documents.ts b/server/src/routes/documents.ts index 8967fcc..2a5be08 100644 --- a/server/src/routes/documents.ts +++ b/server/src/routes/documents.ts @@ -3,12 +3,13 @@ import { Router, type Request, type Response, type NextFunction } from 'express'; import logger from '../logger.js'; import * as notebookStore from '../stores/notebookStore.js'; +import type { TextChunk, Source, SourceGroup } from '../stores/notebookStore.js'; import * as documentCacheStore from '../stores/documentCacheStore.js'; +import type { CacheEntry } from '../stores/documentCacheStore.js'; import { generateStudyGuide, generateFaq, generateExecutiveBrief, - type SourceGroup, type StudyGuide, type Faq, type ExecutiveBrief, @@ -47,28 +48,28 @@ router.post('/generate', async (req: Request, re return; } - const chunks = notebookStore.getChunksForNotebook(notebookId); + const chunks: TextChunk[] = notebookStore.getChunksForNotebook(notebookId); if (chunks.length === 0) { res.status(422).json({ error: 'No source material available in this notebook' }); return; } - const sources = notebookStore.getSources(notebookId); - const cached = documentCacheStore.getCachedDocument(notebookId, type); + const sources: Source[] = notebookStore.getSources(notebookId); + const cached: CacheEntry | null = documentCacheStore.getCachedDocument(notebookId, type); if (cached && documentCacheStore.isFresh(notebookId, type, sources)) { logger.info({ notebookId, type }, 'serving cached document'); res.json({ type, document: cached.document }); return; } - const sourceGroups = notebookStore.buildSourceGroups(notebookId, chunks); + const sourceGroups: SourceGroup[] = notebookStore.buildSourceGroups(notebookId, chunks); logger.info( { notebookId, type, sourceCount: sourceGroups.length, chunkCount: chunks.length }, 'document generation started' ); - const document = await generator(sourceGroups); + const document: StudyGuide | Faq | ExecutiveBrief = await generator(sourceGroups); documentCacheStore.setCachedDocument(notebookId, type, document, sources); logger.info({ notebookId, type }, 'document generation complete'); diff --git a/server/src/routes/query.ts b/server/src/routes/query.ts index 364ca2a..c75fd29 100644 --- a/server/src/routes/query.ts +++ b/server/src/routes/query.ts @@ -3,9 +3,12 @@ import { Router, type Request, type Response, type NextFunction } from 'express'; import logger from '../logger.js'; import { embedTexts, search, rerank } from '../services/retrievalService.js'; +import type { ScoredChunk, RankedChunk, TextChunk } from '../services/retrievalService.js'; import { generate } from '../services/generationService.js'; +import type { GenerationResult } from '../services/generationService.js'; import { computeGroundedness } from '../services/scoringService.js'; import * as notebookStore from '../stores/notebookStore.js'; +import type { SourceGroup } from '../stores/notebookStore.js'; const TOP_K_SEARCH = 20; const TOP_K_RERANK = 5; @@ -17,6 +20,20 @@ interface QueryBody { question?: string; } +interface Citation { + sourceIndex: number; + sourceId: string | null; + name: string | null; + chunkTexts: string[]; +} + +interface QueryResponse { + answer: string; + citations: Citation[]; + groundednessScore: number | null; + followUpQuestions: string[]; +} + router.post('/', async (req: Request, res: Response, next: NextFunction) => { try { const { notebookId, question } = req.body; @@ -27,9 +44,9 @@ router.post('/', async (req: Request, res: Response logger.info({ notebookId, question }, 'query received'); - const [queryEmbedding] = await embedTexts([question], 'query'); + const [queryEmbedding]: number[][] = await embedTexts([question], 'query'); - const searchResults = search(queryEmbedding, notebookId, TOP_K_SEARCH); + const searchResults: ScoredChunk[] = search(queryEmbedding, notebookId, TOP_K_SEARCH); if (searchResults.length === 0) { res.json({ answer: 'No sources found for this notebook. Upload some documents first.', @@ -40,9 +57,9 @@ router.post('/', async (req: Request, res: Response return; } - const candidateChunks = searchResults.map((r) => r.chunk); - const reranked = await rerank(question, candidateChunks); - const topChunks = reranked.slice(0, TOP_K_RERANK).map((r) => r.chunk); + const candidateChunks: TextChunk[] = searchResults.map((r) => r.chunk); + const reranked: RankedChunk[] = await rerank(question, candidateChunks); + const topChunks: TextChunk[] = reranked.slice(0, TOP_K_RERANK).map((r) => r.chunk); logger.debug( { @@ -52,24 +69,24 @@ router.post('/', async (req: Request, res: Response 'retrieval complete' ); - const sourceGroups = notebookStore.buildSourceGroups(notebookId, topChunks); + const sourceGroups: SourceGroup[] = notebookStore.buildSourceGroups(notebookId, topChunks); - const { answer, citedSourceIndices, followUpQuestions } = await generate(question, sourceGroups); + const { answer, citedSourceIndices, followUpQuestions }: GenerationResult = await generate(question, sourceGroups); const citedChunkIds: string[] = []; for (const idx of citedSourceIndices) { - const group = sourceGroups.find((g) => g.docIndex === idx); + const group: SourceGroup | undefined = sourceGroups.find((g) => g.docIndex === idx); if (group) { citedChunkIds.push(...group.chunks.map((c) => c.id)); } } - const groundednessScore = await computeGroundedness(answer, citedChunkIds); + const groundednessScore: number = await computeGroundedness(answer, citedChunkIds); logger.info({ groundednessScore: groundednessScore.toFixed(3) }, 'query complete'); - const citations = citedSourceIndices.map((idx) => { - const group = sourceGroups.find((g) => g.docIndex === idx); + const citations: Citation[] = citedSourceIndices.map((idx) => { + const group: SourceGroup | undefined = sourceGroups.find((g) => g.docIndex === idx); return group ? { sourceIndex: idx, @@ -77,15 +94,17 @@ router.post('/', async (req: Request, res: Response name: group.name, chunkTexts: group.chunks.map((c) => c.text), } - : { sourceIndex: idx, sourceId: null, name: null, chunkTexts: [] }; + : { sourceIndex: idx, sourceId: null, name: null, chunkTexts: [] as string[] }; }); - res.json({ + const response: QueryResponse = { answer, citations, groundednessScore, followUpQuestions, - }); + }; + + res.json(response); } catch (err) { next(err); } diff --git a/server/src/routes/sources.ts b/server/src/routes/sources.ts index da1abb9..593df0b 100644 --- a/server/src/routes/sources.ts +++ b/server/src/routes/sources.ts @@ -5,7 +5,9 @@ import multer from 'multer'; import { v4 as uuidv4 } from 'uuid'; import logger from '../logger.js'; import * as notebookStore from '../stores/notebookStore.js'; +import type { Source } from '../stores/notebookStore.js'; import { parseFile, chunkText, parseUrl, parseYoutubeUrl, isYoutubeUrl } from '../services/sourceService.js'; +import type { TextChunk } from '../services/sourceService.js'; import { embedTexts, storeChunkEmbeddings } from '../services/retrievalService.js'; import { triggerPreGeneration } from '../services/preGenerationService.js'; import { @@ -50,11 +52,11 @@ router.post( requireFile, async (req: NotebookRequest, res: Response, next: NextFunction) => { try { - const sourceId = uuidv4(); - const rawName = Buffer.from(req.file!.originalname, 'latin1').toString('utf-8'); - const displayName = cleanFilename(rawName); - const text = await parseFile(req.file!.buffer, req.file!.mimetype, displayName); - const chunks = chunkText(text, sourceId); + const sourceId: string = uuidv4(); + const rawName: string = Buffer.from(req.file!.originalname, 'latin1').toString('utf-8'); + const displayName: string = cleanFilename(rawName); + const text: string = await parseFile(req.file!.buffer, req.file!.mimetype, displayName); + const chunks: TextChunk[] = chunkText(text, sourceId); logger.info( { @@ -67,11 +69,11 @@ router.post( notebookStore.addChunksToNotebook(req.notebookId!, chunks); - const texts = chunks.map((c) => c.text); - const embeddings = await embedTexts(texts, 'document'); + const texts: string[] = chunks.map((c) => c.text); + const embeddings: number[][] = await embedTexts(texts, 'document'); storeChunkEmbeddings(chunks, embeddings); - const source = notebookStore.addSource(req.notebookId!, { + const source: Source | null = notebookStore.addSource(req.notebookId!, { id: sourceId, name: displayName, mimetype: req.file!.mimetype, @@ -100,14 +102,14 @@ router.post( async (req: NotebookRequest & Request, res: Response, next: NextFunction) => { try { const { url } = req.body; - const sourceId = uuidv4(); - const isYT = isYoutubeUrl(url!); - const displayName = isYT ? `YouTube: ${url}` : url!.replace(/^https?:\/\//, '').slice(0, 60); + const sourceId: string = uuidv4(); + const isYT: boolean = isYoutubeUrl(url!); + const displayName: string = isYT ? `YouTube: ${url}` : url!.replace(/^https?:\/\//, '').slice(0, 60); logger.info({ sourceId, url, isYT }, 'processing URL source'); - const text = isYT ? await parseYoutubeUrl(url!) : await parseUrl(url!); - const chunks = chunkText(text, sourceId); + const text: string = isYT ? await parseYoutubeUrl(url!) : await parseUrl(url!); + const chunks: TextChunk[] = chunkText(text, sourceId); if (chunks.length === 0) { res.status(422).json({ error: 'No usable text could be extracted from the URL' }); @@ -116,11 +118,11 @@ router.post( notebookStore.addChunksToNotebook(req.notebookId!, chunks); - const texts = chunks.map((c) => c.text); - const embeddings = await embedTexts(texts, 'document'); + const texts: string[] = chunks.map((c) => c.text); + const embeddings: number[][] = await embedTexts(texts, 'document'); storeChunkEmbeddings(chunks, embeddings); - const source = notebookStore.addSource(req.notebookId!, { + const source: Source | null = notebookStore.addSource(req.notebookId!, { id: sourceId, name: displayName, mimetype: isYT ? 'video/youtube' : 'text/html', diff --git a/server/src/stores/documentCacheStore.ts b/server/src/stores/documentCacheStore.ts index 30f4ab9..cb24aae 100644 --- a/server/src/stores/documentCacheStore.ts +++ b/server/src/stores/documentCacheStore.ts @@ -1,10 +1,10 @@ 'use strict'; -interface Source { +export interface CacheSource { id: string; } -interface CacheEntry { +export interface CacheEntry { document: unknown; sourceHash: string; createdAt: number; @@ -12,7 +12,7 @@ interface CacheEntry { const cache = new Map(); -function buildSourceHash(sources: Source[]): string { +function buildSourceHash(sources: CacheSource[]): string { return sources .map((s) => s.id) .sort() @@ -29,7 +29,7 @@ export function setCachedDocument( notebookId: string, type: string, document: unknown, - sources: Source[] + sources: CacheSource[] ): void { const key = `${notebookId}:${type}`; cache.set(key, { @@ -39,7 +39,7 @@ export function setCachedDocument( }); } -export function isFresh(notebookId: string, type: string, currentSources: Source[]): boolean { +export function isFresh(notebookId: string, type: string, currentSources: CacheSource[]): boolean { const entry = cache.get(`${notebookId}:${type}`); if (!entry || typeof entry === 'boolean') return false; return entry.sourceHash === buildSourceHash(currentSources);