kjannette/citation_sentinel
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #1 from kjannette/ts-refactor-two

Browse Source 
addiitonal ts implementation
This commit is contained in:
S Jannette
2026-05-09 17:18:32 -04:00
committed by GitHub
parent b3bc99c331 d918b41bba
commit f570a1e711
4 changed files with 63 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
server/src/routes/documents.ts
View File

@@ -3,12 +3,13 @@
import { Router, type Request, type Response, type NextFunction } from 'express';
import logger from '../logger.js';
import * as notebookStore from '../stores/notebookStore.js';
import type { TextChunk, Source, SourceGroup } from '../stores/notebookStore.js';
import * as documentCacheStore from '../stores/documentCacheStore.js';
import type { CacheEntry } from '../stores/documentCacheStore.js';
import {
generateStudyGuide,
generateFaq,
generateExecutiveBrief,
type SourceGroup,
type StudyGuide,
type Faq,
type ExecutiveBrief,
@@ -47,28 +48,28 @@ router.post('/generate', async (req: Request<unknown, unknown, GenerateBody>, re
return;
}
const chunks = notebookStore.getChunksForNotebook(notebookId);
const chunks: TextChunk[] = notebookStore.getChunksForNotebook(notebookId);
if (chunks.length === 0) {
res.status(422).json({ error: 'No source material available in this notebook' });
return;
}
const sources = notebookStore.getSources(notebookId);
const cached = documentCacheStore.getCachedDocument(notebookId, type);
const sources: Source[] = notebookStore.getSources(notebookId);
const cached: CacheEntry | null = documentCacheStore.getCachedDocument(notebookId, type);
if (cached && documentCacheStore.isFresh(notebookId, type, sources)) {
logger.info({ notebookId, type }, 'serving cached document');
res.json({ type, document: cached.document });
return;
}
const sourceGroups = notebookStore.buildSourceGroups(notebookId, chunks);
const sourceGroups: SourceGroup[] = notebookStore.buildSourceGroups(notebookId, chunks);
logger.info(
{ notebookId, type, sourceCount: sourceGroups.length, chunkCount: chunks.length },
'document generation started'
);
const document = await generator(sourceGroups);
const document: StudyGuide | Faq | ExecutiveBrief = await generator(sourceGroups);
documentCacheStore.setCachedDocument(notebookId, type, document, sources);
logger.info({ notebookId, type }, 'document generation complete');

47
server/src/routes/query.ts
View File

@@ -3,9 +3,12 @@
import { Router, type Request, type Response, type NextFunction } from 'express';
import logger from '../logger.js';
import { embedTexts, search, rerank } from '../services/retrievalService.js';
import type { ScoredChunk, RankedChunk, TextChunk } from '../services/retrievalService.js';
import { generate } from '../services/generationService.js';
import type { GenerationResult } from '../services/generationService.js';
import { computeGroundedness } from '../services/scoringService.js';
import * as notebookStore from '../stores/notebookStore.js';
import type { SourceGroup } from '../stores/notebookStore.js';
const TOP_K_SEARCH = 20;
const TOP_K_RERANK = 5;
@@ -17,6 +20,20 @@ interface QueryBody {
question?: string;
}
interface Citation {
sourceIndex: number;
sourceId: string | null;
name: string | null;
chunkTexts: string[];
}
interface QueryResponse {
answer: string;
citations: Citation[];
groundednessScore: number | null;
followUpQuestions: string[];
}
router.post('/', async (req: Request<unknown, unknown, QueryBody>, res: Response, next: NextFunction) => {
try {
const { notebookId, question } = req.body;
@@ -27,9 +44,9 @@ router.post('/', async (req: Request<unknown, unknown, QueryBody>, res: Response
logger.info({ notebookId, question }, 'query received');
const [queryEmbedding] = await embedTexts([question], 'query');
const [queryEmbedding]: number[][] = await embedTexts([question], 'query');
const searchResults = search(queryEmbedding, notebookId, TOP_K_SEARCH);
const searchResults: ScoredChunk[] = search(queryEmbedding, notebookId, TOP_K_SEARCH);
if (searchResults.length === 0) {
res.json({
answer: 'No sources found for this notebook. Upload some documents first.',
@@ -40,9 +57,9 @@ router.post('/', async (req: Request<unknown, unknown, QueryBody>, res: Response
return;
}
const candidateChunks = searchResults.map((r) => r.chunk);
const reranked = await rerank(question, candidateChunks);
const topChunks = reranked.slice(0, TOP_K_RERANK).map((r) => r.chunk);
const candidateChunks: TextChunk[] = searchResults.map((r) => r.chunk);
const reranked: RankedChunk[] = await rerank(question, candidateChunks);
const topChunks: TextChunk[] = reranked.slice(0, TOP_K_RERANK).map((r) => r.chunk);
logger.debug(
{
@@ -52,24 +69,24 @@ router.post('/', async (req: Request<unknown, unknown, QueryBody>, res: Response
'retrieval complete'
);
const sourceGroups = notebookStore.buildSourceGroups(notebookId, topChunks);
const sourceGroups: SourceGroup[] = notebookStore.buildSourceGroups(notebookId, topChunks);
const { answer, citedSourceIndices, followUpQuestions } = await generate(question, sourceGroups);
const { answer, citedSourceIndices, followUpQuestions }: GenerationResult = await generate(question, sourceGroups);
const citedChunkIds: string[] = [];
for (const idx of citedSourceIndices) {
const group = sourceGroups.find((g) => g.docIndex === idx);
const group: SourceGroup | undefined = sourceGroups.find((g) => g.docIndex === idx);
if (group) {
citedChunkIds.push(...group.chunks.map((c) => c.id));
}
}
const groundednessScore = await computeGroundedness(answer, citedChunkIds);
const groundednessScore: number = await computeGroundedness(answer, citedChunkIds);
logger.info({ groundednessScore: groundednessScore.toFixed(3) }, 'query complete');
const citations = citedSourceIndices.map((idx) => {
const group = sourceGroups.find((g) => g.docIndex === idx);
const citations: Citation[] = citedSourceIndices.map((idx) => {
const group: SourceGroup | undefined = sourceGroups.find((g) => g.docIndex === idx);
return group
? {
sourceIndex: idx,
@@ -77,15 +94,17 @@ router.post('/', async (req: Request<unknown, unknown, QueryBody>, res: Response
name: group.name,
chunkTexts: group.chunks.map((c) => c.text),
}
: { sourceIndex: idx, sourceId: null, name: null, chunkTexts: [] };
: { sourceIndex: idx, sourceId: null, name: null, chunkTexts: [] as string[] };
});
res.json({
const response: QueryResponse = {
answer,
citations,
groundednessScore,
followUpQuestions,
});
};
res.json(response);
} catch (err) {
next(err);
}

34
server/src/routes/sources.ts
View File

@@ -5,7 +5,9 @@ import multer from 'multer';
import { v4 as uuidv4 } from 'uuid';
import logger from '../logger.js';
import * as notebookStore from '../stores/notebookStore.js';
import type { Source } from '../stores/notebookStore.js';
import { parseFile, chunkText, parseUrl, parseYoutubeUrl, isYoutubeUrl } from '../services/sourceService.js';
import type { TextChunk } from '../services/sourceService.js';
import { embedTexts, storeChunkEmbeddings } from '../services/retrievalService.js';
import { triggerPreGeneration } from '../services/preGenerationService.js';
import {
@@ -50,11 +52,11 @@ router.post(
requireFile,
async (req: NotebookRequest, res: Response, next: NextFunction) => {
try {
const sourceId = uuidv4();
const rawName = Buffer.from(req.file!.originalname, 'latin1').toString('utf-8');
const displayName = cleanFilename(rawName);
const text = await parseFile(req.file!.buffer, req.file!.mimetype, displayName);
const chunks = chunkText(text, sourceId);
const sourceId: string = uuidv4();
const rawName: string = Buffer.from(req.file!.originalname, 'latin1').toString('utf-8');
const displayName: string = cleanFilename(rawName);
const text: string = await parseFile(req.file!.buffer, req.file!.mimetype, displayName);
const chunks: TextChunk[] = chunkText(text, sourceId);
logger.info(
{
@@ -67,11 +69,11 @@ router.post(
notebookStore.addChunksToNotebook(req.notebookId!, chunks);
const texts = chunks.map((c) => c.text);
const embeddings = await embedTexts(texts, 'document');
const texts: string[] = chunks.map((c) => c.text);
const embeddings: number[][] = await embedTexts(texts, 'document');
storeChunkEmbeddings(chunks, embeddings);
const source = notebookStore.addSource(req.notebookId!, {
const source: Source | null = notebookStore.addSource(req.notebookId!, {
id: sourceId,
name: displayName,
mimetype: req.file!.mimetype,
@@ -100,14 +102,14 @@ router.post(
async (req: NotebookRequest & Request<unknown, unknown, UrlBody>, res: Response, next: NextFunction) => {
try {
const { url } = req.body;
const sourceId = uuidv4();
const isYT = isYoutubeUrl(url!);
const displayName = isYT ? `YouTube: ${url}` : url!.replace(/^https?:\/\//, '').slice(0, 60);
const sourceId: string = uuidv4();
const isYT: boolean = isYoutubeUrl(url!);
const displayName: string = isYT ? `YouTube: ${url}` : url!.replace(/^https?:\/\//, '').slice(0, 60);
logger.info({ sourceId, url, isYT }, 'processing URL source');
const text = isYT ? await parseYoutubeUrl(url!) : await parseUrl(url!);
const chunks = chunkText(text, sourceId);
const text: string = isYT ? await parseYoutubeUrl(url!) : await parseUrl(url!);
const chunks: TextChunk[] = chunkText(text, sourceId);
if (chunks.length === 0) {
res.status(422).json({ error: 'No usable text could be extracted from the URL' });
@@ -116,11 +118,11 @@ router.post(
notebookStore.addChunksToNotebook(req.notebookId!, chunks);
const texts = chunks.map((c) => c.text);
const embeddings = await embedTexts(texts, 'document');
const texts: string[] = chunks.map((c) => c.text);
const embeddings: number[][] = await embedTexts(texts, 'document');
storeChunkEmbeddings(chunks, embeddings);
const source = notebookStore.addSource(req.notebookId!, {
const source: Source | null = notebookStore.addSource(req.notebookId!, {
id: sourceId,
name: displayName,
mimetype: isYT ? 'video/youtube' : 'text/html',

10
server/src/stores/documentCacheStore.ts
View File

@@ -1,10 +1,10 @@
'use strict';
interface Source {
export interface CacheSource {
id: string;
}
interface CacheEntry {
export interface CacheEntry {
document: unknown;
sourceHash: string;
createdAt: number;
@@ -12,7 +12,7 @@ interface CacheEntry {
const cache = new Map<string, CacheEntry | boolean>();
function buildSourceHash(sources: Source[]): string {
function buildSourceHash(sources: CacheSource[]): string {
return sources
.map((s) => s.id)
.sort()
@@ -29,7 +29,7 @@ export function setCachedDocument(
notebookId: string,
type: string,
document: unknown,
sources: Source[]
sources: CacheSource[]
): void {
const key = `${notebookId}:${type}`;
cache.set(key, {
@@ -39,7 +39,7 @@ export function setCachedDocument(
});
}
export function isFresh(notebookId: string, type: string, currentSources: Source[]): boolean {
export function isFresh(notebookId: string, type: string, currentSources: CacheSource[]): boolean {
const entry = cache.get(`${notebookId}:${type}`);
if (!entry || typeof entry === 'boolean') return false;
return entry.sourceHash === buildSourceHash(currentSources);