continue refactor of backend to utilize ts, deprecate js

This commit is contained in:
KS Jannette
2026-05-09 16:57:37 -04:00
parent 1a1161b509
commit c9a69a4d27
20 changed files with 509 additions and 240 deletions

View File

@@ -1,7 +1,7 @@
'use strict';
import 'dotenv/config';
import express from 'express';
import express, { type Request, type Response, type NextFunction } from 'express';
import cors from 'cors';
import logger from './logger.js';
import notebookRoutes from './routes/notebooks.js';
@@ -21,11 +21,10 @@ for (const key of REQUIRED_KEYS) {
const app = express();
// CORS: wide-open for local dev. In production, lock this to specific origins.
app.use(cors());
app.use(express.json());
app.use((req, res, next) => {
app.use((req: Request, res: Response, next: NextFunction) => {
const start = Date.now();
res.on('finish', () => {
logger.info({
@@ -38,7 +37,7 @@ app.use((req, res, next) => {
next();
});
app.get('/.well-known/healthcheck', (req, res) => {
app.get('/.well-known/healthcheck', (_req: Request, res: Response) => {
res.json({
service: 'notebook-clone',
status: 'ok',
@@ -52,7 +51,11 @@ app.use('/api/query', queryRoutes);
app.use('/api/citation-detail', citationDetailRoutes);
app.use('/api/documents', documentRoutes);
app.use((err, req, res, _next) => {
interface ErrorWithStatus extends Error {
status?: number;
}
app.use((err: ErrorWithStatus, req: Request, res: Response, _next: NextFunction) => {
logger.error({ err, method: req.method, url: req.originalUrl });
res.status(err.status || 500).json({
error: err.message || 'internal server error',

View File

@@ -6,9 +6,7 @@ const DEBUG = !!process.env.DEBUG;
const logger = pino({
level: DEBUG ? 'debug' : 'info',
transport: process.stdout.isTTY
? { target: 'pino-pretty' }
: undefined,
transport: process.stdout.isTTY ? { target: 'pino-pretty' } : undefined,
});
export default logger;

View File

@@ -1,44 +0,0 @@
'use strict';
import * as notebookStore from '../stores/notebookStore.js';
/**
* Ensures notebookId is present in req.body or req.query.
* Normalises the value onto req.notebookId for downstream handlers.
*/
export function requireNotebookId(req, res, next) {
const notebookId = req.body?.notebookId ?? req.query?.notebookId;
if (!notebookId) {
return res.status(400).json({ error: 'notebookId is required' });
}
req.notebookId = notebookId;
next();
}
/**
* Must run after requireNotebookId.
* Loads the notebook from the store and attaches it as req.notebook.
* Returns 404 if the notebook doesn't exist.
*/
export function requireNotebook(req, res, next) {
const notebook = notebookStore.getNotebook(req.notebookId);
if (!notebook) {
return res.status(404).json({ error: 'notebook not found' });
}
req.notebook = notebook;
next();
}
export function requireFile(req, res, next) {
if (!req.file) {
return res.status(400).json({ error: 'file is required' });
}
next();
}
export function requireUrl(req, res, next) {
if (!req.body?.url) {
return res.status(400).json({ error: 'url is required' });
}
next();
}

View File

@@ -0,0 +1,46 @@
'use strict';
import type { Request, Response, NextFunction } from 'express';
import * as notebookStore from '../stores/notebookStore.js';
import type { Notebook } from '../stores/notebookStore.js';
export interface NotebookRequest extends Request {
notebookId?: string;
notebook?: Notebook;
}
export function requireNotebookId(req: NotebookRequest, res: Response, next: NextFunction): void {
const notebookId = (req.body?.notebookId ?? req.query?.notebookId) as string | undefined;
if (!notebookId) {
res.status(400).json({ error: 'notebookId is required' });
return;
}
req.notebookId = notebookId;
next();
}
export function requireNotebook(req: NotebookRequest, res: Response, next: NextFunction): void {
const notebook = notebookStore.getNotebook(req.notebookId!);
if (!notebook) {
res.status(404).json({ error: 'notebook not found' });
return;
}
req.notebook = notebook;
next();
}
export function requireFile(req: Request, res: Response, next: NextFunction): void {
if (!req.file) {
res.status(400).json({ error: 'file is required' });
return;
}
next();
}
export function requireUrl(req: Request, res: Response, next: NextFunction): void {
if (!req.body?.url) {
res.status(400).json({ error: 'url is required' });
return;
}
next();
}

View File

@@ -1,19 +1,27 @@
'use strict';
import { Router } from 'express';
import { Router, type Request, type Response, type NextFunction } from 'express';
import logger from '../logger.js';
import { generateCitationDetail } from '../services/generationService.js';
const router = Router();
router.post('/', async (req, res, next) => {
interface CitationDetailBody {
chunkTexts?: string[];
sourceName?: string;
answer?: string;
citationIndex?: number;
}
router.post('/', async (req: Request<unknown, unknown, CitationDetailBody>, res: Response, next: NextFunction) => {
try {
const { chunkTexts, sourceName, answer, citationIndex } = req.body;
if (!chunkTexts?.length || !answer || citationIndex == null) {
return res.status(400).json({
res.status(400).json({
error: 'chunkTexts, answer, and citationIndex are required',
});
return;
}
logger.info({ citationIndex, sourceName }, 'citation detail requested');

View File

@@ -1,6 +1,6 @@
'use strict';
import { Router } from 'express';
import { Router, type Request, type Response, type NextFunction } from 'express';
import logger from '../logger.js';
import * as notebookStore from '../stores/notebookStore.js';
import * as documentCacheStore from '../stores/documentCacheStore.js';
@@ -8,9 +8,16 @@ import {
generateStudyGuide,
generateFaq,
generateExecutiveBrief,
type SourceGroup,
type StudyGuide,
type Faq,
type ExecutiveBrief,
} from '../services/documentService.js';
const GENERATORS = {
type DocumentType = 'study-guide' | 'faq' | 'executive-brief';
type GeneratorFn = (sourceGroups: SourceGroup[]) => Promise<StudyGuide | Faq | ExecutiveBrief>;
const GENERATORS: Record<DocumentType, GeneratorFn> = {
'study-guide': generateStudyGuide,
'faq': generateFaq,
'executive-brief': generateExecutiveBrief,
@@ -18,36 +25,48 @@ const GENERATORS = {
const router = Router();
router.post('/generate', async (req, res, next) => {
interface GenerateBody {
notebookId?: string;
type?: string;
}
router.post('/generate', async (req: Request<unknown, unknown, GenerateBody>, res: Response, next: NextFunction) => {
try {
const { notebookId, type } = req.body;
if (!notebookId || !type) {
return res.status(400).json({ error: 'notebookId and type are required' });
res.status(400).json({ error: 'notebookId and type are required' });
return;
}
const generator = GENERATORS[type];
const generator = GENERATORS[type as DocumentType];
if (!generator) {
return res.status(400).json({
res.status(400).json({
error: `Invalid type. Must be one of: ${Object.keys(GENERATORS).join(', ')}`,
});
return;
}
const chunks = notebookStore.getChunksForNotebook(notebookId);
if (chunks.length === 0) {
return res.status(422).json({ error: 'No source material available in this notebook' });
res.status(422).json({ error: 'No source material available in this notebook' });
return;
}
const sources = notebookStore.getSources(notebookId);
const cached = documentCacheStore.getCachedDocument(notebookId, type);
if (cached && documentCacheStore.isFresh(notebookId, type, sources)) {
logger.info({ notebookId, type }, 'serving cached document');
return res.json({ type, document: cached.document });
res.json({ type, document: cached.document });
return;
}
const sourceGroups = notebookStore.buildSourceGroups(notebookId, chunks);
logger.info({ notebookId, type, sourceCount: sourceGroups.length, chunkCount: chunks.length }, 'document generation started');
logger.info(
{ notebookId, type, sourceCount: sourceGroups.length, chunkCount: chunks.length },
'document generation started'
);
const document = await generator(sourceGroups);

View File

@@ -1,20 +1,25 @@
'use strict';
import { Router } from 'express';
import { Router, type Request, type Response } from 'express';
import { v4 as uuidv4 } from 'uuid';
import * as notebookStore from '../stores/notebookStore.js';
import { deleteVectorsForChunks } from '../stores/vectorStore.js';
const router = Router();
router.get('/', (req, res) => {
router.get('/', (_req: Request, res: Response) => {
res.json(notebookStore.getAllNotebooks());
});
router.post('/', (req, res) => {
interface CreateNotebookBody {
name?: string;
}
router.post('/', (req: Request<unknown, unknown, CreateNotebookBody>, res: Response) => {
const { name } = req.body;
if (!name || !name.trim()) {
return res.status(400).json({ error: 'name is required' });
res.status(400).json({ error: 'name is required' });
return;
}
const notebook = notebookStore.createNotebook({
@@ -26,11 +31,16 @@ router.post('/', (req, res) => {
res.status(201).json(notebook);
});
router.delete('/:id', (req, res) => {
interface DeleteParams {
id: string;
}
router.delete('/:id', (req: Request<DeleteParams>, res: Response) => {
const chunks = notebookStore.getChunksForNotebook(req.params.id);
const deleted = notebookStore.deleteNotebook(req.params.id);
if (!deleted) {
return res.status(404).json({ error: 'notebook not found' });
res.status(404).json({ error: 'notebook not found' });
return;
}
if (chunks.length) {
deleteVectorsForChunks(chunks.map((c) => c.id));
@@ -39,4 +49,3 @@ router.delete('/:id', (req, res) => {
});
export default router;

View File

@@ -1,6 +1,6 @@
'use strict';
import { Router } from 'express';
import { Router, type Request, type Response, type NextFunction } from 'express';
import logger from '../logger.js';
import { embedTexts, search, rerank } from '../services/retrievalService.js';
import { generate } from '../services/generationService.js';
@@ -12,11 +12,17 @@ const TOP_K_RERANK = 5;
const router = Router();
router.post('/', async (req, res, next) => {
interface QueryBody {
notebookId?: string;
question?: string;
}
router.post('/', async (req: Request<unknown, unknown, QueryBody>, res: Response, next: NextFunction) => {
try {
const { notebookId, question } = req.body;
if (!notebookId || !question) {
return res.status(400).json({ error: 'notebookId and question are required' });
res.status(400).json({ error: 'notebookId and question are required' });
return;
}
logger.info({ notebookId, question }, 'query received');
@@ -25,31 +31,32 @@ router.post('/', async (req, res, next) => {
const searchResults = search(queryEmbedding, notebookId, TOP_K_SEARCH);
if (searchResults.length === 0) {
return res.json({
res.json({
answer: 'No sources found for this notebook. Upload some documents first.',
citations: [],
groundednessScore: null,
followUpQuestions: [],
});
return;
}
const candidateChunks = searchResults.map((r) => r.chunk);
const reranked = await rerank(question, candidateChunks);
const topChunks = reranked.slice(0, TOP_K_RERANK).map((r) => r.chunk);
logger.debug({
searchHits: searchResults.length,
rerankedTop: topChunks.length,
}, 'retrieval complete');
logger.debug(
{
searchHits: searchResults.length,
rerankedTop: topChunks.length,
},
'retrieval complete'
);
const sourceGroups = notebookStore.buildSourceGroups(notebookId, topChunks);
const { answer, citedSourceIndices, followUpQuestions } = await generate(
question,
sourceGroups,
);
const { answer, citedSourceIndices, followUpQuestions } = await generate(question, sourceGroups);
const citedChunkIds = [];
const citedChunkIds: string[] = [];
for (const idx of citedSourceIndices) {
const group = sourceGroups.find((g) => g.docIndex === idx);
if (group) {
@@ -65,11 +72,11 @@ router.post('/', async (req, res, next) => {
const group = sourceGroups.find((g) => g.docIndex === idx);
return group
? {
sourceIndex: idx,
sourceId: group.sourceId,
name: group.name,
chunkTexts: group.chunks.map((c) => c.text),
}
sourceIndex: idx,
sourceId: group.sourceId,
name: group.name,
chunkTexts: group.chunks.map((c) => c.text),
}
: { sourceIndex: idx, sourceId: null, name: null, chunkTexts: [] };
});
@@ -85,4 +92,3 @@ router.post('/', async (req, res, next) => {
});
export default router;

View File

@@ -1,17 +1,11 @@
'use strict';
import { Router } from 'express';
import { Router, type Request, type Response, type NextFunction } from 'express';
import multer from 'multer';
import { v4 as uuidv4 } from 'uuid';
import logger from '../logger.js';
import * as notebookStore from '../stores/notebookStore.js';
import {
parseFile,
chunkText,
parseUrl,
parseYoutubeUrl,
isYoutubeUrl,
} from '../services/sourceService.js';
import { parseFile, chunkText, parseUrl, parseYoutubeUrl, isYoutubeUrl } from '../services/sourceService.js';
import { embedTexts, storeChunkEmbeddings } from '../services/retrievalService.js';
import { triggerPreGeneration } from '../services/preGenerationService.js';
import {
@@ -19,28 +13,30 @@ import {
requireNotebook,
requireFile,
requireUrl,
type NotebookRequest,
} from '../middleware/validation.js';
const upload = multer({ storage: multer.memoryStorage(), limits: { fileSize: 50 * 1024 * 1024 } });
const router = Router();
const TIMESTAMP_RE = /\s+\d{1,2}\.\d{2}\.\d{2}\s*[\u2018\u2019''\s]?\s*[AP]M(?=\.\w+$)/i;
function cleanFilename(name) {
function cleanFilename(name: string): string {
return name.replace(TIMESTAMP_RE, '');
}
router.get('/', requireNotebookId, (req, res) => {
res.json(notebookStore.getSources(req.notebookId));
router.get('/', requireNotebookId, (req: NotebookRequest, res: Response) => {
res.json(notebookStore.getSources(req.notebookId!));
});
function multerUpload(req, res, next) {
function multerUpload(req: Request, res: Response, next: NextFunction): void {
upload.single('file')(req, res, (err) => {
if (err) {
if (err.code === 'LIMIT_FILE_SIZE') {
return res.status(413).json({ error: 'File too large. Maximum size is 50 MB.' });
if ((err as multer.MulterError).code === 'LIMIT_FILE_SIZE') {
res.status(413).json({ error: 'File too large. Maximum size is 50 MB.' });
return;
}
return next(err);
next(err);
return;
}
next();
});
@@ -52,73 +48,79 @@ router.post(
requireNotebookId,
requireNotebook,
requireFile,
async (req, res, next) => {
async (req: NotebookRequest, res: Response, next: NextFunction) => {
try {
const sourceId = uuidv4();
const rawName = Buffer.from(req.file.originalname, 'latin1').toString('utf-8');
const rawName = Buffer.from(req.file!.originalname, 'latin1').toString('utf-8');
const displayName = cleanFilename(rawName);
const text = await parseFile(req.file.buffer, req.file.mimetype, displayName);
const text = await parseFile(req.file!.buffer, req.file!.mimetype, displayName);
const chunks = chunkText(text, sourceId);
logger.info({
sourceId,
filename: displayName,
chunkCount: chunks.length,
}, 'parsed and chunked source');
logger.info(
{
sourceId,
filename: displayName,
chunkCount: chunks.length,
},
'parsed and chunked source'
);
notebookStore.addChunksToNotebook(req.notebookId, chunks);
notebookStore.addChunksToNotebook(req.notebookId!, chunks);
const texts = chunks.map((c) => c.text);
const embeddings = await embedTexts(texts, 'document');
storeChunkEmbeddings(chunks, embeddings);
const source = notebookStore.addSource(req.notebookId, {
const source = notebookStore.addSource(req.notebookId!, {
id: sourceId,
name: displayName,
mimetype: req.file.mimetype,
mimetype: req.file!.mimetype,
chunkCount: chunks.length,
uploadedAt: new Date().toISOString(),
});
triggerPreGeneration(req.notebookId);
triggerPreGeneration(req.notebookId!);
res.status(201).json(source);
} catch (err) {
next(err);
}
},
}
);
interface UrlBody {
url?: string;
}
router.post(
'/url',
requireNotebookId,
requireNotebook,
requireUrl,
async (req, res, next) => {
async (req: NotebookRequest & Request<unknown, unknown, UrlBody>, res: Response, next: NextFunction) => {
try {
const { url } = req.body;
const sourceId = uuidv4();
const isYT = isYoutubeUrl(url);
const displayName = isYT
? `YouTube: ${url}`
: url.replace(/^https?:\/\//, '').slice(0, 60);
const isYT = isYoutubeUrl(url!);
const displayName = isYT ? `YouTube: ${url}` : url!.replace(/^https?:\/\//, '').slice(0, 60);
logger.info({ sourceId, url, isYT }, 'processing URL source');
const text = isYT ? await parseYoutubeUrl(url) : await parseUrl(url);
const text = isYT ? await parseYoutubeUrl(url!) : await parseUrl(url!);
const chunks = chunkText(text, sourceId);
if (chunks.length === 0) {
return res.status(422).json({ error: 'No usable text could be extracted from the URL' });
res.status(422).json({ error: 'No usable text could be extracted from the URL' });
return;
}
notebookStore.addChunksToNotebook(req.notebookId, chunks);
notebookStore.addChunksToNotebook(req.notebookId!, chunks);
const texts = chunks.map((c) => c.text);
const embeddings = await embedTexts(texts, 'document');
storeChunkEmbeddings(chunks, embeddings);
const source = notebookStore.addSource(req.notebookId, {
const source = notebookStore.addSource(req.notebookId!, {
id: sourceId,
name: displayName,
mimetype: isYT ? 'video/youtube' : 'text/html',
@@ -126,13 +128,13 @@ router.post(
uploadedAt: new Date().toISOString(),
});
triggerPreGeneration(req.notebookId);
triggerPreGeneration(req.notebookId!);
res.status(201).json(source);
} catch (err) {
next(err);
}
},
}
);
export default router;

View File

@@ -20,7 +20,7 @@ type GeneratorFn = (sourceGroups: SourceGroup[]) => Promise<StudyGuide | Faq | E
const GENERATORS: Record<DocumentType, GeneratorFn> = {
'study-guide': generateStudyGuide,
faq: generateFaq,
'faq': generateFaq,
'executive-brief': generateExecutiveBrief,
};

View File

@@ -1,52 +0,0 @@
'use strict';
const cache = new Map();
function buildSourceHash(sources) {
return sources
.map((s) => s.id)
.sort()
.join('|');
}
export function getCachedDocument(notebookId, type) {
const entry = cache.get(`${notebookId}:${type}`);
if (!entry) return null;
return entry;
}
export function setCachedDocument(notebookId, type, document, sources) {
const key = `${notebookId}:${type}`;
cache.set(key, {
document,
sourceHash: buildSourceHash(sources),
createdAt: Date.now(),
});
}
export function isFresh(notebookId, type, currentSources) {
const entry = cache.get(`${notebookId}:${type}`);
if (!entry) return false;
return entry.sourceHash === buildSourceHash(currentSources);
}
export function invalidate(notebookId) {
for (const key of cache.keys()) {
if (key.startsWith(`${notebookId}:`)) {
cache.delete(key);
}
}
}
export function markGenerating(notebookId) {
const key = `${notebookId}:__generating`;
cache.set(key, true);
}
export function clearGenerating(notebookId) {
cache.delete(`${notebookId}:__generating`);
}
export function isGenerating(notebookId) {
return cache.get(`${notebookId}:__generating`) === true;
}

View File

@@ -10,7 +10,6 @@ import {
} from './documentCacheStore.js';
describe('documentCacheStore', () => {
describe('getCachedDocument / setCachedDocument', () => {
it('returns null for an uncached entry', () => {
expect(getCachedDocument('miss-1', 'faq')).toBeNull();
@@ -22,22 +21,22 @@ describe('documentCacheStore', () => {
setCachedDocument('rt-1', 'study-guide', doc, sources);
const entry = getCachedDocument('rt-1', 'study-guide');
expect(entry.document).toEqual(doc);
expect(entry.sourceHash).toBe('s1|s2');
expect(entry.createdAt).toBeTypeOf('number');
expect(entry!.document).toEqual(doc);
expect(entry!.sourceHash).toBe('s1|s2');
expect(entry!.createdAt).toBeTypeOf('number');
});
it('overwrites an existing entry for the same key', () => {
setCachedDocument('ow-1', 'faq', { v: 1 }, [{ id: 'a' }]);
setCachedDocument('ow-1', 'faq', { v: 2 }, [{ id: 'b' }]);
expect(getCachedDocument('ow-1', 'faq').document).toEqual({ v: 2 });
expect(getCachedDocument('ow-1', 'faq')!.document).toEqual({ v: 2 });
});
it('keeps entries for different types separate', () => {
setCachedDocument('sep-1', 'faq', { kind: 'faq' }, [{ id: 'x' }]);
setCachedDocument('sep-1', 'study-guide', { kind: 'sg' }, [{ id: 'x' }]);
expect(getCachedDocument('sep-1', 'faq').document.kind).toBe('faq');
expect(getCachedDocument('sep-1', 'study-guide').document.kind).toBe('sg');
expect((getCachedDocument('sep-1', 'faq')!.document as { kind: string }).kind).toBe('faq');
expect((getCachedDocument('sep-1', 'study-guide')!.document as { kind: string }).kind).toBe('sg');
});
});
@@ -71,7 +70,7 @@ describe('documentCacheStore', () => {
setCachedDocument('inv-b', 'faq', { b: 2 }, [{ id: 'x' }]);
invalidate('inv-a');
expect(getCachedDocument('inv-a', 'faq')).toBeNull();
expect(getCachedDocument('inv-b', 'faq').document).toEqual({ b: 2 });
expect(getCachedDocument('inv-b', 'faq')!.document).toEqual({ b: 2 });
});
});

View File

@@ -0,0 +1,67 @@
'use strict';
interface Source {
id: string;
}
interface CacheEntry {
document: unknown;
sourceHash: string;
createdAt: number;
}
const cache = new Map<string, CacheEntry | boolean>();
function buildSourceHash(sources: Source[]): string {
return sources
.map((s) => s.id)
.sort()
.join('|');
}
export function getCachedDocument(notebookId: string, type: string): CacheEntry | null {
const entry = cache.get(`${notebookId}:${type}`);
if (!entry || typeof entry === 'boolean') return null;
return entry;
}
export function setCachedDocument(
notebookId: string,
type: string,
document: unknown,
sources: Source[]
): void {
const key = `${notebookId}:${type}`;
cache.set(key, {
document,
sourceHash: buildSourceHash(sources),
createdAt: Date.now(),
});
}
export function isFresh(notebookId: string, type: string, currentSources: Source[]): boolean {
const entry = cache.get(`${notebookId}:${type}`);
if (!entry || typeof entry === 'boolean') return false;
return entry.sourceHash === buildSourceHash(currentSources);
}
export function invalidate(notebookId: string): void {
for (const key of cache.keys()) {
if (key.startsWith(`${notebookId}:`)) {
cache.delete(key);
}
}
}
export function markGenerating(notebookId: string): void {
const key = `${notebookId}:__generating`;
cache.set(key, true);
}
export function clearGenerating(notebookId: string): void {
cache.delete(`${notebookId}:__generating`);
}
export function isGenerating(notebookId: string): boolean {
return cache.get(`${notebookId}:__generating`) === true;
}

View File

@@ -2,7 +2,7 @@ import { describe, it, expect, beforeEach } from 'vitest';
import { NotebookStore } from './notebookStore.js';
describe('notebookStore', () => {
let store;
let store: NotebookStore;
beforeEach(() => {
store = new NotebookStore();
@@ -15,7 +15,7 @@ describe('notebookStore', () => {
const found = store.getNotebook('nb-1');
expect(found).not.toBeNull();
expect(found.name).toBe('Test');
expect(found!.name).toBe('Test');
});
it('lists all notebooks without chunks', () => {
@@ -51,8 +51,8 @@ describe('notebookStore', () => {
it('adds and retrieves chunks', () => {
store.createNotebook({ id: 'nb-1', name: 'A', createdAt: '2026-01-01' });
store.addChunksToNotebook('nb-1', [
{ id: 'c-1', text: 'chunk one' },
{ id: 'c-2', text: 'chunk two' },
{ id: 'c-1', text: 'chunk one', sourceId: 's-1', index: 0 },
{ id: 'c-2', text: 'chunk two', sourceId: 's-1', index: 1 },
]);
const chunks = store.getChunksForNotebook('nb-1');
expect(chunks).toHaveLength(2);

View File

@@ -1,62 +1,100 @@
'use strict';
export interface TextChunk {
id: string;
text: string;
sourceId: string;
index: number;
}
export interface Source {
id: string;
name: string;
mimetype?: string;
chunkCount?: number;
uploadedAt?: string;
}
export interface Notebook {
id: string;
name: string;
createdAt: string;
sources: Source[];
chunks: TextChunk[];
}
export interface NotebookMeta {
id: string;
name: string;
createdAt: string;
}
export interface SourceGroup {
docIndex: number;
sourceId: string;
name: string;
chunks: TextChunk[];
}
export class NotebookStore {
constructor() {
this.notebooks = new Map();
private notebooks = new Map<string, Notebook>();
getAllNotebooks(): NotebookMeta[] {
return Array.from(this.notebooks.values()).map(({ id, name, createdAt }) => ({
id,
name,
createdAt,
}));
}
getAllNotebooks() {
return Array.from(this.notebooks.values()).map(({ chunks, ...meta }) => meta);
}
getNotebook(id) {
getNotebook(id: string): Notebook | null {
return this.notebooks.get(id) || null;
}
createNotebook(notebook) {
const record = { ...notebook, sources: [], chunks: [] };
createNotebook(notebook: { id: string; name: string; createdAt: string }): NotebookMeta {
const record: Notebook = { ...notebook, sources: [], chunks: [] };
this.notebooks.set(record.id, record);
return { id: record.id, name: record.name, createdAt: record.createdAt };
}
deleteNotebook(id) {
deleteNotebook(id: string): boolean {
return this.notebooks.delete(id);
}
addSource(notebookId, source) {
addSource(notebookId: string, source: Source): Source | null {
const notebook = this.notebooks.get(notebookId);
if (!notebook) return null;
notebook.sources.push(source);
return source;
}
getSources(notebookId) {
getSources(notebookId: string): Source[] {
const notebook = this.notebooks.get(notebookId);
if (!notebook) return [];
return notebook.sources;
}
addChunksToNotebook(notebookId, chunks) {
addChunksToNotebook(notebookId: string, chunks: TextChunk[]): Notebook | null {
const notebook = this.notebooks.get(notebookId);
if (!notebook) return null;
notebook.chunks = notebook.chunks.concat(chunks);
return notebook;
}
getChunksForNotebook(notebookId) {
getChunksForNotebook(notebookId: string): TextChunk[] {
const notebook = this.notebooks.get(notebookId);
if (!notebook) return [];
return notebook.chunks;
}
buildSourceGroups(notebookId, chunks) {
buildSourceGroups(notebookId: string, chunks: TextChunk[]): SourceGroup[] {
const sources = this.getSources(notebookId);
const sourceIndexMap = new Map();
const sourceIndexMap = new Map<string, number>();
sources.forEach((src, i) => {
sourceIndexMap.set(src.id, i + 1);
});
const groupMap = new Map();
const groupMap = new Map<string, SourceGroup>();
for (const chunk of chunks) {
const docIndex = sourceIndexMap.get(chunk.sourceId) || 0;
if (!groupMap.has(chunk.sourceId)) {
@@ -68,7 +106,7 @@ export class NotebookStore {
chunks: [],
});
}
groupMap.get(chunk.sourceId).chunks.push(chunk);
groupMap.get(chunk.sourceId)!.chunks.push(chunk);
}
return Array.from(groupMap.values()).sort((a, b) => a.docIndex - b.docIndex);

View File

@@ -1,21 +0,0 @@
'use strict';
const vectors = new Map();
export function storeVector(chunkId, embedding) {
vectors.set(chunkId, new Float32Array(embedding));
}
export function getVector(chunkId) {
return vectors.get(chunkId) || null;
}
export function getAllVectors() {
return vectors;
}
export function deleteVectorsForChunks(chunkIds) {
for (const id of chunkIds) {
vectors.delete(id);
}
}

View File

@@ -0,0 +1,21 @@
'use strict';
const vectors = new Map<string, Float32Array>();
export function storeVector(chunkId: string, embedding: number[]): void {
vectors.set(chunkId, new Float32Array(embedding));
}
export function getVector(chunkId: string): Float32Array | null {
return vectors.get(chunkId) || null;
}
export function getAllVectors(): Map<string, Float32Array> {
return vectors;
}
export function deleteVectorsForChunks(chunkIds: string[]): void {
for (const id of chunkIds) {
vectors.delete(id);
}
}