continue refactor of backend to utilize ts, deprecate js
This commit is contained in:
KS Jannette
165
server/package-lock.json
generated
165
server/package-lock.json
generated
@@ -22,10 +22,15 @@
|
||||
"uuid": "^11.1.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/cors": "^2.8.19",
|
||||
"@types/express": "^5.0.6",
|
||||
"@types/multer": "^2.1.0",
|
||||
"@types/node": "^25.6.2",
|
||||
"@types/pdf-parse": "^1.1.5",
|
||||
"@types/uuid": "^10.0.0",
|
||||
"eslint": "^9.22.0",
|
||||
"prettier": "^3.5.3",
|
||||
"tsx": "^4.21.0",
|
||||
"typescript": "^6.0.3",
|
||||
"vitest": "^3.0.0"
|
||||
},
|
||||
@@ -1063,6 +1068,17 @@
|
||||
"win32"
|
||||
]
|
||||
},
|
||||
"node_modules/@types/body-parser": {
|
||||
"version": "1.19.6",
|
||||
"resolved": "https://registry.npmjs.org/@types/body-parser/-/body-parser-1.19.6.tgz",
|
||||
"integrity": "sha512-HLFeCYgz89uk22N5Qg3dvGvsv46B8GLvKKo1zKG4NybA8U2DiEO3w9lqGg29t/tfLRJpJ6iQxnVw4OnB7MoM9g==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@types/connect": "*",
|
||||
"@types/node": "*"
|
||||
}
|
||||
},
|
||||
"node_modules/@types/chai": {
|
||||
"version": "5.2.3",
|
||||
"resolved": "https://registry.npmjs.org/@types/chai/-/chai-5.2.3.tgz",
|
||||
@@ -1074,6 +1090,26 @@
|
||||
"assertion-error": "^2.0.1"
|
||||
}
|
||||
},
|
||||
"node_modules/@types/connect": {
|
||||
"version": "3.4.38",
|
||||
"resolved": "https://registry.npmjs.org/@types/connect/-/connect-3.4.38.tgz",
|
||||
"integrity": "sha512-K6uROf1LD88uDQqJCktA4yzL1YYAK6NgfsI0v/mTgyPKWsX1CnJ0XPSDhViejru1GcRkLWb8RlzFYJRqGUbaug==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@types/node": "*"
|
||||
}
|
||||
},
|
||||
"node_modules/@types/cors": {
|
||||
"version": "2.8.19",
|
||||
"resolved": "https://registry.npmjs.org/@types/cors/-/cors-2.8.19.tgz",
|
||||
"integrity": "sha512-mFNylyeyqN93lfe/9CSxOGREz8cpzAhH+E93xJ4xWQf62V8sQ/24reV2nyzUWM6H6Xji+GGHpkbLe7pVoUEskg==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@types/node": "*"
|
||||
}
|
||||
},
|
||||
"node_modules/@types/deep-eql": {
|
||||
"version": "4.0.2",
|
||||
"resolved": "https://registry.npmjs.org/@types/deep-eql/-/deep-eql-4.0.2.tgz",
|
||||
@@ -1088,6 +1124,38 @@
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/@types/express": {
|
||||
"version": "5.0.6",
|
||||
"resolved": "https://registry.npmjs.org/@types/express/-/express-5.0.6.tgz",
|
||||
"integrity": "sha512-sKYVuV7Sv9fbPIt/442koC7+IIwK5olP1KWeD88e/idgoJqDm3JV/YUiPwkoKK92ylff2MGxSz1CSjsXelx0YA==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@types/body-parser": "*",
|
||||
"@types/express-serve-static-core": "^5.0.0",
|
||||
"@types/serve-static": "^2"
|
||||
}
|
||||
},
|
||||
"node_modules/@types/express-serve-static-core": {
|
||||
"version": "5.1.1",
|
||||
"resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-5.1.1.tgz",
|
||||
"integrity": "sha512-v4zIMr/cX7/d2BpAEX3KNKL/JrT1s43s96lLvvdTmza1oEvDudCqK9aF/djc/SWgy8Yh0h30TZx5VpzqFCxk5A==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@types/node": "*",
|
||||
"@types/qs": "*",
|
||||
"@types/range-parser": "*",
|
||||
"@types/send": "*"
|
||||
}
|
||||
},
|
||||
"node_modules/@types/http-errors": {
|
||||
"version": "2.0.5",
|
||||
"resolved": "https://registry.npmjs.org/@types/http-errors/-/http-errors-2.0.5.tgz",
|
||||
"integrity": "sha512-r8Tayk8HJnX0FztbZN7oVqGccWgw98T/0neJphO91KkmOzug1KkofZURD4UaD5uH8AqcFLfdPErnBod0u71/qg==",
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/@types/json-schema": {
|
||||
"version": "7.0.15",
|
||||
"resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.15.tgz",
|
||||
@@ -1095,13 +1163,22 @@
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/@types/multer": {
|
||||
"version": "2.1.0",
|
||||
"resolved": "https://registry.npmjs.org/@types/multer/-/multer-2.1.0.tgz",
|
||||
"integrity": "sha512-zYZb0+nJhOHtPpGDb3vqPjwpdeGlGC157VpkqNQL+UU2qwoacoQ7MpsAmUptI/0Oa127X32JzWDqQVEXp2RcIA==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@types/express": "*"
|
||||
}
|
||||
},
|
||||
"node_modules/@types/node": {
|
||||
"version": "25.6.2",
|
||||
"resolved": "https://registry.npmjs.org/@types/node/-/node-25.6.2.tgz",
|
||||
"integrity": "sha512-sokuT28dxf9JT5Kady1fsXOvI4HVpjZa95NKT5y9PNTIrs2AsobR4GFAA90ZG8M+nxVRLysCXsVj6eGC7Vbrlw==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"peer": true,
|
||||
"dependencies": {
|
||||
"undici-types": "~7.19.0"
|
||||
}
|
||||
@@ -1116,6 +1193,48 @@
|
||||
"@types/node": "*"
|
||||
}
|
||||
},
|
||||
"node_modules/@types/qs": {
|
||||
"version": "6.15.1",
|
||||
"resolved": "https://registry.npmjs.org/@types/qs/-/qs-6.15.1.tgz",
|
||||
"integrity": "sha512-GZHUBZR9hckSUhrxmp1nG6NwdpM9fCunJwyThLW1X3AyHgd9IlHb6VANpQQqDr2o/qQp6McZ3y/IA2rVzKzSbw==",
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/@types/range-parser": {
|
||||
"version": "1.2.7",
|
||||
"resolved": "https://registry.npmjs.org/@types/range-parser/-/range-parser-1.2.7.tgz",
|
||||
"integrity": "sha512-hKormJbkJqzQGhziax5PItDUTMAM9uE2XXQmM37dyd4hVM+5aVl7oVxMVUiVQn2oCQFN/LKCZdvSM0pFRqbSmQ==",
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/@types/send": {
|
||||
"version": "1.2.1",
|
||||
"resolved": "https://registry.npmjs.org/@types/send/-/send-1.2.1.tgz",
|
||||
"integrity": "sha512-arsCikDvlU99zl1g69TcAB3mzZPpxgw0UQnaHeC1Nwb015xp8bknZv5rIfri9xTOcMuaVgvabfIRA7PSZVuZIQ==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@types/node": "*"
|
||||
}
|
||||
},
|
||||
"node_modules/@types/serve-static": {
|
||||
"version": "2.2.0",
|
||||
"resolved": "https://registry.npmjs.org/@types/serve-static/-/serve-static-2.2.0.tgz",
|
||||
"integrity": "sha512-8mam4H1NHLtu7nmtalF7eyBH14QyOASmcxHhSfEoRyr0nP/YdoesEtU+uSRvMe96TW/HPTtkoKqQLl53N7UXMQ==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@types/http-errors": "*",
|
||||
"@types/node": "*"
|
||||
}
|
||||
},
|
||||
"node_modules/@types/uuid": {
|
||||
"version": "10.0.0",
|
||||
"resolved": "https://registry.npmjs.org/@types/uuid/-/uuid-10.0.0.tgz",
|
||||
"integrity": "sha512-7gqG38EyHgyP1S+7+xomFtL+ZNHcKv6DwNaCZmJmo1vgMugyF3TCnXVg4t1uk89mLNwnLtnY3TpOpCOyp1/xHQ==",
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/@vitest/expect": {
|
||||
"version": "3.2.4",
|
||||
"resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-3.2.4.tgz",
|
||||
@@ -2525,6 +2644,19 @@
|
||||
"node": ">= 0.4"
|
||||
}
|
||||
},
|
||||
"node_modules/get-tsconfig": {
|
||||
"version": "4.14.0",
|
||||
"resolved": "https://registry.npmjs.org/get-tsconfig/-/get-tsconfig-4.14.0.tgz",
|
||||
"integrity": "sha512-yTb+8DXzDREzgvYmh6s9vHsSVCHeC0G3PI5bEXNBHtmshPnO+S5O7qgLEOn0I5QvMy6kpZN8K1NKGyilLb93wA==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"resolve-pkg-maps": "^1.0.0"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://github.com/privatenumber/get-tsconfig?sponsor=1"
|
||||
}
|
||||
},
|
||||
"node_modules/glob-parent": {
|
||||
"version": "6.0.2",
|
||||
"resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz",
|
||||
@@ -3704,6 +3836,16 @@
|
||||
"node": ">=4"
|
||||
}
|
||||
},
|
||||
"node_modules/resolve-pkg-maps": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/resolve-pkg-maps/-/resolve-pkg-maps-1.0.0.tgz",
|
||||
"integrity": "sha512-seS2Tj26TBVOC2NIc2rOe2y2ZO7efxITtLZcGSOnHHNOQ7CkiUBfw0Iw2ck6xkIhPwLhKNLS8BO+hEpngQlqzw==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"funding": {
|
||||
"url": "https://github.com/privatenumber/resolve-pkg-maps?sponsor=1"
|
||||
}
|
||||
},
|
||||
"node_modules/rollup": {
|
||||
"version": "4.59.0",
|
||||
"resolved": "https://registry.npmjs.org/rollup/-/rollup-4.59.0.tgz",
|
||||
@@ -4172,6 +4314,27 @@
|
||||
"integrity": "sha512-FPAhNPFMrkwz76P7cdjdmiShwMynZYN6SgOujD1urY4oNm80Ou9oMdmbR45LotcKOXoy7wSmHkRFE6Mxbrhefw==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/tsx": {
|
||||
"version": "4.21.0",
|
||||
"resolved": "https://registry.npmjs.org/tsx/-/tsx-4.21.0.tgz",
|
||||
"integrity": "sha512-5C1sg4USs1lfG0GFb2RLXsdpXqBSEhAaA/0kPL01wxzpMqLILNxIxIOKiILz+cdg/pLnOUxFYOR5yhHU666wbw==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"peer": true,
|
||||
"dependencies": {
|
||||
"esbuild": "~0.27.0",
|
||||
"get-tsconfig": "^4.7.5"
|
||||
},
|
||||
"bin": {
|
||||
"tsx": "dist/cli.mjs"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=18.0.0"
|
||||
},
|
||||
"optionalDependencies": {
|
||||
"fsevents": "~2.3.3"
|
||||
}
|
||||
},
|
||||
"node_modules/type-check": {
|
||||
"version": "0.4.0",
|
||||
"resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz",
|
||||
|
||||
@@ -7,7 +7,7 @@
|
||||
},
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"start": "node src/index.js",
|
||||
"start": "tsx src/index.ts",
|
||||
"test": "vitest run",
|
||||
"lint": "eslint src/",
|
||||
"fmt": "prettier --write .",
|
||||
@@ -28,10 +28,15 @@
|
||||
"uuid": "^11.1.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/cors": "^2.8.19",
|
||||
"@types/express": "^5.0.6",
|
||||
"@types/multer": "^2.1.0",
|
||||
"@types/node": "^25.6.2",
|
||||
"@types/pdf-parse": "^1.1.5",
|
||||
"@types/uuid": "^10.0.0",
|
||||
"eslint": "^9.22.0",
|
||||
"prettier": "^3.5.3",
|
||||
"tsx": "^4.21.0",
|
||||
"typescript": "^6.0.3",
|
||||
"vitest": "^3.0.0"
|
||||
}
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
'use strict';
|
||||
|
||||
import 'dotenv/config';
|
||||
import express from 'express';
|
||||
import express, { type Request, type Response, type NextFunction } from 'express';
|
||||
import cors from 'cors';
|
||||
import logger from './logger.js';
|
||||
import notebookRoutes from './routes/notebooks.js';
|
||||
@@ -21,11 +21,10 @@ for (const key of REQUIRED_KEYS) {
|
||||
|
||||
const app = express();
|
||||
|
||||
// CORS: wide-open for local dev. In production, lock this to specific origins.
|
||||
app.use(cors());
|
||||
app.use(express.json());
|
||||
|
||||
app.use((req, res, next) => {
|
||||
app.use((req: Request, res: Response, next: NextFunction) => {
|
||||
const start = Date.now();
|
||||
res.on('finish', () => {
|
||||
logger.info({
|
||||
@@ -38,7 +37,7 @@ app.use((req, res, next) => {
|
||||
next();
|
||||
});
|
||||
|
||||
app.get('/.well-known/healthcheck', (req, res) => {
|
||||
app.get('/.well-known/healthcheck', (_req: Request, res: Response) => {
|
||||
res.json({
|
||||
service: 'notebook-clone',
|
||||
status: 'ok',
|
||||
@@ -52,7 +51,11 @@ app.use('/api/query', queryRoutes);
|
||||
app.use('/api/citation-detail', citationDetailRoutes);
|
||||
app.use('/api/documents', documentRoutes);
|
||||
|
||||
app.use((err, req, res, _next) => {
|
||||
interface ErrorWithStatus extends Error {
|
||||
status?: number;
|
||||
}
|
||||
|
||||
app.use((err: ErrorWithStatus, req: Request, res: Response, _next: NextFunction) => {
|
||||
logger.error({ err, method: req.method, url: req.originalUrl });
|
||||
res.status(err.status || 500).json({
|
||||
error: err.message || 'internal server error',
|
||||
@@ -6,9 +6,7 @@ const DEBUG = !!process.env.DEBUG;
|
||||
|
||||
const logger = pino({
|
||||
level: DEBUG ? 'debug' : 'info',
|
||||
transport: process.stdout.isTTY
|
||||
? { target: 'pino-pretty' }
|
||||
: undefined,
|
||||
transport: process.stdout.isTTY ? { target: 'pino-pretty' } : undefined,
|
||||
});
|
||||
|
||||
export default logger;
|
||||
@@ -1,44 +0,0 @@
|
||||
'use strict';
|
||||
|
||||
import * as notebookStore from '../stores/notebookStore.js';
|
||||
|
||||
/**
|
||||
* Ensures notebookId is present in req.body or req.query.
|
||||
* Normalises the value onto req.notebookId for downstream handlers.
|
||||
*/
|
||||
export function requireNotebookId(req, res, next) {
|
||||
const notebookId = req.body?.notebookId ?? req.query?.notebookId;
|
||||
if (!notebookId) {
|
||||
return res.status(400).json({ error: 'notebookId is required' });
|
||||
}
|
||||
req.notebookId = notebookId;
|
||||
next();
|
||||
}
|
||||
|
||||
/**
|
||||
* Must run after requireNotebookId.
|
||||
* Loads the notebook from the store and attaches it as req.notebook.
|
||||
* Returns 404 if the notebook doesn't exist.
|
||||
*/
|
||||
export function requireNotebook(req, res, next) {
|
||||
const notebook = notebookStore.getNotebook(req.notebookId);
|
||||
if (!notebook) {
|
||||
return res.status(404).json({ error: 'notebook not found' });
|
||||
}
|
||||
req.notebook = notebook;
|
||||
next();
|
||||
}
|
||||
|
||||
export function requireFile(req, res, next) {
|
||||
if (!req.file) {
|
||||
return res.status(400).json({ error: 'file is required' });
|
||||
}
|
||||
next();
|
||||
}
|
||||
|
||||
export function requireUrl(req, res, next) {
|
||||
if (!req.body?.url) {
|
||||
return res.status(400).json({ error: 'url is required' });
|
||||
}
|
||||
next();
|
||||
}
|
||||
46
server/src/middleware/validation.ts
Normal file
46
server/src/middleware/validation.ts
Normal file
@@ -0,0 +1,46 @@
|
||||
'use strict';
|
||||
|
||||
import type { Request, Response, NextFunction } from 'express';
|
||||
import * as notebookStore from '../stores/notebookStore.js';
|
||||
import type { Notebook } from '../stores/notebookStore.js';
|
||||
|
||||
export interface NotebookRequest extends Request {
|
||||
notebookId?: string;
|
||||
notebook?: Notebook;
|
||||
}
|
||||
|
||||
export function requireNotebookId(req: NotebookRequest, res: Response, next: NextFunction): void {
|
||||
const notebookId = (req.body?.notebookId ?? req.query?.notebookId) as string | undefined;
|
||||
if (!notebookId) {
|
||||
res.status(400).json({ error: 'notebookId is required' });
|
||||
return;
|
||||
}
|
||||
req.notebookId = notebookId;
|
||||
next();
|
||||
}
|
||||
|
||||
export function requireNotebook(req: NotebookRequest, res: Response, next: NextFunction): void {
|
||||
const notebook = notebookStore.getNotebook(req.notebookId!);
|
||||
if (!notebook) {
|
||||
res.status(404).json({ error: 'notebook not found' });
|
||||
return;
|
||||
}
|
||||
req.notebook = notebook;
|
||||
next();
|
||||
}
|
||||
|
||||
export function requireFile(req: Request, res: Response, next: NextFunction): void {
|
||||
if (!req.file) {
|
||||
res.status(400).json({ error: 'file is required' });
|
||||
return;
|
||||
}
|
||||
next();
|
||||
}
|
||||
|
||||
export function requireUrl(req: Request, res: Response, next: NextFunction): void {
|
||||
if (!req.body?.url) {
|
||||
res.status(400).json({ error: 'url is required' });
|
||||
return;
|
||||
}
|
||||
next();
|
||||
}
|
||||
@@ -1,19 +1,27 @@
|
||||
'use strict';
|
||||
|
||||
import { Router } from 'express';
|
||||
import { Router, type Request, type Response, type NextFunction } from 'express';
|
||||
import logger from '../logger.js';
|
||||
import { generateCitationDetail } from '../services/generationService.js';
|
||||
|
||||
const router = Router();
|
||||
|
||||
router.post('/', async (req, res, next) => {
|
||||
interface CitationDetailBody {
|
||||
chunkTexts?: string[];
|
||||
sourceName?: string;
|
||||
answer?: string;
|
||||
citationIndex?: number;
|
||||
}
|
||||
|
||||
router.post('/', async (req: Request<unknown, unknown, CitationDetailBody>, res: Response, next: NextFunction) => {
|
||||
try {
|
||||
const { chunkTexts, sourceName, answer, citationIndex } = req.body;
|
||||
|
||||
if (!chunkTexts?.length || !answer || citationIndex == null) {
|
||||
return res.status(400).json({
|
||||
res.status(400).json({
|
||||
error: 'chunkTexts, answer, and citationIndex are required',
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
logger.info({ citationIndex, sourceName }, 'citation detail requested');
|
||||
@@ -1,6 +1,6 @@
|
||||
'use strict';
|
||||
|
||||
import { Router } from 'express';
|
||||
import { Router, type Request, type Response, type NextFunction } from 'express';
|
||||
import logger from '../logger.js';
|
||||
import * as notebookStore from '../stores/notebookStore.js';
|
||||
import * as documentCacheStore from '../stores/documentCacheStore.js';
|
||||
@@ -8,9 +8,16 @@ import {
|
||||
generateStudyGuide,
|
||||
generateFaq,
|
||||
generateExecutiveBrief,
|
||||
type SourceGroup,
|
||||
type StudyGuide,
|
||||
type Faq,
|
||||
type ExecutiveBrief,
|
||||
} from '../services/documentService.js';
|
||||
|
||||
const GENERATORS = {
|
||||
type DocumentType = 'study-guide' | 'faq' | 'executive-brief';
|
||||
type GeneratorFn = (sourceGroups: SourceGroup[]) => Promise<StudyGuide | Faq | ExecutiveBrief>;
|
||||
|
||||
const GENERATORS: Record<DocumentType, GeneratorFn> = {
|
||||
'study-guide': generateStudyGuide,
|
||||
'faq': generateFaq,
|
||||
'executive-brief': generateExecutiveBrief,
|
||||
@@ -18,36 +25,48 @@ const GENERATORS = {
|
||||
|
||||
const router = Router();
|
||||
|
||||
router.post('/generate', async (req, res, next) => {
|
||||
interface GenerateBody {
|
||||
notebookId?: string;
|
||||
type?: string;
|
||||
}
|
||||
|
||||
router.post('/generate', async (req: Request<unknown, unknown, GenerateBody>, res: Response, next: NextFunction) => {
|
||||
try {
|
||||
const { notebookId, type } = req.body;
|
||||
|
||||
if (!notebookId || !type) {
|
||||
return res.status(400).json({ error: 'notebookId and type are required' });
|
||||
res.status(400).json({ error: 'notebookId and type are required' });
|
||||
return;
|
||||
}
|
||||
|
||||
const generator = GENERATORS[type];
|
||||
const generator = GENERATORS[type as DocumentType];
|
||||
if (!generator) {
|
||||
return res.status(400).json({
|
||||
res.status(400).json({
|
||||
error: `Invalid type. Must be one of: ${Object.keys(GENERATORS).join(', ')}`,
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
const chunks = notebookStore.getChunksForNotebook(notebookId);
|
||||
if (chunks.length === 0) {
|
||||
return res.status(422).json({ error: 'No source material available in this notebook' });
|
||||
res.status(422).json({ error: 'No source material available in this notebook' });
|
||||
return;
|
||||
}
|
||||
|
||||
const sources = notebookStore.getSources(notebookId);
|
||||
const cached = documentCacheStore.getCachedDocument(notebookId, type);
|
||||
if (cached && documentCacheStore.isFresh(notebookId, type, sources)) {
|
||||
logger.info({ notebookId, type }, 'serving cached document');
|
||||
return res.json({ type, document: cached.document });
|
||||
res.json({ type, document: cached.document });
|
||||
return;
|
||||
}
|
||||
|
||||
const sourceGroups = notebookStore.buildSourceGroups(notebookId, chunks);
|
||||
|
||||
logger.info({ notebookId, type, sourceCount: sourceGroups.length, chunkCount: chunks.length }, 'document generation started');
|
||||
logger.info(
|
||||
{ notebookId, type, sourceCount: sourceGroups.length, chunkCount: chunks.length },
|
||||
'document generation started'
|
||||
);
|
||||
|
||||
const document = await generator(sourceGroups);
|
||||
|
||||
@@ -1,20 +1,25 @@
|
||||
'use strict';
|
||||
|
||||
import { Router } from 'express';
|
||||
import { Router, type Request, type Response } from 'express';
|
||||
import { v4 as uuidv4 } from 'uuid';
|
||||
import * as notebookStore from '../stores/notebookStore.js';
|
||||
import { deleteVectorsForChunks } from '../stores/vectorStore.js';
|
||||
|
||||
const router = Router();
|
||||
|
||||
router.get('/', (req, res) => {
|
||||
router.get('/', (_req: Request, res: Response) => {
|
||||
res.json(notebookStore.getAllNotebooks());
|
||||
});
|
||||
|
||||
router.post('/', (req, res) => {
|
||||
interface CreateNotebookBody {
|
||||
name?: string;
|
||||
}
|
||||
|
||||
router.post('/', (req: Request<unknown, unknown, CreateNotebookBody>, res: Response) => {
|
||||
const { name } = req.body;
|
||||
if (!name || !name.trim()) {
|
||||
return res.status(400).json({ error: 'name is required' });
|
||||
res.status(400).json({ error: 'name is required' });
|
||||
return;
|
||||
}
|
||||
|
||||
const notebook = notebookStore.createNotebook({
|
||||
@@ -26,11 +31,16 @@ router.post('/', (req, res) => {
|
||||
res.status(201).json(notebook);
|
||||
});
|
||||
|
||||
router.delete('/:id', (req, res) => {
|
||||
interface DeleteParams {
|
||||
id: string;
|
||||
}
|
||||
|
||||
router.delete('/:id', (req: Request<DeleteParams>, res: Response) => {
|
||||
const chunks = notebookStore.getChunksForNotebook(req.params.id);
|
||||
const deleted = notebookStore.deleteNotebook(req.params.id);
|
||||
if (!deleted) {
|
||||
return res.status(404).json({ error: 'notebook not found' });
|
||||
res.status(404).json({ error: 'notebook not found' });
|
||||
return;
|
||||
}
|
||||
if (chunks.length) {
|
||||
deleteVectorsForChunks(chunks.map((c) => c.id));
|
||||
@@ -39,4 +49,3 @@ router.delete('/:id', (req, res) => {
|
||||
});
|
||||
|
||||
export default router;
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
'use strict';
|
||||
|
||||
import { Router } from 'express';
|
||||
import { Router, type Request, type Response, type NextFunction } from 'express';
|
||||
import logger from '../logger.js';
|
||||
import { embedTexts, search, rerank } from '../services/retrievalService.js';
|
||||
import { generate } from '../services/generationService.js';
|
||||
@@ -12,11 +12,17 @@ const TOP_K_RERANK = 5;
|
||||
|
||||
const router = Router();
|
||||
|
||||
router.post('/', async (req, res, next) => {
|
||||
interface QueryBody {
|
||||
notebookId?: string;
|
||||
question?: string;
|
||||
}
|
||||
|
||||
router.post('/', async (req: Request<unknown, unknown, QueryBody>, res: Response, next: NextFunction) => {
|
||||
try {
|
||||
const { notebookId, question } = req.body;
|
||||
if (!notebookId || !question) {
|
||||
return res.status(400).json({ error: 'notebookId and question are required' });
|
||||
res.status(400).json({ error: 'notebookId and question are required' });
|
||||
return;
|
||||
}
|
||||
|
||||
logger.info({ notebookId, question }, 'query received');
|
||||
@@ -25,31 +31,32 @@ router.post('/', async (req, res, next) => {
|
||||
|
||||
const searchResults = search(queryEmbedding, notebookId, TOP_K_SEARCH);
|
||||
if (searchResults.length === 0) {
|
||||
return res.json({
|
||||
res.json({
|
||||
answer: 'No sources found for this notebook. Upload some documents first.',
|
||||
citations: [],
|
||||
groundednessScore: null,
|
||||
followUpQuestions: [],
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
const candidateChunks = searchResults.map((r) => r.chunk);
|
||||
const reranked = await rerank(question, candidateChunks);
|
||||
const topChunks = reranked.slice(0, TOP_K_RERANK).map((r) => r.chunk);
|
||||
|
||||
logger.debug({
|
||||
searchHits: searchResults.length,
|
||||
rerankedTop: topChunks.length,
|
||||
}, 'retrieval complete');
|
||||
logger.debug(
|
||||
{
|
||||
searchHits: searchResults.length,
|
||||
rerankedTop: topChunks.length,
|
||||
},
|
||||
'retrieval complete'
|
||||
);
|
||||
|
||||
const sourceGroups = notebookStore.buildSourceGroups(notebookId, topChunks);
|
||||
|
||||
const { answer, citedSourceIndices, followUpQuestions } = await generate(
|
||||
question,
|
||||
sourceGroups,
|
||||
);
|
||||
const { answer, citedSourceIndices, followUpQuestions } = await generate(question, sourceGroups);
|
||||
|
||||
const citedChunkIds = [];
|
||||
const citedChunkIds: string[] = [];
|
||||
for (const idx of citedSourceIndices) {
|
||||
const group = sourceGroups.find((g) => g.docIndex === idx);
|
||||
if (group) {
|
||||
@@ -65,11 +72,11 @@ router.post('/', async (req, res, next) => {
|
||||
const group = sourceGroups.find((g) => g.docIndex === idx);
|
||||
return group
|
||||
? {
|
||||
sourceIndex: idx,
|
||||
sourceId: group.sourceId,
|
||||
name: group.name,
|
||||
chunkTexts: group.chunks.map((c) => c.text),
|
||||
}
|
||||
sourceIndex: idx,
|
||||
sourceId: group.sourceId,
|
||||
name: group.name,
|
||||
chunkTexts: group.chunks.map((c) => c.text),
|
||||
}
|
||||
: { sourceIndex: idx, sourceId: null, name: null, chunkTexts: [] };
|
||||
});
|
||||
|
||||
@@ -85,4 +92,3 @@ router.post('/', async (req, res, next) => {
|
||||
});
|
||||
|
||||
export default router;
|
||||
|
||||
@@ -1,17 +1,11 @@
|
||||
'use strict';
|
||||
|
||||
import { Router } from 'express';
|
||||
import { Router, type Request, type Response, type NextFunction } from 'express';
|
||||
import multer from 'multer';
|
||||
import { v4 as uuidv4 } from 'uuid';
|
||||
import logger from '../logger.js';
|
||||
import * as notebookStore from '../stores/notebookStore.js';
|
||||
import {
|
||||
parseFile,
|
||||
chunkText,
|
||||
parseUrl,
|
||||
parseYoutubeUrl,
|
||||
isYoutubeUrl,
|
||||
} from '../services/sourceService.js';
|
||||
import { parseFile, chunkText, parseUrl, parseYoutubeUrl, isYoutubeUrl } from '../services/sourceService.js';
|
||||
import { embedTexts, storeChunkEmbeddings } from '../services/retrievalService.js';
|
||||
import { triggerPreGeneration } from '../services/preGenerationService.js';
|
||||
import {
|
||||
@@ -19,28 +13,30 @@ import {
|
||||
requireNotebook,
|
||||
requireFile,
|
||||
requireUrl,
|
||||
type NotebookRequest,
|
||||
} from '../middleware/validation.js';
|
||||
|
||||
const upload = multer({ storage: multer.memoryStorage(), limits: { fileSize: 50 * 1024 * 1024 } });
|
||||
const router = Router();
|
||||
|
||||
const TIMESTAMP_RE = /\s+\d{1,2}\.\d{2}\.\d{2}\s*[\u2018\u2019''\s]?\s*[AP]M(?=\.\w+$)/i;
|
||||
function cleanFilename(name) {
|
||||
function cleanFilename(name: string): string {
|
||||
return name.replace(TIMESTAMP_RE, '');
|
||||
}
|
||||
|
||||
router.get('/', requireNotebookId, (req, res) => {
|
||||
res.json(notebookStore.getSources(req.notebookId));
|
||||
router.get('/', requireNotebookId, (req: NotebookRequest, res: Response) => {
|
||||
res.json(notebookStore.getSources(req.notebookId!));
|
||||
});
|
||||
|
||||
|
||||
function multerUpload(req, res, next) {
|
||||
function multerUpload(req: Request, res: Response, next: NextFunction): void {
|
||||
upload.single('file')(req, res, (err) => {
|
||||
if (err) {
|
||||
if (err.code === 'LIMIT_FILE_SIZE') {
|
||||
return res.status(413).json({ error: 'File too large. Maximum size is 50 MB.' });
|
||||
if ((err as multer.MulterError).code === 'LIMIT_FILE_SIZE') {
|
||||
res.status(413).json({ error: 'File too large. Maximum size is 50 MB.' });
|
||||
return;
|
||||
}
|
||||
return next(err);
|
||||
next(err);
|
||||
return;
|
||||
}
|
||||
next();
|
||||
});
|
||||
@@ -52,73 +48,79 @@ router.post(
|
||||
requireNotebookId,
|
||||
requireNotebook,
|
||||
requireFile,
|
||||
async (req, res, next) => {
|
||||
async (req: NotebookRequest, res: Response, next: NextFunction) => {
|
||||
try {
|
||||
const sourceId = uuidv4();
|
||||
const rawName = Buffer.from(req.file.originalname, 'latin1').toString('utf-8');
|
||||
const rawName = Buffer.from(req.file!.originalname, 'latin1').toString('utf-8');
|
||||
const displayName = cleanFilename(rawName);
|
||||
const text = await parseFile(req.file.buffer, req.file.mimetype, displayName);
|
||||
const text = await parseFile(req.file!.buffer, req.file!.mimetype, displayName);
|
||||
const chunks = chunkText(text, sourceId);
|
||||
|
||||
logger.info({
|
||||
sourceId,
|
||||
filename: displayName,
|
||||
chunkCount: chunks.length,
|
||||
}, 'parsed and chunked source');
|
||||
logger.info(
|
||||
{
|
||||
sourceId,
|
||||
filename: displayName,
|
||||
chunkCount: chunks.length,
|
||||
},
|
||||
'parsed and chunked source'
|
||||
);
|
||||
|
||||
notebookStore.addChunksToNotebook(req.notebookId, chunks);
|
||||
notebookStore.addChunksToNotebook(req.notebookId!, chunks);
|
||||
|
||||
const texts = chunks.map((c) => c.text);
|
||||
const embeddings = await embedTexts(texts, 'document');
|
||||
storeChunkEmbeddings(chunks, embeddings);
|
||||
|
||||
const source = notebookStore.addSource(req.notebookId, {
|
||||
const source = notebookStore.addSource(req.notebookId!, {
|
||||
id: sourceId,
|
||||
name: displayName,
|
||||
mimetype: req.file.mimetype,
|
||||
mimetype: req.file!.mimetype,
|
||||
chunkCount: chunks.length,
|
||||
uploadedAt: new Date().toISOString(),
|
||||
});
|
||||
|
||||
triggerPreGeneration(req.notebookId);
|
||||
triggerPreGeneration(req.notebookId!);
|
||||
|
||||
res.status(201).json(source);
|
||||
} catch (err) {
|
||||
next(err);
|
||||
}
|
||||
},
|
||||
}
|
||||
);
|
||||
|
||||
interface UrlBody {
|
||||
url?: string;
|
||||
}
|
||||
|
||||
router.post(
|
||||
'/url',
|
||||
requireNotebookId,
|
||||
requireNotebook,
|
||||
requireUrl,
|
||||
async (req, res, next) => {
|
||||
async (req: NotebookRequest & Request<unknown, unknown, UrlBody>, res: Response, next: NextFunction) => {
|
||||
try {
|
||||
const { url } = req.body;
|
||||
const sourceId = uuidv4();
|
||||
const isYT = isYoutubeUrl(url);
|
||||
const displayName = isYT
|
||||
? `YouTube: ${url}`
|
||||
: url.replace(/^https?:\/\//, '').slice(0, 60);
|
||||
const isYT = isYoutubeUrl(url!);
|
||||
const displayName = isYT ? `YouTube: ${url}` : url!.replace(/^https?:\/\//, '').slice(0, 60);
|
||||
|
||||
logger.info({ sourceId, url, isYT }, 'processing URL source');
|
||||
|
||||
const text = isYT ? await parseYoutubeUrl(url) : await parseUrl(url);
|
||||
const text = isYT ? await parseYoutubeUrl(url!) : await parseUrl(url!);
|
||||
const chunks = chunkText(text, sourceId);
|
||||
|
||||
if (chunks.length === 0) {
|
||||
return res.status(422).json({ error: 'No usable text could be extracted from the URL' });
|
||||
res.status(422).json({ error: 'No usable text could be extracted from the URL' });
|
||||
return;
|
||||
}
|
||||
|
||||
notebookStore.addChunksToNotebook(req.notebookId, chunks);
|
||||
notebookStore.addChunksToNotebook(req.notebookId!, chunks);
|
||||
|
||||
const texts = chunks.map((c) => c.text);
|
||||
const embeddings = await embedTexts(texts, 'document');
|
||||
storeChunkEmbeddings(chunks, embeddings);
|
||||
|
||||
const source = notebookStore.addSource(req.notebookId, {
|
||||
const source = notebookStore.addSource(req.notebookId!, {
|
||||
id: sourceId,
|
||||
name: displayName,
|
||||
mimetype: isYT ? 'video/youtube' : 'text/html',
|
||||
@@ -126,13 +128,13 @@ router.post(
|
||||
uploadedAt: new Date().toISOString(),
|
||||
});
|
||||
|
||||
triggerPreGeneration(req.notebookId);
|
||||
triggerPreGeneration(req.notebookId!);
|
||||
|
||||
res.status(201).json(source);
|
||||
} catch (err) {
|
||||
next(err);
|
||||
}
|
||||
},
|
||||
}
|
||||
);
|
||||
|
||||
export default router;
|
||||
@@ -20,7 +20,7 @@ type GeneratorFn = (sourceGroups: SourceGroup[]) => Promise<StudyGuide | Faq | E
|
||||
|
||||
const GENERATORS: Record<DocumentType, GeneratorFn> = {
|
||||
'study-guide': generateStudyGuide,
|
||||
faq: generateFaq,
|
||||
'faq': generateFaq,
|
||||
'executive-brief': generateExecutiveBrief,
|
||||
};
|
||||
|
||||
|
||||
@@ -1,52 +0,0 @@
|
||||
'use strict';
|
||||
|
||||
const cache = new Map();
|
||||
|
||||
function buildSourceHash(sources) {
|
||||
return sources
|
||||
.map((s) => s.id)
|
||||
.sort()
|
||||
.join('|');
|
||||
}
|
||||
|
||||
export function getCachedDocument(notebookId, type) {
|
||||
const entry = cache.get(`${notebookId}:${type}`);
|
||||
if (!entry) return null;
|
||||
return entry;
|
||||
}
|
||||
|
||||
export function setCachedDocument(notebookId, type, document, sources) {
|
||||
const key = `${notebookId}:${type}`;
|
||||
cache.set(key, {
|
||||
document,
|
||||
sourceHash: buildSourceHash(sources),
|
||||
createdAt: Date.now(),
|
||||
});
|
||||
}
|
||||
|
||||
export function isFresh(notebookId, type, currentSources) {
|
||||
const entry = cache.get(`${notebookId}:${type}`);
|
||||
if (!entry) return false;
|
||||
return entry.sourceHash === buildSourceHash(currentSources);
|
||||
}
|
||||
|
||||
export function invalidate(notebookId) {
|
||||
for (const key of cache.keys()) {
|
||||
if (key.startsWith(`${notebookId}:`)) {
|
||||
cache.delete(key);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export function markGenerating(notebookId) {
|
||||
const key = `${notebookId}:__generating`;
|
||||
cache.set(key, true);
|
||||
}
|
||||
|
||||
export function clearGenerating(notebookId) {
|
||||
cache.delete(`${notebookId}:__generating`);
|
||||
}
|
||||
|
||||
export function isGenerating(notebookId) {
|
||||
return cache.get(`${notebookId}:__generating`) === true;
|
||||
}
|
||||
@@ -10,7 +10,6 @@ import {
|
||||
} from './documentCacheStore.js';
|
||||
|
||||
describe('documentCacheStore', () => {
|
||||
|
||||
describe('getCachedDocument / setCachedDocument', () => {
|
||||
it('returns null for an uncached entry', () => {
|
||||
expect(getCachedDocument('miss-1', 'faq')).toBeNull();
|
||||
@@ -22,22 +21,22 @@ describe('documentCacheStore', () => {
|
||||
setCachedDocument('rt-1', 'study-guide', doc, sources);
|
||||
|
||||
const entry = getCachedDocument('rt-1', 'study-guide');
|
||||
expect(entry.document).toEqual(doc);
|
||||
expect(entry.sourceHash).toBe('s1|s2');
|
||||
expect(entry.createdAt).toBeTypeOf('number');
|
||||
expect(entry!.document).toEqual(doc);
|
||||
expect(entry!.sourceHash).toBe('s1|s2');
|
||||
expect(entry!.createdAt).toBeTypeOf('number');
|
||||
});
|
||||
|
||||
it('overwrites an existing entry for the same key', () => {
|
||||
setCachedDocument('ow-1', 'faq', { v: 1 }, [{ id: 'a' }]);
|
||||
setCachedDocument('ow-1', 'faq', { v: 2 }, [{ id: 'b' }]);
|
||||
expect(getCachedDocument('ow-1', 'faq').document).toEqual({ v: 2 });
|
||||
expect(getCachedDocument('ow-1', 'faq')!.document).toEqual({ v: 2 });
|
||||
});
|
||||
|
||||
it('keeps entries for different types separate', () => {
|
||||
setCachedDocument('sep-1', 'faq', { kind: 'faq' }, [{ id: 'x' }]);
|
||||
setCachedDocument('sep-1', 'study-guide', { kind: 'sg' }, [{ id: 'x' }]);
|
||||
expect(getCachedDocument('sep-1', 'faq').document.kind).toBe('faq');
|
||||
expect(getCachedDocument('sep-1', 'study-guide').document.kind).toBe('sg');
|
||||
expect((getCachedDocument('sep-1', 'faq')!.document as { kind: string }).kind).toBe('faq');
|
||||
expect((getCachedDocument('sep-1', 'study-guide')!.document as { kind: string }).kind).toBe('sg');
|
||||
});
|
||||
});
|
||||
|
||||
@@ -71,7 +70,7 @@ describe('documentCacheStore', () => {
|
||||
setCachedDocument('inv-b', 'faq', { b: 2 }, [{ id: 'x' }]);
|
||||
invalidate('inv-a');
|
||||
expect(getCachedDocument('inv-a', 'faq')).toBeNull();
|
||||
expect(getCachedDocument('inv-b', 'faq').document).toEqual({ b: 2 });
|
||||
expect(getCachedDocument('inv-b', 'faq')!.document).toEqual({ b: 2 });
|
||||
});
|
||||
});
|
||||
|
||||
67
server/src/stores/documentCacheStore.ts
Normal file
67
server/src/stores/documentCacheStore.ts
Normal file
@@ -0,0 +1,67 @@
|
||||
'use strict';
|
||||
|
||||
interface Source {
|
||||
id: string;
|
||||
}
|
||||
|
||||
interface CacheEntry {
|
||||
document: unknown;
|
||||
sourceHash: string;
|
||||
createdAt: number;
|
||||
}
|
||||
|
||||
const cache = new Map<string, CacheEntry | boolean>();
|
||||
|
||||
function buildSourceHash(sources: Source[]): string {
|
||||
return sources
|
||||
.map((s) => s.id)
|
||||
.sort()
|
||||
.join('|');
|
||||
}
|
||||
|
||||
export function getCachedDocument(notebookId: string, type: string): CacheEntry | null {
|
||||
const entry = cache.get(`${notebookId}:${type}`);
|
||||
if (!entry || typeof entry === 'boolean') return null;
|
||||
return entry;
|
||||
}
|
||||
|
||||
export function setCachedDocument(
|
||||
notebookId: string,
|
||||
type: string,
|
||||
document: unknown,
|
||||
sources: Source[]
|
||||
): void {
|
||||
const key = `${notebookId}:${type}`;
|
||||
cache.set(key, {
|
||||
document,
|
||||
sourceHash: buildSourceHash(sources),
|
||||
createdAt: Date.now(),
|
||||
});
|
||||
}
|
||||
|
||||
export function isFresh(notebookId: string, type: string, currentSources: Source[]): boolean {
|
||||
const entry = cache.get(`${notebookId}:${type}`);
|
||||
if (!entry || typeof entry === 'boolean') return false;
|
||||
return entry.sourceHash === buildSourceHash(currentSources);
|
||||
}
|
||||
|
||||
export function invalidate(notebookId: string): void {
|
||||
for (const key of cache.keys()) {
|
||||
if (key.startsWith(`${notebookId}:`)) {
|
||||
cache.delete(key);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export function markGenerating(notebookId: string): void {
|
||||
const key = `${notebookId}:__generating`;
|
||||
cache.set(key, true);
|
||||
}
|
||||
|
||||
export function clearGenerating(notebookId: string): void {
|
||||
cache.delete(`${notebookId}:__generating`);
|
||||
}
|
||||
|
||||
export function isGenerating(notebookId: string): boolean {
|
||||
return cache.get(`${notebookId}:__generating`) === true;
|
||||
}
|
||||
@@ -2,7 +2,7 @@ import { describe, it, expect, beforeEach } from 'vitest';
|
||||
import { NotebookStore } from './notebookStore.js';
|
||||
|
||||
describe('notebookStore', () => {
|
||||
let store;
|
||||
let store: NotebookStore;
|
||||
|
||||
beforeEach(() => {
|
||||
store = new NotebookStore();
|
||||
@@ -15,7 +15,7 @@ describe('notebookStore', () => {
|
||||
|
||||
const found = store.getNotebook('nb-1');
|
||||
expect(found).not.toBeNull();
|
||||
expect(found.name).toBe('Test');
|
||||
expect(found!.name).toBe('Test');
|
||||
});
|
||||
|
||||
it('lists all notebooks without chunks', () => {
|
||||
@@ -51,8 +51,8 @@ describe('notebookStore', () => {
|
||||
it('adds and retrieves chunks', () => {
|
||||
store.createNotebook({ id: 'nb-1', name: 'A', createdAt: '2026-01-01' });
|
||||
store.addChunksToNotebook('nb-1', [
|
||||
{ id: 'c-1', text: 'chunk one' },
|
||||
{ id: 'c-2', text: 'chunk two' },
|
||||
{ id: 'c-1', text: 'chunk one', sourceId: 's-1', index: 0 },
|
||||
{ id: 'c-2', text: 'chunk two', sourceId: 's-1', index: 1 },
|
||||
]);
|
||||
const chunks = store.getChunksForNotebook('nb-1');
|
||||
expect(chunks).toHaveLength(2);
|
||||
@@ -1,62 +1,100 @@
|
||||
'use strict';
|
||||
|
||||
export interface TextChunk {
|
||||
id: string;
|
||||
text: string;
|
||||
sourceId: string;
|
||||
index: number;
|
||||
}
|
||||
|
||||
export interface Source {
|
||||
id: string;
|
||||
name: string;
|
||||
mimetype?: string;
|
||||
chunkCount?: number;
|
||||
uploadedAt?: string;
|
||||
}
|
||||
|
||||
export interface Notebook {
|
||||
id: string;
|
||||
name: string;
|
||||
createdAt: string;
|
||||
sources: Source[];
|
||||
chunks: TextChunk[];
|
||||
}
|
||||
|
||||
export interface NotebookMeta {
|
||||
id: string;
|
||||
name: string;
|
||||
createdAt: string;
|
||||
}
|
||||
|
||||
export interface SourceGroup {
|
||||
docIndex: number;
|
||||
sourceId: string;
|
||||
name: string;
|
||||
chunks: TextChunk[];
|
||||
}
|
||||
|
||||
export class NotebookStore {
|
||||
constructor() {
|
||||
this.notebooks = new Map();
|
||||
private notebooks = new Map<string, Notebook>();
|
||||
|
||||
getAllNotebooks(): NotebookMeta[] {
|
||||
return Array.from(this.notebooks.values()).map(({ id, name, createdAt }) => ({
|
||||
id,
|
||||
name,
|
||||
createdAt,
|
||||
}));
|
||||
}
|
||||
|
||||
getAllNotebooks() {
|
||||
return Array.from(this.notebooks.values()).map(({ chunks, ...meta }) => meta);
|
||||
}
|
||||
|
||||
getNotebook(id) {
|
||||
getNotebook(id: string): Notebook | null {
|
||||
return this.notebooks.get(id) || null;
|
||||
}
|
||||
|
||||
createNotebook(notebook) {
|
||||
const record = { ...notebook, sources: [], chunks: [] };
|
||||
createNotebook(notebook: { id: string; name: string; createdAt: string }): NotebookMeta {
|
||||
const record: Notebook = { ...notebook, sources: [], chunks: [] };
|
||||
this.notebooks.set(record.id, record);
|
||||
return { id: record.id, name: record.name, createdAt: record.createdAt };
|
||||
}
|
||||
|
||||
deleteNotebook(id) {
|
||||
deleteNotebook(id: string): boolean {
|
||||
return this.notebooks.delete(id);
|
||||
}
|
||||
|
||||
addSource(notebookId, source) {
|
||||
addSource(notebookId: string, source: Source): Source | null {
|
||||
const notebook = this.notebooks.get(notebookId);
|
||||
if (!notebook) return null;
|
||||
notebook.sources.push(source);
|
||||
return source;
|
||||
}
|
||||
|
||||
getSources(notebookId) {
|
||||
getSources(notebookId: string): Source[] {
|
||||
const notebook = this.notebooks.get(notebookId);
|
||||
if (!notebook) return [];
|
||||
return notebook.sources;
|
||||
}
|
||||
|
||||
addChunksToNotebook(notebookId, chunks) {
|
||||
addChunksToNotebook(notebookId: string, chunks: TextChunk[]): Notebook | null {
|
||||
const notebook = this.notebooks.get(notebookId);
|
||||
if (!notebook) return null;
|
||||
notebook.chunks = notebook.chunks.concat(chunks);
|
||||
return notebook;
|
||||
}
|
||||
|
||||
getChunksForNotebook(notebookId) {
|
||||
getChunksForNotebook(notebookId: string): TextChunk[] {
|
||||
const notebook = this.notebooks.get(notebookId);
|
||||
if (!notebook) return [];
|
||||
return notebook.chunks;
|
||||
}
|
||||
|
||||
buildSourceGroups(notebookId, chunks) {
|
||||
buildSourceGroups(notebookId: string, chunks: TextChunk[]): SourceGroup[] {
|
||||
const sources = this.getSources(notebookId);
|
||||
const sourceIndexMap = new Map();
|
||||
const sourceIndexMap = new Map<string, number>();
|
||||
sources.forEach((src, i) => {
|
||||
sourceIndexMap.set(src.id, i + 1);
|
||||
});
|
||||
|
||||
const groupMap = new Map();
|
||||
const groupMap = new Map<string, SourceGroup>();
|
||||
for (const chunk of chunks) {
|
||||
const docIndex = sourceIndexMap.get(chunk.sourceId) || 0;
|
||||
if (!groupMap.has(chunk.sourceId)) {
|
||||
@@ -68,7 +106,7 @@ export class NotebookStore {
|
||||
chunks: [],
|
||||
});
|
||||
}
|
||||
groupMap.get(chunk.sourceId).chunks.push(chunk);
|
||||
groupMap.get(chunk.sourceId)!.chunks.push(chunk);
|
||||
}
|
||||
|
||||
return Array.from(groupMap.values()).sort((a, b) => a.docIndex - b.docIndex);
|
||||
@@ -1,21 +0,0 @@
|
||||
'use strict';
|
||||
|
||||
const vectors = new Map();
|
||||
|
||||
export function storeVector(chunkId, embedding) {
|
||||
vectors.set(chunkId, new Float32Array(embedding));
|
||||
}
|
||||
|
||||
export function getVector(chunkId) {
|
||||
return vectors.get(chunkId) || null;
|
||||
}
|
||||
|
||||
export function getAllVectors() {
|
||||
return vectors;
|
||||
}
|
||||
|
||||
export function deleteVectorsForChunks(chunkIds) {
|
||||
for (const id of chunkIds) {
|
||||
vectors.delete(id);
|
||||
}
|
||||
}
|
||||
21
server/src/stores/vectorStore.ts
Normal file
21
server/src/stores/vectorStore.ts
Normal file
@@ -0,0 +1,21 @@
|
||||
'use strict';
|
||||
|
||||
const vectors = new Map<string, Float32Array>();
|
||||
|
||||
export function storeVector(chunkId: string, embedding: number[]): void {
|
||||
vectors.set(chunkId, new Float32Array(embedding));
|
||||
}
|
||||
|
||||
export function getVector(chunkId: string): Float32Array | null {
|
||||
return vectors.get(chunkId) || null;
|
||||
}
|
||||
|
||||
export function getAllVectors(): Map<string, Float32Array> {
|
||||
return vectors;
|
||||
}
|
||||
|
||||
export function deleteVectorsForChunks(chunkIds: string[]): void {
|
||||
for (const id of chunkIds) {
|
||||
vectors.delete(id);
|
||||
}
|
||||
}
|
||||
@@ -1,13 +1,15 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"allowJs": true, // Allow JavaScript files to be imported and compiled
|
||||
"allowJs": true,
|
||||
"checkJs": true,
|
||||
"target": "es2022",
|
||||
"module": "commonjs",
|
||||
"module": "NodeNext",
|
||||
"moduleResolution": "NodeNext",
|
||||
"outDir": "./dist",
|
||||
"rootDir": "./src", // Root of input files
|
||||
"rootDir": "./src",
|
||||
"strict": true,
|
||||
"skipLibCheck": true // Skip checking .d.ts files for faster builds
|
||||
"skipLibCheck": true,
|
||||
"esModuleInterop": true
|
||||
},
|
||||
"include": [
|
||||
"src/**/*"
|
||||
|
||||
Reference in New Issue
Block a user