continue refactor of backend to utilize ts, deprecate js
This commit is contained in:
165
server/package-lock.json
generated
165
server/package-lock.json
generated
@@ -22,10 +22,15 @@
|
|||||||
"uuid": "^11.1.0"
|
"uuid": "^11.1.0"
|
||||||
},
|
},
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
|
"@types/cors": "^2.8.19",
|
||||||
|
"@types/express": "^5.0.6",
|
||||||
|
"@types/multer": "^2.1.0",
|
||||||
"@types/node": "^25.6.2",
|
"@types/node": "^25.6.2",
|
||||||
"@types/pdf-parse": "^1.1.5",
|
"@types/pdf-parse": "^1.1.5",
|
||||||
|
"@types/uuid": "^10.0.0",
|
||||||
"eslint": "^9.22.0",
|
"eslint": "^9.22.0",
|
||||||
"prettier": "^3.5.3",
|
"prettier": "^3.5.3",
|
||||||
|
"tsx": "^4.21.0",
|
||||||
"typescript": "^6.0.3",
|
"typescript": "^6.0.3",
|
||||||
"vitest": "^3.0.0"
|
"vitest": "^3.0.0"
|
||||||
},
|
},
|
||||||
@@ -1063,6 +1068,17 @@
|
|||||||
"win32"
|
"win32"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
|
"node_modules/@types/body-parser": {
|
||||||
|
"version": "1.19.6",
|
||||||
|
"resolved": "https://registry.npmjs.org/@types/body-parser/-/body-parser-1.19.6.tgz",
|
||||||
|
"integrity": "sha512-HLFeCYgz89uk22N5Qg3dvGvsv46B8GLvKKo1zKG4NybA8U2DiEO3w9lqGg29t/tfLRJpJ6iQxnVw4OnB7MoM9g==",
|
||||||
|
"dev": true,
|
||||||
|
"license": "MIT",
|
||||||
|
"dependencies": {
|
||||||
|
"@types/connect": "*",
|
||||||
|
"@types/node": "*"
|
||||||
|
}
|
||||||
|
},
|
||||||
"node_modules/@types/chai": {
|
"node_modules/@types/chai": {
|
||||||
"version": "5.2.3",
|
"version": "5.2.3",
|
||||||
"resolved": "https://registry.npmjs.org/@types/chai/-/chai-5.2.3.tgz",
|
"resolved": "https://registry.npmjs.org/@types/chai/-/chai-5.2.3.tgz",
|
||||||
@@ -1074,6 +1090,26 @@
|
|||||||
"assertion-error": "^2.0.1"
|
"assertion-error": "^2.0.1"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"node_modules/@types/connect": {
|
||||||
|
"version": "3.4.38",
|
||||||
|
"resolved": "https://registry.npmjs.org/@types/connect/-/connect-3.4.38.tgz",
|
||||||
|
"integrity": "sha512-K6uROf1LD88uDQqJCktA4yzL1YYAK6NgfsI0v/mTgyPKWsX1CnJ0XPSDhViejru1GcRkLWb8RlzFYJRqGUbaug==",
|
||||||
|
"dev": true,
|
||||||
|
"license": "MIT",
|
||||||
|
"dependencies": {
|
||||||
|
"@types/node": "*"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/@types/cors": {
|
||||||
|
"version": "2.8.19",
|
||||||
|
"resolved": "https://registry.npmjs.org/@types/cors/-/cors-2.8.19.tgz",
|
||||||
|
"integrity": "sha512-mFNylyeyqN93lfe/9CSxOGREz8cpzAhH+E93xJ4xWQf62V8sQ/24reV2nyzUWM6H6Xji+GGHpkbLe7pVoUEskg==",
|
||||||
|
"dev": true,
|
||||||
|
"license": "MIT",
|
||||||
|
"dependencies": {
|
||||||
|
"@types/node": "*"
|
||||||
|
}
|
||||||
|
},
|
||||||
"node_modules/@types/deep-eql": {
|
"node_modules/@types/deep-eql": {
|
||||||
"version": "4.0.2",
|
"version": "4.0.2",
|
||||||
"resolved": "https://registry.npmjs.org/@types/deep-eql/-/deep-eql-4.0.2.tgz",
|
"resolved": "https://registry.npmjs.org/@types/deep-eql/-/deep-eql-4.0.2.tgz",
|
||||||
@@ -1088,6 +1124,38 @@
|
|||||||
"dev": true,
|
"dev": true,
|
||||||
"license": "MIT"
|
"license": "MIT"
|
||||||
},
|
},
|
||||||
|
"node_modules/@types/express": {
|
||||||
|
"version": "5.0.6",
|
||||||
|
"resolved": "https://registry.npmjs.org/@types/express/-/express-5.0.6.tgz",
|
||||||
|
"integrity": "sha512-sKYVuV7Sv9fbPIt/442koC7+IIwK5olP1KWeD88e/idgoJqDm3JV/YUiPwkoKK92ylff2MGxSz1CSjsXelx0YA==",
|
||||||
|
"dev": true,
|
||||||
|
"license": "MIT",
|
||||||
|
"dependencies": {
|
||||||
|
"@types/body-parser": "*",
|
||||||
|
"@types/express-serve-static-core": "^5.0.0",
|
||||||
|
"@types/serve-static": "^2"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/@types/express-serve-static-core": {
|
||||||
|
"version": "5.1.1",
|
||||||
|
"resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-5.1.1.tgz",
|
||||||
|
"integrity": "sha512-v4zIMr/cX7/d2BpAEX3KNKL/JrT1s43s96lLvvdTmza1oEvDudCqK9aF/djc/SWgy8Yh0h30TZx5VpzqFCxk5A==",
|
||||||
|
"dev": true,
|
||||||
|
"license": "MIT",
|
||||||
|
"dependencies": {
|
||||||
|
"@types/node": "*",
|
||||||
|
"@types/qs": "*",
|
||||||
|
"@types/range-parser": "*",
|
||||||
|
"@types/send": "*"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/@types/http-errors": {
|
||||||
|
"version": "2.0.5",
|
||||||
|
"resolved": "https://registry.npmjs.org/@types/http-errors/-/http-errors-2.0.5.tgz",
|
||||||
|
"integrity": "sha512-r8Tayk8HJnX0FztbZN7oVqGccWgw98T/0neJphO91KkmOzug1KkofZURD4UaD5uH8AqcFLfdPErnBod0u71/qg==",
|
||||||
|
"dev": true,
|
||||||
|
"license": "MIT"
|
||||||
|
},
|
||||||
"node_modules/@types/json-schema": {
|
"node_modules/@types/json-schema": {
|
||||||
"version": "7.0.15",
|
"version": "7.0.15",
|
||||||
"resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.15.tgz",
|
"resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.15.tgz",
|
||||||
@@ -1095,13 +1163,22 @@
|
|||||||
"dev": true,
|
"dev": true,
|
||||||
"license": "MIT"
|
"license": "MIT"
|
||||||
},
|
},
|
||||||
|
"node_modules/@types/multer": {
|
||||||
|
"version": "2.1.0",
|
||||||
|
"resolved": "https://registry.npmjs.org/@types/multer/-/multer-2.1.0.tgz",
|
||||||
|
"integrity": "sha512-zYZb0+nJhOHtPpGDb3vqPjwpdeGlGC157VpkqNQL+UU2qwoacoQ7MpsAmUptI/0Oa127X32JzWDqQVEXp2RcIA==",
|
||||||
|
"dev": true,
|
||||||
|
"license": "MIT",
|
||||||
|
"dependencies": {
|
||||||
|
"@types/express": "*"
|
||||||
|
}
|
||||||
|
},
|
||||||
"node_modules/@types/node": {
|
"node_modules/@types/node": {
|
||||||
"version": "25.6.2",
|
"version": "25.6.2",
|
||||||
"resolved": "https://registry.npmjs.org/@types/node/-/node-25.6.2.tgz",
|
"resolved": "https://registry.npmjs.org/@types/node/-/node-25.6.2.tgz",
|
||||||
"integrity": "sha512-sokuT28dxf9JT5Kady1fsXOvI4HVpjZa95NKT5y9PNTIrs2AsobR4GFAA90ZG8M+nxVRLysCXsVj6eGC7Vbrlw==",
|
"integrity": "sha512-sokuT28dxf9JT5Kady1fsXOvI4HVpjZa95NKT5y9PNTIrs2AsobR4GFAA90ZG8M+nxVRLysCXsVj6eGC7Vbrlw==",
|
||||||
"dev": true,
|
"dev": true,
|
||||||
"license": "MIT",
|
"license": "MIT",
|
||||||
"peer": true,
|
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"undici-types": "~7.19.0"
|
"undici-types": "~7.19.0"
|
||||||
}
|
}
|
||||||
@@ -1116,6 +1193,48 @@
|
|||||||
"@types/node": "*"
|
"@types/node": "*"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"node_modules/@types/qs": {
|
||||||
|
"version": "6.15.1",
|
||||||
|
"resolved": "https://registry.npmjs.org/@types/qs/-/qs-6.15.1.tgz",
|
||||||
|
"integrity": "sha512-GZHUBZR9hckSUhrxmp1nG6NwdpM9fCunJwyThLW1X3AyHgd9IlHb6VANpQQqDr2o/qQp6McZ3y/IA2rVzKzSbw==",
|
||||||
|
"dev": true,
|
||||||
|
"license": "MIT"
|
||||||
|
},
|
||||||
|
"node_modules/@types/range-parser": {
|
||||||
|
"version": "1.2.7",
|
||||||
|
"resolved": "https://registry.npmjs.org/@types/range-parser/-/range-parser-1.2.7.tgz",
|
||||||
|
"integrity": "sha512-hKormJbkJqzQGhziax5PItDUTMAM9uE2XXQmM37dyd4hVM+5aVl7oVxMVUiVQn2oCQFN/LKCZdvSM0pFRqbSmQ==",
|
||||||
|
"dev": true,
|
||||||
|
"license": "MIT"
|
||||||
|
},
|
||||||
|
"node_modules/@types/send": {
|
||||||
|
"version": "1.2.1",
|
||||||
|
"resolved": "https://registry.npmjs.org/@types/send/-/send-1.2.1.tgz",
|
||||||
|
"integrity": "sha512-arsCikDvlU99zl1g69TcAB3mzZPpxgw0UQnaHeC1Nwb015xp8bknZv5rIfri9xTOcMuaVgvabfIRA7PSZVuZIQ==",
|
||||||
|
"dev": true,
|
||||||
|
"license": "MIT",
|
||||||
|
"dependencies": {
|
||||||
|
"@types/node": "*"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/@types/serve-static": {
|
||||||
|
"version": "2.2.0",
|
||||||
|
"resolved": "https://registry.npmjs.org/@types/serve-static/-/serve-static-2.2.0.tgz",
|
||||||
|
"integrity": "sha512-8mam4H1NHLtu7nmtalF7eyBH14QyOASmcxHhSfEoRyr0nP/YdoesEtU+uSRvMe96TW/HPTtkoKqQLl53N7UXMQ==",
|
||||||
|
"dev": true,
|
||||||
|
"license": "MIT",
|
||||||
|
"dependencies": {
|
||||||
|
"@types/http-errors": "*",
|
||||||
|
"@types/node": "*"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/@types/uuid": {
|
||||||
|
"version": "10.0.0",
|
||||||
|
"resolved": "https://registry.npmjs.org/@types/uuid/-/uuid-10.0.0.tgz",
|
||||||
|
"integrity": "sha512-7gqG38EyHgyP1S+7+xomFtL+ZNHcKv6DwNaCZmJmo1vgMugyF3TCnXVg4t1uk89mLNwnLtnY3TpOpCOyp1/xHQ==",
|
||||||
|
"dev": true,
|
||||||
|
"license": "MIT"
|
||||||
|
},
|
||||||
"node_modules/@vitest/expect": {
|
"node_modules/@vitest/expect": {
|
||||||
"version": "3.2.4",
|
"version": "3.2.4",
|
||||||
"resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-3.2.4.tgz",
|
"resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-3.2.4.tgz",
|
||||||
@@ -2525,6 +2644,19 @@
|
|||||||
"node": ">= 0.4"
|
"node": ">= 0.4"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"node_modules/get-tsconfig": {
|
||||||
|
"version": "4.14.0",
|
||||||
|
"resolved": "https://registry.npmjs.org/get-tsconfig/-/get-tsconfig-4.14.0.tgz",
|
||||||
|
"integrity": "sha512-yTb+8DXzDREzgvYmh6s9vHsSVCHeC0G3PI5bEXNBHtmshPnO+S5O7qgLEOn0I5QvMy6kpZN8K1NKGyilLb93wA==",
|
||||||
|
"dev": true,
|
||||||
|
"license": "MIT",
|
||||||
|
"dependencies": {
|
||||||
|
"resolve-pkg-maps": "^1.0.0"
|
||||||
|
},
|
||||||
|
"funding": {
|
||||||
|
"url": "https://github.com/privatenumber/get-tsconfig?sponsor=1"
|
||||||
|
}
|
||||||
|
},
|
||||||
"node_modules/glob-parent": {
|
"node_modules/glob-parent": {
|
||||||
"version": "6.0.2",
|
"version": "6.0.2",
|
||||||
"resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz",
|
"resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz",
|
||||||
@@ -3704,6 +3836,16 @@
|
|||||||
"node": ">=4"
|
"node": ">=4"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"node_modules/resolve-pkg-maps": {
|
||||||
|
"version": "1.0.0",
|
||||||
|
"resolved": "https://registry.npmjs.org/resolve-pkg-maps/-/resolve-pkg-maps-1.0.0.tgz",
|
||||||
|
"integrity": "sha512-seS2Tj26TBVOC2NIc2rOe2y2ZO7efxITtLZcGSOnHHNOQ7CkiUBfw0Iw2ck6xkIhPwLhKNLS8BO+hEpngQlqzw==",
|
||||||
|
"dev": true,
|
||||||
|
"license": "MIT",
|
||||||
|
"funding": {
|
||||||
|
"url": "https://github.com/privatenumber/resolve-pkg-maps?sponsor=1"
|
||||||
|
}
|
||||||
|
},
|
||||||
"node_modules/rollup": {
|
"node_modules/rollup": {
|
||||||
"version": "4.59.0",
|
"version": "4.59.0",
|
||||||
"resolved": "https://registry.npmjs.org/rollup/-/rollup-4.59.0.tgz",
|
"resolved": "https://registry.npmjs.org/rollup/-/rollup-4.59.0.tgz",
|
||||||
@@ -4172,6 +4314,27 @@
|
|||||||
"integrity": "sha512-FPAhNPFMrkwz76P7cdjdmiShwMynZYN6SgOujD1urY4oNm80Ou9oMdmbR45LotcKOXoy7wSmHkRFE6Mxbrhefw==",
|
"integrity": "sha512-FPAhNPFMrkwz76P7cdjdmiShwMynZYN6SgOujD1urY4oNm80Ou9oMdmbR45LotcKOXoy7wSmHkRFE6Mxbrhefw==",
|
||||||
"license": "MIT"
|
"license": "MIT"
|
||||||
},
|
},
|
||||||
|
"node_modules/tsx": {
|
||||||
|
"version": "4.21.0",
|
||||||
|
"resolved": "https://registry.npmjs.org/tsx/-/tsx-4.21.0.tgz",
|
||||||
|
"integrity": "sha512-5C1sg4USs1lfG0GFb2RLXsdpXqBSEhAaA/0kPL01wxzpMqLILNxIxIOKiILz+cdg/pLnOUxFYOR5yhHU666wbw==",
|
||||||
|
"dev": true,
|
||||||
|
"license": "MIT",
|
||||||
|
"peer": true,
|
||||||
|
"dependencies": {
|
||||||
|
"esbuild": "~0.27.0",
|
||||||
|
"get-tsconfig": "^4.7.5"
|
||||||
|
},
|
||||||
|
"bin": {
|
||||||
|
"tsx": "dist/cli.mjs"
|
||||||
|
},
|
||||||
|
"engines": {
|
||||||
|
"node": ">=18.0.0"
|
||||||
|
},
|
||||||
|
"optionalDependencies": {
|
||||||
|
"fsevents": "~2.3.3"
|
||||||
|
}
|
||||||
|
},
|
||||||
"node_modules/type-check": {
|
"node_modules/type-check": {
|
||||||
"version": "0.4.0",
|
"version": "0.4.0",
|
||||||
"resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz",
|
"resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz",
|
||||||
|
|||||||
@@ -7,7 +7,7 @@
|
|||||||
},
|
},
|
||||||
"type": "module",
|
"type": "module",
|
||||||
"scripts": {
|
"scripts": {
|
||||||
"start": "node src/index.js",
|
"start": "tsx src/index.ts",
|
||||||
"test": "vitest run",
|
"test": "vitest run",
|
||||||
"lint": "eslint src/",
|
"lint": "eslint src/",
|
||||||
"fmt": "prettier --write .",
|
"fmt": "prettier --write .",
|
||||||
@@ -28,10 +28,15 @@
|
|||||||
"uuid": "^11.1.0"
|
"uuid": "^11.1.0"
|
||||||
},
|
},
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
|
"@types/cors": "^2.8.19",
|
||||||
|
"@types/express": "^5.0.6",
|
||||||
|
"@types/multer": "^2.1.0",
|
||||||
"@types/node": "^25.6.2",
|
"@types/node": "^25.6.2",
|
||||||
"@types/pdf-parse": "^1.1.5",
|
"@types/pdf-parse": "^1.1.5",
|
||||||
|
"@types/uuid": "^10.0.0",
|
||||||
"eslint": "^9.22.0",
|
"eslint": "^9.22.0",
|
||||||
"prettier": "^3.5.3",
|
"prettier": "^3.5.3",
|
||||||
|
"tsx": "^4.21.0",
|
||||||
"typescript": "^6.0.3",
|
"typescript": "^6.0.3",
|
||||||
"vitest": "^3.0.0"
|
"vitest": "^3.0.0"
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
'use strict';
|
'use strict';
|
||||||
|
|
||||||
import 'dotenv/config';
|
import 'dotenv/config';
|
||||||
import express from 'express';
|
import express, { type Request, type Response, type NextFunction } from 'express';
|
||||||
import cors from 'cors';
|
import cors from 'cors';
|
||||||
import logger from './logger.js';
|
import logger from './logger.js';
|
||||||
import notebookRoutes from './routes/notebooks.js';
|
import notebookRoutes from './routes/notebooks.js';
|
||||||
@@ -21,11 +21,10 @@ for (const key of REQUIRED_KEYS) {
|
|||||||
|
|
||||||
const app = express();
|
const app = express();
|
||||||
|
|
||||||
// CORS: wide-open for local dev. In production, lock this to specific origins.
|
|
||||||
app.use(cors());
|
app.use(cors());
|
||||||
app.use(express.json());
|
app.use(express.json());
|
||||||
|
|
||||||
app.use((req, res, next) => {
|
app.use((req: Request, res: Response, next: NextFunction) => {
|
||||||
const start = Date.now();
|
const start = Date.now();
|
||||||
res.on('finish', () => {
|
res.on('finish', () => {
|
||||||
logger.info({
|
logger.info({
|
||||||
@@ -38,7 +37,7 @@ app.use((req, res, next) => {
|
|||||||
next();
|
next();
|
||||||
});
|
});
|
||||||
|
|
||||||
app.get('/.well-known/healthcheck', (req, res) => {
|
app.get('/.well-known/healthcheck', (_req: Request, res: Response) => {
|
||||||
res.json({
|
res.json({
|
||||||
service: 'notebook-clone',
|
service: 'notebook-clone',
|
||||||
status: 'ok',
|
status: 'ok',
|
||||||
@@ -52,7 +51,11 @@ app.use('/api/query', queryRoutes);
|
|||||||
app.use('/api/citation-detail', citationDetailRoutes);
|
app.use('/api/citation-detail', citationDetailRoutes);
|
||||||
app.use('/api/documents', documentRoutes);
|
app.use('/api/documents', documentRoutes);
|
||||||
|
|
||||||
app.use((err, req, res, _next) => {
|
interface ErrorWithStatus extends Error {
|
||||||
|
status?: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
app.use((err: ErrorWithStatus, req: Request, res: Response, _next: NextFunction) => {
|
||||||
logger.error({ err, method: req.method, url: req.originalUrl });
|
logger.error({ err, method: req.method, url: req.originalUrl });
|
||||||
res.status(err.status || 500).json({
|
res.status(err.status || 500).json({
|
||||||
error: err.message || 'internal server error',
|
error: err.message || 'internal server error',
|
||||||
@@ -6,9 +6,7 @@ const DEBUG = !!process.env.DEBUG;
|
|||||||
|
|
||||||
const logger = pino({
|
const logger = pino({
|
||||||
level: DEBUG ? 'debug' : 'info',
|
level: DEBUG ? 'debug' : 'info',
|
||||||
transport: process.stdout.isTTY
|
transport: process.stdout.isTTY ? { target: 'pino-pretty' } : undefined,
|
||||||
? { target: 'pino-pretty' }
|
|
||||||
: undefined,
|
|
||||||
});
|
});
|
||||||
|
|
||||||
export default logger;
|
export default logger;
|
||||||
@@ -1,44 +0,0 @@
|
|||||||
'use strict';
|
|
||||||
|
|
||||||
import * as notebookStore from '../stores/notebookStore.js';
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Ensures notebookId is present in req.body or req.query.
|
|
||||||
* Normalises the value onto req.notebookId for downstream handlers.
|
|
||||||
*/
|
|
||||||
export function requireNotebookId(req, res, next) {
|
|
||||||
const notebookId = req.body?.notebookId ?? req.query?.notebookId;
|
|
||||||
if (!notebookId) {
|
|
||||||
return res.status(400).json({ error: 'notebookId is required' });
|
|
||||||
}
|
|
||||||
req.notebookId = notebookId;
|
|
||||||
next();
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Must run after requireNotebookId.
|
|
||||||
* Loads the notebook from the store and attaches it as req.notebook.
|
|
||||||
* Returns 404 if the notebook doesn't exist.
|
|
||||||
*/
|
|
||||||
export function requireNotebook(req, res, next) {
|
|
||||||
const notebook = notebookStore.getNotebook(req.notebookId);
|
|
||||||
if (!notebook) {
|
|
||||||
return res.status(404).json({ error: 'notebook not found' });
|
|
||||||
}
|
|
||||||
req.notebook = notebook;
|
|
||||||
next();
|
|
||||||
}
|
|
||||||
|
|
||||||
export function requireFile(req, res, next) {
|
|
||||||
if (!req.file) {
|
|
||||||
return res.status(400).json({ error: 'file is required' });
|
|
||||||
}
|
|
||||||
next();
|
|
||||||
}
|
|
||||||
|
|
||||||
export function requireUrl(req, res, next) {
|
|
||||||
if (!req.body?.url) {
|
|
||||||
return res.status(400).json({ error: 'url is required' });
|
|
||||||
}
|
|
||||||
next();
|
|
||||||
}
|
|
||||||
46
server/src/middleware/validation.ts
Normal file
46
server/src/middleware/validation.ts
Normal file
@@ -0,0 +1,46 @@
|
|||||||
|
'use strict';
|
||||||
|
|
||||||
|
import type { Request, Response, NextFunction } from 'express';
|
||||||
|
import * as notebookStore from '../stores/notebookStore.js';
|
||||||
|
import type { Notebook } from '../stores/notebookStore.js';
|
||||||
|
|
||||||
|
export interface NotebookRequest extends Request {
|
||||||
|
notebookId?: string;
|
||||||
|
notebook?: Notebook;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function requireNotebookId(req: NotebookRequest, res: Response, next: NextFunction): void {
|
||||||
|
const notebookId = (req.body?.notebookId ?? req.query?.notebookId) as string | undefined;
|
||||||
|
if (!notebookId) {
|
||||||
|
res.status(400).json({ error: 'notebookId is required' });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
req.notebookId = notebookId;
|
||||||
|
next();
|
||||||
|
}
|
||||||
|
|
||||||
|
export function requireNotebook(req: NotebookRequest, res: Response, next: NextFunction): void {
|
||||||
|
const notebook = notebookStore.getNotebook(req.notebookId!);
|
||||||
|
if (!notebook) {
|
||||||
|
res.status(404).json({ error: 'notebook not found' });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
req.notebook = notebook;
|
||||||
|
next();
|
||||||
|
}
|
||||||
|
|
||||||
|
export function requireFile(req: Request, res: Response, next: NextFunction): void {
|
||||||
|
if (!req.file) {
|
||||||
|
res.status(400).json({ error: 'file is required' });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
next();
|
||||||
|
}
|
||||||
|
|
||||||
|
export function requireUrl(req: Request, res: Response, next: NextFunction): void {
|
||||||
|
if (!req.body?.url) {
|
||||||
|
res.status(400).json({ error: 'url is required' });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
next();
|
||||||
|
}
|
||||||
@@ -1,19 +1,27 @@
|
|||||||
'use strict';
|
'use strict';
|
||||||
|
|
||||||
import { Router } from 'express';
|
import { Router, type Request, type Response, type NextFunction } from 'express';
|
||||||
import logger from '../logger.js';
|
import logger from '../logger.js';
|
||||||
import { generateCitationDetail } from '../services/generationService.js';
|
import { generateCitationDetail } from '../services/generationService.js';
|
||||||
|
|
||||||
const router = Router();
|
const router = Router();
|
||||||
|
|
||||||
router.post('/', async (req, res, next) => {
|
interface CitationDetailBody {
|
||||||
|
chunkTexts?: string[];
|
||||||
|
sourceName?: string;
|
||||||
|
answer?: string;
|
||||||
|
citationIndex?: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
router.post('/', async (req: Request<unknown, unknown, CitationDetailBody>, res: Response, next: NextFunction) => {
|
||||||
try {
|
try {
|
||||||
const { chunkTexts, sourceName, answer, citationIndex } = req.body;
|
const { chunkTexts, sourceName, answer, citationIndex } = req.body;
|
||||||
|
|
||||||
if (!chunkTexts?.length || !answer || citationIndex == null) {
|
if (!chunkTexts?.length || !answer || citationIndex == null) {
|
||||||
return res.status(400).json({
|
res.status(400).json({
|
||||||
error: 'chunkTexts, answer, and citationIndex are required',
|
error: 'chunkTexts, answer, and citationIndex are required',
|
||||||
});
|
});
|
||||||
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
logger.info({ citationIndex, sourceName }, 'citation detail requested');
|
logger.info({ citationIndex, sourceName }, 'citation detail requested');
|
||||||
@@ -1,6 +1,6 @@
|
|||||||
'use strict';
|
'use strict';
|
||||||
|
|
||||||
import { Router } from 'express';
|
import { Router, type Request, type Response, type NextFunction } from 'express';
|
||||||
import logger from '../logger.js';
|
import logger from '../logger.js';
|
||||||
import * as notebookStore from '../stores/notebookStore.js';
|
import * as notebookStore from '../stores/notebookStore.js';
|
||||||
import * as documentCacheStore from '../stores/documentCacheStore.js';
|
import * as documentCacheStore from '../stores/documentCacheStore.js';
|
||||||
@@ -8,9 +8,16 @@ import {
|
|||||||
generateStudyGuide,
|
generateStudyGuide,
|
||||||
generateFaq,
|
generateFaq,
|
||||||
generateExecutiveBrief,
|
generateExecutiveBrief,
|
||||||
|
type SourceGroup,
|
||||||
|
type StudyGuide,
|
||||||
|
type Faq,
|
||||||
|
type ExecutiveBrief,
|
||||||
} from '../services/documentService.js';
|
} from '../services/documentService.js';
|
||||||
|
|
||||||
const GENERATORS = {
|
type DocumentType = 'study-guide' | 'faq' | 'executive-brief';
|
||||||
|
type GeneratorFn = (sourceGroups: SourceGroup[]) => Promise<StudyGuide | Faq | ExecutiveBrief>;
|
||||||
|
|
||||||
|
const GENERATORS: Record<DocumentType, GeneratorFn> = {
|
||||||
'study-guide': generateStudyGuide,
|
'study-guide': generateStudyGuide,
|
||||||
'faq': generateFaq,
|
'faq': generateFaq,
|
||||||
'executive-brief': generateExecutiveBrief,
|
'executive-brief': generateExecutiveBrief,
|
||||||
@@ -18,36 +25,48 @@ const GENERATORS = {
|
|||||||
|
|
||||||
const router = Router();
|
const router = Router();
|
||||||
|
|
||||||
router.post('/generate', async (req, res, next) => {
|
interface GenerateBody {
|
||||||
|
notebookId?: string;
|
||||||
|
type?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
router.post('/generate', async (req: Request<unknown, unknown, GenerateBody>, res: Response, next: NextFunction) => {
|
||||||
try {
|
try {
|
||||||
const { notebookId, type } = req.body;
|
const { notebookId, type } = req.body;
|
||||||
|
|
||||||
if (!notebookId || !type) {
|
if (!notebookId || !type) {
|
||||||
return res.status(400).json({ error: 'notebookId and type are required' });
|
res.status(400).json({ error: 'notebookId and type are required' });
|
||||||
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
const generator = GENERATORS[type];
|
const generator = GENERATORS[type as DocumentType];
|
||||||
if (!generator) {
|
if (!generator) {
|
||||||
return res.status(400).json({
|
res.status(400).json({
|
||||||
error: `Invalid type. Must be one of: ${Object.keys(GENERATORS).join(', ')}`,
|
error: `Invalid type. Must be one of: ${Object.keys(GENERATORS).join(', ')}`,
|
||||||
});
|
});
|
||||||
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
const chunks = notebookStore.getChunksForNotebook(notebookId);
|
const chunks = notebookStore.getChunksForNotebook(notebookId);
|
||||||
if (chunks.length === 0) {
|
if (chunks.length === 0) {
|
||||||
return res.status(422).json({ error: 'No source material available in this notebook' });
|
res.status(422).json({ error: 'No source material available in this notebook' });
|
||||||
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
const sources = notebookStore.getSources(notebookId);
|
const sources = notebookStore.getSources(notebookId);
|
||||||
const cached = documentCacheStore.getCachedDocument(notebookId, type);
|
const cached = documentCacheStore.getCachedDocument(notebookId, type);
|
||||||
if (cached && documentCacheStore.isFresh(notebookId, type, sources)) {
|
if (cached && documentCacheStore.isFresh(notebookId, type, sources)) {
|
||||||
logger.info({ notebookId, type }, 'serving cached document');
|
logger.info({ notebookId, type }, 'serving cached document');
|
||||||
return res.json({ type, document: cached.document });
|
res.json({ type, document: cached.document });
|
||||||
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
const sourceGroups = notebookStore.buildSourceGroups(notebookId, chunks);
|
const sourceGroups = notebookStore.buildSourceGroups(notebookId, chunks);
|
||||||
|
|
||||||
logger.info({ notebookId, type, sourceCount: sourceGroups.length, chunkCount: chunks.length }, 'document generation started');
|
logger.info(
|
||||||
|
{ notebookId, type, sourceCount: sourceGroups.length, chunkCount: chunks.length },
|
||||||
|
'document generation started'
|
||||||
|
);
|
||||||
|
|
||||||
const document = await generator(sourceGroups);
|
const document = await generator(sourceGroups);
|
||||||
|
|
||||||
@@ -1,20 +1,25 @@
|
|||||||
'use strict';
|
'use strict';
|
||||||
|
|
||||||
import { Router } from 'express';
|
import { Router, type Request, type Response } from 'express';
|
||||||
import { v4 as uuidv4 } from 'uuid';
|
import { v4 as uuidv4 } from 'uuid';
|
||||||
import * as notebookStore from '../stores/notebookStore.js';
|
import * as notebookStore from '../stores/notebookStore.js';
|
||||||
import { deleteVectorsForChunks } from '../stores/vectorStore.js';
|
import { deleteVectorsForChunks } from '../stores/vectorStore.js';
|
||||||
|
|
||||||
const router = Router();
|
const router = Router();
|
||||||
|
|
||||||
router.get('/', (req, res) => {
|
router.get('/', (_req: Request, res: Response) => {
|
||||||
res.json(notebookStore.getAllNotebooks());
|
res.json(notebookStore.getAllNotebooks());
|
||||||
});
|
});
|
||||||
|
|
||||||
router.post('/', (req, res) => {
|
interface CreateNotebookBody {
|
||||||
|
name?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
router.post('/', (req: Request<unknown, unknown, CreateNotebookBody>, res: Response) => {
|
||||||
const { name } = req.body;
|
const { name } = req.body;
|
||||||
if (!name || !name.trim()) {
|
if (!name || !name.trim()) {
|
||||||
return res.status(400).json({ error: 'name is required' });
|
res.status(400).json({ error: 'name is required' });
|
||||||
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
const notebook = notebookStore.createNotebook({
|
const notebook = notebookStore.createNotebook({
|
||||||
@@ -26,11 +31,16 @@ router.post('/', (req, res) => {
|
|||||||
res.status(201).json(notebook);
|
res.status(201).json(notebook);
|
||||||
});
|
});
|
||||||
|
|
||||||
router.delete('/:id', (req, res) => {
|
interface DeleteParams {
|
||||||
|
id: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
router.delete('/:id', (req: Request<DeleteParams>, res: Response) => {
|
||||||
const chunks = notebookStore.getChunksForNotebook(req.params.id);
|
const chunks = notebookStore.getChunksForNotebook(req.params.id);
|
||||||
const deleted = notebookStore.deleteNotebook(req.params.id);
|
const deleted = notebookStore.deleteNotebook(req.params.id);
|
||||||
if (!deleted) {
|
if (!deleted) {
|
||||||
return res.status(404).json({ error: 'notebook not found' });
|
res.status(404).json({ error: 'notebook not found' });
|
||||||
|
return;
|
||||||
}
|
}
|
||||||
if (chunks.length) {
|
if (chunks.length) {
|
||||||
deleteVectorsForChunks(chunks.map((c) => c.id));
|
deleteVectorsForChunks(chunks.map((c) => c.id));
|
||||||
@@ -39,4 +49,3 @@ router.delete('/:id', (req, res) => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
export default router;
|
export default router;
|
||||||
|
|
||||||
@@ -1,6 +1,6 @@
|
|||||||
'use strict';
|
'use strict';
|
||||||
|
|
||||||
import { Router } from 'express';
|
import { Router, type Request, type Response, type NextFunction } from 'express';
|
||||||
import logger from '../logger.js';
|
import logger from '../logger.js';
|
||||||
import { embedTexts, search, rerank } from '../services/retrievalService.js';
|
import { embedTexts, search, rerank } from '../services/retrievalService.js';
|
||||||
import { generate } from '../services/generationService.js';
|
import { generate } from '../services/generationService.js';
|
||||||
@@ -12,11 +12,17 @@ const TOP_K_RERANK = 5;
|
|||||||
|
|
||||||
const router = Router();
|
const router = Router();
|
||||||
|
|
||||||
router.post('/', async (req, res, next) => {
|
interface QueryBody {
|
||||||
|
notebookId?: string;
|
||||||
|
question?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
router.post('/', async (req: Request<unknown, unknown, QueryBody>, res: Response, next: NextFunction) => {
|
||||||
try {
|
try {
|
||||||
const { notebookId, question } = req.body;
|
const { notebookId, question } = req.body;
|
||||||
if (!notebookId || !question) {
|
if (!notebookId || !question) {
|
||||||
return res.status(400).json({ error: 'notebookId and question are required' });
|
res.status(400).json({ error: 'notebookId and question are required' });
|
||||||
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
logger.info({ notebookId, question }, 'query received');
|
logger.info({ notebookId, question }, 'query received');
|
||||||
@@ -25,31 +31,32 @@ router.post('/', async (req, res, next) => {
|
|||||||
|
|
||||||
const searchResults = search(queryEmbedding, notebookId, TOP_K_SEARCH);
|
const searchResults = search(queryEmbedding, notebookId, TOP_K_SEARCH);
|
||||||
if (searchResults.length === 0) {
|
if (searchResults.length === 0) {
|
||||||
return res.json({
|
res.json({
|
||||||
answer: 'No sources found for this notebook. Upload some documents first.',
|
answer: 'No sources found for this notebook. Upload some documents first.',
|
||||||
citations: [],
|
citations: [],
|
||||||
groundednessScore: null,
|
groundednessScore: null,
|
||||||
followUpQuestions: [],
|
followUpQuestions: [],
|
||||||
});
|
});
|
||||||
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
const candidateChunks = searchResults.map((r) => r.chunk);
|
const candidateChunks = searchResults.map((r) => r.chunk);
|
||||||
const reranked = await rerank(question, candidateChunks);
|
const reranked = await rerank(question, candidateChunks);
|
||||||
const topChunks = reranked.slice(0, TOP_K_RERANK).map((r) => r.chunk);
|
const topChunks = reranked.slice(0, TOP_K_RERANK).map((r) => r.chunk);
|
||||||
|
|
||||||
logger.debug({
|
logger.debug(
|
||||||
|
{
|
||||||
searchHits: searchResults.length,
|
searchHits: searchResults.length,
|
||||||
rerankedTop: topChunks.length,
|
rerankedTop: topChunks.length,
|
||||||
}, 'retrieval complete');
|
},
|
||||||
|
'retrieval complete'
|
||||||
|
);
|
||||||
|
|
||||||
const sourceGroups = notebookStore.buildSourceGroups(notebookId, topChunks);
|
const sourceGroups = notebookStore.buildSourceGroups(notebookId, topChunks);
|
||||||
|
|
||||||
const { answer, citedSourceIndices, followUpQuestions } = await generate(
|
const { answer, citedSourceIndices, followUpQuestions } = await generate(question, sourceGroups);
|
||||||
question,
|
|
||||||
sourceGroups,
|
|
||||||
);
|
|
||||||
|
|
||||||
const citedChunkIds = [];
|
const citedChunkIds: string[] = [];
|
||||||
for (const idx of citedSourceIndices) {
|
for (const idx of citedSourceIndices) {
|
||||||
const group = sourceGroups.find((g) => g.docIndex === idx);
|
const group = sourceGroups.find((g) => g.docIndex === idx);
|
||||||
if (group) {
|
if (group) {
|
||||||
@@ -85,4 +92,3 @@ router.post('/', async (req, res, next) => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
export default router;
|
export default router;
|
||||||
|
|
||||||
@@ -1,17 +1,11 @@
|
|||||||
'use strict';
|
'use strict';
|
||||||
|
|
||||||
import { Router } from 'express';
|
import { Router, type Request, type Response, type NextFunction } from 'express';
|
||||||
import multer from 'multer';
|
import multer from 'multer';
|
||||||
import { v4 as uuidv4 } from 'uuid';
|
import { v4 as uuidv4 } from 'uuid';
|
||||||
import logger from '../logger.js';
|
import logger from '../logger.js';
|
||||||
import * as notebookStore from '../stores/notebookStore.js';
|
import * as notebookStore from '../stores/notebookStore.js';
|
||||||
import {
|
import { parseFile, chunkText, parseUrl, parseYoutubeUrl, isYoutubeUrl } from '../services/sourceService.js';
|
||||||
parseFile,
|
|
||||||
chunkText,
|
|
||||||
parseUrl,
|
|
||||||
parseYoutubeUrl,
|
|
||||||
isYoutubeUrl,
|
|
||||||
} from '../services/sourceService.js';
|
|
||||||
import { embedTexts, storeChunkEmbeddings } from '../services/retrievalService.js';
|
import { embedTexts, storeChunkEmbeddings } from '../services/retrievalService.js';
|
||||||
import { triggerPreGeneration } from '../services/preGenerationService.js';
|
import { triggerPreGeneration } from '../services/preGenerationService.js';
|
||||||
import {
|
import {
|
||||||
@@ -19,28 +13,30 @@ import {
|
|||||||
requireNotebook,
|
requireNotebook,
|
||||||
requireFile,
|
requireFile,
|
||||||
requireUrl,
|
requireUrl,
|
||||||
|
type NotebookRequest,
|
||||||
} from '../middleware/validation.js';
|
} from '../middleware/validation.js';
|
||||||
|
|
||||||
const upload = multer({ storage: multer.memoryStorage(), limits: { fileSize: 50 * 1024 * 1024 } });
|
const upload = multer({ storage: multer.memoryStorage(), limits: { fileSize: 50 * 1024 * 1024 } });
|
||||||
const router = Router();
|
const router = Router();
|
||||||
|
|
||||||
const TIMESTAMP_RE = /\s+\d{1,2}\.\d{2}\.\d{2}\s*[\u2018\u2019''\s]?\s*[AP]M(?=\.\w+$)/i;
|
const TIMESTAMP_RE = /\s+\d{1,2}\.\d{2}\.\d{2}\s*[\u2018\u2019''\s]?\s*[AP]M(?=\.\w+$)/i;
|
||||||
function cleanFilename(name) {
|
function cleanFilename(name: string): string {
|
||||||
return name.replace(TIMESTAMP_RE, '');
|
return name.replace(TIMESTAMP_RE, '');
|
||||||
}
|
}
|
||||||
|
|
||||||
router.get('/', requireNotebookId, (req, res) => {
|
router.get('/', requireNotebookId, (req: NotebookRequest, res: Response) => {
|
||||||
res.json(notebookStore.getSources(req.notebookId));
|
res.json(notebookStore.getSources(req.notebookId!));
|
||||||
});
|
});
|
||||||
|
|
||||||
|
function multerUpload(req: Request, res: Response, next: NextFunction): void {
|
||||||
function multerUpload(req, res, next) {
|
|
||||||
upload.single('file')(req, res, (err) => {
|
upload.single('file')(req, res, (err) => {
|
||||||
if (err) {
|
if (err) {
|
||||||
if (err.code === 'LIMIT_FILE_SIZE') {
|
if ((err as multer.MulterError).code === 'LIMIT_FILE_SIZE') {
|
||||||
return res.status(413).json({ error: 'File too large. Maximum size is 50 MB.' });
|
res.status(413).json({ error: 'File too large. Maximum size is 50 MB.' });
|
||||||
|
return;
|
||||||
}
|
}
|
||||||
return next(err);
|
next(err);
|
||||||
|
return;
|
||||||
}
|
}
|
||||||
next();
|
next();
|
||||||
});
|
});
|
||||||
@@ -52,73 +48,79 @@ router.post(
|
|||||||
requireNotebookId,
|
requireNotebookId,
|
||||||
requireNotebook,
|
requireNotebook,
|
||||||
requireFile,
|
requireFile,
|
||||||
async (req, res, next) => {
|
async (req: NotebookRequest, res: Response, next: NextFunction) => {
|
||||||
try {
|
try {
|
||||||
const sourceId = uuidv4();
|
const sourceId = uuidv4();
|
||||||
const rawName = Buffer.from(req.file.originalname, 'latin1').toString('utf-8');
|
const rawName = Buffer.from(req.file!.originalname, 'latin1').toString('utf-8');
|
||||||
const displayName = cleanFilename(rawName);
|
const displayName = cleanFilename(rawName);
|
||||||
const text = await parseFile(req.file.buffer, req.file.mimetype, displayName);
|
const text = await parseFile(req.file!.buffer, req.file!.mimetype, displayName);
|
||||||
const chunks = chunkText(text, sourceId);
|
const chunks = chunkText(text, sourceId);
|
||||||
|
|
||||||
logger.info({
|
logger.info(
|
||||||
|
{
|
||||||
sourceId,
|
sourceId,
|
||||||
filename: displayName,
|
filename: displayName,
|
||||||
chunkCount: chunks.length,
|
chunkCount: chunks.length,
|
||||||
}, 'parsed and chunked source');
|
},
|
||||||
|
'parsed and chunked source'
|
||||||
|
);
|
||||||
|
|
||||||
notebookStore.addChunksToNotebook(req.notebookId, chunks);
|
notebookStore.addChunksToNotebook(req.notebookId!, chunks);
|
||||||
|
|
||||||
const texts = chunks.map((c) => c.text);
|
const texts = chunks.map((c) => c.text);
|
||||||
const embeddings = await embedTexts(texts, 'document');
|
const embeddings = await embedTexts(texts, 'document');
|
||||||
storeChunkEmbeddings(chunks, embeddings);
|
storeChunkEmbeddings(chunks, embeddings);
|
||||||
|
|
||||||
const source = notebookStore.addSource(req.notebookId, {
|
const source = notebookStore.addSource(req.notebookId!, {
|
||||||
id: sourceId,
|
id: sourceId,
|
||||||
name: displayName,
|
name: displayName,
|
||||||
mimetype: req.file.mimetype,
|
mimetype: req.file!.mimetype,
|
||||||
chunkCount: chunks.length,
|
chunkCount: chunks.length,
|
||||||
uploadedAt: new Date().toISOString(),
|
uploadedAt: new Date().toISOString(),
|
||||||
});
|
});
|
||||||
|
|
||||||
triggerPreGeneration(req.notebookId);
|
triggerPreGeneration(req.notebookId!);
|
||||||
|
|
||||||
res.status(201).json(source);
|
res.status(201).json(source);
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
next(err);
|
next(err);
|
||||||
}
|
}
|
||||||
},
|
}
|
||||||
);
|
);
|
||||||
|
|
||||||
|
interface UrlBody {
|
||||||
|
url?: string;
|
||||||
|
}
|
||||||
|
|
||||||
router.post(
|
router.post(
|
||||||
'/url',
|
'/url',
|
||||||
requireNotebookId,
|
requireNotebookId,
|
||||||
requireNotebook,
|
requireNotebook,
|
||||||
requireUrl,
|
requireUrl,
|
||||||
async (req, res, next) => {
|
async (req: NotebookRequest & Request<unknown, unknown, UrlBody>, res: Response, next: NextFunction) => {
|
||||||
try {
|
try {
|
||||||
const { url } = req.body;
|
const { url } = req.body;
|
||||||
const sourceId = uuidv4();
|
const sourceId = uuidv4();
|
||||||
const isYT = isYoutubeUrl(url);
|
const isYT = isYoutubeUrl(url!);
|
||||||
const displayName = isYT
|
const displayName = isYT ? `YouTube: ${url}` : url!.replace(/^https?:\/\//, '').slice(0, 60);
|
||||||
? `YouTube: ${url}`
|
|
||||||
: url.replace(/^https?:\/\//, '').slice(0, 60);
|
|
||||||
|
|
||||||
logger.info({ sourceId, url, isYT }, 'processing URL source');
|
logger.info({ sourceId, url, isYT }, 'processing URL source');
|
||||||
|
|
||||||
const text = isYT ? await parseYoutubeUrl(url) : await parseUrl(url);
|
const text = isYT ? await parseYoutubeUrl(url!) : await parseUrl(url!);
|
||||||
const chunks = chunkText(text, sourceId);
|
const chunks = chunkText(text, sourceId);
|
||||||
|
|
||||||
if (chunks.length === 0) {
|
if (chunks.length === 0) {
|
||||||
return res.status(422).json({ error: 'No usable text could be extracted from the URL' });
|
res.status(422).json({ error: 'No usable text could be extracted from the URL' });
|
||||||
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
notebookStore.addChunksToNotebook(req.notebookId, chunks);
|
notebookStore.addChunksToNotebook(req.notebookId!, chunks);
|
||||||
|
|
||||||
const texts = chunks.map((c) => c.text);
|
const texts = chunks.map((c) => c.text);
|
||||||
const embeddings = await embedTexts(texts, 'document');
|
const embeddings = await embedTexts(texts, 'document');
|
||||||
storeChunkEmbeddings(chunks, embeddings);
|
storeChunkEmbeddings(chunks, embeddings);
|
||||||
|
|
||||||
const source = notebookStore.addSource(req.notebookId, {
|
const source = notebookStore.addSource(req.notebookId!, {
|
||||||
id: sourceId,
|
id: sourceId,
|
||||||
name: displayName,
|
name: displayName,
|
||||||
mimetype: isYT ? 'video/youtube' : 'text/html',
|
mimetype: isYT ? 'video/youtube' : 'text/html',
|
||||||
@@ -126,13 +128,13 @@ router.post(
|
|||||||
uploadedAt: new Date().toISOString(),
|
uploadedAt: new Date().toISOString(),
|
||||||
});
|
});
|
||||||
|
|
||||||
triggerPreGeneration(req.notebookId);
|
triggerPreGeneration(req.notebookId!);
|
||||||
|
|
||||||
res.status(201).json(source);
|
res.status(201).json(source);
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
next(err);
|
next(err);
|
||||||
}
|
}
|
||||||
},
|
}
|
||||||
);
|
);
|
||||||
|
|
||||||
export default router;
|
export default router;
|
||||||
@@ -20,7 +20,7 @@ type GeneratorFn = (sourceGroups: SourceGroup[]) => Promise<StudyGuide | Faq | E
|
|||||||
|
|
||||||
const GENERATORS: Record<DocumentType, GeneratorFn> = {
|
const GENERATORS: Record<DocumentType, GeneratorFn> = {
|
||||||
'study-guide': generateStudyGuide,
|
'study-guide': generateStudyGuide,
|
||||||
faq: generateFaq,
|
'faq': generateFaq,
|
||||||
'executive-brief': generateExecutiveBrief,
|
'executive-brief': generateExecutiveBrief,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|||||||
@@ -1,52 +0,0 @@
|
|||||||
'use strict';
|
|
||||||
|
|
||||||
const cache = new Map();
|
|
||||||
|
|
||||||
function buildSourceHash(sources) {
|
|
||||||
return sources
|
|
||||||
.map((s) => s.id)
|
|
||||||
.sort()
|
|
||||||
.join('|');
|
|
||||||
}
|
|
||||||
|
|
||||||
export function getCachedDocument(notebookId, type) {
|
|
||||||
const entry = cache.get(`${notebookId}:${type}`);
|
|
||||||
if (!entry) return null;
|
|
||||||
return entry;
|
|
||||||
}
|
|
||||||
|
|
||||||
export function setCachedDocument(notebookId, type, document, sources) {
|
|
||||||
const key = `${notebookId}:${type}`;
|
|
||||||
cache.set(key, {
|
|
||||||
document,
|
|
||||||
sourceHash: buildSourceHash(sources),
|
|
||||||
createdAt: Date.now(),
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
export function isFresh(notebookId, type, currentSources) {
|
|
||||||
const entry = cache.get(`${notebookId}:${type}`);
|
|
||||||
if (!entry) return false;
|
|
||||||
return entry.sourceHash === buildSourceHash(currentSources);
|
|
||||||
}
|
|
||||||
|
|
||||||
export function invalidate(notebookId) {
|
|
||||||
for (const key of cache.keys()) {
|
|
||||||
if (key.startsWith(`${notebookId}:`)) {
|
|
||||||
cache.delete(key);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
export function markGenerating(notebookId) {
|
|
||||||
const key = `${notebookId}:__generating`;
|
|
||||||
cache.set(key, true);
|
|
||||||
}
|
|
||||||
|
|
||||||
export function clearGenerating(notebookId) {
|
|
||||||
cache.delete(`${notebookId}:__generating`);
|
|
||||||
}
|
|
||||||
|
|
||||||
export function isGenerating(notebookId) {
|
|
||||||
return cache.get(`${notebookId}:__generating`) === true;
|
|
||||||
}
|
|
||||||
@@ -10,7 +10,6 @@ import {
|
|||||||
} from './documentCacheStore.js';
|
} from './documentCacheStore.js';
|
||||||
|
|
||||||
describe('documentCacheStore', () => {
|
describe('documentCacheStore', () => {
|
||||||
|
|
||||||
describe('getCachedDocument / setCachedDocument', () => {
|
describe('getCachedDocument / setCachedDocument', () => {
|
||||||
it('returns null for an uncached entry', () => {
|
it('returns null for an uncached entry', () => {
|
||||||
expect(getCachedDocument('miss-1', 'faq')).toBeNull();
|
expect(getCachedDocument('miss-1', 'faq')).toBeNull();
|
||||||
@@ -22,22 +21,22 @@ describe('documentCacheStore', () => {
|
|||||||
setCachedDocument('rt-1', 'study-guide', doc, sources);
|
setCachedDocument('rt-1', 'study-guide', doc, sources);
|
||||||
|
|
||||||
const entry = getCachedDocument('rt-1', 'study-guide');
|
const entry = getCachedDocument('rt-1', 'study-guide');
|
||||||
expect(entry.document).toEqual(doc);
|
expect(entry!.document).toEqual(doc);
|
||||||
expect(entry.sourceHash).toBe('s1|s2');
|
expect(entry!.sourceHash).toBe('s1|s2');
|
||||||
expect(entry.createdAt).toBeTypeOf('number');
|
expect(entry!.createdAt).toBeTypeOf('number');
|
||||||
});
|
});
|
||||||
|
|
||||||
it('overwrites an existing entry for the same key', () => {
|
it('overwrites an existing entry for the same key', () => {
|
||||||
setCachedDocument('ow-1', 'faq', { v: 1 }, [{ id: 'a' }]);
|
setCachedDocument('ow-1', 'faq', { v: 1 }, [{ id: 'a' }]);
|
||||||
setCachedDocument('ow-1', 'faq', { v: 2 }, [{ id: 'b' }]);
|
setCachedDocument('ow-1', 'faq', { v: 2 }, [{ id: 'b' }]);
|
||||||
expect(getCachedDocument('ow-1', 'faq').document).toEqual({ v: 2 });
|
expect(getCachedDocument('ow-1', 'faq')!.document).toEqual({ v: 2 });
|
||||||
});
|
});
|
||||||
|
|
||||||
it('keeps entries for different types separate', () => {
|
it('keeps entries for different types separate', () => {
|
||||||
setCachedDocument('sep-1', 'faq', { kind: 'faq' }, [{ id: 'x' }]);
|
setCachedDocument('sep-1', 'faq', { kind: 'faq' }, [{ id: 'x' }]);
|
||||||
setCachedDocument('sep-1', 'study-guide', { kind: 'sg' }, [{ id: 'x' }]);
|
setCachedDocument('sep-1', 'study-guide', { kind: 'sg' }, [{ id: 'x' }]);
|
||||||
expect(getCachedDocument('sep-1', 'faq').document.kind).toBe('faq');
|
expect((getCachedDocument('sep-1', 'faq')!.document as { kind: string }).kind).toBe('faq');
|
||||||
expect(getCachedDocument('sep-1', 'study-guide').document.kind).toBe('sg');
|
expect((getCachedDocument('sep-1', 'study-guide')!.document as { kind: string }).kind).toBe('sg');
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -71,7 +70,7 @@ describe('documentCacheStore', () => {
|
|||||||
setCachedDocument('inv-b', 'faq', { b: 2 }, [{ id: 'x' }]);
|
setCachedDocument('inv-b', 'faq', { b: 2 }, [{ id: 'x' }]);
|
||||||
invalidate('inv-a');
|
invalidate('inv-a');
|
||||||
expect(getCachedDocument('inv-a', 'faq')).toBeNull();
|
expect(getCachedDocument('inv-a', 'faq')).toBeNull();
|
||||||
expect(getCachedDocument('inv-b', 'faq').document).toEqual({ b: 2 });
|
expect(getCachedDocument('inv-b', 'faq')!.document).toEqual({ b: 2 });
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
67
server/src/stores/documentCacheStore.ts
Normal file
67
server/src/stores/documentCacheStore.ts
Normal file
@@ -0,0 +1,67 @@
|
|||||||
|
'use strict';
|
||||||
|
|
||||||
|
interface Source {
|
||||||
|
id: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface CacheEntry {
|
||||||
|
document: unknown;
|
||||||
|
sourceHash: string;
|
||||||
|
createdAt: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
const cache = new Map<string, CacheEntry | boolean>();
|
||||||
|
|
||||||
|
function buildSourceHash(sources: Source[]): string {
|
||||||
|
return sources
|
||||||
|
.map((s) => s.id)
|
||||||
|
.sort()
|
||||||
|
.join('|');
|
||||||
|
}
|
||||||
|
|
||||||
|
export function getCachedDocument(notebookId: string, type: string): CacheEntry | null {
|
||||||
|
const entry = cache.get(`${notebookId}:${type}`);
|
||||||
|
if (!entry || typeof entry === 'boolean') return null;
|
||||||
|
return entry;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function setCachedDocument(
|
||||||
|
notebookId: string,
|
||||||
|
type: string,
|
||||||
|
document: unknown,
|
||||||
|
sources: Source[]
|
||||||
|
): void {
|
||||||
|
const key = `${notebookId}:${type}`;
|
||||||
|
cache.set(key, {
|
||||||
|
document,
|
||||||
|
sourceHash: buildSourceHash(sources),
|
||||||
|
createdAt: Date.now(),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
export function isFresh(notebookId: string, type: string, currentSources: Source[]): boolean {
|
||||||
|
const entry = cache.get(`${notebookId}:${type}`);
|
||||||
|
if (!entry || typeof entry === 'boolean') return false;
|
||||||
|
return entry.sourceHash === buildSourceHash(currentSources);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function invalidate(notebookId: string): void {
|
||||||
|
for (const key of cache.keys()) {
|
||||||
|
if (key.startsWith(`${notebookId}:`)) {
|
||||||
|
cache.delete(key);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export function markGenerating(notebookId: string): void {
|
||||||
|
const key = `${notebookId}:__generating`;
|
||||||
|
cache.set(key, true);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function clearGenerating(notebookId: string): void {
|
||||||
|
cache.delete(`${notebookId}:__generating`);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function isGenerating(notebookId: string): boolean {
|
||||||
|
return cache.get(`${notebookId}:__generating`) === true;
|
||||||
|
}
|
||||||
@@ -2,7 +2,7 @@ import { describe, it, expect, beforeEach } from 'vitest';
|
|||||||
import { NotebookStore } from './notebookStore.js';
|
import { NotebookStore } from './notebookStore.js';
|
||||||
|
|
||||||
describe('notebookStore', () => {
|
describe('notebookStore', () => {
|
||||||
let store;
|
let store: NotebookStore;
|
||||||
|
|
||||||
beforeEach(() => {
|
beforeEach(() => {
|
||||||
store = new NotebookStore();
|
store = new NotebookStore();
|
||||||
@@ -15,7 +15,7 @@ describe('notebookStore', () => {
|
|||||||
|
|
||||||
const found = store.getNotebook('nb-1');
|
const found = store.getNotebook('nb-1');
|
||||||
expect(found).not.toBeNull();
|
expect(found).not.toBeNull();
|
||||||
expect(found.name).toBe('Test');
|
expect(found!.name).toBe('Test');
|
||||||
});
|
});
|
||||||
|
|
||||||
it('lists all notebooks without chunks', () => {
|
it('lists all notebooks without chunks', () => {
|
||||||
@@ -51,8 +51,8 @@ describe('notebookStore', () => {
|
|||||||
it('adds and retrieves chunks', () => {
|
it('adds and retrieves chunks', () => {
|
||||||
store.createNotebook({ id: 'nb-1', name: 'A', createdAt: '2026-01-01' });
|
store.createNotebook({ id: 'nb-1', name: 'A', createdAt: '2026-01-01' });
|
||||||
store.addChunksToNotebook('nb-1', [
|
store.addChunksToNotebook('nb-1', [
|
||||||
{ id: 'c-1', text: 'chunk one' },
|
{ id: 'c-1', text: 'chunk one', sourceId: 's-1', index: 0 },
|
||||||
{ id: 'c-2', text: 'chunk two' },
|
{ id: 'c-2', text: 'chunk two', sourceId: 's-1', index: 1 },
|
||||||
]);
|
]);
|
||||||
const chunks = store.getChunksForNotebook('nb-1');
|
const chunks = store.getChunksForNotebook('nb-1');
|
||||||
expect(chunks).toHaveLength(2);
|
expect(chunks).toHaveLength(2);
|
||||||
@@ -1,62 +1,100 @@
|
|||||||
'use strict';
|
'use strict';
|
||||||
|
|
||||||
|
export interface TextChunk {
|
||||||
|
id: string;
|
||||||
|
text: string;
|
||||||
|
sourceId: string;
|
||||||
|
index: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface Source {
|
||||||
|
id: string;
|
||||||
|
name: string;
|
||||||
|
mimetype?: string;
|
||||||
|
chunkCount?: number;
|
||||||
|
uploadedAt?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface Notebook {
|
||||||
|
id: string;
|
||||||
|
name: string;
|
||||||
|
createdAt: string;
|
||||||
|
sources: Source[];
|
||||||
|
chunks: TextChunk[];
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface NotebookMeta {
|
||||||
|
id: string;
|
||||||
|
name: string;
|
||||||
|
createdAt: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface SourceGroup {
|
||||||
|
docIndex: number;
|
||||||
|
sourceId: string;
|
||||||
|
name: string;
|
||||||
|
chunks: TextChunk[];
|
||||||
|
}
|
||||||
|
|
||||||
export class NotebookStore {
|
export class NotebookStore {
|
||||||
constructor() {
|
private notebooks = new Map<string, Notebook>();
|
||||||
this.notebooks = new Map();
|
|
||||||
|
getAllNotebooks(): NotebookMeta[] {
|
||||||
|
return Array.from(this.notebooks.values()).map(({ id, name, createdAt }) => ({
|
||||||
|
id,
|
||||||
|
name,
|
||||||
|
createdAt,
|
||||||
|
}));
|
||||||
}
|
}
|
||||||
|
|
||||||
getAllNotebooks() {
|
getNotebook(id: string): Notebook | null {
|
||||||
return Array.from(this.notebooks.values()).map(({ chunks, ...meta }) => meta);
|
|
||||||
}
|
|
||||||
|
|
||||||
getNotebook(id) {
|
|
||||||
return this.notebooks.get(id) || null;
|
return this.notebooks.get(id) || null;
|
||||||
}
|
}
|
||||||
|
|
||||||
createNotebook(notebook) {
|
createNotebook(notebook: { id: string; name: string; createdAt: string }): NotebookMeta {
|
||||||
const record = { ...notebook, sources: [], chunks: [] };
|
const record: Notebook = { ...notebook, sources: [], chunks: [] };
|
||||||
this.notebooks.set(record.id, record);
|
this.notebooks.set(record.id, record);
|
||||||
return { id: record.id, name: record.name, createdAt: record.createdAt };
|
return { id: record.id, name: record.name, createdAt: record.createdAt };
|
||||||
}
|
}
|
||||||
|
|
||||||
deleteNotebook(id) {
|
deleteNotebook(id: string): boolean {
|
||||||
return this.notebooks.delete(id);
|
return this.notebooks.delete(id);
|
||||||
}
|
}
|
||||||
|
|
||||||
addSource(notebookId, source) {
|
addSource(notebookId: string, source: Source): Source | null {
|
||||||
const notebook = this.notebooks.get(notebookId);
|
const notebook = this.notebooks.get(notebookId);
|
||||||
if (!notebook) return null;
|
if (!notebook) return null;
|
||||||
notebook.sources.push(source);
|
notebook.sources.push(source);
|
||||||
return source;
|
return source;
|
||||||
}
|
}
|
||||||
|
|
||||||
getSources(notebookId) {
|
getSources(notebookId: string): Source[] {
|
||||||
const notebook = this.notebooks.get(notebookId);
|
const notebook = this.notebooks.get(notebookId);
|
||||||
if (!notebook) return [];
|
if (!notebook) return [];
|
||||||
return notebook.sources;
|
return notebook.sources;
|
||||||
}
|
}
|
||||||
|
|
||||||
addChunksToNotebook(notebookId, chunks) {
|
addChunksToNotebook(notebookId: string, chunks: TextChunk[]): Notebook | null {
|
||||||
const notebook = this.notebooks.get(notebookId);
|
const notebook = this.notebooks.get(notebookId);
|
||||||
if (!notebook) return null;
|
if (!notebook) return null;
|
||||||
notebook.chunks = notebook.chunks.concat(chunks);
|
notebook.chunks = notebook.chunks.concat(chunks);
|
||||||
return notebook;
|
return notebook;
|
||||||
}
|
}
|
||||||
|
|
||||||
getChunksForNotebook(notebookId) {
|
getChunksForNotebook(notebookId: string): TextChunk[] {
|
||||||
const notebook = this.notebooks.get(notebookId);
|
const notebook = this.notebooks.get(notebookId);
|
||||||
if (!notebook) return [];
|
if (!notebook) return [];
|
||||||
return notebook.chunks;
|
return notebook.chunks;
|
||||||
}
|
}
|
||||||
|
|
||||||
buildSourceGroups(notebookId, chunks) {
|
buildSourceGroups(notebookId: string, chunks: TextChunk[]): SourceGroup[] {
|
||||||
const sources = this.getSources(notebookId);
|
const sources = this.getSources(notebookId);
|
||||||
const sourceIndexMap = new Map();
|
const sourceIndexMap = new Map<string, number>();
|
||||||
sources.forEach((src, i) => {
|
sources.forEach((src, i) => {
|
||||||
sourceIndexMap.set(src.id, i + 1);
|
sourceIndexMap.set(src.id, i + 1);
|
||||||
});
|
});
|
||||||
|
|
||||||
const groupMap = new Map();
|
const groupMap = new Map<string, SourceGroup>();
|
||||||
for (const chunk of chunks) {
|
for (const chunk of chunks) {
|
||||||
const docIndex = sourceIndexMap.get(chunk.sourceId) || 0;
|
const docIndex = sourceIndexMap.get(chunk.sourceId) || 0;
|
||||||
if (!groupMap.has(chunk.sourceId)) {
|
if (!groupMap.has(chunk.sourceId)) {
|
||||||
@@ -68,7 +106,7 @@ export class NotebookStore {
|
|||||||
chunks: [],
|
chunks: [],
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
groupMap.get(chunk.sourceId).chunks.push(chunk);
|
groupMap.get(chunk.sourceId)!.chunks.push(chunk);
|
||||||
}
|
}
|
||||||
|
|
||||||
return Array.from(groupMap.values()).sort((a, b) => a.docIndex - b.docIndex);
|
return Array.from(groupMap.values()).sort((a, b) => a.docIndex - b.docIndex);
|
||||||
@@ -1,21 +0,0 @@
|
|||||||
'use strict';
|
|
||||||
|
|
||||||
const vectors = new Map();
|
|
||||||
|
|
||||||
export function storeVector(chunkId, embedding) {
|
|
||||||
vectors.set(chunkId, new Float32Array(embedding));
|
|
||||||
}
|
|
||||||
|
|
||||||
export function getVector(chunkId) {
|
|
||||||
return vectors.get(chunkId) || null;
|
|
||||||
}
|
|
||||||
|
|
||||||
export function getAllVectors() {
|
|
||||||
return vectors;
|
|
||||||
}
|
|
||||||
|
|
||||||
export function deleteVectorsForChunks(chunkIds) {
|
|
||||||
for (const id of chunkIds) {
|
|
||||||
vectors.delete(id);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
21
server/src/stores/vectorStore.ts
Normal file
21
server/src/stores/vectorStore.ts
Normal file
@@ -0,0 +1,21 @@
|
|||||||
|
'use strict';
|
||||||
|
|
||||||
|
const vectors = new Map<string, Float32Array>();
|
||||||
|
|
||||||
|
export function storeVector(chunkId: string, embedding: number[]): void {
|
||||||
|
vectors.set(chunkId, new Float32Array(embedding));
|
||||||
|
}
|
||||||
|
|
||||||
|
export function getVector(chunkId: string): Float32Array | null {
|
||||||
|
return vectors.get(chunkId) || null;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function getAllVectors(): Map<string, Float32Array> {
|
||||||
|
return vectors;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function deleteVectorsForChunks(chunkIds: string[]): void {
|
||||||
|
for (const id of chunkIds) {
|
||||||
|
vectors.delete(id);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,13 +1,15 @@
|
|||||||
{
|
{
|
||||||
"compilerOptions": {
|
"compilerOptions": {
|
||||||
"allowJs": true, // Allow JavaScript files to be imported and compiled
|
"allowJs": true,
|
||||||
"checkJs": true,
|
"checkJs": true,
|
||||||
"target": "es2022",
|
"target": "es2022",
|
||||||
"module": "commonjs",
|
"module": "NodeNext",
|
||||||
|
"moduleResolution": "NodeNext",
|
||||||
"outDir": "./dist",
|
"outDir": "./dist",
|
||||||
"rootDir": "./src", // Root of input files
|
"rootDir": "./src",
|
||||||
"strict": true,
|
"strict": true,
|
||||||
"skipLibCheck": true // Skip checking .d.ts files for faster builds
|
"skipLibCheck": true,
|
||||||
|
"esModuleInterop": true
|
||||||
},
|
},
|
||||||
"include": [
|
"include": [
|
||||||
"src/**/*"
|
"src/**/*"
|
||||||
|
|||||||
Reference in New Issue
Block a user