Compare commits
22 Commits
FEAT-new-r
...
65d3a18edc
| Author | SHA1 | Date | |
|---|---|---|---|
| 65d3a18edc | |||
|
|
5e0189be61 | ||
| 9708d36d51 | |||
| 247ba97a3a | |||
| fc6ceb558b | |||
| 676c86cf43 | |||
| 6810704173 | |||
| 4a1c67c518 | |||
| 3a4ac641cd | |||
| 861983531a | |||
| fe36ea6aaa | |||
|
|
62dd6de60c | ||
| d481b79465 | |||
| b8c65348e8 | |||
| d6902daa49 | |||
|
|
4f803c14f4 | ||
|
|
720d34b663 | ||
| a79c5ad1cc | |||
|
|
bf89c7544f | ||
| 8ec849b103 | |||
|
|
0352bdf516 | ||
| 4a5e5d6612 |
6
Makefile
6
Makefile
@@ -20,7 +20,7 @@ fmt-check:
|
||||
check: test lint fmt-check
|
||||
|
||||
docker:
|
||||
docker build -t notebook-clone .
|
||||
docker build -t source-sentinel .
|
||||
|
||||
hooks:
|
||||
printf '#!/usr/bin/env bash\nset -euo pipefail\nmake check\n' > .git/hooks/pre-commit
|
||||
@@ -28,12 +28,12 @@ hooks:
|
||||
|
||||
dev:
|
||||
@echo "Starting backend and frontend..."
|
||||
cd $(SERVER_DIR) && node src/index.js &
|
||||
cd $(SERVER_DIR) && npx --no-install tsx src/index.ts &
|
||||
cd $(WEB_DIR) && npx --no-install vite --port 5173 &
|
||||
wait
|
||||
|
||||
run:
|
||||
cd $(SERVER_DIR) && node src/index.js
|
||||
cd $(SERVER_DIR) && npx --no-install tsx src/index.ts
|
||||
|
||||
build:
|
||||
cd $(WEB_DIR) && npx --no-install vite build
|
||||
|
||||
95
README.md
95
README.md
@@ -1,99 +1,156 @@
|
||||
# Citation Sentinel
|
||||
# Citation Sentinel - 2025 @sjDev - LICENSE: MIT
|
||||
|
||||
A source-grounded research assistant that employs cosine similarity scoring for LLM-generated query responses to provide a "groundedness" score. This is a metric in Retrieval-Augmented Generation (RAG) systems that quantifies how well an AI-generated answer is supported by retrieved context. It measures "faithfulness" to source documents, ensuring the answer is not hallucinated or pulled from the model's pre-training data.
|
||||
|
||||
Users upload source documents, or provide links to online sources including audio/video (i.e. links to youtube videos). Users may then ask questions and receive answers (with inline citations) verifiably grounded in the provided information sources.
|
||||

|
||||
|
||||
Image one: source-grounded research assistant for applied use of LLMs.
|
||||
|
||||
**Citation Sentinel leverages multiple Large Language Models for analyzing and conveniently working with large data corpora, when query-result accuracy is of the highest value. Examples include complex litigation (i.e. Patent/IP), technical medical data exploration (reviewing health records, test results, or studies to find hidden patterns, trends, and answers), advanced LLM development and refinement.**
|
||||
|
||||
Citation Sentinel empowers users to transform an otherwise-unmanageably-large corpus of data on any topic of interest into refined, easily-digested subtopics and pose focused inquiries, to instantly receive concise, quantifiably-evaluated, accurate responses. The system also provides suggested follow-up questions further refining in user inquiries by detecting the query goals.
|
||||
|
||||
ALSO NOTE IN DEMO: clickable inline citations (in blue) that take user to source-grounded research basis/knowledge corpus supporting query response, with rated “groundedness” score.
|
||||
|
||||
|
||||
# Reliability and Understandability
|
||||
|
||||
|
||||
Citation Sentinel's Retrieval-Augmented Generation (RAG) pipeline uses two-stage pass-through to Voyage AI embedding and ranking models, which uses cosine similarity methodology to scoring to evaluate query-source embeddings in multi-dimensional vector space. This yields a "groundedness" score.
|
||||
|
||||
|
||||
Groundedness metrics quantify how well an AI-generated answer is supported by retrieved context. It measures "faithfulness" to source documents, ensuring the answer is not hallucinated, inaccurate or stale (pulled from the model's training data).
|
||||
|
||||
|
||||
# Architecture - Basic Overview
|
||||
|
||||
|
||||
Built with a React/Vite frontend and a Typescript/Node/Express backend, using Anthropic Claude for NL response generation, OpenAI Whisper for video audio track transcription, Voyage AI voyage-3 for embeddings and Voyage AI rerank-2 for response cosine similarity scoring (to generate the "groundedness" score) presented as an easily-understandable red/yellow/green "badge" style tooltip in the response (see image above.)
|
||||
|
||||
Built with a React/Vite frontend and a Typescript/Node/Express backend, using Anthropic Claude for generation, OpenAI Whisper for video audio track transcription, Voyage AI for embeddings and response cosine similarity scoring (the "groundedness" score).
|
||||
|
||||
## Query pipeline
|
||||
|
||||
|
||||
Source Ingestion -> Parsing -> Chunking -> Embedding (using Voyage AI voyage-3 model) -> Storage (vector store) -> { user query submission } -> Evaluation of User Query -> Retrieval -> Ranking -> Response Generation (Using Anthopic's claude-opus-4-6 model) -> Response Groundedness Scoring (using Voyage AI rerank-r model)
|
||||
|
||||
## Why Database Math Can't Replace Groundedness Scoring
|
||||
|
||||
Many recent additions to the Vector database/store product space use distance metrics (Cosine, L2, Inner Product) for Bi-Encoder Similarity. They compare the vector of the user query against the vectors of chunks independently.
|
||||
|
||||
The rerank-r model at the end of the Source Sentinel pipeline performs Cross-Encoder Evaluation, taking two entirely separate text inputs:
|
||||
|
||||
1) The generated response from claude-opus-4-6 and
|
||||
|
||||
2) The raw source chunks
|
||||
|
||||
It processes them simultaneously through deep attention layers to check for hallucinations, missing context, and factual alignment.
|
||||
|
||||
A vector database cosine metric only calculates how close two embeddings are in coordinate space. It **does not read the generated text of a response to verify if it accurately reflects the source chunks**.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
|
||||
- **Node.js** (v18+)
|
||||
- **yt-dlp** -- Required for YouTube video source support (`brew install yt-dlp`)
|
||||
|
||||
|
||||
## Getting Started
|
||||
|
||||
|
||||
```bash
|
||||
# clone and install
|
||||
git clone <repo-url> && cd citation_sentinel
|
||||
cd server && npm install && cd ..
|
||||
cd client && npm install && cd ..
|
||||
|
||||
|
||||
# configure
|
||||
cp server/.env.example server/.env
|
||||
# edit server/.env and add your ANTHROPIC_API_KEY, VOYAGE_API_KEY,
|
||||
# and optionally OPENAI_API_KEY (only needed for audio source transcription via Whisper)
|
||||
|
||||
|
||||
# run
|
||||
make dev
|
||||
```
|
||||
|
||||
|
||||
## Rationale
|
||||
|
||||
|
||||
This app demonstrates the core source-grounded Q&A pattern with transparent retrieval, generation, and groundedness scoring (LLM response quality cosine scoring) -- all with swappable models and is fully open source.
|
||||
|
||||
|
||||
## Methodology
|
||||
|
||||
|
||||
This is a RAG (Retrieval-Augmented Generation) and query-response source-groundedness assurance system allowing users to create a research knowledge corpus, including:
|
||||
|
||||
|
||||
1. Documents — PDFs, DOCX files, plain text audio files.
|
||||
2. Internet sources: via web URLs.
|
||||
2. Audio sources: i.e. YouTube videos, audio from which is transcribed to text.
|
||||
|
||||
|
||||
These are then parsed, split into ~2000-character overlapping chunks, and converted into vector embeddings using Voyage AI's voyage-3 model.
|
||||
|
||||
|
||||
The embeddings are stored in an in-memory vector store.
|
||||
|
||||
|
||||
When a user submits a query, the system enforces groundedness through a multi-layered strategy:
|
||||
|
||||
1. **Retrieval constraint** — The query is embedded (via Voyage AI voyage-3) and compared against
|
||||
stored chunk vectors using cosine similarity, returning the top 20 candidates.
|
||||
|
||||
2. **Reranking for precision** — Those 20 candidates are sent to Voyage AI's rerank-2 cross-encoder,
|
||||
which re-scores each query-chunk pair with deeper semantic analysis. Only the top 5 survive.
|
||||
1. **Retrieval constraint** — The query is embedded (via Voyage AI voyage-3) and compared against stored chunk vectors using cosine similarity, returning the top 20 candidates.
|
||||
|
||||
3. **Prompt-level constraint** — The top 5 chunks are passed to the "Primary LLM" (Claude opus-4-6)
|
||||
for natural language query response.
|
||||
The Primary LLM must cite sources using bracketed indices (e.g., [1], [2]) and admit when sources are
|
||||
insufficient.
|
||||
|
||||
4. **Schema enforcement** — The Primary LLM's response is constrained to a JSON schema requiring structured
|
||||
fields (answer, citedSourceIndices, followUpQuestions). Any cited source indices that do not
|
||||
correspond to real source groups are programmatically removed.
|
||||
2. **Reranking for precision** — Those 20 candidates are sent to Voyage AI's rerank-2 cross-encoder, which re-scores each query-chunk pair with deeper semantic analysis. Only the top 5 survive.
|
||||
|
||||
|
||||
3. **Prompt-level constraint** — The top 5 chunks are passed to the "Primary LLM" (Claude opus-4-6) for natural language query response. The Primary LLM must cite sources using bracketed indices (e.g., [1], [2]) and admit when sources are insufficient.
|
||||
|
||||
|
||||
4. **Schema enforcement** — The Primary LLM's response is constrained to a JSON schema requiring structured fields (answer, citedSourceIndices, followUpQuestions). Any cited source indices that do not correspond to real source groups are programmatically removed.
|
||||
|
||||
|
||||
5. **Post-generation groundedness scoring** — The answer is split into individual sentences. Voyage AI voyage-3 embeds each sentence, and compares it (using cosine similarity) against the vectors of the cited chunks. The similarity is calibrated to a 0–1 scale and averaged, producing a groundedness score surfaced to the user with a visual indicator (“badge” in green/yellow/red).
|
||||
|
||||
5. **Post-generation groundedness scoring** — The answer is split into individual sentences. Voyage
|
||||
AI voyage-3 embeds each sentence, and compares it (using cosine similarity) against the vectors
|
||||
of the cited chunks. The similarity is calibrated to a 0–1 scale and averaged, producing a single
|
||||
groundedness score that is surfaced to the user with a visual indicator (green/yellow/red).
|
||||
|
||||
## Similarity Metrics for Semantic Understanding
|
||||
|
||||
|
||||
Cosine similarity measures how closely two vectors (representing data like words, images, or preferences) are aligned in a multi-dimensional space by calculating the cosine of the angle between them.
|
||||
|
||||
|
||||
Archetypical cosine scores range from -1 to 1.
|
||||
|
||||
|
||||
1: Vectors point in the exact same direction (highly similar).
|
||||
0: Vectors are at a 90-degree angle (orthogonal/unrelated).
|
||||
-1: Vectors point in opposite directions.
|
||||
|
||||
|
||||
Cosine distance between high-dimensional text embeddings is compressed: unrelated notes sit close to orthogonal, so both the within-cluster and nearest-cluster distances land near 0.8. In this application, scores are normalized to a bounded range wherein unrelated query-response pair scores are nearly orthogonal.
|
||||
|
||||
|
||||
## Design
|
||||
|
||||
|
||||
Two-package monorepo:
|
||||
|
||||
|
||||
- `server/` -- single Express backend with layered architecture (routes -> services -> stores). Routes orchestrate; services contain business logic; stores manage in-memory state.
|
||||
|
||||
|
||||
- `client/` -- React 19 SPA via Vite. Two-panel layout: sidebar for data sources, main area for LLM chat with explorable citations, groundedness badges (cosine similarity scoring of LLM responses), and follow-up question chips.
|
||||
|
||||
|
||||
## License
|
||||
|
||||
|
||||
MIT. See [LICENSE](./LICENSE).
|
||||
|
||||
|
||||
## Author
|
||||
|
||||
|
||||
@sjdev
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
<meta charset="UTF-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<link rel="icon" href="data:image/svg+xml,<svg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 100 100'><circle cx='50' cy='50' r='50' fill='black'/></svg>">
|
||||
<title>NotebookLM Clone</title>
|
||||
<title>Source Sentinel</title>
|
||||
</head>
|
||||
<body>
|
||||
<div id="root"></div>
|
||||
|
||||
4
client/package-lock.json
generated
4
client/package-lock.json
generated
@@ -1,11 +1,11 @@
|
||||
{
|
||||
"name": "notebook-clone-web",
|
||||
"name": "source-sentinel-web",
|
||||
"version": "0.1.0",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "notebook-clone-web",
|
||||
"name": "source-sentinel-web",
|
||||
"version": "0.1.0",
|
||||
"dependencies": {
|
||||
"react": "^19.0.0",
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
{
|
||||
"name": "notebook-clone-web",
|
||||
"name": "source-sentinel-web",
|
||||
"version": "0.1.0",
|
||||
"private": true,
|
||||
"engines": { "node": ">=18" },
|
||||
|
||||
@@ -1,20 +1,20 @@
|
||||
import { useState } from 'react';
|
||||
import NotebookList from './components/NotebookList.jsx';
|
||||
import SourceCorpusList from './components/SourceCorpusList.jsx';
|
||||
import SourcePanel from './components/SourcePanel.jsx';
|
||||
import DocumentButtons from './components/DocumentButtons.jsx';
|
||||
import ChatPanel from './components/ChatPanel.jsx';
|
||||
import DocumentModal from './components/DocumentModal.jsx';
|
||||
import { useNotebook } from './hooks/useNotebook.js';
|
||||
import { useSourceCorpus } from './hooks/useSourceCorpus.js';
|
||||
import { generateDocument } from './api/documents.js';
|
||||
|
||||
function App() {
|
||||
const {
|
||||
notebooks,
|
||||
activeNotebook,
|
||||
selectNotebook,
|
||||
createNotebook,
|
||||
deleteNotebook,
|
||||
} = useNotebook();
|
||||
sourceCorpora,
|
||||
activeSourceCorpus,
|
||||
selectSourceCorpus,
|
||||
createSourceCorpus,
|
||||
deleteSourceCorpus,
|
||||
} = useSourceCorpus();
|
||||
|
||||
const [hoverState, setHoverState] = useState(null);
|
||||
const [sourceCount, setSourceCount] = useState(0);
|
||||
@@ -22,9 +22,9 @@ function App() {
|
||||
const [docModal, setDocModal] = useState({ open: false, type: null, document: null, loading: false });
|
||||
const [chatReady, setChatReady] = useState(false);
|
||||
|
||||
const handleSelectNotebook = (id) => {
|
||||
const handleSelectSourceCorpus = (id) => {
|
||||
setChatReady(false);
|
||||
selectNotebook(id);
|
||||
selectSourceCorpus(id);
|
||||
};
|
||||
|
||||
const handleSourceHover = (val) => {
|
||||
@@ -42,7 +42,7 @@ function App() {
|
||||
setDocModal({ open: true, type, document: null, loading: true });
|
||||
|
||||
try {
|
||||
const res = await generateDocument(activeNotebook.id, type);
|
||||
const res = await generateDocument(activeSourceCorpus.id, type);
|
||||
setDocModal({ open: true, type, document: res.document, loading: false });
|
||||
} catch (err) {
|
||||
console.error('document generation failed', err);
|
||||
@@ -58,17 +58,17 @@ function App() {
|
||||
return (
|
||||
<div className="app">
|
||||
<aside className="sidebar">
|
||||
<NotebookList
|
||||
notebooks={notebooks}
|
||||
activeId={activeNotebook?.id}
|
||||
onSelect={handleSelectNotebook}
|
||||
onCreate={createNotebook}
|
||||
onDelete={deleteNotebook}
|
||||
<SourceCorpusList
|
||||
sourceCorpora={sourceCorpora}
|
||||
activeId={activeSourceCorpus?.id}
|
||||
onSelect={handleSelectSourceCorpus}
|
||||
onCreate={createSourceCorpus}
|
||||
onDelete={deleteSourceCorpus}
|
||||
/>
|
||||
{activeNotebook && (
|
||||
{activeSourceCorpus && (
|
||||
<>
|
||||
<SourcePanel
|
||||
notebookId={activeNotebook.id}
|
||||
sourceCorpusId={activeSourceCorpus.id}
|
||||
hoveredSourceIndex={hoveredDocIndex}
|
||||
onSourceHover={handleSourceHover}
|
||||
onSourcesChange={setSourceCount}
|
||||
@@ -85,16 +85,16 @@ function App() {
|
||||
)}
|
||||
</aside>
|
||||
<main className="main">
|
||||
{activeNotebook ? (
|
||||
{activeSourceCorpus ? (
|
||||
<ChatPanel
|
||||
notebookId={activeNotebook.id}
|
||||
sourceCorpusId={activeSourceCorpus.id}
|
||||
hoveredSource={hoveredInstanceId}
|
||||
hoveredDocIndex={hoveredDocIndex}
|
||||
onSourceHover={handleSourceHover}
|
||||
onFirstResponse={() => setChatReady(true)}
|
||||
/>
|
||||
) : (
|
||||
<p>Select or create a notebook to get started.</p>
|
||||
<p>Select or create a source corpus to get started.</p>
|
||||
)}
|
||||
</main>
|
||||
<DocumentModal
|
||||
|
||||
@@ -4,11 +4,11 @@ import { handleResponse } from './client.js';
|
||||
|
||||
const BASE = '/api/documents';
|
||||
|
||||
export async function generateDocument(notebookId, type) {
|
||||
export async function generateDocument(sourceCorpusId, type) {
|
||||
const res = await fetch(`${BASE}/generate`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ notebookId, type }),
|
||||
body: JSON.stringify({ sourceCorpusId, type }),
|
||||
});
|
||||
return handleResponse(res);
|
||||
}
|
||||
|
||||
@@ -5,11 +5,11 @@ import { handleResponse } from './client.js';
|
||||
const BASE = '/api/query';
|
||||
const CITATION_DETAIL_BASE = '/api/citation-detail';
|
||||
|
||||
export async function sendQuery(notebookId, question, onCitationDetails) {
|
||||
export async function sendQuery(sourceCorpusId, question, onCitationDetails) {
|
||||
const res = await fetch(BASE, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ notebookId, question }),
|
||||
body: JSON.stringify({ sourceCorpusId, question }),
|
||||
});
|
||||
const data = await handleResponse(res);
|
||||
|
||||
|
||||
@@ -16,7 +16,7 @@ describe('sendQuery', () => {
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
it('sends POST to /api/query with notebookId and question', async () => {
|
||||
it('sends POST to /api/query with sourceCorpusId and question', async () => {
|
||||
const data = { answer: 'The answer', citations: [] };
|
||||
fetch.mockResolvedValue(okResponse(data));
|
||||
|
||||
@@ -25,7 +25,7 @@ describe('sendQuery', () => {
|
||||
expect(fetch).toHaveBeenCalledWith('/api/query', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ notebookId: 'nb-1', question: 'What is AI?' }),
|
||||
body: JSON.stringify({ sourceCorpusId: 'nb-1', question: 'What is AI?' }),
|
||||
});
|
||||
expect(result).toEqual(data);
|
||||
});
|
||||
|
||||
@@ -2,14 +2,14 @@
|
||||
|
||||
import { handleResponse } from './client.js';
|
||||
|
||||
const BASE = '/api/notebooks';
|
||||
const BASE = '/api/source-corpus';
|
||||
|
||||
export async function listNotebooks() {
|
||||
export async function listSourceCorpora() {
|
||||
const res = await fetch(BASE);
|
||||
return handleResponse(res);
|
||||
}
|
||||
|
||||
export async function createNotebook(name) {
|
||||
export async function createSourceCorpus(name) {
|
||||
const res = await fetch(BASE, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
@@ -18,7 +18,7 @@ export async function createNotebook(name) {
|
||||
return handleResponse(res);
|
||||
}
|
||||
|
||||
export async function deleteNotebook(id) {
|
||||
export async function deleteSourceCorpus(id) {
|
||||
const res = await fetch(`${BASE}/${id}`, { method: 'DELETE' });
|
||||
return handleResponse(res);
|
||||
}
|
||||
@@ -1,5 +1,5 @@
|
||||
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
|
||||
import { listNotebooks, createNotebook, deleteNotebook } from './notebooks.js';
|
||||
import { listSourceCorpora, createSourceCorpus, deleteSourceCorpus } from './sourceCorpus.js';
|
||||
|
||||
const okResponse = (body) => ({
|
||||
ok: true,
|
||||
@@ -7,7 +7,7 @@ const okResponse = (body) => ({
|
||||
json: () => Promise.resolve(body),
|
||||
});
|
||||
|
||||
describe('notebooks API', () => {
|
||||
describe('sourceCorpora API', () => {
|
||||
beforeEach(() => {
|
||||
vi.stubGlobal('fetch', vi.fn());
|
||||
});
|
||||
@@ -16,41 +16,41 @@ describe('notebooks API', () => {
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
describe('listNotebooks', () => {
|
||||
it('fetches GET /api/notebooks', async () => {
|
||||
describe('listSourceCorpora', () => {
|
||||
it('fetches GET /api/source-corpus', async () => {
|
||||
const data = [{ id: '1', name: 'NB' }];
|
||||
fetch.mockResolvedValue(okResponse(data));
|
||||
|
||||
const result = await listNotebooks();
|
||||
const result = await listSourceCorpora();
|
||||
|
||||
expect(fetch).toHaveBeenCalledWith('/api/notebooks');
|
||||
expect(fetch).toHaveBeenCalledWith('/api/source-corpus');
|
||||
expect(result).toEqual(data);
|
||||
});
|
||||
});
|
||||
|
||||
describe('createNotebook', () => {
|
||||
describe('createSourceCorpus', () => {
|
||||
it('sends POST with name in JSON body', async () => {
|
||||
const nb = { id: '2', name: 'New' };
|
||||
fetch.mockResolvedValue(okResponse(nb));
|
||||
const corpus = { id: '2', name: 'New' };
|
||||
fetch.mockResolvedValue(okResponse(corpus));
|
||||
|
||||
const result = await createNotebook('New');
|
||||
const result = await createSourceCorpus('New');
|
||||
|
||||
expect(fetch).toHaveBeenCalledWith('/api/notebooks', {
|
||||
expect(fetch).toHaveBeenCalledWith('/api/source-corpus', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ name: 'New' }),
|
||||
});
|
||||
expect(result).toEqual(nb);
|
||||
expect(result).toEqual(corpus);
|
||||
});
|
||||
});
|
||||
|
||||
describe('deleteNotebook', () => {
|
||||
it('sends DELETE to /api/notebooks/:id', async () => {
|
||||
describe('deleteSourceCorpus', () => {
|
||||
it('sends DELETE to /api/source-corpus/:id', async () => {
|
||||
fetch.mockResolvedValue(okResponse({}));
|
||||
|
||||
await deleteNotebook('abc-123');
|
||||
await deleteSourceCorpus('abc-123');
|
||||
|
||||
expect(fetch).toHaveBeenCalledWith('/api/notebooks/abc-123', {
|
||||
expect(fetch).toHaveBeenCalledWith('/api/source-corpus/abc-123', {
|
||||
method: 'DELETE',
|
||||
});
|
||||
});
|
||||
@@ -63,6 +63,6 @@ describe('notebooks API', () => {
|
||||
json: () => Promise.resolve({ error: 'Server down' }),
|
||||
});
|
||||
|
||||
await expect(listNotebooks()).rejects.toThrow('Server down');
|
||||
await expect(listSourceCorpora()).rejects.toThrow('Server down');
|
||||
});
|
||||
});
|
||||
@@ -4,15 +4,15 @@ import { handleResponse } from './client.js';
|
||||
|
||||
const BASE = '/api/sources';
|
||||
|
||||
export async function listSources(notebookId) {
|
||||
const res = await fetch(`${BASE}?notebookId=${notebookId}`);
|
||||
export async function listSources(sourceCorpusId) {
|
||||
const res = await fetch(`${BASE}?sourceCorpusId=${sourceCorpusId}`);
|
||||
return handleResponse(res);
|
||||
}
|
||||
|
||||
export async function uploadSource(notebookId, file) {
|
||||
export async function uploadSource(sourceCorpusId, file) {
|
||||
const formData = new FormData();
|
||||
formData.append('file', file);
|
||||
formData.append('notebookId', notebookId);
|
||||
formData.append('sourceCorpusId', sourceCorpusId);
|
||||
const res = await fetch(BASE, {
|
||||
method: 'POST',
|
||||
body: formData,
|
||||
@@ -20,11 +20,11 @@ export async function uploadSource(notebookId, file) {
|
||||
return handleResponse(res);
|
||||
}
|
||||
|
||||
export async function addUrlSource(notebookId, url) {
|
||||
export async function addUrlSource(sourceCorpusId, url) {
|
||||
const res = await fetch(`${BASE}/url`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ notebookId, url }),
|
||||
body: JSON.stringify({ sourceCorpusId, url }),
|
||||
});
|
||||
return handleResponse(res);
|
||||
}
|
||||
|
||||
@@ -17,19 +17,19 @@ describe('sources API', () => {
|
||||
});
|
||||
|
||||
describe('listSources', () => {
|
||||
it('fetches GET /api/sources with notebookId query param', async () => {
|
||||
it('fetches GET /api/sources with sourceCorpusId query param', async () => {
|
||||
const data = [{ id: 's1', name: 'file.pdf' }];
|
||||
fetch.mockResolvedValue(okResponse(data));
|
||||
|
||||
const result = await listSources('nb-42');
|
||||
|
||||
expect(fetch).toHaveBeenCalledWith('/api/sources?notebookId=nb-42');
|
||||
expect(fetch).toHaveBeenCalledWith('/api/sources?sourceCorpusId=nb-42');
|
||||
expect(result).toEqual(data);
|
||||
});
|
||||
});
|
||||
|
||||
describe('uploadSource', () => {
|
||||
it('sends POST with FormData containing file and notebookId', async () => {
|
||||
it('sends POST with FormData containing file and sourceCorpusId', async () => {
|
||||
const source = { id: 's2', name: 'doc.pdf' };
|
||||
fetch.mockResolvedValue(okResponse(source));
|
||||
|
||||
@@ -42,7 +42,7 @@ describe('sources API', () => {
|
||||
});
|
||||
|
||||
const formData = fetch.mock.calls[0][1].body;
|
||||
expect(formData.get('notebookId')).toBe('nb-1');
|
||||
expect(formData.get('sourceCorpusId')).toBe('nb-1');
|
||||
expect(formData.get('file')).toBeInstanceOf(File);
|
||||
expect(result).toEqual(source);
|
||||
});
|
||||
@@ -58,7 +58,7 @@ describe('sources API', () => {
|
||||
expect(fetch).toHaveBeenCalledWith('/api/sources/url', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ notebookId: 'nb-5', url: 'https://example.com' }),
|
||||
body: JSON.stringify({ sourceCorpusId: 'nb-5', url: 'https://example.com' }),
|
||||
});
|
||||
expect(result).toEqual(source);
|
||||
});
|
||||
|
||||
@@ -3,7 +3,7 @@ import { sendQuery } from '../api/query.js';
|
||||
import ChatMessage from './ChatMessage.jsx';
|
||||
import CitationDetailModal from './CitationDetailModal.jsx';
|
||||
|
||||
function ChatPanel({ notebookId, hoveredSource, hoveredDocIndex, onSourceHover, onFirstResponse }) {
|
||||
function ChatPanel({ sourceCorpusId, hoveredSource, hoveredDocIndex, onSourceHover, onFirstResponse }) {
|
||||
const [messages, setMessages] = useState([]);
|
||||
const [input, setInput] = useState('');
|
||||
const [loading, setLoading] = useState(false);
|
||||
@@ -18,7 +18,7 @@ function ChatPanel({ notebookId, hoveredSource, hoveredDocIndex, onSourceHover,
|
||||
setCitationDetails({});
|
||||
setActiveCitation(null);
|
||||
firstResponseFired.current = false;
|
||||
}, [notebookId]);
|
||||
}, [sourceCorpusId]);
|
||||
|
||||
useEffect(() => {
|
||||
bottomRef.current?.scrollIntoView({ behavior: 'smooth' });
|
||||
@@ -35,7 +35,7 @@ function ChatPanel({ notebookId, hoveredSource, hoveredDocIndex, onSourceHover,
|
||||
const msgId = crypto.randomUUID();
|
||||
|
||||
try {
|
||||
const res = await sendQuery(notebookId, question, (details) => {
|
||||
const res = await sendQuery(sourceCorpusId, question, (details) => {
|
||||
const detailMap = {};
|
||||
for (const d of details) {
|
||||
detailMap[d.sourceIndex] = d;
|
||||
|
||||
@@ -1,99 +0,0 @@
|
||||
import { describe, it, expect, vi } from 'vitest';
|
||||
import { render, screen, waitFor } from '@testing-library/react';
|
||||
import userEvent from '@testing-library/user-event';
|
||||
import CreateNotebookModal from './CreateNotebookModal.jsx';
|
||||
|
||||
describe('CreateNotebookModal', () => {
|
||||
it('renders nothing when open is false', () => {
|
||||
const { container } = render(
|
||||
<CreateNotebookModal open={false} onConfirm={() => {}} onCancel={() => {}} />,
|
||||
);
|
||||
expect(container.innerHTML).toBe('');
|
||||
});
|
||||
|
||||
it('renders the modal with title when open', () => {
|
||||
render(<CreateNotebookModal open={true} onConfirm={() => {}} onCancel={() => {}} />);
|
||||
expect(screen.getByText('New Notebook')).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it('renders input and buttons', () => {
|
||||
render(<CreateNotebookModal open={true} onConfirm={() => {}} onCancel={() => {}} />);
|
||||
expect(screen.getByPlaceholderText('Enter notebook name')).toBeInTheDocument();
|
||||
expect(screen.getByText('Cancel')).toBeInTheDocument();
|
||||
expect(screen.getByText('Create')).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it('disables Create button when input is empty', () => {
|
||||
render(<CreateNotebookModal open={true} onConfirm={() => {}} onCancel={() => {}} />);
|
||||
expect(screen.getByText('Create')).toBeDisabled();
|
||||
});
|
||||
|
||||
it('enables Create button when input has text', async () => {
|
||||
const user = userEvent.setup();
|
||||
render(<CreateNotebookModal open={true} onConfirm={() => {}} onCancel={() => {}} />);
|
||||
|
||||
await user.type(screen.getByPlaceholderText('Enter notebook name'), 'My Notebook');
|
||||
expect(screen.getByText('Create')).toBeEnabled();
|
||||
});
|
||||
|
||||
it('calls onConfirm with trimmed name on submit', async () => {
|
||||
const user = userEvent.setup();
|
||||
const onConfirm = vi.fn();
|
||||
render(<CreateNotebookModal open={true} onConfirm={onConfirm} onCancel={() => {}} />);
|
||||
|
||||
await user.type(screen.getByPlaceholderText('Enter notebook name'), ' Research Notes ');
|
||||
await user.click(screen.getByText('Create'));
|
||||
expect(onConfirm).toHaveBeenCalledWith('Research Notes');
|
||||
});
|
||||
|
||||
it('calls onConfirm on Enter key', async () => {
|
||||
const user = userEvent.setup();
|
||||
const onConfirm = vi.fn();
|
||||
render(<CreateNotebookModal open={true} onConfirm={onConfirm} onCancel={() => {}} />);
|
||||
|
||||
await user.type(screen.getByPlaceholderText('Enter notebook name'), 'Test{Enter}');
|
||||
expect(onConfirm).toHaveBeenCalledWith('Test');
|
||||
});
|
||||
|
||||
it('does not call onConfirm when input is only whitespace', async () => {
|
||||
const user = userEvent.setup();
|
||||
const onConfirm = vi.fn();
|
||||
render(<CreateNotebookModal open={true} onConfirm={onConfirm} onCancel={() => {}} />);
|
||||
|
||||
const input = screen.getByPlaceholderText('Enter notebook name');
|
||||
await user.type(input, ' ');
|
||||
await user.type(input, '{Enter}');
|
||||
expect(onConfirm).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('calls onCancel when Cancel button is clicked', async () => {
|
||||
const user = userEvent.setup();
|
||||
const onCancel = vi.fn();
|
||||
render(<CreateNotebookModal open={true} onConfirm={() => {}} onCancel={onCancel} />);
|
||||
|
||||
await user.click(screen.getByText('Cancel'));
|
||||
expect(onCancel).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it('resets input when reopened', async () => {
|
||||
const user = userEvent.setup();
|
||||
const { rerender } = render(
|
||||
<CreateNotebookModal open={true} onConfirm={() => {}} onCancel={() => {}} />,
|
||||
);
|
||||
|
||||
await user.type(screen.getByPlaceholderText('Enter notebook name'), 'Old name');
|
||||
|
||||
rerender(<CreateNotebookModal open={false} onConfirm={() => {}} onCancel={() => {}} />);
|
||||
rerender(<CreateNotebookModal open={true} onConfirm={() => {}} onCancel={() => {}} />);
|
||||
|
||||
expect(screen.getByPlaceholderText('Enter notebook name')).toHaveValue('');
|
||||
});
|
||||
|
||||
it('auto-focuses the input when opened', async () => {
|
||||
render(<CreateNotebookModal open={true} onConfirm={() => {}} onCancel={() => {}} />);
|
||||
|
||||
await waitFor(() => {
|
||||
expect(screen.getByPlaceholderText('Enter notebook name')).toHaveFocus();
|
||||
}, { timeout: 200 });
|
||||
});
|
||||
});
|
||||
@@ -1,7 +1,7 @@
|
||||
import { useState, useEffect, useRef } from 'react';
|
||||
import Modal from './Modal.jsx';
|
||||
|
||||
function CreateNotebookModal({ open, onConfirm, onCancel }) {
|
||||
function CreateSourceCorpusModal({ open, onConfirm, onCancel }) {
|
||||
const [name, setName] = useState('');
|
||||
const inputRef = useRef(null);
|
||||
|
||||
@@ -19,13 +19,13 @@ function CreateNotebookModal({ open, onConfirm, onCancel }) {
|
||||
};
|
||||
|
||||
return (
|
||||
<Modal open={open} onClose={onCancel} title="New Notebook">
|
||||
<Modal open={open} onClose={onCancel} title="New Source Corpus">
|
||||
<form onSubmit={handleSubmit}>
|
||||
<input
|
||||
ref={inputRef}
|
||||
className="modal-input"
|
||||
type="text"
|
||||
placeholder="Enter notebook name"
|
||||
placeholder="Enter source corpus name"
|
||||
value={name}
|
||||
onChange={(e) => setName(e.target.value)}
|
||||
/>
|
||||
@@ -50,4 +50,4 @@ function CreateNotebookModal({ open, onConfirm, onCancel }) {
|
||||
);
|
||||
}
|
||||
|
||||
export default CreateNotebookModal;
|
||||
export default CreateSourceCorpusModal;
|
||||
99
client/src/components/CreateSourceCorpusModal.test.jsx
Normal file
99
client/src/components/CreateSourceCorpusModal.test.jsx
Normal file
@@ -0,0 +1,99 @@
|
||||
import { describe, it, expect, vi } from 'vitest';
|
||||
import { render, screen, waitFor } from '@testing-library/react';
|
||||
import userEvent from '@testing-library/user-event';
|
||||
import CreateSourceCorpusModal from './CreateSourceCorpusModal.jsx';
|
||||
|
||||
describe('CreateSourceCorpusModal', () => {
|
||||
it('renders nothing when open is false', () => {
|
||||
const { container } = render(
|
||||
<CreateSourceCorpusModal open={false} onConfirm={() => {}} onCancel={() => {}} />,
|
||||
);
|
||||
expect(container.innerHTML).toBe('');
|
||||
});
|
||||
|
||||
it('renders the modal with title when open', () => {
|
||||
render(<CreateSourceCorpusModal open={true} onConfirm={() => {}} onCancel={() => {}} />);
|
||||
expect(screen.getByText('New Source Corpus')).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it('renders input and buttons', () => {
|
||||
render(<CreateSourceCorpusModal open={true} onConfirm={() => {}} onCancel={() => {}} />);
|
||||
expect(screen.getByPlaceholderText('Enter source corpus name')).toBeInTheDocument();
|
||||
expect(screen.getByText('Cancel')).toBeInTheDocument();
|
||||
expect(screen.getByText('Create')).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it('disables Create button when input is empty', () => {
|
||||
render(<CreateSourceCorpusModal open={true} onConfirm={() => {}} onCancel={() => {}} />);
|
||||
expect(screen.getByText('Create')).toBeDisabled();
|
||||
});
|
||||
|
||||
it('enables Create button when input has text', async () => {
|
||||
const user = userEvent.setup();
|
||||
render(<CreateSourceCorpusModal open={true} onConfirm={() => {}} onCancel={() => {}} />);
|
||||
|
||||
await user.type(screen.getByPlaceholderText('Enter source corpus name'), 'My SourceCorpus');
|
||||
expect(screen.getByText('Create')).toBeEnabled();
|
||||
});
|
||||
|
||||
it('calls onConfirm with trimmed name on submit', async () => {
|
||||
const user = userEvent.setup();
|
||||
const onConfirm = vi.fn();
|
||||
render(<CreateSourceCorpusModal open={true} onConfirm={onConfirm} onCancel={() => {}} />);
|
||||
|
||||
await user.type(screen.getByPlaceholderText('Enter source corpus name'), ' Research Notes ');
|
||||
await user.click(screen.getByText('Create'));
|
||||
expect(onConfirm).toHaveBeenCalledWith('Research Notes');
|
||||
});
|
||||
|
||||
it('calls onConfirm on Enter key', async () => {
|
||||
const user = userEvent.setup();
|
||||
const onConfirm = vi.fn();
|
||||
render(<CreateSourceCorpusModal open={true} onConfirm={onConfirm} onCancel={() => {}} />);
|
||||
|
||||
await user.type(screen.getByPlaceholderText('Enter source corpus name'), 'Test{Enter}');
|
||||
expect(onConfirm).toHaveBeenCalledWith('Test');
|
||||
});
|
||||
|
||||
it('does not call onConfirm when input is only whitespace', async () => {
|
||||
const user = userEvent.setup();
|
||||
const onConfirm = vi.fn();
|
||||
render(<CreateSourceCorpusModal open={true} onConfirm={onConfirm} onCancel={() => {}} />);
|
||||
|
||||
const input = screen.getByPlaceholderText('Enter source corpus name');
|
||||
await user.type(input, ' ');
|
||||
await user.type(input, '{Enter}');
|
||||
expect(onConfirm).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('calls onCancel when Cancel button is clicked', async () => {
|
||||
const user = userEvent.setup();
|
||||
const onCancel = vi.fn();
|
||||
render(<CreateSourceCorpusModal open={true} onConfirm={() => {}} onCancel={onCancel} />);
|
||||
|
||||
await user.click(screen.getByText('Cancel'));
|
||||
expect(onCancel).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it('resets input when reopened', async () => {
|
||||
const user = userEvent.setup();
|
||||
const { rerender } = render(
|
||||
<CreateSourceCorpusModal open={true} onConfirm={() => {}} onCancel={() => {}} />,
|
||||
);
|
||||
|
||||
await user.type(screen.getByPlaceholderText('Enter source corpus name'), 'Old name');
|
||||
|
||||
rerender(<CreateSourceCorpusModal open={false} onConfirm={() => {}} onCancel={() => {}} />);
|
||||
rerender(<CreateSourceCorpusModal open={true} onConfirm={() => {}} onCancel={() => {}} />);
|
||||
|
||||
expect(screen.getByPlaceholderText('Enter source corpus name')).toHaveValue('');
|
||||
});
|
||||
|
||||
it('auto-focuses the input when opened', async () => {
|
||||
render(<CreateSourceCorpusModal open={true} onConfirm={() => {}} onCancel={() => {}} />);
|
||||
|
||||
await waitFor(() => {
|
||||
expect(screen.getByPlaceholderText('Enter source corpus name')).toHaveFocus();
|
||||
}, { timeout: 200 });
|
||||
});
|
||||
});
|
||||
@@ -1,53 +0,0 @@
|
||||
import { useState } from 'react';
|
||||
import CreateNotebookModal from './CreateNotebookModal.jsx';
|
||||
|
||||
function NotebookList({ notebooks, activeId, onSelect, onCreate, onDelete }) {
|
||||
const [modalOpen, setModalOpen] = useState(false);
|
||||
|
||||
const handleDelete = (e, id) => {
|
||||
e.stopPropagation();
|
||||
onDelete(id);
|
||||
};
|
||||
|
||||
const handleConfirm = (name) => {
|
||||
setModalOpen(false);
|
||||
onCreate(name);
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="notebook-list">
|
||||
<h2>Notebooks</h2>
|
||||
<ul>
|
||||
{notebooks.map((nb) => (
|
||||
<li
|
||||
key={nb.id}
|
||||
className={nb.id === activeId ? 'active' : ''}
|
||||
onClick={() => onSelect(nb.id)}
|
||||
>
|
||||
<span className="nb-name">{nb.name}</span>
|
||||
<button
|
||||
className="nb-delete"
|
||||
onClick={(e) => handleDelete(e, nb.id)}
|
||||
title="Delete notebook"
|
||||
>
|
||||
×
|
||||
</button>
|
||||
</li>
|
||||
))}
|
||||
</ul>
|
||||
<button
|
||||
className="btn-new-notebook"
|
||||
onClick={() => setModalOpen(true)}
|
||||
>
|
||||
+ New Notebook
|
||||
</button>
|
||||
<CreateNotebookModal
|
||||
open={modalOpen}
|
||||
onConfirm={handleConfirm}
|
||||
onCancel={() => setModalOpen(false)}
|
||||
/>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
export default NotebookList;
|
||||
53
client/src/components/SourceCorpusList.jsx
Normal file
53
client/src/components/SourceCorpusList.jsx
Normal file
@@ -0,0 +1,53 @@
|
||||
import { useState } from 'react';
|
||||
import CreateSourceCorpusModal from './CreateSourceCorpusModal.jsx';
|
||||
|
||||
function SourceCorpusList({ sourceCorpora, activeId, onSelect, onCreate, onDelete }) {
|
||||
const [modalOpen, setModalOpen] = useState(false);
|
||||
|
||||
const handleDelete = (e, id) => {
|
||||
e.stopPropagation();
|
||||
onDelete(id);
|
||||
};
|
||||
|
||||
const handleConfirm = (name) => {
|
||||
setModalOpen(false);
|
||||
onCreate(name);
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="source-corpus-list">
|
||||
<h2>Source Corpora</h2>
|
||||
<ul>
|
||||
{sourceCorpora.map((corpus) => (
|
||||
<li
|
||||
key={corpus.id}
|
||||
className={corpus.id === activeId ? 'active' : ''}
|
||||
onClick={() => onSelect(corpus.id)}
|
||||
>
|
||||
<span className="sc-name">{corpus.name}</span>
|
||||
<button
|
||||
className="sc-delete"
|
||||
onClick={(e) => handleDelete(e, corpus.id)}
|
||||
title="Delete source corpus"
|
||||
>
|
||||
×
|
||||
</button>
|
||||
</li>
|
||||
))}
|
||||
</ul>
|
||||
<button
|
||||
className="btn-new-corpus"
|
||||
onClick={() => setModalOpen(true)}
|
||||
>
|
||||
+ New Source Corpus
|
||||
</button>
|
||||
<CreateSourceCorpusModal
|
||||
open={modalOpen}
|
||||
onConfirm={handleConfirm}
|
||||
onCancel={() => setModalOpen(false)}
|
||||
/>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
export default SourceCorpusList;
|
||||
@@ -10,7 +10,7 @@ function isFileAllowed(file) {
|
||||
return ALLOWED_EXTENSIONS.includes(ext);
|
||||
}
|
||||
|
||||
function SourcePanel({ notebookId, hoveredSourceIndex, onSourceHover, onSourcesChange, children }) {
|
||||
function SourcePanel({ sourceCorpusId, hoveredSourceIndex, onSourceHover, onSourcesChange, children }) {
|
||||
const [sources, setSources] = useState([]);
|
||||
const [uploading, setUploading] = useState(false);
|
||||
const [isDragging, setIsDragging] = useState(false);
|
||||
@@ -30,11 +30,11 @@ function SourcePanel({ notebookId, hoveredSourceIndex, onSourceHover, onSourcesC
|
||||
useEffect(() => {
|
||||
setSources([]);
|
||||
onSourcesChange?.(0);
|
||||
sourcesApi.listSources(notebookId).then((s) => {
|
||||
sourcesApi.listSources(sourceCorpusId).then((s) => {
|
||||
setSources(s);
|
||||
onSourcesChange?.(s.length);
|
||||
}).catch((err) => showError(err.message || 'Failed to load sources'));
|
||||
}, [notebookId, onSourcesChange]);
|
||||
}, [sourceCorpusId, onSourcesChange]);
|
||||
|
||||
useEffect(() => () => clearTimeout(errorTimer.current), []);
|
||||
|
||||
@@ -47,7 +47,7 @@ function SourcePanel({ notebookId, hoveredSourceIndex, onSourceHover, onSourcesC
|
||||
setError(null);
|
||||
setUploading(true);
|
||||
try {
|
||||
const src = await sourcesApi.uploadSource(notebookId, file);
|
||||
const src = await sourcesApi.uploadSource(sourceCorpusId, file);
|
||||
setSources((prev) => {
|
||||
const next = [...prev, src];
|
||||
onSourcesChange?.(next.length);
|
||||
@@ -72,7 +72,7 @@ function SourcePanel({ notebookId, hoveredSourceIndex, onSourceHover, onSourcesC
|
||||
setError(null);
|
||||
setAddingUrl(true);
|
||||
try {
|
||||
const src = await sourcesApi.addUrlSource(notebookId, url);
|
||||
const src = await sourcesApi.addUrlSource(sourceCorpusId, url);
|
||||
if (src.error) {
|
||||
showError(src.error);
|
||||
} else {
|
||||
|
||||
@@ -1,50 +0,0 @@
|
||||
import { useState, useEffect, useCallback } from 'react';
|
||||
import * as notebooksApi from '../api/notebooks.js';
|
||||
|
||||
export function useNotebook() {
|
||||
const [notebooks, setNotebooks] = useState([]);
|
||||
const [activeNotebook, setActiveNotebook] = useState(null);
|
||||
|
||||
useEffect(() => {
|
||||
notebooksApi.listNotebooks().then(setNotebooks).catch(console.error);
|
||||
}, []);
|
||||
|
||||
const selectNotebook = useCallback(
|
||||
(id) => {
|
||||
const nb = notebooks.find((n) => n.id === id) || null;
|
||||
setActiveNotebook(nb);
|
||||
},
|
||||
[notebooks],
|
||||
);
|
||||
|
||||
const createNotebook = useCallback(async (name) => {
|
||||
if (!name) return;
|
||||
const nb = await notebooksApi.createNotebook(name);
|
||||
setNotebooks((prev) => [...prev, nb]);
|
||||
setActiveNotebook(nb);
|
||||
}, []);
|
||||
|
||||
const deleteNotebook = useCallback(
|
||||
async (id) => {
|
||||
try {
|
||||
await notebooksApi.deleteNotebook(id);
|
||||
} catch (err) {
|
||||
console.error('Failed to delete notebook', err);
|
||||
return;
|
||||
}
|
||||
setNotebooks((prev) => prev.filter((n) => n.id !== id));
|
||||
if (activeNotebook?.id === id) {
|
||||
setActiveNotebook(null);
|
||||
}
|
||||
},
|
||||
[activeNotebook],
|
||||
);
|
||||
|
||||
return {
|
||||
notebooks,
|
||||
activeNotebook,
|
||||
selectNotebook,
|
||||
createNotebook,
|
||||
deleteNotebook,
|
||||
};
|
||||
}
|
||||
@@ -1,176 +0,0 @@
|
||||
import { describe, it, expect, vi, beforeEach } from 'vitest';
|
||||
import { renderHook, act, waitFor } from '@testing-library/react';
|
||||
|
||||
vi.mock('../api/notebooks.js', () => ({
|
||||
listNotebooks: vi.fn(),
|
||||
createNotebook: vi.fn(),
|
||||
deleteNotebook: vi.fn(),
|
||||
}));
|
||||
|
||||
import { useNotebook } from './useNotebook.js';
|
||||
import * as notebooksApi from '../api/notebooks.js';
|
||||
|
||||
const notebooks = [
|
||||
{ id: 'nb-1', name: 'Research' },
|
||||
{ id: 'nb-2', name: 'Personal' },
|
||||
];
|
||||
|
||||
beforeEach(() => {
|
||||
vi.resetAllMocks();
|
||||
notebooksApi.listNotebooks.mockResolvedValue(notebooks);
|
||||
notebooksApi.createNotebook.mockImplementation(async (name) => ({
|
||||
id: `nb-${Date.now()}`,
|
||||
name,
|
||||
}));
|
||||
notebooksApi.deleteNotebook.mockResolvedValue({});
|
||||
});
|
||||
|
||||
describe('useNotebook', () => {
|
||||
it('loads notebooks on mount', async () => {
|
||||
const { result } = renderHook(() => useNotebook());
|
||||
|
||||
await waitFor(() => {
|
||||
expect(result.current.notebooks).toEqual(notebooks);
|
||||
});
|
||||
expect(notebooksApi.listNotebooks).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it('starts with no active notebook', () => {
|
||||
const { result } = renderHook(() => useNotebook());
|
||||
expect(result.current.activeNotebook).toBeNull();
|
||||
});
|
||||
|
||||
it('selectNotebook sets the active notebook', async () => {
|
||||
const { result } = renderHook(() => useNotebook());
|
||||
|
||||
await waitFor(() => {
|
||||
expect(result.current.notebooks).toHaveLength(2);
|
||||
});
|
||||
|
||||
act(() => {
|
||||
result.current.selectNotebook('nb-2');
|
||||
});
|
||||
|
||||
expect(result.current.activeNotebook).toEqual({ id: 'nb-2', name: 'Personal' });
|
||||
});
|
||||
|
||||
it('selectNotebook sets null for unknown id', async () => {
|
||||
const { result } = renderHook(() => useNotebook());
|
||||
|
||||
await waitFor(() => {
|
||||
expect(result.current.notebooks).toHaveLength(2);
|
||||
});
|
||||
|
||||
act(() => {
|
||||
result.current.selectNotebook('nb-2');
|
||||
});
|
||||
expect(result.current.activeNotebook).not.toBeNull();
|
||||
|
||||
act(() => {
|
||||
result.current.selectNotebook('nonexistent');
|
||||
});
|
||||
expect(result.current.activeNotebook).toBeNull();
|
||||
});
|
||||
|
||||
it('createNotebook calls API and adds to list', async () => {
|
||||
const newNb = { id: 'nb-new', name: 'New One' };
|
||||
notebooksApi.createNotebook.mockResolvedValue(newNb);
|
||||
|
||||
const { result } = renderHook(() => useNotebook());
|
||||
await waitFor(() => expect(result.current.notebooks).toHaveLength(2));
|
||||
|
||||
await act(async () => {
|
||||
await result.current.createNotebook('New One');
|
||||
});
|
||||
|
||||
expect(notebooksApi.createNotebook).toHaveBeenCalledWith('New One');
|
||||
expect(result.current.notebooks).toHaveLength(3);
|
||||
expect(result.current.notebooks[2]).toEqual(newNb);
|
||||
expect(result.current.activeNotebook).toEqual(newNb);
|
||||
});
|
||||
|
||||
it('createNotebook does nothing for empty name', async () => {
|
||||
const { result } = renderHook(() => useNotebook());
|
||||
await waitFor(() => expect(result.current.notebooks).toHaveLength(2));
|
||||
|
||||
await act(async () => {
|
||||
await result.current.createNotebook('');
|
||||
});
|
||||
|
||||
expect(notebooksApi.createNotebook).not.toHaveBeenCalled();
|
||||
expect(result.current.notebooks).toHaveLength(2);
|
||||
});
|
||||
|
||||
it('deleteNotebook removes from list', async () => {
|
||||
const { result } = renderHook(() => useNotebook());
|
||||
await waitFor(() => expect(result.current.notebooks).toHaveLength(2));
|
||||
|
||||
await act(async () => {
|
||||
await result.current.deleteNotebook('nb-1');
|
||||
});
|
||||
|
||||
expect(notebooksApi.deleteNotebook).toHaveBeenCalledWith('nb-1');
|
||||
expect(result.current.notebooks).toHaveLength(1);
|
||||
expect(result.current.notebooks[0].id).toBe('nb-2');
|
||||
});
|
||||
|
||||
it('deleteNotebook clears activeNotebook if it was the deleted one', async () => {
|
||||
const { result } = renderHook(() => useNotebook());
|
||||
await waitFor(() => expect(result.current.notebooks).toHaveLength(2));
|
||||
|
||||
act(() => {
|
||||
result.current.selectNotebook('nb-1');
|
||||
});
|
||||
expect(result.current.activeNotebook?.id).toBe('nb-1');
|
||||
|
||||
await act(async () => {
|
||||
await result.current.deleteNotebook('nb-1');
|
||||
});
|
||||
|
||||
expect(result.current.activeNotebook).toBeNull();
|
||||
});
|
||||
|
||||
it('deleteNotebook preserves activeNotebook if different one deleted', async () => {
|
||||
const { result } = renderHook(() => useNotebook());
|
||||
await waitFor(() => expect(result.current.notebooks).toHaveLength(2));
|
||||
|
||||
act(() => {
|
||||
result.current.selectNotebook('nb-2');
|
||||
});
|
||||
|
||||
await act(async () => {
|
||||
await result.current.deleteNotebook('nb-1');
|
||||
});
|
||||
|
||||
expect(result.current.activeNotebook).toEqual({ id: 'nb-2', name: 'Personal' });
|
||||
});
|
||||
|
||||
it('deleteNotebook does not remove from list on API error', async () => {
|
||||
notebooksApi.deleteNotebook.mockRejectedValue(new Error('Server error'));
|
||||
vi.spyOn(console, 'error').mockImplementation(() => {});
|
||||
|
||||
const { result } = renderHook(() => useNotebook());
|
||||
await waitFor(() => expect(result.current.notebooks).toHaveLength(2));
|
||||
|
||||
await act(async () => {
|
||||
await result.current.deleteNotebook('nb-1');
|
||||
});
|
||||
|
||||
expect(result.current.notebooks).toHaveLength(2);
|
||||
console.error.mockRestore();
|
||||
});
|
||||
|
||||
it('handles listNotebooks API failure gracefully', async () => {
|
||||
notebooksApi.listNotebooks.mockRejectedValue(new Error('Network error'));
|
||||
vi.spyOn(console, 'error').mockImplementation(() => {});
|
||||
|
||||
const { result } = renderHook(() => useNotebook());
|
||||
|
||||
await waitFor(() => {
|
||||
expect(notebooksApi.listNotebooks).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
expect(result.current.notebooks).toEqual([]);
|
||||
console.error.mockRestore();
|
||||
});
|
||||
});
|
||||
50
client/src/hooks/useSourceCorpus.js
Normal file
50
client/src/hooks/useSourceCorpus.js
Normal file
@@ -0,0 +1,50 @@
|
||||
import { useState, useEffect, useCallback } from 'react';
|
||||
import * as sourceCorpusApi from '../api/sourceCorpus.js';
|
||||
|
||||
export function useSourceCorpus() {
|
||||
const [sourceCorpora, setSourceCorpora] = useState([]);
|
||||
const [activeSourceCorpus, setActiveSourceCorpus] = useState(null);
|
||||
|
||||
useEffect(() => {
|
||||
sourceCorpusApi.listSourceCorpora().then(setSourceCorpora).catch(console.error);
|
||||
}, []);
|
||||
|
||||
const selectSourceCorpus = useCallback(
|
||||
(id) => {
|
||||
const corpus = sourceCorpora.find((n) => n.id === id) || null;
|
||||
setActiveSourceCorpus(corpus);
|
||||
},
|
||||
[sourceCorpora],
|
||||
);
|
||||
|
||||
const createSourceCorpus = useCallback(async (name) => {
|
||||
if (!name) return;
|
||||
const corpus = await sourceCorpusApi.createSourceCorpus(name);
|
||||
setSourceCorpora((prev) => [...prev, corpus]);
|
||||
setActiveSourceCorpus(corpus);
|
||||
}, []);
|
||||
|
||||
const deleteSourceCorpus = useCallback(
|
||||
async (id) => {
|
||||
try {
|
||||
await sourceCorpusApi.deleteSourceCorpus(id);
|
||||
} catch (err) {
|
||||
console.error('Failed to delete sourceCorpus', err);
|
||||
return;
|
||||
}
|
||||
setSourceCorpora((prev) => prev.filter((n) => n.id !== id));
|
||||
if (activeSourceCorpus?.id === id) {
|
||||
setActiveSourceCorpus(null);
|
||||
}
|
||||
},
|
||||
[activeSourceCorpus],
|
||||
);
|
||||
|
||||
return {
|
||||
sourceCorpora,
|
||||
activeSourceCorpus,
|
||||
selectSourceCorpus,
|
||||
createSourceCorpus,
|
||||
deleteSourceCorpus,
|
||||
};
|
||||
}
|
||||
176
client/src/hooks/useSourceCorpus.test.js
Normal file
176
client/src/hooks/useSourceCorpus.test.js
Normal file
@@ -0,0 +1,176 @@
|
||||
import { describe, it, expect, vi, beforeEach } from 'vitest';
|
||||
import { renderHook, act, waitFor } from '@testing-library/react';
|
||||
|
||||
vi.mock('../api/sourceCorpus.js', () => ({
|
||||
listSourceCorpora: vi.fn(),
|
||||
createSourceCorpus: vi.fn(),
|
||||
deleteSourceCorpus: vi.fn(),
|
||||
}));
|
||||
|
||||
import { useSourceCorpus } from './useSourceCorpus.js';
|
||||
import * as sourceCorpusApi from '../api/sourceCorpus.js';
|
||||
|
||||
const sourceCorpora = [
|
||||
{ id: 'nb-1', name: 'Research' },
|
||||
{ id: 'nb-2', name: 'Personal' },
|
||||
];
|
||||
|
||||
beforeEach(() => {
|
||||
vi.resetAllMocks();
|
||||
sourceCorpusApi.listSourceCorpora.mockResolvedValue(sourceCorpora);
|
||||
sourceCorpusApi.createSourceCorpus.mockImplementation(async (name) => ({
|
||||
id: `nb-${Date.now()}`,
|
||||
name,
|
||||
}));
|
||||
sourceCorpusApi.deleteSourceCorpus.mockResolvedValue({});
|
||||
});
|
||||
|
||||
describe('useSourceCorpus', () => {
|
||||
it('loads sourceCorpora on mount', async () => {
|
||||
const { result } = renderHook(() => useSourceCorpus());
|
||||
|
||||
await waitFor(() => {
|
||||
expect(result.current.sourceCorpora).toEqual(sourceCorpora);
|
||||
});
|
||||
expect(sourceCorpusApi.listSourceCorpora).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it('starts with no active sourceCorpus', () => {
|
||||
const { result } = renderHook(() => useSourceCorpus());
|
||||
expect(result.current.activeSourceCorpus).toBeNull();
|
||||
});
|
||||
|
||||
it('selectSourceCorpus sets the active sourceCorpus', async () => {
|
||||
const { result } = renderHook(() => useSourceCorpus());
|
||||
|
||||
await waitFor(() => {
|
||||
expect(result.current.sourceCorpora).toHaveLength(2);
|
||||
});
|
||||
|
||||
act(() => {
|
||||
result.current.selectSourceCorpus('nb-2');
|
||||
});
|
||||
|
||||
expect(result.current.activeSourceCorpus).toEqual({ id: 'nb-2', name: 'Personal' });
|
||||
});
|
||||
|
||||
it('selectSourceCorpus sets null for unknown id', async () => {
|
||||
const { result } = renderHook(() => useSourceCorpus());
|
||||
|
||||
await waitFor(() => {
|
||||
expect(result.current.sourceCorpora).toHaveLength(2);
|
||||
});
|
||||
|
||||
act(() => {
|
||||
result.current.selectSourceCorpus('nb-2');
|
||||
});
|
||||
expect(result.current.activeSourceCorpus).not.toBeNull();
|
||||
|
||||
act(() => {
|
||||
result.current.selectSourceCorpus('nonexistent');
|
||||
});
|
||||
expect(result.current.activeSourceCorpus).toBeNull();
|
||||
});
|
||||
|
||||
it('createSourceCorpus calls API and adds to list', async () => {
|
||||
const newCorpus = { id: 'nb-new', name: 'New One' };
|
||||
sourceCorpusApi.createSourceCorpus.mockResolvedValue(newCorpus);
|
||||
|
||||
const { result } = renderHook(() => useSourceCorpus());
|
||||
await waitFor(() => expect(result.current.sourceCorpora).toHaveLength(2));
|
||||
|
||||
await act(async () => {
|
||||
await result.current.createSourceCorpus('New One');
|
||||
});
|
||||
|
||||
expect(sourceCorpusApi.createSourceCorpus).toHaveBeenCalledWith('New One');
|
||||
expect(result.current.sourceCorpora).toHaveLength(3);
|
||||
expect(result.current.sourceCorpora[2]).toEqual(newCorpus);
|
||||
expect(result.current.activeSourceCorpus).toEqual(newCorpus);
|
||||
});
|
||||
|
||||
it('createSourceCorpus does nothing for empty name', async () => {
|
||||
const { result } = renderHook(() => useSourceCorpus());
|
||||
await waitFor(() => expect(result.current.sourceCorpora).toHaveLength(2));
|
||||
|
||||
await act(async () => {
|
||||
await result.current.createSourceCorpus('');
|
||||
});
|
||||
|
||||
expect(sourceCorpusApi.createSourceCorpus).not.toHaveBeenCalled();
|
||||
expect(result.current.sourceCorpora).toHaveLength(2);
|
||||
});
|
||||
|
||||
it('deleteSourceCorpus removes from list', async () => {
|
||||
const { result } = renderHook(() => useSourceCorpus());
|
||||
await waitFor(() => expect(result.current.sourceCorpora).toHaveLength(2));
|
||||
|
||||
await act(async () => {
|
||||
await result.current.deleteSourceCorpus('nb-1');
|
||||
});
|
||||
|
||||
expect(sourceCorpusApi.deleteSourceCorpus).toHaveBeenCalledWith('nb-1');
|
||||
expect(result.current.sourceCorpora).toHaveLength(1);
|
||||
expect(result.current.sourceCorpora[0].id).toBe('nb-2');
|
||||
});
|
||||
|
||||
it('deleteSourceCorpus clears activeSourceCorpus if it was the deleted one', async () => {
|
||||
const { result } = renderHook(() => useSourceCorpus());
|
||||
await waitFor(() => expect(result.current.sourceCorpora).toHaveLength(2));
|
||||
|
||||
act(() => {
|
||||
result.current.selectSourceCorpus('nb-1');
|
||||
});
|
||||
expect(result.current.activeSourceCorpus?.id).toBe('nb-1');
|
||||
|
||||
await act(async () => {
|
||||
await result.current.deleteSourceCorpus('nb-1');
|
||||
});
|
||||
|
||||
expect(result.current.activeSourceCorpus).toBeNull();
|
||||
});
|
||||
|
||||
it('deleteSourceCorpus preserves activeSourceCorpus if different one deleted', async () => {
|
||||
const { result } = renderHook(() => useSourceCorpus());
|
||||
await waitFor(() => expect(result.current.sourceCorpora).toHaveLength(2));
|
||||
|
||||
act(() => {
|
||||
result.current.selectSourceCorpus('nb-2');
|
||||
});
|
||||
|
||||
await act(async () => {
|
||||
await result.current.deleteSourceCorpus('nb-1');
|
||||
});
|
||||
|
||||
expect(result.current.activeSourceCorpus).toEqual({ id: 'nb-2', name: 'Personal' });
|
||||
});
|
||||
|
||||
it('deleteSourceCorpus does not remove from list on API error', async () => {
|
||||
sourceCorpusApi.deleteSourceCorpus.mockRejectedValue(new Error('Server error'));
|
||||
vi.spyOn(console, 'error').mockImplementation(() => {});
|
||||
|
||||
const { result } = renderHook(() => useSourceCorpus());
|
||||
await waitFor(() => expect(result.current.sourceCorpora).toHaveLength(2));
|
||||
|
||||
await act(async () => {
|
||||
await result.current.deleteSourceCorpus('nb-1');
|
||||
});
|
||||
|
||||
expect(result.current.sourceCorpora).toHaveLength(2);
|
||||
console.error.mockRestore();
|
||||
});
|
||||
|
||||
it('handles listSourceCorpora API failure gracefully', async () => {
|
||||
sourceCorpusApi.listSourceCorpora.mockRejectedValue(new Error('Network error'));
|
||||
vi.spyOn(console, 'error').mockImplementation(() => {});
|
||||
|
||||
const { result } = renderHook(() => useSourceCorpus());
|
||||
|
||||
await waitFor(() => {
|
||||
expect(sourceCorpusApi.listSourceCorpora).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
expect(result.current.sourceCorpora).toEqual([]);
|
||||
console.error.mockRestore();
|
||||
});
|
||||
});
|
||||
@@ -75,14 +75,14 @@ body {
|
||||
font-size: 16px;
|
||||
}
|
||||
|
||||
/* ── Notebook List ─────────────────────────────────── */
|
||||
/* ── SourceCorpus List ─────────────────────────────────── */
|
||||
|
||||
.notebook-list {
|
||||
.source-corpus-list {
|
||||
padding: 20px 16px 12px;
|
||||
border-bottom: 1px solid rgba(255, 255, 255, 0.08);
|
||||
}
|
||||
|
||||
.notebook-list h2 {
|
||||
.source-corpus-list h2 {
|
||||
font-size: 11px;
|
||||
font-weight: 600;
|
||||
text-transform: uppercase;
|
||||
@@ -91,12 +91,12 @@ body {
|
||||
margin-bottom: 12px;
|
||||
}
|
||||
|
||||
.notebook-list ul {
|
||||
.source-corpus-list ul {
|
||||
list-style: none;
|
||||
margin-bottom: 8px;
|
||||
}
|
||||
|
||||
.notebook-list li {
|
||||
.source-corpus-list li {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: space-between;
|
||||
@@ -107,23 +107,23 @@ body {
|
||||
transition: background 0.15s;
|
||||
}
|
||||
|
||||
.notebook-list li:hover {
|
||||
.source-corpus-list li:hover {
|
||||
background: var(--sidebar-hover);
|
||||
}
|
||||
|
||||
.notebook-list li.active {
|
||||
.source-corpus-list li.active {
|
||||
background: var(--sidebar-active);
|
||||
color: #fff;
|
||||
font-weight: 500;
|
||||
}
|
||||
|
||||
.notebook-list li .nb-name {
|
||||
.source-corpus-list li .sc-name {
|
||||
overflow: hidden;
|
||||
text-overflow: ellipsis;
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.notebook-list li .nb-delete {
|
||||
.source-corpus-list li .sc-delete {
|
||||
opacity: 0;
|
||||
background: none;
|
||||
border: none;
|
||||
@@ -135,15 +135,15 @@ body {
|
||||
transition: opacity 0.15s, color 0.15s;
|
||||
}
|
||||
|
||||
.notebook-list li:hover .nb-delete {
|
||||
.source-corpus-list li:hover .sc-delete {
|
||||
opacity: 1;
|
||||
}
|
||||
|
||||
.notebook-list li .nb-delete:hover {
|
||||
.source-corpus-list li .sc-delete:hover {
|
||||
color: var(--danger);
|
||||
}
|
||||
|
||||
.btn-new-notebook {
|
||||
.btn-new-corpus {
|
||||
width: 100%;
|
||||
padding: 8px;
|
||||
border: 1px dashed rgba(255, 255, 255, 0.2);
|
||||
@@ -155,7 +155,7 @@ body {
|
||||
transition: border-color 0.15s, color 0.15s;
|
||||
}
|
||||
|
||||
.btn-new-notebook:hover {
|
||||
.btn-new-corpus:hover {
|
||||
border-color: rgba(255, 255, 255, 0.5);
|
||||
color: #fff;
|
||||
}
|
||||
|
||||
BIN
sample screen scaps/DEMO FOR SOURCE SENT.jpg
Normal file
BIN
sample screen scaps/DEMO FOR SOURCE SENT.jpg
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 365 KiB |
BIN
sample screen scaps/DEMO FOR SOURCE SENT_SMALL.jpg
Normal file
BIN
sample screen scaps/DEMO FOR SOURCE SENT_SMALL.jpg
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 133 KiB |
BIN
sample screen scaps/SMALL_DEMO FOR SOURCE SENT.jpg
Normal file
BIN
sample screen scaps/SMALL_DEMO FOR SOURCE SENT.jpg
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 157 KiB |
BIN
sample screen scaps/Screenshot 2026-08-03 at 12.03.34 AM.png
Normal file
BIN
sample screen scaps/Screenshot 2026-08-03 at 12.03.34 AM.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 307 KiB |
BIN
sample screen scaps/Screenshot 2026-08-03 at 12.04.10 AM.png
Normal file
BIN
sample screen scaps/Screenshot 2026-08-03 at 12.04.10 AM.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 309 KiB |
4
server/package-lock.json
generated
4
server/package-lock.json
generated
@@ -1,11 +1,11 @@
|
||||
{
|
||||
"name": "notebook-clone-server",
|
||||
"name": "source-sentinel-server",
|
||||
"version": "0.1.0",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "notebook-clone-server",
|
||||
"name": "source-sentinel-server",
|
||||
"version": "0.1.0",
|
||||
"dependencies": {
|
||||
"@anthropic-ai/sdk": "^0.78.0",
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
{
|
||||
"name": "notebook-clone-server",
|
||||
"name": "source-sentinel-server",
|
||||
"version": "0.1.0",
|
||||
"private": true,
|
||||
"engines": {
|
||||
|
||||
@@ -4,7 +4,7 @@ import 'dotenv/config';
|
||||
import express, { type Request, type Response, type NextFunction } from 'express';
|
||||
import cors from 'cors';
|
||||
import logger from './logger.js';
|
||||
import notebookRoutes from './routes/notebooks.js';
|
||||
import sourceCorpusRoutes from './routes/sourceCorpus.js';
|
||||
import sourceRoutes from './routes/sources.js';
|
||||
import queryRoutes from './routes/query.js';
|
||||
import citationDetailRoutes from './routes/citationDetail.js';
|
||||
@@ -39,13 +39,13 @@ app.use((req: Request, res: Response, next: NextFunction) => {
|
||||
|
||||
app.get('/.well-known/healthcheck', (_req: Request, res: Response) => {
|
||||
res.json({
|
||||
service: 'notebook-clone',
|
||||
service: 'source-sentinel',
|
||||
status: 'ok',
|
||||
uptime: process.uptime(),
|
||||
});
|
||||
});
|
||||
|
||||
app.use('/api/notebooks', notebookRoutes);
|
||||
app.use('/api/source-corpus', sourceCorpusRoutes);
|
||||
app.use('/api/sources', sourceRoutes);
|
||||
app.use('/api/query', queryRoutes);
|
||||
app.use('/api/citation-detail', citationDetailRoutes);
|
||||
|
||||
@@ -1,31 +1,31 @@
|
||||
'use strict';
|
||||
|
||||
import type { Request, Response, NextFunction } from 'express';
|
||||
import * as notebookStore from '../stores/notebookStore.js';
|
||||
import type { Notebook } from '../stores/notebookStore.js';
|
||||
import * as sourceCorpusStore from '../stores/sourceCorpusStore.js';
|
||||
import type { SourceCorpus } from '../stores/sourceCorpusStore.js';
|
||||
|
||||
export interface NotebookRequest extends Request {
|
||||
notebookId?: string;
|
||||
notebook?: Notebook;
|
||||
export interface SourceCorpusRequest extends Request {
|
||||
sourceCorpusId?: string;
|
||||
sourceCorpus?: SourceCorpus;
|
||||
}
|
||||
|
||||
export function requireNotebookId(req: NotebookRequest, res: Response, next: NextFunction): void {
|
||||
const notebookId = (req.body?.notebookId ?? req.query?.notebookId) as string | undefined;
|
||||
if (!notebookId) {
|
||||
res.status(400).json({ error: 'notebookId is required' });
|
||||
export function requireSourceCorpusId(req: SourceCorpusRequest, res: Response, next: NextFunction): void {
|
||||
const sourceCorpusId = (req.body?.sourceCorpusId ?? req.query?.sourceCorpusId) as string | undefined;
|
||||
if (!sourceCorpusId) {
|
||||
res.status(400).json({ error: 'sourceCorpusId is required' });
|
||||
return;
|
||||
}
|
||||
req.notebookId = notebookId;
|
||||
req.sourceCorpusId = sourceCorpusId;
|
||||
next();
|
||||
}
|
||||
|
||||
export function requireNotebook(req: NotebookRequest, res: Response, next: NextFunction): void {
|
||||
const notebook = notebookStore.getNotebook(req.notebookId!);
|
||||
if (!notebook) {
|
||||
res.status(404).json({ error: 'notebook not found' });
|
||||
export function requireSourceCorpus(req: SourceCorpusRequest, res: Response, next: NextFunction): void {
|
||||
const sourceCorpus = sourceCorpusStore.getSourceCorpus(req.sourceCorpusId!);
|
||||
if (!sourceCorpus) {
|
||||
res.status(404).json({ error: 'source corpus not found' });
|
||||
return;
|
||||
}
|
||||
req.notebook = notebook;
|
||||
req.sourceCorpus = sourceCorpus;
|
||||
next();
|
||||
}
|
||||
|
||||
|
||||
@@ -2,8 +2,8 @@
|
||||
|
||||
import { Router, type Request, type Response, type NextFunction } from 'express';
|
||||
import logger from '../logger.js';
|
||||
import * as notebookStore from '../stores/notebookStore.js';
|
||||
import type { TextChunk, Source, SourceGroup } from '../stores/notebookStore.js';
|
||||
import * as sourceCorpusStore from '../stores/sourceCorpusStore.js';
|
||||
import type { TextChunk, Source, SourceGroup } from '../stores/sourceCorpusStore.js';
|
||||
import * as documentCacheStore from '../stores/documentCacheStore.js';
|
||||
import type { CacheEntry } from '../stores/documentCacheStore.js';
|
||||
import {
|
||||
@@ -27,16 +27,16 @@ const GENERATORS: Record<DocumentType, GeneratorFn> = {
|
||||
const router = Router();
|
||||
|
||||
interface GenerateBody {
|
||||
notebookId?: string;
|
||||
sourceCorpusId?: string;
|
||||
type?: string;
|
||||
}
|
||||
|
||||
router.post('/generate', async (req: Request<unknown, unknown, GenerateBody>, res: Response, next: NextFunction) => {
|
||||
try {
|
||||
const { notebookId, type } = req.body;
|
||||
const { sourceCorpusId, type } = req.body;
|
||||
|
||||
if (!notebookId || !type) {
|
||||
res.status(400).json({ error: 'notebookId and type are required' });
|
||||
if (!sourceCorpusId || !type) {
|
||||
res.status(400).json({ error: 'sourceCorpusId and type are required' });
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -48,31 +48,31 @@ router.post('/generate', async (req: Request<unknown, unknown, GenerateBody>, re
|
||||
return;
|
||||
}
|
||||
|
||||
const chunks: TextChunk[] = notebookStore.getChunksForNotebook(notebookId);
|
||||
const chunks: TextChunk[] = sourceCorpusStore.getChunksForSourceCorpus(sourceCorpusId);
|
||||
if (chunks.length === 0) {
|
||||
res.status(422).json({ error: 'No source material available in this notebook' });
|
||||
res.status(422).json({ error: 'No source material available in this source corpus' });
|
||||
return;
|
||||
}
|
||||
|
||||
const sources: Source[] = notebookStore.getSources(notebookId);
|
||||
const cached: CacheEntry | null = documentCacheStore.getCachedDocument(notebookId, type);
|
||||
if (cached && documentCacheStore.isFresh(notebookId, type, sources)) {
|
||||
logger.info({ notebookId, type }, 'serving cached document');
|
||||
const sources: Source[] = sourceCorpusStore.getSources(sourceCorpusId);
|
||||
const cached: CacheEntry | null = documentCacheStore.getCachedDocument(sourceCorpusId, type);
|
||||
if (cached && documentCacheStore.isFresh(sourceCorpusId, type, sources)) {
|
||||
logger.info({ sourceCorpusId, type }, 'serving cached document');
|
||||
res.json({ type, document: cached.document });
|
||||
return;
|
||||
}
|
||||
|
||||
const sourceGroups: SourceGroup[] = notebookStore.buildSourceGroups(notebookId, chunks);
|
||||
const sourceGroups: SourceGroup[] = sourceCorpusStore.buildSourceGroups(sourceCorpusId, chunks);
|
||||
|
||||
logger.info(
|
||||
{ notebookId, type, sourceCount: sourceGroups.length, chunkCount: chunks.length },
|
||||
{ sourceCorpusId, type, sourceCount: sourceGroups.length, chunkCount: chunks.length },
|
||||
'document generation started'
|
||||
);
|
||||
|
||||
const document: StudyGuide | Faq | ExecutiveBrief = await generator(sourceGroups);
|
||||
|
||||
documentCacheStore.setCachedDocument(notebookId, type, document, sources);
|
||||
logger.info({ notebookId, type }, 'document generation complete');
|
||||
documentCacheStore.setCachedDocument(sourceCorpusId, type, document, sources);
|
||||
logger.info({ sourceCorpusId, type }, 'document generation complete');
|
||||
|
||||
res.json({ type, document });
|
||||
} catch (err) {
|
||||
|
||||
@@ -7,8 +7,8 @@ import type { ScoredChunk, RankedChunk, TextChunk } from '../services/retrievalS
|
||||
import { generate } from '../services/generationService.js';
|
||||
import type { GenerationResult } from '../services/generationService.js';
|
||||
import { computeGroundedness } from '../services/scoringService.js';
|
||||
import * as notebookStore from '../stores/notebookStore.js';
|
||||
import type { SourceGroup } from '../stores/notebookStore.js';
|
||||
import * as sourceCorpusStore from '../stores/sourceCorpusStore.js';
|
||||
import type { SourceGroup } from '../stores/sourceCorpusStore.js';
|
||||
|
||||
const TOP_K_SEARCH = 20;
|
||||
const TOP_K_RERANK = 5;
|
||||
@@ -16,7 +16,7 @@ const TOP_K_RERANK = 5;
|
||||
const router = Router();
|
||||
|
||||
interface QueryBody {
|
||||
notebookId?: string;
|
||||
sourceCorpusId?: string;
|
||||
question?: string;
|
||||
}
|
||||
|
||||
@@ -36,20 +36,20 @@ interface QueryResponse {
|
||||
|
||||
router.post('/', async (req: Request<unknown, unknown, QueryBody>, res: Response, next: NextFunction) => {
|
||||
try {
|
||||
const { notebookId, question } = req.body;
|
||||
if (!notebookId || !question) {
|
||||
res.status(400).json({ error: 'notebookId and question are required' });
|
||||
const { sourceCorpusId, question } = req.body;
|
||||
if (!sourceCorpusId || !question) {
|
||||
res.status(400).json({ error: 'sourceCorpusId and question are required' });
|
||||
return;
|
||||
}
|
||||
|
||||
logger.info({ notebookId, question }, 'query received');
|
||||
logger.info({ sourceCorpusId, question }, 'query received');
|
||||
|
||||
const [queryEmbedding]: number[][] = await embedTexts([question], 'query');
|
||||
|
||||
const searchResults: ScoredChunk[] = search(queryEmbedding, notebookId, TOP_K_SEARCH);
|
||||
const searchResults: ScoredChunk[] = search(queryEmbedding, sourceCorpusId, TOP_K_SEARCH);
|
||||
if (searchResults.length === 0) {
|
||||
res.json({
|
||||
answer: 'No sources found for this notebook. Upload some documents first.',
|
||||
answer: 'No sources found for this source corpus. Upload some documents first.',
|
||||
citations: [],
|
||||
groundednessScore: null,
|
||||
followUpQuestions: [],
|
||||
@@ -69,7 +69,7 @@ router.post('/', async (req: Request<unknown, unknown, QueryBody>, res: Response
|
||||
'retrieval complete'
|
||||
);
|
||||
|
||||
const sourceGroups: SourceGroup[] = notebookStore.buildSourceGroups(notebookId, topChunks);
|
||||
const sourceGroups: SourceGroup[] = sourceCorpusStore.buildSourceGroups(sourceCorpusId, topChunks);
|
||||
|
||||
const { answer, citedSourceIndices, followUpQuestions }: GenerationResult = await generate(question, sourceGroups);
|
||||
|
||||
|
||||
@@ -2,33 +2,33 @@
|
||||
|
||||
import { Router, type Request, type Response } from 'express';
|
||||
import { v4 as uuidv4 } from 'uuid';
|
||||
import * as notebookStore from '../stores/notebookStore.js';
|
||||
import * as sourceCorpusStore from '../stores/sourceCorpusStore.js';
|
||||
import { deleteVectorsForChunks } from '../stores/vectorStore.js';
|
||||
|
||||
const router = Router();
|
||||
|
||||
router.get('/', (_req: Request, res: Response) => {
|
||||
res.json(notebookStore.getAllNotebooks());
|
||||
res.json(sourceCorpusStore.getAllSourceCorpora());
|
||||
});
|
||||
|
||||
interface CreateNotebookBody {
|
||||
interface CreateSourceCorpusBody {
|
||||
name?: string;
|
||||
}
|
||||
|
||||
router.post('/', (req: Request<unknown, unknown, CreateNotebookBody>, res: Response) => {
|
||||
router.post('/', (req: Request<unknown, unknown, CreateSourceCorpusBody>, res: Response) => {
|
||||
const { name } = req.body;
|
||||
if (!name || !name.trim()) {
|
||||
res.status(400).json({ error: 'name is required' });
|
||||
return;
|
||||
}
|
||||
|
||||
const notebook = notebookStore.createNotebook({
|
||||
const sourceCorpus = sourceCorpusStore.createSourceCorpus({
|
||||
id: uuidv4(),
|
||||
name: name.trim(),
|
||||
createdAt: new Date().toISOString(),
|
||||
});
|
||||
|
||||
res.status(201).json(notebook);
|
||||
res.status(201).json(sourceCorpus);
|
||||
});
|
||||
|
||||
interface DeleteParams {
|
||||
@@ -36,10 +36,10 @@ interface DeleteParams {
|
||||
}
|
||||
|
||||
router.delete('/:id', (req: Request<DeleteParams>, res: Response) => {
|
||||
const chunks = notebookStore.getChunksForNotebook(req.params.id);
|
||||
const deleted = notebookStore.deleteNotebook(req.params.id);
|
||||
const chunks = sourceCorpusStore.getChunksForSourceCorpus(req.params.id);
|
||||
const deleted = sourceCorpusStore.deleteSourceCorpus(req.params.id);
|
||||
if (!deleted) {
|
||||
res.status(404).json({ error: 'notebook not found' });
|
||||
res.status(404).json({ error: 'source corpus not found' });
|
||||
return;
|
||||
}
|
||||
if (chunks.length) {
|
||||
@@ -4,18 +4,18 @@ import { Router, type Request, type Response, type NextFunction } from 'express'
|
||||
import multer from 'multer';
|
||||
import { v4 as uuidv4 } from 'uuid';
|
||||
import logger from '../logger.js';
|
||||
import * as notebookStore from '../stores/notebookStore.js';
|
||||
import type { Source } from '../stores/notebookStore.js';
|
||||
import * as sourceCorpusStore from '../stores/sourceCorpusStore.js';
|
||||
import type { Source } from '../stores/sourceCorpusStore.js';
|
||||
import { parseFile, chunkText, parseUrl, parseYoutubeUrl, isYoutubeUrl } from '../services/sourceService.js';
|
||||
import type { TextChunk } from '../services/sourceService.js';
|
||||
import { embedTexts, storeChunkEmbeddings } from '../services/retrievalService.js';
|
||||
import { triggerPreGeneration } from '../services/preGenerationService.js';
|
||||
import {
|
||||
requireNotebookId,
|
||||
requireNotebook,
|
||||
requireSourceCorpusId,
|
||||
requireSourceCorpus,
|
||||
requireFile,
|
||||
requireUrl,
|
||||
type NotebookRequest,
|
||||
type SourceCorpusRequest,
|
||||
} from '../middleware/validation.js';
|
||||
|
||||
const upload = multer({ storage: multer.memoryStorage(), limits: { fileSize: 50 * 1024 * 1024 } });
|
||||
@@ -26,8 +26,8 @@ function cleanFilename(name: string): string {
|
||||
return name.replace(TIMESTAMP_RE, '');
|
||||
}
|
||||
|
||||
router.get('/', requireNotebookId, (req: NotebookRequest, res: Response) => {
|
||||
res.json(notebookStore.getSources(req.notebookId!));
|
||||
router.get('/', requireSourceCorpusId, (req: SourceCorpusRequest, res: Response) => {
|
||||
res.json(sourceCorpusStore.getSources(req.sourceCorpusId!));
|
||||
});
|
||||
|
||||
function multerUpload(req: Request, res: Response, next: NextFunction): void {
|
||||
@@ -47,10 +47,10 @@ function multerUpload(req: Request, res: Response, next: NextFunction): void {
|
||||
router.post(
|
||||
'/',
|
||||
multerUpload,
|
||||
requireNotebookId,
|
||||
requireNotebook,
|
||||
requireSourceCorpusId,
|
||||
requireSourceCorpus,
|
||||
requireFile,
|
||||
async (req: NotebookRequest, res: Response, next: NextFunction) => {
|
||||
async (req: SourceCorpusRequest, res: Response, next: NextFunction) => {
|
||||
try {
|
||||
const sourceId: string = uuidv4();
|
||||
const rawName: string = Buffer.from(req.file!.originalname, 'latin1').toString('utf-8');
|
||||
@@ -67,13 +67,13 @@ router.post(
|
||||
'parsed and chunked source'
|
||||
);
|
||||
|
||||
notebookStore.addChunksToNotebook(req.notebookId!, chunks);
|
||||
sourceCorpusStore.addChunksToSourceCorpus(req.sourceCorpusId!, chunks);
|
||||
|
||||
const texts: string[] = chunks.map((c) => c.text);
|
||||
const embeddings: number[][] = await embedTexts(texts, 'document');
|
||||
storeChunkEmbeddings(chunks, embeddings);
|
||||
|
||||
const source: Source | null = notebookStore.addSource(req.notebookId!, {
|
||||
const source: Source | null = sourceCorpusStore.addSource(req.sourceCorpusId!, {
|
||||
id: sourceId,
|
||||
name: displayName,
|
||||
mimetype: req.file!.mimetype,
|
||||
@@ -81,7 +81,7 @@ router.post(
|
||||
uploadedAt: new Date().toISOString(),
|
||||
});
|
||||
|
||||
triggerPreGeneration(req.notebookId!);
|
||||
triggerPreGeneration(req.sourceCorpusId!);
|
||||
|
||||
res.status(201).json(source);
|
||||
} catch (err) {
|
||||
@@ -96,10 +96,10 @@ interface UrlBody {
|
||||
|
||||
router.post(
|
||||
'/url',
|
||||
requireNotebookId,
|
||||
requireNotebook,
|
||||
requireSourceCorpusId,
|
||||
requireSourceCorpus,
|
||||
requireUrl,
|
||||
async (req: NotebookRequest & Request<unknown, unknown, UrlBody>, res: Response, next: NextFunction) => {
|
||||
async (req: SourceCorpusRequest & Request<unknown, unknown, UrlBody>, res: Response, next: NextFunction) => {
|
||||
try {
|
||||
const { url } = req.body;
|
||||
const sourceId: string = uuidv4();
|
||||
@@ -116,13 +116,13 @@ router.post(
|
||||
return;
|
||||
}
|
||||
|
||||
notebookStore.addChunksToNotebook(req.notebookId!, chunks);
|
||||
sourceCorpusStore.addChunksToSourceCorpus(req.sourceCorpusId!, chunks);
|
||||
|
||||
const texts: string[] = chunks.map((c) => c.text);
|
||||
const embeddings: number[][] = await embedTexts(texts, 'document');
|
||||
storeChunkEmbeddings(chunks, embeddings);
|
||||
|
||||
const source: Source | null = notebookStore.addSource(req.notebookId!, {
|
||||
const source: Source | null = sourceCorpusStore.addSource(req.sourceCorpusId!, {
|
||||
id: sourceId,
|
||||
name: displayName,
|
||||
mimetype: isYT ? 'video/youtube' : 'text/html',
|
||||
@@ -130,7 +130,7 @@ router.post(
|
||||
uploadedAt: new Date().toISOString(),
|
||||
});
|
||||
|
||||
triggerPreGeneration(req.notebookId!);
|
||||
triggerPreGeneration(req.sourceCorpusId!);
|
||||
|
||||
res.status(201).json(source);
|
||||
} catch (err) {
|
||||
|
||||
@@ -104,11 +104,11 @@ export async function generateStudyGuide(sourceGroups: SourceGroup[]): Promise<S
|
||||
const sources = buildSourceBlock(sourceGroups);
|
||||
const start = Date.now();
|
||||
|
||||
const prompt = `You are an expert educator. Given the source documents below, produce a comprehensive study guide. Follow these rules:
|
||||
const prompt = `You are an expert reasearch assistant. Given the source documents below, produce a comprehensive guide. Follow these rules:
|
||||
|
||||
1. Create a structured outline of the main ideas organized into logical sections.
|
||||
2. For each section, provide: bullet-point key concepts, key terms with definitions, and 2-3 self-review questions.
|
||||
3. At the end, provide mnemonic devices or simplified restatements to help memorize the hardest concepts.
|
||||
3. At the end, provide mnemonic devices or simplified restatements of the hardest concepts.
|
||||
4. Be concise but thorough. Use simple, clear language.
|
||||
|
||||
--- SOURCE DOCUMENTS ---
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
import { describe, it, expect, vi, beforeEach, afterEach, type Mock } from 'vitest';
|
||||
|
||||
vi.mock('../stores/notebookStore.js', () => ({
|
||||
getNotebook: vi.fn(),
|
||||
vi.mock('../stores/sourceCorpusStore.js', () => ({
|
||||
getSourceCorpus: vi.fn(),
|
||||
getSources: vi.fn(),
|
||||
getChunksForNotebook: vi.fn(),
|
||||
getChunksForSourceCorpus: vi.fn(),
|
||||
buildSourceGroups: vi.fn(),
|
||||
}));
|
||||
|
||||
@@ -26,14 +26,14 @@ vi.mock('../logger.js', () => ({
|
||||
}));
|
||||
|
||||
import { triggerPreGeneration } from './preGenerationService.js';
|
||||
import * as notebookStore from '../stores/notebookStore.js';
|
||||
import * as sourceCorpusStore from '../stores/sourceCorpusStore.js';
|
||||
import * as documentCacheStore from '../stores/documentCacheStore.js';
|
||||
import { generateStudyGuide, generateFaq, generateExecutiveBrief } from './documentService.js';
|
||||
|
||||
const mockedGetNotebook = notebookStore.getNotebook as Mock;
|
||||
const mockedGetSources = notebookStore.getSources as Mock;
|
||||
const mockedGetChunksForNotebook = notebookStore.getChunksForNotebook as Mock;
|
||||
const mockedBuildSourceGroups = notebookStore.buildSourceGroups as Mock;
|
||||
const mockedGetSourceCorpus = sourceCorpusStore.getSourceCorpus as Mock;
|
||||
const mockedGetSources = sourceCorpusStore.getSources as Mock;
|
||||
const mockedGetChunksForSourceCorpus = sourceCorpusStore.getChunksForSourceCorpus as Mock;
|
||||
const mockedBuildSourceGroups = sourceCorpusStore.buildSourceGroups as Mock;
|
||||
const mockedIsGenerating = documentCacheStore.isGenerating as Mock;
|
||||
const mockedInvalidate = documentCacheStore.invalidate as Mock;
|
||||
const mockedMarkGenerating = documentCacheStore.markGenerating as Mock;
|
||||
@@ -43,10 +43,10 @@ const mockedGenerateStudyGuide = generateStudyGuide as Mock;
|
||||
const mockedGenerateFaq = generateFaq as Mock;
|
||||
const mockedGenerateExecutiveBrief = generateExecutiveBrief as Mock;
|
||||
|
||||
function setupNotebook(id: string): void {
|
||||
mockedGetNotebook.mockReturnValue({ id });
|
||||
function setupSourceCorpus(id: string): void {
|
||||
mockedGetSourceCorpus.mockReturnValue({ id });
|
||||
mockedGetSources.mockReturnValue([{ id: 's1' }, { id: 's2' }]);
|
||||
mockedGetChunksForNotebook.mockReturnValue([{ id: 'c1', text: 'hello' }]);
|
||||
mockedGetChunksForSourceCorpus.mockReturnValue([{ id: 'c1', text: 'hello' }]);
|
||||
mockedBuildSourceGroups.mockReturnValue([{ docIndex: 1, name: 'Doc', chunks: [{ text: 'hello' }] }]);
|
||||
}
|
||||
|
||||
@@ -66,14 +66,14 @@ afterEach(() => {
|
||||
});
|
||||
|
||||
describe('triggerPreGeneration guards', () => {
|
||||
it('does nothing if the notebook does not exist', () => {
|
||||
mockedGetNotebook.mockReturnValue(null);
|
||||
it('does nothing if the sourceCorpus does not exist', () => {
|
||||
mockedGetSourceCorpus.mockReturnValue(null);
|
||||
triggerPreGeneration('nb-missing');
|
||||
expect(mockedGetSources).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('does nothing if fewer than 2 sources', () => {
|
||||
mockedGetNotebook.mockReturnValue({ id: 'nb-few' });
|
||||
mockedGetSourceCorpus.mockReturnValue({ id: 'nb-few' });
|
||||
mockedGetSources.mockReturnValue([{ id: 's1' }]);
|
||||
triggerPreGeneration('nb-few');
|
||||
expect(mockedMarkGenerating).not.toHaveBeenCalled();
|
||||
@@ -82,7 +82,7 @@ describe('triggerPreGeneration guards', () => {
|
||||
|
||||
describe('queuing when already generating', () => {
|
||||
it('invalidates cache and skips runPreGeneration', () => {
|
||||
setupNotebook('nb-q');
|
||||
setupSourceCorpus('nb-q');
|
||||
mockedIsGenerating.mockReturnValue(true);
|
||||
|
||||
triggerPreGeneration('nb-q');
|
||||
@@ -94,7 +94,7 @@ describe('queuing when already generating', () => {
|
||||
|
||||
describe('debounce', () => {
|
||||
it('does not run generation immediately', () => {
|
||||
setupNotebook('nb-debounce');
|
||||
setupSourceCorpus('nb-debounce');
|
||||
mockedIsGenerating.mockReturnValue(false);
|
||||
stubGeneratorsOk();
|
||||
|
||||
@@ -104,7 +104,7 @@ describe('debounce', () => {
|
||||
});
|
||||
|
||||
it('runs generation after the debounce delay', async () => {
|
||||
setupNotebook('nb-delay');
|
||||
setupSourceCorpus('nb-delay');
|
||||
mockedIsGenerating.mockReturnValue(false);
|
||||
stubGeneratorsOk();
|
||||
|
||||
@@ -119,7 +119,7 @@ describe('debounce', () => {
|
||||
});
|
||||
|
||||
it('resets the timer on repeated calls, running generation only once', async () => {
|
||||
setupNotebook('nb-batch');
|
||||
setupSourceCorpus('nb-batch');
|
||||
mockedIsGenerating.mockReturnValue(false);
|
||||
stubGeneratorsOk();
|
||||
|
||||
@@ -140,7 +140,7 @@ describe('debounce', () => {
|
||||
|
||||
describe('runPreGeneration (via triggerPreGeneration)', () => {
|
||||
it('runs all three generators and caches results', async () => {
|
||||
setupNotebook('nb-happy');
|
||||
setupSourceCorpus('nb-happy');
|
||||
mockedIsGenerating.mockReturnValue(false);
|
||||
stubGeneratorsOk();
|
||||
|
||||
@@ -158,7 +158,7 @@ describe('runPreGeneration (via triggerPreGeneration)', () => {
|
||||
});
|
||||
|
||||
it('marks generating before starting and clears it after', async () => {
|
||||
setupNotebook('nb-flag');
|
||||
setupSourceCorpus('nb-flag');
|
||||
mockedIsGenerating.mockReturnValue(false);
|
||||
stubGeneratorsOk();
|
||||
|
||||
@@ -173,7 +173,7 @@ describe('runPreGeneration (via triggerPreGeneration)', () => {
|
||||
});
|
||||
|
||||
it('still clears generating flag when a generator fails', async () => {
|
||||
setupNotebook('nb-partial');
|
||||
setupSourceCorpus('nb-partial');
|
||||
mockedIsGenerating.mockReturnValue(false);
|
||||
mockedGenerateStudyGuide.mockRejectedValue(new Error('boom'));
|
||||
mockedGenerateFaq.mockResolvedValue({ subject: 'ok' });
|
||||
@@ -190,9 +190,9 @@ describe('runPreGeneration (via triggerPreGeneration)', () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe('re-trigger for queued notebooks', () => {
|
||||
describe('re-trigger for queued sourceCorpora', () => {
|
||||
it('runs a second generation cycle after the first finishes', async () => {
|
||||
setupNotebook('nb-re');
|
||||
setupSourceCorpus('nb-re');
|
||||
mockedIsGenerating.mockReturnValueOnce(false).mockReturnValueOnce(true);
|
||||
|
||||
let resolveFirst: (value: { title: string }) => void;
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
'use strict';
|
||||
|
||||
import logger from '../logger.js';
|
||||
import * as notebookStore from '../stores/notebookStore.js';
|
||||
import * as sourceCorpusStore from '../stores/sourceCorpusStore.js';
|
||||
import * as documentCacheStore from '../stores/documentCacheStore.js';
|
||||
import {
|
||||
generateStudyGuide,
|
||||
@@ -29,45 +29,45 @@ const DEBOUNCE_MS = 5000;
|
||||
const pendingReGen = new Set<string>();
|
||||
const debounceTimers = new Map<string, ReturnType<typeof setTimeout>>();
|
||||
|
||||
export function triggerPreGeneration(notebookId: string): void {
|
||||
const notebook = notebookStore.getNotebook(notebookId);
|
||||
if (!notebook) return;
|
||||
export function triggerPreGeneration(sourceCorpusId: string): void {
|
||||
const sourceCorpus = sourceCorpusStore.getSourceCorpus(sourceCorpusId);
|
||||
if (!sourceCorpus) return;
|
||||
|
||||
const sources = notebookStore.getSources(notebookId);
|
||||
const sources = sourceCorpusStore.getSources(sourceCorpusId);
|
||||
if (sources.length < MIN_SOURCES) return;
|
||||
|
||||
if (documentCacheStore.isGenerating(notebookId)) {
|
||||
pendingReGen.add(notebookId);
|
||||
documentCacheStore.invalidate(notebookId);
|
||||
logger.debug({ notebookId }, 'pre-generation in progress, queued re-generation');
|
||||
if (documentCacheStore.isGenerating(sourceCorpusId)) {
|
||||
pendingReGen.add(sourceCorpusId);
|
||||
documentCacheStore.invalidate(sourceCorpusId);
|
||||
logger.debug({ sourceCorpusId }, 'pre-generation in progress, queued re-generation');
|
||||
return;
|
||||
}
|
||||
|
||||
const existingTimer = debounceTimers.get(notebookId);
|
||||
const existingTimer = debounceTimers.get(sourceCorpusId);
|
||||
if (existingTimer !== undefined) {
|
||||
clearTimeout(existingTimer);
|
||||
}
|
||||
debounceTimers.set(
|
||||
notebookId,
|
||||
sourceCorpusId,
|
||||
setTimeout(() => {
|
||||
debounceTimers.delete(notebookId);
|
||||
runPreGeneration(notebookId);
|
||||
debounceTimers.delete(sourceCorpusId);
|
||||
runPreGeneration(sourceCorpusId);
|
||||
}, DEBOUNCE_MS)
|
||||
);
|
||||
logger.debug({ notebookId, debounceMs: DEBOUNCE_MS }, 'pre-generation debounced');
|
||||
logger.debug({ sourceCorpusId, debounceMs: DEBOUNCE_MS }, 'pre-generation debounced');
|
||||
}
|
||||
|
||||
function runPreGeneration(notebookId: string): void {
|
||||
function runPreGeneration(sourceCorpusId: string): void {
|
||||
try {
|
||||
const sources = notebookStore.getSources(notebookId);
|
||||
const chunks = notebookStore.getChunksForNotebook(notebookId);
|
||||
const sourceGroups = notebookStore.buildSourceGroups(notebookId, chunks) as SourceGroup[];
|
||||
const sources = sourceCorpusStore.getSources(sourceCorpusId);
|
||||
const chunks = sourceCorpusStore.getChunksForSourceCorpus(sourceCorpusId);
|
||||
const sourceGroups = sourceCorpusStore.buildSourceGroups(sourceCorpusId, chunks) as SourceGroup[];
|
||||
|
||||
documentCacheStore.invalidate(notebookId);
|
||||
documentCacheStore.markGenerating(notebookId);
|
||||
documentCacheStore.invalidate(sourceCorpusId);
|
||||
documentCacheStore.markGenerating(sourceCorpusId);
|
||||
|
||||
logger.info(
|
||||
{ notebookId, sourceCount: sources.length, chunkCount: chunks.length },
|
||||
{ sourceCorpusId, sourceCount: sources.length, chunkCount: chunks.length },
|
||||
'background pre-generation started'
|
||||
);
|
||||
|
||||
@@ -75,31 +75,31 @@ function runPreGeneration(notebookId: string): void {
|
||||
async ([type, generator]) => {
|
||||
try {
|
||||
const document = await generator(sourceGroups);
|
||||
documentCacheStore.setCachedDocument(notebookId, type, document, sources);
|
||||
logger.info({ notebookId, type }, 'background pre-generation complete for type');
|
||||
documentCacheStore.setCachedDocument(sourceCorpusId, type, document, sources);
|
||||
logger.info({ sourceCorpusId, type }, 'background pre-generation complete for type');
|
||||
} catch (err) {
|
||||
const error = err as Error;
|
||||
logger.error({ notebookId, type, err: error.message }, 'background pre-generation failed for type');
|
||||
logger.error({ sourceCorpusId, type, err: error.message }, 'background pre-generation failed for type');
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
Promise.all(jobs)
|
||||
.then(() => {
|
||||
logger.info({ notebookId }, 'all background pre-generation complete');
|
||||
logger.info({ sourceCorpusId }, 'all background pre-generation complete');
|
||||
})
|
||||
.finally(() => {
|
||||
documentCacheStore.clearGenerating(notebookId);
|
||||
documentCacheStore.clearGenerating(sourceCorpusId);
|
||||
|
||||
if (pendingReGen.has(notebookId)) {
|
||||
pendingReGen.delete(notebookId);
|
||||
logger.info({ notebookId }, 're-triggering pre-generation for updated sources');
|
||||
runPreGeneration(notebookId);
|
||||
if (pendingReGen.has(sourceCorpusId)) {
|
||||
pendingReGen.delete(sourceCorpusId);
|
||||
logger.info({ sourceCorpusId }, 're-triggering pre-generation for updated sources');
|
||||
runPreGeneration(sourceCorpusId);
|
||||
}
|
||||
});
|
||||
} catch (err) {
|
||||
const error = err as Error;
|
||||
logger.error({ notebookId, err: error.message }, 'pre-generation setup failed');
|
||||
documentCacheStore.clearGenerating(notebookId);
|
||||
logger.error({ sourceCorpusId, err: error.message }, 'pre-generation setup failed');
|
||||
documentCacheStore.clearGenerating(sourceCorpusId);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
import logger from '../logger.js';
|
||||
import * as vectorStore from '../stores/vectorStore.js';
|
||||
import * as notebookStore from '../stores/notebookStore.js';
|
||||
import * as sourceCorpusStore from '../stores/sourceCorpusStore.js';
|
||||
|
||||
const VOYAGE_API_URL = 'https://api.voyageai.com/v1';
|
||||
const EMBED_MODEL = 'voyage-3';
|
||||
@@ -72,8 +72,8 @@ export function storeChunkEmbeddings(chunks: TextChunk[], embeddings: number[][]
|
||||
logger.debug({ count: chunks.length }, 'stored chunk embeddings');
|
||||
}
|
||||
|
||||
export function search(queryEmbedding: number[], notebookId: string, topK = 10): ScoredChunk[] {
|
||||
const chunks = notebookStore.getChunksForNotebook(notebookId) as TextChunk[];
|
||||
export function search(queryEmbedding: number[], sourceCorpusId: string, topK = 10): ScoredChunk[] {
|
||||
const chunks = sourceCorpusStore.getChunksForSourceCorpus(sourceCorpusId) as TextChunk[];
|
||||
if (chunks.length === 0) return [];
|
||||
|
||||
const scored: ScoredChunk[] = [];
|
||||
|
||||
@@ -136,7 +136,7 @@ export async function parseUrl(url: string): Promise<string> {
|
||||
|
||||
const res = await fetch(safeUrl, {
|
||||
headers: {
|
||||
'User-Agent': 'Mozilla/5.0 (compatible; NotebookClone/1.0)',
|
||||
'User-Agent': 'Mozilla/5.0 (compatible; SourceSentinel/1.0)',
|
||||
Accept: 'text/html,application/xhtml+xml,text/plain',
|
||||
},
|
||||
signal: AbortSignal.timeout(15000),
|
||||
|
||||
@@ -57,7 +57,7 @@ describe('documentCacheStore', () => {
|
||||
});
|
||||
|
||||
describe('invalidate', () => {
|
||||
it('removes all entries for a notebook', () => {
|
||||
it('removes all entries for a sourceCorpus', () => {
|
||||
setCachedDocument('inv-1', 'faq', { a: 1 }, [{ id: 'x' }]);
|
||||
setCachedDocument('inv-1', 'study-guide', { b: 2 }, [{ id: 'x' }]);
|
||||
invalidate('inv-1');
|
||||
@@ -65,7 +65,7 @@ describe('documentCacheStore', () => {
|
||||
expect(getCachedDocument('inv-1', 'study-guide')).toBeNull();
|
||||
});
|
||||
|
||||
it('does not affect other notebooks', () => {
|
||||
it('does not affect other sourceCorpora', () => {
|
||||
setCachedDocument('inv-a', 'faq', { a: 1 }, [{ id: 'x' }]);
|
||||
setCachedDocument('inv-b', 'faq', { b: 2 }, [{ id: 'x' }]);
|
||||
invalidate('inv-a');
|
||||
@@ -86,7 +86,7 @@ describe('documentCacheStore', () => {
|
||||
expect(isGenerating('gen-cycle')).toBe(false);
|
||||
});
|
||||
|
||||
it('is cleared when the notebook is invalidated', () => {
|
||||
it('is cleared when the sourceCorpus is invalidated', () => {
|
||||
markGenerating('gen-inv');
|
||||
invalidate('gen-inv');
|
||||
expect(isGenerating('gen-inv')).toBe(false);
|
||||
|
||||
@@ -19,19 +19,19 @@ function buildSourceHash(sources: CacheSource[]): string {
|
||||
.join('|');
|
||||
}
|
||||
|
||||
export function getCachedDocument(notebookId: string, type: string): CacheEntry | null {
|
||||
const entry = cache.get(`${notebookId}:${type}`);
|
||||
export function getCachedDocument(sourceCorpusId: string, type: string): CacheEntry | null {
|
||||
const entry = cache.get(`${sourceCorpusId}:${type}`);
|
||||
if (!entry || typeof entry === 'boolean') return null;
|
||||
return entry;
|
||||
}
|
||||
|
||||
export function setCachedDocument(
|
||||
notebookId: string,
|
||||
sourceCorpusId: string,
|
||||
type: string,
|
||||
document: unknown,
|
||||
sources: CacheSource[]
|
||||
): void {
|
||||
const key = `${notebookId}:${type}`;
|
||||
const key = `${sourceCorpusId}:${type}`;
|
||||
cache.set(key, {
|
||||
document,
|
||||
sourceHash: buildSourceHash(sources),
|
||||
@@ -39,29 +39,29 @@ export function setCachedDocument(
|
||||
});
|
||||
}
|
||||
|
||||
export function isFresh(notebookId: string, type: string, currentSources: CacheSource[]): boolean {
|
||||
const entry = cache.get(`${notebookId}:${type}`);
|
||||
export function isFresh(sourceCorpusId: string, type: string, currentSources: CacheSource[]): boolean {
|
||||
const entry = cache.get(`${sourceCorpusId}:${type}`);
|
||||
if (!entry || typeof entry === 'boolean') return false;
|
||||
return entry.sourceHash === buildSourceHash(currentSources);
|
||||
}
|
||||
|
||||
export function invalidate(notebookId: string): void {
|
||||
export function invalidate(sourceCorpusId: string): void {
|
||||
for (const key of cache.keys()) {
|
||||
if (key.startsWith(`${notebookId}:`)) {
|
||||
if (key.startsWith(`${sourceCorpusId}:`)) {
|
||||
cache.delete(key);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export function markGenerating(notebookId: string): void {
|
||||
const key = `${notebookId}:__generating`;
|
||||
export function markGenerating(sourceCorpusId: string): void {
|
||||
const key = `${sourceCorpusId}:__generating`;
|
||||
cache.set(key, true);
|
||||
}
|
||||
|
||||
export function clearGenerating(notebookId: string): void {
|
||||
cache.delete(`${notebookId}:__generating`);
|
||||
export function clearGenerating(sourceCorpusId: string): void {
|
||||
cache.delete(`${sourceCorpusId}:__generating`);
|
||||
}
|
||||
|
||||
export function isGenerating(notebookId: string): boolean {
|
||||
return cache.get(`${notebookId}:__generating`) === true;
|
||||
export function isGenerating(sourceCorpusId: string): boolean {
|
||||
return cache.get(`${sourceCorpusId}:__generating`) === true;
|
||||
}
|
||||
|
||||
@@ -1,65 +0,0 @@
|
||||
import { describe, it, expect, beforeEach } from 'vitest';
|
||||
import { NotebookStore } from './notebookStore.js';
|
||||
|
||||
describe('notebookStore', () => {
|
||||
let store: NotebookStore;
|
||||
|
||||
beforeEach(() => {
|
||||
store = new NotebookStore();
|
||||
});
|
||||
|
||||
it('creates and retrieves a notebook', () => {
|
||||
const nb = store.createNotebook({ id: 'nb-1', name: 'Test', createdAt: '2026-01-01' });
|
||||
expect(nb.id).toBe('nb-1');
|
||||
expect(nb.name).toBe('Test');
|
||||
|
||||
const found = store.getNotebook('nb-1');
|
||||
expect(found).not.toBeNull();
|
||||
expect(found!.name).toBe('Test');
|
||||
});
|
||||
|
||||
it('lists all notebooks without chunks', () => {
|
||||
store.createNotebook({ id: 'nb-1', name: 'A', createdAt: '2026-01-01' });
|
||||
store.createNotebook({ id: 'nb-2', name: 'B', createdAt: '2026-01-02' });
|
||||
const all = store.getAllNotebooks();
|
||||
expect(all).toHaveLength(2);
|
||||
expect(all[0]).not.toHaveProperty('chunks');
|
||||
});
|
||||
|
||||
it('deletes a notebook', () => {
|
||||
store.createNotebook({ id: 'nb-1', name: 'A', createdAt: '2026-01-01' });
|
||||
expect(store.deleteNotebook('nb-1')).toBe(true);
|
||||
expect(store.getNotebook('nb-1')).toBeNull();
|
||||
});
|
||||
|
||||
it('returns false when deleting a non-existent notebook', () => {
|
||||
expect(store.deleteNotebook('nope')).toBe(false);
|
||||
});
|
||||
|
||||
it('adds and retrieves sources', () => {
|
||||
store.createNotebook({ id: 'nb-1', name: 'A', createdAt: '2026-01-01' });
|
||||
store.addSource('nb-1', { id: 's-1', name: 'doc.pdf' });
|
||||
const sources = store.getSources('nb-1');
|
||||
expect(sources).toHaveLength(1);
|
||||
expect(sources[0].name).toBe('doc.pdf');
|
||||
});
|
||||
|
||||
it('returns empty sources for non-existent notebook', () => {
|
||||
expect(store.getSources('nope')).toEqual([]);
|
||||
});
|
||||
|
||||
it('adds and retrieves chunks', () => {
|
||||
store.createNotebook({ id: 'nb-1', name: 'A', createdAt: '2026-01-01' });
|
||||
store.addChunksToNotebook('nb-1', [
|
||||
{ id: 'c-1', text: 'chunk one', sourceId: 's-1', index: 0 },
|
||||
{ id: 'c-2', text: 'chunk two', sourceId: 's-1', index: 1 },
|
||||
]);
|
||||
const chunks = store.getChunksForNotebook('nb-1');
|
||||
expect(chunks).toHaveLength(2);
|
||||
expect(chunks[0].text).toBe('chunk one');
|
||||
});
|
||||
|
||||
it('returns empty chunks for non-existent notebook', () => {
|
||||
expect(store.getChunksForNotebook('nope')).toEqual([]);
|
||||
});
|
||||
});
|
||||
@@ -1,126 +0,0 @@
|
||||
'use strict';
|
||||
|
||||
export interface TextChunk {
|
||||
id: string;
|
||||
text: string;
|
||||
sourceId: string;
|
||||
index: number;
|
||||
}
|
||||
|
||||
export interface Source {
|
||||
id: string;
|
||||
name: string;
|
||||
mimetype?: string;
|
||||
chunkCount?: number;
|
||||
uploadedAt?: string;
|
||||
}
|
||||
|
||||
export interface Notebook {
|
||||
id: string;
|
||||
name: string;
|
||||
createdAt: string;
|
||||
sources: Source[];
|
||||
chunks: TextChunk[];
|
||||
}
|
||||
|
||||
export interface NotebookMeta {
|
||||
id: string;
|
||||
name: string;
|
||||
createdAt: string;
|
||||
}
|
||||
|
||||
export interface SourceGroup {
|
||||
docIndex: number;
|
||||
sourceId: string;
|
||||
name: string;
|
||||
chunks: TextChunk[];
|
||||
}
|
||||
|
||||
export class NotebookStore {
|
||||
private notebooks = new Map<string, Notebook>();
|
||||
|
||||
getAllNotebooks(): NotebookMeta[] {
|
||||
return Array.from(this.notebooks.values()).map(({ id, name, createdAt }) => ({
|
||||
id,
|
||||
name,
|
||||
createdAt,
|
||||
}));
|
||||
}
|
||||
|
||||
getNotebook(id: string): Notebook | null {
|
||||
return this.notebooks.get(id) || null;
|
||||
}
|
||||
|
||||
createNotebook(notebook: { id: string; name: string; createdAt: string }): NotebookMeta {
|
||||
const record: Notebook = { ...notebook, sources: [], chunks: [] };
|
||||
this.notebooks.set(record.id, record);
|
||||
return { id: record.id, name: record.name, createdAt: record.createdAt };
|
||||
}
|
||||
|
||||
deleteNotebook(id: string): boolean {
|
||||
return this.notebooks.delete(id);
|
||||
}
|
||||
|
||||
addSource(notebookId: string, source: Source): Source | null {
|
||||
const notebook = this.notebooks.get(notebookId);
|
||||
if (!notebook) return null;
|
||||
notebook.sources.push(source);
|
||||
return source;
|
||||
}
|
||||
|
||||
getSources(notebookId: string): Source[] {
|
||||
const notebook = this.notebooks.get(notebookId);
|
||||
if (!notebook) return [];
|
||||
return notebook.sources;
|
||||
}
|
||||
|
||||
addChunksToNotebook(notebookId: string, chunks: TextChunk[]): Notebook | null {
|
||||
const notebook = this.notebooks.get(notebookId);
|
||||
if (!notebook) return null;
|
||||
notebook.chunks = notebook.chunks.concat(chunks);
|
||||
return notebook;
|
||||
}
|
||||
|
||||
getChunksForNotebook(notebookId: string): TextChunk[] {
|
||||
const notebook = this.notebooks.get(notebookId);
|
||||
if (!notebook) return [];
|
||||
return notebook.chunks;
|
||||
}
|
||||
|
||||
buildSourceGroups(notebookId: string, chunks: TextChunk[]): SourceGroup[] {
|
||||
const sources = this.getSources(notebookId);
|
||||
const sourceIndexMap = new Map<string, number>();
|
||||
sources.forEach((src, i) => {
|
||||
sourceIndexMap.set(src.id, i + 1);
|
||||
});
|
||||
|
||||
const groupMap = new Map<string, SourceGroup>();
|
||||
for (const chunk of chunks) {
|
||||
const docIndex = sourceIndexMap.get(chunk.sourceId) || 0;
|
||||
if (!groupMap.has(chunk.sourceId)) {
|
||||
const src = sources.find((s) => s.id === chunk.sourceId);
|
||||
groupMap.set(chunk.sourceId, {
|
||||
docIndex,
|
||||
sourceId: chunk.sourceId,
|
||||
name: src?.name || 'Unknown',
|
||||
chunks: [],
|
||||
});
|
||||
}
|
||||
groupMap.get(chunk.sourceId)!.chunks.push(chunk);
|
||||
}
|
||||
|
||||
return Array.from(groupMap.values()).sort((a, b) => a.docIndex - b.docIndex);
|
||||
}
|
||||
}
|
||||
|
||||
const defaultStore = new NotebookStore();
|
||||
|
||||
export const getAllNotebooks = defaultStore.getAllNotebooks.bind(defaultStore);
|
||||
export const getNotebook = defaultStore.getNotebook.bind(defaultStore);
|
||||
export const createNotebook = defaultStore.createNotebook.bind(defaultStore);
|
||||
export const deleteNotebook = defaultStore.deleteNotebook.bind(defaultStore);
|
||||
export const addSource = defaultStore.addSource.bind(defaultStore);
|
||||
export const getSources = defaultStore.getSources.bind(defaultStore);
|
||||
export const addChunksToNotebook = defaultStore.addChunksToNotebook.bind(defaultStore);
|
||||
export const getChunksForNotebook = defaultStore.getChunksForNotebook.bind(defaultStore);
|
||||
export const buildSourceGroups = defaultStore.buildSourceGroups.bind(defaultStore);
|
||||
65
server/src/stores/sourceCorpusStore.test.ts
Normal file
65
server/src/stores/sourceCorpusStore.test.ts
Normal file
@@ -0,0 +1,65 @@
|
||||
import { describe, it, expect, beforeEach } from 'vitest';
|
||||
import { SourceCorpusStore } from './sourceCorpusStore.js';
|
||||
|
||||
describe('sourceCorpusStore', () => {
|
||||
let store: SourceCorpusStore;
|
||||
|
||||
beforeEach(() => {
|
||||
store = new SourceCorpusStore();
|
||||
});
|
||||
|
||||
it('creates and retrieves a sourceCorpus', () => {
|
||||
const corpus = store.createSourceCorpus({ id: 'nb-1', name: 'Test', createdAt: '2026-01-01' });
|
||||
expect(corpus.id).toBe('nb-1');
|
||||
expect(corpus.name).toBe('Test');
|
||||
|
||||
const found = store.getSourceCorpus('nb-1');
|
||||
expect(found).not.toBeNull();
|
||||
expect(found!.name).toBe('Test');
|
||||
});
|
||||
|
||||
it('lists all sourceCorpora without chunks', () => {
|
||||
store.createSourceCorpus({ id: 'nb-1', name: 'A', createdAt: '2026-01-01' });
|
||||
store.createSourceCorpus({ id: 'nb-2', name: 'B', createdAt: '2026-01-02' });
|
||||
const all = store.getAllSourceCorpora();
|
||||
expect(all).toHaveLength(2);
|
||||
expect(all[0]).not.toHaveProperty('chunks');
|
||||
});
|
||||
|
||||
it('deletes a sourceCorpus', () => {
|
||||
store.createSourceCorpus({ id: 'nb-1', name: 'A', createdAt: '2026-01-01' });
|
||||
expect(store.deleteSourceCorpus('nb-1')).toBe(true);
|
||||
expect(store.getSourceCorpus('nb-1')).toBeNull();
|
||||
});
|
||||
|
||||
it('returns false when deleting a non-existent sourceCorpus', () => {
|
||||
expect(store.deleteSourceCorpus('nope')).toBe(false);
|
||||
});
|
||||
|
||||
it('adds and retrieves sources', () => {
|
||||
store.createSourceCorpus({ id: 'nb-1', name: 'A', createdAt: '2026-01-01' });
|
||||
store.addSource('nb-1', { id: 's-1', name: 'doc.pdf' });
|
||||
const sources = store.getSources('nb-1');
|
||||
expect(sources).toHaveLength(1);
|
||||
expect(sources[0].name).toBe('doc.pdf');
|
||||
});
|
||||
|
||||
it('returns empty sources for non-existent sourceCorpus', () => {
|
||||
expect(store.getSources('nope')).toEqual([]);
|
||||
});
|
||||
|
||||
it('adds and retrieves chunks', () => {
|
||||
store.createSourceCorpus({ id: 'nb-1', name: 'A', createdAt: '2026-01-01' });
|
||||
store.addChunksToSourceCorpus('nb-1', [
|
||||
{ id: 'c-1', text: 'chunk one', sourceId: 's-1', index: 0 },
|
||||
{ id: 'c-2', text: 'chunk two', sourceId: 's-1', index: 1 },
|
||||
]);
|
||||
const chunks = store.getChunksForSourceCorpus('nb-1');
|
||||
expect(chunks).toHaveLength(2);
|
||||
expect(chunks[0].text).toBe('chunk one');
|
||||
});
|
||||
|
||||
it('returns empty chunks for non-existent sourceCorpus', () => {
|
||||
expect(store.getChunksForSourceCorpus('nope')).toEqual([]);
|
||||
});
|
||||
});
|
||||
126
server/src/stores/sourceCorpusStore.ts
Normal file
126
server/src/stores/sourceCorpusStore.ts
Normal file
@@ -0,0 +1,126 @@
|
||||
'use strict';
|
||||
|
||||
export interface TextChunk {
|
||||
id: string;
|
||||
text: string;
|
||||
sourceId: string;
|
||||
index: number;
|
||||
}
|
||||
|
||||
export interface Source {
|
||||
id: string;
|
||||
name: string;
|
||||
mimetype?: string;
|
||||
chunkCount?: number;
|
||||
uploadedAt?: string;
|
||||
}
|
||||
|
||||
export interface SourceCorpus {
|
||||
id: string;
|
||||
name: string;
|
||||
createdAt: string;
|
||||
sources: Source[];
|
||||
chunks: TextChunk[];
|
||||
}
|
||||
|
||||
export interface SourceCorpusMeta {
|
||||
id: string;
|
||||
name: string;
|
||||
createdAt: string;
|
||||
}
|
||||
|
||||
export interface SourceGroup {
|
||||
docIndex: number;
|
||||
sourceId: string;
|
||||
name: string;
|
||||
chunks: TextChunk[];
|
||||
}
|
||||
|
||||
export class SourceCorpusStore {
|
||||
private sourceCorpora = new Map<string, SourceCorpus>();
|
||||
|
||||
getAllSourceCorpora(): SourceCorpusMeta[] {
|
||||
return Array.from(this.sourceCorpora.values()).map(({ id, name, createdAt }) => ({
|
||||
id,
|
||||
name,
|
||||
createdAt,
|
||||
}));
|
||||
}
|
||||
|
||||
getSourceCorpus(id: string): SourceCorpus | null {
|
||||
return this.sourceCorpora.get(id) || null;
|
||||
}
|
||||
|
||||
createSourceCorpus(sourceCorpus: { id: string; name: string; createdAt: string }): SourceCorpusMeta {
|
||||
const record: SourceCorpus = { ...sourceCorpus, sources: [], chunks: [] };
|
||||
this.sourceCorpora.set(record.id, record);
|
||||
return { id: record.id, name: record.name, createdAt: record.createdAt };
|
||||
}
|
||||
|
||||
deleteSourceCorpus(id: string): boolean {
|
||||
return this.sourceCorpora.delete(id);
|
||||
}
|
||||
|
||||
addSource(sourceCorpusId: string, source: Source): Source | null {
|
||||
const sourceCorpus = this.sourceCorpora.get(sourceCorpusId);
|
||||
if (!sourceCorpus) return null;
|
||||
sourceCorpus.sources.push(source);
|
||||
return source;
|
||||
}
|
||||
|
||||
getSources(sourceCorpusId: string): Source[] {
|
||||
const sourceCorpus = this.sourceCorpora.get(sourceCorpusId);
|
||||
if (!sourceCorpus) return [];
|
||||
return sourceCorpus.sources;
|
||||
}
|
||||
|
||||
addChunksToSourceCorpus(sourceCorpusId: string, chunks: TextChunk[]): SourceCorpus | null {
|
||||
const sourceCorpus = this.sourceCorpora.get(sourceCorpusId);
|
||||
if (!sourceCorpus) return null;
|
||||
sourceCorpus.chunks = sourceCorpus.chunks.concat(chunks);
|
||||
return sourceCorpus;
|
||||
}
|
||||
|
||||
getChunksForSourceCorpus(sourceCorpusId: string): TextChunk[] {
|
||||
const sourceCorpus = this.sourceCorpora.get(sourceCorpusId);
|
||||
if (!sourceCorpus) return [];
|
||||
return sourceCorpus.chunks;
|
||||
}
|
||||
|
||||
buildSourceGroups(sourceCorpusId: string, chunks: TextChunk[]): SourceGroup[] {
|
||||
const sources = this.getSources(sourceCorpusId);
|
||||
const sourceIndexMap = new Map<string, number>();
|
||||
sources.forEach((src, i) => {
|
||||
sourceIndexMap.set(src.id, i + 1);
|
||||
});
|
||||
|
||||
const groupMap = new Map<string, SourceGroup>();
|
||||
for (const chunk of chunks) {
|
||||
const docIndex = sourceIndexMap.get(chunk.sourceId) || 0;
|
||||
if (!groupMap.has(chunk.sourceId)) {
|
||||
const src = sources.find((s) => s.id === chunk.sourceId);
|
||||
groupMap.set(chunk.sourceId, {
|
||||
docIndex,
|
||||
sourceId: chunk.sourceId,
|
||||
name: src?.name || 'Unknown',
|
||||
chunks: [],
|
||||
});
|
||||
}
|
||||
groupMap.get(chunk.sourceId)!.chunks.push(chunk);
|
||||
}
|
||||
|
||||
return Array.from(groupMap.values()).sort((a, b) => a.docIndex - b.docIndex);
|
||||
}
|
||||
}
|
||||
|
||||
const defaultStore = new SourceCorpusStore();
|
||||
|
||||
export const getAllSourceCorpora = defaultStore.getAllSourceCorpora.bind(defaultStore);
|
||||
export const getSourceCorpus = defaultStore.getSourceCorpus.bind(defaultStore);
|
||||
export const createSourceCorpus = defaultStore.createSourceCorpus.bind(defaultStore);
|
||||
export const deleteSourceCorpus = defaultStore.deleteSourceCorpus.bind(defaultStore);
|
||||
export const addSource = defaultStore.addSource.bind(defaultStore);
|
||||
export const getSources = defaultStore.getSources.bind(defaultStore);
|
||||
export const addChunksToSourceCorpus = defaultStore.addChunksToSourceCorpus.bind(defaultStore);
|
||||
export const getChunksForSourceCorpus = defaultStore.getChunksForSourceCorpus.bind(defaultStore);
|
||||
export const buildSourceGroups = defaultStore.buildSourceGroups.bind(defaultStore);
|
||||
Reference in New Issue
Block a user