Compare commits
24 Commits
test
...
FEAT-minor
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
5e0189be61 | ||
| fc6ceb558b | |||
| 676c86cf43 | |||
| 6810704173 | |||
| 4a1c67c518 | |||
| 3a4ac641cd | |||
| 861983531a | |||
| fe36ea6aaa | |||
|
|
62dd6de60c | ||
| d481b79465 | |||
| b8c65348e8 | |||
| d6902daa49 | |||
|
|
4f803c14f4 | ||
|
|
720d34b663 | ||
| a79c5ad1cc | |||
|
|
bf89c7544f | ||
| 8ec849b103 | |||
|
|
0352bdf516 | ||
| 4a5e5d6612 | |||
|
|
878cbaea6c | ||
| 6387a84558 | |||
| ef44f52fa2 | |||
| ab068ba43f | |||
| 58e85c00f8 |
6
Makefile
6
Makefile
@@ -20,7 +20,7 @@ fmt-check:
|
||||
check: test lint fmt-check
|
||||
|
||||
docker:
|
||||
docker build -t notebook-clone .
|
||||
docker build -t source-sentinel .
|
||||
|
||||
hooks:
|
||||
printf '#!/usr/bin/env bash\nset -euo pipefail\nmake check\n' > .git/hooks/pre-commit
|
||||
@@ -28,12 +28,12 @@ hooks:
|
||||
|
||||
dev:
|
||||
@echo "Starting backend and frontend..."
|
||||
cd $(SERVER_DIR) && node src/index.js &
|
||||
cd $(SERVER_DIR) && npx --no-install tsx src/index.ts &
|
||||
cd $(WEB_DIR) && npx --no-install vite --port 5173 &
|
||||
wait
|
||||
|
||||
run:
|
||||
cd $(SERVER_DIR) && node src/index.js
|
||||
cd $(SERVER_DIR) && npx --no-install tsx src/index.ts
|
||||
|
||||
build:
|
||||
cd $(WEB_DIR) && npx --no-install vite build
|
||||
|
||||
124
README.md
124
README.md
@@ -1,78 +1,156 @@
|
||||
# Citation Sentinel
|
||||
# Source Sentinel - 2025 @sjDev - LICENSE: MIT
|
||||
|
||||
A source-grounded research assistant that employs cosine similarity scoring for LLM-generated query responses to provide a "groundedness" score. This is a metric in Retrieval-Augmented Generation (RAG) systems that quantifies how well an AI-generated answer is supported by retrieved context. It measures "faithfulness" to source documents, ensuring the answer is not hallucinated or pulled from the model's pre-training data.
|
||||
|
||||
Users upload source documents, or provide links to online sources including audio/video (i.e. links to youtube videos). Users may then ask questions and receive answers (with inline citations) verifiably grounded in the provided information sources.
|
||||

|
||||
|
||||
Image one: source-grounded research assistant for applied use of LLMs.
|
||||
|
||||
**Source Sentinel's value proposition: analyzing and conveniently working with large data corpora, when query-result accuracy is of the highest value. Examples include technical medical data exploration (reviewing health records, test results, or studies to find hidden patterns, trends, and answers), complex litigation (i.e. Patent/IP), advanced LLM development and refinement.**
|
||||
|
||||
Source Sentinel empowers users to transform an otherwise-unmanageably-large corpus of data on any topic of interest into refined, easily-digested subtopics and pose focused inquiries, to instantly receive concise, quantifiably-evaluated, accurate responses. The system also provides suggested follow-up questions further refining in user inquiries by detecting the query goals.
|
||||
|
||||
ALSO NOTE IN DEMO: clickable inline citations (in blue) that take user to source-grounded research basis/knowledge corpus supporting query response, with rated “groundedness” score.
|
||||
|
||||
|
||||
# Reliability and Understandability
|
||||
|
||||
|
||||
Source Sentinel's Retrieval-Augmented Generation (RAG) pipeline uses two-stage pass-through to Voyage AI embedding and ranking models, which uses cosine similarity methodology to scoring to evaluate query-source embeddings in multi-dimensional vector space. This yields a "groundedness" score.
|
||||
|
||||
|
||||
Groundedness metrics quantify how well an AI-generated answer is supported by retrieved context. It measures "faithfulness" to source documents, ensuring the answer is not hallucinated, inaccurate or stale (pulled from the model's training data).
|
||||
|
||||
|
||||
# Architecture - Basic Overview
|
||||
|
||||
|
||||
Built with a React/Vite frontend and a Typescript/Node/Express backend, using Anthropic Claude for NL response generation, OpenAI Whisper for video audio track transcription, Voyage AI voyage-3 for embeddings and Voyage AI rerank-2 for response cosine similarity scoring (to generate the "groundedness" score) presented as an easily-understandable red/yellow/green "badge" style tooltip in the response (see image above.)
|
||||
|
||||
Built with a React/Vite frontend and a Typescript/Node/Express backend, using Anthropic Claude for generation, OpenAI Whisper for video audio track transcription, Voyage AI for embeddings and response cosine similarity scoring (the "groundedness" score).
|
||||
|
||||
## Query pipeline
|
||||
|
||||
Source Ingestion -> Parsing -> Chunking -> Embedding (using Voyage AI voyage-3 model) -> Storage (vector store) -> { user query submission } -> Evaluation of User Query -> Retrieval -> Ranking -> Response Generation (Using Anthopic's claude-opus-4-6 model) -> Response Groundedness Scoring (using Voyage AI rerank-r model)
|
||||
|
||||
Source Ingestion -> Parsing -> Chunking -> Embedding (using Voyage AI voyage-3 model) -> Storage (vector store) -> { user query submission } -> Evaluation of User Query -> Retrieval -> Ranking -> Response Generation (Using Anthopic's claude-opus-4-6 model) -> Response Groundedness Scoring (using Voyage AI rerank-r model)
|
||||
|
||||
## Why Database Math Can't Replace Groundedness Scoring
|
||||
|
||||
Many recent additions to the Vector database/store product space use distance metrics (Cosine, L2, Inner Product) for Bi-Encoder Similarity. They compare the vector of the user query against the vectors of chunks independently.
|
||||
|
||||
The rerank-r model at the end of the Source Sentinel pipeline performs Cross-Encoder Evaluation, taking two entirely separate text inputs:
|
||||
|
||||
1) The generated response from claude-opus-4-6 and
|
||||
|
||||
2) The raw source chunks
|
||||
|
||||
It processes them simultaneously through deep attention layers to check for hallucinations, missing context, and factual alignment.
|
||||
|
||||
A vector database cosine metric only calculates how close two embeddings are in coordinate space. It **does not read the generated text of a response to verify if it accurately reflects the source chunks**.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
|
||||
- **Node.js** (v18+)
|
||||
- **yt-dlp** -- required for YouTube video source support (`brew install yt-dlp`)
|
||||
- **yt-dlp** -- Required for YouTube video source support (`brew install yt-dlp`)
|
||||
|
||||
|
||||
## Getting Started
|
||||
|
||||
|
||||
```bash
|
||||
# clone and install
|
||||
git clone <repo-url> && cd citation_sentinel
|
||||
cd server && npm install && cd ..
|
||||
cd client && npm install && cd ..
|
||||
|
||||
|
||||
# configure
|
||||
cp server/.env.example server/.env
|
||||
# edit server/.env and add your ANTHROPIC_API_KEY, VOYAGE_API_KEY,
|
||||
# and optionally OPENAI_API_KEY (only needed for audio source transcription via Whisper)
|
||||
|
||||
|
||||
# run
|
||||
make dev
|
||||
```
|
||||
|
||||
|
||||
## Rationale
|
||||
|
||||
|
||||
This app demonstrates the core source-grounded Q&A pattern with transparent retrieval, generation, and groundedness scoring (LLM response quality cosine scoring) -- all with swappable models and is fully open source.
|
||||
|
||||
|
||||
## Methodology
|
||||
|
||||
This application is a RAG (Retrieval-Augmented Generation) system that allows users to upload source documents — PDFs, DOCX files, plain text, audio files, web URLs, and YouTube videos — which are then parsed, split into ~2000-character overlapping chunks, and converted into vector embeddings using Voyage AI's voyage-3 model. Those embeddings are stored in an in-memory vector store.
|
||||
|
||||
When a user submits a query, the system enforces groundedness through a multi-layered strategy:
|
||||
This is a RAG (Retrieval-Augmented Generation) and query-response source-groundedness assurance system allowing users to create a research knowledge corpus, including:
|
||||
|
||||
1. **Retrieval constraint** — The query is embedded (via Voyage AI voyage-3) and compared against
|
||||
stored chunk vectors using cosine similarity, returning the top 20 candidates.
|
||||
|
||||
2. **Reranking for precision** — Those 20 candidates are sent to Voyage AI's rerank-2 cross-encoder,
|
||||
which re-scores each query-chunk pair with deeper semantic analysis. Only the top 5 survive.
|
||||
1. Documents — PDFs, DOCX files, plain text audio files.
|
||||
2. Internet sources: via web URLs.
|
||||
2. Audio sources: i.e. YouTube videos, audio from which is transcribed to text.
|
||||
|
||||
3. **Prompt-level constraint** — The top 5 chunks are passed to the "Primary LLM" (Claude opus-4-6).
|
||||
The LLM must cite sources using bracketed indices (e.g., [1], [2]) and admit when sources are
|
||||
insufficient.
|
||||
|
||||
4. **Schema enforcement** — The LLM's response is constrained to a JSON schema requiring structured
|
||||
fields (answer, citedSourceIndices, followUpQuestions). Any cited source indices that do not
|
||||
correspond to real source groups are programmatically stripped out.
|
||||
These are then parsed, split into ~2000-character overlapping chunks, and converted into vector embeddings using Voyage AI's voyage-3 model.
|
||||
|
||||
|
||||
The embeddings are stored in an in-memory vector store.
|
||||
|
||||
|
||||
When a user submits a query, the system enforces groundedness through a multi-layered strategy:
|
||||
|
||||
|
||||
1. **Retrieval constraint** — The query is embedded (via Voyage AI voyage-3) and compared against stored chunk vectors using cosine similarity, returning the top 20 candidates.
|
||||
|
||||
|
||||
2. **Reranking for precision** — Those 20 candidates are sent to Voyage AI's rerank-2 cross-encoder, which re-scores each query-chunk pair with deeper semantic analysis. Only the top 5 survive.
|
||||
|
||||
|
||||
3. **Prompt-level constraint** — The top 5 chunks are passed to the "Primary LLM" (Claude opus-4-6) for natural language query response. The Primary LLM must cite sources using bracketed indices (e.g., [1], [2]) and admit when sources are insufficient.
|
||||
|
||||
|
||||
4. **Schema enforcement** — The Primary LLM's response is constrained to a JSON schema requiring structured fields (answer, citedSourceIndices, followUpQuestions). Any cited source indices that do not correspond to real source groups are programmatically removed.
|
||||
|
||||
|
||||
5. **Post-generation groundedness scoring** — The answer is split into individual sentences. Voyage AI voyage-3 embeds each sentence, and compares it (using cosine similarity) against the vectors of the cited chunks. The similarity is calibrated to a 0–1 scale and averaged, producing a groundedness score surfaced to the user with a visual indicator (“badge” in green/yellow/red).
|
||||
|
||||
|
||||
## Similarity Metrics for Semantic Understanding
|
||||
|
||||
|
||||
Cosine similarity measures how closely two vectors (representing data like words, images, or preferences) are aligned in a multi-dimensional space by calculating the cosine of the angle between them.
|
||||
|
||||
|
||||
Archetypical cosine scores range from -1 to 1.
|
||||
|
||||
|
||||
1: Vectors point in the exact same direction (highly similar).
|
||||
0: Vectors are at a 90-degree angle (orthogonal/unrelated).
|
||||
-1: Vectors point in opposite directions.
|
||||
|
||||
|
||||
Cosine distance between high-dimensional text embeddings is compressed: unrelated notes sit close to orthogonal, so both the within-cluster and nearest-cluster distances land near 0.8. In this application, scores are normalized to a bounded range wherein unrelated query-response pair scores are nearly orthogonal.
|
||||
|
||||
5. **Post-generation groundedness scoring** — The answer is split into individual sentences. Voyage
|
||||
AI voyage-3 embeds each sentence, and compares it (using cosine similarity) against the vectors
|
||||
of the cited chunks. The similarity is calibrated to a 0–1 scale and averaged, producing a single
|
||||
groundedness score that is surfaced to the user with a visual indicator (green/yellow/red).
|
||||
|
||||
## Design
|
||||
|
||||
|
||||
Two-package monorepo:
|
||||
|
||||
|
||||
- `server/` -- single Express backend with layered architecture (routes -> services -> stores). Routes orchestrate; services contain business logic; stores manage in-memory state.
|
||||
|
||||
|
||||
- `client/` -- React 19 SPA via Vite. Two-panel layout: sidebar for data sources, main area for LLM chat with explorable citations, groundedness badges (cosine similarity scoring of LLM responses), and follow-up question chips.
|
||||
|
||||
|
||||
## License
|
||||
|
||||
|
||||
MIT. See [LICENSE](./LICENSE).
|
||||
|
||||
|
||||
## Author
|
||||
|
||||
[@sjdev](https://sjdev.co)
|
||||
|
||||
@sjdev
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
<meta charset="UTF-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<link rel="icon" href="data:image/svg+xml,<svg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 100 100'><circle cx='50' cy='50' r='50' fill='black'/></svg>">
|
||||
<title>NotebookLM Clone</title>
|
||||
<title>Source Sentinel</title>
|
||||
</head>
|
||||
<body>
|
||||
<div id="root"></div>
|
||||
|
||||
4
client/package-lock.json
generated
4
client/package-lock.json
generated
@@ -1,11 +1,11 @@
|
||||
{
|
||||
"name": "notebook-clone-web",
|
||||
"name": "source-sentinel-web",
|
||||
"version": "0.1.0",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "notebook-clone-web",
|
||||
"name": "source-sentinel-web",
|
||||
"version": "0.1.0",
|
||||
"dependencies": {
|
||||
"react": "^19.0.0",
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
{
|
||||
"name": "notebook-clone-web",
|
||||
"name": "source-sentinel-web",
|
||||
"version": "0.1.0",
|
||||
"private": true,
|
||||
"engines": { "node": ">=18" },
|
||||
|
||||
@@ -1,20 +1,20 @@
|
||||
import { useState } from 'react';
|
||||
import NotebookList from './components/NotebookList.jsx';
|
||||
import SourceCorpusList from './components/SourceCorpusList.jsx';
|
||||
import SourcePanel from './components/SourcePanel.jsx';
|
||||
import DocumentButtons from './components/DocumentButtons.jsx';
|
||||
import ChatPanel from './components/ChatPanel.jsx';
|
||||
import DocumentModal from './components/DocumentModal.jsx';
|
||||
import { useNotebook } from './hooks/useNotebook.js';
|
||||
import { useSourceCorpus } from './hooks/useSourceCorpus.js';
|
||||
import { generateDocument } from './api/documents.js';
|
||||
|
||||
function App() {
|
||||
const {
|
||||
notebooks,
|
||||
activeNotebook,
|
||||
selectNotebook,
|
||||
createNotebook,
|
||||
deleteNotebook,
|
||||
} = useNotebook();
|
||||
sourceCorpora,
|
||||
activeSourceCorpus,
|
||||
selectSourceCorpus,
|
||||
createSourceCorpus,
|
||||
deleteSourceCorpus,
|
||||
} = useSourceCorpus();
|
||||
|
||||
const [hoverState, setHoverState] = useState(null);
|
||||
const [sourceCount, setSourceCount] = useState(0);
|
||||
@@ -22,9 +22,9 @@ function App() {
|
||||
const [docModal, setDocModal] = useState({ open: false, type: null, document: null, loading: false });
|
||||
const [chatReady, setChatReady] = useState(false);
|
||||
|
||||
const handleSelectNotebook = (id) => {
|
||||
const handleSelectSourceCorpus = (id) => {
|
||||
setChatReady(false);
|
||||
selectNotebook(id);
|
||||
selectSourceCorpus(id);
|
||||
};
|
||||
|
||||
const handleSourceHover = (val) => {
|
||||
@@ -42,7 +42,7 @@ function App() {
|
||||
setDocModal({ open: true, type, document: null, loading: true });
|
||||
|
||||
try {
|
||||
const res = await generateDocument(activeNotebook.id, type);
|
||||
const res = await generateDocument(activeSourceCorpus.id, type);
|
||||
setDocModal({ open: true, type, document: res.document, loading: false });
|
||||
} catch (err) {
|
||||
console.error('document generation failed', err);
|
||||
@@ -58,17 +58,17 @@ function App() {
|
||||
return (
|
||||
<div className="app">
|
||||
<aside className="sidebar">
|
||||
<NotebookList
|
||||
notebooks={notebooks}
|
||||
activeId={activeNotebook?.id}
|
||||
onSelect={handleSelectNotebook}
|
||||
onCreate={createNotebook}
|
||||
onDelete={deleteNotebook}
|
||||
<SourceCorpusList
|
||||
sourceCorpora={sourceCorpora}
|
||||
activeId={activeSourceCorpus?.id}
|
||||
onSelect={handleSelectSourceCorpus}
|
||||
onCreate={createSourceCorpus}
|
||||
onDelete={deleteSourceCorpus}
|
||||
/>
|
||||
{activeNotebook && (
|
||||
{activeSourceCorpus && (
|
||||
<>
|
||||
<SourcePanel
|
||||
notebookId={activeNotebook.id}
|
||||
sourceCorpusId={activeSourceCorpus.id}
|
||||
hoveredSourceIndex={hoveredDocIndex}
|
||||
onSourceHover={handleSourceHover}
|
||||
onSourcesChange={setSourceCount}
|
||||
@@ -85,16 +85,16 @@ function App() {
|
||||
)}
|
||||
</aside>
|
||||
<main className="main">
|
||||
{activeNotebook ? (
|
||||
{activeSourceCorpus ? (
|
||||
<ChatPanel
|
||||
notebookId={activeNotebook.id}
|
||||
sourceCorpusId={activeSourceCorpus.id}
|
||||
hoveredSource={hoveredInstanceId}
|
||||
hoveredDocIndex={hoveredDocIndex}
|
||||
onSourceHover={handleSourceHover}
|
||||
onFirstResponse={() => setChatReady(true)}
|
||||
/>
|
||||
) : (
|
||||
<p>Select or create a notebook to get started.</p>
|
||||
<p>Select or create a source corpus to get started.</p>
|
||||
)}
|
||||
</main>
|
||||
<DocumentModal
|
||||
|
||||
@@ -4,11 +4,11 @@ import { handleResponse } from './client.js';
|
||||
|
||||
const BASE = '/api/documents';
|
||||
|
||||
export async function generateDocument(notebookId, type) {
|
||||
export async function generateDocument(sourceCorpusId, type) {
|
||||
const res = await fetch(`${BASE}/generate`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ notebookId, type }),
|
||||
body: JSON.stringify({ sourceCorpusId, type }),
|
||||
});
|
||||
return handleResponse(res);
|
||||
}
|
||||
|
||||
@@ -5,11 +5,11 @@ import { handleResponse } from './client.js';
|
||||
const BASE = '/api/query';
|
||||
const CITATION_DETAIL_BASE = '/api/citation-detail';
|
||||
|
||||
export async function sendQuery(notebookId, question, onCitationDetails) {
|
||||
export async function sendQuery(sourceCorpusId, question, onCitationDetails) {
|
||||
const res = await fetch(BASE, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ notebookId, question }),
|
||||
body: JSON.stringify({ sourceCorpusId, question }),
|
||||
});
|
||||
const data = await handleResponse(res);
|
||||
|
||||
|
||||
@@ -16,7 +16,7 @@ describe('sendQuery', () => {
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
it('sends POST to /api/query with notebookId and question', async () => {
|
||||
it('sends POST to /api/query with sourceCorpusId and question', async () => {
|
||||
const data = { answer: 'The answer', citations: [] };
|
||||
fetch.mockResolvedValue(okResponse(data));
|
||||
|
||||
@@ -25,7 +25,7 @@ describe('sendQuery', () => {
|
||||
expect(fetch).toHaveBeenCalledWith('/api/query', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ notebookId: 'nb-1', question: 'What is AI?' }),
|
||||
body: JSON.stringify({ sourceCorpusId: 'nb-1', question: 'What is AI?' }),
|
||||
});
|
||||
expect(result).toEqual(data);
|
||||
});
|
||||
|
||||
@@ -2,14 +2,14 @@
|
||||
|
||||
import { handleResponse } from './client.js';
|
||||
|
||||
const BASE = '/api/notebooks';
|
||||
const BASE = '/api/source-corpus';
|
||||
|
||||
export async function listNotebooks() {
|
||||
export async function listSourceCorpora() {
|
||||
const res = await fetch(BASE);
|
||||
return handleResponse(res);
|
||||
}
|
||||
|
||||
export async function createNotebook(name) {
|
||||
export async function createSourceCorpus(name) {
|
||||
const res = await fetch(BASE, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
@@ -18,7 +18,7 @@ export async function createNotebook(name) {
|
||||
return handleResponse(res);
|
||||
}
|
||||
|
||||
export async function deleteNotebook(id) {
|
||||
export async function deleteSourceCorpus(id) {
|
||||
const res = await fetch(`${BASE}/${id}`, { method: 'DELETE' });
|
||||
return handleResponse(res);
|
||||
}
|
||||
@@ -1,5 +1,5 @@
|
||||
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
|
||||
import { listNotebooks, createNotebook, deleteNotebook } from './notebooks.js';
|
||||
import { listSourceCorpora, createSourceCorpus, deleteSourceCorpus } from './sourceCorpus.js';
|
||||
|
||||
const okResponse = (body) => ({
|
||||
ok: true,
|
||||
@@ -7,7 +7,7 @@ const okResponse = (body) => ({
|
||||
json: () => Promise.resolve(body),
|
||||
});
|
||||
|
||||
describe('notebooks API', () => {
|
||||
describe('sourceCorpora API', () => {
|
||||
beforeEach(() => {
|
||||
vi.stubGlobal('fetch', vi.fn());
|
||||
});
|
||||
@@ -16,41 +16,41 @@ describe('notebooks API', () => {
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
describe('listNotebooks', () => {
|
||||
it('fetches GET /api/notebooks', async () => {
|
||||
describe('listSourceCorpora', () => {
|
||||
it('fetches GET /api/source-corpus', async () => {
|
||||
const data = [{ id: '1', name: 'NB' }];
|
||||
fetch.mockResolvedValue(okResponse(data));
|
||||
|
||||
const result = await listNotebooks();
|
||||
const result = await listSourceCorpora();
|
||||
|
||||
expect(fetch).toHaveBeenCalledWith('/api/notebooks');
|
||||
expect(fetch).toHaveBeenCalledWith('/api/source-corpus');
|
||||
expect(result).toEqual(data);
|
||||
});
|
||||
});
|
||||
|
||||
describe('createNotebook', () => {
|
||||
describe('createSourceCorpus', () => {
|
||||
it('sends POST with name in JSON body', async () => {
|
||||
const nb = { id: '2', name: 'New' };
|
||||
fetch.mockResolvedValue(okResponse(nb));
|
||||
const corpus = { id: '2', name: 'New' };
|
||||
fetch.mockResolvedValue(okResponse(corpus));
|
||||
|
||||
const result = await createNotebook('New');
|
||||
const result = await createSourceCorpus('New');
|
||||
|
||||
expect(fetch).toHaveBeenCalledWith('/api/notebooks', {
|
||||
expect(fetch).toHaveBeenCalledWith('/api/source-corpus', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ name: 'New' }),
|
||||
});
|
||||
expect(result).toEqual(nb);
|
||||
expect(result).toEqual(corpus);
|
||||
});
|
||||
});
|
||||
|
||||
describe('deleteNotebook', () => {
|
||||
it('sends DELETE to /api/notebooks/:id', async () => {
|
||||
describe('deleteSourceCorpus', () => {
|
||||
it('sends DELETE to /api/source-corpus/:id', async () => {
|
||||
fetch.mockResolvedValue(okResponse({}));
|
||||
|
||||
await deleteNotebook('abc-123');
|
||||
await deleteSourceCorpus('abc-123');
|
||||
|
||||
expect(fetch).toHaveBeenCalledWith('/api/notebooks/abc-123', {
|
||||
expect(fetch).toHaveBeenCalledWith('/api/source-corpus/abc-123', {
|
||||
method: 'DELETE',
|
||||
});
|
||||
});
|
||||
@@ -63,6 +63,6 @@ describe('notebooks API', () => {
|
||||
json: () => Promise.resolve({ error: 'Server down' }),
|
||||
});
|
||||
|
||||
await expect(listNotebooks()).rejects.toThrow('Server down');
|
||||
await expect(listSourceCorpora()).rejects.toThrow('Server down');
|
||||
});
|
||||
});
|
||||
@@ -4,15 +4,15 @@ import { handleResponse } from './client.js';
|
||||
|
||||
const BASE = '/api/sources';
|
||||
|
||||
export async function listSources(notebookId) {
|
||||
const res = await fetch(`${BASE}?notebookId=${notebookId}`);
|
||||
export async function listSources(sourceCorpusId) {
|
||||
const res = await fetch(`${BASE}?sourceCorpusId=${sourceCorpusId}`);
|
||||
return handleResponse(res);
|
||||
}
|
||||
|
||||
export async function uploadSource(notebookId, file) {
|
||||
export async function uploadSource(sourceCorpusId, file) {
|
||||
const formData = new FormData();
|
||||
formData.append('file', file);
|
||||
formData.append('notebookId', notebookId);
|
||||
formData.append('sourceCorpusId', sourceCorpusId);
|
||||
const res = await fetch(BASE, {
|
||||
method: 'POST',
|
||||
body: formData,
|
||||
@@ -20,11 +20,11 @@ export async function uploadSource(notebookId, file) {
|
||||
return handleResponse(res);
|
||||
}
|
||||
|
||||
export async function addUrlSource(notebookId, url) {
|
||||
export async function addUrlSource(sourceCorpusId, url) {
|
||||
const res = await fetch(`${BASE}/url`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ notebookId, url }),
|
||||
body: JSON.stringify({ sourceCorpusId, url }),
|
||||
});
|
||||
return handleResponse(res);
|
||||
}
|
||||
|
||||
@@ -17,19 +17,19 @@ describe('sources API', () => {
|
||||
});
|
||||
|
||||
describe('listSources', () => {
|
||||
it('fetches GET /api/sources with notebookId query param', async () => {
|
||||
it('fetches GET /api/sources with sourceCorpusId query param', async () => {
|
||||
const data = [{ id: 's1', name: 'file.pdf' }];
|
||||
fetch.mockResolvedValue(okResponse(data));
|
||||
|
||||
const result = await listSources('nb-42');
|
||||
|
||||
expect(fetch).toHaveBeenCalledWith('/api/sources?notebookId=nb-42');
|
||||
expect(fetch).toHaveBeenCalledWith('/api/sources?sourceCorpusId=nb-42');
|
||||
expect(result).toEqual(data);
|
||||
});
|
||||
});
|
||||
|
||||
describe('uploadSource', () => {
|
||||
it('sends POST with FormData containing file and notebookId', async () => {
|
||||
it('sends POST with FormData containing file and sourceCorpusId', async () => {
|
||||
const source = { id: 's2', name: 'doc.pdf' };
|
||||
fetch.mockResolvedValue(okResponse(source));
|
||||
|
||||
@@ -42,7 +42,7 @@ describe('sources API', () => {
|
||||
});
|
||||
|
||||
const formData = fetch.mock.calls[0][1].body;
|
||||
expect(formData.get('notebookId')).toBe('nb-1');
|
||||
expect(formData.get('sourceCorpusId')).toBe('nb-1');
|
||||
expect(formData.get('file')).toBeInstanceOf(File);
|
||||
expect(result).toEqual(source);
|
||||
});
|
||||
@@ -58,7 +58,7 @@ describe('sources API', () => {
|
||||
expect(fetch).toHaveBeenCalledWith('/api/sources/url', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ notebookId: 'nb-5', url: 'https://example.com' }),
|
||||
body: JSON.stringify({ sourceCorpusId: 'nb-5', url: 'https://example.com' }),
|
||||
});
|
||||
expect(result).toEqual(source);
|
||||
});
|
||||
|
||||
@@ -3,7 +3,7 @@ import { sendQuery } from '../api/query.js';
|
||||
import ChatMessage from './ChatMessage.jsx';
|
||||
import CitationDetailModal from './CitationDetailModal.jsx';
|
||||
|
||||
function ChatPanel({ notebookId, hoveredSource, hoveredDocIndex, onSourceHover, onFirstResponse }) {
|
||||
function ChatPanel({ sourceCorpusId, hoveredSource, hoveredDocIndex, onSourceHover, onFirstResponse }) {
|
||||
const [messages, setMessages] = useState([]);
|
||||
const [input, setInput] = useState('');
|
||||
const [loading, setLoading] = useState(false);
|
||||
@@ -18,7 +18,7 @@ function ChatPanel({ notebookId, hoveredSource, hoveredDocIndex, onSourceHover,
|
||||
setCitationDetails({});
|
||||
setActiveCitation(null);
|
||||
firstResponseFired.current = false;
|
||||
}, [notebookId]);
|
||||
}, [sourceCorpusId]);
|
||||
|
||||
useEffect(() => {
|
||||
bottomRef.current?.scrollIntoView({ behavior: 'smooth' });
|
||||
@@ -35,7 +35,7 @@ function ChatPanel({ notebookId, hoveredSource, hoveredDocIndex, onSourceHover,
|
||||
const msgId = crypto.randomUUID();
|
||||
|
||||
try {
|
||||
const res = await sendQuery(notebookId, question, (details) => {
|
||||
const res = await sendQuery(sourceCorpusId, question, (details) => {
|
||||
const detailMap = {};
|
||||
for (const d of details) {
|
||||
detailMap[d.sourceIndex] = d;
|
||||
|
||||
@@ -1,99 +0,0 @@
|
||||
import { describe, it, expect, vi } from 'vitest';
|
||||
import { render, screen, waitFor } from '@testing-library/react';
|
||||
import userEvent from '@testing-library/user-event';
|
||||
import CreateNotebookModal from './CreateNotebookModal.jsx';
|
||||
|
||||
describe('CreateNotebookModal', () => {
|
||||
it('renders nothing when open is false', () => {
|
||||
const { container } = render(
|
||||
<CreateNotebookModal open={false} onConfirm={() => {}} onCancel={() => {}} />,
|
||||
);
|
||||
expect(container.innerHTML).toBe('');
|
||||
});
|
||||
|
||||
it('renders the modal with title when open', () => {
|
||||
render(<CreateNotebookModal open={true} onConfirm={() => {}} onCancel={() => {}} />);
|
||||
expect(screen.getByText('New Notebook')).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it('renders input and buttons', () => {
|
||||
render(<CreateNotebookModal open={true} onConfirm={() => {}} onCancel={() => {}} />);
|
||||
expect(screen.getByPlaceholderText('Enter notebook name')).toBeInTheDocument();
|
||||
expect(screen.getByText('Cancel')).toBeInTheDocument();
|
||||
expect(screen.getByText('Create')).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it('disables Create button when input is empty', () => {
|
||||
render(<CreateNotebookModal open={true} onConfirm={() => {}} onCancel={() => {}} />);
|
||||
expect(screen.getByText('Create')).toBeDisabled();
|
||||
});
|
||||
|
||||
it('enables Create button when input has text', async () => {
|
||||
const user = userEvent.setup();
|
||||
render(<CreateNotebookModal open={true} onConfirm={() => {}} onCancel={() => {}} />);
|
||||
|
||||
await user.type(screen.getByPlaceholderText('Enter notebook name'), 'My Notebook');
|
||||
expect(screen.getByText('Create')).toBeEnabled();
|
||||
});
|
||||
|
||||
it('calls onConfirm with trimmed name on submit', async () => {
|
||||
const user = userEvent.setup();
|
||||
const onConfirm = vi.fn();
|
||||
render(<CreateNotebookModal open={true} onConfirm={onConfirm} onCancel={() => {}} />);
|
||||
|
||||
await user.type(screen.getByPlaceholderText('Enter notebook name'), ' Research Notes ');
|
||||
await user.click(screen.getByText('Create'));
|
||||
expect(onConfirm).toHaveBeenCalledWith('Research Notes');
|
||||
});
|
||||
|
||||
it('calls onConfirm on Enter key', async () => {
|
||||
const user = userEvent.setup();
|
||||
const onConfirm = vi.fn();
|
||||
render(<CreateNotebookModal open={true} onConfirm={onConfirm} onCancel={() => {}} />);
|
||||
|
||||
await user.type(screen.getByPlaceholderText('Enter notebook name'), 'Test{Enter}');
|
||||
expect(onConfirm).toHaveBeenCalledWith('Test');
|
||||
});
|
||||
|
||||
it('does not call onConfirm when input is only whitespace', async () => {
|
||||
const user = userEvent.setup();
|
||||
const onConfirm = vi.fn();
|
||||
render(<CreateNotebookModal open={true} onConfirm={onConfirm} onCancel={() => {}} />);
|
||||
|
||||
const input = screen.getByPlaceholderText('Enter notebook name');
|
||||
await user.type(input, ' ');
|
||||
await user.type(input, '{Enter}');
|
||||
expect(onConfirm).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('calls onCancel when Cancel button is clicked', async () => {
|
||||
const user = userEvent.setup();
|
||||
const onCancel = vi.fn();
|
||||
render(<CreateNotebookModal open={true} onConfirm={() => {}} onCancel={onCancel} />);
|
||||
|
||||
await user.click(screen.getByText('Cancel'));
|
||||
expect(onCancel).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it('resets input when reopened', async () => {
|
||||
const user = userEvent.setup();
|
||||
const { rerender } = render(
|
||||
<CreateNotebookModal open={true} onConfirm={() => {}} onCancel={() => {}} />,
|
||||
);
|
||||
|
||||
await user.type(screen.getByPlaceholderText('Enter notebook name'), 'Old name');
|
||||
|
||||
rerender(<CreateNotebookModal open={false} onConfirm={() => {}} onCancel={() => {}} />);
|
||||
rerender(<CreateNotebookModal open={true} onConfirm={() => {}} onCancel={() => {}} />);
|
||||
|
||||
expect(screen.getByPlaceholderText('Enter notebook name')).toHaveValue('');
|
||||
});
|
||||
|
||||
it('auto-focuses the input when opened', async () => {
|
||||
render(<CreateNotebookModal open={true} onConfirm={() => {}} onCancel={() => {}} />);
|
||||
|
||||
await waitFor(() => {
|
||||
expect(screen.getByPlaceholderText('Enter notebook name')).toHaveFocus();
|
||||
}, { timeout: 200 });
|
||||
});
|
||||
});
|
||||
@@ -1,7 +1,7 @@
|
||||
import { useState, useEffect, useRef } from 'react';
|
||||
import Modal from './Modal.jsx';
|
||||
|
||||
function CreateNotebookModal({ open, onConfirm, onCancel }) {
|
||||
function CreateSourceCorpusModal({ open, onConfirm, onCancel }) {
|
||||
const [name, setName] = useState('');
|
||||
const inputRef = useRef(null);
|
||||
|
||||
@@ -19,13 +19,13 @@ function CreateNotebookModal({ open, onConfirm, onCancel }) {
|
||||
};
|
||||
|
||||
return (
|
||||
<Modal open={open} onClose={onCancel} title="New Notebook">
|
||||
<Modal open={open} onClose={onCancel} title="New Source Corpus">
|
||||
<form onSubmit={handleSubmit}>
|
||||
<input
|
||||
ref={inputRef}
|
||||
className="modal-input"
|
||||
type="text"
|
||||
placeholder="Enter notebook name"
|
||||
placeholder="Enter source corpus name"
|
||||
value={name}
|
||||
onChange={(e) => setName(e.target.value)}
|
||||
/>
|
||||
@@ -50,4 +50,4 @@ function CreateNotebookModal({ open, onConfirm, onCancel }) {
|
||||
);
|
||||
}
|
||||
|
||||
export default CreateNotebookModal;
|
||||
export default CreateSourceCorpusModal;
|
||||
99
client/src/components/CreateSourceCorpusModal.test.jsx
Normal file
99
client/src/components/CreateSourceCorpusModal.test.jsx
Normal file
@@ -0,0 +1,99 @@
|
||||
import { describe, it, expect, vi } from 'vitest';
|
||||
import { render, screen, waitFor } from '@testing-library/react';
|
||||
import userEvent from '@testing-library/user-event';
|
||||
import CreateSourceCorpusModal from './CreateSourceCorpusModal.jsx';
|
||||
|
||||
describe('CreateSourceCorpusModal', () => {
|
||||
it('renders nothing when open is false', () => {
|
||||
const { container } = render(
|
||||
<CreateSourceCorpusModal open={false} onConfirm={() => {}} onCancel={() => {}} />,
|
||||
);
|
||||
expect(container.innerHTML).toBe('');
|
||||
});
|
||||
|
||||
it('renders the modal with title when open', () => {
|
||||
render(<CreateSourceCorpusModal open={true} onConfirm={() => {}} onCancel={() => {}} />);
|
||||
expect(screen.getByText('New Source Corpus')).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it('renders input and buttons', () => {
|
||||
render(<CreateSourceCorpusModal open={true} onConfirm={() => {}} onCancel={() => {}} />);
|
||||
expect(screen.getByPlaceholderText('Enter source corpus name')).toBeInTheDocument();
|
||||
expect(screen.getByText('Cancel')).toBeInTheDocument();
|
||||
expect(screen.getByText('Create')).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it('disables Create button when input is empty', () => {
|
||||
render(<CreateSourceCorpusModal open={true} onConfirm={() => {}} onCancel={() => {}} />);
|
||||
expect(screen.getByText('Create')).toBeDisabled();
|
||||
});
|
||||
|
||||
it('enables Create button when input has text', async () => {
|
||||
const user = userEvent.setup();
|
||||
render(<CreateSourceCorpusModal open={true} onConfirm={() => {}} onCancel={() => {}} />);
|
||||
|
||||
await user.type(screen.getByPlaceholderText('Enter source corpus name'), 'My SourceCorpus');
|
||||
expect(screen.getByText('Create')).toBeEnabled();
|
||||
});
|
||||
|
||||
it('calls onConfirm with trimmed name on submit', async () => {
|
||||
const user = userEvent.setup();
|
||||
const onConfirm = vi.fn();
|
||||
render(<CreateSourceCorpusModal open={true} onConfirm={onConfirm} onCancel={() => {}} />);
|
||||
|
||||
await user.type(screen.getByPlaceholderText('Enter source corpus name'), ' Research Notes ');
|
||||
await user.click(screen.getByText('Create'));
|
||||
expect(onConfirm).toHaveBeenCalledWith('Research Notes');
|
||||
});
|
||||
|
||||
it('calls onConfirm on Enter key', async () => {
|
||||
const user = userEvent.setup();
|
||||
const onConfirm = vi.fn();
|
||||
render(<CreateSourceCorpusModal open={true} onConfirm={onConfirm} onCancel={() => {}} />);
|
||||
|
||||
await user.type(screen.getByPlaceholderText('Enter source corpus name'), 'Test{Enter}');
|
||||
expect(onConfirm).toHaveBeenCalledWith('Test');
|
||||
});
|
||||
|
||||
it('does not call onConfirm when input is only whitespace', async () => {
|
||||
const user = userEvent.setup();
|
||||
const onConfirm = vi.fn();
|
||||
render(<CreateSourceCorpusModal open={true} onConfirm={onConfirm} onCancel={() => {}} />);
|
||||
|
||||
const input = screen.getByPlaceholderText('Enter source corpus name');
|
||||
await user.type(input, ' ');
|
||||
await user.type(input, '{Enter}');
|
||||
expect(onConfirm).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('calls onCancel when Cancel button is clicked', async () => {
|
||||
const user = userEvent.setup();
|
||||
const onCancel = vi.fn();
|
||||
render(<CreateSourceCorpusModal open={true} onConfirm={() => {}} onCancel={onCancel} />);
|
||||
|
||||
await user.click(screen.getByText('Cancel'));
|
||||
expect(onCancel).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it('resets input when reopened', async () => {
|
||||
const user = userEvent.setup();
|
||||
const { rerender } = render(
|
||||
<CreateSourceCorpusModal open={true} onConfirm={() => {}} onCancel={() => {}} />,
|
||||
);
|
||||
|
||||
await user.type(screen.getByPlaceholderText('Enter source corpus name'), 'Old name');
|
||||
|
||||
rerender(<CreateSourceCorpusModal open={false} onConfirm={() => {}} onCancel={() => {}} />);
|
||||
rerender(<CreateSourceCorpusModal open={true} onConfirm={() => {}} onCancel={() => {}} />);
|
||||
|
||||
expect(screen.getByPlaceholderText('Enter source corpus name')).toHaveValue('');
|
||||
});
|
||||
|
||||
it('auto-focuses the input when opened', async () => {
|
||||
render(<CreateSourceCorpusModal open={true} onConfirm={() => {}} onCancel={() => {}} />);
|
||||
|
||||
await waitFor(() => {
|
||||
expect(screen.getByPlaceholderText('Enter source corpus name')).toHaveFocus();
|
||||
}, { timeout: 200 });
|
||||
});
|
||||
});
|
||||
@@ -1,53 +0,0 @@
|
||||
import { useState } from 'react';
|
||||
import CreateNotebookModal from './CreateNotebookModal.jsx';
|
||||
|
||||
function NotebookList({ notebooks, activeId, onSelect, onCreate, onDelete }) {
|
||||
const [modalOpen, setModalOpen] = useState(false);
|
||||
|
||||
const handleDelete = (e, id) => {
|
||||
e.stopPropagation();
|
||||
onDelete(id);
|
||||
};
|
||||
|
||||
const handleConfirm = (name) => {
|
||||
setModalOpen(false);
|
||||
onCreate(name);
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="notebook-list">
|
||||
<h2>Notebooks</h2>
|
||||
<ul>
|
||||
{notebooks.map((nb) => (
|
||||
<li
|
||||
key={nb.id}
|
||||
className={nb.id === activeId ? 'active' : ''}
|
||||
onClick={() => onSelect(nb.id)}
|
||||
>
|
||||
<span className="nb-name">{nb.name}</span>
|
||||
<button
|
||||
className="nb-delete"
|
||||
onClick={(e) => handleDelete(e, nb.id)}
|
||||
title="Delete notebook"
|
||||
>
|
||||
×
|
||||
</button>
|
||||
</li>
|
||||
))}
|
||||
</ul>
|
||||
<button
|
||||
className="btn-new-notebook"
|
||||
onClick={() => setModalOpen(true)}
|
||||
>
|
||||
+ New Notebook
|
||||
</button>
|
||||
<CreateNotebookModal
|
||||
open={modalOpen}
|
||||
onConfirm={handleConfirm}
|
||||
onCancel={() => setModalOpen(false)}
|
||||
/>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
export default NotebookList;
|
||||
53
client/src/components/SourceCorpusList.jsx
Normal file
53
client/src/components/SourceCorpusList.jsx
Normal file
@@ -0,0 +1,53 @@
|
||||
import { useState } from 'react';
|
||||
import CreateSourceCorpusModal from './CreateSourceCorpusModal.jsx';
|
||||
|
||||
function SourceCorpusList({ sourceCorpora, activeId, onSelect, onCreate, onDelete }) {
|
||||
const [modalOpen, setModalOpen] = useState(false);
|
||||
|
||||
const handleDelete = (e, id) => {
|
||||
e.stopPropagation();
|
||||
onDelete(id);
|
||||
};
|
||||
|
||||
const handleConfirm = (name) => {
|
||||
setModalOpen(false);
|
||||
onCreate(name);
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="source-corpus-list">
|
||||
<h2>Source Corpora</h2>
|
||||
<ul>
|
||||
{sourceCorpora.map((corpus) => (
|
||||
<li
|
||||
key={corpus.id}
|
||||
className={corpus.id === activeId ? 'active' : ''}
|
||||
onClick={() => onSelect(corpus.id)}
|
||||
>
|
||||
<span className="sc-name">{corpus.name}</span>
|
||||
<button
|
||||
className="sc-delete"
|
||||
onClick={(e) => handleDelete(e, corpus.id)}
|
||||
title="Delete source corpus"
|
||||
>
|
||||
×
|
||||
</button>
|
||||
</li>
|
||||
))}
|
||||
</ul>
|
||||
<button
|
||||
className="btn-new-corpus"
|
||||
onClick={() => setModalOpen(true)}
|
||||
>
|
||||
+ New Source Corpus
|
||||
</button>
|
||||
<CreateSourceCorpusModal
|
||||
open={modalOpen}
|
||||
onConfirm={handleConfirm}
|
||||
onCancel={() => setModalOpen(false)}
|
||||
/>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
export default SourceCorpusList;
|
||||
@@ -10,7 +10,7 @@ function isFileAllowed(file) {
|
||||
return ALLOWED_EXTENSIONS.includes(ext);
|
||||
}
|
||||
|
||||
function SourcePanel({ notebookId, hoveredSourceIndex, onSourceHover, onSourcesChange, children }) {
|
||||
function SourcePanel({ sourceCorpusId, hoveredSourceIndex, onSourceHover, onSourcesChange, children }) {
|
||||
const [sources, setSources] = useState([]);
|
||||
const [uploading, setUploading] = useState(false);
|
||||
const [isDragging, setIsDragging] = useState(false);
|
||||
@@ -30,11 +30,11 @@ function SourcePanel({ notebookId, hoveredSourceIndex, onSourceHover, onSourcesC
|
||||
useEffect(() => {
|
||||
setSources([]);
|
||||
onSourcesChange?.(0);
|
||||
sourcesApi.listSources(notebookId).then((s) => {
|
||||
sourcesApi.listSources(sourceCorpusId).then((s) => {
|
||||
setSources(s);
|
||||
onSourcesChange?.(s.length);
|
||||
}).catch((err) => showError(err.message || 'Failed to load sources'));
|
||||
}, [notebookId, onSourcesChange]);
|
||||
}, [sourceCorpusId, onSourcesChange]);
|
||||
|
||||
useEffect(() => () => clearTimeout(errorTimer.current), []);
|
||||
|
||||
@@ -47,7 +47,7 @@ function SourcePanel({ notebookId, hoveredSourceIndex, onSourceHover, onSourcesC
|
||||
setError(null);
|
||||
setUploading(true);
|
||||
try {
|
||||
const src = await sourcesApi.uploadSource(notebookId, file);
|
||||
const src = await sourcesApi.uploadSource(sourceCorpusId, file);
|
||||
setSources((prev) => {
|
||||
const next = [...prev, src];
|
||||
onSourcesChange?.(next.length);
|
||||
@@ -72,7 +72,7 @@ function SourcePanel({ notebookId, hoveredSourceIndex, onSourceHover, onSourcesC
|
||||
setError(null);
|
||||
setAddingUrl(true);
|
||||
try {
|
||||
const src = await sourcesApi.addUrlSource(notebookId, url);
|
||||
const src = await sourcesApi.addUrlSource(sourceCorpusId, url);
|
||||
if (src.error) {
|
||||
showError(src.error);
|
||||
} else {
|
||||
|
||||
@@ -1,50 +0,0 @@
|
||||
import { useState, useEffect, useCallback } from 'react';
|
||||
import * as notebooksApi from '../api/notebooks.js';
|
||||
|
||||
export function useNotebook() {
|
||||
const [notebooks, setNotebooks] = useState([]);
|
||||
const [activeNotebook, setActiveNotebook] = useState(null);
|
||||
|
||||
useEffect(() => {
|
||||
notebooksApi.listNotebooks().then(setNotebooks).catch(console.error);
|
||||
}, []);
|
||||
|
||||
const selectNotebook = useCallback(
|
||||
(id) => {
|
||||
const nb = notebooks.find((n) => n.id === id) || null;
|
||||
setActiveNotebook(nb);
|
||||
},
|
||||
[notebooks],
|
||||
);
|
||||
|
||||
const createNotebook = useCallback(async (name) => {
|
||||
if (!name) return;
|
||||
const nb = await notebooksApi.createNotebook(name);
|
||||
setNotebooks((prev) => [...prev, nb]);
|
||||
setActiveNotebook(nb);
|
||||
}, []);
|
||||
|
||||
const deleteNotebook = useCallback(
|
||||
async (id) => {
|
||||
try {
|
||||
await notebooksApi.deleteNotebook(id);
|
||||
} catch (err) {
|
||||
console.error('Failed to delete notebook', err);
|
||||
return;
|
||||
}
|
||||
setNotebooks((prev) => prev.filter((n) => n.id !== id));
|
||||
if (activeNotebook?.id === id) {
|
||||
setActiveNotebook(null);
|
||||
}
|
||||
},
|
||||
[activeNotebook],
|
||||
);
|
||||
|
||||
return {
|
||||
notebooks,
|
||||
activeNotebook,
|
||||
selectNotebook,
|
||||
createNotebook,
|
||||
deleteNotebook,
|
||||
};
|
||||
}
|
||||
@@ -1,176 +0,0 @@
|
||||
import { describe, it, expect, vi, beforeEach } from 'vitest';
|
||||
import { renderHook, act, waitFor } from '@testing-library/react';
|
||||
|
||||
vi.mock('../api/notebooks.js', () => ({
|
||||
listNotebooks: vi.fn(),
|
||||
createNotebook: vi.fn(),
|
||||
deleteNotebook: vi.fn(),
|
||||
}));
|
||||
|
||||
import { useNotebook } from './useNotebook.js';
|
||||
import * as notebooksApi from '../api/notebooks.js';
|
||||
|
||||
const notebooks = [
|
||||
{ id: 'nb-1', name: 'Research' },
|
||||
{ id: 'nb-2', name: 'Personal' },
|
||||
];
|
||||
|
||||
beforeEach(() => {
|
||||
vi.resetAllMocks();
|
||||
notebooksApi.listNotebooks.mockResolvedValue(notebooks);
|
||||
notebooksApi.createNotebook.mockImplementation(async (name) => ({
|
||||
id: `nb-${Date.now()}`,
|
||||
name,
|
||||
}));
|
||||
notebooksApi.deleteNotebook.mockResolvedValue({});
|
||||
});
|
||||
|
||||
describe('useNotebook', () => {
|
||||
it('loads notebooks on mount', async () => {
|
||||
const { result } = renderHook(() => useNotebook());
|
||||
|
||||
await waitFor(() => {
|
||||
expect(result.current.notebooks).toEqual(notebooks);
|
||||
});
|
||||
expect(notebooksApi.listNotebooks).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it('starts with no active notebook', () => {
|
||||
const { result } = renderHook(() => useNotebook());
|
||||
expect(result.current.activeNotebook).toBeNull();
|
||||
});
|
||||
|
||||
it('selectNotebook sets the active notebook', async () => {
|
||||
const { result } = renderHook(() => useNotebook());
|
||||
|
||||
await waitFor(() => {
|
||||
expect(result.current.notebooks).toHaveLength(2);
|
||||
});
|
||||
|
||||
act(() => {
|
||||
result.current.selectNotebook('nb-2');
|
||||
});
|
||||
|
||||
expect(result.current.activeNotebook).toEqual({ id: 'nb-2', name: 'Personal' });
|
||||
});
|
||||
|
||||
it('selectNotebook sets null for unknown id', async () => {
|
||||
const { result } = renderHook(() => useNotebook());
|
||||
|
||||
await waitFor(() => {
|
||||
expect(result.current.notebooks).toHaveLength(2);
|
||||
});
|
||||
|
||||
act(() => {
|
||||
result.current.selectNotebook('nb-2');
|
||||
});
|
||||
expect(result.current.activeNotebook).not.toBeNull();
|
||||
|
||||
act(() => {
|
||||
result.current.selectNotebook('nonexistent');
|
||||
});
|
||||
expect(result.current.activeNotebook).toBeNull();
|
||||
});
|
||||
|
||||
it('createNotebook calls API and adds to list', async () => {
|
||||
const newNb = { id: 'nb-new', name: 'New One' };
|
||||
notebooksApi.createNotebook.mockResolvedValue(newNb);
|
||||
|
||||
const { result } = renderHook(() => useNotebook());
|
||||
await waitFor(() => expect(result.current.notebooks).toHaveLength(2));
|
||||
|
||||
await act(async () => {
|
||||
await result.current.createNotebook('New One');
|
||||
});
|
||||
|
||||
expect(notebooksApi.createNotebook).toHaveBeenCalledWith('New One');
|
||||
expect(result.current.notebooks).toHaveLength(3);
|
||||
expect(result.current.notebooks[2]).toEqual(newNb);
|
||||
expect(result.current.activeNotebook).toEqual(newNb);
|
||||
});
|
||||
|
||||
it('createNotebook does nothing for empty name', async () => {
|
||||
const { result } = renderHook(() => useNotebook());
|
||||
await waitFor(() => expect(result.current.notebooks).toHaveLength(2));
|
||||
|
||||
await act(async () => {
|
||||
await result.current.createNotebook('');
|
||||
});
|
||||
|
||||
expect(notebooksApi.createNotebook).not.toHaveBeenCalled();
|
||||
expect(result.current.notebooks).toHaveLength(2);
|
||||
});
|
||||
|
||||
it('deleteNotebook removes from list', async () => {
|
||||
const { result } = renderHook(() => useNotebook());
|
||||
await waitFor(() => expect(result.current.notebooks).toHaveLength(2));
|
||||
|
||||
await act(async () => {
|
||||
await result.current.deleteNotebook('nb-1');
|
||||
});
|
||||
|
||||
expect(notebooksApi.deleteNotebook).toHaveBeenCalledWith('nb-1');
|
||||
expect(result.current.notebooks).toHaveLength(1);
|
||||
expect(result.current.notebooks[0].id).toBe('nb-2');
|
||||
});
|
||||
|
||||
it('deleteNotebook clears activeNotebook if it was the deleted one', async () => {
|
||||
const { result } = renderHook(() => useNotebook());
|
||||
await waitFor(() => expect(result.current.notebooks).toHaveLength(2));
|
||||
|
||||
act(() => {
|
||||
result.current.selectNotebook('nb-1');
|
||||
});
|
||||
expect(result.current.activeNotebook?.id).toBe('nb-1');
|
||||
|
||||
await act(async () => {
|
||||
await result.current.deleteNotebook('nb-1');
|
||||
});
|
||||
|
||||
expect(result.current.activeNotebook).toBeNull();
|
||||
});
|
||||
|
||||
it('deleteNotebook preserves activeNotebook if different one deleted', async () => {
|
||||
const { result } = renderHook(() => useNotebook());
|
||||
await waitFor(() => expect(result.current.notebooks).toHaveLength(2));
|
||||
|
||||
act(() => {
|
||||
result.current.selectNotebook('nb-2');
|
||||
});
|
||||
|
||||
await act(async () => {
|
||||
await result.current.deleteNotebook('nb-1');
|
||||
});
|
||||
|
||||
expect(result.current.activeNotebook).toEqual({ id: 'nb-2', name: 'Personal' });
|
||||
});
|
||||
|
||||
it('deleteNotebook does not remove from list on API error', async () => {
|
||||
notebooksApi.deleteNotebook.mockRejectedValue(new Error('Server error'));
|
||||
vi.spyOn(console, 'error').mockImplementation(() => {});
|
||||
|
||||
const { result } = renderHook(() => useNotebook());
|
||||
await waitFor(() => expect(result.current.notebooks).toHaveLength(2));
|
||||
|
||||
await act(async () => {
|
||||
await result.current.deleteNotebook('nb-1');
|
||||
});
|
||||
|
||||
expect(result.current.notebooks).toHaveLength(2);
|
||||
console.error.mockRestore();
|
||||
});
|
||||
|
||||
it('handles listNotebooks API failure gracefully', async () => {
|
||||
notebooksApi.listNotebooks.mockRejectedValue(new Error('Network error'));
|
||||
vi.spyOn(console, 'error').mockImplementation(() => {});
|
||||
|
||||
const { result } = renderHook(() => useNotebook());
|
||||
|
||||
await waitFor(() => {
|
||||
expect(notebooksApi.listNotebooks).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
expect(result.current.notebooks).toEqual([]);
|
||||
console.error.mockRestore();
|
||||
});
|
||||
});
|
||||
50
client/src/hooks/useSourceCorpus.js
Normal file
50
client/src/hooks/useSourceCorpus.js
Normal file
@@ -0,0 +1,50 @@
|
||||
import { useState, useEffect, useCallback } from 'react';
|
||||
import * as sourceCorpusApi from '../api/sourceCorpus.js';
|
||||
|
||||
export function useSourceCorpus() {
|
||||
const [sourceCorpora, setSourceCorpora] = useState([]);
|
||||
const [activeSourceCorpus, setActiveSourceCorpus] = useState(null);
|
||||
|
||||
useEffect(() => {
|
||||
sourceCorpusApi.listSourceCorpora().then(setSourceCorpora).catch(console.error);
|
||||
}, []);
|
||||
|
||||
const selectSourceCorpus = useCallback(
|
||||
(id) => {
|
||||
const corpus = sourceCorpora.find((n) => n.id === id) || null;
|
||||
setActiveSourceCorpus(corpus);
|
||||
},
|
||||
[sourceCorpora],
|
||||
);
|
||||
|
||||
const createSourceCorpus = useCallback(async (name) => {
|
||||
if (!name) return;
|
||||
const corpus = await sourceCorpusApi.createSourceCorpus(name);
|
||||
setSourceCorpora((prev) => [...prev, corpus]);
|
||||
setActiveSourceCorpus(corpus);
|
||||
}, []);
|
||||
|
||||
const deleteSourceCorpus = useCallback(
|
||||
async (id) => {
|
||||
try {
|
||||
await sourceCorpusApi.deleteSourceCorpus(id);
|
||||
} catch (err) {
|
||||
console.error('Failed to delete sourceCorpus', err);
|
||||
return;
|
||||
}
|
||||
setSourceCorpora((prev) => prev.filter((n) => n.id !== id));
|
||||
if (activeSourceCorpus?.id === id) {
|
||||
setActiveSourceCorpus(null);
|
||||
}
|
||||
},
|
||||
[activeSourceCorpus],
|
||||
);
|
||||
|
||||
return {
|
||||
sourceCorpora,
|
||||
activeSourceCorpus,
|
||||
selectSourceCorpus,
|
||||
createSourceCorpus,
|
||||
deleteSourceCorpus,
|
||||
};
|
||||
}
|
||||
176
client/src/hooks/useSourceCorpus.test.js
Normal file
176
client/src/hooks/useSourceCorpus.test.js
Normal file
@@ -0,0 +1,176 @@
|
||||
import { describe, it, expect, vi, beforeEach } from 'vitest';
|
||||
import { renderHook, act, waitFor } from '@testing-library/react';
|
||||
|
||||
vi.mock('../api/sourceCorpus.js', () => ({
|
||||
listSourceCorpora: vi.fn(),
|
||||
createSourceCorpus: vi.fn(),
|
||||
deleteSourceCorpus: vi.fn(),
|
||||
}));
|
||||
|
||||
import { useSourceCorpus } from './useSourceCorpus.js';
|
||||
import * as sourceCorpusApi from '../api/sourceCorpus.js';
|
||||
|
||||
const sourceCorpora = [
|
||||
{ id: 'nb-1', name: 'Research' },
|
||||
{ id: 'nb-2', name: 'Personal' },
|
||||
];
|
||||
|
||||
beforeEach(() => {
|
||||
vi.resetAllMocks();
|
||||
sourceCorpusApi.listSourceCorpora.mockResolvedValue(sourceCorpora);
|
||||
sourceCorpusApi.createSourceCorpus.mockImplementation(async (name) => ({
|
||||
id: `nb-${Date.now()}`,
|
||||
name,
|
||||
}));
|
||||
sourceCorpusApi.deleteSourceCorpus.mockResolvedValue({});
|
||||
});
|
||||
|
||||
describe('useSourceCorpus', () => {
|
||||
it('loads sourceCorpora on mount', async () => {
|
||||
const { result } = renderHook(() => useSourceCorpus());
|
||||
|
||||
await waitFor(() => {
|
||||
expect(result.current.sourceCorpora).toEqual(sourceCorpora);
|
||||
});
|
||||
expect(sourceCorpusApi.listSourceCorpora).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it('starts with no active sourceCorpus', () => {
|
||||
const { result } = renderHook(() => useSourceCorpus());
|
||||
expect(result.current.activeSourceCorpus).toBeNull();
|
||||
});
|
||||
|
||||
it('selectSourceCorpus sets the active sourceCorpus', async () => {
|
||||
const { result } = renderHook(() => useSourceCorpus());
|
||||
|
||||
await waitFor(() => {
|
||||
expect(result.current.sourceCorpora).toHaveLength(2);
|
||||
});
|
||||
|
||||
act(() => {
|
||||
result.current.selectSourceCorpus('nb-2');
|
||||
});
|
||||
|
||||
expect(result.current.activeSourceCorpus).toEqual({ id: 'nb-2', name: 'Personal' });
|
||||
});
|
||||
|
||||
it('selectSourceCorpus sets null for unknown id', async () => {
|
||||
const { result } = renderHook(() => useSourceCorpus());
|
||||
|
||||
await waitFor(() => {
|
||||
expect(result.current.sourceCorpora).toHaveLength(2);
|
||||
});
|
||||
|
||||
act(() => {
|
||||
result.current.selectSourceCorpus('nb-2');
|
||||
});
|
||||
expect(result.current.activeSourceCorpus).not.toBeNull();
|
||||
|
||||
act(() => {
|
||||
result.current.selectSourceCorpus('nonexistent');
|
||||
});
|
||||
expect(result.current.activeSourceCorpus).toBeNull();
|
||||
});
|
||||
|
||||
it('createSourceCorpus calls API and adds to list', async () => {
|
||||
const newCorpus = { id: 'nb-new', name: 'New One' };
|
||||
sourceCorpusApi.createSourceCorpus.mockResolvedValue(newCorpus);
|
||||
|
||||
const { result } = renderHook(() => useSourceCorpus());
|
||||
await waitFor(() => expect(result.current.sourceCorpora).toHaveLength(2));
|
||||
|
||||
await act(async () => {
|
||||
await result.current.createSourceCorpus('New One');
|
||||
});
|
||||
|
||||
expect(sourceCorpusApi.createSourceCorpus).toHaveBeenCalledWith('New One');
|
||||
expect(result.current.sourceCorpora).toHaveLength(3);
|
||||
expect(result.current.sourceCorpora[2]).toEqual(newCorpus);
|
||||
expect(result.current.activeSourceCorpus).toEqual(newCorpus);
|
||||
});
|
||||
|
||||
it('createSourceCorpus does nothing for empty name', async () => {
|
||||
const { result } = renderHook(() => useSourceCorpus());
|
||||
await waitFor(() => expect(result.current.sourceCorpora).toHaveLength(2));
|
||||
|
||||
await act(async () => {
|
||||
await result.current.createSourceCorpus('');
|
||||
});
|
||||
|
||||
expect(sourceCorpusApi.createSourceCorpus).not.toHaveBeenCalled();
|
||||
expect(result.current.sourceCorpora).toHaveLength(2);
|
||||
});
|
||||
|
||||
it('deleteSourceCorpus removes from list', async () => {
|
||||
const { result } = renderHook(() => useSourceCorpus());
|
||||
await waitFor(() => expect(result.current.sourceCorpora).toHaveLength(2));
|
||||
|
||||
await act(async () => {
|
||||
await result.current.deleteSourceCorpus('nb-1');
|
||||
});
|
||||
|
||||
expect(sourceCorpusApi.deleteSourceCorpus).toHaveBeenCalledWith('nb-1');
|
||||
expect(result.current.sourceCorpora).toHaveLength(1);
|
||||
expect(result.current.sourceCorpora[0].id).toBe('nb-2');
|
||||
});
|
||||
|
||||
it('deleteSourceCorpus clears activeSourceCorpus if it was the deleted one', async () => {
|
||||
const { result } = renderHook(() => useSourceCorpus());
|
||||
await waitFor(() => expect(result.current.sourceCorpora).toHaveLength(2));
|
||||
|
||||
act(() => {
|
||||
result.current.selectSourceCorpus('nb-1');
|
||||
});
|
||||
expect(result.current.activeSourceCorpus?.id).toBe('nb-1');
|
||||
|
||||
await act(async () => {
|
||||
await result.current.deleteSourceCorpus('nb-1');
|
||||
});
|
||||
|
||||
expect(result.current.activeSourceCorpus).toBeNull();
|
||||
});
|
||||
|
||||
it('deleteSourceCorpus preserves activeSourceCorpus if different one deleted', async () => {
|
||||
const { result } = renderHook(() => useSourceCorpus());
|
||||
await waitFor(() => expect(result.current.sourceCorpora).toHaveLength(2));
|
||||
|
||||
act(() => {
|
||||
result.current.selectSourceCorpus('nb-2');
|
||||
});
|
||||
|
||||
await act(async () => {
|
||||
await result.current.deleteSourceCorpus('nb-1');
|
||||
});
|
||||
|
||||
expect(result.current.activeSourceCorpus).toEqual({ id: 'nb-2', name: 'Personal' });
|
||||
});
|
||||
|
||||
it('deleteSourceCorpus does not remove from list on API error', async () => {
|
||||
sourceCorpusApi.deleteSourceCorpus.mockRejectedValue(new Error('Server error'));
|
||||
vi.spyOn(console, 'error').mockImplementation(() => {});
|
||||
|
||||
const { result } = renderHook(() => useSourceCorpus());
|
||||
await waitFor(() => expect(result.current.sourceCorpora).toHaveLength(2));
|
||||
|
||||
await act(async () => {
|
||||
await result.current.deleteSourceCorpus('nb-1');
|
||||
});
|
||||
|
||||
expect(result.current.sourceCorpora).toHaveLength(2);
|
||||
console.error.mockRestore();
|
||||
});
|
||||
|
||||
it('handles listSourceCorpora API failure gracefully', async () => {
|
||||
sourceCorpusApi.listSourceCorpora.mockRejectedValue(new Error('Network error'));
|
||||
vi.spyOn(console, 'error').mockImplementation(() => {});
|
||||
|
||||
const { result } = renderHook(() => useSourceCorpus());
|
||||
|
||||
await waitFor(() => {
|
||||
expect(sourceCorpusApi.listSourceCorpora).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
expect(result.current.sourceCorpora).toEqual([]);
|
||||
console.error.mockRestore();
|
||||
});
|
||||
});
|
||||
@@ -75,14 +75,14 @@ body {
|
||||
font-size: 16px;
|
||||
}
|
||||
|
||||
/* ── Notebook List ─────────────────────────────────── */
|
||||
/* ── SourceCorpus List ─────────────────────────────────── */
|
||||
|
||||
.notebook-list {
|
||||
.source-corpus-list {
|
||||
padding: 20px 16px 12px;
|
||||
border-bottom: 1px solid rgba(255, 255, 255, 0.08);
|
||||
}
|
||||
|
||||
.notebook-list h2 {
|
||||
.source-corpus-list h2 {
|
||||
font-size: 11px;
|
||||
font-weight: 600;
|
||||
text-transform: uppercase;
|
||||
@@ -91,12 +91,12 @@ body {
|
||||
margin-bottom: 12px;
|
||||
}
|
||||
|
||||
.notebook-list ul {
|
||||
.source-corpus-list ul {
|
||||
list-style: none;
|
||||
margin-bottom: 8px;
|
||||
}
|
||||
|
||||
.notebook-list li {
|
||||
.source-corpus-list li {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: space-between;
|
||||
@@ -107,23 +107,23 @@ body {
|
||||
transition: background 0.15s;
|
||||
}
|
||||
|
||||
.notebook-list li:hover {
|
||||
.source-corpus-list li:hover {
|
||||
background: var(--sidebar-hover);
|
||||
}
|
||||
|
||||
.notebook-list li.active {
|
||||
.source-corpus-list li.active {
|
||||
background: var(--sidebar-active);
|
||||
color: #fff;
|
||||
font-weight: 500;
|
||||
}
|
||||
|
||||
.notebook-list li .nb-name {
|
||||
.source-corpus-list li .sc-name {
|
||||
overflow: hidden;
|
||||
text-overflow: ellipsis;
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.notebook-list li .nb-delete {
|
||||
.source-corpus-list li .sc-delete {
|
||||
opacity: 0;
|
||||
background: none;
|
||||
border: none;
|
||||
@@ -135,15 +135,15 @@ body {
|
||||
transition: opacity 0.15s, color 0.15s;
|
||||
}
|
||||
|
||||
.notebook-list li:hover .nb-delete {
|
||||
.source-corpus-list li:hover .sc-delete {
|
||||
opacity: 1;
|
||||
}
|
||||
|
||||
.notebook-list li .nb-delete:hover {
|
||||
.source-corpus-list li .sc-delete:hover {
|
||||
color: var(--danger);
|
||||
}
|
||||
|
||||
.btn-new-notebook {
|
||||
.btn-new-corpus {
|
||||
width: 100%;
|
||||
padding: 8px;
|
||||
border: 1px dashed rgba(255, 255, 255, 0.2);
|
||||
@@ -155,7 +155,7 @@ body {
|
||||
transition: border-color 0.15s, color 0.15s;
|
||||
}
|
||||
|
||||
.btn-new-notebook:hover {
|
||||
.btn-new-corpus:hover {
|
||||
border-color: rgba(255, 255, 255, 0.5);
|
||||
color: #fff;
|
||||
}
|
||||
|
||||
BIN
sample screen scaps/DEMO FOR SOURCE SENT.jpg
Normal file
BIN
sample screen scaps/DEMO FOR SOURCE SENT.jpg
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 365 KiB |
BIN
sample screen scaps/DEMO FOR SOURCE SENT_SMALL.jpg
Normal file
BIN
sample screen scaps/DEMO FOR SOURCE SENT_SMALL.jpg
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 133 KiB |
BIN
sample screen scaps/SMALL_DEMO FOR SOURCE SENT.jpg
Normal file
BIN
sample screen scaps/SMALL_DEMO FOR SOURCE SENT.jpg
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 157 KiB |
BIN
sample screen scaps/Screenshot 2026-08-03 at 12.03.34 AM.png
Normal file
BIN
sample screen scaps/Screenshot 2026-08-03 at 12.03.34 AM.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 307 KiB |
BIN
sample screen scaps/Screenshot 2026-08-03 at 12.04.10 AM.png
Normal file
BIN
sample screen scaps/Screenshot 2026-08-03 at 12.04.10 AM.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 309 KiB |
4
server/package-lock.json
generated
4
server/package-lock.json
generated
@@ -1,11 +1,11 @@
|
||||
{
|
||||
"name": "notebook-clone-server",
|
||||
"name": "source-sentinel-server",
|
||||
"version": "0.1.0",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "notebook-clone-server",
|
||||
"name": "source-sentinel-server",
|
||||
"version": "0.1.0",
|
||||
"dependencies": {
|
||||
"@anthropic-ai/sdk": "^0.78.0",
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
{
|
||||
"name": "notebook-clone-server",
|
||||
"name": "source-sentinel-server",
|
||||
"version": "0.1.0",
|
||||
"private": true,
|
||||
"engines": {
|
||||
|
||||
@@ -4,7 +4,7 @@ import 'dotenv/config';
|
||||
import express, { type Request, type Response, type NextFunction } from 'express';
|
||||
import cors from 'cors';
|
||||
import logger from './logger.js';
|
||||
import notebookRoutes from './routes/notebooks.js';
|
||||
import sourceCorpusRoutes from './routes/sourceCorpus.js';
|
||||
import sourceRoutes from './routes/sources.js';
|
||||
import queryRoutes from './routes/query.js';
|
||||
import citationDetailRoutes from './routes/citationDetail.js';
|
||||
@@ -39,13 +39,13 @@ app.use((req: Request, res: Response, next: NextFunction) => {
|
||||
|
||||
app.get('/.well-known/healthcheck', (_req: Request, res: Response) => {
|
||||
res.json({
|
||||
service: 'notebook-clone',
|
||||
service: 'source-sentinel',
|
||||
status: 'ok',
|
||||
uptime: process.uptime(),
|
||||
});
|
||||
});
|
||||
|
||||
app.use('/api/notebooks', notebookRoutes);
|
||||
app.use('/api/source-corpus', sourceCorpusRoutes);
|
||||
app.use('/api/sources', sourceRoutes);
|
||||
app.use('/api/query', queryRoutes);
|
||||
app.use('/api/citation-detail', citationDetailRoutes);
|
||||
|
||||
@@ -1,31 +1,31 @@
|
||||
'use strict';
|
||||
|
||||
import type { Request, Response, NextFunction } from 'express';
|
||||
import * as notebookStore from '../stores/notebookStore.js';
|
||||
import type { Notebook } from '../stores/notebookStore.js';
|
||||
import * as sourceCorpusStore from '../stores/sourceCorpusStore.js';
|
||||
import type { SourceCorpus } from '../stores/sourceCorpusStore.js';
|
||||
|
||||
export interface NotebookRequest extends Request {
|
||||
notebookId?: string;
|
||||
notebook?: Notebook;
|
||||
export interface SourceCorpusRequest extends Request {
|
||||
sourceCorpusId?: string;
|
||||
sourceCorpus?: SourceCorpus;
|
||||
}
|
||||
|
||||
export function requireNotebookId(req: NotebookRequest, res: Response, next: NextFunction): void {
|
||||
const notebookId = (req.body?.notebookId ?? req.query?.notebookId) as string | undefined;
|
||||
if (!notebookId) {
|
||||
res.status(400).json({ error: 'notebookId is required' });
|
||||
export function requireSourceCorpusId(req: SourceCorpusRequest, res: Response, next: NextFunction): void {
|
||||
const sourceCorpusId = (req.body?.sourceCorpusId ?? req.query?.sourceCorpusId) as string | undefined;
|
||||
if (!sourceCorpusId) {
|
||||
res.status(400).json({ error: 'sourceCorpusId is required' });
|
||||
return;
|
||||
}
|
||||
req.notebookId = notebookId;
|
||||
req.sourceCorpusId = sourceCorpusId;
|
||||
next();
|
||||
}
|
||||
|
||||
export function requireNotebook(req: NotebookRequest, res: Response, next: NextFunction): void {
|
||||
const notebook = notebookStore.getNotebook(req.notebookId!);
|
||||
if (!notebook) {
|
||||
res.status(404).json({ error: 'notebook not found' });
|
||||
export function requireSourceCorpus(req: SourceCorpusRequest, res: Response, next: NextFunction): void {
|
||||
const sourceCorpus = sourceCorpusStore.getSourceCorpus(req.sourceCorpusId!);
|
||||
if (!sourceCorpus) {
|
||||
res.status(404).json({ error: 'source corpus not found' });
|
||||
return;
|
||||
}
|
||||
req.notebook = notebook;
|
||||
req.sourceCorpus = sourceCorpus;
|
||||
next();
|
||||
}
|
||||
|
||||
|
||||
@@ -2,8 +2,8 @@
|
||||
|
||||
import { Router, type Request, type Response, type NextFunction } from 'express';
|
||||
import logger from '../logger.js';
|
||||
import * as notebookStore from '../stores/notebookStore.js';
|
||||
import type { TextChunk, Source, SourceGroup } from '../stores/notebookStore.js';
|
||||
import * as sourceCorpusStore from '../stores/sourceCorpusStore.js';
|
||||
import type { TextChunk, Source, SourceGroup } from '../stores/sourceCorpusStore.js';
|
||||
import * as documentCacheStore from '../stores/documentCacheStore.js';
|
||||
import type { CacheEntry } from '../stores/documentCacheStore.js';
|
||||
import {
|
||||
@@ -27,16 +27,16 @@ const GENERATORS: Record<DocumentType, GeneratorFn> = {
|
||||
const router = Router();
|
||||
|
||||
interface GenerateBody {
|
||||
notebookId?: string;
|
||||
sourceCorpusId?: string;
|
||||
type?: string;
|
||||
}
|
||||
|
||||
router.post('/generate', async (req: Request<unknown, unknown, GenerateBody>, res: Response, next: NextFunction) => {
|
||||
try {
|
||||
const { notebookId, type } = req.body;
|
||||
const { sourceCorpusId, type } = req.body;
|
||||
|
||||
if (!notebookId || !type) {
|
||||
res.status(400).json({ error: 'notebookId and type are required' });
|
||||
if (!sourceCorpusId || !type) {
|
||||
res.status(400).json({ error: 'sourceCorpusId and type are required' });
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -48,31 +48,31 @@ router.post('/generate', async (req: Request<unknown, unknown, GenerateBody>, re
|
||||
return;
|
||||
}
|
||||
|
||||
const chunks: TextChunk[] = notebookStore.getChunksForNotebook(notebookId);
|
||||
const chunks: TextChunk[] = sourceCorpusStore.getChunksForSourceCorpus(sourceCorpusId);
|
||||
if (chunks.length === 0) {
|
||||
res.status(422).json({ error: 'No source material available in this notebook' });
|
||||
res.status(422).json({ error: 'No source material available in this source corpus' });
|
||||
return;
|
||||
}
|
||||
|
||||
const sources: Source[] = notebookStore.getSources(notebookId);
|
||||
const cached: CacheEntry | null = documentCacheStore.getCachedDocument(notebookId, type);
|
||||
if (cached && documentCacheStore.isFresh(notebookId, type, sources)) {
|
||||
logger.info({ notebookId, type }, 'serving cached document');
|
||||
const sources: Source[] = sourceCorpusStore.getSources(sourceCorpusId);
|
||||
const cached: CacheEntry | null = documentCacheStore.getCachedDocument(sourceCorpusId, type);
|
||||
if (cached && documentCacheStore.isFresh(sourceCorpusId, type, sources)) {
|
||||
logger.info({ sourceCorpusId, type }, 'serving cached document');
|
||||
res.json({ type, document: cached.document });
|
||||
return;
|
||||
}
|
||||
|
||||
const sourceGroups: SourceGroup[] = notebookStore.buildSourceGroups(notebookId, chunks);
|
||||
const sourceGroups: SourceGroup[] = sourceCorpusStore.buildSourceGroups(sourceCorpusId, chunks);
|
||||
|
||||
logger.info(
|
||||
{ notebookId, type, sourceCount: sourceGroups.length, chunkCount: chunks.length },
|
||||
{ sourceCorpusId, type, sourceCount: sourceGroups.length, chunkCount: chunks.length },
|
||||
'document generation started'
|
||||
);
|
||||
|
||||
const document: StudyGuide | Faq | ExecutiveBrief = await generator(sourceGroups);
|
||||
|
||||
documentCacheStore.setCachedDocument(notebookId, type, document, sources);
|
||||
logger.info({ notebookId, type }, 'document generation complete');
|
||||
documentCacheStore.setCachedDocument(sourceCorpusId, type, document, sources);
|
||||
logger.info({ sourceCorpusId, type }, 'document generation complete');
|
||||
|
||||
res.json({ type, document });
|
||||
} catch (err) {
|
||||
|
||||
@@ -7,8 +7,8 @@ import type { ScoredChunk, RankedChunk, TextChunk } from '../services/retrievalS
|
||||
import { generate } from '../services/generationService.js';
|
||||
import type { GenerationResult } from '../services/generationService.js';
|
||||
import { computeGroundedness } from '../services/scoringService.js';
|
||||
import * as notebookStore from '../stores/notebookStore.js';
|
||||
import type { SourceGroup } from '../stores/notebookStore.js';
|
||||
import * as sourceCorpusStore from '../stores/sourceCorpusStore.js';
|
||||
import type { SourceGroup } from '../stores/sourceCorpusStore.js';
|
||||
|
||||
const TOP_K_SEARCH = 20;
|
||||
const TOP_K_RERANK = 5;
|
||||
@@ -16,7 +16,7 @@ const TOP_K_RERANK = 5;
|
||||
const router = Router();
|
||||
|
||||
interface QueryBody {
|
||||
notebookId?: string;
|
||||
sourceCorpusId?: string;
|
||||
question?: string;
|
||||
}
|
||||
|
||||
@@ -36,20 +36,20 @@ interface QueryResponse {
|
||||
|
||||
router.post('/', async (req: Request<unknown, unknown, QueryBody>, res: Response, next: NextFunction) => {
|
||||
try {
|
||||
const { notebookId, question } = req.body;
|
||||
if (!notebookId || !question) {
|
||||
res.status(400).json({ error: 'notebookId and question are required' });
|
||||
const { sourceCorpusId, question } = req.body;
|
||||
if (!sourceCorpusId || !question) {
|
||||
res.status(400).json({ error: 'sourceCorpusId and question are required' });
|
||||
return;
|
||||
}
|
||||
|
||||
logger.info({ notebookId, question }, 'query received');
|
||||
logger.info({ sourceCorpusId, question }, 'query received');
|
||||
|
||||
const [queryEmbedding]: number[][] = await embedTexts([question], 'query');
|
||||
|
||||
const searchResults: ScoredChunk[] = search(queryEmbedding, notebookId, TOP_K_SEARCH);
|
||||
const searchResults: ScoredChunk[] = search(queryEmbedding, sourceCorpusId, TOP_K_SEARCH);
|
||||
if (searchResults.length === 0) {
|
||||
res.json({
|
||||
answer: 'No sources found for this notebook. Upload some documents first.',
|
||||
answer: 'No sources found for this source corpus. Upload some documents first.',
|
||||
citations: [],
|
||||
groundednessScore: null,
|
||||
followUpQuestions: [],
|
||||
@@ -69,7 +69,7 @@ router.post('/', async (req: Request<unknown, unknown, QueryBody>, res: Response
|
||||
'retrieval complete'
|
||||
);
|
||||
|
||||
const sourceGroups: SourceGroup[] = notebookStore.buildSourceGroups(notebookId, topChunks);
|
||||
const sourceGroups: SourceGroup[] = sourceCorpusStore.buildSourceGroups(sourceCorpusId, topChunks);
|
||||
|
||||
const { answer, citedSourceIndices, followUpQuestions }: GenerationResult = await generate(question, sourceGroups);
|
||||
|
||||
|
||||
@@ -2,33 +2,33 @@
|
||||
|
||||
import { Router, type Request, type Response } from 'express';
|
||||
import { v4 as uuidv4 } from 'uuid';
|
||||
import * as notebookStore from '../stores/notebookStore.js';
|
||||
import * as sourceCorpusStore from '../stores/sourceCorpusStore.js';
|
||||
import { deleteVectorsForChunks } from '../stores/vectorStore.js';
|
||||
|
||||
const router = Router();
|
||||
|
||||
router.get('/', (_req: Request, res: Response) => {
|
||||
res.json(notebookStore.getAllNotebooks());
|
||||
res.json(sourceCorpusStore.getAllSourceCorpora());
|
||||
});
|
||||
|
||||
interface CreateNotebookBody {
|
||||
interface CreateSourceCorpusBody {
|
||||
name?: string;
|
||||
}
|
||||
|
||||
router.post('/', (req: Request<unknown, unknown, CreateNotebookBody>, res: Response) => {
|
||||
router.post('/', (req: Request<unknown, unknown, CreateSourceCorpusBody>, res: Response) => {
|
||||
const { name } = req.body;
|
||||
if (!name || !name.trim()) {
|
||||
res.status(400).json({ error: 'name is required' });
|
||||
return;
|
||||
}
|
||||
|
||||
const notebook = notebookStore.createNotebook({
|
||||
const sourceCorpus = sourceCorpusStore.createSourceCorpus({
|
||||
id: uuidv4(),
|
||||
name: name.trim(),
|
||||
createdAt: new Date().toISOString(),
|
||||
});
|
||||
|
||||
res.status(201).json(notebook);
|
||||
res.status(201).json(sourceCorpus);
|
||||
});
|
||||
|
||||
interface DeleteParams {
|
||||
@@ -36,10 +36,10 @@ interface DeleteParams {
|
||||
}
|
||||
|
||||
router.delete('/:id', (req: Request<DeleteParams>, res: Response) => {
|
||||
const chunks = notebookStore.getChunksForNotebook(req.params.id);
|
||||
const deleted = notebookStore.deleteNotebook(req.params.id);
|
||||
const chunks = sourceCorpusStore.getChunksForSourceCorpus(req.params.id);
|
||||
const deleted = sourceCorpusStore.deleteSourceCorpus(req.params.id);
|
||||
if (!deleted) {
|
||||
res.status(404).json({ error: 'notebook not found' });
|
||||
res.status(404).json({ error: 'source corpus not found' });
|
||||
return;
|
||||
}
|
||||
if (chunks.length) {
|
||||
@@ -4,18 +4,18 @@ import { Router, type Request, type Response, type NextFunction } from 'express'
|
||||
import multer from 'multer';
|
||||
import { v4 as uuidv4 } from 'uuid';
|
||||
import logger from '../logger.js';
|
||||
import * as notebookStore from '../stores/notebookStore.js';
|
||||
import type { Source } from '../stores/notebookStore.js';
|
||||
import * as sourceCorpusStore from '../stores/sourceCorpusStore.js';
|
||||
import type { Source } from '../stores/sourceCorpusStore.js';
|
||||
import { parseFile, chunkText, parseUrl, parseYoutubeUrl, isYoutubeUrl } from '../services/sourceService.js';
|
||||
import type { TextChunk } from '../services/sourceService.js';
|
||||
import { embedTexts, storeChunkEmbeddings } from '../services/retrievalService.js';
|
||||
import { triggerPreGeneration } from '../services/preGenerationService.js';
|
||||
import {
|
||||
requireNotebookId,
|
||||
requireNotebook,
|
||||
requireSourceCorpusId,
|
||||
requireSourceCorpus,
|
||||
requireFile,
|
||||
requireUrl,
|
||||
type NotebookRequest,
|
||||
type SourceCorpusRequest,
|
||||
} from '../middleware/validation.js';
|
||||
|
||||
const upload = multer({ storage: multer.memoryStorage(), limits: { fileSize: 50 * 1024 * 1024 } });
|
||||
@@ -26,8 +26,8 @@ function cleanFilename(name: string): string {
|
||||
return name.replace(TIMESTAMP_RE, '');
|
||||
}
|
||||
|
||||
router.get('/', requireNotebookId, (req: NotebookRequest, res: Response) => {
|
||||
res.json(notebookStore.getSources(req.notebookId!));
|
||||
router.get('/', requireSourceCorpusId, (req: SourceCorpusRequest, res: Response) => {
|
||||
res.json(sourceCorpusStore.getSources(req.sourceCorpusId!));
|
||||
});
|
||||
|
||||
function multerUpload(req: Request, res: Response, next: NextFunction): void {
|
||||
@@ -47,10 +47,10 @@ function multerUpload(req: Request, res: Response, next: NextFunction): void {
|
||||
router.post(
|
||||
'/',
|
||||
multerUpload,
|
||||
requireNotebookId,
|
||||
requireNotebook,
|
||||
requireSourceCorpusId,
|
||||
requireSourceCorpus,
|
||||
requireFile,
|
||||
async (req: NotebookRequest, res: Response, next: NextFunction) => {
|
||||
async (req: SourceCorpusRequest, res: Response, next: NextFunction) => {
|
||||
try {
|
||||
const sourceId: string = uuidv4();
|
||||
const rawName: string = Buffer.from(req.file!.originalname, 'latin1').toString('utf-8');
|
||||
@@ -67,13 +67,13 @@ router.post(
|
||||
'parsed and chunked source'
|
||||
);
|
||||
|
||||
notebookStore.addChunksToNotebook(req.notebookId!, chunks);
|
||||
sourceCorpusStore.addChunksToSourceCorpus(req.sourceCorpusId!, chunks);
|
||||
|
||||
const texts: string[] = chunks.map((c) => c.text);
|
||||
const embeddings: number[][] = await embedTexts(texts, 'document');
|
||||
storeChunkEmbeddings(chunks, embeddings);
|
||||
|
||||
const source: Source | null = notebookStore.addSource(req.notebookId!, {
|
||||
const source: Source | null = sourceCorpusStore.addSource(req.sourceCorpusId!, {
|
||||
id: sourceId,
|
||||
name: displayName,
|
||||
mimetype: req.file!.mimetype,
|
||||
@@ -81,7 +81,7 @@ router.post(
|
||||
uploadedAt: new Date().toISOString(),
|
||||
});
|
||||
|
||||
triggerPreGeneration(req.notebookId!);
|
||||
triggerPreGeneration(req.sourceCorpusId!);
|
||||
|
||||
res.status(201).json(source);
|
||||
} catch (err) {
|
||||
@@ -96,10 +96,10 @@ interface UrlBody {
|
||||
|
||||
router.post(
|
||||
'/url',
|
||||
requireNotebookId,
|
||||
requireNotebook,
|
||||
requireSourceCorpusId,
|
||||
requireSourceCorpus,
|
||||
requireUrl,
|
||||
async (req: NotebookRequest & Request<unknown, unknown, UrlBody>, res: Response, next: NextFunction) => {
|
||||
async (req: SourceCorpusRequest & Request<unknown, unknown, UrlBody>, res: Response, next: NextFunction) => {
|
||||
try {
|
||||
const { url } = req.body;
|
||||
const sourceId: string = uuidv4();
|
||||
@@ -116,13 +116,13 @@ router.post(
|
||||
return;
|
||||
}
|
||||
|
||||
notebookStore.addChunksToNotebook(req.notebookId!, chunks);
|
||||
sourceCorpusStore.addChunksToSourceCorpus(req.sourceCorpusId!, chunks);
|
||||
|
||||
const texts: string[] = chunks.map((c) => c.text);
|
||||
const embeddings: number[][] = await embedTexts(texts, 'document');
|
||||
storeChunkEmbeddings(chunks, embeddings);
|
||||
|
||||
const source: Source | null = notebookStore.addSource(req.notebookId!, {
|
||||
const source: Source | null = sourceCorpusStore.addSource(req.sourceCorpusId!, {
|
||||
id: sourceId,
|
||||
name: displayName,
|
||||
mimetype: isYT ? 'video/youtube' : 'text/html',
|
||||
@@ -130,7 +130,7 @@ router.post(
|
||||
uploadedAt: new Date().toISOString(),
|
||||
});
|
||||
|
||||
triggerPreGeneration(req.notebookId!);
|
||||
triggerPreGeneration(req.sourceCorpusId!);
|
||||
|
||||
res.status(201).json(source);
|
||||
} catch (err) {
|
||||
|
||||
@@ -104,11 +104,11 @@ export async function generateStudyGuide(sourceGroups: SourceGroup[]): Promise<S
|
||||
const sources = buildSourceBlock(sourceGroups);
|
||||
const start = Date.now();
|
||||
|
||||
const prompt = `You are an expert educator. Given the source documents below, produce a comprehensive study guide. Follow these rules:
|
||||
const prompt = `You are an expert reasearch assistant. Given the source documents below, produce a comprehensive guide. Follow these rules:
|
||||
|
||||
1. Create a structured outline of the main ideas organized into logical sections.
|
||||
2. For each section, provide: bullet-point key concepts, key terms with definitions, and 2-3 self-review questions.
|
||||
3. At the end, provide mnemonic devices or simplified restatements to help memorize the hardest concepts.
|
||||
3. At the end, provide mnemonic devices or simplified restatements of the hardest concepts.
|
||||
4. Be concise but thorough. Use simple, clear language.
|
||||
|
||||
--- SOURCE DOCUMENTS ---
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
import { describe, it, expect, vi, beforeEach, afterEach, type Mock } from 'vitest';
|
||||
|
||||
vi.mock('../stores/notebookStore.js', () => ({
|
||||
getNotebook: vi.fn(),
|
||||
vi.mock('../stores/sourceCorpusStore.js', () => ({
|
||||
getSourceCorpus: vi.fn(),
|
||||
getSources: vi.fn(),
|
||||
getChunksForNotebook: vi.fn(),
|
||||
getChunksForSourceCorpus: vi.fn(),
|
||||
buildSourceGroups: vi.fn(),
|
||||
}));
|
||||
|
||||
@@ -26,14 +26,14 @@ vi.mock('../logger.js', () => ({
|
||||
}));
|
||||
|
||||
import { triggerPreGeneration } from './preGenerationService.js';
|
||||
import * as notebookStore from '../stores/notebookStore.js';
|
||||
import * as sourceCorpusStore from '../stores/sourceCorpusStore.js';
|
||||
import * as documentCacheStore from '../stores/documentCacheStore.js';
|
||||
import { generateStudyGuide, generateFaq, generateExecutiveBrief } from './documentService.js';
|
||||
|
||||
const mockedGetNotebook = notebookStore.getNotebook as Mock;
|
||||
const mockedGetSources = notebookStore.getSources as Mock;
|
||||
const mockedGetChunksForNotebook = notebookStore.getChunksForNotebook as Mock;
|
||||
const mockedBuildSourceGroups = notebookStore.buildSourceGroups as Mock;
|
||||
const mockedGetSourceCorpus = sourceCorpusStore.getSourceCorpus as Mock;
|
||||
const mockedGetSources = sourceCorpusStore.getSources as Mock;
|
||||
const mockedGetChunksForSourceCorpus = sourceCorpusStore.getChunksForSourceCorpus as Mock;
|
||||
const mockedBuildSourceGroups = sourceCorpusStore.buildSourceGroups as Mock;
|
||||
const mockedIsGenerating = documentCacheStore.isGenerating as Mock;
|
||||
const mockedInvalidate = documentCacheStore.invalidate as Mock;
|
||||
const mockedMarkGenerating = documentCacheStore.markGenerating as Mock;
|
||||
@@ -43,10 +43,10 @@ const mockedGenerateStudyGuide = generateStudyGuide as Mock;
|
||||
const mockedGenerateFaq = generateFaq as Mock;
|
||||
const mockedGenerateExecutiveBrief = generateExecutiveBrief as Mock;
|
||||
|
||||
function setupNotebook(id: string): void {
|
||||
mockedGetNotebook.mockReturnValue({ id });
|
||||
function setupSourceCorpus(id: string): void {
|
||||
mockedGetSourceCorpus.mockReturnValue({ id });
|
||||
mockedGetSources.mockReturnValue([{ id: 's1' }, { id: 's2' }]);
|
||||
mockedGetChunksForNotebook.mockReturnValue([{ id: 'c1', text: 'hello' }]);
|
||||
mockedGetChunksForSourceCorpus.mockReturnValue([{ id: 'c1', text: 'hello' }]);
|
||||
mockedBuildSourceGroups.mockReturnValue([{ docIndex: 1, name: 'Doc', chunks: [{ text: 'hello' }] }]);
|
||||
}
|
||||
|
||||
@@ -66,14 +66,14 @@ afterEach(() => {
|
||||
});
|
||||
|
||||
describe('triggerPreGeneration guards', () => {
|
||||
it('does nothing if the notebook does not exist', () => {
|
||||
mockedGetNotebook.mockReturnValue(null);
|
||||
it('does nothing if the sourceCorpus does not exist', () => {
|
||||
mockedGetSourceCorpus.mockReturnValue(null);
|
||||
triggerPreGeneration('nb-missing');
|
||||
expect(mockedGetSources).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('does nothing if fewer than 2 sources', () => {
|
||||
mockedGetNotebook.mockReturnValue({ id: 'nb-few' });
|
||||
mockedGetSourceCorpus.mockReturnValue({ id: 'nb-few' });
|
||||
mockedGetSources.mockReturnValue([{ id: 's1' }]);
|
||||
triggerPreGeneration('nb-few');
|
||||
expect(mockedMarkGenerating).not.toHaveBeenCalled();
|
||||
@@ -82,7 +82,7 @@ describe('triggerPreGeneration guards', () => {
|
||||
|
||||
describe('queuing when already generating', () => {
|
||||
it('invalidates cache and skips runPreGeneration', () => {
|
||||
setupNotebook('nb-q');
|
||||
setupSourceCorpus('nb-q');
|
||||
mockedIsGenerating.mockReturnValue(true);
|
||||
|
||||
triggerPreGeneration('nb-q');
|
||||
@@ -94,7 +94,7 @@ describe('queuing when already generating', () => {
|
||||
|
||||
describe('debounce', () => {
|
||||
it('does not run generation immediately', () => {
|
||||
setupNotebook('nb-debounce');
|
||||
setupSourceCorpus('nb-debounce');
|
||||
mockedIsGenerating.mockReturnValue(false);
|
||||
stubGeneratorsOk();
|
||||
|
||||
@@ -104,7 +104,7 @@ describe('debounce', () => {
|
||||
});
|
||||
|
||||
it('runs generation after the debounce delay', async () => {
|
||||
setupNotebook('nb-delay');
|
||||
setupSourceCorpus('nb-delay');
|
||||
mockedIsGenerating.mockReturnValue(false);
|
||||
stubGeneratorsOk();
|
||||
|
||||
@@ -119,7 +119,7 @@ describe('debounce', () => {
|
||||
});
|
||||
|
||||
it('resets the timer on repeated calls, running generation only once', async () => {
|
||||
setupNotebook('nb-batch');
|
||||
setupSourceCorpus('nb-batch');
|
||||
mockedIsGenerating.mockReturnValue(false);
|
||||
stubGeneratorsOk();
|
||||
|
||||
@@ -140,7 +140,7 @@ describe('debounce', () => {
|
||||
|
||||
describe('runPreGeneration (via triggerPreGeneration)', () => {
|
||||
it('runs all three generators and caches results', async () => {
|
||||
setupNotebook('nb-happy');
|
||||
setupSourceCorpus('nb-happy');
|
||||
mockedIsGenerating.mockReturnValue(false);
|
||||
stubGeneratorsOk();
|
||||
|
||||
@@ -158,7 +158,7 @@ describe('runPreGeneration (via triggerPreGeneration)', () => {
|
||||
});
|
||||
|
||||
it('marks generating before starting and clears it after', async () => {
|
||||
setupNotebook('nb-flag');
|
||||
setupSourceCorpus('nb-flag');
|
||||
mockedIsGenerating.mockReturnValue(false);
|
||||
stubGeneratorsOk();
|
||||
|
||||
@@ -173,7 +173,7 @@ describe('runPreGeneration (via triggerPreGeneration)', () => {
|
||||
});
|
||||
|
||||
it('still clears generating flag when a generator fails', async () => {
|
||||
setupNotebook('nb-partial');
|
||||
setupSourceCorpus('nb-partial');
|
||||
mockedIsGenerating.mockReturnValue(false);
|
||||
mockedGenerateStudyGuide.mockRejectedValue(new Error('boom'));
|
||||
mockedGenerateFaq.mockResolvedValue({ subject: 'ok' });
|
||||
@@ -190,9 +190,9 @@ describe('runPreGeneration (via triggerPreGeneration)', () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe('re-trigger for queued notebooks', () => {
|
||||
describe('re-trigger for queued sourceCorpora', () => {
|
||||
it('runs a second generation cycle after the first finishes', async () => {
|
||||
setupNotebook('nb-re');
|
||||
setupSourceCorpus('nb-re');
|
||||
mockedIsGenerating.mockReturnValueOnce(false).mockReturnValueOnce(true);
|
||||
|
||||
let resolveFirst: (value: { title: string }) => void;
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
'use strict';
|
||||
|
||||
import logger from '../logger.js';
|
||||
import * as notebookStore from '../stores/notebookStore.js';
|
||||
import * as sourceCorpusStore from '../stores/sourceCorpusStore.js';
|
||||
import * as documentCacheStore from '../stores/documentCacheStore.js';
|
||||
import {
|
||||
generateStudyGuide,
|
||||
@@ -29,45 +29,45 @@ const DEBOUNCE_MS = 5000;
|
||||
const pendingReGen = new Set<string>();
|
||||
const debounceTimers = new Map<string, ReturnType<typeof setTimeout>>();
|
||||
|
||||
export function triggerPreGeneration(notebookId: string): void {
|
||||
const notebook = notebookStore.getNotebook(notebookId);
|
||||
if (!notebook) return;
|
||||
export function triggerPreGeneration(sourceCorpusId: string): void {
|
||||
const sourceCorpus = sourceCorpusStore.getSourceCorpus(sourceCorpusId);
|
||||
if (!sourceCorpus) return;
|
||||
|
||||
const sources = notebookStore.getSources(notebookId);
|
||||
const sources = sourceCorpusStore.getSources(sourceCorpusId);
|
||||
if (sources.length < MIN_SOURCES) return;
|
||||
|
||||
if (documentCacheStore.isGenerating(notebookId)) {
|
||||
pendingReGen.add(notebookId);
|
||||
documentCacheStore.invalidate(notebookId);
|
||||
logger.debug({ notebookId }, 'pre-generation in progress, queued re-generation');
|
||||
if (documentCacheStore.isGenerating(sourceCorpusId)) {
|
||||
pendingReGen.add(sourceCorpusId);
|
||||
documentCacheStore.invalidate(sourceCorpusId);
|
||||
logger.debug({ sourceCorpusId }, 'pre-generation in progress, queued re-generation');
|
||||
return;
|
||||
}
|
||||
|
||||
const existingTimer = debounceTimers.get(notebookId);
|
||||
const existingTimer = debounceTimers.get(sourceCorpusId);
|
||||
if (existingTimer !== undefined) {
|
||||
clearTimeout(existingTimer);
|
||||
}
|
||||
debounceTimers.set(
|
||||
notebookId,
|
||||
sourceCorpusId,
|
||||
setTimeout(() => {
|
||||
debounceTimers.delete(notebookId);
|
||||
runPreGeneration(notebookId);
|
||||
debounceTimers.delete(sourceCorpusId);
|
||||
runPreGeneration(sourceCorpusId);
|
||||
}, DEBOUNCE_MS)
|
||||
);
|
||||
logger.debug({ notebookId, debounceMs: DEBOUNCE_MS }, 'pre-generation debounced');
|
||||
logger.debug({ sourceCorpusId, debounceMs: DEBOUNCE_MS }, 'pre-generation debounced');
|
||||
}
|
||||
|
||||
function runPreGeneration(notebookId: string): void {
|
||||
function runPreGeneration(sourceCorpusId: string): void {
|
||||
try {
|
||||
const sources = notebookStore.getSources(notebookId);
|
||||
const chunks = notebookStore.getChunksForNotebook(notebookId);
|
||||
const sourceGroups = notebookStore.buildSourceGroups(notebookId, chunks) as SourceGroup[];
|
||||
const sources = sourceCorpusStore.getSources(sourceCorpusId);
|
||||
const chunks = sourceCorpusStore.getChunksForSourceCorpus(sourceCorpusId);
|
||||
const sourceGroups = sourceCorpusStore.buildSourceGroups(sourceCorpusId, chunks) as SourceGroup[];
|
||||
|
||||
documentCacheStore.invalidate(notebookId);
|
||||
documentCacheStore.markGenerating(notebookId);
|
||||
documentCacheStore.invalidate(sourceCorpusId);
|
||||
documentCacheStore.markGenerating(sourceCorpusId);
|
||||
|
||||
logger.info(
|
||||
{ notebookId, sourceCount: sources.length, chunkCount: chunks.length },
|
||||
{ sourceCorpusId, sourceCount: sources.length, chunkCount: chunks.length },
|
||||
'background pre-generation started'
|
||||
);
|
||||
|
||||
@@ -75,31 +75,31 @@ function runPreGeneration(notebookId: string): void {
|
||||
async ([type, generator]) => {
|
||||
try {
|
||||
const document = await generator(sourceGroups);
|
||||
documentCacheStore.setCachedDocument(notebookId, type, document, sources);
|
||||
logger.info({ notebookId, type }, 'background pre-generation complete for type');
|
||||
documentCacheStore.setCachedDocument(sourceCorpusId, type, document, sources);
|
||||
logger.info({ sourceCorpusId, type }, 'background pre-generation complete for type');
|
||||
} catch (err) {
|
||||
const error = err as Error;
|
||||
logger.error({ notebookId, type, err: error.message }, 'background pre-generation failed for type');
|
||||
logger.error({ sourceCorpusId, type, err: error.message }, 'background pre-generation failed for type');
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
Promise.all(jobs)
|
||||
.then(() => {
|
||||
logger.info({ notebookId }, 'all background pre-generation complete');
|
||||
logger.info({ sourceCorpusId }, 'all background pre-generation complete');
|
||||
})
|
||||
.finally(() => {
|
||||
documentCacheStore.clearGenerating(notebookId);
|
||||
documentCacheStore.clearGenerating(sourceCorpusId);
|
||||
|
||||
if (pendingReGen.has(notebookId)) {
|
||||
pendingReGen.delete(notebookId);
|
||||
logger.info({ notebookId }, 're-triggering pre-generation for updated sources');
|
||||
runPreGeneration(notebookId);
|
||||
if (pendingReGen.has(sourceCorpusId)) {
|
||||
pendingReGen.delete(sourceCorpusId);
|
||||
logger.info({ sourceCorpusId }, 're-triggering pre-generation for updated sources');
|
||||
runPreGeneration(sourceCorpusId);
|
||||
}
|
||||
});
|
||||
} catch (err) {
|
||||
const error = err as Error;
|
||||
logger.error({ notebookId, err: error.message }, 'pre-generation setup failed');
|
||||
documentCacheStore.clearGenerating(notebookId);
|
||||
logger.error({ sourceCorpusId, err: error.message }, 'pre-generation setup failed');
|
||||
documentCacheStore.clearGenerating(sourceCorpusId);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
import logger from '../logger.js';
|
||||
import * as vectorStore from '../stores/vectorStore.js';
|
||||
import * as notebookStore from '../stores/notebookStore.js';
|
||||
import * as sourceCorpusStore from '../stores/sourceCorpusStore.js';
|
||||
|
||||
const VOYAGE_API_URL = 'https://api.voyageai.com/v1';
|
||||
const EMBED_MODEL = 'voyage-3';
|
||||
@@ -72,8 +72,8 @@ export function storeChunkEmbeddings(chunks: TextChunk[], embeddings: number[][]
|
||||
logger.debug({ count: chunks.length }, 'stored chunk embeddings');
|
||||
}
|
||||
|
||||
export function search(queryEmbedding: number[], notebookId: string, topK = 10): ScoredChunk[] {
|
||||
const chunks = notebookStore.getChunksForNotebook(notebookId) as TextChunk[];
|
||||
export function search(queryEmbedding: number[], sourceCorpusId: string, topK = 10): ScoredChunk[] {
|
||||
const chunks = sourceCorpusStore.getChunksForSourceCorpus(sourceCorpusId) as TextChunk[];
|
||||
if (chunks.length === 0) return [];
|
||||
|
||||
const scored: ScoredChunk[] = [];
|
||||
|
||||
@@ -136,7 +136,7 @@ export async function parseUrl(url: string): Promise<string> {
|
||||
|
||||
const res = await fetch(safeUrl, {
|
||||
headers: {
|
||||
'User-Agent': 'Mozilla/5.0 (compatible; NotebookClone/1.0)',
|
||||
'User-Agent': 'Mozilla/5.0 (compatible; SourceSentinel/1.0)',
|
||||
Accept: 'text/html,application/xhtml+xml,text/plain',
|
||||
},
|
||||
signal: AbortSignal.timeout(15000),
|
||||
|
||||
@@ -57,7 +57,7 @@ describe('documentCacheStore', () => {
|
||||
});
|
||||
|
||||
describe('invalidate', () => {
|
||||
it('removes all entries for a notebook', () => {
|
||||
it('removes all entries for a sourceCorpus', () => {
|
||||
setCachedDocument('inv-1', 'faq', { a: 1 }, [{ id: 'x' }]);
|
||||
setCachedDocument('inv-1', 'study-guide', { b: 2 }, [{ id: 'x' }]);
|
||||
invalidate('inv-1');
|
||||
@@ -65,7 +65,7 @@ describe('documentCacheStore', () => {
|
||||
expect(getCachedDocument('inv-1', 'study-guide')).toBeNull();
|
||||
});
|
||||
|
||||
it('does not affect other notebooks', () => {
|
||||
it('does not affect other sourceCorpora', () => {
|
||||
setCachedDocument('inv-a', 'faq', { a: 1 }, [{ id: 'x' }]);
|
||||
setCachedDocument('inv-b', 'faq', { b: 2 }, [{ id: 'x' }]);
|
||||
invalidate('inv-a');
|
||||
@@ -86,7 +86,7 @@ describe('documentCacheStore', () => {
|
||||
expect(isGenerating('gen-cycle')).toBe(false);
|
||||
});
|
||||
|
||||
it('is cleared when the notebook is invalidated', () => {
|
||||
it('is cleared when the sourceCorpus is invalidated', () => {
|
||||
markGenerating('gen-inv');
|
||||
invalidate('gen-inv');
|
||||
expect(isGenerating('gen-inv')).toBe(false);
|
||||
|
||||
@@ -19,19 +19,19 @@ function buildSourceHash(sources: CacheSource[]): string {
|
||||
.join('|');
|
||||
}
|
||||
|
||||
export function getCachedDocument(notebookId: string, type: string): CacheEntry | null {
|
||||
const entry = cache.get(`${notebookId}:${type}`);
|
||||
export function getCachedDocument(sourceCorpusId: string, type: string): CacheEntry | null {
|
||||
const entry = cache.get(`${sourceCorpusId}:${type}`);
|
||||
if (!entry || typeof entry === 'boolean') return null;
|
||||
return entry;
|
||||
}
|
||||
|
||||
export function setCachedDocument(
|
||||
notebookId: string,
|
||||
sourceCorpusId: string,
|
||||
type: string,
|
||||
document: unknown,
|
||||
sources: CacheSource[]
|
||||
): void {
|
||||
const key = `${notebookId}:${type}`;
|
||||
const key = `${sourceCorpusId}:${type}`;
|
||||
cache.set(key, {
|
||||
document,
|
||||
sourceHash: buildSourceHash(sources),
|
||||
@@ -39,29 +39,29 @@ export function setCachedDocument(
|
||||
});
|
||||
}
|
||||
|
||||
export function isFresh(notebookId: string, type: string, currentSources: CacheSource[]): boolean {
|
||||
const entry = cache.get(`${notebookId}:${type}`);
|
||||
export function isFresh(sourceCorpusId: string, type: string, currentSources: CacheSource[]): boolean {
|
||||
const entry = cache.get(`${sourceCorpusId}:${type}`);
|
||||
if (!entry || typeof entry === 'boolean') return false;
|
||||
return entry.sourceHash === buildSourceHash(currentSources);
|
||||
}
|
||||
|
||||
export function invalidate(notebookId: string): void {
|
||||
export function invalidate(sourceCorpusId: string): void {
|
||||
for (const key of cache.keys()) {
|
||||
if (key.startsWith(`${notebookId}:`)) {
|
||||
if (key.startsWith(`${sourceCorpusId}:`)) {
|
||||
cache.delete(key);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export function markGenerating(notebookId: string): void {
|
||||
const key = `${notebookId}:__generating`;
|
||||
export function markGenerating(sourceCorpusId: string): void {
|
||||
const key = `${sourceCorpusId}:__generating`;
|
||||
cache.set(key, true);
|
||||
}
|
||||
|
||||
export function clearGenerating(notebookId: string): void {
|
||||
cache.delete(`${notebookId}:__generating`);
|
||||
export function clearGenerating(sourceCorpusId: string): void {
|
||||
cache.delete(`${sourceCorpusId}:__generating`);
|
||||
}
|
||||
|
||||
export function isGenerating(notebookId: string): boolean {
|
||||
return cache.get(`${notebookId}:__generating`) === true;
|
||||
export function isGenerating(sourceCorpusId: string): boolean {
|
||||
return cache.get(`${sourceCorpusId}:__generating`) === true;
|
||||
}
|
||||
|
||||
@@ -1,65 +0,0 @@
|
||||
import { describe, it, expect, beforeEach } from 'vitest';
|
||||
import { NotebookStore } from './notebookStore.js';
|
||||
|
||||
describe('notebookStore', () => {
|
||||
let store: NotebookStore;
|
||||
|
||||
beforeEach(() => {
|
||||
store = new NotebookStore();
|
||||
});
|
||||
|
||||
it('creates and retrieves a notebook', () => {
|
||||
const nb = store.createNotebook({ id: 'nb-1', name: 'Test', createdAt: '2026-01-01' });
|
||||
expect(nb.id).toBe('nb-1');
|
||||
expect(nb.name).toBe('Test');
|
||||
|
||||
const found = store.getNotebook('nb-1');
|
||||
expect(found).not.toBeNull();
|
||||
expect(found!.name).toBe('Test');
|
||||
});
|
||||
|
||||
it('lists all notebooks without chunks', () => {
|
||||
store.createNotebook({ id: 'nb-1', name: 'A', createdAt: '2026-01-01' });
|
||||
store.createNotebook({ id: 'nb-2', name: 'B', createdAt: '2026-01-02' });
|
||||
const all = store.getAllNotebooks();
|
||||
expect(all).toHaveLength(2);
|
||||
expect(all[0]).not.toHaveProperty('chunks');
|
||||
});
|
||||
|
||||
it('deletes a notebook', () => {
|
||||
store.createNotebook({ id: 'nb-1', name: 'A', createdAt: '2026-01-01' });
|
||||
expect(store.deleteNotebook('nb-1')).toBe(true);
|
||||
expect(store.getNotebook('nb-1')).toBeNull();
|
||||
});
|
||||
|
||||
it('returns false when deleting a non-existent notebook', () => {
|
||||
expect(store.deleteNotebook('nope')).toBe(false);
|
||||
});
|
||||
|
||||
it('adds and retrieves sources', () => {
|
||||
store.createNotebook({ id: 'nb-1', name: 'A', createdAt: '2026-01-01' });
|
||||
store.addSource('nb-1', { id: 's-1', name: 'doc.pdf' });
|
||||
const sources = store.getSources('nb-1');
|
||||
expect(sources).toHaveLength(1);
|
||||
expect(sources[0].name).toBe('doc.pdf');
|
||||
});
|
||||
|
||||
it('returns empty sources for non-existent notebook', () => {
|
||||
expect(store.getSources('nope')).toEqual([]);
|
||||
});
|
||||
|
||||
it('adds and retrieves chunks', () => {
|
||||
store.createNotebook({ id: 'nb-1', name: 'A', createdAt: '2026-01-01' });
|
||||
store.addChunksToNotebook('nb-1', [
|
||||
{ id: 'c-1', text: 'chunk one', sourceId: 's-1', index: 0 },
|
||||
{ id: 'c-2', text: 'chunk two', sourceId: 's-1', index: 1 },
|
||||
]);
|
||||
const chunks = store.getChunksForNotebook('nb-1');
|
||||
expect(chunks).toHaveLength(2);
|
||||
expect(chunks[0].text).toBe('chunk one');
|
||||
});
|
||||
|
||||
it('returns empty chunks for non-existent notebook', () => {
|
||||
expect(store.getChunksForNotebook('nope')).toEqual([]);
|
||||
});
|
||||
});
|
||||
@@ -1,126 +0,0 @@
|
||||
'use strict';
|
||||
|
||||
export interface TextChunk {
|
||||
id: string;
|
||||
text: string;
|
||||
sourceId: string;
|
||||
index: number;
|
||||
}
|
||||
|
||||
export interface Source {
|
||||
id: string;
|
||||
name: string;
|
||||
mimetype?: string;
|
||||
chunkCount?: number;
|
||||
uploadedAt?: string;
|
||||
}
|
||||
|
||||
export interface Notebook {
|
||||
id: string;
|
||||
name: string;
|
||||
createdAt: string;
|
||||
sources: Source[];
|
||||
chunks: TextChunk[];
|
||||
}
|
||||
|
||||
export interface NotebookMeta {
|
||||
id: string;
|
||||
name: string;
|
||||
createdAt: string;
|
||||
}
|
||||
|
||||
export interface SourceGroup {
|
||||
docIndex: number;
|
||||
sourceId: string;
|
||||
name: string;
|
||||
chunks: TextChunk[];
|
||||
}
|
||||
|
||||
export class NotebookStore {
|
||||
private notebooks = new Map<string, Notebook>();
|
||||
|
||||
getAllNotebooks(): NotebookMeta[] {
|
||||
return Array.from(this.notebooks.values()).map(({ id, name, createdAt }) => ({
|
||||
id,
|
||||
name,
|
||||
createdAt,
|
||||
}));
|
||||
}
|
||||
|
||||
getNotebook(id: string): Notebook | null {
|
||||
return this.notebooks.get(id) || null;
|
||||
}
|
||||
|
||||
createNotebook(notebook: { id: string; name: string; createdAt: string }): NotebookMeta {
|
||||
const record: Notebook = { ...notebook, sources: [], chunks: [] };
|
||||
this.notebooks.set(record.id, record);
|
||||
return { id: record.id, name: record.name, createdAt: record.createdAt };
|
||||
}
|
||||
|
||||
deleteNotebook(id: string): boolean {
|
||||
return this.notebooks.delete(id);
|
||||
}
|
||||
|
||||
addSource(notebookId: string, source: Source): Source | null {
|
||||
const notebook = this.notebooks.get(notebookId);
|
||||
if (!notebook) return null;
|
||||
notebook.sources.push(source);
|
||||
return source;
|
||||
}
|
||||
|
||||
getSources(notebookId: string): Source[] {
|
||||
const notebook = this.notebooks.get(notebookId);
|
||||
if (!notebook) return [];
|
||||
return notebook.sources;
|
||||
}
|
||||
|
||||
addChunksToNotebook(notebookId: string, chunks: TextChunk[]): Notebook | null {
|
||||
const notebook = this.notebooks.get(notebookId);
|
||||
if (!notebook) return null;
|
||||
notebook.chunks = notebook.chunks.concat(chunks);
|
||||
return notebook;
|
||||
}
|
||||
|
||||
getChunksForNotebook(notebookId: string): TextChunk[] {
|
||||
const notebook = this.notebooks.get(notebookId);
|
||||
if (!notebook) return [];
|
||||
return notebook.chunks;
|
||||
}
|
||||
|
||||
buildSourceGroups(notebookId: string, chunks: TextChunk[]): SourceGroup[] {
|
||||
const sources = this.getSources(notebookId);
|
||||
const sourceIndexMap = new Map<string, number>();
|
||||
sources.forEach((src, i) => {
|
||||
sourceIndexMap.set(src.id, i + 1);
|
||||
});
|
||||
|
||||
const groupMap = new Map<string, SourceGroup>();
|
||||
for (const chunk of chunks) {
|
||||
const docIndex = sourceIndexMap.get(chunk.sourceId) || 0;
|
||||
if (!groupMap.has(chunk.sourceId)) {
|
||||
const src = sources.find((s) => s.id === chunk.sourceId);
|
||||
groupMap.set(chunk.sourceId, {
|
||||
docIndex,
|
||||
sourceId: chunk.sourceId,
|
||||
name: src?.name || 'Unknown',
|
||||
chunks: [],
|
||||
});
|
||||
}
|
||||
groupMap.get(chunk.sourceId)!.chunks.push(chunk);
|
||||
}
|
||||
|
||||
return Array.from(groupMap.values()).sort((a, b) => a.docIndex - b.docIndex);
|
||||
}
|
||||
}
|
||||
|
||||
const defaultStore = new NotebookStore();
|
||||
|
||||
export const getAllNotebooks = defaultStore.getAllNotebooks.bind(defaultStore);
|
||||
export const getNotebook = defaultStore.getNotebook.bind(defaultStore);
|
||||
export const createNotebook = defaultStore.createNotebook.bind(defaultStore);
|
||||
export const deleteNotebook = defaultStore.deleteNotebook.bind(defaultStore);
|
||||
export const addSource = defaultStore.addSource.bind(defaultStore);
|
||||
export const getSources = defaultStore.getSources.bind(defaultStore);
|
||||
export const addChunksToNotebook = defaultStore.addChunksToNotebook.bind(defaultStore);
|
||||
export const getChunksForNotebook = defaultStore.getChunksForNotebook.bind(defaultStore);
|
||||
export const buildSourceGroups = defaultStore.buildSourceGroups.bind(defaultStore);
|
||||
65
server/src/stores/sourceCorpusStore.test.ts
Normal file
65
server/src/stores/sourceCorpusStore.test.ts
Normal file
@@ -0,0 +1,65 @@
|
||||
import { describe, it, expect, beforeEach } from 'vitest';
|
||||
import { SourceCorpusStore } from './sourceCorpusStore.js';
|
||||
|
||||
describe('sourceCorpusStore', () => {
|
||||
let store: SourceCorpusStore;
|
||||
|
||||
beforeEach(() => {
|
||||
store = new SourceCorpusStore();
|
||||
});
|
||||
|
||||
it('creates and retrieves a sourceCorpus', () => {
|
||||
const corpus = store.createSourceCorpus({ id: 'nb-1', name: 'Test', createdAt: '2026-01-01' });
|
||||
expect(corpus.id).toBe('nb-1');
|
||||
expect(corpus.name).toBe('Test');
|
||||
|
||||
const found = store.getSourceCorpus('nb-1');
|
||||
expect(found).not.toBeNull();
|
||||
expect(found!.name).toBe('Test');
|
||||
});
|
||||
|
||||
it('lists all sourceCorpora without chunks', () => {
|
||||
store.createSourceCorpus({ id: 'nb-1', name: 'A', createdAt: '2026-01-01' });
|
||||
store.createSourceCorpus({ id: 'nb-2', name: 'B', createdAt: '2026-01-02' });
|
||||
const all = store.getAllSourceCorpora();
|
||||
expect(all).toHaveLength(2);
|
||||
expect(all[0]).not.toHaveProperty('chunks');
|
||||
});
|
||||
|
||||
it('deletes a sourceCorpus', () => {
|
||||
store.createSourceCorpus({ id: 'nb-1', name: 'A', createdAt: '2026-01-01' });
|
||||
expect(store.deleteSourceCorpus('nb-1')).toBe(true);
|
||||
expect(store.getSourceCorpus('nb-1')).toBeNull();
|
||||
});
|
||||
|
||||
it('returns false when deleting a non-existent sourceCorpus', () => {
|
||||
expect(store.deleteSourceCorpus('nope')).toBe(false);
|
||||
});
|
||||
|
||||
it('adds and retrieves sources', () => {
|
||||
store.createSourceCorpus({ id: 'nb-1', name: 'A', createdAt: '2026-01-01' });
|
||||
store.addSource('nb-1', { id: 's-1', name: 'doc.pdf' });
|
||||
const sources = store.getSources('nb-1');
|
||||
expect(sources).toHaveLength(1);
|
||||
expect(sources[0].name).toBe('doc.pdf');
|
||||
});
|
||||
|
||||
it('returns empty sources for non-existent sourceCorpus', () => {
|
||||
expect(store.getSources('nope')).toEqual([]);
|
||||
});
|
||||
|
||||
it('adds and retrieves chunks', () => {
|
||||
store.createSourceCorpus({ id: 'nb-1', name: 'A', createdAt: '2026-01-01' });
|
||||
store.addChunksToSourceCorpus('nb-1', [
|
||||
{ id: 'c-1', text: 'chunk one', sourceId: 's-1', index: 0 },
|
||||
{ id: 'c-2', text: 'chunk two', sourceId: 's-1', index: 1 },
|
||||
]);
|
||||
const chunks = store.getChunksForSourceCorpus('nb-1');
|
||||
expect(chunks).toHaveLength(2);
|
||||
expect(chunks[0].text).toBe('chunk one');
|
||||
});
|
||||
|
||||
it('returns empty chunks for non-existent sourceCorpus', () => {
|
||||
expect(store.getChunksForSourceCorpus('nope')).toEqual([]);
|
||||
});
|
||||
});
|
||||
126
server/src/stores/sourceCorpusStore.ts
Normal file
126
server/src/stores/sourceCorpusStore.ts
Normal file
@@ -0,0 +1,126 @@
|
||||
'use strict';
|
||||
|
||||
export interface TextChunk {
|
||||
id: string;
|
||||
text: string;
|
||||
sourceId: string;
|
||||
index: number;
|
||||
}
|
||||
|
||||
export interface Source {
|
||||
id: string;
|
||||
name: string;
|
||||
mimetype?: string;
|
||||
chunkCount?: number;
|
||||
uploadedAt?: string;
|
||||
}
|
||||
|
||||
export interface SourceCorpus {
|
||||
id: string;
|
||||
name: string;
|
||||
createdAt: string;
|
||||
sources: Source[];
|
||||
chunks: TextChunk[];
|
||||
}
|
||||
|
||||
export interface SourceCorpusMeta {
|
||||
id: string;
|
||||
name: string;
|
||||
createdAt: string;
|
||||
}
|
||||
|
||||
export interface SourceGroup {
|
||||
docIndex: number;
|
||||
sourceId: string;
|
||||
name: string;
|
||||
chunks: TextChunk[];
|
||||
}
|
||||
|
||||
export class SourceCorpusStore {
|
||||
private sourceCorpora = new Map<string, SourceCorpus>();
|
||||
|
||||
getAllSourceCorpora(): SourceCorpusMeta[] {
|
||||
return Array.from(this.sourceCorpora.values()).map(({ id, name, createdAt }) => ({
|
||||
id,
|
||||
name,
|
||||
createdAt,
|
||||
}));
|
||||
}
|
||||
|
||||
getSourceCorpus(id: string): SourceCorpus | null {
|
||||
return this.sourceCorpora.get(id) || null;
|
||||
}
|
||||
|
||||
createSourceCorpus(sourceCorpus: { id: string; name: string; createdAt: string }): SourceCorpusMeta {
|
||||
const record: SourceCorpus = { ...sourceCorpus, sources: [], chunks: [] };
|
||||
this.sourceCorpora.set(record.id, record);
|
||||
return { id: record.id, name: record.name, createdAt: record.createdAt };
|
||||
}
|
||||
|
||||
deleteSourceCorpus(id: string): boolean {
|
||||
return this.sourceCorpora.delete(id);
|
||||
}
|
||||
|
||||
addSource(sourceCorpusId: string, source: Source): Source | null {
|
||||
const sourceCorpus = this.sourceCorpora.get(sourceCorpusId);
|
||||
if (!sourceCorpus) return null;
|
||||
sourceCorpus.sources.push(source);
|
||||
return source;
|
||||
}
|
||||
|
||||
getSources(sourceCorpusId: string): Source[] {
|
||||
const sourceCorpus = this.sourceCorpora.get(sourceCorpusId);
|
||||
if (!sourceCorpus) return [];
|
||||
return sourceCorpus.sources;
|
||||
}
|
||||
|
||||
addChunksToSourceCorpus(sourceCorpusId: string, chunks: TextChunk[]): SourceCorpus | null {
|
||||
const sourceCorpus = this.sourceCorpora.get(sourceCorpusId);
|
||||
if (!sourceCorpus) return null;
|
||||
sourceCorpus.chunks = sourceCorpus.chunks.concat(chunks);
|
||||
return sourceCorpus;
|
||||
}
|
||||
|
||||
getChunksForSourceCorpus(sourceCorpusId: string): TextChunk[] {
|
||||
const sourceCorpus = this.sourceCorpora.get(sourceCorpusId);
|
||||
if (!sourceCorpus) return [];
|
||||
return sourceCorpus.chunks;
|
||||
}
|
||||
|
||||
buildSourceGroups(sourceCorpusId: string, chunks: TextChunk[]): SourceGroup[] {
|
||||
const sources = this.getSources(sourceCorpusId);
|
||||
const sourceIndexMap = new Map<string, number>();
|
||||
sources.forEach((src, i) => {
|
||||
sourceIndexMap.set(src.id, i + 1);
|
||||
});
|
||||
|
||||
const groupMap = new Map<string, SourceGroup>();
|
||||
for (const chunk of chunks) {
|
||||
const docIndex = sourceIndexMap.get(chunk.sourceId) || 0;
|
||||
if (!groupMap.has(chunk.sourceId)) {
|
||||
const src = sources.find((s) => s.id === chunk.sourceId);
|
||||
groupMap.set(chunk.sourceId, {
|
||||
docIndex,
|
||||
sourceId: chunk.sourceId,
|
||||
name: src?.name || 'Unknown',
|
||||
chunks: [],
|
||||
});
|
||||
}
|
||||
groupMap.get(chunk.sourceId)!.chunks.push(chunk);
|
||||
}
|
||||
|
||||
return Array.from(groupMap.values()).sort((a, b) => a.docIndex - b.docIndex);
|
||||
}
|
||||
}
|
||||
|
||||
const defaultStore = new SourceCorpusStore();
|
||||
|
||||
export const getAllSourceCorpora = defaultStore.getAllSourceCorpora.bind(defaultStore);
|
||||
export const getSourceCorpus = defaultStore.getSourceCorpus.bind(defaultStore);
|
||||
export const createSourceCorpus = defaultStore.createSourceCorpus.bind(defaultStore);
|
||||
export const deleteSourceCorpus = defaultStore.deleteSourceCorpus.bind(defaultStore);
|
||||
export const addSource = defaultStore.addSource.bind(defaultStore);
|
||||
export const getSources = defaultStore.getSources.bind(defaultStore);
|
||||
export const addChunksToSourceCorpus = defaultStore.addChunksToSourceCorpus.bind(defaultStore);
|
||||
export const getChunksForSourceCorpus = defaultStore.getChunksForSourceCorpus.bind(defaultStore);
|
||||
export const buildSourceGroups = defaultStore.buildSourceGroups.bind(defaultStore);
|
||||
Reference in New Issue
Block a user