10 Commits

44 changed files with 873 additions and 831 deletions

View File

@@ -20,7 +20,7 @@ fmt-check:
check: test lint fmt-check
docker:
docker build -t notebook-clone .
docker build -t source-sentinel .
hooks:
printf '#!/usr/bin/env bash\nset -euo pipefail\nmake check\n' > .git/hooks/pre-commit
@@ -28,12 +28,12 @@ hooks:
dev:
@echo "Starting backend and frontend..."
cd $(SERVER_DIR) && node src/index.js &
cd $(SERVER_DIR) && npx --no-install tsx src/index.ts &
cd $(WEB_DIR) && npx --no-install vite --port 5173 &
wait
run:
cd $(SERVER_DIR) && node src/index.js
cd $(SERVER_DIR) && npx --no-install tsx src/index.ts
build:
cd $(WEB_DIR) && npx --no-install vite build

View File

@@ -1,99 +1,141 @@
# Citation Sentinel
# Citation Sentinel © 2023 @sjDev
A source-grounded research assistant that employs cosine similarity scoring for LLM-generated query responses to provide a "groundedness" score. This is a metric in Retrieval-Augmented Generation (RAG) systems that quantifies how well an AI-generated answer is supported by retrieved context. It measures "faithfulness" to source documents, ensuring the answer is not hallucinated or pulled from the model's pre-training data.
Users upload source documents, or provide links to online sources including audio/video (i.e. links to youtube videos). Users may then ask questions and receive answers (with inline citations) verifiably grounded in the provided information sources.
![Citation Sentinel UI showing the source sidebar, chat answer with citations, and groundedness badge](demo.jpg)
Image one: source-grounded research assistant that empowers users to transform an otherwise-unmanageably-large corpus of data on any topic of interest into refined, easily-digested subtopics and pose focused inquiries, to instantly receive concise, quantifiably-evaluated, accurate responses. The system also provides suggested follow-up questions further refining in user inquiries by detecting the query goals.
Image Two: demo co clickable inline citations that take user to source-grounded research basis/knowledge corpus supporting query response, with rated “groundedness” score.
# Reliability and Understandability
Citation Sentinel's Retrieval-Augmented Generation (RAG) pipeline uses two-stage pass-through to Voyage AI embedding and ranking models, which uses cosine similarity methodology to scoring to evaluate query-source embeddings in multi-dimensional vector space. This yields a "groundedness" score.
Groundedness metrics quantify how well an AI-generated answer is supported by retrieved context. It measures "faithfulness" to source documents, ensuring the answer is not hallucinated, inaccurate or stale (pulled from the model's training data).
# Architecture - Basic Overview
Built with a React/Vote frontend and a Typescript/Node/Express backend, using Anthropic Claude for generation, OpenAI Whisper for video audio track transcription, Voyage AI voyage-3 for embeddings and Voyage AI rerank-2 for response cosine similarity scoring, to generate the "groundedness" score, presented as an easily-understandable red/yellow/green "badge" style tooltip in the response (see above.)
Built with a React/Vite frontend and a Typescript/Node/Express backend, using Anthropic Claude for generation, OpenAI Whisper for video audio track transcription, Voyage AI for embeddings and response cosine similarity scoring (the "groundedness" score).
## Query pipeline
Source Ingestion -> Parsing -> Chunking -> Embedding (using Voyage AI voyage-3 model) -> Storage (vector store) -> { user query submission } -> Evaluation of User Query -> Retrieval -> Ranking -> Response Generation (Using Anthopic's claude-opus-4-6 model) -> Response Groundedness Scoring (using Voyage AI rerank-r model)
## Prerequisites
- **Node.js** (v18+)
- **yt-dlp** -- Required for YouTube video source support (`brew install yt-dlp`)
## Getting Started
```bash
# clone and install
git clone <repo-url> && cd citation_sentinel
cd server && npm install && cd ..
cd client && npm install && cd ..
# configure
cp server/.env.example server/.env
# edit server/.env and add your ANTHROPIC_API_KEY, VOYAGE_API_KEY,
# and optionally OPENAI_API_KEY (only needed for audio source transcription via Whisper)
# run
make dev
```
## Rationale
This app demonstrates the core source-grounded Q&A pattern with transparent retrieval, generation, and groundedness scoring (LLM response quality cosine scoring) -- all with swappable models and is fully open source.
## Methodology
This is a RAG (Retrieval-Augmented Generation) and query-response source-groundedness assurance system allowing users to create a research knowledge corpus, including:
1. Documents — PDFs, DOCX files, plain text audio files.
2. Internet sources: via web URLs.
2. Audio sources: i.e. YouTube videos, audio from which is transcribed to text.
These are then parsed, split into ~2000-character overlapping chunks, and converted into vector embeddings using Voyage AI's voyage-3 model.
The embeddings are stored in an in-memory vector store.
When a user submits a query, the system enforces groundedness through a multi-layered strategy:
1. **Retrieval constraint** — The query is embedded (via Voyage AI voyage-3) and compared against
stored chunk vectors using cosine similarity, returning the top 20 candidates.
2. **Reranking for precision** — Those 20 candidates are sent to Voyage AI's rerank-2 cross-encoder,
which re-scores each query-chunk pair with deeper semantic analysis. Only the top 5 survive.
1. **Retrieval constraint** — The query is embedded (via Voyage AI voyage-3) and compared against stored chunk vectors using cosine similarity, returning the top 20 candidates.
3. **Prompt-level constraint** — The top 5 chunks are passed to the "Primary LLM" (Claude opus-4-6)
for natural language query response.
The Primary LLM must cite sources using bracketed indices (e.g., [1], [2]) and admit when sources are
insufficient.
4. **Schema enforcement** — The Primary LLM's response is constrained to a JSON schema requiring structured
fields (answer, citedSourceIndices, followUpQuestions). Any cited source indices that do not
correspond to real source groups are programmatically removed.
2. **Reranking for precision** — Those 20 candidates are sent to Voyage AI's rerank-2 cross-encoder, which re-scores each query-chunk pair with deeper semantic analysis. Only the top 5 survive.
3. **Prompt-level constraint** — The top 5 chunks are passed to the "Primary LLM" (Claude opus-4-6) for natural language query response. The Primary LLM must cite sources using bracketed indices (e.g., [1], [2]) and admit when sources are insufficient.
4. **Schema enforcement** — The Primary LLM's response is constrained to a JSON schema requiring structured fields (answer, citedSourceIndices, followUpQuestions). Any cited source indices that do not correspond to real source groups are programmatically removed.
5. **Post-generation groundedness scoring** — The answer is split into individual sentences. Voyage AI voyage-3 embeds each sentence, and compares it (using cosine similarity) against the vectors of the cited chunks. The similarity is calibrated to a 0–1 scale and averaged, producing a groundedness score surfaced to the user with a visual indicator (“badge” in green/yellow/red).
5. **Post-generation groundedness scoring** — The answer is split into individual sentences. Voyage
AI voyage-3 embeds each sentence, and compares it (using cosine similarity) against the vectors
of the cited chunks. The similarity is calibrated to a 0–1 scale and averaged, producing a single
groundedness score that is surfaced to the user with a visual indicator (green/yellow/red).
## Similarity Metrics for Semantic Understanding
Cosine similarity measures how closely two vectors (representing data like words, images, or preferences) are aligned in a multi-dimensional space by calculating the cosine of the angle between them.
Archetypical cosine scores range from -1 to 1.
1: Vectors point in the exact same direction (highly similar).
0: Vectors are at a 90-degree angle (orthogonal/unrelated).
-1: Vectors point in opposite directions.
Cosine distance between high-dimensional text embeddings is compressed: unrelated notes sit close to orthogonal, so both the within-cluster and nearest-cluster distances land near 0.8. In this application, scores are normalized to a bounded range wherein unrelated query-response pair scores are nearly orthogonal.
## Design
Two-package monorepo:
- `server/` -- single Express backend with layered architecture (routes -> services -> stores). Routes orchestrate; services contain business logic; stores manage in-memory state.
- `client/` -- React 19 SPA via Vite. Two-panel layout: sidebar for data sources, main area for LLM chat with explorable citations, groundedness badges (cosine similarity scoring of LLM responses), and follow-up question chips.
## License
MIT. See [LICENSE](./LICENSE).
## Author
@sjdev

View File

@@ -4,7 +4,7 @@
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<link rel="icon" href="data:image/svg+xml,<svg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 100 100'><circle cx='50' cy='50' r='50' fill='black'/></svg>">
<title>NotebookLM Clone</title>
<title>Source Sentinel</title>
</head>
<body>
<div id="root"></div>

View File

@@ -1,11 +1,11 @@
{
"name": "notebook-clone-web",
"name": "source-sentinel-web",
"version": "0.1.0",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "notebook-clone-web",
"name": "source-sentinel-web",
"version": "0.1.0",
"dependencies": {
"react": "^19.0.0",

View File

@@ -1,5 +1,5 @@
{
"name": "notebook-clone-web",
"name": "source-sentinel-web",
"version": "0.1.0",
"private": true,
"engines": { "node": ">=18" },

View File

@@ -1,20 +1,20 @@
import { useState } from 'react';
import NotebookList from './components/NotebookList.jsx';
import SourceCorpusList from './components/SourceCorpusList.jsx';
import SourcePanel from './components/SourcePanel.jsx';
import DocumentButtons from './components/DocumentButtons.jsx';
import ChatPanel from './components/ChatPanel.jsx';
import DocumentModal from './components/DocumentModal.jsx';
import { useNotebook } from './hooks/useNotebook.js';
import { useSourceCorpus } from './hooks/useSourceCorpus.js';
import { generateDocument } from './api/documents.js';
function App() {
const {
notebooks,
activeNotebook,
selectNotebook,
createNotebook,
deleteNotebook,
} = useNotebook();
sourceCorpora,
activeSourceCorpus,
selectSourceCorpus,
createSourceCorpus,
deleteSourceCorpus,
} = useSourceCorpus();
const [hoverState, setHoverState] = useState(null);
const [sourceCount, setSourceCount] = useState(0);
@@ -22,9 +22,9 @@ function App() {
const [docModal, setDocModal] = useState({ open: false, type: null, document: null, loading: false });
const [chatReady, setChatReady] = useState(false);
const handleSelectNotebook = (id) => {
const handleSelectSourceCorpus = (id) => {
setChatReady(false);
selectNotebook(id);
selectSourceCorpus(id);
};
const handleSourceHover = (val) => {
@@ -42,7 +42,7 @@ function App() {
setDocModal({ open: true, type, document: null, loading: true });
try {
const res = await generateDocument(activeNotebook.id, type);
const res = await generateDocument(activeSourceCorpus.id, type);
setDocModal({ open: true, type, document: res.document, loading: false });
} catch (err) {
console.error('document generation failed', err);
@@ -58,17 +58,17 @@ function App() {
return (
<div className="app">
<aside className="sidebar">
<NotebookList
notebooks={notebooks}
activeId={activeNotebook?.id}
onSelect={handleSelectNotebook}
onCreate={createNotebook}
onDelete={deleteNotebook}
<SourceCorpusList
sourceCorpora={sourceCorpora}
activeId={activeSourceCorpus?.id}
onSelect={handleSelectSourceCorpus}
onCreate={createSourceCorpus}
onDelete={deleteSourceCorpus}
/>
{activeNotebook && (
{activeSourceCorpus && (
<>
<SourcePanel
notebookId={activeNotebook.id}
sourceCorpusId={activeSourceCorpus.id}
hoveredSourceIndex={hoveredDocIndex}
onSourceHover={handleSourceHover}
onSourcesChange={setSourceCount}
@@ -85,16 +85,16 @@ function App() {
)}
</aside>
<main className="main">
{activeNotebook ? (
{activeSourceCorpus ? (
<ChatPanel
notebookId={activeNotebook.id}
sourceCorpusId={activeSourceCorpus.id}
hoveredSource={hoveredInstanceId}
hoveredDocIndex={hoveredDocIndex}
onSourceHover={handleSourceHover}
onFirstResponse={() => setChatReady(true)}
/>
) : (
<p>Select or create a notebook to get started.</p>
<p>Select or create a source corpus to get started.</p>
)}
</main>
<DocumentModal

View File

@@ -4,11 +4,11 @@ import { handleResponse } from './client.js';
const BASE = '/api/documents';
export async function generateDocument(notebookId, type) {
export async function generateDocument(sourceCorpusId, type) {
const res = await fetch(`${BASE}/generate`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ notebookId, type }),
body: JSON.stringify({ sourceCorpusId, type }),
});
return handleResponse(res);
}

View File

@@ -5,11 +5,11 @@ import { handleResponse } from './client.js';
const BASE = '/api/query';
const CITATION_DETAIL_BASE = '/api/citation-detail';
export async function sendQuery(notebookId, question, onCitationDetails) {
export async function sendQuery(sourceCorpusId, question, onCitationDetails) {
const res = await fetch(BASE, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ notebookId, question }),
body: JSON.stringify({ sourceCorpusId, question }),
});
const data = await handleResponse(res);

View File

@@ -16,7 +16,7 @@ describe('sendQuery', () => {
vi.restoreAllMocks();
});
it('sends POST to /api/query with notebookId and question', async () => {
it('sends POST to /api/query with sourceCorpusId and question', async () => {
const data = { answer: 'The answer', citations: [] };
fetch.mockResolvedValue(okResponse(data));
@@ -25,7 +25,7 @@ describe('sendQuery', () => {
expect(fetch).toHaveBeenCalledWith('/api/query', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ notebookId: 'nb-1', question: 'What is AI?' }),
body: JSON.stringify({ sourceCorpusId: 'nb-1', question: 'What is AI?' }),
});
expect(result).toEqual(data);
});

View File

@@ -2,14 +2,14 @@
import { handleResponse } from './client.js';
const BASE = '/api/notebooks';
const BASE = '/api/source-corpus';
export async function listNotebooks() {
export async function listSourceCorpora() {
const res = await fetch(BASE);
return handleResponse(res);
}
export async function createNotebook(name) {
export async function createSourceCorpus(name) {
const res = await fetch(BASE, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
@@ -18,7 +18,7 @@ export async function createNotebook(name) {
return handleResponse(res);
}
export async function deleteNotebook(id) {
export async function deleteSourceCorpus(id) {
const res = await fetch(`${BASE}/${id}`, { method: 'DELETE' });
return handleResponse(res);
}

View File

@@ -1,5 +1,5 @@
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
import { listNotebooks, createNotebook, deleteNotebook } from './notebooks.js';
import { listSourceCorpora, createSourceCorpus, deleteSourceCorpus } from './sourceCorpus.js';
const okResponse = (body) => ({
ok: true,
@@ -7,7 +7,7 @@ const okResponse = (body) => ({
json: () => Promise.resolve(body),
});
describe('notebooks API', () => {
describe('sourceCorpora API', () => {
beforeEach(() => {
vi.stubGlobal('fetch', vi.fn());
});
@@ -16,41 +16,41 @@ describe('notebooks API', () => {
vi.restoreAllMocks();
});
describe('listNotebooks', () => {
it('fetches GET /api/notebooks', async () => {
describe('listSourceCorpora', () => {
it('fetches GET /api/source-corpus', async () => {
const data = [{ id: '1', name: 'NB' }];
fetch.mockResolvedValue(okResponse(data));
const result = await listNotebooks();
const result = await listSourceCorpora();
expect(fetch).toHaveBeenCalledWith('/api/notebooks');
expect(fetch).toHaveBeenCalledWith('/api/source-corpus');
expect(result).toEqual(data);
});
});
describe('createNotebook', () => {
describe('createSourceCorpus', () => {
it('sends POST with name in JSON body', async () => {
const nb = { id: '2', name: 'New' };
fetch.mockResolvedValue(okResponse(nb));
const corpus = { id: '2', name: 'New' };
fetch.mockResolvedValue(okResponse(corpus));
const result = await createNotebook('New');
const result = await createSourceCorpus('New');
expect(fetch).toHaveBeenCalledWith('/api/notebooks', {
expect(fetch).toHaveBeenCalledWith('/api/source-corpus', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ name: 'New' }),
});
expect(result).toEqual(nb);
expect(result).toEqual(corpus);
});
});
describe('deleteNotebook', () => {
it('sends DELETE to /api/notebooks/:id', async () => {
describe('deleteSourceCorpus', () => {
it('sends DELETE to /api/source-corpus/:id', async () => {
fetch.mockResolvedValue(okResponse({}));
await deleteNotebook('abc-123');
await deleteSourceCorpus('abc-123');
expect(fetch).toHaveBeenCalledWith('/api/notebooks/abc-123', {
expect(fetch).toHaveBeenCalledWith('/api/source-corpus/abc-123', {
method: 'DELETE',
});
});
@@ -63,6 +63,6 @@ describe('notebooks API', () => {
json: () => Promise.resolve({ error: 'Server down' }),
});
await expect(listNotebooks()).rejects.toThrow('Server down');
await expect(listSourceCorpora()).rejects.toThrow('Server down');
});
});

View File

@@ -4,15 +4,15 @@ import { handleResponse } from './client.js';
const BASE = '/api/sources';
export async function listSources(notebookId) {
const res = await fetch(`${BASE}?notebookId=${notebookId}`);
export async function listSources(sourceCorpusId) {
const res = await fetch(`${BASE}?sourceCorpusId=${sourceCorpusId}`);
return handleResponse(res);
}
export async function uploadSource(notebookId, file) {
export async function uploadSource(sourceCorpusId, file) {
const formData = new FormData();
formData.append('file', file);
formData.append('notebookId', notebookId);
formData.append('sourceCorpusId', sourceCorpusId);
const res = await fetch(BASE, {
method: 'POST',
body: formData,
@@ -20,11 +20,11 @@ export async function uploadSource(notebookId, file) {
return handleResponse(res);
}
export async function addUrlSource(notebookId, url) {
export async function addUrlSource(sourceCorpusId, url) {
const res = await fetch(`${BASE}/url`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ notebookId, url }),
body: JSON.stringify({ sourceCorpusId, url }),
});
return handleResponse(res);
}

View File

@@ -17,19 +17,19 @@ describe('sources API', () => {
});
describe('listSources', () => {
it('fetches GET /api/sources with notebookId query param', async () => {
it('fetches GET /api/sources with sourceCorpusId query param', async () => {
const data = [{ id: 's1', name: 'file.pdf' }];
fetch.mockResolvedValue(okResponse(data));
const result = await listSources('nb-42');
expect(fetch).toHaveBeenCalledWith('/api/sources?notebookId=nb-42');
expect(fetch).toHaveBeenCalledWith('/api/sources?sourceCorpusId=nb-42');
expect(result).toEqual(data);
});
});
describe('uploadSource', () => {
it('sends POST with FormData containing file and notebookId', async () => {
it('sends POST with FormData containing file and sourceCorpusId', async () => {
const source = { id: 's2', name: 'doc.pdf' };
fetch.mockResolvedValue(okResponse(source));
@@ -42,7 +42,7 @@ describe('sources API', () => {
});
const formData = fetch.mock.calls[0][1].body;
expect(formData.get('notebookId')).toBe('nb-1');
expect(formData.get('sourceCorpusId')).toBe('nb-1');
expect(formData.get('file')).toBeInstanceOf(File);
expect(result).toEqual(source);
});
@@ -58,7 +58,7 @@ describe('sources API', () => {
expect(fetch).toHaveBeenCalledWith('/api/sources/url', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ notebookId: 'nb-5', url: 'https://example.com' }),
body: JSON.stringify({ sourceCorpusId: 'nb-5', url: 'https://example.com' }),
});
expect(result).toEqual(source);
});

View File

@@ -3,7 +3,7 @@ import { sendQuery } from '../api/query.js';
import ChatMessage from './ChatMessage.jsx';
import CitationDetailModal from './CitationDetailModal.jsx';
function ChatPanel({ notebookId, hoveredSource, hoveredDocIndex, onSourceHover, onFirstResponse }) {
function ChatPanel({ sourceCorpusId, hoveredSource, hoveredDocIndex, onSourceHover, onFirstResponse }) {
const [messages, setMessages] = useState([]);
const [input, setInput] = useState('');
const [loading, setLoading] = useState(false);
@@ -18,7 +18,7 @@ function ChatPanel({ notebookId, hoveredSource, hoveredDocIndex, onSourceHover,
setCitationDetails({});
setActiveCitation(null);
firstResponseFired.current = false;
}, [notebookId]);
}, [sourceCorpusId]);
useEffect(() => {
bottomRef.current?.scrollIntoView({ behavior: 'smooth' });
@@ -35,7 +35,7 @@ function ChatPanel({ notebookId, hoveredSource, hoveredDocIndex, onSourceHover,
const msgId = crypto.randomUUID();
try {
const res = await sendQuery(notebookId, question, (details) => {
const res = await sendQuery(sourceCorpusId, question, (details) => {
const detailMap = {};
for (const d of details) {
detailMap[d.sourceIndex] = d;

View File

@@ -1,99 +0,0 @@
import { describe, it, expect, vi } from 'vitest';
import { render, screen, waitFor } from '@testing-library/react';
import userEvent from '@testing-library/user-event';
import CreateNotebookModal from './CreateNotebookModal.jsx';
describe('CreateNotebookModal', () => {
it('renders nothing when open is false', () => {
const { container } = render(
<CreateNotebookModal open={false} onConfirm={() => {}} onCancel={() => {}} />,
);
expect(container.innerHTML).toBe('');
});
it('renders the modal with title when open', () => {
render(<CreateNotebookModal open={true} onConfirm={() => {}} onCancel={() => {}} />);
expect(screen.getByText('New Notebook')).toBeInTheDocument();
});
it('renders input and buttons', () => {
render(<CreateNotebookModal open={true} onConfirm={() => {}} onCancel={() => {}} />);
expect(screen.getByPlaceholderText('Enter notebook name')).toBeInTheDocument();
expect(screen.getByText('Cancel')).toBeInTheDocument();
expect(screen.getByText('Create')).toBeInTheDocument();
});
it('disables Create button when input is empty', () => {
render(<CreateNotebookModal open={true} onConfirm={() => {}} onCancel={() => {}} />);
expect(screen.getByText('Create')).toBeDisabled();
});
it('enables Create button when input has text', async () => {
const user = userEvent.setup();
render(<CreateNotebookModal open={true} onConfirm={() => {}} onCancel={() => {}} />);
await user.type(screen.getByPlaceholderText('Enter notebook name'), 'My Notebook');
expect(screen.getByText('Create')).toBeEnabled();
});
it('calls onConfirm with trimmed name on submit', async () => {
const user = userEvent.setup();
const onConfirm = vi.fn();
render(<CreateNotebookModal open={true} onConfirm={onConfirm} onCancel={() => {}} />);
await user.type(screen.getByPlaceholderText('Enter notebook name'), ' Research Notes ');
await user.click(screen.getByText('Create'));
expect(onConfirm).toHaveBeenCalledWith('Research Notes');
});
it('calls onConfirm on Enter key', async () => {
const user = userEvent.setup();
const onConfirm = vi.fn();
render(<CreateNotebookModal open={true} onConfirm={onConfirm} onCancel={() => {}} />);
await user.type(screen.getByPlaceholderText('Enter notebook name'), 'Test{Enter}');
expect(onConfirm).toHaveBeenCalledWith('Test');
});
it('does not call onConfirm when input is only whitespace', async () => {
const user = userEvent.setup();
const onConfirm = vi.fn();
render(<CreateNotebookModal open={true} onConfirm={onConfirm} onCancel={() => {}} />);
const input = screen.getByPlaceholderText('Enter notebook name');
await user.type(input, ' ');
await user.type(input, '{Enter}');
expect(onConfirm).not.toHaveBeenCalled();
});
it('calls onCancel when Cancel button is clicked', async () => {
const user = userEvent.setup();
const onCancel = vi.fn();
render(<CreateNotebookModal open={true} onConfirm={() => {}} onCancel={onCancel} />);
await user.click(screen.getByText('Cancel'));
expect(onCancel).toHaveBeenCalledOnce();
});
it('resets input when reopened', async () => {
const user = userEvent.setup();
const { rerender } = render(
<CreateNotebookModal open={true} onConfirm={() => {}} onCancel={() => {}} />,
);
await user.type(screen.getByPlaceholderText('Enter notebook name'), 'Old name');
rerender(<CreateNotebookModal open={false} onConfirm={() => {}} onCancel={() => {}} />);
rerender(<CreateNotebookModal open={true} onConfirm={() => {}} onCancel={() => {}} />);
expect(screen.getByPlaceholderText('Enter notebook name')).toHaveValue('');
});
it('auto-focuses the input when opened', async () => {
render(<CreateNotebookModal open={true} onConfirm={() => {}} onCancel={() => {}} />);
await waitFor(() => {
expect(screen.getByPlaceholderText('Enter notebook name')).toHaveFocus();
}, { timeout: 200 });
});
});

View File

@@ -1,7 +1,7 @@
import { useState, useEffect, useRef } from 'react';
import Modal from './Modal.jsx';
function CreateNotebookModal({ open, onConfirm, onCancel }) {
function CreateSourceCorpusModal({ open, onConfirm, onCancel }) {
const [name, setName] = useState('');
const inputRef = useRef(null);
@@ -19,13 +19,13 @@ function CreateNotebookModal({ open, onConfirm, onCancel }) {
};
return (
<Modal open={open} onClose={onCancel} title="New Notebook">
<Modal open={open} onClose={onCancel} title="New Source Corpus">
<form onSubmit={handleSubmit}>
<input
ref={inputRef}
className="modal-input"
type="text"
placeholder="Enter notebook name"
placeholder="Enter source corpus name"
value={name}
onChange={(e) => setName(e.target.value)}
/>
@@ -50,4 +50,4 @@ function CreateNotebookModal({ open, onConfirm, onCancel }) {
);
}
export default CreateNotebookModal;
export default CreateSourceCorpusModal;

View File

@@ -0,0 +1,99 @@
import { describe, it, expect, vi } from 'vitest';
import { render, screen, waitFor } from '@testing-library/react';
import userEvent from '@testing-library/user-event';
import CreateSourceCorpusModal from './CreateSourceCorpusModal.jsx';
describe('CreateSourceCorpusModal', () => {
it('renders nothing when open is false', () => {
const { container } = render(
<CreateSourceCorpusModal open={false} onConfirm={() => {}} onCancel={() => {}} />,
);
expect(container.innerHTML).toBe('');
});
it('renders the modal with title when open', () => {
render(<CreateSourceCorpusModal open={true} onConfirm={() => {}} onCancel={() => {}} />);
expect(screen.getByText('New Source Corpus')).toBeInTheDocument();
});
it('renders input and buttons', () => {
render(<CreateSourceCorpusModal open={true} onConfirm={() => {}} onCancel={() => {}} />);
expect(screen.getByPlaceholderText('Enter source corpus name')).toBeInTheDocument();
expect(screen.getByText('Cancel')).toBeInTheDocument();
expect(screen.getByText('Create')).toBeInTheDocument();
});
it('disables Create button when input is empty', () => {
render(<CreateSourceCorpusModal open={true} onConfirm={() => {}} onCancel={() => {}} />);
expect(screen.getByText('Create')).toBeDisabled();
});
it('enables Create button when input has text', async () => {
const user = userEvent.setup();
render(<CreateSourceCorpusModal open={true} onConfirm={() => {}} onCancel={() => {}} />);
await user.type(screen.getByPlaceholderText('Enter source corpus name'), 'My SourceCorpus');
expect(screen.getByText('Create')).toBeEnabled();
});
it('calls onConfirm with trimmed name on submit', async () => {
const user = userEvent.setup();
const onConfirm = vi.fn();
render(<CreateSourceCorpusModal open={true} onConfirm={onConfirm} onCancel={() => {}} />);
await user.type(screen.getByPlaceholderText('Enter source corpus name'), ' Research Notes ');
await user.click(screen.getByText('Create'));
expect(onConfirm).toHaveBeenCalledWith('Research Notes');
});
it('calls onConfirm on Enter key', async () => {
const user = userEvent.setup();
const onConfirm = vi.fn();
render(<CreateSourceCorpusModal open={true} onConfirm={onConfirm} onCancel={() => {}} />);
await user.type(screen.getByPlaceholderText('Enter source corpus name'), 'Test{Enter}');
expect(onConfirm).toHaveBeenCalledWith('Test');
});
it('does not call onConfirm when input is only whitespace', async () => {
const user = userEvent.setup();
const onConfirm = vi.fn();
render(<CreateSourceCorpusModal open={true} onConfirm={onConfirm} onCancel={() => {}} />);
const input = screen.getByPlaceholderText('Enter source corpus name');
await user.type(input, ' ');
await user.type(input, '{Enter}');
expect(onConfirm).not.toHaveBeenCalled();
});
it('calls onCancel when Cancel button is clicked', async () => {
const user = userEvent.setup();
const onCancel = vi.fn();
render(<CreateSourceCorpusModal open={true} onConfirm={() => {}} onCancel={onCancel} />);
await user.click(screen.getByText('Cancel'));
expect(onCancel).toHaveBeenCalledOnce();
});
it('resets input when reopened', async () => {
const user = userEvent.setup();
const { rerender } = render(
<CreateSourceCorpusModal open={true} onConfirm={() => {}} onCancel={() => {}} />,
);
await user.type(screen.getByPlaceholderText('Enter source corpus name'), 'Old name');
rerender(<CreateSourceCorpusModal open={false} onConfirm={() => {}} onCancel={() => {}} />);
rerender(<CreateSourceCorpusModal open={true} onConfirm={() => {}} onCancel={() => {}} />);
expect(screen.getByPlaceholderText('Enter source corpus name')).toHaveValue('');
});
it('auto-focuses the input when opened', async () => {
render(<CreateSourceCorpusModal open={true} onConfirm={() => {}} onCancel={() => {}} />);
await waitFor(() => {
expect(screen.getByPlaceholderText('Enter source corpus name')).toHaveFocus();
}, { timeout: 200 });
});
});

View File

@@ -1,53 +0,0 @@
import { useState } from 'react';
import CreateNotebookModal from './CreateNotebookModal.jsx';
function NotebookList({ notebooks, activeId, onSelect, onCreate, onDelete }) {
const [modalOpen, setModalOpen] = useState(false);
const handleDelete = (e, id) => {
e.stopPropagation();
onDelete(id);
};
const handleConfirm = (name) => {
setModalOpen(false);
onCreate(name);
};
return (
<div className="notebook-list">
<h2>Notebooks</h2>
<ul>
{notebooks.map((nb) => (
<li
key={nb.id}
className={nb.id === activeId ? 'active' : ''}
onClick={() => onSelect(nb.id)}
>
<span className="nb-name">{nb.name}</span>
<button
className="nb-delete"
onClick={(e) => handleDelete(e, nb.id)}
title="Delete notebook"
>
&times;
</button>
</li>
))}
</ul>
<button
className="btn-new-notebook"
onClick={() => setModalOpen(true)}
>
+ New Notebook
</button>
<CreateNotebookModal
open={modalOpen}
onConfirm={handleConfirm}
onCancel={() => setModalOpen(false)}
/>
</div>
);
}
export default NotebookList;

View File

@@ -0,0 +1,53 @@
import { useState } from 'react';
import CreateSourceCorpusModal from './CreateSourceCorpusModal.jsx';
function SourceCorpusList({ sourceCorpora, activeId, onSelect, onCreate, onDelete }) {
const [modalOpen, setModalOpen] = useState(false);
const handleDelete = (e, id) => {
e.stopPropagation();
onDelete(id);
};
const handleConfirm = (name) => {
setModalOpen(false);
onCreate(name);
};
return (
<div className="source-corpus-list">
<h2>Source Corpora</h2>
<ul>
{sourceCorpora.map((corpus) => (
<li
key={corpus.id}
className={corpus.id === activeId ? 'active' : ''}
onClick={() => onSelect(corpus.id)}
>
<span className="sc-name">{corpus.name}</span>
<button
className="sc-delete"
onClick={(e) => handleDelete(e, corpus.id)}
title="Delete source corpus"
>
&times;
</button>
</li>
))}
</ul>
<button
className="btn-new-corpus"
onClick={() => setModalOpen(true)}
>
+ New Source Corpus
</button>
<CreateSourceCorpusModal
open={modalOpen}
onConfirm={handleConfirm}
onCancel={() => setModalOpen(false)}
/>
</div>
);
}
export default SourceCorpusList;

View File

@@ -10,7 +10,7 @@ function isFileAllowed(file) {
return ALLOWED_EXTENSIONS.includes(ext);
}
function SourcePanel({ notebookId, hoveredSourceIndex, onSourceHover, onSourcesChange, children }) {
function SourcePanel({ sourceCorpusId, hoveredSourceIndex, onSourceHover, onSourcesChange, children }) {
const [sources, setSources] = useState([]);
const [uploading, setUploading] = useState(false);
const [isDragging, setIsDragging] = useState(false);
@@ -30,11 +30,11 @@ function SourcePanel({ notebookId, hoveredSourceIndex, onSourceHover, onSourcesC
useEffect(() => {
setSources([]);
onSourcesChange?.(0);
sourcesApi.listSources(notebookId).then((s) => {
sourcesApi.listSources(sourceCorpusId).then((s) => {
setSources(s);
onSourcesChange?.(s.length);
}).catch((err) => showError(err.message || 'Failed to load sources'));
}, [notebookId, onSourcesChange]);
}, [sourceCorpusId, onSourcesChange]);
useEffect(() => () => clearTimeout(errorTimer.current), []);
@@ -47,7 +47,7 @@ function SourcePanel({ notebookId, hoveredSourceIndex, onSourceHover, onSourcesC
setError(null);
setUploading(true);
try {
const src = await sourcesApi.uploadSource(notebookId, file);
const src = await sourcesApi.uploadSource(sourceCorpusId, file);
setSources((prev) => {
const next = [...prev, src];
onSourcesChange?.(next.length);
@@ -72,7 +72,7 @@ function SourcePanel({ notebookId, hoveredSourceIndex, onSourceHover, onSourcesC
setError(null);
setAddingUrl(true);
try {
const src = await sourcesApi.addUrlSource(notebookId, url);
const src = await sourcesApi.addUrlSource(sourceCorpusId, url);
if (src.error) {
showError(src.error);
} else {

View File

@@ -1,50 +0,0 @@
import { useState, useEffect, useCallback } from 'react';
import * as notebooksApi from '../api/notebooks.js';
export function useNotebook() {
const [notebooks, setNotebooks] = useState([]);
const [activeNotebook, setActiveNotebook] = useState(null);
useEffect(() => {
notebooksApi.listNotebooks().then(setNotebooks).catch(console.error);
}, []);
const selectNotebook = useCallback(
(id) => {
const nb = notebooks.find((n) => n.id === id) || null;
setActiveNotebook(nb);
},
[notebooks],
);
const createNotebook = useCallback(async (name) => {
if (!name) return;
const nb = await notebooksApi.createNotebook(name);
setNotebooks((prev) => [...prev, nb]);
setActiveNotebook(nb);
}, []);
const deleteNotebook = useCallback(
async (id) => {
try {
await notebooksApi.deleteNotebook(id);
} catch (err) {
console.error('Failed to delete notebook', err);
return;
}
setNotebooks((prev) => prev.filter((n) => n.id !== id));
if (activeNotebook?.id === id) {
setActiveNotebook(null);
}
},
[activeNotebook],
);
return {
notebooks,
activeNotebook,
selectNotebook,
createNotebook,
deleteNotebook,
};
}

View File

@@ -1,176 +0,0 @@
import { describe, it, expect, vi, beforeEach } from 'vitest';
import { renderHook, act, waitFor } from '@testing-library/react';
vi.mock('../api/notebooks.js', () => ({
listNotebooks: vi.fn(),
createNotebook: vi.fn(),
deleteNotebook: vi.fn(),
}));
import { useNotebook } from './useNotebook.js';
import * as notebooksApi from '../api/notebooks.js';
const notebooks = [
{ id: 'nb-1', name: 'Research' },
{ id: 'nb-2', name: 'Personal' },
];
beforeEach(() => {
vi.resetAllMocks();
notebooksApi.listNotebooks.mockResolvedValue(notebooks);
notebooksApi.createNotebook.mockImplementation(async (name) => ({
id: `nb-${Date.now()}`,
name,
}));
notebooksApi.deleteNotebook.mockResolvedValue({});
});
describe('useNotebook', () => {
it('loads notebooks on mount', async () => {
const { result } = renderHook(() => useNotebook());
await waitFor(() => {
expect(result.current.notebooks).toEqual(notebooks);
});
expect(notebooksApi.listNotebooks).toHaveBeenCalledOnce();
});
it('starts with no active notebook', () => {
const { result } = renderHook(() => useNotebook());
expect(result.current.activeNotebook).toBeNull();
});
it('selectNotebook sets the active notebook', async () => {
const { result } = renderHook(() => useNotebook());
await waitFor(() => {
expect(result.current.notebooks).toHaveLength(2);
});
act(() => {
result.current.selectNotebook('nb-2');
});
expect(result.current.activeNotebook).toEqual({ id: 'nb-2', name: 'Personal' });
});
it('selectNotebook sets null for unknown id', async () => {
const { result } = renderHook(() => useNotebook());
await waitFor(() => {
expect(result.current.notebooks).toHaveLength(2);
});
act(() => {
result.current.selectNotebook('nb-2');
});
expect(result.current.activeNotebook).not.toBeNull();
act(() => {
result.current.selectNotebook('nonexistent');
});
expect(result.current.activeNotebook).toBeNull();
});
it('createNotebook calls API and adds to list', async () => {
const newNb = { id: 'nb-new', name: 'New One' };
notebooksApi.createNotebook.mockResolvedValue(newNb);
const { result } = renderHook(() => useNotebook());
await waitFor(() => expect(result.current.notebooks).toHaveLength(2));
await act(async () => {
await result.current.createNotebook('New One');
});
expect(notebooksApi.createNotebook).toHaveBeenCalledWith('New One');
expect(result.current.notebooks).toHaveLength(3);
expect(result.current.notebooks[2]).toEqual(newNb);
expect(result.current.activeNotebook).toEqual(newNb);
});
it('createNotebook does nothing for empty name', async () => {
const { result } = renderHook(() => useNotebook());
await waitFor(() => expect(result.current.notebooks).toHaveLength(2));
await act(async () => {
await result.current.createNotebook('');
});
expect(notebooksApi.createNotebook).not.toHaveBeenCalled();
expect(result.current.notebooks).toHaveLength(2);
});
it('deleteNotebook removes from list', async () => {
const { result } = renderHook(() => useNotebook());
await waitFor(() => expect(result.current.notebooks).toHaveLength(2));
await act(async () => {
await result.current.deleteNotebook('nb-1');
});
expect(notebooksApi.deleteNotebook).toHaveBeenCalledWith('nb-1');
expect(result.current.notebooks).toHaveLength(1);
expect(result.current.notebooks[0].id).toBe('nb-2');
});
it('deleteNotebook clears activeNotebook if it was the deleted one', async () => {
const { result } = renderHook(() => useNotebook());
await waitFor(() => expect(result.current.notebooks).toHaveLength(2));
act(() => {
result.current.selectNotebook('nb-1');
});
expect(result.current.activeNotebook?.id).toBe('nb-1');
await act(async () => {
await result.current.deleteNotebook('nb-1');
});
expect(result.current.activeNotebook).toBeNull();
});
it('deleteNotebook preserves activeNotebook if different one deleted', async () => {
const { result } = renderHook(() => useNotebook());
await waitFor(() => expect(result.current.notebooks).toHaveLength(2));
act(() => {
result.current.selectNotebook('nb-2');
});
await act(async () => {
await result.current.deleteNotebook('nb-1');
});
expect(result.current.activeNotebook).toEqual({ id: 'nb-2', name: 'Personal' });
});
it('deleteNotebook does not remove from list on API error', async () => {
notebooksApi.deleteNotebook.mockRejectedValue(new Error('Server error'));
vi.spyOn(console, 'error').mockImplementation(() => {});
const { result } = renderHook(() => useNotebook());
await waitFor(() => expect(result.current.notebooks).toHaveLength(2));
await act(async () => {
await result.current.deleteNotebook('nb-1');
});
expect(result.current.notebooks).toHaveLength(2);
console.error.mockRestore();
});
it('handles listNotebooks API failure gracefully', async () => {
notebooksApi.listNotebooks.mockRejectedValue(new Error('Network error'));
vi.spyOn(console, 'error').mockImplementation(() => {});
const { result } = renderHook(() => useNotebook());
await waitFor(() => {
expect(notebooksApi.listNotebooks).toHaveBeenCalled();
});
expect(result.current.notebooks).toEqual([]);
console.error.mockRestore();
});
});

View File

@@ -0,0 +1,50 @@
import { useState, useEffect, useCallback } from 'react';
import * as sourceCorpusApi from '../api/sourceCorpus.js';
export function useSourceCorpus() {
const [sourceCorpora, setSourceCorpora] = useState([]);
const [activeSourceCorpus, setActiveSourceCorpus] = useState(null);
useEffect(() => {
sourceCorpusApi.listSourceCorpora().then(setSourceCorpora).catch(console.error);
}, []);
const selectSourceCorpus = useCallback(
(id) => {
const corpus = sourceCorpora.find((n) => n.id === id) || null;
setActiveSourceCorpus(corpus);
},
[sourceCorpora],
);
const createSourceCorpus = useCallback(async (name) => {
if (!name) return;
const corpus = await sourceCorpusApi.createSourceCorpus(name);
setSourceCorpora((prev) => [...prev, corpus]);
setActiveSourceCorpus(corpus);
}, []);
const deleteSourceCorpus = useCallback(
async (id) => {
try {
await sourceCorpusApi.deleteSourceCorpus(id);
} catch (err) {
console.error('Failed to delete sourceCorpus', err);
return;
}
setSourceCorpora((prev) => prev.filter((n) => n.id !== id));
if (activeSourceCorpus?.id === id) {
setActiveSourceCorpus(null);
}
},
[activeSourceCorpus],
);
return {
sourceCorpora,
activeSourceCorpus,
selectSourceCorpus,
createSourceCorpus,
deleteSourceCorpus,
};
}

View File

@@ -0,0 +1,176 @@
import { describe, it, expect, vi, beforeEach } from 'vitest';
import { renderHook, act, waitFor } from '@testing-library/react';
vi.mock('../api/sourceCorpus.js', () => ({
listSourceCorpora: vi.fn(),
createSourceCorpus: vi.fn(),
deleteSourceCorpus: vi.fn(),
}));
import { useSourceCorpus } from './useSourceCorpus.js';
import * as sourceCorpusApi from '../api/sourceCorpus.js';
const sourceCorpora = [
{ id: 'nb-1', name: 'Research' },
{ id: 'nb-2', name: 'Personal' },
];
beforeEach(() => {
vi.resetAllMocks();
sourceCorpusApi.listSourceCorpora.mockResolvedValue(sourceCorpora);
sourceCorpusApi.createSourceCorpus.mockImplementation(async (name) => ({
id: `nb-${Date.now()}`,
name,
}));
sourceCorpusApi.deleteSourceCorpus.mockResolvedValue({});
});
describe('useSourceCorpus', () => {
it('loads sourceCorpora on mount', async () => {
const { result } = renderHook(() => useSourceCorpus());
await waitFor(() => {
expect(result.current.sourceCorpora).toEqual(sourceCorpora);
});
expect(sourceCorpusApi.listSourceCorpora).toHaveBeenCalledOnce();
});
it('starts with no active sourceCorpus', () => {
const { result } = renderHook(() => useSourceCorpus());
expect(result.current.activeSourceCorpus).toBeNull();
});
it('selectSourceCorpus sets the active sourceCorpus', async () => {
const { result } = renderHook(() => useSourceCorpus());
await waitFor(() => {
expect(result.current.sourceCorpora).toHaveLength(2);
});
act(() => {
result.current.selectSourceCorpus('nb-2');
});
expect(result.current.activeSourceCorpus).toEqual({ id: 'nb-2', name: 'Personal' });
});
it('selectSourceCorpus sets null for unknown id', async () => {
const { result } = renderHook(() => useSourceCorpus());
await waitFor(() => {
expect(result.current.sourceCorpora).toHaveLength(2);
});
act(() => {
result.current.selectSourceCorpus('nb-2');
});
expect(result.current.activeSourceCorpus).not.toBeNull();
act(() => {
result.current.selectSourceCorpus('nonexistent');
});
expect(result.current.activeSourceCorpus).toBeNull();
});
it('createSourceCorpus calls API and adds to list', async () => {
const newCorpus = { id: 'nb-new', name: 'New One' };
sourceCorpusApi.createSourceCorpus.mockResolvedValue(newCorpus);
const { result } = renderHook(() => useSourceCorpus());
await waitFor(() => expect(result.current.sourceCorpora).toHaveLength(2));
await act(async () => {
await result.current.createSourceCorpus('New One');
});
expect(sourceCorpusApi.createSourceCorpus).toHaveBeenCalledWith('New One');
expect(result.current.sourceCorpora).toHaveLength(3);
expect(result.current.sourceCorpora[2]).toEqual(newCorpus);
expect(result.current.activeSourceCorpus).toEqual(newCorpus);
});
it('createSourceCorpus does nothing for empty name', async () => {
const { result } = renderHook(() => useSourceCorpus());
await waitFor(() => expect(result.current.sourceCorpora).toHaveLength(2));
await act(async () => {
await result.current.createSourceCorpus('');
});
expect(sourceCorpusApi.createSourceCorpus).not.toHaveBeenCalled();
expect(result.current.sourceCorpora).toHaveLength(2);
});
it('deleteSourceCorpus removes from list', async () => {
const { result } = renderHook(() => useSourceCorpus());
await waitFor(() => expect(result.current.sourceCorpora).toHaveLength(2));
await act(async () => {
await result.current.deleteSourceCorpus('nb-1');
});
expect(sourceCorpusApi.deleteSourceCorpus).toHaveBeenCalledWith('nb-1');
expect(result.current.sourceCorpora).toHaveLength(1);
expect(result.current.sourceCorpora[0].id).toBe('nb-2');
});
it('deleteSourceCorpus clears activeSourceCorpus if it was the deleted one', async () => {
const { result } = renderHook(() => useSourceCorpus());
await waitFor(() => expect(result.current.sourceCorpora).toHaveLength(2));
act(() => {
result.current.selectSourceCorpus('nb-1');
});
expect(result.current.activeSourceCorpus?.id).toBe('nb-1');
await act(async () => {
await result.current.deleteSourceCorpus('nb-1');
});
expect(result.current.activeSourceCorpus).toBeNull();
});
it('deleteSourceCorpus preserves activeSourceCorpus if different one deleted', async () => {
const { result } = renderHook(() => useSourceCorpus());
await waitFor(() => expect(result.current.sourceCorpora).toHaveLength(2));
act(() => {
result.current.selectSourceCorpus('nb-2');
});
await act(async () => {
await result.current.deleteSourceCorpus('nb-1');
});
expect(result.current.activeSourceCorpus).toEqual({ id: 'nb-2', name: 'Personal' });
});
it('deleteSourceCorpus does not remove from list on API error', async () => {
sourceCorpusApi.deleteSourceCorpus.mockRejectedValue(new Error('Server error'));
vi.spyOn(console, 'error').mockImplementation(() => {});
const { result } = renderHook(() => useSourceCorpus());
await waitFor(() => expect(result.current.sourceCorpora).toHaveLength(2));
await act(async () => {
await result.current.deleteSourceCorpus('nb-1');
});
expect(result.current.sourceCorpora).toHaveLength(2);
console.error.mockRestore();
});
it('handles listSourceCorpora API failure gracefully', async () => {
sourceCorpusApi.listSourceCorpora.mockRejectedValue(new Error('Network error'));
vi.spyOn(console, 'error').mockImplementation(() => {});
const { result } = renderHook(() => useSourceCorpus());
await waitFor(() => {
expect(sourceCorpusApi.listSourceCorpora).toHaveBeenCalled();
});
expect(result.current.sourceCorpora).toEqual([]);
console.error.mockRestore();
});
});

View File

@@ -75,14 +75,14 @@ body {
font-size: 16px;
}
/* ── Notebook List ─────────────────────────────────── */
/* ── SourceCorpus List ─────────────────────────────────── */
.notebook-list {
.source-corpus-list {
padding: 20px 16px 12px;
border-bottom: 1px solid rgba(255, 255, 255, 0.08);
}
.notebook-list h2 {
.source-corpus-list h2 {
font-size: 11px;
font-weight: 600;
text-transform: uppercase;
@@ -91,12 +91,12 @@ body {
margin-bottom: 12px;
}
.notebook-list ul {
.source-corpus-list ul {
list-style: none;
margin-bottom: 8px;
}
.notebook-list li {
.source-corpus-list li {
display: flex;
align-items: center;
justify-content: space-between;
@@ -107,23 +107,23 @@ body {
transition: background 0.15s;
}
.notebook-list li:hover {
.source-corpus-list li:hover {
background: var(--sidebar-hover);
}
.notebook-list li.active {
.source-corpus-list li.active {
background: var(--sidebar-active);
color: #fff;
font-weight: 500;
}
.notebook-list li .nb-name {
.source-corpus-list li .sc-name {
overflow: hidden;
text-overflow: ellipsis;
white-space: nowrap;
}
.notebook-list li .nb-delete {
.source-corpus-list li .sc-delete {
opacity: 0;
background: none;
border: none;
@@ -135,15 +135,15 @@ body {
transition: opacity 0.15s, color 0.15s;
}
.notebook-list li:hover .nb-delete {
.source-corpus-list li:hover .sc-delete {
opacity: 1;
}
.notebook-list li .nb-delete:hover {
.source-corpus-list li .sc-delete:hover {
color: var(--danger);
}
.btn-new-notebook {
.btn-new-corpus {
width: 100%;
padding: 8px;
border: 1px dashed rgba(255, 255, 255, 0.2);
@@ -155,7 +155,7 @@ body {
transition: border-color 0.15s, color 0.15s;
}
.btn-new-notebook:hover {
.btn-new-corpus:hover {
border-color: rgba(255, 255, 255, 0.5);
color: #fff;
}

BIN
demo.jpg Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 109 KiB

View File

@@ -1,11 +1,11 @@
{
"name": "notebook-clone-server",
"name": "source-sentinel-server",
"version": "0.1.0",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "notebook-clone-server",
"name": "source-sentinel-server",
"version": "0.1.0",
"dependencies": {
"@anthropic-ai/sdk": "^0.78.0",

View File

@@ -1,5 +1,5 @@
{
"name": "notebook-clone-server",
"name": "source-sentinel-server",
"version": "0.1.0",
"private": true,
"engines": {

View File

@@ -4,7 +4,7 @@ import 'dotenv/config';
import express, { type Request, type Response, type NextFunction } from 'express';
import cors from 'cors';
import logger from './logger.js';
import notebookRoutes from './routes/notebooks.js';
import sourceCorpusRoutes from './routes/sourceCorpus.js';
import sourceRoutes from './routes/sources.js';
import queryRoutes from './routes/query.js';
import citationDetailRoutes from './routes/citationDetail.js';
@@ -39,13 +39,13 @@ app.use((req: Request, res: Response, next: NextFunction) => {
app.get('/.well-known/healthcheck', (_req: Request, res: Response) => {
res.json({
service: 'notebook-clone',
service: 'source-sentinel',
status: 'ok',
uptime: process.uptime(),
});
});
app.use('/api/notebooks', notebookRoutes);
app.use('/api/source-corpus', sourceCorpusRoutes);
app.use('/api/sources', sourceRoutes);
app.use('/api/query', queryRoutes);
app.use('/api/citation-detail', citationDetailRoutes);

View File

@@ -1,31 +1,31 @@
'use strict';
import type { Request, Response, NextFunction } from 'express';
import * as notebookStore from '../stores/notebookStore.js';
import type { Notebook } from '../stores/notebookStore.js';
import * as sourceCorpusStore from '../stores/sourceCorpusStore.js';
import type { SourceCorpus } from '../stores/sourceCorpusStore.js';
export interface NotebookRequest extends Request {
notebookId?: string;
notebook?: Notebook;
export interface SourceCorpusRequest extends Request {
sourceCorpusId?: string;
sourceCorpus?: SourceCorpus;
}
export function requireNotebookId(req: NotebookRequest, res: Response, next: NextFunction): void {
const notebookId = (req.body?.notebookId ?? req.query?.notebookId) as string | undefined;
if (!notebookId) {
res.status(400).json({ error: 'notebookId is required' });
export function requireSourceCorpusId(req: SourceCorpusRequest, res: Response, next: NextFunction): void {
const sourceCorpusId = (req.body?.sourceCorpusId ?? req.query?.sourceCorpusId) as string | undefined;
if (!sourceCorpusId) {
res.status(400).json({ error: 'sourceCorpusId is required' });
return;
}
req.notebookId = notebookId;
req.sourceCorpusId = sourceCorpusId;
next();
}
export function requireNotebook(req: NotebookRequest, res: Response, next: NextFunction): void {
const notebook = notebookStore.getNotebook(req.notebookId!);
if (!notebook) {
res.status(404).json({ error: 'notebook not found' });
export function requireSourceCorpus(req: SourceCorpusRequest, res: Response, next: NextFunction): void {
const sourceCorpus = sourceCorpusStore.getSourceCorpus(req.sourceCorpusId!);
if (!sourceCorpus) {
res.status(404).json({ error: 'source corpus not found' });
return;
}
req.notebook = notebook;
req.sourceCorpus = sourceCorpus;
next();
}

View File

@@ -2,8 +2,8 @@
import { Router, type Request, type Response, type NextFunction } from 'express';
import logger from '../logger.js';
import * as notebookStore from '../stores/notebookStore.js';
import type { TextChunk, Source, SourceGroup } from '../stores/notebookStore.js';
import * as sourceCorpusStore from '../stores/sourceCorpusStore.js';
import type { TextChunk, Source, SourceGroup } from '../stores/sourceCorpusStore.js';
import * as documentCacheStore from '../stores/documentCacheStore.js';
import type { CacheEntry } from '../stores/documentCacheStore.js';
import {
@@ -27,16 +27,16 @@ const GENERATORS: Record<DocumentType, GeneratorFn> = {
const router = Router();
interface GenerateBody {
notebookId?: string;
sourceCorpusId?: string;
type?: string;
}
router.post('/generate', async (req: Request<unknown, unknown, GenerateBody>, res: Response, next: NextFunction) => {
try {
const { notebookId, type } = req.body;
const { sourceCorpusId, type } = req.body;
if (!notebookId || !type) {
res.status(400).json({ error: 'notebookId and type are required' });
if (!sourceCorpusId || !type) {
res.status(400).json({ error: 'sourceCorpusId and type are required' });
return;
}
@@ -48,31 +48,31 @@ router.post('/generate', async (req: Request<unknown, unknown, GenerateBody>, re
return;
}
const chunks: TextChunk[] = notebookStore.getChunksForNotebook(notebookId);
const chunks: TextChunk[] = sourceCorpusStore.getChunksForSourceCorpus(sourceCorpusId);
if (chunks.length === 0) {
res.status(422).json({ error: 'No source material available in this notebook' });
res.status(422).json({ error: 'No source material available in this source corpus' });
return;
}
const sources: Source[] = notebookStore.getSources(notebookId);
const cached: CacheEntry | null = documentCacheStore.getCachedDocument(notebookId, type);
if (cached && documentCacheStore.isFresh(notebookId, type, sources)) {
logger.info({ notebookId, type }, 'serving cached document');
const sources: Source[] = sourceCorpusStore.getSources(sourceCorpusId);
const cached: CacheEntry | null = documentCacheStore.getCachedDocument(sourceCorpusId, type);
if (cached && documentCacheStore.isFresh(sourceCorpusId, type, sources)) {
logger.info({ sourceCorpusId, type }, 'serving cached document');
res.json({ type, document: cached.document });
return;
}
const sourceGroups: SourceGroup[] = notebookStore.buildSourceGroups(notebookId, chunks);
const sourceGroups: SourceGroup[] = sourceCorpusStore.buildSourceGroups(sourceCorpusId, chunks);
logger.info(
{ notebookId, type, sourceCount: sourceGroups.length, chunkCount: chunks.length },
{ sourceCorpusId, type, sourceCount: sourceGroups.length, chunkCount: chunks.length },
'document generation started'
);
const document: StudyGuide | Faq | ExecutiveBrief = await generator(sourceGroups);
documentCacheStore.setCachedDocument(notebookId, type, document, sources);
logger.info({ notebookId, type }, 'document generation complete');
documentCacheStore.setCachedDocument(sourceCorpusId, type, document, sources);
logger.info({ sourceCorpusId, type }, 'document generation complete');
res.json({ type, document });
} catch (err) {

View File

@@ -7,8 +7,8 @@ import type { ScoredChunk, RankedChunk, TextChunk } from '../services/retrievalS
import { generate } from '../services/generationService.js';
import type { GenerationResult } from '../services/generationService.js';
import { computeGroundedness } from '../services/scoringService.js';
import * as notebookStore from '../stores/notebookStore.js';
import type { SourceGroup } from '../stores/notebookStore.js';
import * as sourceCorpusStore from '../stores/sourceCorpusStore.js';
import type { SourceGroup } from '../stores/sourceCorpusStore.js';
const TOP_K_SEARCH = 20;
const TOP_K_RERANK = 5;
@@ -16,7 +16,7 @@ const TOP_K_RERANK = 5;
const router = Router();
interface QueryBody {
notebookId?: string;
sourceCorpusId?: string;
question?: string;
}
@@ -36,20 +36,20 @@ interface QueryResponse {
router.post('/', async (req: Request<unknown, unknown, QueryBody>, res: Response, next: NextFunction) => {
try {
const { notebookId, question } = req.body;
if (!notebookId || !question) {
res.status(400).json({ error: 'notebookId and question are required' });
const { sourceCorpusId, question } = req.body;
if (!sourceCorpusId || !question) {
res.status(400).json({ error: 'sourceCorpusId and question are required' });
return;
}
logger.info({ notebookId, question }, 'query received');
logger.info({ sourceCorpusId, question }, 'query received');
const [queryEmbedding]: number[][] = await embedTexts([question], 'query');
const searchResults: ScoredChunk[] = search(queryEmbedding, notebookId, TOP_K_SEARCH);
const searchResults: ScoredChunk[] = search(queryEmbedding, sourceCorpusId, TOP_K_SEARCH);
if (searchResults.length === 0) {
res.json({
answer: 'No sources found for this notebook. Upload some documents first.',
answer: 'No sources found for this source corpus. Upload some documents first.',
citations: [],
groundednessScore: null,
followUpQuestions: [],
@@ -69,7 +69,7 @@ router.post('/', async (req: Request<unknown, unknown, QueryBody>, res: Response
'retrieval complete'
);
const sourceGroups: SourceGroup[] = notebookStore.buildSourceGroups(notebookId, topChunks);
const sourceGroups: SourceGroup[] = sourceCorpusStore.buildSourceGroups(sourceCorpusId, topChunks);
const { answer, citedSourceIndices, followUpQuestions }: GenerationResult = await generate(question, sourceGroups);

View File

@@ -2,33 +2,33 @@
import { Router, type Request, type Response } from 'express';
import { v4 as uuidv4 } from 'uuid';
import * as notebookStore from '../stores/notebookStore.js';
import * as sourceCorpusStore from '../stores/sourceCorpusStore.js';
import { deleteVectorsForChunks } from '../stores/vectorStore.js';
const router = Router();
router.get('/', (_req: Request, res: Response) => {
res.json(notebookStore.getAllNotebooks());
res.json(sourceCorpusStore.getAllSourceCorpora());
});
interface CreateNotebookBody {
interface CreateSourceCorpusBody {
name?: string;
}
router.post('/', (req: Request<unknown, unknown, CreateNotebookBody>, res: Response) => {
router.post('/', (req: Request<unknown, unknown, CreateSourceCorpusBody>, res: Response) => {
const { name } = req.body;
if (!name || !name.trim()) {
res.status(400).json({ error: 'name is required' });
return;
}
const notebook = notebookStore.createNotebook({
const sourceCorpus = sourceCorpusStore.createSourceCorpus({
id: uuidv4(),
name: name.trim(),
createdAt: new Date().toISOString(),
});
res.status(201).json(notebook);
res.status(201).json(sourceCorpus);
});
interface DeleteParams {
@@ -36,10 +36,10 @@ interface DeleteParams {
}
router.delete('/:id', (req: Request<DeleteParams>, res: Response) => {
const chunks = notebookStore.getChunksForNotebook(req.params.id);
const deleted = notebookStore.deleteNotebook(req.params.id);
const chunks = sourceCorpusStore.getChunksForSourceCorpus(req.params.id);
const deleted = sourceCorpusStore.deleteSourceCorpus(req.params.id);
if (!deleted) {
res.status(404).json({ error: 'notebook not found' });
res.status(404).json({ error: 'source corpus not found' });
return;
}
if (chunks.length) {

View File

@@ -4,18 +4,18 @@ import { Router, type Request, type Response, type NextFunction } from 'express'
import multer from 'multer';
import { v4 as uuidv4 } from 'uuid';
import logger from '../logger.js';
import * as notebookStore from '../stores/notebookStore.js';
import type { Source } from '../stores/notebookStore.js';
import * as sourceCorpusStore from '../stores/sourceCorpusStore.js';
import type { Source } from '../stores/sourceCorpusStore.js';
import { parseFile, chunkText, parseUrl, parseYoutubeUrl, isYoutubeUrl } from '../services/sourceService.js';
import type { TextChunk } from '../services/sourceService.js';
import { embedTexts, storeChunkEmbeddings } from '../services/retrievalService.js';
import { triggerPreGeneration } from '../services/preGenerationService.js';
import {
requireNotebookId,
requireNotebook,
requireSourceCorpusId,
requireSourceCorpus,
requireFile,
requireUrl,
type NotebookRequest,
type SourceCorpusRequest,
} from '../middleware/validation.js';
const upload = multer({ storage: multer.memoryStorage(), limits: { fileSize: 50 * 1024 * 1024 } });
@@ -26,8 +26,8 @@ function cleanFilename(name: string): string {
return name.replace(TIMESTAMP_RE, '');
}
router.get('/', requireNotebookId, (req: NotebookRequest, res: Response) => {
res.json(notebookStore.getSources(req.notebookId!));
router.get('/', requireSourceCorpusId, (req: SourceCorpusRequest, res: Response) => {
res.json(sourceCorpusStore.getSources(req.sourceCorpusId!));
});
function multerUpload(req: Request, res: Response, next: NextFunction): void {
@@ -47,10 +47,10 @@ function multerUpload(req: Request, res: Response, next: NextFunction): void {
router.post(
'/',
multerUpload,
requireNotebookId,
requireNotebook,
requireSourceCorpusId,
requireSourceCorpus,
requireFile,
async (req: NotebookRequest, res: Response, next: NextFunction) => {
async (req: SourceCorpusRequest, res: Response, next: NextFunction) => {
try {
const sourceId: string = uuidv4();
const rawName: string = Buffer.from(req.file!.originalname, 'latin1').toString('utf-8');
@@ -67,13 +67,13 @@ router.post(
'parsed and chunked source'
);
notebookStore.addChunksToNotebook(req.notebookId!, chunks);
sourceCorpusStore.addChunksToSourceCorpus(req.sourceCorpusId!, chunks);
const texts: string[] = chunks.map((c) => c.text);
const embeddings: number[][] = await embedTexts(texts, 'document');
storeChunkEmbeddings(chunks, embeddings);
const source: Source | null = notebookStore.addSource(req.notebookId!, {
const source: Source | null = sourceCorpusStore.addSource(req.sourceCorpusId!, {
id: sourceId,
name: displayName,
mimetype: req.file!.mimetype,
@@ -81,7 +81,7 @@ router.post(
uploadedAt: new Date().toISOString(),
});
triggerPreGeneration(req.notebookId!);
triggerPreGeneration(req.sourceCorpusId!);
res.status(201).json(source);
} catch (err) {
@@ -96,10 +96,10 @@ interface UrlBody {
router.post(
'/url',
requireNotebookId,
requireNotebook,
requireSourceCorpusId,
requireSourceCorpus,
requireUrl,
async (req: NotebookRequest & Request<unknown, unknown, UrlBody>, res: Response, next: NextFunction) => {
async (req: SourceCorpusRequest & Request<unknown, unknown, UrlBody>, res: Response, next: NextFunction) => {
try {
const { url } = req.body;
const sourceId: string = uuidv4();
@@ -116,13 +116,13 @@ router.post(
return;
}
notebookStore.addChunksToNotebook(req.notebookId!, chunks);
sourceCorpusStore.addChunksToSourceCorpus(req.sourceCorpusId!, chunks);
const texts: string[] = chunks.map((c) => c.text);
const embeddings: number[][] = await embedTexts(texts, 'document');
storeChunkEmbeddings(chunks, embeddings);
const source: Source | null = notebookStore.addSource(req.notebookId!, {
const source: Source | null = sourceCorpusStore.addSource(req.sourceCorpusId!, {
id: sourceId,
name: displayName,
mimetype: isYT ? 'video/youtube' : 'text/html',
@@ -130,7 +130,7 @@ router.post(
uploadedAt: new Date().toISOString(),
});
triggerPreGeneration(req.notebookId!);
triggerPreGeneration(req.sourceCorpusId!);
res.status(201).json(source);
} catch (err) {

View File

@@ -1,9 +1,9 @@
import { describe, it, expect, vi, beforeEach, afterEach, type Mock } from 'vitest';
vi.mock('../stores/notebookStore.js', () => ({
getNotebook: vi.fn(),
vi.mock('../stores/sourceCorpusStore.js', () => ({
getSourceCorpus: vi.fn(),
getSources: vi.fn(),
getChunksForNotebook: vi.fn(),
getChunksForSourceCorpus: vi.fn(),
buildSourceGroups: vi.fn(),
}));
@@ -26,14 +26,14 @@ vi.mock('../logger.js', () => ({
}));
import { triggerPreGeneration } from './preGenerationService.js';
import * as notebookStore from '../stores/notebookStore.js';
import * as sourceCorpusStore from '../stores/sourceCorpusStore.js';
import * as documentCacheStore from '../stores/documentCacheStore.js';
import { generateStudyGuide, generateFaq, generateExecutiveBrief } from './documentService.js';
const mockedGetNotebook = notebookStore.getNotebook as Mock;
const mockedGetSources = notebookStore.getSources as Mock;
const mockedGetChunksForNotebook = notebookStore.getChunksForNotebook as Mock;
const mockedBuildSourceGroups = notebookStore.buildSourceGroups as Mock;
const mockedGetSourceCorpus = sourceCorpusStore.getSourceCorpus as Mock;
const mockedGetSources = sourceCorpusStore.getSources as Mock;
const mockedGetChunksForSourceCorpus = sourceCorpusStore.getChunksForSourceCorpus as Mock;
const mockedBuildSourceGroups = sourceCorpusStore.buildSourceGroups as Mock;
const mockedIsGenerating = documentCacheStore.isGenerating as Mock;
const mockedInvalidate = documentCacheStore.invalidate as Mock;
const mockedMarkGenerating = documentCacheStore.markGenerating as Mock;
@@ -43,10 +43,10 @@ const mockedGenerateStudyGuide = generateStudyGuide as Mock;
const mockedGenerateFaq = generateFaq as Mock;
const mockedGenerateExecutiveBrief = generateExecutiveBrief as Mock;
function setupNotebook(id: string): void {
mockedGetNotebook.mockReturnValue({ id });
function setupSourceCorpus(id: string): void {
mockedGetSourceCorpus.mockReturnValue({ id });
mockedGetSources.mockReturnValue([{ id: 's1' }, { id: 's2' }]);
mockedGetChunksForNotebook.mockReturnValue([{ id: 'c1', text: 'hello' }]);
mockedGetChunksForSourceCorpus.mockReturnValue([{ id: 'c1', text: 'hello' }]);
mockedBuildSourceGroups.mockReturnValue([{ docIndex: 1, name: 'Doc', chunks: [{ text: 'hello' }] }]);
}
@@ -66,14 +66,14 @@ afterEach(() => {
});
describe('triggerPreGeneration guards', () => {
it('does nothing if the notebook does not exist', () => {
mockedGetNotebook.mockReturnValue(null);
it('does nothing if the sourceCorpus does not exist', () => {
mockedGetSourceCorpus.mockReturnValue(null);
triggerPreGeneration('nb-missing');
expect(mockedGetSources).not.toHaveBeenCalled();
});
it('does nothing if fewer than 2 sources', () => {
mockedGetNotebook.mockReturnValue({ id: 'nb-few' });
mockedGetSourceCorpus.mockReturnValue({ id: 'nb-few' });
mockedGetSources.mockReturnValue([{ id: 's1' }]);
triggerPreGeneration('nb-few');
expect(mockedMarkGenerating).not.toHaveBeenCalled();
@@ -82,7 +82,7 @@ describe('triggerPreGeneration guards', () => {
describe('queuing when already generating', () => {
it('invalidates cache and skips runPreGeneration', () => {
setupNotebook('nb-q');
setupSourceCorpus('nb-q');
mockedIsGenerating.mockReturnValue(true);
triggerPreGeneration('nb-q');
@@ -94,7 +94,7 @@ describe('queuing when already generating', () => {
describe('debounce', () => {
it('does not run generation immediately', () => {
setupNotebook('nb-debounce');
setupSourceCorpus('nb-debounce');
mockedIsGenerating.mockReturnValue(false);
stubGeneratorsOk();
@@ -104,7 +104,7 @@ describe('debounce', () => {
});
it('runs generation after the debounce delay', async () => {
setupNotebook('nb-delay');
setupSourceCorpus('nb-delay');
mockedIsGenerating.mockReturnValue(false);
stubGeneratorsOk();
@@ -119,7 +119,7 @@ describe('debounce', () => {
});
it('resets the timer on repeated calls, running generation only once', async () => {
setupNotebook('nb-batch');
setupSourceCorpus('nb-batch');
mockedIsGenerating.mockReturnValue(false);
stubGeneratorsOk();
@@ -140,7 +140,7 @@ describe('debounce', () => {
describe('runPreGeneration (via triggerPreGeneration)', () => {
it('runs all three generators and caches results', async () => {
setupNotebook('nb-happy');
setupSourceCorpus('nb-happy');
mockedIsGenerating.mockReturnValue(false);
stubGeneratorsOk();
@@ -158,7 +158,7 @@ describe('runPreGeneration (via triggerPreGeneration)', () => {
});
it('marks generating before starting and clears it after', async () => {
setupNotebook('nb-flag');
setupSourceCorpus('nb-flag');
mockedIsGenerating.mockReturnValue(false);
stubGeneratorsOk();
@@ -173,7 +173,7 @@ describe('runPreGeneration (via triggerPreGeneration)', () => {
});
it('still clears generating flag when a generator fails', async () => {
setupNotebook('nb-partial');
setupSourceCorpus('nb-partial');
mockedIsGenerating.mockReturnValue(false);
mockedGenerateStudyGuide.mockRejectedValue(new Error('boom'));
mockedGenerateFaq.mockResolvedValue({ subject: 'ok' });
@@ -190,9 +190,9 @@ describe('runPreGeneration (via triggerPreGeneration)', () => {
});
});
describe('re-trigger for queued notebooks', () => {
describe('re-trigger for queued sourceCorpora', () => {
it('runs a second generation cycle after the first finishes', async () => {
setupNotebook('nb-re');
setupSourceCorpus('nb-re');
mockedIsGenerating.mockReturnValueOnce(false).mockReturnValueOnce(true);
let resolveFirst: (value: { title: string }) => void;

View File

@@ -1,7 +1,7 @@
'use strict';
import logger from '../logger.js';
import * as notebookStore from '../stores/notebookStore.js';
import * as sourceCorpusStore from '../stores/sourceCorpusStore.js';
import * as documentCacheStore from '../stores/documentCacheStore.js';
import {
generateStudyGuide,
@@ -29,45 +29,45 @@ const DEBOUNCE_MS = 5000;
const pendingReGen = new Set<string>();
const debounceTimers = new Map<string, ReturnType<typeof setTimeout>>();
export function triggerPreGeneration(notebookId: string): void {
const notebook = notebookStore.getNotebook(notebookId);
if (!notebook) return;
export function triggerPreGeneration(sourceCorpusId: string): void {
const sourceCorpus = sourceCorpusStore.getSourceCorpus(sourceCorpusId);
if (!sourceCorpus) return;
const sources = notebookStore.getSources(notebookId);
const sources = sourceCorpusStore.getSources(sourceCorpusId);
if (sources.length < MIN_SOURCES) return;
if (documentCacheStore.isGenerating(notebookId)) {
pendingReGen.add(notebookId);
documentCacheStore.invalidate(notebookId);
logger.debug({ notebookId }, 'pre-generation in progress, queued re-generation');
if (documentCacheStore.isGenerating(sourceCorpusId)) {
pendingReGen.add(sourceCorpusId);
documentCacheStore.invalidate(sourceCorpusId);
logger.debug({ sourceCorpusId }, 'pre-generation in progress, queued re-generation');
return;
}
const existingTimer = debounceTimers.get(notebookId);
const existingTimer = debounceTimers.get(sourceCorpusId);
if (existingTimer !== undefined) {
clearTimeout(existingTimer);
}
debounceTimers.set(
notebookId,
sourceCorpusId,
setTimeout(() => {
debounceTimers.delete(notebookId);
runPreGeneration(notebookId);
debounceTimers.delete(sourceCorpusId);
runPreGeneration(sourceCorpusId);
}, DEBOUNCE_MS)
);
logger.debug({ notebookId, debounceMs: DEBOUNCE_MS }, 'pre-generation debounced');
logger.debug({ sourceCorpusId, debounceMs: DEBOUNCE_MS }, 'pre-generation debounced');
}
function runPreGeneration(notebookId: string): void {
function runPreGeneration(sourceCorpusId: string): void {
try {
const sources = notebookStore.getSources(notebookId);
const chunks = notebookStore.getChunksForNotebook(notebookId);
const sourceGroups = notebookStore.buildSourceGroups(notebookId, chunks) as SourceGroup[];
const sources = sourceCorpusStore.getSources(sourceCorpusId);
const chunks = sourceCorpusStore.getChunksForSourceCorpus(sourceCorpusId);
const sourceGroups = sourceCorpusStore.buildSourceGroups(sourceCorpusId, chunks) as SourceGroup[];
documentCacheStore.invalidate(notebookId);
documentCacheStore.markGenerating(notebookId);
documentCacheStore.invalidate(sourceCorpusId);
documentCacheStore.markGenerating(sourceCorpusId);
logger.info(
{ notebookId, sourceCount: sources.length, chunkCount: chunks.length },
{ sourceCorpusId, sourceCount: sources.length, chunkCount: chunks.length },
'background pre-generation started'
);
@@ -75,31 +75,31 @@ function runPreGeneration(notebookId: string): void {
async ([type, generator]) => {
try {
const document = await generator(sourceGroups);
documentCacheStore.setCachedDocument(notebookId, type, document, sources);
logger.info({ notebookId, type }, 'background pre-generation complete for type');
documentCacheStore.setCachedDocument(sourceCorpusId, type, document, sources);
logger.info({ sourceCorpusId, type }, 'background pre-generation complete for type');
} catch (err) {
const error = err as Error;
logger.error({ notebookId, type, err: error.message }, 'background pre-generation failed for type');
logger.error({ sourceCorpusId, type, err: error.message }, 'background pre-generation failed for type');
}
}
);
Promise.all(jobs)
.then(() => {
logger.info({ notebookId }, 'all background pre-generation complete');
logger.info({ sourceCorpusId }, 'all background pre-generation complete');
})
.finally(() => {
documentCacheStore.clearGenerating(notebookId);
documentCacheStore.clearGenerating(sourceCorpusId);
if (pendingReGen.has(notebookId)) {
pendingReGen.delete(notebookId);
logger.info({ notebookId }, 're-triggering pre-generation for updated sources');
runPreGeneration(notebookId);
if (pendingReGen.has(sourceCorpusId)) {
pendingReGen.delete(sourceCorpusId);
logger.info({ sourceCorpusId }, 're-triggering pre-generation for updated sources');
runPreGeneration(sourceCorpusId);
}
});
} catch (err) {
const error = err as Error;
logger.error({ notebookId, err: error.message }, 'pre-generation setup failed');
documentCacheStore.clearGenerating(notebookId);
logger.error({ sourceCorpusId, err: error.message }, 'pre-generation setup failed');
documentCacheStore.clearGenerating(sourceCorpusId);
}
}

View File

@@ -2,7 +2,7 @@
import logger from '../logger.js';
import * as vectorStore from '../stores/vectorStore.js';
import * as notebookStore from '../stores/notebookStore.js';
import * as sourceCorpusStore from '../stores/sourceCorpusStore.js';
const VOYAGE_API_URL = 'https://api.voyageai.com/v1';
const EMBED_MODEL = 'voyage-3';
@@ -72,8 +72,8 @@ export function storeChunkEmbeddings(chunks: TextChunk[], embeddings: number[][]
logger.debug({ count: chunks.length }, 'stored chunk embeddings');
}
export function search(queryEmbedding: number[], notebookId: string, topK = 10): ScoredChunk[] {
const chunks = notebookStore.getChunksForNotebook(notebookId) as TextChunk[];
export function search(queryEmbedding: number[], sourceCorpusId: string, topK = 10): ScoredChunk[] {
const chunks = sourceCorpusStore.getChunksForSourceCorpus(sourceCorpusId) as TextChunk[];
if (chunks.length === 0) return [];
const scored: ScoredChunk[] = [];

View File

@@ -136,7 +136,7 @@ export async function parseUrl(url: string): Promise<string> {
const res = await fetch(safeUrl, {
headers: {
'User-Agent': 'Mozilla/5.0 (compatible; NotebookClone/1.0)',
'User-Agent': 'Mozilla/5.0 (compatible; SourceSentinel/1.0)',
Accept: 'text/html,application/xhtml+xml,text/plain',
},
signal: AbortSignal.timeout(15000),

View File

@@ -57,7 +57,7 @@ describe('documentCacheStore', () => {
});
describe('invalidate', () => {
it('removes all entries for a notebook', () => {
it('removes all entries for a sourceCorpus', () => {
setCachedDocument('inv-1', 'faq', { a: 1 }, [{ id: 'x' }]);
setCachedDocument('inv-1', 'study-guide', { b: 2 }, [{ id: 'x' }]);
invalidate('inv-1');
@@ -65,7 +65,7 @@ describe('documentCacheStore', () => {
expect(getCachedDocument('inv-1', 'study-guide')).toBeNull();
});
it('does not affect other notebooks', () => {
it('does not affect other sourceCorpora', () => {
setCachedDocument('inv-a', 'faq', { a: 1 }, [{ id: 'x' }]);
setCachedDocument('inv-b', 'faq', { b: 2 }, [{ id: 'x' }]);
invalidate('inv-a');
@@ -86,7 +86,7 @@ describe('documentCacheStore', () => {
expect(isGenerating('gen-cycle')).toBe(false);
});
it('is cleared when the notebook is invalidated', () => {
it('is cleared when the sourceCorpus is invalidated', () => {
markGenerating('gen-inv');
invalidate('gen-inv');
expect(isGenerating('gen-inv')).toBe(false);

View File

@@ -19,19 +19,19 @@ function buildSourceHash(sources: CacheSource[]): string {
.join('|');
}
export function getCachedDocument(notebookId: string, type: string): CacheEntry | null {
const entry = cache.get(`${notebookId}:${type}`);
export function getCachedDocument(sourceCorpusId: string, type: string): CacheEntry | null {
const entry = cache.get(`${sourceCorpusId}:${type}`);
if (!entry || typeof entry === 'boolean') return null;
return entry;
}
export function setCachedDocument(
notebookId: string,
sourceCorpusId: string,
type: string,
document: unknown,
sources: CacheSource[]
): void {
const key = `${notebookId}:${type}`;
const key = `${sourceCorpusId}:${type}`;
cache.set(key, {
document,
sourceHash: buildSourceHash(sources),
@@ -39,29 +39,29 @@ export function setCachedDocument(
});
}
export function isFresh(notebookId: string, type: string, currentSources: CacheSource[]): boolean {
const entry = cache.get(`${notebookId}:${type}`);
export function isFresh(sourceCorpusId: string, type: string, currentSources: CacheSource[]): boolean {
const entry = cache.get(`${sourceCorpusId}:${type}`);
if (!entry || typeof entry === 'boolean') return false;
return entry.sourceHash === buildSourceHash(currentSources);
}
export function invalidate(notebookId: string): void {
export function invalidate(sourceCorpusId: string): void {
for (const key of cache.keys()) {
if (key.startsWith(`${notebookId}:`)) {
if (key.startsWith(`${sourceCorpusId}:`)) {
cache.delete(key);
}
}
}
export function markGenerating(notebookId: string): void {
const key = `${notebookId}:__generating`;
export function markGenerating(sourceCorpusId: string): void {
const key = `${sourceCorpusId}:__generating`;
cache.set(key, true);
}
export function clearGenerating(notebookId: string): void {
cache.delete(`${notebookId}:__generating`);
export function clearGenerating(sourceCorpusId: string): void {
cache.delete(`${sourceCorpusId}:__generating`);
}
export function isGenerating(notebookId: string): boolean {
return cache.get(`${notebookId}:__generating`) === true;
export function isGenerating(sourceCorpusId: string): boolean {
return cache.get(`${sourceCorpusId}:__generating`) === true;
}

View File

@@ -1,65 +0,0 @@
import { describe, it, expect, beforeEach } from 'vitest';
import { NotebookStore } from './notebookStore.js';
describe('notebookStore', () => {
let store: NotebookStore;
beforeEach(() => {
store = new NotebookStore();
});
it('creates and retrieves a notebook', () => {
const nb = store.createNotebook({ id: 'nb-1', name: 'Test', createdAt: '2026-01-01' });
expect(nb.id).toBe('nb-1');
expect(nb.name).toBe('Test');
const found = store.getNotebook('nb-1');
expect(found).not.toBeNull();
expect(found!.name).toBe('Test');
});
it('lists all notebooks without chunks', () => {
store.createNotebook({ id: 'nb-1', name: 'A', createdAt: '2026-01-01' });
store.createNotebook({ id: 'nb-2', name: 'B', createdAt: '2026-01-02' });
const all = store.getAllNotebooks();
expect(all).toHaveLength(2);
expect(all[0]).not.toHaveProperty('chunks');
});
it('deletes a notebook', () => {
store.createNotebook({ id: 'nb-1', name: 'A', createdAt: '2026-01-01' });
expect(store.deleteNotebook('nb-1')).toBe(true);
expect(store.getNotebook('nb-1')).toBeNull();
});
it('returns false when deleting a non-existent notebook', () => {
expect(store.deleteNotebook('nope')).toBe(false);
});
it('adds and retrieves sources', () => {
store.createNotebook({ id: 'nb-1', name: 'A', createdAt: '2026-01-01' });
store.addSource('nb-1', { id: 's-1', name: 'doc.pdf' });
const sources = store.getSources('nb-1');
expect(sources).toHaveLength(1);
expect(sources[0].name).toBe('doc.pdf');
});
it('returns empty sources for non-existent notebook', () => {
expect(store.getSources('nope')).toEqual([]);
});
it('adds and retrieves chunks', () => {
store.createNotebook({ id: 'nb-1', name: 'A', createdAt: '2026-01-01' });
store.addChunksToNotebook('nb-1', [
{ id: 'c-1', text: 'chunk one', sourceId: 's-1', index: 0 },
{ id: 'c-2', text: 'chunk two', sourceId: 's-1', index: 1 },
]);
const chunks = store.getChunksForNotebook('nb-1');
expect(chunks).toHaveLength(2);
expect(chunks[0].text).toBe('chunk one');
});
it('returns empty chunks for non-existent notebook', () => {
expect(store.getChunksForNotebook('nope')).toEqual([]);
});
});

View File

@@ -1,126 +0,0 @@
'use strict';
export interface TextChunk {
id: string;
text: string;
sourceId: string;
index: number;
}
export interface Source {
id: string;
name: string;
mimetype?: string;
chunkCount?: number;
uploadedAt?: string;
}
export interface Notebook {
id: string;
name: string;
createdAt: string;
sources: Source[];
chunks: TextChunk[];
}
export interface NotebookMeta {
id: string;
name: string;
createdAt: string;
}
export interface SourceGroup {
docIndex: number;
sourceId: string;
name: string;
chunks: TextChunk[];
}
export class NotebookStore {
private notebooks = new Map<string, Notebook>();
getAllNotebooks(): NotebookMeta[] {
return Array.from(this.notebooks.values()).map(({ id, name, createdAt }) => ({
id,
name,
createdAt,
}));
}
getNotebook(id: string): Notebook | null {
return this.notebooks.get(id) || null;
}
createNotebook(notebook: { id: string; name: string; createdAt: string }): NotebookMeta {
const record: Notebook = { ...notebook, sources: [], chunks: [] };
this.notebooks.set(record.id, record);
return { id: record.id, name: record.name, createdAt: record.createdAt };
}
deleteNotebook(id: string): boolean {
return this.notebooks.delete(id);
}
addSource(notebookId: string, source: Source): Source | null {
const notebook = this.notebooks.get(notebookId);
if (!notebook) return null;
notebook.sources.push(source);
return source;
}
getSources(notebookId: string): Source[] {
const notebook = this.notebooks.get(notebookId);
if (!notebook) return [];
return notebook.sources;
}
addChunksToNotebook(notebookId: string, chunks: TextChunk[]): Notebook | null {
const notebook = this.notebooks.get(notebookId);
if (!notebook) return null;
notebook.chunks = notebook.chunks.concat(chunks);
return notebook;
}
getChunksForNotebook(notebookId: string): TextChunk[] {
const notebook = this.notebooks.get(notebookId);
if (!notebook) return [];
return notebook.chunks;
}
buildSourceGroups(notebookId: string, chunks: TextChunk[]): SourceGroup[] {
const sources = this.getSources(notebookId);
const sourceIndexMap = new Map<string, number>();
sources.forEach((src, i) => {
sourceIndexMap.set(src.id, i + 1);
});
const groupMap = new Map<string, SourceGroup>();
for (const chunk of chunks) {
const docIndex = sourceIndexMap.get(chunk.sourceId) || 0;
if (!groupMap.has(chunk.sourceId)) {
const src = sources.find((s) => s.id === chunk.sourceId);
groupMap.set(chunk.sourceId, {
docIndex,
sourceId: chunk.sourceId,
name: src?.name || 'Unknown',
chunks: [],
});
}
groupMap.get(chunk.sourceId)!.chunks.push(chunk);
}
return Array.from(groupMap.values()).sort((a, b) => a.docIndex - b.docIndex);
}
}
const defaultStore = new NotebookStore();
export const getAllNotebooks = defaultStore.getAllNotebooks.bind(defaultStore);
export const getNotebook = defaultStore.getNotebook.bind(defaultStore);
export const createNotebook = defaultStore.createNotebook.bind(defaultStore);
export const deleteNotebook = defaultStore.deleteNotebook.bind(defaultStore);
export const addSource = defaultStore.addSource.bind(defaultStore);
export const getSources = defaultStore.getSources.bind(defaultStore);
export const addChunksToNotebook = defaultStore.addChunksToNotebook.bind(defaultStore);
export const getChunksForNotebook = defaultStore.getChunksForNotebook.bind(defaultStore);
export const buildSourceGroups = defaultStore.buildSourceGroups.bind(defaultStore);

View File

@@ -0,0 +1,65 @@
import { describe, it, expect, beforeEach } from 'vitest';
import { SourceCorpusStore } from './sourceCorpusStore.js';
describe('sourceCorpusStore', () => {
let store: SourceCorpusStore;
beforeEach(() => {
store = new SourceCorpusStore();
});
it('creates and retrieves a sourceCorpus', () => {
const corpus = store.createSourceCorpus({ id: 'nb-1', name: 'Test', createdAt: '2026-01-01' });
expect(corpus.id).toBe('nb-1');
expect(corpus.name).toBe('Test');
const found = store.getSourceCorpus('nb-1');
expect(found).not.toBeNull();
expect(found!.name).toBe('Test');
});
it('lists all sourceCorpora without chunks', () => {
store.createSourceCorpus({ id: 'nb-1', name: 'A', createdAt: '2026-01-01' });
store.createSourceCorpus({ id: 'nb-2', name: 'B', createdAt: '2026-01-02' });
const all = store.getAllSourceCorpora();
expect(all).toHaveLength(2);
expect(all[0]).not.toHaveProperty('chunks');
});
it('deletes a sourceCorpus', () => {
store.createSourceCorpus({ id: 'nb-1', name: 'A', createdAt: '2026-01-01' });
expect(store.deleteSourceCorpus('nb-1')).toBe(true);
expect(store.getSourceCorpus('nb-1')).toBeNull();
});
it('returns false when deleting a non-existent sourceCorpus', () => {
expect(store.deleteSourceCorpus('nope')).toBe(false);
});
it('adds and retrieves sources', () => {
store.createSourceCorpus({ id: 'nb-1', name: 'A', createdAt: '2026-01-01' });
store.addSource('nb-1', { id: 's-1', name: 'doc.pdf' });
const sources = store.getSources('nb-1');
expect(sources).toHaveLength(1);
expect(sources[0].name).toBe('doc.pdf');
});
it('returns empty sources for non-existent sourceCorpus', () => {
expect(store.getSources('nope')).toEqual([]);
});
it('adds and retrieves chunks', () => {
store.createSourceCorpus({ id: 'nb-1', name: 'A', createdAt: '2026-01-01' });
store.addChunksToSourceCorpus('nb-1', [
{ id: 'c-1', text: 'chunk one', sourceId: 's-1', index: 0 },
{ id: 'c-2', text: 'chunk two', sourceId: 's-1', index: 1 },
]);
const chunks = store.getChunksForSourceCorpus('nb-1');
expect(chunks).toHaveLength(2);
expect(chunks[0].text).toBe('chunk one');
});
it('returns empty chunks for non-existent sourceCorpus', () => {
expect(store.getChunksForSourceCorpus('nope')).toEqual([]);
});
});

View File

@@ -0,0 +1,126 @@
'use strict';
export interface TextChunk {
id: string;
text: string;
sourceId: string;
index: number;
}
export interface Source {
id: string;
name: string;
mimetype?: string;
chunkCount?: number;
uploadedAt?: string;
}
export interface SourceCorpus {
id: string;
name: string;
createdAt: string;
sources: Source[];
chunks: TextChunk[];
}
export interface SourceCorpusMeta {
id: string;
name: string;
createdAt: string;
}
export interface SourceGroup {
docIndex: number;
sourceId: string;
name: string;
chunks: TextChunk[];
}
export class SourceCorpusStore {
private sourceCorpora = new Map<string, SourceCorpus>();
getAllSourceCorpora(): SourceCorpusMeta[] {
return Array.from(this.sourceCorpora.values()).map(({ id, name, createdAt }) => ({
id,
name,
createdAt,
}));
}
getSourceCorpus(id: string): SourceCorpus | null {
return this.sourceCorpora.get(id) || null;
}
createSourceCorpus(sourceCorpus: { id: string; name: string; createdAt: string }): SourceCorpusMeta {
const record: SourceCorpus = { ...sourceCorpus, sources: [], chunks: [] };
this.sourceCorpora.set(record.id, record);
return { id: record.id, name: record.name, createdAt: record.createdAt };
}
deleteSourceCorpus(id: string): boolean {
return this.sourceCorpora.delete(id);
}
addSource(sourceCorpusId: string, source: Source): Source | null {
const sourceCorpus = this.sourceCorpora.get(sourceCorpusId);
if (!sourceCorpus) return null;
sourceCorpus.sources.push(source);
return source;
}
getSources(sourceCorpusId: string): Source[] {
const sourceCorpus = this.sourceCorpora.get(sourceCorpusId);
if (!sourceCorpus) return [];
return sourceCorpus.sources;
}
addChunksToSourceCorpus(sourceCorpusId: string, chunks: TextChunk[]): SourceCorpus | null {
const sourceCorpus = this.sourceCorpora.get(sourceCorpusId);
if (!sourceCorpus) return null;
sourceCorpus.chunks = sourceCorpus.chunks.concat(chunks);
return sourceCorpus;
}
getChunksForSourceCorpus(sourceCorpusId: string): TextChunk[] {
const sourceCorpus = this.sourceCorpora.get(sourceCorpusId);
if (!sourceCorpus) return [];
return sourceCorpus.chunks;
}
buildSourceGroups(sourceCorpusId: string, chunks: TextChunk[]): SourceGroup[] {
const sources = this.getSources(sourceCorpusId);
const sourceIndexMap = new Map<string, number>();
sources.forEach((src, i) => {
sourceIndexMap.set(src.id, i + 1);
});
const groupMap = new Map<string, SourceGroup>();
for (const chunk of chunks) {
const docIndex = sourceIndexMap.get(chunk.sourceId) || 0;
if (!groupMap.has(chunk.sourceId)) {
const src = sources.find((s) => s.id === chunk.sourceId);
groupMap.set(chunk.sourceId, {
docIndex,
sourceId: chunk.sourceId,
name: src?.name || 'Unknown',
chunks: [],
});
}
groupMap.get(chunk.sourceId)!.chunks.push(chunk);
}
return Array.from(groupMap.values()).sort((a, b) => a.docIndex - b.docIndex);
}
}
const defaultStore = new SourceCorpusStore();
export const getAllSourceCorpora = defaultStore.getAllSourceCorpora.bind(defaultStore);
export const getSourceCorpus = defaultStore.getSourceCorpus.bind(defaultStore);
export const createSourceCorpus = defaultStore.createSourceCorpus.bind(defaultStore);
export const deleteSourceCorpus = defaultStore.deleteSourceCorpus.bind(defaultStore);
export const addSource = defaultStore.addSource.bind(defaultStore);
export const getSources = defaultStore.getSources.bind(defaultStore);
export const addChunksToSourceCorpus = defaultStore.addChunksToSourceCorpus.bind(defaultStore);
export const getChunksForSourceCorpus = defaultStore.getChunksForSourceCorpus.bind(defaultStore);
export const buildSourceGroups = defaultStore.buildSourceGroups.bind(defaultStore);