diff --git a/LLM_DEV_PROMPTS/.dockerignore b/LLM_DEV_PROMPTS/.dockerignore deleted file mode 100644 index 5414d56..0000000 --- a/LLM_DEV_PROMPTS/.dockerignore +++ /dev/null @@ -1,3 +0,0 @@ -.git -node_modules -.DS_Store diff --git a/LLM_DEV_PROMPTS/ANGULAR PROJECTS/ANGULAR_BEST_PRACTICES.md b/LLM_DEV_PROMPTS/ANGULAR PROJECTS/ANGULAR_BEST_PRACTICES.md index 1505d25..8ecf0d6 100644 --- a/LLM_DEV_PROMPTS/ANGULAR PROJECTS/ANGULAR_BEST_PRACTICES.md +++ b/LLM_DEV_PROMPTS/ANGULAR PROJECTS/ANGULAR_BEST_PRACTICES.md @@ -9,31 +9,43 @@ code and move between projects. TypeScript, check out [Google's TypeScript style guide](https://google.github.io/styleguide/tsguide.html). -### Prefer consistency +### When in doubt, prefer consistency -1. If these rules contradict the style of a particular file, prioritize maintaining consistency within a file. Mixing different style conventions in a single file creates more confusion than diverging from the recommendations in this guide. +1. Whenever you encounter a situation in which these rules contradict the style of a particular file, +prioritize maintaining consistency within a file. Mixing different style conventions in a single +file creates more confusion than diverging from the recommendations in this guide. ## Naming ### Separate words in file names with hyphens -1. Separate words within a file name with hyphens (`-`). For example, a component named `UserProfile` has a file name `user-profile.ts`. +1. Separate words within a file name with hyphens (`-`). For example, a component named `UserProfile` +has a file name `user-profile.ts`. ### Use the same name for a file's tests with `.spec` at the end -1. For unit tests, end file names with `.spec.ts`. For example, the unit test file for the `UserProfile` component has the file name `user-profile.spec.ts`. +1. For unit tests, end file names with `.spec.ts`. For example, the unit test file for +the `UserProfile` component has the file name `user-profile.spec.ts`. ### Match file names to the TypeScript identifier within -1. File names should generally describe the contents of the code in the file. When the file contains a TypeScript class, the file name should reflect that class name. For example, a file containing a component named `UserProfile` has the name `user-profile.ts`. +1. File names should generally describe the contents of the code in the file. When the file contains a +TypeScript class, the file name should reflect that class name. For example, a file containing a +component named `UserProfile` has the name `user-profile.ts`. -2. If the file contains more than one primary namable identifier, choose a name that describes the common theme to the code within. If the code in a file does not fit within a common theme or feature area, consider breaking the code up into different files. Do not use overly generic file names like `helpers.ts`, `utils.ts`, or `common.ts`. +2. If the file contains more than one primary namable identifier, choose a name that describes the +common theme to the code within. If the code in a file does not fit within a common theme or feature +area, consider breaking the code up into different files. Do not use overly generic file names +like `helpers.ts`, `utils.ts`, or `common.ts`. ### Use the same file name for a component's TypeScript, template, and styles -1. Components typically consist of one TypeScript file, one template file, and one style file. These files should share the same name with different file extensions. For example, a `UserProfile` component can have the files `user-profile.ts`, `user-profile.html`, and `user-profile.css`. +1. Components typically consist of one TypeScript file, one template file, and one style file. These +files should share the same name with different file extensions. For example, a `UserProfile` +component can have the files `user-profile.ts`, `user-profile.html`, and `user-profile.css`. -2. If a component has more than one style file, append the name with additional words that describe the styles specific to that file. For example, `UserProfile` might have style +2. If a component has more than one style file, append the name with additional words that describe the +styles specific to that file. For example, `UserProfile` might have style files `user-profile-settings.css` and `user-profile-subscription.css`. ## Project structure @@ -44,7 +56,8 @@ files `user-profile-settings.css` and `user-profile-subscription.css`. named `src`. Code that's not related to UI, such as configuration files or scripts, should live outside the `src` directory. -2. This keeps the root application directory consistent between different Angular projects and creates a clear separation between UI code and other code in your project. +2. This keeps the root application directory consistent between different Angular projects and creates +a clear separation between UI code and other code in your project. ### Bootstrap your application in a file named `main.ts` directly inside `src` @@ -53,14 +66,17 @@ named `main.ts`. This represents the primary entry point to the application. ### Group closely related files together in the same directory -1. Angular components consist of a TypeScript file and, optionally, a template and one or more style files. You should group these together in the same directory. +1. Angular components consist of a TypeScript file and, optionally, a template and one or more style +files. You should group these together in the same directory. 2. Unit tests should live in the same directory as the code-under-test. Do not collect unrelated tests into a single `tests` directory. ### Organize your project by feature areas -1. Organize your project into subdirectories based on the features of your application or common themes to the code in those directories. For example, the project structure for a movie theater site, MovieReel, might look like this: +1. Organize your project into subdirectories based on the features of your application or common themes +to the code in those directories. For example, the project structure for a movie theater site, +MovieReel, might look like this: ``` src/ @@ -81,7 +97,10 @@ number of files in a directory grows, consider splitting further into additional ### One concept per file -1. Prefer focusing source files on a single _concept_. For Angular classes specifically, this usually means one component, directive, or service per file. However, it's okay if a file contains more than one component or directive if your classes are relatively small and they tie together as part of a single concept. +1. Prefer focusing source files on a single _concept_. For Angular classes specifically, this usually +means one component, directive, or service per file. However, it's okay if a file contains more than +one component or directive if your classes are relatively small and they tie together as part of a +single concept. 2. When in doubt, go with the approach that leads to smaller files. @@ -102,7 +121,8 @@ number of files in a directory grows, consider splitting further into additional ### Choosing component selectors -1. See the [Components guide for details on choosing component selectors](guide/components/selectors#choosing-a-selector). +1. See +the [Components guide for details on choosing component selectors](guide/components/selectors#choosing-a-selector). ### Naming component and directive members diff --git a/LLM_DEV_PROMPTS/TYPESCRIPT PROJECTS/CODE_STYLEGUIDE_TYPESCRIPT.md b/LLM_DEV_PROMPTS/TYPESCRIPT PROJECTS/CODE_STYLEGUIDE_TYPESCRIPT.md new file mode 100644 index 0000000..2ffea82 --- /dev/null +++ b/LLM_DEV_PROMPTS/TYPESCRIPT PROJECTS/CODE_STYLEGUIDE_TYPESCRIPT.md @@ -0,0 +1,214 @@ +1. General Types + +Don't ever use the types Number, String, Boolean, Symbol, or Object These types refer to non-primitive boxed objects that are almost never used appropriately in JavaScript code. + +/* WRONG */ +function reverse(s: String): String; + +Do use the types number, string, boolean, and symbol. + +/* OK */ +function reverse(s: string): string; + +Instead of Object, use the non-primitive object type. + +2. Use const and let + +JavaScript first searches to see if a variable exists locally, then searches progressively in higher levels of scope until global variables. var is function scope, but, let and const are block scope. + +Using let and const where appropriate makes the intention of the declarations clearer. + +It will also help in identifying issues when a value is reassigned to a constant accidentally by throwing a compile time error. + +Use a linter that automates checking and fixing this so that changing let to const doesn't become a delay in code review. + +3. Use === instead of == + +JavaScript utilizes two different kinds of equality operators: === | !== and == | !=. + +It is considered best practice to always use the former set when comparing. + +If two operands are of the same type and value, then === produces true and !== produces false. + +However, when working with == and !=, we'll run into issues when working with different types. In these cases, they'll try to coerce the values, unsuccessfully. + +4. Use the fastest way to loop arrays + +There are many ways to loop through array. The first way is a for loop. Other ways include the for...of loop, the forEach method for arrays, map, filter, and others. There is also the while loop. + +The for loop is the fastest way. Caching the length makes the loop perform better. Some browser engines have optimized the for loop without manually caching the length property. The forEach is slower than the for loop, so it's probably better to avoid it, especially for large arrays. However, unless we are desperate for performance at the code level (which is rare), make it readable. For example, we can use the for loop in server-side applications and the array methods in client-side applications, because, in general, we don't have expensive operations on the client-side. + +5. Prefer array methods + +It is recommended to use a functional approach without intermediate variables. The base JavaScript for loop can be more performant in some browsers but the benefit can be measured only by iterating over millions of items. It is the job of compiler and runtime to remove the penalty of using new array methods. + +6. Do not trust any data - Validate + +Make sure that all the data that goes into our system is clean and exactly what we need. This is most important on the back end when writing out parameters retrieved from the URL. + +The same applies to forms that validate only on the client side. Another very insecure practice is to read information from the DOM and use it without validation. + +7. Generics + +Don't ever have a generic type which doesn't use its type parameter. + +8. Any + +Don't use any as a type unless you are in the process of migrating a JavaScript project to TypeScript. The compiler effectively treats any as "please turn off type checking for this thing". It is similar to putting an @ts-ignore comment around every usage of the variable. This can be very helpful when you are first migrating a JavaScript project to TypeScript as you can set the type for stuff you haven't migrated yet as any, but in a full TypeScript project you are disabling type checking for any parts of your program that use it. +In cases where you don't know what type you want to accept, or when you want to accept anything because you will be blindly passing it through without interacting with it, you can use unknown. + +9. Strict configuration + +The stricter configuration is mandatory. Otherwise, types will be too permissive, and it is what we are trying to avoid as much as possible with Typescript. + +{ + "forceConsistentCasingInFileNames": true, + "noImplicitReturns": true, + "strict": true, + "noUnusedLocals": true, +} + +The most important one here is the strict flag which actually covers four other flags: noImplicitAny, noImplicitThis, alwaysStrict and strictNullChecks. + +10. Callback Types - Return Types of Callbacks + +Don't use the return type any for callbacks whose value will be ignored: +/* WRONG */ +function fn(x: () => any) { + x(); +} + +Do use the return type void for callbacks whose value will be ignored: +/* OK */ +function fn(x: () => void) { + x(); +} + +Why: Using void is safer because it prevents you from accidentally using the return value of x in an unchecked way: +function fn(x: () => void) { + const k = x(); // oops! meant to do something else + k.doSomething(); // error, but would be OK if the return type had been 'any' +} + +11. Optional Parameters in Callbacks + +Don't use optional parameters in callbacks unless you really mean it: + +/* WRONG */ +interface Fetcher { + getObject(done: (data: unknown, elapsedTime?: number) => void): void; +} + +This has a very specific meaning: the done callback might be invoked with 1 argument or might be invoked with 2 arguments. The author probably intended to say that the callback might not care about the elapsedTime parameter, but there's no need to make the parameter optional to accomplish this — it's always legal to provide a callback that accepts fewer arguments. + +Do write callback parameters as non-optional: +/* OK */ +interface Fetcher { + getObject(done: (data: unknown, elapsedTime: number) => void): void; +} + +12. Overloads and Callbacks + +Don't write separate overloads that differ only on callback arity: + +/* WRONG */ +declare function beforeAll(action: () => void, timeout?: number): void; +declare function beforeAll( + action: (done: DoneFn) => void, + timeout?: number +): void; + +Do write a single overload using the maximum arity: + +/* OK */ +declare function beforeAll( + action: (done: DoneFn) => void, + timeout?: number +): void; + +Why: It's always legal for a callback to disregard a parameter, so there's no need for the shorter overload. Providing a shorter callback first allows incorrectly-typed functions to be passed in because they match the first overload. + +13. Function Overloads + +13.1 Ordering + +Don't put more general overloads before more specific overloads: +/* WRONG */ +declare function fn(x: unknown): unknown; +declare function fn(x: HTMLElement): number; +declare function fn(x: HTMLDivElement): string; +const myElem: HTMLDivElement; +const x = fn(myElem); // x: unknown, wat? + +Do sort overloads by putting the more general signatures after more specific signatures: + +/* OK */ +declare function fn(x: HTMLDivElement): string; +declare function fn(x: HTMLElement): number; +declare function fn(x: unknown): unknown; +const myElem: HTMLDivElement; +const x = fn(myElem); // x: string, :) + +Why: TypeScript chooses the first matching overload when resolving function calls. When an earlier overload is "more general" than a later one, the later one is effectively hidden and cannot be called. + +13.2 Use Optional Parameters + +Don't write several overloads that differ only in trailing parameters: + +/* WRONG */ +interface Example { + diff(one: string): number; + diff(one: string, two: string): number; + diff(one: string, two: string, three: boolean): number; +} + +Do use optional parameters whenever possible: + +/* OK */ +interface Example { + diff(one: string, two?: string, three?: boolean): number; +} + +Note that this collapsing should only occur when all overloads have the same return type. + +Why: This is important for two reasons. +TypeScript resolves signature compatibility by seeing if any signature of the target can be invoked with the arguments of the source, and extraneous arguments are allowed. This code, for example, exposes a bug only when the signature is correctly written using optional parameters: +function fn(x: (a: string, b: number, c: number) => void) {} +const example: Example; +// When written with overloads, OK -- used first overload +// When written with optionals, correctly an error +fn(example.diff); + +The second reason is when a consumer uses the "strict null checking" feature of TypeScript. + +Because unspecified parameters appear as undefined in JavaScript, it's usually fine to pass an explicit undefined to a function with optional arguments. This code, for example, should be OK under strict nulls: +const example2: Example; +// When written with overloads, incorrectly an error because of passing 'undefined' to 'string' +// When written with optionals, correctly OK +example2.diff("something", true ? undefined : "hour"); + +13. Use Union Types + +Don't write overloads that differ by type in only one argument position: +/* WRONG */ +interface Moment { + utcOffset(): number; + utcOffset(b: number): Moment; + utcOffset(b: string): Moment; +} + +Do use union types whenever possible: +/* OK */ +interface Moment { + utcOffset(): number; + utcOffset(b: number | string): Moment; +} +Note that we didn't make b optional here because the return types of the signatures differ. +❔ Why: This is important for people who are "passing through" a value to your function: +function fn(x: string): Moment; +function fn(x: number): Moment; +function fn(x: number | string) { + // When written with separate overloads, incorrectly an error + // When written with union types, correctly OK + return moment().utcOffset(x); +} diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 0000000..39d66d6 --- /dev/null +++ b/package-lock.json @@ -0,0 +1,6 @@ +{ + "name": "citation_sentinel", + "lockfileVersion": 3, + "requires": true, + "packages": {} +} diff --git a/server/package-lock.json b/server/package-lock.json index 8e797a6..a13358a 100644 --- a/server/package-lock.json +++ b/server/package-lock.json @@ -22,8 +22,11 @@ "uuid": "^11.1.0" }, "devDependencies": { + "@types/node": "^25.6.2", + "@types/pdf-parse": "^1.1.5", "eslint": "^9.22.0", "prettier": "^3.5.3", + "typescript": "^6.0.3", "vitest": "^3.0.0" }, "engines": { @@ -1092,6 +1095,27 @@ "dev": true, "license": "MIT" }, + "node_modules/@types/node": { + "version": "25.6.2", + "resolved": "https://registry.npmjs.org/@types/node/-/node-25.6.2.tgz", + "integrity": "sha512-sokuT28dxf9JT5Kady1fsXOvI4HVpjZa95NKT5y9PNTIrs2AsobR4GFAA90ZG8M+nxVRLysCXsVj6eGC7Vbrlw==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "undici-types": "~7.19.0" + } + }, + "node_modules/@types/pdf-parse": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@types/pdf-parse/-/pdf-parse-1.1.5.tgz", + "integrity": "sha512-kBfrSXsloMnUJOKi25s3+hRmkycHfLK6A09eRGqF/N8BkQoPUmaCr+q8Cli5FnfohEz/rsv82zAiPz/LXtOGhA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@vitest/expect": { "version": "3.2.4", "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-3.2.4.tgz", @@ -4180,6 +4204,20 @@ "integrity": "sha512-/aCDEGatGvZ2BIk+HmLf4ifCJFwvKFNb9/JeZPMulfgFracn9QFcAf5GO8B/mweUjSoblS5In0cWhqpfs/5PQA==", "license": "MIT" }, + "node_modules/typescript": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-6.0.3.tgz", + "integrity": "sha512-y2TvuxSZPDyQakkFRPZHKFm+KKVqIisdg9/CZwm9ftvKXLP8NRWj38/ODjNbr43SsoXqNuAisEf1GdCxqWcdBw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, "node_modules/underscore": { "version": "1.13.8", "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.13.8.tgz", @@ -4195,6 +4233,13 @@ "node": ">=20.18.1" } }, + "node_modules/undici-types": { + "version": "7.19.2", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.19.2.tgz", + "integrity": "sha512-qYVnV5OEm2AW8cJMCpdV20CDyaN3g0AjDlOGf1OW4iaDEx8MwdtChUp4zu4H0VP3nDRF/8RKWH+IPp9uW0YGZg==", + "dev": true, + "license": "MIT" + }, "node_modules/unpipe": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", diff --git a/server/package.json b/server/package.json index 309b1c1..9412a40 100644 --- a/server/package.json +++ b/server/package.json @@ -2,7 +2,9 @@ "name": "notebook-clone-server", "version": "0.1.0", "private": true, - "engines": { "node": ">=18" }, + "engines": { + "node": ">=18" + }, "type": "module", "scripts": { "start": "node src/index.js", @@ -26,8 +28,11 @@ "uuid": "^11.1.0" }, "devDependencies": { + "@types/node": "^25.6.2", + "@types/pdf-parse": "^1.1.5", "eslint": "^9.22.0", "prettier": "^3.5.3", + "typescript": "^6.0.3", "vitest": "^3.0.0" } } diff --git a/server/src/services/documentService.test.js b/server/src/services/documentService.test.ts similarity index 77% rename from server/src/services/documentService.test.js rename to server/src/services/documentService.test.ts index fbb2808..7af8bd1 100644 --- a/server/src/services/documentService.test.js +++ b/server/src/services/documentService.test.ts @@ -1,4 +1,4 @@ -import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; +import { describe, it, expect, vi, beforeEach, afterEach, type Mock } from 'vitest'; const { mockCreate } = vi.hoisted(() => ({ mockCreate: vi.fn() })); @@ -14,14 +14,18 @@ import { generateStudyGuide, generateFaq, generateExecutiveBrief, + type SourceGroup, + type StudyGuide, + type Faq, + type ExecutiveBrief, } from './documentService.js'; -const SOURCE_GROUPS = [ +const SOURCE_GROUPS: SourceGroup[] = [ { docIndex: 1, name: 'Doc A', chunks: [{ text: 'Alpha content.' }] }, { docIndex: 2, name: 'Doc B', chunks: [{ text: 'Beta first.' }, { text: 'Beta second.' }] }, ]; -function apiResponse(obj) { +function apiResponse(obj: T): { content: Array<{ text: string }> } { return { content: [{ text: JSON.stringify(obj) }] }; } @@ -35,7 +39,7 @@ afterEach(() => { }); describe('generateStudyGuide', () => { - const STUDY_GUIDE = { + const STUDY_GUIDE: StudyGuide = { title: 'Test Guide', sections: [{ heading: 'Intro', bullets: ['b1'], keyTerms: [], reviewQuestions: [] }], mnemonics: ['ABC'], @@ -51,7 +55,7 @@ describe('generateStudyGuide', () => { mockCreate.mockResolvedValue(apiResponse(STUDY_GUIDE)); await generateStudyGuide(SOURCE_GROUPS); - const prompt = mockCreate.mock.calls[0][0].messages[0].content; + const prompt = (mockCreate as Mock).mock.calls[0][0].messages[0].content as string; expect(prompt).toContain('[Source 1] (Doc A)'); expect(prompt).toContain('Alpha content.'); expect(prompt).toContain('[Source 2] (Doc B)'); @@ -63,13 +67,13 @@ describe('generateStudyGuide', () => { mockCreate.mockResolvedValue(apiResponse(STUDY_GUIDE)); await generateStudyGuide(SOURCE_GROUPS); - const prompt = mockCreate.mock.calls[0][0].messages[0].content; + const prompt = (mockCreate as Mock).mock.calls[0][0].messages[0].content as string; expect(prompt).toContain('---'); }); }); describe('generateFaq', () => { - const FAQ = { + const FAQ: Faq = { subject: 'Testing', faqPairs: [{ question: 'Q?', answer: 'A.' }], }; @@ -84,14 +88,14 @@ describe('generateFaq', () => { mockCreate.mockResolvedValue(apiResponse(FAQ)); await generateFaq(SOURCE_GROUPS); - const prompt = mockCreate.mock.calls[0][0].messages[0].content; + const prompt = (mockCreate as Mock).mock.calls[0][0].messages[0].content as string; expect(prompt).toContain('[Source 1] (Doc A)'); expect(prompt).toContain('[Source 2] (Doc B)'); }); }); describe('generateExecutiveBrief', () => { - const BRIEF = { + const BRIEF: ExecutiveBrief = { title: 'Exec Brief', sections: [{ subhead: 'Overview', prose: 'Some prose.' }], }; @@ -106,7 +110,7 @@ describe('generateExecutiveBrief', () => { mockCreate.mockResolvedValue(apiResponse(BRIEF)); await generateExecutiveBrief(SOURCE_GROUPS); - const prompt = mockCreate.mock.calls[0][0].messages[0].content; + const prompt = (mockCreate as Mock).mock.calls[0][0].messages[0].content as string; expect(prompt).toContain('[Source 1] (Doc A)'); expect(prompt).toContain('[Source 2] (Doc B)'); }); @@ -126,31 +130,25 @@ describe('shared behaviour', () => { it('truncates sources that exceed the per-source character budget', async () => { const longText = 'x'.repeat(35_000); - const bigGroups = [ - { docIndex: 1, name: 'Big', chunks: [{ text: longText }] }, - ]; - mockCreate.mockResolvedValue( - apiResponse({ title: 't', sections: [], mnemonics: [] }), - ); + const bigGroups: SourceGroup[] = [{ docIndex: 1, name: 'Big', chunks: [{ text: longText }] }]; + mockCreate.mockResolvedValue(apiResponse({ title: 't', sections: [], mnemonics: [] })); await generateStudyGuide(bigGroups); - const prompt = mockCreate.mock.calls[0][0].messages[0].content; + const prompt = (mockCreate as Mock).mock.calls[0][0].messages[0].content as string; expect(prompt).toContain('[...truncated]'); expect(prompt.length).toBeLessThan(longText.length); }); it('combines multiple chunks within a source with double newlines', async () => { - const groups = [ + const groups: SourceGroup[] = [ { docIndex: 1, name: 'Multi', chunks: [{ text: 'AAA' }, { text: 'BBB' }] }, ]; - mockCreate.mockResolvedValue( - apiResponse({ title: 't', sections: [], mnemonics: [] }), - ); + mockCreate.mockResolvedValue(apiResponse({ title: 't', sections: [], mnemonics: [] })); await generateStudyGuide(groups); - const prompt = mockCreate.mock.calls[0][0].messages[0].content; + const prompt = (mockCreate as Mock).mock.calls[0][0].messages[0].content as string; expect(prompt).toContain('AAA\n\nBBB'); }); }); diff --git a/server/src/services/documentService.js b/server/src/services/documentService.ts similarity index 76% rename from server/src/services/documentService.js rename to server/src/services/documentService.ts index 78bd03c..01d13cd 100644 --- a/server/src/services/documentService.js +++ b/server/src/services/documentService.ts @@ -5,7 +5,7 @@ import logger from '../logger.js'; const MODEL = 'claude-sonnet-4-5-20250929'; -function getClient() { +function getClient(): Anthropic { const key = process.env.ANTHROPIC_API_KEY; if (!key) throw new Error('ANTHROPIC_API_KEY is not set'); return new Anthropic({ apiKey: key }); @@ -13,7 +13,17 @@ function getClient() { const MAX_SOURCE_CHARS = 30000; -function buildSourceBlock(sourceGroups) { +export interface SourceChunk { + text: string; +} + +export interface SourceGroup { + docIndex: number; + name: string; + chunks: SourceChunk[]; +} + +function buildSourceBlock(sourceGroups: SourceGroup[]): string { const perSourceBudget = Math.floor(MAX_SOURCE_CHARS / (sourceGroups.length || 1)); return sourceGroups @@ -51,7 +61,11 @@ const STUDY_GUIDE_SCHEMA = { additionalProperties: false, }, }, - reviewQuestions: { type: 'array', items: { type: 'string' }, description: 'Self-test questions for this section' }, + reviewQuestions: { + type: 'array', + items: { type: 'string' }, + description: 'Self-test questions for this section', + }, }, required: ['heading', 'bullets', 'keyTerms', 'reviewQuestions'], additionalProperties: false, @@ -65,9 +79,27 @@ const STUDY_GUIDE_SCHEMA = { }, required: ['title', 'sections', 'mnemonics'], additionalProperties: false, -}; +} as const; -export async function generateStudyGuide(sourceGroups) { +export interface KeyTerm { + term: string; + definition: string; +} + +export interface StudyGuideSection { + heading: string; + bullets: string[]; + keyTerms: KeyTerm[]; + reviewQuestions: string[]; +} + +export interface StudyGuide { + title: string; + sections: StudyGuideSection[]; + mnemonics: string[]; +} + +export async function generateStudyGuide(sourceGroups: SourceGroup[]): Promise { const client = getClient(); const sources = buildSourceBlock(sourceGroups); const start = Date.now(); @@ -90,7 +122,9 @@ ${sources}`; output_config: { format: { type: 'json_schema', schema: STUDY_GUIDE_SCHEMA } }, }); - const parsed = JSON.parse(message.content[0]?.text || '{}'); + const textBlock = message.content[0]; + const raw = textBlock && 'text' in textBlock && textBlock.text ? textBlock.text : '{}'; + const parsed = JSON.parse(raw) as StudyGuide; const elapsed = ((Date.now() - start) / 1000).toFixed(1); logger.info({ sections: parsed.sections?.length, elapsedSec: elapsed }, 'study guide generated'); return parsed; @@ -116,9 +150,19 @@ const FAQ_SCHEMA = { }, required: ['subject', 'faqPairs'], additionalProperties: false, -}; +} as const; -export async function generateFaq(sourceGroups) { +export interface FaqPair { + question: string; + answer: string; +} + +export interface Faq { + subject: string; + faqPairs: FaqPair[]; +} + +export async function generateFaq(sourceGroups: SourceGroup[]): Promise { const client = getClient(); const sources = buildSourceBlock(sourceGroups); const start = Date.now(); @@ -141,7 +185,9 @@ ${sources}`; output_config: { format: { type: 'json_schema', schema: FAQ_SCHEMA } }, }); - const parsed = JSON.parse(message.content[0]?.text || '{}'); + const textBlock = message.content[0]; + const raw = textBlock && 'text' in textBlock && textBlock.text ? textBlock.text : '{}'; + const parsed = JSON.parse(raw) as Faq; const elapsed = ((Date.now() - start) / 1000).toFixed(1); logger.info({ subject: parsed.subject, pairs: parsed.faqPairs?.length, elapsedSec: elapsed }, 'FAQ generated'); return parsed; @@ -166,9 +212,19 @@ const EXEC_BRIEF_SCHEMA = { }, required: ['title', 'sections'], additionalProperties: false, -}; +} as const; -export async function generateExecutiveBrief(sourceGroups) { +export interface ExecutiveBriefSection { + subhead: string; + prose: string; +} + +export interface ExecutiveBrief { + title: string; + sections: ExecutiveBriefSection[]; +} + +export async function generateExecutiveBrief(sourceGroups: SourceGroup[]): Promise { const client = getClient(); const sources = buildSourceBlock(sourceGroups); const start = Date.now(); @@ -192,8 +248,13 @@ ${sources}`; output_config: { format: { type: 'json_schema', schema: EXEC_BRIEF_SCHEMA } }, }); - const parsed = JSON.parse(message.content[0]?.text || '{}'); + const textBlock = message.content[0]; + const raw = textBlock && 'text' in textBlock && textBlock.text ? textBlock.text : '{}'; + const parsed = JSON.parse(raw) as ExecutiveBrief; const elapsed = ((Date.now() - start) / 1000).toFixed(1); - logger.info({ title: parsed.title, sections: parsed.sections?.length, elapsedSec: elapsed }, 'executive brief generated'); + logger.info( + { title: parsed.title, sections: parsed.sections?.length, elapsedSec: elapsed }, + 'executive brief generated' + ); return parsed; } diff --git a/server/src/services/generationService.test.js b/server/src/services/generationService.test.ts similarity index 91% rename from server/src/services/generationService.test.js rename to server/src/services/generationService.test.ts index 6854513..2cd9a2e 100644 --- a/server/src/services/generationService.test.js +++ b/server/src/services/generationService.test.ts @@ -1,8 +1,8 @@ import { describe, it, expect } from 'vitest'; -import { buildPrompt } from './generationService.js'; +import { buildPrompt, type SourceGroup } from './generationService.js'; describe('buildPrompt', () => { - const sourceGroups = [ + const sourceGroups: SourceGroup[] = [ { docIndex: 1, name: 'Weather Facts', @@ -11,10 +11,7 @@ describe('buildPrompt', () => { { docIndex: 2, name: 'Water Science', - chunks: [ - { text: 'Water is wet.' }, - { text: 'Water boils at 100°C.' }, - ], + chunks: [{ text: 'Water is wet.' }, { text: 'Water boils at 100°C.' }], }, ]; diff --git a/server/src/services/generationService.js b/server/src/services/generationService.ts similarity index 73% rename from server/src/services/generationService.js rename to server/src/services/generationService.ts index bec2f85..be32122 100644 --- a/server/src/services/generationService.js +++ b/server/src/services/generationService.ts @@ -26,15 +26,50 @@ const RESPONSE_SCHEMA = { }, required: ['answer', 'citedSourceIndices', 'followUpQuestions'], additionalProperties: false, -}; +} as const; -function getClient() { +export interface SourceChunk { + text: string; +} + +export interface SourceGroup { + docIndex: number; + name: string; + chunks: SourceChunk[]; +} + +export interface GenerationResult { + answer: string; + citedSourceIndices: number[]; + followUpQuestions: string[]; +} + +export interface CitationDetailParams { + chunkTexts: string[]; + sourceName: string; + answer: string; + citationIndex: number; +} + +export interface RelevantLink { + title: string; + url: string; +} + +export interface CitationDetail { + citedSentence: string; + topicSummary: string; + additionalInsight: string; + relevantLinks: RelevantLink[]; +} + +function getClient(): Anthropic { const key = process.env.ANTHROPIC_API_KEY; if (!key) throw new Error('ANTHROPIC_API_KEY is not set'); return new Anthropic({ apiKey: key }); } -export function buildPrompt(query, sourceGroups) { +export function buildPrompt(query: string, sourceGroups: SourceGroup[]): string { const today = new Date().toLocaleDateString('en-US', { weekday: 'long', year: 'numeric', @@ -64,15 +99,13 @@ ${sources} ${query}`; } -export async function generate(query, sourceGroups) { +export async function generate(query: string, sourceGroups: SourceGroup[]): Promise { const client = getClient(); const message = await client.messages.create({ model: MODEL, max_tokens: 2048, - messages: [ - { role: 'user', content: buildPrompt(query, sourceGroups) }, - ], + messages: [{ role: 'user', content: buildPrompt(query, sourceGroups) }], output_config: { format: { type: 'json_schema', @@ -81,16 +114,19 @@ export async function generate(query, sourceGroups) { }, }); - const raw = message.content[0]?.text || ''; + const textBlock = message.content[0]; + const raw = textBlock && 'text' in textBlock ? textBlock.text : ''; logger.debug({ rawLength: raw.length }, 'claude response received'); - const parsed = JSON.parse(raw); + const parsed = JSON.parse(raw) as { + answer?: string; + citedSourceIndices?: number[]; + followUpQuestions?: string[]; + }; const validIndices = new Set(sourceGroups.map((g) => g.docIndex)); const citedSourceIndices = [ - ...new Set( - (parsed.citedSourceIndices || []).filter((idx) => validIndices.has(idx)), - ), + ...new Set((parsed.citedSourceIndices || []).filter((idx) => validIndices.has(idx))), ]; return { @@ -113,7 +149,8 @@ const CITATION_DETAIL_SCHEMA = { }, additionalInsight: { type: 'string', - description: 'Additional expert insight, analysis, or context on the subject beyond what the source states', + description: + 'Additional expert insight, analysis, or context on the subject beyond what the source states', }, relevantLinks: { type: 'array', @@ -131,9 +168,14 @@ const CITATION_DETAIL_SCHEMA = { }, required: ['citedSentence', 'topicSummary', 'additionalInsight', 'relevantLinks'], additionalProperties: false, -}; +} as const; -export async function generateCitationDetail({ chunkTexts, sourceName, answer, citationIndex }) { +export async function generateCitationDetail({ + chunkTexts, + sourceName, + answer, + citationIndex, +}: CitationDetailParams): Promise { const client = getClient(); const sourceText = chunkTexts.join('\n\n'); @@ -166,8 +208,9 @@ ${sourceText}`; }, }); - const raw = message.content[0]?.text || ''; + const textBlock = message.content[0]; + const raw = textBlock && 'text' in textBlock ? textBlock.text : ''; logger.debug({ citationIndex, rawLength: raw.length }, 'citation detail response received'); - return JSON.parse(raw); + return JSON.parse(raw) as CitationDetail; } diff --git a/server/src/services/preGenerationService.test.js b/server/src/services/preGenerationService.test.js deleted file mode 100644 index e3997a9..0000000 --- a/server/src/services/preGenerationService.test.js +++ /dev/null @@ -1,216 +0,0 @@ -import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; - -vi.mock('../stores/notebookStore.js', () => ({ - getNotebook: vi.fn(), - getSources: vi.fn(), - getChunksForNotebook: vi.fn(), - buildSourceGroups: vi.fn(), -})); - -vi.mock('../stores/documentCacheStore.js', () => ({ - isGenerating: vi.fn(), - invalidate: vi.fn(), - markGenerating: vi.fn(), - clearGenerating: vi.fn(), - setCachedDocument: vi.fn(), -})); - -vi.mock('./documentService.js', () => ({ - generateStudyGuide: vi.fn(), - generateFaq: vi.fn(), - generateExecutiveBrief: vi.fn(), -})); - -vi.mock('../logger.js', () => ({ - default: { debug: vi.fn(), info: vi.fn(), warn: vi.fn(), error: vi.fn() }, -})); - -import { triggerPreGeneration } from './preGenerationService.js'; -import * as notebookStore from '../stores/notebookStore.js'; -import * as documentCacheStore from '../stores/documentCacheStore.js'; -import { - generateStudyGuide, - generateFaq, - generateExecutiveBrief, -} from './documentService.js'; - -function setupNotebook(id) { - notebookStore.getNotebook.mockReturnValue({ id }); - notebookStore.getSources.mockReturnValue([{ id: 's1' }, { id: 's2' }]); - notebookStore.getChunksForNotebook.mockReturnValue([{ id: 'c1', text: 'hello' }]); - notebookStore.buildSourceGroups.mockReturnValue([ - { docIndex: 1, name: 'Doc', chunks: [{ text: 'hello' }] }, - ]); -} - -function stubGeneratorsOk() { - generateStudyGuide.mockResolvedValue({ title: 'guide' }); - generateFaq.mockResolvedValue({ subject: 'topic' }); - generateExecutiveBrief.mockResolvedValue({ title: 'brief' }); -} - -beforeEach(() => { - vi.clearAllMocks(); - vi.useFakeTimers(); -}); - -afterEach(() => { - vi.useRealTimers(); -}); - -describe('triggerPreGeneration guards', () => { - it('does nothing if the notebook does not exist', () => { - notebookStore.getNotebook.mockReturnValue(null); - triggerPreGeneration('nb-missing'); - expect(notebookStore.getSources).not.toHaveBeenCalled(); - }); - - it('does nothing if fewer than 2 sources', () => { - notebookStore.getNotebook.mockReturnValue({ id: 'nb-few' }); - notebookStore.getSources.mockReturnValue([{ id: 's1' }]); - triggerPreGeneration('nb-few'); - expect(documentCacheStore.markGenerating).not.toHaveBeenCalled(); - }); -}); - -describe('queuing when already generating', () => { - it('invalidates cache and skips runPreGeneration', () => { - setupNotebook('nb-q'); - documentCacheStore.isGenerating.mockReturnValue(true); - - triggerPreGeneration('nb-q'); - - expect(documentCacheStore.invalidate).toHaveBeenCalledWith('nb-q'); - expect(documentCacheStore.markGenerating).not.toHaveBeenCalled(); - }); -}); - -describe('debounce', () => { - it('does not run generation immediately', () => { - setupNotebook('nb-debounce'); - documentCacheStore.isGenerating.mockReturnValue(false); - stubGeneratorsOk(); - - triggerPreGeneration('nb-debounce'); - - expect(documentCacheStore.markGenerating).not.toHaveBeenCalled(); - }); - - it('runs generation after the debounce delay', async () => { - setupNotebook('nb-delay'); - documentCacheStore.isGenerating.mockReturnValue(false); - stubGeneratorsOk(); - - triggerPreGeneration('nb-delay'); - await vi.advanceTimersByTimeAsync(5000); - - await vi.waitFor(() => { - expect(documentCacheStore.clearGenerating).toHaveBeenCalledWith('nb-delay'); - }); - - expect(documentCacheStore.markGenerating).toHaveBeenCalledWith('nb-delay'); - }); - - it('resets the timer on repeated calls, running generation only once', async () => { - setupNotebook('nb-batch'); - documentCacheStore.isGenerating.mockReturnValue(false); - stubGeneratorsOk(); - - triggerPreGeneration('nb-batch'); - await vi.advanceTimersByTimeAsync(3000); - triggerPreGeneration('nb-batch'); - await vi.advanceTimersByTimeAsync(3000); - triggerPreGeneration('nb-batch'); - await vi.advanceTimersByTimeAsync(5000); - - await vi.waitFor(() => { - expect(documentCacheStore.clearGenerating).toHaveBeenCalled(); - }); - - expect(documentCacheStore.markGenerating).toHaveBeenCalledTimes(1); - }); -}); - -describe('runPreGeneration (via triggerPreGeneration)', () => { - it('runs all three generators and caches results', async () => { - setupNotebook('nb-happy'); - documentCacheStore.isGenerating.mockReturnValue(false); - stubGeneratorsOk(); - - triggerPreGeneration('nb-happy'); - await vi.advanceTimersByTimeAsync(5000); - - await vi.waitFor(() => { - expect(documentCacheStore.clearGenerating).toHaveBeenCalledWith('nb-happy'); - }); - - expect(generateStudyGuide).toHaveBeenCalled(); - expect(generateFaq).toHaveBeenCalled(); - expect(generateExecutiveBrief).toHaveBeenCalled(); - expect(documentCacheStore.setCachedDocument).toHaveBeenCalledTimes(3); - }); - - it('marks generating before starting and clears it after', async () => { - setupNotebook('nb-flag'); - documentCacheStore.isGenerating.mockReturnValue(false); - stubGeneratorsOk(); - - triggerPreGeneration('nb-flag'); - await vi.advanceTimersByTimeAsync(5000); - - expect(documentCacheStore.markGenerating).toHaveBeenCalledWith('nb-flag'); - - await vi.waitFor(() => { - expect(documentCacheStore.clearGenerating).toHaveBeenCalledWith('nb-flag'); - }); - }); - - it('still clears generating flag when a generator fails', async () => { - setupNotebook('nb-partial'); - documentCacheStore.isGenerating.mockReturnValue(false); - generateStudyGuide.mockRejectedValue(new Error('boom')); - generateFaq.mockResolvedValue({ subject: 'ok' }); - generateExecutiveBrief.mockResolvedValue({ title: 'ok' }); - - triggerPreGeneration('nb-partial'); - await vi.advanceTimersByTimeAsync(5000); - - await vi.waitFor(() => { - expect(documentCacheStore.clearGenerating).toHaveBeenCalledWith('nb-partial'); - }); - - expect(documentCacheStore.setCachedDocument).toHaveBeenCalledTimes(2); - }); -}); - -describe('re-trigger for queued notebooks', () => { - it('runs a second generation cycle after the first finishes', async () => { - setupNotebook('nb-re'); - documentCacheStore.isGenerating - .mockReturnValueOnce(false) // first trigger → debounced - .mockReturnValueOnce(true); // second trigger → queues - - let resolveFirst; - generateStudyGuide - .mockImplementationOnce(() => new Promise((r) => { resolveFirst = r; })) - .mockResolvedValue({ title: 'guide' }); - generateFaq.mockResolvedValue({ subject: 'topic' }); - generateExecutiveBrief.mockResolvedValue({ title: 'brief' }); - - triggerPreGeneration('nb-re'); - await vi.advanceTimersByTimeAsync(5000); - // First run is in-flight, blocked on generateStudyGuide - - triggerPreGeneration('nb-re'); - // Queued via pendingReGen since isGenerating returns true - - resolveFirst({ title: 'guide' }); - - await vi.waitFor(() => { - expect(documentCacheStore.markGenerating).toHaveBeenCalledTimes(2); - }); - - expect(documentCacheStore.clearGenerating).toHaveBeenCalledTimes(2); - expect(documentCacheStore.setCachedDocument).toHaveBeenCalledTimes(6); - }); -}); diff --git a/server/src/services/preGenerationService.test.ts b/server/src/services/preGenerationService.test.ts new file mode 100644 index 0000000..f7963ae --- /dev/null +++ b/server/src/services/preGenerationService.test.ts @@ -0,0 +1,224 @@ +import { describe, it, expect, vi, beforeEach, afterEach, type Mock } from 'vitest'; + +vi.mock('../stores/notebookStore.js', () => ({ + getNotebook: vi.fn(), + getSources: vi.fn(), + getChunksForNotebook: vi.fn(), + buildSourceGroups: vi.fn(), +})); + +vi.mock('../stores/documentCacheStore.js', () => ({ + isGenerating: vi.fn(), + invalidate: vi.fn(), + markGenerating: vi.fn(), + clearGenerating: vi.fn(), + setCachedDocument: vi.fn(), +})); + +vi.mock('./documentService.js', () => ({ + generateStudyGuide: vi.fn(), + generateFaq: vi.fn(), + generateExecutiveBrief: vi.fn(), +})); + +vi.mock('../logger.js', () => ({ + default: { debug: vi.fn(), info: vi.fn(), warn: vi.fn(), error: vi.fn() }, +})); + +import { triggerPreGeneration } from './preGenerationService.js'; +import * as notebookStore from '../stores/notebookStore.js'; +import * as documentCacheStore from '../stores/documentCacheStore.js'; +import { generateStudyGuide, generateFaq, generateExecutiveBrief } from './documentService.js'; + +const mockedGetNotebook = notebookStore.getNotebook as Mock; +const mockedGetSources = notebookStore.getSources as Mock; +const mockedGetChunksForNotebook = notebookStore.getChunksForNotebook as Mock; +const mockedBuildSourceGroups = notebookStore.buildSourceGroups as Mock; +const mockedIsGenerating = documentCacheStore.isGenerating as Mock; +const mockedInvalidate = documentCacheStore.invalidate as Mock; +const mockedMarkGenerating = documentCacheStore.markGenerating as Mock; +const mockedClearGenerating = documentCacheStore.clearGenerating as Mock; +const mockedSetCachedDocument = documentCacheStore.setCachedDocument as Mock; +const mockedGenerateStudyGuide = generateStudyGuide as Mock; +const mockedGenerateFaq = generateFaq as Mock; +const mockedGenerateExecutiveBrief = generateExecutiveBrief as Mock; + +function setupNotebook(id: string): void { + mockedGetNotebook.mockReturnValue({ id }); + mockedGetSources.mockReturnValue([{ id: 's1' }, { id: 's2' }]); + mockedGetChunksForNotebook.mockReturnValue([{ id: 'c1', text: 'hello' }]); + mockedBuildSourceGroups.mockReturnValue([{ docIndex: 1, name: 'Doc', chunks: [{ text: 'hello' }] }]); +} + +function stubGeneratorsOk(): void { + mockedGenerateStudyGuide.mockResolvedValue({ title: 'guide' }); + mockedGenerateFaq.mockResolvedValue({ subject: 'topic' }); + mockedGenerateExecutiveBrief.mockResolvedValue({ title: 'brief' }); +} + +beforeEach(() => { + vi.clearAllMocks(); + vi.useFakeTimers(); +}); + +afterEach(() => { + vi.useRealTimers(); +}); + +describe('triggerPreGeneration guards', () => { + it('does nothing if the notebook does not exist', () => { + mockedGetNotebook.mockReturnValue(null); + triggerPreGeneration('nb-missing'); + expect(mockedGetSources).not.toHaveBeenCalled(); + }); + + it('does nothing if fewer than 2 sources', () => { + mockedGetNotebook.mockReturnValue({ id: 'nb-few' }); + mockedGetSources.mockReturnValue([{ id: 's1' }]); + triggerPreGeneration('nb-few'); + expect(mockedMarkGenerating).not.toHaveBeenCalled(); + }); +}); + +describe('queuing when already generating', () => { + it('invalidates cache and skips runPreGeneration', () => { + setupNotebook('nb-q'); + mockedIsGenerating.mockReturnValue(true); + + triggerPreGeneration('nb-q'); + + expect(mockedInvalidate).toHaveBeenCalledWith('nb-q'); + expect(mockedMarkGenerating).not.toHaveBeenCalled(); + }); +}); + +describe('debounce', () => { + it('does not run generation immediately', () => { + setupNotebook('nb-debounce'); + mockedIsGenerating.mockReturnValue(false); + stubGeneratorsOk(); + + triggerPreGeneration('nb-debounce'); + + expect(mockedMarkGenerating).not.toHaveBeenCalled(); + }); + + it('runs generation after the debounce delay', async () => { + setupNotebook('nb-delay'); + mockedIsGenerating.mockReturnValue(false); + stubGeneratorsOk(); + + triggerPreGeneration('nb-delay'); + await vi.advanceTimersByTimeAsync(5000); + + await vi.waitFor(() => { + expect(mockedClearGenerating).toHaveBeenCalledWith('nb-delay'); + }); + + expect(mockedMarkGenerating).toHaveBeenCalledWith('nb-delay'); + }); + + it('resets the timer on repeated calls, running generation only once', async () => { + setupNotebook('nb-batch'); + mockedIsGenerating.mockReturnValue(false); + stubGeneratorsOk(); + + triggerPreGeneration('nb-batch'); + await vi.advanceTimersByTimeAsync(3000); + triggerPreGeneration('nb-batch'); + await vi.advanceTimersByTimeAsync(3000); + triggerPreGeneration('nb-batch'); + await vi.advanceTimersByTimeAsync(5000); + + await vi.waitFor(() => { + expect(mockedClearGenerating).toHaveBeenCalled(); + }); + + expect(mockedMarkGenerating).toHaveBeenCalledTimes(1); + }); +}); + +describe('runPreGeneration (via triggerPreGeneration)', () => { + it('runs all three generators and caches results', async () => { + setupNotebook('nb-happy'); + mockedIsGenerating.mockReturnValue(false); + stubGeneratorsOk(); + + triggerPreGeneration('nb-happy'); + await vi.advanceTimersByTimeAsync(5000); + + await vi.waitFor(() => { + expect(mockedClearGenerating).toHaveBeenCalledWith('nb-happy'); + }); + + expect(mockedGenerateStudyGuide).toHaveBeenCalled(); + expect(mockedGenerateFaq).toHaveBeenCalled(); + expect(mockedGenerateExecutiveBrief).toHaveBeenCalled(); + expect(mockedSetCachedDocument).toHaveBeenCalledTimes(3); + }); + + it('marks generating before starting and clears it after', async () => { + setupNotebook('nb-flag'); + mockedIsGenerating.mockReturnValue(false); + stubGeneratorsOk(); + + triggerPreGeneration('nb-flag'); + await vi.advanceTimersByTimeAsync(5000); + + expect(mockedMarkGenerating).toHaveBeenCalledWith('nb-flag'); + + await vi.waitFor(() => { + expect(mockedClearGenerating).toHaveBeenCalledWith('nb-flag'); + }); + }); + + it('still clears generating flag when a generator fails', async () => { + setupNotebook('nb-partial'); + mockedIsGenerating.mockReturnValue(false); + mockedGenerateStudyGuide.mockRejectedValue(new Error('boom')); + mockedGenerateFaq.mockResolvedValue({ subject: 'ok' }); + mockedGenerateExecutiveBrief.mockResolvedValue({ title: 'ok' }); + + triggerPreGeneration('nb-partial'); + await vi.advanceTimersByTimeAsync(5000); + + await vi.waitFor(() => { + expect(mockedClearGenerating).toHaveBeenCalledWith('nb-partial'); + }); + + expect(mockedSetCachedDocument).toHaveBeenCalledTimes(2); + }); +}); + +describe('re-trigger for queued notebooks', () => { + it('runs a second generation cycle after the first finishes', async () => { + setupNotebook('nb-re'); + mockedIsGenerating.mockReturnValueOnce(false).mockReturnValueOnce(true); + + let resolveFirst: (value: { title: string }) => void; + mockedGenerateStudyGuide + .mockImplementationOnce( + () => + new Promise<{ title: string }>((r) => { + resolveFirst = r; + }) + ) + .mockResolvedValue({ title: 'guide' }); + mockedGenerateFaq.mockResolvedValue({ subject: 'topic' }); + mockedGenerateExecutiveBrief.mockResolvedValue({ title: 'brief' }); + + triggerPreGeneration('nb-re'); + await vi.advanceTimersByTimeAsync(5000); + + triggerPreGeneration('nb-re'); + + resolveFirst!({ title: 'guide' }); + + await vi.waitFor(() => { + expect(mockedMarkGenerating).toHaveBeenCalledTimes(2); + }); + + expect(mockedClearGenerating).toHaveBeenCalledTimes(2); + expect(mockedSetCachedDocument).toHaveBeenCalledTimes(6); + }); +}); diff --git a/server/src/services/preGenerationService.js b/server/src/services/preGenerationService.ts similarity index 58% rename from server/src/services/preGenerationService.js rename to server/src/services/preGenerationService.ts index b2e300a..2b64709 100644 --- a/server/src/services/preGenerationService.js +++ b/server/src/services/preGenerationService.ts @@ -7,22 +7,29 @@ import { generateStudyGuide, generateFaq, generateExecutiveBrief, + type SourceGroup, + type StudyGuide, + type Faq, + type ExecutiveBrief, } from './documentService.js'; const MIN_SOURCES = 2; -const GENERATORS = { +type DocumentType = 'study-guide' | 'faq' | 'executive-brief'; +type GeneratorFn = (sourceGroups: SourceGroup[]) => Promise; + +const GENERATORS: Record = { 'study-guide': generateStudyGuide, - 'faq': generateFaq, + faq: generateFaq, 'executive-brief': generateExecutiveBrief, }; const DEBOUNCE_MS = 5000; -const pendingReGen = new Set(); -const debounceTimers = new Map(); +const pendingReGen = new Set(); +const debounceTimers = new Map>(); -export function triggerPreGeneration(notebookId) { +export function triggerPreGeneration(notebookId: string): void { const notebook = notebookStore.getNotebook(notebookId); if (!notebook) return; @@ -36,40 +43,46 @@ export function triggerPreGeneration(notebookId) { return; } - clearTimeout(debounceTimers.get(notebookId)); + const existingTimer = debounceTimers.get(notebookId); + if (existingTimer !== undefined) { + clearTimeout(existingTimer); + } debounceTimers.set( notebookId, setTimeout(() => { debounceTimers.delete(notebookId); runPreGeneration(notebookId); - }, DEBOUNCE_MS), + }, DEBOUNCE_MS) ); logger.debug({ notebookId, debounceMs: DEBOUNCE_MS }, 'pre-generation debounced'); } -function runPreGeneration(notebookId) { +function runPreGeneration(notebookId: string): void { try { const sources = notebookStore.getSources(notebookId); const chunks = notebookStore.getChunksForNotebook(notebookId); - const sourceGroups = notebookStore.buildSourceGroups(notebookId, chunks); + const sourceGroups = notebookStore.buildSourceGroups(notebookId, chunks) as SourceGroup[]; documentCacheStore.invalidate(notebookId); documentCacheStore.markGenerating(notebookId); logger.info( { notebookId, sourceCount: sources.length, chunkCount: chunks.length }, - 'background pre-generation started', + 'background pre-generation started' ); - const jobs = Object.entries(GENERATORS).map(async ([type, generator]) => { - try { - const document = await generator(sourceGroups); - documentCacheStore.setCachedDocument(notebookId, type, document, sources); - logger.info({ notebookId, type }, 'background pre-generation complete for type'); - } catch (err) { - logger.error({ notebookId, type, err: err.message }, 'background pre-generation failed for type'); + const jobs = (Object.entries(GENERATORS) as Array<[DocumentType, GeneratorFn]>).map( + async ([type, generator]) => { + try { + const document = await generator(sourceGroups); + documentCacheStore.setCachedDocument(notebookId, type, document, sources); + logger.info({ notebookId, type }, 'background pre-generation complete for type'); + } catch (err) { + const error = err as Error; + logger.error({ notebookId, type, err: error.message }, 'background pre-generation failed for type'); + } } - }); + ); Promise.all(jobs) .then(() => { @@ -85,7 +98,8 @@ function runPreGeneration(notebookId) { } }); } catch (err) { - logger.error({ notebookId, err: err.message }, 'pre-generation setup failed'); + const error = err as Error; + logger.error({ notebookId, err: error.message }, 'pre-generation setup failed'); documentCacheStore.clearGenerating(notebookId); } } diff --git a/server/src/services/retrievalService.js b/server/src/services/retrievalService.js deleted file mode 100644 index 947b2ca..0000000 --- a/server/src/services/retrievalService.js +++ /dev/null @@ -1,138 +0,0 @@ -'use strict'; - -import logger from '../logger.js'; -import * as vectorStore from '../stores/vectorStore.js'; -import * as notebookStore from '../stores/notebookStore.js'; - -const VOYAGE_API_URL = 'https://api.voyageai.com/v1'; -const EMBED_MODEL = 'voyage-3'; -const RERANK_MODEL = 'rerank-2'; - -function getApiKey() { - const key = process.env.VOYAGE_API_KEY; - if (!key) throw new Error('VOYAGE_API_KEY is not set'); - return key; -} - -export async function embedTexts(texts, inputType = 'document') { - const res = await fetch(`${VOYAGE_API_URL}/embeddings`, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - Authorization: `Bearer ${getApiKey()}`, - }, - body: JSON.stringify({ - model: EMBED_MODEL, - input: texts, - input_type: inputType, - }), - }); - - if (!res.ok) { - const body = await res.text(); - throw new Error(`Voyage embed failed (${res.status}): ${body}`); - } - - const json = await res.json(); - return json.data.map((d) => d.embedding); -} - -export function storeChunkEmbeddings(chunks, embeddings) { - for (let i = 0; i < chunks.length; i++) { - vectorStore.storeVector(chunks[i].id, embeddings[i]); - } - logger.debug({ count: chunks.length }, 'stored chunk embeddings'); -} - -export function search(queryEmbedding, notebookId, topK = 10) { - const chunks = notebookStore.getChunksForNotebook(notebookId); - if (chunks.length === 0) return []; - - const scored = []; - for (const chunk of chunks) { - const vec = vectorStore.getVector(chunk.id); - if (!vec) continue; - scored.push({ chunk, score: cosineSimilarity(queryEmbedding, vec) }); - } - - scored.sort((a, b) => b.score - a.score); - return scored.slice(0, topK); -} - -export async function rerank(query, chunks) { - if (chunks.length === 0) return []; - - const res = await fetch(`${VOYAGE_API_URL}/rerank`, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - Authorization: `Bearer ${getApiKey()}`, - }, - body: JSON.stringify({ - model: RERANK_MODEL, - query, - documents: chunks.map((c) => c.text), - }), - }); - - if (!res.ok) { - const body = await res.text(); - throw new Error(`Voyage rerank failed (${res.status}): ${body}`); - } - - const json = await res.json(); - return json.data.map((item) => ({ - chunk: chunks[item.index], - relevanceScore: item.relevance_score, - })); -} - -// Computes cosine similarity between two vectors `a` and `b`. -// Cosine similarity measures how similar two vectors' *directions* are, -// ignoring their magnitudes. It returns a value from -1 (opposite) to 1 (identical direction). -// Formula: cos(θ) = (a · b) / (||a|| * ||b||) -function cosineSimilarity(a, b) { - // Accumulator for the dot product (a · b): the sum of element-wise products. - // The dot product captures how much the two vectors "agree" — it grows - // when corresponding elements point the same way and shrinks when they oppose. - let dot = 0; - - // Accumulator for the squared magnitude of vector `a` (sum of a[i]²). - // After the loop, Math.sqrt(normA) will give ||a||, the Euclidean length of `a`. - // This is needed for the denominator, which normalizes out each vector's magnitude - // so the result reflects only directional similarity, not scale. - let normA = 0; - - // Same as above, but for vector `b`. Together with normA, these two values - // will form the denominator ||a|| * ||b|| that scales the dot product into - // the -1..1 cosine similarity range. - let normB = 0; - - // Walk through every dimension of the two vectors in lockstep. - for (let i = 0; i < a.length; i++) { - // Multiply the i-th elements of `a` and `b` and add to the running dot product. - // Each term a[i]*b[i] contributes positively when both components share the same - // sign (both positive or both negative) and negatively when they differ. - dot += a[i] * b[i]; - - // Square the i-th element of `a` and accumulate it toward `a`'s squared magnitude. - // Squaring ensures every component contributes positively regardless of sign. - normA += a[i] * a[i]; - - // Same for vector `b` — accumulate toward `b`'s squared magnitude. - normB += b[i] * b[i]; - } - - // Compute the denominator: the product of the two vectors' Euclidean lengths. - // Taking the square root of each accumulated sum-of-squares converts them from - // squared magnitudes back into actual magnitudes (lengths). - const denom = Math.sqrt(normA) * Math.sqrt(normB); - - // If either vector has zero magnitude (all zeros), the denominator is 0 and - // division would produce NaN, so we return 0 (no meaningful similarity). - // Otherwise, dividing the dot product by the combined magnitudes yields the - // cosine of the angle between the vectors — our similarity score. - return denom === 0 ? 0 : dot / denom; -} - -export { cosineSimilarity }; diff --git a/server/src/services/retrievalService.test.js b/server/src/services/retrievalService.test.ts similarity index 100% rename from server/src/services/retrievalService.test.js rename to server/src/services/retrievalService.test.ts diff --git a/server/src/services/retrievalService.ts b/server/src/services/retrievalService.ts new file mode 100644 index 0000000..0002017 --- /dev/null +++ b/server/src/services/retrievalService.ts @@ -0,0 +1,131 @@ +'use strict'; + +import logger from '../logger.js'; +import * as vectorStore from '../stores/vectorStore.js'; +import * as notebookStore from '../stores/notebookStore.js'; + +const VOYAGE_API_URL = 'https://api.voyageai.com/v1'; +const EMBED_MODEL = 'voyage-3'; +const RERANK_MODEL = 'rerank-2'; + +function getApiKey(): string { + const key = process.env.VOYAGE_API_KEY; + if (!key) throw new Error('VOYAGE_API_KEY is not set'); + return key; +} + +export interface TextChunk { + id: string; + text: string; + sourceId: string; + index: number; +} + +interface VoyageEmbeddingResponse { + data: Array<{ embedding: number[] }>; +} + +interface VoyageRerankResponse { + data: Array<{ index: number; relevance_score: number }>; +} + +export interface ScoredChunk { + chunk: TextChunk; + score: number; +} + +export interface RankedChunk { + chunk: TextChunk; + relevanceScore: number; +} + +export async function embedTexts( + texts: string[], + inputType: 'document' | 'query' = 'document' +): Promise { + const res = await fetch(`${VOYAGE_API_URL}/embeddings`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + Authorization: `Bearer ${getApiKey()}`, + }, + body: JSON.stringify({ + model: EMBED_MODEL, + input: texts, + input_type: inputType, + }), + }); + + if (!res.ok) { + const body = await res.text(); + throw new Error(`Voyage embed failed (${res.status}): ${body}`); + } + + const json = (await res.json()) as VoyageEmbeddingResponse; + return json.data.map((d) => d.embedding); +} + +export function storeChunkEmbeddings(chunks: TextChunk[], embeddings: number[][]): void { + for (let i = 0; i < chunks.length; i++) { + vectorStore.storeVector(chunks[i].id, embeddings[i]); + } + logger.debug({ count: chunks.length }, 'stored chunk embeddings'); +} + +export function search(queryEmbedding: number[], notebookId: string, topK = 10): ScoredChunk[] { + const chunks = notebookStore.getChunksForNotebook(notebookId) as TextChunk[]; + if (chunks.length === 0) return []; + + const scored: ScoredChunk[] = []; + for (const chunk of chunks) { + const vec = vectorStore.getVector(chunk.id); + if (!vec) continue; + scored.push({ chunk, score: cosineSimilarity(queryEmbedding, vec) }); + } + + scored.sort((a, b) => b.score - a.score); + return scored.slice(0, topK); +} + +export async function rerank(query: string, chunks: TextChunk[]): Promise { + if (chunks.length === 0) return []; + + const res = await fetch(`${VOYAGE_API_URL}/rerank`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + Authorization: `Bearer ${getApiKey()}`, + }, + body: JSON.stringify({ + model: RERANK_MODEL, + query, + documents: chunks.map((c) => c.text), + }), + }); + + if (!res.ok) { + const body = await res.text(); + throw new Error(`Voyage rerank failed (${res.status}): ${body}`); + } + + const json = (await res.json()) as VoyageRerankResponse; + return json.data.map((item) => ({ + chunk: chunks[item.index], + relevanceScore: item.relevance_score, + })); +} + +export function cosineSimilarity(a: number[], b: number[]): number { + let dot = 0; + let normA = 0; + let normB = 0; + + for (let i = 0; i < a.length; i++) { + dot += a[i] * b[i]; + normA += a[i] * a[i]; + normB += b[i] * b[i]; + } + + const denom = Math.sqrt(normA) * Math.sqrt(normB); + return denom === 0 ? 0 : dot / denom; +} diff --git a/server/src/services/scoringService.test.js b/server/src/services/scoringService.test.ts similarity index 71% rename from server/src/services/scoringService.test.js rename to server/src/services/scoringService.test.ts index c7db76f..0661067 100644 --- a/server/src/services/scoringService.test.js +++ b/server/src/services/scoringService.test.ts @@ -1,4 +1,4 @@ -import { describe, it, expect, vi, beforeEach } from 'vitest'; +import { describe, it, expect, vi, beforeEach, type Mock } from 'vitest'; vi.mock('./retrievalService.js', () => ({ embedTexts: vi.fn(), @@ -17,6 +17,10 @@ import { computeGroundedness } from './scoringService.js'; import { embedTexts, cosineSimilarity } from './retrievalService.js'; import * as vectorStore from '../stores/vectorStore.js'; +const mockedEmbedTexts = embedTexts as Mock; +const mockedCosineSimilarity = cosineSimilarity as Mock; +const mockedGetVector = vectorStore.getVector as Mock; + beforeEach(() => { vi.clearAllMocks(); }); @@ -36,16 +40,16 @@ describe('computeGroundedness', () => { }); it('returns 0 when no chunk vectors are found', async () => { - vectorStore.getVector.mockReturnValue(null); + mockedGetVector.mockReturnValue(null); const answer = 'This is a sentence that is definitely long enough to pass the filter.'; expect(await computeGroundedness(answer, ['chunk-1'])).toBe(0); }); it('returns 1.0 when all sentences have similarity at or above the ceiling', async () => { const fakeVec = new Float32Array([1, 0, 0]); - vectorStore.getVector.mockReturnValue(fakeVec); - embedTexts.mockResolvedValue([new Float32Array([1, 0, 0])]); - cosineSimilarity.mockReturnValue(0.70); + mockedGetVector.mockReturnValue(fakeVec); + mockedEmbedTexts.mockResolvedValue([new Float32Array([1, 0, 0])]); + mockedCosineSimilarity.mockReturnValue(0.7); const answer = 'This sentence is long enough to be included in the analysis.'; const score = await computeGroundedness(answer, ['chunk-1']); @@ -54,9 +58,9 @@ describe('computeGroundedness', () => { it('returns 0 when all sentences have similarity at or below the floor', async () => { const fakeVec = new Float32Array([1, 0, 0]); - vectorStore.getVector.mockReturnValue(fakeVec); - embedTexts.mockResolvedValue([new Float32Array([0, 1, 0])]); - cosineSimilarity.mockReturnValue(0.20); + mockedGetVector.mockReturnValue(fakeVec); + mockedEmbedTexts.mockResolvedValue([new Float32Array([0, 1, 0])]); + mockedCosineSimilarity.mockReturnValue(0.2); const answer = 'This sentence is long enough to be included in the analysis.'; const score = await computeGroundedness(answer, ['chunk-1']); @@ -65,9 +69,9 @@ describe('computeGroundedness', () => { it('returns a value between 0 and 1 for mid-range similarity', async () => { const fakeVec = new Float32Array([1, 0, 0]); - vectorStore.getVector.mockReturnValue(fakeVec); - embedTexts.mockResolvedValue([new Float32Array([1, 0, 0])]); - cosineSimilarity.mockReturnValue(0.50); + mockedGetVector.mockReturnValue(fakeVec); + mockedEmbedTexts.mockResolvedValue([new Float32Array([1, 0, 0])]); + mockedCosineSimilarity.mockReturnValue(0.5); const answer = 'This sentence is long enough to be included in the analysis.'; const score = await computeGroundedness(answer, ['chunk-1']); @@ -78,14 +82,9 @@ describe('computeGroundedness', () => { it('averages across multiple sentences', async () => { const fakeVec = new Float32Array([1, 0, 0]); - vectorStore.getVector.mockReturnValue(fakeVec); - embedTexts.mockResolvedValue([ - new Float32Array([1, 0, 0]), - new Float32Array([0, 1, 0]), - ]); - cosineSimilarity - .mockReturnValueOnce(0.65) - .mockReturnValueOnce(0.35); + mockedGetVector.mockReturnValue(fakeVec); + mockedEmbedTexts.mockResolvedValue([new Float32Array([1, 0, 0]), new Float32Array([0, 1, 0])]); + mockedCosineSimilarity.mockReturnValueOnce(0.65).mockReturnValueOnce(0.35); const answer = 'This is the first sentence that is long enough. This is the second sentence that is also long enough.'; @@ -94,13 +93,11 @@ describe('computeGroundedness', () => { }); it('picks the best similarity across multiple chunk vectors', async () => { - vectorStore.getVector + mockedGetVector .mockReturnValueOnce(new Float32Array([1, 0, 0])) .mockReturnValueOnce(new Float32Array([0, 1, 0])); - embedTexts.mockResolvedValue([new Float32Array([1, 0, 0])]); - cosineSimilarity - .mockReturnValueOnce(0.40) - .mockReturnValueOnce(0.65); + mockedEmbedTexts.mockResolvedValue([new Float32Array([1, 0, 0])]); + mockedCosineSimilarity.mockReturnValueOnce(0.4).mockReturnValueOnce(0.65); const answer = 'This sentence is long enough to be included in the analysis.'; const score = await computeGroundedness(answer, ['chunk-1', 'chunk-2']); @@ -109,14 +106,14 @@ describe('computeGroundedness', () => { it('strips citation markers [1] [2] before splitting sentences', async () => { const fakeVec = new Float32Array([1, 0, 0]); - vectorStore.getVector.mockReturnValue(fakeVec); - embedTexts.mockResolvedValue([new Float32Array([1, 0, 0])]); - cosineSimilarity.mockReturnValue(0.50); + mockedGetVector.mockReturnValue(fakeVec); + mockedEmbedTexts.mockResolvedValue([new Float32Array([1, 0, 0])]); + mockedCosineSimilarity.mockReturnValue(0.5); const answer = 'The sky is blue according to science [1]. Water covers most of the earth [2].'; await computeGroundedness(answer, ['chunk-1']); - const embeddedTexts = embedTexts.mock.calls[0][0]; + const embeddedTexts = mockedEmbedTexts.mock.calls[0][0] as string[]; for (const text of embeddedTexts) { expect(text).not.toMatch(/\[\d+\]/); } diff --git a/server/src/services/scoringService.js b/server/src/services/scoringService.ts similarity index 72% rename from server/src/services/scoringService.js rename to server/src/services/scoringService.ts index 1d65b34..ec6dda6 100644 --- a/server/src/services/scoringService.js +++ b/server/src/services/scoringService.ts @@ -4,42 +4,33 @@ import { embedTexts, cosineSimilarity } from './retrievalService.js'; import * as vectorStore from '../stores/vectorStore.js'; import logger from '../logger.js'; -// Calibration bounds for voyage-3 cosine similarity. -// Empirically, near-direct-quote sentences top out around 0.68-0.72 -// raw cosine similarity; unrelated text falls below ~0.35. const SIM_FLOOR = 0.35; const SIM_CEILING = 0.65; const MIN_SENTENCE_LENGTH = 20; -function splitIntoSentences(text) { +function splitIntoSentences(text: string): string[] { const cleaned = text.replace(/\[\d+\]/g, '').trim(); const raw = cleaned.split(/(?<=[.!?])\s+/); - return raw - .map((s) => s.trim()) - .filter((s) => s.length >= MIN_SENTENCE_LENGTH); + return raw.map((s) => s.trim()).filter((s) => s.length >= MIN_SENTENCE_LENGTH); } -function calibrate(rawSim) { - const scaled = - (rawSim - SIM_FLOOR) / (SIM_CEILING - SIM_FLOOR); +function calibrate(rawSim: number): number { + const scaled = (rawSim - SIM_FLOOR) / (SIM_CEILING - SIM_FLOOR); return Math.max(0, Math.min(1, scaled)); } export async function computeGroundedness( - answerText, - citedChunkIds -) { + answerText: string, + citedChunkIds: string[] | null | undefined +): Promise { if (!citedChunkIds || citedChunkIds.length === 0) return 0; const sentences = splitIntoSentences(answerText); if (sentences.length === 0) return 0; - const sentenceEmbeddings = await embedTexts( - sentences, - 'document' - ); + const sentenceEmbeddings = await embedTexts(sentences, 'document'); - const chunkVectors = []; + const chunkVectors: number[][] = []; for (const chunkId of citedChunkIds) { const vec = vectorStore.getVector(chunkId); if (vec) { @@ -73,4 +64,3 @@ export async function computeGroundedness( return totalCalibrated / sentenceEmbeddings.length; } - diff --git a/server/src/services/sourceService.test.js b/server/src/services/sourceService.test.ts similarity index 96% rename from server/src/services/sourceService.test.js rename to server/src/services/sourceService.test.ts index c3a8d45..5a6d03d 100644 --- a/server/src/services/sourceService.test.js +++ b/server/src/services/sourceService.test.ts @@ -1,9 +1,16 @@ import { describe, it, expect } from 'vitest'; -import { chunkText, parseFile, isYoutubeUrl, extractVideoId, parseUrl } from './sourceService.js'; +import { + chunkText, + parseFile, + isYoutubeUrl, + extractVideoId, + parseUrl, + type TextChunk, +} from './sourceService.js'; describe('chunkText', () => { it('returns a single chunk for short text', () => { - const chunks = chunkText('Hello world.', 'src-1'); + const chunks: TextChunk[] = chunkText('Hello world.', 'src-1'); expect(chunks).toHaveLength(1); expect(chunks[0].text).toBe('Hello world.'); expect(chunks[0].sourceId).toBe('src-1'); @@ -42,7 +49,6 @@ describe('chunkText', () => { const chunks = chunkText(text, 'src-5'); if (chunks.length >= 2) { const end0 = chunks[0].text.slice(-100); - const start1 = chunks[1].text.slice(0, 100); const hasOverlap = chunks[1].text.includes(end0.slice(-50)); expect(hasOverlap).toBe(true); } diff --git a/server/src/services/sourceService.js b/server/src/services/sourceService.ts similarity index 74% rename from server/src/services/sourceService.js rename to server/src/services/sourceService.ts index 8b45358..a00efc9 100644 --- a/server/src/services/sourceService.js +++ b/server/src/services/sourceService.ts @@ -19,7 +19,7 @@ const execFileAsync = promisify(execFile); const CHARS_PER_CHUNK = 2000; const OVERLAP_CHARS = 200; -const AUDIO_MIMETYPES = new Set([ +const AUDIO_MIMETYPES = new Set([ 'audio/mpeg', 'audio/mp3', 'audio/wav', @@ -31,11 +31,15 @@ const AUDIO_MIMETYPES = new Set([ 'audio/webm', ]); -export function isAudioMimetype(mimetype) { +export function isAudioMimetype(mimetype: string): boolean { return AUDIO_MIMETYPES.has(mimetype); } -export async function parseFile(buffer, mimetype, filename) { +export async function parseFile( + buffer: Buffer, + mimetype: string, + filename?: string +): Promise { if (mimetype === 'application/pdf') { const result = await pdfParse(buffer); return result.text; @@ -58,7 +62,7 @@ export async function parseFile(buffer, mimetype, filename) { const WHISPER_MAX_BYTES = 25 * 1024 * 1024; -async function transcribeAudio(buffer, filename) { +async function transcribeAudio(buffer: Buffer, filename?: string): Promise { const key = process.env.OPENAI_API_KEY; if (!key) throw new Error('OPENAI_API_KEY is required for audio transcription'); @@ -69,7 +73,7 @@ async function transcribeAudio(buffer, filename) { const client = new OpenAI({ apiKey: key }); const ext = filename?.split('.').pop()?.toLowerCase() || 'mp3'; - const file = new File([buffer], `audio.${ext}`, { type: `audio/${ext}` }); + const file = new File([new Uint8Array(buffer)], `audio.${ext}`, { type: `audio/${ext}` }); logger.info({ filename, bytes: buffer.length }, 'transcribing audio with Whisper'); @@ -81,7 +85,7 @@ async function transcribeAudio(buffer, filename) { return response.text; } -const BLOCKED_IP_RANGES = [ +const BLOCKED_IP_RANGES: RegExp[] = [ /^127\./, /^10\./, /^172\.(1[6-9]|2\d|3[01])\./, @@ -94,12 +98,12 @@ const BLOCKED_IP_RANGES = [ /^fd/i, ]; -function isBlockedIp(ip) { +function isBlockedIp(ip: string): boolean { return BLOCKED_IP_RANGES.some((re) => re.test(ip)); } -async function validateUrl(raw) { - let parsed; +async function validateUrl(raw: string): Promise { + let parsed: URL; try { parsed = new URL(raw); } catch { @@ -126,14 +130,14 @@ async function validateUrl(raw) { return parsed.href; } -export async function parseUrl(url) { +export async function parseUrl(url: string): Promise { const safeUrl = await validateUrl(url); logger.info({ url: safeUrl }, 'fetching URL content'); const res = await fetch(safeUrl, { headers: { 'User-Agent': 'Mozilla/5.0 (compatible; NotebookClone/1.0)', - 'Accept': 'text/html,application/xhtml+xml,text/plain', + Accept: 'text/html,application/xhtml+xml,text/plain', }, signal: AbortSignal.timeout(15000), }); @@ -164,16 +168,24 @@ export async function parseUrl(url) { const YT_URL_RE = /(?:youtube\.com\/watch\?v=|youtu\.be\/|youtube\.com\/embed\/)([a-zA-Z0-9_-]{11})/; -export function isYoutubeUrl(url) { +export function isYoutubeUrl(url: string): boolean { return YT_URL_RE.test(url); } -export function extractVideoId(url) { +export function extractVideoId(url: string): string | null { const match = url.match(YT_URL_RE); return match ? match[1] : null; } -export async function parseYoutubeUrl(url) { +interface YouTubeSubtitleEvent { + segs?: Array<{ utf8?: string }>; +} + +interface YouTubeSubtitleData { + events?: YouTubeSubtitleEvent[]; +} + +export async function parseYoutubeUrl(url: string): Promise { const videoId = extractVideoId(url); if (!videoId) { throw new Error('Could not extract YouTube video ID from URL'); @@ -183,16 +195,23 @@ export async function parseYoutubeUrl(url) { const tmpDir = mkdtempSync(join(tmpdir(), 'yt-')); try { - const { stderr } = await execFileAsync('yt-dlp', [ - '--skip-download', - '--write-auto-sub', - '--write-sub', - '--sub-lang', 'en', - '--sub-format', 'json3', - '--no-warnings', - '-o', join(tmpDir, '%(id)s'), - `https://www.youtube.com/watch?v=${videoId}`, - ], { timeout: 20000 }); + const { stderr } = await execFileAsync( + 'yt-dlp', + [ + '--skip-download', + '--write-auto-sub', + '--write-sub', + '--sub-lang', + 'en', + '--sub-format', + 'json3', + '--no-warnings', + '-o', + join(tmpDir, '%(id)s'), + `https://www.youtube.com/watch?v=${videoId}`, + ], + { timeout: 20000 } + ); if (stderr) { logger.warn({ videoId, stderr: stderr.slice(0, 500) }, 'yt-dlp stderr output'); @@ -204,10 +223,10 @@ export async function parseYoutubeUrl(url) { throw new Error('No English subtitles available for this YouTube video'); } - const data = JSON.parse(readFileSync(join(tmpDir, subFile), 'utf-8')); + const data: YouTubeSubtitleData = JSON.parse(readFileSync(join(tmpDir, subFile), 'utf-8')); const events = (data.events || []).filter((e) => e.segs); const text = events - .map((e) => e.segs.map((s) => s.utf8 || '').join('')) + .map((e) => e.segs!.map((s) => s.utf8 || '').join('')) .join(' ') .replace(/\n/g, ' ') .replace(/\s+/g, ' ') @@ -220,27 +239,35 @@ export async function parseYoutubeUrl(url) { logger.info({ videoId, textLength: text.length }, 'YouTube subtitles extracted'); return text; } catch (err) { - if (err.code === 'ENOENT') { + const error = err as NodeJS.ErrnoException & { killed?: boolean; stderr?: string }; + if (error.code === 'ENOENT') { throw new Error('YouTube support requires yt-dlp to be installed (brew install yt-dlp)'); } - if (err.killed) { + if (error.killed) { throw new Error('yt-dlp timed out fetching subtitles'); } - if (err.message.startsWith('No English') || err.message.startsWith('Subtitles')) { - throw err; + if (error.message?.startsWith('No English') || error.message?.startsWith('Subtitles')) { + throw error; } - const detail = err.stderr?.slice(0, 300) || err.message; + const detail = error.stderr?.slice(0, 300) || error.message; throw new Error(`Failed to extract YouTube subtitles: ${detail}`); } finally { rmSync(tmpDir, { recursive: true, force: true }); } } -export function chunkText(text, sourceId) { +export interface TextChunk { + id: string; + text: string; + sourceId: string; + index: number; +} + +export function chunkText(text: string, sourceId: string): TextChunk[] { const cleaned = text.replace(/\r\n/g, '\n').trim(); if (!cleaned) return []; - const chunks = []; + const chunks: TextChunk[] = []; let start = 0; let index = 0; diff --git a/server/tsconfig.json b/server/tsconfig.json new file mode 100644 index 0000000..555d1d4 --- /dev/null +++ b/server/tsconfig.json @@ -0,0 +1,18 @@ +{ + "compilerOptions": { + "allowJs": true, // Allow JavaScript files to be imported and compiled + "checkJs": true, + "target": "es2022", + "module": "commonjs", + "outDir": "./dist", + "rootDir": "./src", // Root of input files + "strict": true, + "skipLibCheck": true // Skip checking .d.ts files for faster builds + }, + "include": [ + "src/**/*" + ], // Files to include in the project + "exclude": [ + "node_modules" + ] // Folders to ignore +} \ No newline at end of file