import { rerank } from "./voyage.js"; import { judgeEntailment } from "./anthropic.js"; // These bounds are a starting ramp for rerank-2; log raw scores and adjust. const RERANK_FLOOR = 0.2; const RERANK_CEILING = 0.8; const MIN_SENTENCE_LENGTH = 20; const MAX_SENTENCES = 12; const MAX_CITED_CHUNKS = 5; function splitIntoSentences(text) { const cleaned = text.replace(/\[\d+\]/g, "").trim(); const raw = cleaned.split(/(?<=[.!?])\s+/); return raw .map((s) => s.trim()) .filter((s) => s.length >= MIN_SENTENCE_LENGTH) .slice(0, MAX_SENTENCES); } function calibrate(rawScore) { const scaled = (rawScore - RERANK_FLOOR) / (RERANK_CEILING - RERANK_FLOOR); return Math.max(0, Math.min(1, scaled)); } function groundednessMode() { const mode = (process.env.GROUNDEDNESS_MODE || "rerank_nli").toLowerCase(); return mode === "rerank" ? "rerank" : "rerank_nli"; } export async function computeGroundedness(answerText, citedChunkTexts) { if (!citedChunkTexts || citedChunkTexts.length === 0) return 0; const sentences = splitIntoSentences(answerText); if (sentences.length === 0) return 0; const chunks = citedChunkTexts.filter(Boolean).slice(0, MAX_CITED_CHUNKS); if (chunks.length === 0) return 0; const mode = groundednessMode(); const perSentence = await Promise.all( sentences.map(async (sentence) => { const ranked = await rerank(sentence, chunks); const raw = ranked.length === 0 ? 0 : Math.max(...ranked.map((r) => r.relevanceScore)); const calibrated = calibrate(raw); let nli = null; if (mode === "rerank_nli") { nli = await judgeEntailment(sentence, chunks); } console.log( JSON.stringify({ event: "sentence_groundedness", sentence: sentence.slice(0, 80), rawRerank: Number(raw.toFixed(4)), calibrated: Number(calibrated.toFixed(4)), nli, mode, }) ); return { raw, calibrated, nli }; }) ); if (perSentence.some((s) => s.nli && s.nli.label === "contradicted")) { console.log(JSON.stringify({ event: "groundedness_floored", reason: "contradicted" })); return 0; } const total = perSentence.reduce((sum, s) => sum + s.calibrated, 0); return total / perSentence.length; }