78 lines
2.5 KiB
JavaScript
78 lines
2.5 KiB
JavaScript
import { rerank } from "./voyage.js";
|
|
import { judgeEntailment } from "./anthropic.js";
|
|
|
|
// These bounds are a starting ramp for rerank-2; log raw scores and adjust.
|
|
const RERANK_FLOOR = 0.2;
|
|
const RERANK_CEILING = 0.8;
|
|
const MIN_SENTENCE_LENGTH = 20;
|
|
const MAX_SENTENCES = 12;
|
|
const MAX_CITED_CHUNKS = 5;
|
|
|
|
function splitIntoSentences(text) {
|
|
const cleaned = text.replace(/\[\d+\]/g, "").trim();
|
|
const raw = cleaned.split(/(?<=[.!?])\s+/);
|
|
return raw
|
|
.map((s) => s.trim())
|
|
.filter((s) => s.length >= MIN_SENTENCE_LENGTH)
|
|
.slice(0, MAX_SENTENCES);
|
|
}
|
|
|
|
function calibrate(rawScore) {
|
|
const scaled = (rawScore - RERANK_FLOOR) / (RERANK_CEILING - RERANK_FLOOR);
|
|
return Math.max(0, Math.min(1, scaled));
|
|
}
|
|
|
|
function groundednessMode() {
|
|
const mode = (process.env.GROUNDEDNESS_MODE || "rerank_nli").toLowerCase();
|
|
return mode === "rerank" ? "rerank" : "rerank_nli";
|
|
}
|
|
|
|
export async function computeGroundedness(answerText, citedChunkTexts) {
|
|
if (!citedChunkTexts || citedChunkTexts.length === 0) return 0;
|
|
|
|
const sentences = splitIntoSentences(answerText);
|
|
if (sentences.length === 0) return 0;
|
|
|
|
const chunks = citedChunkTexts.filter(Boolean).slice(0, MAX_CITED_CHUNKS);
|
|
if (chunks.length === 0) return 0;
|
|
|
|
const mode = groundednessMode();
|
|
|
|
const perSentence = await Promise.all(
|
|
sentences.map(async (sentence) => {
|
|
const ranked = await rerank(sentence, chunks);
|
|
const raw =
|
|
ranked.length === 0
|
|
? 0
|
|
: Math.max(...ranked.map((r) => r.relevanceScore));
|
|
const calibrated = calibrate(raw);
|
|
|
|
let nli = null;
|
|
if (mode === "rerank_nli") {
|
|
nli = await judgeEntailment(sentence, chunks);
|
|
}
|
|
|
|
console.log(
|
|
JSON.stringify({
|
|
event: "sentence_groundedness",
|
|
sentence: sentence.slice(0, 80),
|
|
rawRerank: Number(raw.toFixed(4)),
|
|
calibrated: Number(calibrated.toFixed(4)),
|
|
nli,
|
|
mode,
|
|
})
|
|
);
|
|
|
|
return { raw, calibrated, nli };
|
|
})
|
|
);
|
|
|
|
if (perSentence.some((s) => s.nli && s.nli.label === "contradicted")) {
|
|
console.log(JSON.stringify({ event: "groundedness_floored", reason: "contradicted" }));
|
|
return 0;
|
|
}
|
|
|
|
const total = perSentence.reduce((sum, s) => sum + s.calibrated, 0);
|
|
return total / perSentence.length;
|
|
}
|