Kjell Tore Guttormsen
3e39f2df6b
Deterministisk de-risk-harness for Fase 3-judgen (kjøres på frosset 373-påstands gull-sett): - lib/judge-bakeoff.mjs: P-filter (volatil+fetchbar, price ekskl.), confusion-matrix for 3 armer (staleness/judge/hybrid), Wilson-bånd, forhåndsregistrert gate. 14 tester. - extract-judge-claims.mjs: blind manifest (255 påstander, 0 label-lekkasje — testet invariant). - judge-claim-prompt.md: blind per-påstands groundedness-judge (Opus xhigh, microsoft_docs_fetch). - run-judge-bakeoff.mjs: join gull+results på id, gate-rapport (.json/.md). Gate FORHÅNDSREGISTRERT (operatørvalg, før fan-out): recall ≥0.80, presisjon ≥0.70, OG slår staleness (0/38). Evalueringspop P = 240 verifiserbare, 38 positive. Suite 551/551 (538 + 13 nye).
199 lines
10 KiB
JavaScript
199 lines
10 KiB
JavaScript
// tests/kb-eval/test-judge-bakeoff.test.mjs
|
|
// Unit tests for the S1 judge bake-off aggregation lib.
|
|
//
|
|
// S1 de-risks the Fase 3 correctness judge BEFORE any production scaffolding is
|
|
// built: run a per-claim groundedness judge against the frozen 373-claim gold set
|
|
// and ask whether it beats the cheap staleness baseline (recall 0/40) by enough to
|
|
// justify ~2700 non-batchable fetches per full pass.
|
|
//
|
|
// The eval population P is volatile + fetchable claim_types (price excluded) — the
|
|
// judge is measured exactly where the real errors live, which structurally avoids
|
|
// the "inverted leverage" trap (a judge that wins only by auto-scoring cheap stable
|
|
// claims). Grading is a pure detection confusion matrix vs the gold verdicts:
|
|
// gold positive = verdict ∈ {outdated, wrong} (a real error to catch)
|
|
// gold negative = verdict === 'correct'
|
|
// excluded = verdict === 'unsourced' (no ground-truth value)
|
|
// predicted positive (flag) = judge_verdict === 'not_grounded'
|
|
// All computation is deterministic — no I/O, no Date.now/Math.random.
|
|
|
|
import { test } from 'node:test';
|
|
import assert from 'node:assert/strict';
|
|
import {
|
|
evalPopulation,
|
|
blindClaims,
|
|
gradeArm,
|
|
stalenessFlag,
|
|
judgeFlag,
|
|
hybridFlag,
|
|
gateDecision,
|
|
computeBakeoff,
|
|
} from '../../scripts/kb-eval/lib/judge-bakeoff.mjs';
|
|
|
|
// --- synthetic joined gold+judge set ----------------------------------------
|
|
// Each claim carries its gold fields plus the (blind) judge_verdict the harness
|
|
// joined back by id. Comments give the intended confusion-matrix cell.
|
|
const J = [
|
|
// 1: volatile/version, correct, judge grounded -> P, verifiable, TN
|
|
{ id: 'a#1', skill: 'x', stratum: 'volatile', claim_type: 'version', verdict: 'correct', lastmod_changed: false, judge_verdict: 'grounded' },
|
|
// 2: volatile/sku, outdated, judge not_grounded -> P, TP (judge catches)
|
|
{ id: 'a#2', skill: 'x', stratum: 'volatile', claim_type: 'sku', verdict: 'outdated', lastmod_changed: false, judge_verdict: 'not_grounded' },
|
|
// 3: volatile/tpm, wrong, judge grounded -> P, FN (judge misses)
|
|
{ id: 'a#3', skill: 'x', stratum: 'volatile', claim_type: 'tpm', verdict: 'wrong', lastmod_changed: false, judge_verdict: 'grounded' },
|
|
// 4: volatile/status, correct, judge not_grounded -> P, FP (false alarm)
|
|
{ id: 'a#4', skill: 'x', stratum: 'volatile', claim_type: 'status', verdict: 'correct', lastmod_changed: false, judge_verdict: 'not_grounded' },
|
|
// 5: volatile/region, outdated, lastmod TRUE, judge ng -> P, TP; staleness also catches
|
|
{ id: 'a#5', skill: 'x', stratum: 'volatile', claim_type: 'region', verdict: 'outdated', lastmod_changed: true, judge_verdict: 'not_grounded' },
|
|
// 6: volatile/PRICE, wrong -> excluded from P (price)
|
|
{ id: 'a#6', skill: 'x', stratum: 'volatile', claim_type: 'price', verdict: 'wrong', lastmod_changed: false, judge_verdict: 'not_grounded' },
|
|
// 7: CONTROL/taxonomy, wrong -> excluded from P (control stratum)
|
|
{ id: 'a#7', skill: 'x', stratum: 'control', claim_type: 'taxonomy', verdict: 'wrong', lastmod_changed: false, judge_verdict: 'not_grounded' },
|
|
// 8: volatile/status, unsourced, judge source_silent -> P but not verifiable (excluded from P/R)
|
|
{ id: 'a#8', skill: 'x', stratum: 'volatile', claim_type: 'status', verdict: 'unsourced', lastmod_changed: false, judge_verdict: 'source_silent' },
|
|
// 9: volatile/taxonomy, correct, judge source_silent -> P, verifiable negative, not flagged -> TN
|
|
{ id: 'a#9', skill: 'x', stratum: 'volatile', claim_type: 'taxonomy', verdict: 'correct', lastmod_changed: false, judge_verdict: 'source_silent' },
|
|
];
|
|
|
|
// --- evalPopulation ----------------------------------------------------------
|
|
test('evalPopulation — keeps volatile + fetchable, drops price and control', () => {
|
|
const P = evalPopulation(J);
|
|
const ids = P.map((c) => c.id).sort();
|
|
assert.deepEqual(ids, ['a#1', 'a#2', 'a#3', 'a#4', 'a#5', 'a#8', 'a#9']);
|
|
});
|
|
|
|
// --- blindClaims (no label leakage) -----------------------------------------
|
|
test('blindClaims — emits all of P with only the allowed blind fields', () => {
|
|
const blind = blindClaims(J);
|
|
assert.equal(blind.length, 7); // all of P, unsourced included
|
|
const allowed = new Set(['id', 'file', 'skill', 'claim', 'claim_type', 'evidence_url']);
|
|
for (const c of blind) {
|
|
for (const k of Object.keys(c)) {
|
|
assert.ok(allowed.has(k), `leaked field: ${k}`);
|
|
}
|
|
// the answer key must never appear
|
|
assert.equal(c.verdict, undefined);
|
|
assert.equal(c.notes, undefined);
|
|
assert.equal(c.lastmod_changed, undefined);
|
|
assert.equal(c.stratum, undefined);
|
|
}
|
|
});
|
|
|
|
// --- flag predicates ---------------------------------------------------------
|
|
test('stalenessFlag — true only when lastmod_changed === true', () => {
|
|
assert.equal(stalenessFlag({ lastmod_changed: true }), true);
|
|
assert.equal(stalenessFlag({ lastmod_changed: false }), false);
|
|
assert.equal(stalenessFlag({ lastmod_changed: null }), false);
|
|
});
|
|
test('judgeFlag — true only when judge_verdict === not_grounded', () => {
|
|
assert.equal(judgeFlag({ judge_verdict: 'not_grounded' }), true);
|
|
assert.equal(judgeFlag({ judge_verdict: 'grounded' }), false);
|
|
assert.equal(judgeFlag({ judge_verdict: 'source_silent' }), false);
|
|
});
|
|
test('hybridFlag — union of staleness and judge', () => {
|
|
assert.equal(hybridFlag({ lastmod_changed: true, judge_verdict: 'grounded' }), true);
|
|
assert.equal(hybridFlag({ lastmod_changed: false, judge_verdict: 'not_grounded' }), true);
|
|
assert.equal(hybridFlag({ lastmod_changed: false, judge_verdict: 'source_silent' }), false);
|
|
});
|
|
|
|
// --- gradeArm ----------------------------------------------------------------
|
|
// Verifiable subset of P = ids 1,2,3,4,5,9 (8 excluded: unsourced).
|
|
function verifiableP() {
|
|
return evalPopulation(J).filter((c) => c.verdict !== 'unsourced');
|
|
}
|
|
|
|
test('gradeArm — judge arm confusion matrix', () => {
|
|
const a = gradeArm(verifiableP(), judgeFlag);
|
|
// positives = outdated/wrong = {2,3,5}=3 ; negatives = correct = {1,4,9}=3
|
|
assert.equal(a.positives, 3);
|
|
assert.equal(a.negatives, 3);
|
|
// flags: 2(TP),4(FP),5(TP) ; misses: 3(FN); correct-rejects: 1,9 (TN)
|
|
assert.equal(a.tp, 2);
|
|
assert.equal(a.fp, 1);
|
|
assert.equal(a.fn, 1);
|
|
assert.equal(a.tn, 2);
|
|
assert.ok(Math.abs(a.precision - 2 / 3) < 1e-9);
|
|
assert.ok(Math.abs(a.recall - 2 / 3) < 1e-9);
|
|
assert.ok(Math.abs(a.f1 - 2 / 3) < 1e-9);
|
|
});
|
|
|
|
test('gradeArm — staleness arm catches only the lastmod-changed error', () => {
|
|
const a = gradeArm(verifiableP(), stalenessFlag);
|
|
// only id 5 has lastmod true (a positive) -> tp=1; positives 2,3 missed -> fn=2
|
|
assert.equal(a.tp, 1);
|
|
assert.equal(a.fp, 0);
|
|
assert.equal(a.fn, 2);
|
|
assert.equal(a.tn, 3);
|
|
assert.ok(Math.abs(a.recall - 1 / 3) < 1e-9);
|
|
assert.equal(a.precision, 1); // 1 flag, 1 right
|
|
});
|
|
|
|
test('gradeArm — precision is null when nothing is flagged', () => {
|
|
const noFlag = gradeArm(verifiableP(), () => false);
|
|
assert.equal(noFlag.tp, 0);
|
|
assert.equal(noFlag.fp, 0);
|
|
assert.equal(noFlag.precision, null); // 0/0 -> null, not NaN
|
|
assert.equal(noFlag.recall, 0);
|
|
assert.equal(noFlag.f1, null);
|
|
assert.equal(noFlag.precisionWilson, null); // no flags -> no band
|
|
});
|
|
|
|
test('gradeArm — Wilson bands bracket the point estimates', () => {
|
|
const a = gradeArm(verifiableP(), judgeFlag); // recall .667 (2/3), precision .667 (2/3)
|
|
assert.ok(a.recallWilson.low <= a.recall && a.recall <= a.recallWilson.high);
|
|
assert.ok(a.precisionWilson.low <= a.precision && a.precision <= a.precisionWilson.high);
|
|
});
|
|
|
|
// --- gateDecision ------------------------------------------------------------
|
|
test('gateDecision — passes when judge clears both thresholds and beats staleness', () => {
|
|
const judge = gradeArm(verifiableP(), judgeFlag); // recall .667 prec .667
|
|
const stale = gradeArm(verifiableP(), stalenessFlag); // recall .333
|
|
const g = gateDecision(judge, stale, { minRecall: 0.6, minPrecision: 0.6 });
|
|
assert.equal(g.pass, true);
|
|
assert.equal(g.beatsStaleness, true);
|
|
assert.equal(g.recallOk, true);
|
|
assert.equal(g.precisionOk, true);
|
|
});
|
|
|
|
test('gateDecision — fails when recall below threshold', () => {
|
|
const judge = gradeArm(verifiableP(), judgeFlag); // recall .667
|
|
const stale = gradeArm(verifiableP(), stalenessFlag);
|
|
const g = gateDecision(judge, stale, { minRecall: 0.7, minPrecision: 0.6 });
|
|
assert.equal(g.pass, false);
|
|
assert.equal(g.recallOk, false);
|
|
assert.equal(g.precisionOk, true);
|
|
assert.ok(Array.isArray(g.reasons) && g.reasons.length >= 1);
|
|
});
|
|
|
|
test('gateDecision — beatsStaleness requires strictly greater recall', () => {
|
|
const judge = gradeArm(verifiableP(), judgeFlag); // recall .667
|
|
// staleness with identical recall -> not strictly greater
|
|
const fakeStale = { recall: 0.667, precision: 1, tp: 2, fp: 0, fn: 1, tn: 2, positives: 3, negatives: 3 };
|
|
const g = gateDecision(judge, fakeStale, { minRecall: 0.6, minPrecision: 0.6 });
|
|
assert.equal(g.beatsStaleness, false);
|
|
assert.equal(g.pass, false);
|
|
});
|
|
|
|
// --- computeBakeoff (top-level) ---------------------------------------------
|
|
test('computeBakeoff — wires population, three arms, diagnostics and gate', () => {
|
|
const r = computeBakeoff(J, { minRecall: 0.6, minPrecision: 0.6 });
|
|
assert.equal(r.population.total, 7); // P
|
|
assert.equal(r.population.verifiable, 6);
|
|
assert.equal(r.population.positives, 3);
|
|
assert.equal(r.population.unsourcedInP, 1); // id 8
|
|
// three arms present
|
|
assert.ok(r.arms.judge && r.arms.staleness && r.arms.hybrid);
|
|
assert.equal(r.arms.judge.tp, 2);
|
|
// hybrid = union; here same recall as judge (staleness adds the already-caught #5)
|
|
assert.equal(r.arms.hybrid.tp, 2);
|
|
// source_silent diagnostics
|
|
assert.equal(r.sourceSilent.onVerifiableNegative, 1); // id 9 correct but judge couldn't verify
|
|
assert.equal(r.sourceSilent.agreesWithUnsourced, 1); // id 8 unsourced, judge source_silent
|
|
// gate
|
|
assert.equal(r.gate.pass, true);
|
|
});
|
|
|
|
test('computeBakeoff — per claim_type judge breakdown present', () => {
|
|
const r = computeBakeoff(J, { minRecall: 0.6, minPrecision: 0.6 });
|
|
// version/sku/tpm/status/region/taxonomy each appear in verifiable P
|
|
assert.ok(r.byClaimType.sku);
|
|
assert.equal(r.byClaimType.sku.tp, 1); // id 2 outdated sku, judge ng
|
|
});
|