// tests/kb-eval/test-judge-bakeoff.test.mjs
// Unit tests for the S1 judge bake-off aggregation lib.
//
// S1 de-risks the Fase 3 correctness judge BEFORE any production scaffolding is
// built: run a per-claim groundedness judge against the frozen 373-claim gold set
// and ask whether it beats the cheap staleness baseline (recall 0/40) by enough to
// justify ~2700 non-batchable fetches per full pass.
//
// The eval population P is volatile + fetchable claim_types (price excluded) — the
// judge is measured exactly where the real errors live, which structurally avoids
// the "inverted leverage" trap (a judge that wins only by auto-scoring cheap stable
// claims). Grading is a pure detection confusion matrix vs the gold verdicts:
//   gold positive  = verdict ∈ {outdated, wrong}   (a real error to catch)
//   gold negative  = verdict === 'correct'
//   excluded       = verdict === 'unsourced'        (no ground-truth value)
//   predicted positive (flag) = judge_verdict === 'not_grounded'
// All computation is deterministic — no I/O, no Date.now/Math.random.

import { test } from 'node:test';
import assert from 'node:assert/strict';
import {
  evalPopulation,
  blindClaims,
  gradeArm,
  stalenessFlag,
  judgeFlag,
  hybridFlag,
  gateDecision,
  computeBakeoff,
} from '../../scripts/kb-eval/lib/judge-bakeoff.mjs';

// --- synthetic joined gold+judge set ----------------------------------------
// Each claim carries its gold fields plus the (blind) judge_verdict the harness
// joined back by id. Comments give the intended confusion-matrix cell.
const J = [
  // 1: volatile/version, correct, judge grounded            -> P, verifiable, TN
  { id: 'a#1', skill: 'x', stratum: 'volatile', claim_type: 'version', verdict: 'correct', lastmod_changed: false, judge_verdict: 'grounded' },
  // 2: volatile/sku, outdated, judge not_grounded           -> P, TP (judge catches)
  { id: 'a#2', skill: 'x', stratum: 'volatile', claim_type: 'sku', verdict: 'outdated', lastmod_changed: false, judge_verdict: 'not_grounded' },
  // 3: volatile/tpm, wrong, judge grounded                  -> P, FN (judge misses)
  { id: 'a#3', skill: 'x', stratum: 'volatile', claim_type: 'tpm', verdict: 'wrong', lastmod_changed: false, judge_verdict: 'grounded' },
  // 4: volatile/status, correct, judge not_grounded         -> P, FP (false alarm)
  { id: 'a#4', skill: 'x', stratum: 'volatile', claim_type: 'status', verdict: 'correct', lastmod_changed: false, judge_verdict: 'not_grounded' },
  // 5: volatile/region, outdated, lastmod TRUE, judge ng    -> P, TP; staleness also catches
  { id: 'a#5', skill: 'x', stratum: 'volatile', claim_type: 'region', verdict: 'outdated', lastmod_changed: true, judge_verdict: 'not_grounded' },
  // 6: volatile/PRICE, wrong                                -> excluded from P (price)
  { id: 'a#6', skill: 'x', stratum: 'volatile', claim_type: 'price', verdict: 'wrong', lastmod_changed: false, judge_verdict: 'not_grounded' },
  // 7: CONTROL/taxonomy, wrong                              -> excluded from P (control stratum)
  { id: 'a#7', skill: 'x', stratum: 'control', claim_type: 'taxonomy', verdict: 'wrong', lastmod_changed: false, judge_verdict: 'not_grounded' },
  // 8: volatile/status, unsourced, judge source_silent      -> P but not verifiable (excluded from P/R)
  { id: 'a#8', skill: 'x', stratum: 'volatile', claim_type: 'status', verdict: 'unsourced', lastmod_changed: false, judge_verdict: 'source_silent' },
  // 9: volatile/taxonomy, correct, judge source_silent      -> P, verifiable negative, not flagged -> TN
  { id: 'a#9', skill: 'x', stratum: 'volatile', claim_type: 'taxonomy', verdict: 'correct', lastmod_changed: false, judge_verdict: 'source_silent' },
];

// --- evalPopulation ----------------------------------------------------------
test('evalPopulation — keeps volatile + fetchable, drops price and control', () => {
  const P = evalPopulation(J);
  const ids = P.map((c) => c.id).sort();
  assert.deepEqual(ids, ['a#1', 'a#2', 'a#3', 'a#4', 'a#5', 'a#8', 'a#9']);
});

// --- blindClaims (no label leakage) -----------------------------------------
test('blindClaims — emits all of P with only the allowed blind fields', () => {
  const blind = blindClaims(J);
  assert.equal(blind.length, 7); // all of P, unsourced included
  const allowed = new Set(['id', 'file', 'skill', 'claim', 'claim_type', 'evidence_url']);
  for (const c of blind) {
    for (const k of Object.keys(c)) {
      assert.ok(allowed.has(k), `leaked field: ${k}`);
    }
    // the answer key must never appear
    assert.equal(c.verdict, undefined);
    assert.equal(c.notes, undefined);
    assert.equal(c.lastmod_changed, undefined);
    assert.equal(c.stratum, undefined);
  }
});

// --- flag predicates ---------------------------------------------------------
test('stalenessFlag — true only when lastmod_changed === true', () => {
  assert.equal(stalenessFlag({ lastmod_changed: true }), true);
  assert.equal(stalenessFlag({ lastmod_changed: false }), false);
  assert.equal(stalenessFlag({ lastmod_changed: null }), false);
});
test('judgeFlag — true only when judge_verdict === not_grounded', () => {
  assert.equal(judgeFlag({ judge_verdict: 'not_grounded' }), true);
  assert.equal(judgeFlag({ judge_verdict: 'grounded' }), false);
  assert.equal(judgeFlag({ judge_verdict: 'source_silent' }), false);
});
test('hybridFlag — union of staleness and judge', () => {
  assert.equal(hybridFlag({ lastmod_changed: true, judge_verdict: 'grounded' }), true);
  assert.equal(hybridFlag({ lastmod_changed: false, judge_verdict: 'not_grounded' }), true);
  assert.equal(hybridFlag({ lastmod_changed: false, judge_verdict: 'source_silent' }), false);
});

// --- gradeArm ----------------------------------------------------------------
// Verifiable subset of P = ids 1,2,3,4,5,9 (8 excluded: unsourced).
function verifiableP() {
  return evalPopulation(J).filter((c) => c.verdict !== 'unsourced');
}

test('gradeArm — judge arm confusion matrix', () => {
  const a = gradeArm(verifiableP(), judgeFlag);
  // positives = outdated/wrong = {2,3,5}=3 ; negatives = correct = {1,4,9}=3
  assert.equal(a.positives, 3);
  assert.equal(a.negatives, 3);
  // flags: 2(TP),4(FP),5(TP) ; misses: 3(FN); correct-rejects: 1,9 (TN)
  assert.equal(a.tp, 2);
  assert.equal(a.fp, 1);
  assert.equal(a.fn, 1);
  assert.equal(a.tn, 2);
  assert.ok(Math.abs(a.precision - 2 / 3) < 1e-9);
  assert.ok(Math.abs(a.recall - 2 / 3) < 1e-9);
  assert.ok(Math.abs(a.f1 - 2 / 3) < 1e-9);
});

test('gradeArm — staleness arm catches only the lastmod-changed error', () => {
  const a = gradeArm(verifiableP(), stalenessFlag);
  // only id 5 has lastmod true (a positive) -> tp=1; positives 2,3 missed -> fn=2
  assert.equal(a.tp, 1);
  assert.equal(a.fp, 0);
  assert.equal(a.fn, 2);
  assert.equal(a.tn, 3);
  assert.ok(Math.abs(a.recall - 1 / 3) < 1e-9);
  assert.equal(a.precision, 1); // 1 flag, 1 right
});

test('gradeArm — precision is null when nothing is flagged', () => {
  const noFlag = gradeArm(verifiableP(), () => false);
  assert.equal(noFlag.tp, 0);
  assert.equal(noFlag.fp, 0);
  assert.equal(noFlag.precision, null); // 0/0 -> null, not NaN
  assert.equal(noFlag.recall, 0);
  assert.equal(noFlag.f1, null);
  assert.equal(noFlag.precisionWilson, null); // no flags -> no band
});

test('gradeArm — Wilson bands bracket the point estimates', () => {
  const a = gradeArm(verifiableP(), judgeFlag); // recall .667 (2/3), precision .667 (2/3)
  assert.ok(a.recallWilson.low <= a.recall && a.recall <= a.recallWilson.high);
  assert.ok(a.precisionWilson.low <= a.precision && a.precision <= a.precisionWilson.high);
});

// --- gateDecision ------------------------------------------------------------
test('gateDecision — passes when judge clears both thresholds and beats staleness', () => {
  const judge = gradeArm(verifiableP(), judgeFlag); // recall .667 prec .667
  const stale = gradeArm(verifiableP(), stalenessFlag); // recall .333
  const g = gateDecision(judge, stale, { minRecall: 0.6, minPrecision: 0.6 });
  assert.equal(g.pass, true);
  assert.equal(g.beatsStaleness, true);
  assert.equal(g.recallOk, true);
  assert.equal(g.precisionOk, true);
});

test('gateDecision — fails when recall below threshold', () => {
  const judge = gradeArm(verifiableP(), judgeFlag); // recall .667
  const stale = gradeArm(verifiableP(), stalenessFlag);
  const g = gateDecision(judge, stale, { minRecall: 0.7, minPrecision: 0.6 });
  assert.equal(g.pass, false);
  assert.equal(g.recallOk, false);
  assert.equal(g.precisionOk, true);
  assert.ok(Array.isArray(g.reasons) && g.reasons.length >= 1);
});

test('gateDecision — beatsStaleness requires strictly greater recall', () => {
  const judge = gradeArm(verifiableP(), judgeFlag); // recall .667
  // staleness with identical recall -> not strictly greater
  const fakeStale = { recall: 0.667, precision: 1, tp: 2, fp: 0, fn: 1, tn: 2, positives: 3, negatives: 3 };
  const g = gateDecision(judge, fakeStale, { minRecall: 0.6, minPrecision: 0.6 });
  assert.equal(g.beatsStaleness, false);
  assert.equal(g.pass, false);
});

// --- computeBakeoff (top-level) ---------------------------------------------
test('computeBakeoff — wires population, three arms, diagnostics and gate', () => {
  const r = computeBakeoff(J, { minRecall: 0.6, minPrecision: 0.6 });
  assert.equal(r.population.total, 7); // P
  assert.equal(r.population.verifiable, 6);
  assert.equal(r.population.positives, 3);
  assert.equal(r.population.unsourcedInP, 1); // id 8
  // three arms present
  assert.ok(r.arms.judge && r.arms.staleness && r.arms.hybrid);
  assert.equal(r.arms.judge.tp, 2);
  // hybrid = union; here same recall as judge (staleness adds the already-caught #5)
  assert.equal(r.arms.hybrid.tp, 2);
  // source_silent diagnostics
  assert.equal(r.sourceSilent.onVerifiableNegative, 1); // id 9 correct but judge couldn't verify
  assert.equal(r.sourceSilent.agreesWithUnsourced, 1);  // id 8 unsourced, judge source_silent
  // gate
  assert.equal(r.gate.pass, true);
});

test('computeBakeoff — per claim_type judge breakdown present', () => {
  const r = computeBakeoff(J, { minRecall: 0.6, minPrecision: 0.6 });
  // version/sku/tpm/status/region/taxonomy each appear in verifiable P
  assert.ok(r.byClaimType.sku);
  assert.equal(r.byClaimType.sku.tp, 1); // id 2 outdated sku, judge ng
});