ms-ai-architect/scripts/kb-eval/run-judge-bakeoff.mjs

#!/usr/bin/env node
// run-judge-bakeoff.mjs — S1 final glue: join the blind judge results back to the
// frozen gold set, grade the judge / staleness / hybrid arms, apply the
// PRE-REGISTERED gate, and write the bake-off report. All math lives in tested
// lib/judge-bakeoff.mjs; this CLI is thin wiring (same shape as compute-base-rate.mjs).
//
// The thresholds are required flags, not defaults: the gate must be locked BEFORE the
// judge fan-out runs (pre-registration), and the locked values are recorded in the
// report _meta so the decision is auditable.
//
// Usage:
//   node scripts/kb-eval/run-judge-bakeoff.mjs --min-recall 0.70 --min-precision 0.60 [--write]
//
// Inputs:  data/gold-correctness-set.json  (answer key)
//          data/judge-bakeoff-results.json  ({ results: [{id, judge_verdict, ...}] })
// Outputs: data/judge-bakeoff-report.{json,md}  (with --write)

import fs from 'node:fs';
import path from 'node:path';
import { fileURLToPath } from 'node:url';
import { computeBakeoff, evalPopulation } from './lib/judge-bakeoff.mjs';

const __dirname = path.dirname(fileURLToPath(import.meta.url));
const DATA = path.join(__dirname, 'data');

function flag(name) {
  const i = process.argv.indexOf(name);
  return i >= 0 ? process.argv[i + 1] : undefined;
}
const minRecall = Number(flag('--min-recall'));
const minPrecision = Number(flag('--min-precision'));
if (!Number.isFinite(minRecall) || !Number.isFinite(minPrecision)) {
  console.error('error: --min-recall and --min-precision are required (pre-registered gate)');
  process.exit(2);
}
const thresholds = { minRecall, minPrecision };

const gold = JSON.parse(fs.readFileSync(path.join(DATA, 'gold-correctness-set.json'), 'utf8'));
// --results / --report-prefix let a second iteration (v2) be graded without
// clobbering the frozen v1 artifacts. Defaults preserve the v1 file names.
const resultsPath = path.join(DATA, flag('--results') || 'judge-bakeoff-results.json');
const reportPrefix = flag('--report-prefix') || 'judge-bakeoff-report';
if (!fs.existsSync(resultsPath)) {
  console.error(`error: ${resultsPath} not found — run the judge fan-out first`);
  process.exit(2);
}
const judge = JSON.parse(fs.readFileSync(resultsPath, 'utf8'));

// Join judge verdicts back to gold by id.
const verdictById = new Map((judge.results || []).map((r) => [r.id, r.judge_verdict]));
const joined = gold.claims.map((c) => ({ ...c, judge_verdict: verdictById.get(c.id) }));

// Completeness check: every claim in P must have a judge verdict.
const missing = evalPopulation(joined).filter((c) => c.judge_verdict == null);
if (missing.length && !process.argv.includes('--allow-incomplete')) {
  console.error(`error: ${missing.length} P-claims have no judge verdict (incomplete fan-out).`);
  console.error('first few:', missing.slice(0, 5).map((c) => c.id).join(', '));
  console.error('pass --allow-incomplete to grade anyway (missing claims count as not-flagged).');
  process.exit(2);
}

const r = computeBakeoff(joined, thresholds);

const pct = (x) => (x == null ? 'n/a' : `${(x * 100).toFixed(1)}%`);
const num = (x) => (x == null ? 'n/a' : x.toFixed(3));
const band = (w) => (w == null ? 'n/a' : `[${pct(w.low)}, ${pct(w.high)}]`);

function armRow(name, a) {
  return `| ${name} | ${a.tp} | ${a.fp} | ${a.fn} | ${a.tn} | ${pct(a.precision)} | ${pct(a.recall)} | ${band(a.recallWilson)} | ${num(a.f1)} |`;
}
function typeRow(t, a) {
  return `| ${t} | ${a.positives} | ${a.tp} | ${a.fp} | ${a.fn} | ${pct(a.precision)} | ${pct(a.recall)} |`;
}

const p = r.population;
const g = r.gate;
const ss = r.sourceSilent;
const verdict = g.pass ? '✅ PASS — bygg S3' : '❌ FAIL — stopp, ikke bygg S3';

const md = `# Judge bake-off-rapport — S1 (Fase 3 de-risk)

_Generert deterministisk av \`run-judge-bakeoff.mjs\` over \`gold-correctness-set.json\` + \`judge-bakeoff-results.json\`. Tall fra testet \`lib/judge-bakeoff.mjs\`. Ikke rediger for hånd — regenerer._

**Forhåndsregistrert gate (låst FØR fan-out):** recall ≥ ${minRecall}, presisjon ≥ ${minPrecision}, OG judge-recall > staleness-recall.

## Evaluerings-populasjon (P)

Volatil stratum + fetchbare claim_types (price ekskludert) — der feilene bor; unngår «invertert leverage».

| metrikk | verdi |
|---|---|
| P totalt | ${p.total} |
| Verifiserbare (correct/outdated/wrong) | ${p.verifiable} |
| Positive (reelle feil å fange) | ${p.positives} |
| Negative (correct) | ${p.negatives} |
| Unsourced i P (kjørt, men utenfor P/R) | ${p.unsourcedInP} |

## Arm-sammenligning (detektering over de ${p.verifiable} verifiserbare)

| arm | TP | FP | FN | TN | presisjon | recall | recall Wilson 95% | F1 |
|---|---|---|---|---|---|---|---|---|
${armRow('staleness (billig baseline)', r.arms.staleness)}
${armRow('judge (per-påstand groundedness)', r.arms.judge)}
${armRow('hybrid (union)', r.arms.hybrid)}

## Judge per claim_type (verifiserbar delmengde)

| claim_type | positive | TP | FP | FN | presisjon | recall |
|---|---|---|---|---|---|---|
${Object.entries(r.byClaimType).sort((a, b) => b[1].positives - a[1].positives).map(([t, a]) => typeRow(t, a)).join('\n')}

## source_silent-diagnostikk

Judgen hentet siden men fant ikke verdien. Diagnostisk, ikke et flagg.

| signal | antall | tolkning |
|---|---|---|
| På verifiserbar feil | ${ss.onVerifiableError} | judge-bom: reell feil oversett via «kan ikke verifisere» |
| På verifiserbar correct | ${ss.onVerifiableNegative} | judge reproduserte ikke et korrekt faktum mennesket fant |
| Enig med unsourced | ${ss.agreesWithUnsourced} | judge reproduserer den uverifiserbare grensen (godt) |
| Uenig med unsourced | ${ss.disagreesWithUnsourced} | judge hevdet grunnet/ugrunnet der mennesket ikke fant kilde |

## GATE: ${verdict}

- recall ${num(r.arms.judge.recall)} ≥ ${minRecall}? **${g.recallOk ? 'ja' : 'nei'}**
- presisjon ${num(r.arms.judge.precision)} ≥ ${minPrecision}? **${g.precisionOk ? 'ja' : 'nei'}**
- slår staleness (recall ${num(r.arms.staleness.recall)})? **${g.beatsStaleness ? 'ja' : 'nei'}**
- begrunnelse: ${g.reasons.join('; ')}
`;

if (process.argv.includes('--write')) {
  const jsonOut = path.join(DATA, `${reportPrefix}.json`);
  const mdOut = path.join(DATA, `${reportPrefix}.md`);
  fs.writeFileSync(
    jsonOut,
    JSON.stringify(
      {
        _meta: {
          source: 'gold-correctness-set.json + judge-bakeoff-results.json',
          thresholds,
          judged: (judge.results || []).length,
        },
        ...r,
      },
      null,
      2,
    ) + '\n',
  );
  fs.writeFileSync(mdOut, md);
  console.log(`wrote ${jsonOut}`);
  console.log(`wrote ${mdOut}`);
} else {
  console.log(`P=${p.total} verifiable=${p.verifiable} positives=${p.positives}`);
  console.log(`judge:     precision=${num(r.arms.judge.precision)} recall=${num(r.arms.judge.recall)} f1=${num(r.arms.judge.f1)}`);
  console.log(`staleness: recall=${num(r.arms.staleness.recall)}`);
  console.log(`GATE: ${g.pass ? 'PASS' : 'FAIL'} — ${g.reasons.join('; ')}`);
  console.log('(dry run — pass --write to persist judge-bakeoff-report.json + .md)');
}