Kjell Tore Guttormsen
3e39f2df6b
Deterministisk de-risk-harness for Fase 3-judgen (kjøres på frosset 373-påstands gull-sett): - lib/judge-bakeoff.mjs: P-filter (volatil+fetchbar, price ekskl.), confusion-matrix for 3 armer (staleness/judge/hybrid), Wilson-bånd, forhåndsregistrert gate. 14 tester. - extract-judge-claims.mjs: blind manifest (255 påstander, 0 label-lekkasje — testet invariant). - judge-claim-prompt.md: blind per-påstands groundedness-judge (Opus xhigh, microsoft_docs_fetch). - run-judge-bakeoff.mjs: join gull+results på id, gate-rapport (.json/.md). Gate FORHÅNDSREGISTRERT (operatørvalg, før fan-out): recall ≥0.80, presisjon ≥0.70, OG slår staleness (0/38). Evalueringspop P = 240 verifiserbare, 38 positive. Suite 551/551 (538 + 13 nye).
155 lines
6.7 KiB
JavaScript
155 lines
6.7 KiB
JavaScript
#!/usr/bin/env node
|
|
// run-judge-bakeoff.mjs — S1 final glue: join the blind judge results back to the
|
|
// frozen gold set, grade the judge / staleness / hybrid arms, apply the
|
|
// PRE-REGISTERED gate, and write the bake-off report. All math lives in tested
|
|
// lib/judge-bakeoff.mjs; this CLI is thin wiring (same shape as compute-base-rate.mjs).
|
|
//
|
|
// The thresholds are required flags, not defaults: the gate must be locked BEFORE the
|
|
// judge fan-out runs (pre-registration), and the locked values are recorded in the
|
|
// report _meta so the decision is auditable.
|
|
//
|
|
// Usage:
|
|
// node scripts/kb-eval/run-judge-bakeoff.mjs --min-recall 0.70 --min-precision 0.60 [--write]
|
|
//
|
|
// Inputs: data/gold-correctness-set.json (answer key)
|
|
// data/judge-bakeoff-results.json ({ results: [{id, judge_verdict, ...}] })
|
|
// Outputs: data/judge-bakeoff-report.{json,md} (with --write)
|
|
|
|
import fs from 'node:fs';
|
|
import path from 'node:path';
|
|
import { fileURLToPath } from 'node:url';
|
|
import { computeBakeoff, evalPopulation } from './lib/judge-bakeoff.mjs';
|
|
|
|
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
|
const DATA = path.join(__dirname, 'data');
|
|
|
|
function flag(name) {
|
|
const i = process.argv.indexOf(name);
|
|
return i >= 0 ? process.argv[i + 1] : undefined;
|
|
}
|
|
const minRecall = Number(flag('--min-recall'));
|
|
const minPrecision = Number(flag('--min-precision'));
|
|
if (!Number.isFinite(minRecall) || !Number.isFinite(minPrecision)) {
|
|
console.error('error: --min-recall and --min-precision are required (pre-registered gate)');
|
|
process.exit(2);
|
|
}
|
|
const thresholds = { minRecall, minPrecision };
|
|
|
|
const gold = JSON.parse(fs.readFileSync(path.join(DATA, 'gold-correctness-set.json'), 'utf8'));
|
|
const resultsPath = path.join(DATA, 'judge-bakeoff-results.json');
|
|
if (!fs.existsSync(resultsPath)) {
|
|
console.error(`error: ${resultsPath} not found — run the judge fan-out first`);
|
|
process.exit(2);
|
|
}
|
|
const judge = JSON.parse(fs.readFileSync(resultsPath, 'utf8'));
|
|
|
|
// Join judge verdicts back to gold by id.
|
|
const verdictById = new Map((judge.results || []).map((r) => [r.id, r.judge_verdict]));
|
|
const joined = gold.claims.map((c) => ({ ...c, judge_verdict: verdictById.get(c.id) }));
|
|
|
|
// Completeness check: every claim in P must have a judge verdict.
|
|
const missing = evalPopulation(joined).filter((c) => c.judge_verdict == null);
|
|
if (missing.length && !process.argv.includes('--allow-incomplete')) {
|
|
console.error(`error: ${missing.length} P-claims have no judge verdict (incomplete fan-out).`);
|
|
console.error('first few:', missing.slice(0, 5).map((c) => c.id).join(', '));
|
|
console.error('pass --allow-incomplete to grade anyway (missing claims count as not-flagged).');
|
|
process.exit(2);
|
|
}
|
|
|
|
const r = computeBakeoff(joined, thresholds);
|
|
|
|
const pct = (x) => (x == null ? 'n/a' : `${(x * 100).toFixed(1)}%`);
|
|
const num = (x) => (x == null ? 'n/a' : x.toFixed(3));
|
|
const band = (w) => (w == null ? 'n/a' : `[${pct(w.low)}, ${pct(w.high)}]`);
|
|
|
|
function armRow(name, a) {
|
|
return `| ${name} | ${a.tp} | ${a.fp} | ${a.fn} | ${a.tn} | ${pct(a.precision)} | ${pct(a.recall)} | ${band(a.recallWilson)} | ${num(a.f1)} |`;
|
|
}
|
|
function typeRow(t, a) {
|
|
return `| ${t} | ${a.positives} | ${a.tp} | ${a.fp} | ${a.fn} | ${pct(a.precision)} | ${pct(a.recall)} |`;
|
|
}
|
|
|
|
const p = r.population;
|
|
const g = r.gate;
|
|
const ss = r.sourceSilent;
|
|
const verdict = g.pass ? '✅ PASS — bygg S3' : '❌ FAIL — stopp, ikke bygg S3';
|
|
|
|
const md = `# Judge bake-off-rapport — S1 (Fase 3 de-risk)
|
|
|
|
_Generert deterministisk av \`run-judge-bakeoff.mjs\` over \`gold-correctness-set.json\` + \`judge-bakeoff-results.json\`. Tall fra testet \`lib/judge-bakeoff.mjs\`. Ikke rediger for hånd — regenerer._
|
|
|
|
**Forhåndsregistrert gate (låst FØR fan-out):** recall ≥ ${minRecall}, presisjon ≥ ${minPrecision}, OG judge-recall > staleness-recall.
|
|
|
|
## Evaluerings-populasjon (P)
|
|
|
|
Volatil stratum + fetchbare claim_types (price ekskludert) — der feilene bor; unngår «invertert leverage».
|
|
|
|
| metrikk | verdi |
|
|
|---|---|
|
|
| P totalt | ${p.total} |
|
|
| Verifiserbare (correct/outdated/wrong) | ${p.verifiable} |
|
|
| Positive (reelle feil å fange) | ${p.positives} |
|
|
| Negative (correct) | ${p.negatives} |
|
|
| Unsourced i P (kjørt, men utenfor P/R) | ${p.unsourcedInP} |
|
|
|
|
## Arm-sammenligning (detektering over de ${p.verifiable} verifiserbare)
|
|
|
|
| arm | TP | FP | FN | TN | presisjon | recall | recall Wilson 95% | F1 |
|
|
|---|---|---|---|---|---|---|---|---|
|
|
${armRow('staleness (billig baseline)', r.arms.staleness)}
|
|
${armRow('judge (per-påstand groundedness)', r.arms.judge)}
|
|
${armRow('hybrid (union)', r.arms.hybrid)}
|
|
|
|
## Judge per claim_type (verifiserbar delmengde)
|
|
|
|
| claim_type | positive | TP | FP | FN | presisjon | recall |
|
|
|---|---|---|---|---|---|---|
|
|
${Object.entries(r.byClaimType).sort((a, b) => b[1].positives - a[1].positives).map(([t, a]) => typeRow(t, a)).join('\n')}
|
|
|
|
## source_silent-diagnostikk
|
|
|
|
Judgen hentet siden men fant ikke verdien. Diagnostisk, ikke et flagg.
|
|
|
|
| signal | antall | tolkning |
|
|
|---|---|---|
|
|
| På verifiserbar feil | ${ss.onVerifiableError} | judge-bom: reell feil oversett via «kan ikke verifisere» |
|
|
| På verifiserbar correct | ${ss.onVerifiableNegative} | judge reproduserte ikke et korrekt faktum mennesket fant |
|
|
| Enig med unsourced | ${ss.agreesWithUnsourced} | judge reproduserer den uverifiserbare grensen (godt) |
|
|
| Uenig med unsourced | ${ss.disagreesWithUnsourced} | judge hevdet grunnet/ugrunnet der mennesket ikke fant kilde |
|
|
|
|
## GATE: ${verdict}
|
|
|
|
- recall ${num(r.arms.judge.recall)} ≥ ${minRecall}? **${g.recallOk ? 'ja' : 'nei'}**
|
|
- presisjon ${num(r.arms.judge.precision)} ≥ ${minPrecision}? **${g.precisionOk ? 'ja' : 'nei'}**
|
|
- slår staleness (recall ${num(r.arms.staleness.recall)})? **${g.beatsStaleness ? 'ja' : 'nei'}**
|
|
- begrunnelse: ${g.reasons.join('; ')}
|
|
`;
|
|
|
|
if (process.argv.includes('--write')) {
|
|
const jsonOut = path.join(DATA, 'judge-bakeoff-report.json');
|
|
const mdOut = path.join(DATA, 'judge-bakeoff-report.md');
|
|
fs.writeFileSync(
|
|
jsonOut,
|
|
JSON.stringify(
|
|
{
|
|
_meta: {
|
|
source: 'gold-correctness-set.json + judge-bakeoff-results.json',
|
|
thresholds,
|
|
judged: (judge.results || []).length,
|
|
},
|
|
...r,
|
|
},
|
|
null,
|
|
2,
|
|
) + '\n',
|
|
);
|
|
fs.writeFileSync(mdOut, md);
|
|
console.log(`wrote ${jsonOut}`);
|
|
console.log(`wrote ${mdOut}`);
|
|
} else {
|
|
console.log(`P=${p.total} verifiable=${p.verifiable} positives=${p.positives}`);
|
|
console.log(`judge: precision=${num(r.arms.judge.precision)} recall=${num(r.arms.judge.recall)} f1=${num(r.arms.judge.f1)}`);
|
|
console.log(`staleness: recall=${num(r.arms.staleness.recall)}`);
|
|
console.log(`GATE: ${g.pass ? 'PASS' : 'FAIL'} — ${g.reasons.join('; ')}`);
|
|
console.log('(dry run — pass --write to persist judge-bakeoff-report.json + .md)');
|
|
}
|