Kjell Tore Guttormsen
440594f1b2
Scores committed agent-run fixtures against the golden corpus at (file, rule_key) granularity, building on the deterministic coordinator contract (4a). Offline: committed reviewer payloads, no live agent spawn, no LLM, no network (the LLM-in-the-loop grading is the separate 4c tier). - lib/review/gold-scorer.mjs: scoreFindings (precision/recall/f1 at (file,rule_key) granularity, line+severity ignored) + scoreVerdict; pure, with documented vacuous-set conventions. - tests/fixtures/bakeoff-rich/runs/run-perfect.json: committed run that reproduces all 5 seeded gold findings through runContract. - tests/lib/gold-eval.test.mjs: the scoring RUN (precision/recall/f1 = 1.0, verdict == expected_verdict BLOCK, nothing suppressed/skipped). - lib/util/test-census.mjs: third census category (goldEval) — a scoring run is neither behavior coverage nor a doc-pin; honest-count invariant now 3-way. - docs/eval-corpus/README.md: 4b moved from Future hardening to implemented. Suite 809 -> 822 (820/0/2). gold-scorer covers TP+FP+FN+degenerate paths. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01BJQYC5vpkJWxndS55vQQZ6
49 lines
2.5 KiB
JavaScript
49 lines
2.5 KiB
JavaScript
// tests/lib/gold-eval.test.mjs
|
|
// SKAL-1·4b — the offline gold-scored output eval (the scoring RUN).
|
|
//
|
|
// This is the third test-census category (see lib/util/test-census.mjs):
|
|
// neither a behavior unit test nor a doc-consistency pin, but a SCORING RUN —
|
|
// it feeds a committed agent-run fixture through the deterministic coordinator
|
|
// contract (4a) and scores the result against the golden corpus at
|
|
// (file, rule_key) granularity. Offline: committed reviewer payloads, no live
|
|
// agent spawn, no LLM, no network. The all-agree foundation; the
|
|
// LLM-in-the-loop eval is the separate 4c tier.
|
|
|
|
import { test } from 'node:test';
|
|
import { strict as assert } from 'node:assert';
|
|
import { readFileSync } from 'node:fs';
|
|
import { join, dirname } from 'node:path';
|
|
import { fileURLToPath } from 'node:url';
|
|
import { runContract } from '../../lib/review/coordinator-contract.mjs';
|
|
import { scoreFindings, scoreVerdict } from '../../lib/review/gold-scorer.mjs';
|
|
|
|
const HERE = dirname(fileURLToPath(import.meta.url));
|
|
const ROOT = join(HERE, '..', '..');
|
|
const gold = JSON.parse(readFileSync(join(ROOT, 'tests/fixtures/bakeoff-rich/gold.json'), 'utf-8'));
|
|
const runPerfect = JSON.parse(
|
|
readFileSync(join(ROOT, 'tests/fixtures/bakeoff-rich/runs/run-perfect.json'), 'utf-8'),
|
|
);
|
|
|
|
test('gold-eval — run-perfect reproduces gold at (file,rule_key): precision/recall/f1 = 1.0', () => {
|
|
const result = runContract(runPerfect);
|
|
const score = scoreFindings(result.findings, gold.findings);
|
|
assert.equal(score.precision, 1, `precision ${score.precision}; spurious=${JSON.stringify(score.spurious)}`);
|
|
assert.equal(score.recall, 1, `recall ${score.recall}; missed=${JSON.stringify(score.missed)}`);
|
|
assert.equal(score.f1, 1);
|
|
assert.equal(score.tp, gold.findings.length);
|
|
});
|
|
|
|
test('gold-eval — run-perfect coordinator verdict matches gold expected_verdict (BLOCK)', () => {
|
|
const result = runContract(runPerfect);
|
|
assert.equal(result.verdict, gold.expected_verdict);
|
|
assert.ok(scoreVerdict(result.verdict, gold.expected_verdict));
|
|
});
|
|
|
|
test('gold-eval — run-perfect drops nothing (no suppressed, no schema-skipped payloads)', () => {
|
|
// A clean run is the regression guard: any future contract change that
|
|
// silently suppresses or skips one of the 5 seeded findings breaks here.
|
|
const result = runContract(runPerfect);
|
|
assert.equal(result.skipped.length, 0, `skipped payloads: ${JSON.stringify(result.skipped)}`);
|
|
assert.equal(result.suppressed.length, 0, `suppressed findings: ${result.suppressed.length}`);
|
|
assert.equal(result.findings.length, gold.findings.length);
|
|
});
|