Kjell Tore Guttormsen
5f42bbdb9e
The trekcontinue-stats line_ref follows Phase 5 to 271-275 (the Phase 1.5 cut moved it). gold-eval.test.mjs and its run fixture come back: they are the one end-to-end check that the review contract keeps every seeded finding, and they stay with the scorer until R-b. CHANGELOG: the removed validator leaves the known-issues text; the review's other findings outside R-a's files are listed as known issues. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
49 lines
2.5 KiB
JavaScript
49 lines
2.5 KiB
JavaScript
// tests/lib/gold-eval.test.mjs
|
|
// SKAL-1·4b — the offline gold-scored output eval (the scoring RUN).
|
|
//
|
|
// This is the third test-census category (see lib/util/test-census.mjs):
|
|
// neither a behavior unit test nor a doc-consistency pin, but a SCORING RUN —
|
|
// it feeds a committed agent-run fixture through the deterministic coordinator
|
|
// contract (4a) and scores the result against the golden corpus at
|
|
// (file, rule_key) granularity. Offline: committed reviewer payloads, no live
|
|
// agent spawn, no LLM, no network. The all-agree foundation; the
|
|
// LLM-in-the-loop eval is the separate 4c tier.
|
|
|
|
import { test } from 'node:test';
|
|
import { strict as assert } from 'node:assert';
|
|
import { readFileSync } from 'node:fs';
|
|
import { join, dirname } from 'node:path';
|
|
import { fileURLToPath } from 'node:url';
|
|
import { runContract } from '../../lib/review/coordinator-contract.mjs';
|
|
import { scoreFindings, scoreVerdict } from '../../lib/review/gold-scorer.mjs';
|
|
|
|
const HERE = dirname(fileURLToPath(import.meta.url));
|
|
const ROOT = join(HERE, '..', '..');
|
|
const gold = JSON.parse(readFileSync(join(ROOT, 'tests/fixtures/bakeoff-rich/gold.json'), 'utf-8'));
|
|
const runPerfect = JSON.parse(
|
|
readFileSync(join(ROOT, 'tests/fixtures/bakeoff-rich/runs/run-perfect.json'), 'utf-8'),
|
|
);
|
|
|
|
test('gold-eval — run-perfect reproduces gold at (file,rule_key): precision/recall/f1 = 1.0', () => {
|
|
const result = runContract(runPerfect);
|
|
const score = scoreFindings(result.findings, gold.findings);
|
|
assert.equal(score.precision, 1, `precision ${score.precision}; spurious=${JSON.stringify(score.spurious)}`);
|
|
assert.equal(score.recall, 1, `recall ${score.recall}; missed=${JSON.stringify(score.missed)}`);
|
|
assert.equal(score.f1, 1);
|
|
assert.equal(score.tp, gold.findings.length);
|
|
});
|
|
|
|
test('gold-eval — run-perfect coordinator verdict matches gold expected_verdict (BLOCK)', () => {
|
|
const result = runContract(runPerfect);
|
|
assert.equal(result.verdict, gold.expected_verdict);
|
|
assert.ok(scoreVerdict(result.verdict, gold.expected_verdict));
|
|
});
|
|
|
|
test('gold-eval — run-perfect drops nothing (no suppressed, no schema-skipped payloads)', () => {
|
|
// A clean run is the regression guard: any future contract change that
|
|
// silently suppresses or skips one of the 5 seeded findings breaks here.
|
|
const result = runContract(runPerfect);
|
|
assert.equal(result.skipped.length, 0, `skipped payloads: ${JSON.stringify(result.skipped)}`);
|
|
assert.equal(result.suppressed.length, 0, `suppressed findings: ${result.suppressed.length}`);
|
|
assert.equal(result.findings.length, gold.findings.length);
|
|
});
|