Kjell Tore Guttormsen
440594f1b2
Scores committed agent-run fixtures against the golden corpus at (file, rule_key) granularity, building on the deterministic coordinator contract (4a). Offline: committed reviewer payloads, no live agent spawn, no LLM, no network (the LLM-in-the-loop grading is the separate 4c tier). - lib/review/gold-scorer.mjs: scoreFindings (precision/recall/f1 at (file,rule_key) granularity, line+severity ignored) + scoreVerdict; pure, with documented vacuous-set conventions. - tests/fixtures/bakeoff-rich/runs/run-perfect.json: committed run that reproduces all 5 seeded gold findings through runContract. - tests/lib/gold-eval.test.mjs: the scoring RUN (precision/recall/f1 = 1.0, verdict == expected_verdict BLOCK, nothing suppressed/skipped). - lib/util/test-census.mjs: third census category (goldEval) — a scoring run is neither behavior coverage nor a doc-pin; honest-count invariant now 3-way. - docs/eval-corpus/README.md: 4b moved from Future hardening to implemented. Suite 809 -> 822 (820/0/2). gold-scorer covers TP+FP+FN+degenerate paths. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01BJQYC5vpkJWxndS55vQQZ6
44 lines
2.2 KiB
JavaScript
44 lines
2.2 KiB
JavaScript
// tests/lib/test-census.test.mjs
|
|
// Behavior test for the suite census + the honest-count invariant from the
|
|
// devil's-advocate audit §Top changes #8 (S19): the behavior-test count must
|
|
// be reported SEPARATELY from the doc-consistency prose-pin count. A prose-pin
|
|
// (a string/existence assertion against documentation) is not the same coverage
|
|
// as a behavior test, so a single conflated total oversells the behavior
|
|
// coverage. This census makes the split explicit and drift-proof.
|
|
|
|
import { test } from 'node:test';
|
|
import { strict as assert } from 'node:assert';
|
|
import { dirname, join } from 'node:path';
|
|
import { fileURLToPath } from 'node:url';
|
|
import { censusTests } from '../../lib/util/test-census.mjs';
|
|
|
|
const HERE = dirname(fileURLToPath(import.meta.url));
|
|
const TESTS_ROOT = join(HERE, '..');
|
|
|
|
test('suite census splits behavior / doc-pins / gold-eval (S19 + SKAL-1·4b)', (t) => {
|
|
const c = censusTests(TESTS_ROOT);
|
|
// Honest-count invariant: the three buckets must account for every top-level
|
|
// test() declaration — no silent drift between behavior, pin, and eval counts.
|
|
assert.equal(c.behavior + c.docPins + c.goldEval, c.total,
|
|
'census buckets must sum to the total declaration count');
|
|
assert.ok(c.docPins > 0, 'doc-consistency pin bucket must be non-empty (regex/glob sanity)');
|
|
assert.ok(c.behavior > 0, 'behavior bucket must be non-empty (regex/glob sanity)');
|
|
assert.ok(c.goldEval > 0, 'gold-eval scoring-run bucket must be non-empty (SKAL-1·4b present)');
|
|
// Report the split so the cited count is honest (audit §Top changes #8).
|
|
// Metric = top-level test() declarations; node:test's runtime total counts
|
|
// subtests too and is therefore ≥ this number.
|
|
t.diagnostic(
|
|
`behavior=${c.behavior} doc-consistency-pins=${c.docPins} ` +
|
|
`gold-eval=${c.goldEval} total=${c.total} (top-level test() declarations)`,
|
|
);
|
|
});
|
|
|
|
test("doc-consistency.test.mjs is the suite's prose-pin home (S19)", () => {
|
|
const c = censusTests(TESTS_ROOT);
|
|
const pinFiles = Object.keys(c.byFile)
|
|
.filter((f) => /(doc-consistency|prose-pins)\.test\.mjs$/.test(f));
|
|
assert.ok(
|
|
pinFiles.some((f) => f.endsWith('doc-consistency.test.mjs')),
|
|
'doc-consistency.test.mjs must exist as the canonical prose-pin bucket',
|
|
);
|
|
});
|