// lib/util/test-census.mjs // Census of the test suite: split top-level test() declarations into three // honest categories: // - "behavior" — ordinary behavior coverage (the default bucket). // - "docPins" — doc-consistency pins (string/existence assertions that pin // documentation against a source-of-truth). // - "goldEval" — the offline gold-scored output eval scoring RUN (SKAL-1·4b): // neither behavior nor prose-pin, but a run that scores a // committed agent-run fixture against the golden corpus. // Makes the cited test count honest — a prose-pin is not the same coverage as a // behavior test, and a scoring run is a third thing again, so a single // conflated total oversells behavior coverage. // Devil's-advocate audit §Top changes #8 (S19); third bucket added in SKAL-1·4b. // // Metric: top-level `test(` declarations (static, deterministic, in-process). // This is distinct from node:test's runtime total, which additionally counts // subtests; the runtime total is therefore ≥ this declaration count. import { readFileSync, readdirSync } from 'node:fs'; import { join } from 'node:path'; // Files whose tests are documentation pins, not behavior coverage. Kept as a // regex (not a single filename) so a future split-out (e.g. prose-pins.test.mjs) // is bucketed correctly without editing this module. export const PIN_FILE_RE = /(doc-consistency|prose-pins)\.test\.mjs$/; // Files whose tests are the offline gold-scored output eval scoring run // (SKAL-1·4b). Kept as a regex (not a single filename) so a future split-out // is bucketed correctly without editing this module. export const GOLD_EVAL_FILE_RE = /gold-eval\.test\.mjs$/; function walk(dir) { const out = []; for (const e of readdirSync(dir, { withFileTypes: true })) { const p = join(dir, e.name); if (e.isDirectory()) out.push(...walk(p)); else if (e.isFile() && e.name.endsWith('.test.mjs')) out.push(p); } return out; } function countTests(file) { return (readFileSync(file, 'utf-8').match(/^\s*test\(/gm) || []).length; } // Returns { behavior, docPins, goldEval, total, byFile } for all *.test.mjs // under testsRoot. behavior + docPins + goldEval === total by construction. export function censusTests(testsRoot) { const files = walk(testsRoot).sort(); const byFile = {}; let behavior = 0; let docPins = 0; let goldEval = 0; for (const f of files) { const n = countTests(f); byFile[f] = n; if (PIN_FILE_RE.test(f)) docPins += n; else if (GOLD_EVAL_FILE_RE.test(f)) goldEval += n; else behavior += n; } return { behavior, docPins, goldEval, total: behavior + docPins + goldEval, byFile }; }