voyage/lib/util/test-census.mjs

// lib/util/test-census.mjs
// Census of the test suite: split top-level test() declarations into three
// honest categories:
//   - "behavior"  — ordinary behavior coverage (the default bucket).
//   - "docPins"   — doc-consistency pins (string/existence assertions that pin
//                   documentation against a source-of-truth).
//   - "goldEval"  — the offline gold-scored output eval scoring RUN (SKAL-1·4b):
//                   neither behavior nor prose-pin, but a run that scores a
//                   committed agent-run fixture against the golden corpus.
// Makes the cited test count honest — a prose-pin is not the same coverage as a
// behavior test, and a scoring run is a third thing again, so a single
// conflated total oversells behavior coverage.
// Devil's-advocate audit §Top changes #8 (S19); third bucket added in SKAL-1·4b.
//
// Metric: top-level `test(` declarations (static, deterministic, in-process).
// This is distinct from node:test's runtime total, which additionally counts
// subtests; the runtime total is therefore ≥ this declaration count.

import { readFileSync, readdirSync } from 'node:fs';
import { join } from 'node:path';

// Files whose tests are documentation pins, not behavior coverage. Kept as a
// regex (not a single filename) so a future split-out (e.g. prose-pins.test.mjs)
// is bucketed correctly without editing this module.
export const PIN_FILE_RE = /(doc-consistency|prose-pins)\.test\.mjs$/;

// Files whose tests are the offline gold-scored output eval scoring run
// (SKAL-1·4b). Kept as a regex (not a single filename) so a future split-out
// is bucketed correctly without editing this module.
export const GOLD_EVAL_FILE_RE = /gold-eval\.test\.mjs$/;

function walk(dir) {
  const out = [];
  for (const e of readdirSync(dir, { withFileTypes: true })) {
    const p = join(dir, e.name);
    if (e.isDirectory()) out.push(...walk(p));
    else if (e.isFile() && e.name.endsWith('.test.mjs')) out.push(p);
  }
  return out;
}

function countTests(file) {
  return (readFileSync(file, 'utf-8').match(/^\s*test\(/gm) || []).length;
}

// Returns { behavior, docPins, goldEval, total, byFile } for all *.test.mjs
// under testsRoot. behavior + docPins + goldEval === total by construction.
export function censusTests(testsRoot) {
  const files = walk(testsRoot).sort();
  const byFile = {};
  let behavior = 0;
  let docPins = 0;
  let goldEval = 0;
  for (const f of files) {
    const n = countTests(f);
    byFile[f] = n;
    if (PIN_FILE_RE.test(f)) docPins += n;
    else if (GOLD_EVAL_FILE_RE.test(f)) goldEval += n;
    else behavior += n;
  }
  return { behavior, docPins, goldEval, total: behavior + docPins + goldEval, byFile };
}