voyage/tests/helpers/red-judge.mjs

#!/usr/bin/env node
// tests/helpers/red-judge.mjs
// The judge of the red script of fix round 4-5a (tests/fix-round-4-5a-red.sh).
// It reads the JSON lines tests/helpers/red-reporter.mjs wrote for a run at
// ab6b32c and decides whether every id of the round was seen red there: each
// tagged test (a name that starts with "[id]") failed on an assertion.
//
//   node tests/helpers/red-judge.mjs --ids <file> --results <jsonl>
//        [--other-ids <file>] [--pass-ids <id>[,<id>...]] [--mutation <id>]
//
// --other-ids <file>: the other rounds' ids, one per line (read like --ids).
// A result whose tag is on that list is skipped before every rule (pass,
// fail, crash, skip, todo): another round's test is that round's to judge.
// An id on both lists is a usage error (exit 2).
//
// Plain run, one line per broken rule and exit 1 when:
//   broken: no-test [id]                      a listed id has no test
//   broken: passed [id] {name}                a tagged test passed (unless its
//                                             id is in --pass-ids)
//   broken: skipped [id] {name}               a tagged test was skipped
//   broken: todo [id] {name}                  a tagged test is a todo
//   broken: not-assertion [id] {name} ({code} {cname})
//                                             it failed on anything but
//                                             ERR_ASSERTION / AssertionError
//                                             (a crash is not red)
//   broken: untagged-failure {name} ({file})  a failing event with no tag (a
//                                             file that failed to load)
//   broken: unlisted-tag [tag] {name}         a tag that is not in the list
//   broken: unreadable-tag {name}             a name that starts with "[" but
//                                             holds no readable tag ("[X 1]",
//                                             "[]"), whatever its result
// An aggregate (a suite or file event with failureType subtestsFailed: it
// failed only because a child failed) is not counted.
// --mutation <id>: only that id's tests are judged (plus untagged failures,
// unlisted and unreadable tags); exit 1 with "broken: mutation-not-red [id]" unless at
// least one of them fails on an assertion, and a "broken:" line for any of
// them that fails otherwise, is skipped or is a todo. A passing one is fine.
// Otherwise exit 0, printing "red: [id] {n} test(s)" per id (and
// "pass: [id] {n} test(s) (--pass-ids)" for a passing --pass-ids id).
// Exit 2: a usage error or an unreadable input (never 0). Every exit 2 goes
// through usage().

import { readFileSync } from 'node:fs';

/** Print a usage problem and exit 2. */
function usage(msg) {
  process.stderr.write(`red-judge: ${msg}\n` +
    'usage: red-judge.mjs --ids <file> --results <jsonl> [--other-ids <file>] [--pass-ids <id>[,<id>...]] [--mutation <id>]\n');
  process.exit(2);
}

/** --flag value pairs; every flag takes one value. */
function parse(argv) {
  const known = new Set(['--ids', '--results', '--other-ids', '--pass-ids', '--mutation']);
  const o = {};
  for (let i = 0; i < argv.length; i += 2) {
    if (!known.has(argv[i])) usage(`unknown argument ${argv[i]}`);
    if (i + 1 >= argv.length || argv[i + 1].startsWith('--')) usage(`${argv[i]} needs a value`);
    o[argv[i]] = argv[i + 1];
  }
  if (!o['--ids'] || !o['--results']) usage('--ids and --results are required');
  return o;
}

/** A file's text, or exit 2. */
function readText(path, what) {
  try {
    return readFileSync(path, 'utf8');
  } catch (e) {
    return usage(`cannot read the ${what} file ${path}: ${e.code || e.message}`);
  }
}

/** The id a test name starts with, in brackets, or null. */
function tagOf(name) {
  const m = /^\[([^\]\s]+)\]/.exec(String(name));
  return m ? m[1] : null;
}

/** One line of output: whitespace collapsed. */
const one = (s) => String(s).replace(/\s+/g, ' ').trim();

const opts = parse(process.argv.slice(2));
const ids = readText(opts['--ids'], 'ids').split('\n').map((l) => l.trim()).filter(Boolean);
if (ids.length === 0) usage(`the ids file ${opts['--ids']} lists no id`);
const listed = new Set(ids);
const otherIds = opts['--other-ids']
  ? new Set(readText(opts['--other-ids'], 'other-ids').split('\n').map((l) => l.trim()).filter(Boolean))
  : new Set();
for (const id of ids) {
  if (otherIds.has(id)) usage(`id ${id} is on both --ids and --other-ids`);
}
const passIds = new Set((opts['--pass-ids'] || '').split(',').map((s) => s.trim()).filter(Boolean));
const mutation = opts['--mutation'] || null;
if (mutation && !listed.has(mutation)) usage(`--mutation ${mutation} is not in the ids file`);

const events = [];
readText(opts['--results'], 'results').split('\n').forEach((line, i) => {
  if (!line.trim()) return;
  try {
    events.push(JSON.parse(line));
  } catch {
    usage(`line ${i + 1} of ${opts['--results']} is not JSON`);
  }
});

const broken = [];
const seen = new Map();
const red = new Map();
const passed = new Map();
const bump = (m, k) => m.set(k, (m.get(k) || 0) + 1);

for (const e of events) {
  if (e.ok === false && e.failureType === 'subtestsFailed') continue;
  const name = one(e.name);
  const tag = tagOf(name);
  if (tag && otherIds.has(tag)) continue;
  if (!tag && name.startsWith('[')) {
    broken.push(`broken: unreadable-tag ${name}`);
    continue;
  }
  if (!tag) {
    if (e.ok === false) broken.push(`broken: untagged-failure ${name} (${e.file ?? 'no file'})`);
    continue;
  }
  if (!listed.has(tag)) {
    broken.push(`broken: unlisted-tag [${tag}] ${name}`);
    continue;
  }
  if (mutation && tag !== mutation) continue;
  bump(seen, tag);
  if (e.skip) {
    broken.push(`broken: skipped [${tag}] ${name}`);
  } else if (e.todo) {
    broken.push(`broken: todo [${tag}] ${name}`);
  } else if (e.ok === true) {
    if (mutation || passIds.has(tag)) bump(passed, tag);
    else broken.push(`broken: passed [${tag}] ${name}`);
  } else if (e.code === 'ERR_ASSERTION' && e.cname === 'AssertionError') {
    bump(red, tag);
  } else {
    broken.push(`broken: not-assertion [${tag}] ${name} (${e.code ?? 'no code'} ${e.cname ?? 'no name'})`);
  }
}

const judged = mutation ? [mutation] : ids;
for (const id of judged) {
  if (mutation) {
    if (!red.get(id)) broken.push(`broken: mutation-not-red [${id}]`);
  } else if (!seen.get(id)) {
    broken.push(`broken: no-test [${id}]`);
  }
}

if (broken.length > 0) {
  process.stdout.write(`${broken.join('\n')}\n`);
  process.exit(1);
}
const out = [];
for (const id of judged) {
  if (red.get(id)) out.push(`red: [${id}] ${red.get(id)} test(s)`);
  if (!mutation && passed.get(id)) out.push(`pass: [${id}] ${passed.get(id)} test(s) (--pass-ids)`);
}
process.stdout.write(`${out.join('\n')}\n`);
process.exit(0);