open/voyage
1
0
Fork 0
Code Issues Releases 13 Activity

fix(red-proof): the judge skips other rounds' ids and reports an unreadable tag

Browse source 
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Kjell Tore Guttormsen 2026-10-26 12:19:36 +01:00
commit cde4c1d3ed
Signed by: ktg
SSH key fingerprint: SHA256:JakMjO6FTBBzN0Bhfj9saOoEjaFxlSdYuZQQpM/lF9Q
2 changed files with 105 additions and 7 deletions
Download patch file Download diff file Expand all files Collapse all files

32
tests/helpers/red-judge.mjs
View file

@ -6,7 +6,12 @@
// tagged test (a name that starts with "[id]") failed on an assertion.
//
// node tests/helpers/red-judge.mjs --ids <file> --results <jsonl>
// [--pass-ids <id>[,<id>...]] [--mutation <id>]
// [--other-ids <file>] [--pass-ids <id>[,<id>...]] [--mutation <id>]
//
// --other-ids <file>: the other rounds' ids, one per line (read like --ids).
// A result whose tag is on that list is skipped before every rule (pass,
// fail, crash, skip, todo): another round's test is that round's to judge.
// An id on both lists is a usage error (exit 2).
//
// Plain run, one line per broken rule and exit 1 when:
// broken: no-test [id] a listed id has no test
@ -21,28 +26,32 @@
// broken: untagged-failure {name} ({file}) a failing event with no tag (a
// file that failed to load)
// broken: unlisted-tag [tag] {name} a tag that is not in the list
// broken: unreadable-tag {name} a name that starts with "[" but
// holds no readable tag ("[X 1]",
// "[]"), whatever its result
// An aggregate (a suite or file event with failureType subtestsFailed: it
// failed only because a child failed) is not counted.
// --mutation <id>: only that id's tests are judged (plus untagged failures
// and unlisted tags); exit 1 with "broken: mutation-not-red [id]" unless at
// --mutation <id>: only that id's tests are judged (plus untagged failures,
// unlisted and unreadable tags); exit 1 with "broken: mutation-not-red [id]" unless at
// least one of them fails on an assertion, and a "broken:" line for any of
// them that fails otherwise, is skipped or is a todo. A passing one is fine.
// Otherwise exit 0, printing "red: [id] {n} test(s)" per id (and
// "pass: [id] {n} test(s) (--pass-ids)" for a passing --pass-ids id).
// Exit 2: a usage error or an unreadable input (never 0).
// Exit 2: a usage error or an unreadable input (never 0). Every exit 2 goes
// through usage().
import { readFileSync } from 'node:fs';
/** Print a usage problem and exit 2. */
function usage(msg) {
process.stderr.write(`red-judge: ${msg}\n` +
'usage: red-judge.mjs --ids <file> --results <jsonl> [--pass-ids <id>[,<id>...]] [--mutation <id>]\n');
'usage: red-judge.mjs --ids <file> --results <jsonl> [--other-ids <file>] [--pass-ids <id>[,<id>...]] [--mutation <id>]\n');
process.exit(2);
}
/** --flag value pairs; every flag takes one value. */
function parse(argv) {
const known = new Set(['--ids', '--results', '--pass-ids', '--mutation']);
const known = new Set(['--ids', '--results', '--other-ids', '--pass-ids', '--mutation']);
const o = {};
for (let i = 0; i < argv.length; i += 2) {
if (!known.has(argv[i])) usage(`unknown argument ${argv[i]}`);
@ -75,6 +84,12 @@ const opts = parse(process.argv.slice(2));
const ids = readText(opts['--ids'], 'ids').split('\n').map((l) => l.trim()).filter(Boolean);
if (ids.length === 0) usage(`the ids file ${opts['--ids']} lists no id`);
const listed = new Set(ids);
const otherIds = opts['--other-ids']
? new Set(readText(opts['--other-ids'], 'other-ids').split('\n').map((l) => l.trim()).filter(Boolean))
: new Set();
for (const id of ids) {
if (otherIds.has(id)) usage(`id ${id} is on both --ids and --other-ids`);
}
const passIds = new Set((opts['--pass-ids'] || '').split(',').map((s) => s.trim()).filter(Boolean));
const mutation = opts['--mutation'] || null;
if (mutation && !listed.has(mutation)) usage(`--mutation ${mutation} is not in the ids file`);
@ -99,6 +114,11 @@ for (const e of events) {
if (e.ok === false && e.failureType === 'subtestsFailed') continue;
const name = one(e.name);
const tag = tagOf(name);
if (tag && otherIds.has(tag)) continue;
if (!tag && name.startsWith('[')) {
broken.push(`broken: unreadable-tag ${name}`);
continue;
}
if (!tag) {
if (e.ok === false) broken.push(`broken: untagged-failure ${name} (${e.file ?? 'no file'})`);
continue;