open/voyage
1
0
Fork 0
Code Issues Releases 13 Activity

fix(red-proof): the judge skips other rounds' ids and reports an unreadable tag

Browse source 
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Kjell Tore Guttormsen 2026-10-26 12:19:36 +01:00
commit cde4c1d3ed
Signed by: ktg
SSH key fingerprint: SHA256:JakMjO6FTBBzN0Bhfj9saOoEjaFxlSdYuZQQpM/lF9Q
2 changed files with 105 additions and 7 deletions
Download patch file Download diff file Expand all files Collapse all files

32
tests/helpers/red-judge.mjs
View file

@ -6,7 +6,12 @@
// tagged test (a name that starts with "[id]") failed on an assertion.
//
// node tests/helpers/red-judge.mjs --ids <file> --results <jsonl>
// [--pass-ids <id>[,<id>...]] [--mutation <id>]
// [--other-ids <file>] [--pass-ids <id>[,<id>...]] [--mutation <id>]
//
// --other-ids <file>: the other rounds' ids, one per line (read like --ids).
// A result whose tag is on that list is skipped before every rule (pass,
// fail, crash, skip, todo): another round's test is that round's to judge.
// An id on both lists is a usage error (exit 2).
//
// Plain run, one line per broken rule and exit 1 when:
// broken: no-test [id] a listed id has no test
@ -21,28 +26,32 @@
// broken: untagged-failure {name} ({file}) a failing event with no tag (a
// file that failed to load)
// broken: unlisted-tag [tag] {name} a tag that is not in the list
// broken: unreadable-tag {name} a name that starts with "[" but
// holds no readable tag ("[X 1]",
// "[]"), whatever its result
// An aggregate (a suite or file event with failureType subtestsFailed: it
// failed only because a child failed) is not counted.
// --mutation <id>: only that id's tests are judged (plus untagged failures
// and unlisted tags); exit 1 with "broken: mutation-not-red [id]" unless at
// --mutation <id>: only that id's tests are judged (plus untagged failures,
// unlisted and unreadable tags); exit 1 with "broken: mutation-not-red [id]" unless at
// least one of them fails on an assertion, and a "broken:" line for any of
// them that fails otherwise, is skipped or is a todo. A passing one is fine.
// Otherwise exit 0, printing "red: [id] {n} test(s)" per id (and
// "pass: [id] {n} test(s) (--pass-ids)" for a passing --pass-ids id).
// Exit 2: a usage error or an unreadable input (never 0).
// Exit 2: a usage error or an unreadable input (never 0). Every exit 2 goes
// through usage().
import { readFileSync } from 'node:fs';
/** Print a usage problem and exit 2. */
function usage(msg) {
process.stderr.write(`red-judge: ${msg}\n` +
'usage: red-judge.mjs --ids <file> --results <jsonl> [--pass-ids <id>[,<id>...]] [--mutation <id>]\n');
'usage: red-judge.mjs --ids <file> --results <jsonl> [--other-ids <file>] [--pass-ids <id>[,<id>...]] [--mutation <id>]\n');
process.exit(2);
}
/** --flag value pairs; every flag takes one value. */
function parse(argv) {
const known = new Set(['--ids', '--results', '--pass-ids', '--mutation']);
const known = new Set(['--ids', '--results', '--other-ids', '--pass-ids', '--mutation']);
const o = {};
for (let i = 0; i < argv.length; i += 2) {
if (!known.has(argv[i])) usage(`unknown argument ${argv[i]}`);
@ -75,6 +84,12 @@ const opts = parse(process.argv.slice(2));
const ids = readText(opts['--ids'], 'ids').split('\n').map((l) => l.trim()).filter(Boolean);
if (ids.length === 0) usage(`the ids file ${opts['--ids']} lists no id`);
const listed = new Set(ids);
const otherIds = opts['--other-ids']
? new Set(readText(opts['--other-ids'], 'other-ids').split('\n').map((l) => l.trim()).filter(Boolean))
: new Set();
for (const id of ids) {
if (otherIds.has(id)) usage(`id ${id} is on both --ids and --other-ids`);
}
const passIds = new Set((opts['--pass-ids'] || '').split(',').map((s) => s.trim()).filter(Boolean));
const mutation = opts['--mutation'] || null;
if (mutation && !listed.has(mutation)) usage(`--mutation ${mutation} is not in the ids file`);
@ -99,6 +114,11 @@ for (const e of events) {
if (e.ok === false && e.failureType === 'subtestsFailed') continue;
const name = one(e.name);
const tag = tagOf(name);
if (tag && otherIds.has(tag)) continue;
if (!tag && name.startsWith('[')) {
broken.push(`broken: unreadable-tag ${name}`);
continue;
}
if (!tag) {
if (e.ok === false) broken.push(`broken: untagged-failure ${name} (${e.file ?? 'no file'})`);
continue;

80
tests/lib/red-judge.test.mjs
View file

@ -18,7 +18,10 @@
// broken: not-assertion [id] {name} (...) it failed, but not on an assertion
// broken: untagged-failure {name} (...) a failing event with no tag
// broken: unlisted-tag [tag] {name} a tag that is not in the list
// broken: unreadable-tag {name} a name opens with [ but holds no tag
// broken: mutation-not-red [id] --mutation: no assertion failure
// A tag on --other-ids (another round's id) is never judged; an id on both
// lists is a usage error (exit 2).
// All red: exit 0 and "red: [id] {n} test(s)" per id.
import { test } from 'node:test';
@ -164,6 +167,79 @@ test('red judge: --mutation X1 with one [X1] assertion failure and one [X1] Type
assert.match(r.out, /^broken: not-assertion \[X1\] \[X1\] crash \(ERR_TEST_FAILURE TypeError\)$/m, r.out);
});
/** Run the judge with an --other-ids list written next to the id list, and planted events. */
function judgeOther(ids, otherIds, events, extra = []) {
return inTmp((dir) => {
const results = join(dir, 'results.jsonl');
writeFileSync(results, events.map((e) => JSON.stringify(e)).join('\n') + (events.length ? '\n' : ''));
writeFileSync(join(dir, 'other-ids'), otherIds.join('\n') + '\n');
return judgeFile(dir, ids, results, ['--other-ids', join(dir, 'other-ids'), ...extra]);
});
}
/** A string as a literal inside a RegExp. */
const lit = (s) => s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
test('[RS] judge: a tag on --other-ids is not judged whatever its result', () => {
const other = [
passing('[X2] passes'),
ev('[X2] red'),
ev('[X2] crash', { code: 'ERR_MODULE_NOT_FOUND', cname: 'Error' }),
ev('[X2] skipped', { ok: true, skip: true, code: null, cname: null, failureType: null }),
ev('[X2] someday', { todo: true }),
];
const r = judgeOther(['X1'], ['X2'], [ev('[X1] red'), ...other]);
assert.equal(r.status, 0, r.out);
assert.match(r.out, /^red: \[X1\] 1 test\(s\)$/m, r.out);
assert.doesNotMatch(r.out, /^broken:/m, r.out);
assert.doesNotMatch(r.out, /X2/, 'an other-round tag must not be named');
const m = judgeOther(['X1'], ['X2'], [ev('[X1] red'), ...other], ['--mutation', 'X1']);
assert.equal(m.status, 0, m.out);
assert.match(m.out, /^red: \[X1\] 1 test\(s\)$/m, m.out);
assert.doesNotMatch(m.out, /^broken:/m, m.out);
});
test('[RS] judge: an id on both lists is a usage error', () => {
const r = judgeOther(['X1', 'X2'], ['X2'], [ev('[X1] red'), ev('[X2] red')]);
assert.equal(r.status, 2, r.out);
assert.match(r.out, /id X2 is on both --ids and --other-ids/, r.out);
assert.doesNotMatch(r.out, /^red:/m, r.out);
});
test('[E23a] judge: a name that opens with [ but has no readable tag is broken', () => {
for (const name of ['[X 1] typo', '[] empty', '[#5 2] spaced']) {
const p = judge(['X1'], [ev('[X1] red'), passing(name)]);
assert.equal(p.status, 1, `${name} passing: ${p.out}`);
assert.match(p.out, new RegExp(`^broken: unreadable-tag ${lit(name)}$`, 'm'), p.out);
const f = judge(['X1'], [ev('[X1] red'), ev(name)]);
assert.equal(f.status, 1, `${name} failing: ${f.out}`);
assert.match(f.out, new RegExp(`^broken: unreadable-tag ${lit(name)}$`, 'm'), f.out);
}
});
test('[E23d] judge: a missing --ids file exits 2', () => {
inTmp((dir) => {
const results = join(dir, 'results.jsonl');
writeFileSync(results, `${JSON.stringify(ev('[X1] red'))}\n`);
const r = spawnSync(process.execPath, [JUDGE, '--ids', join(dir, 'no-such-ids'), '--results', results],
{ encoding: 'utf8', env: childEnv() });
assert.equal(r.status, 2, `${r.stdout}${r.stderr}`);
assert.match(r.stderr, /cannot read the ids file/, r.stderr);
assert.doesNotMatch(r.stdout, /^red:/m, r.stdout);
});
});
test('[E23d] judge: a non-JSON results line exits 2', () => {
inTmp((dir) => {
const results = join(dir, 'results.jsonl');
writeFileSync(results, `${JSON.stringify(ev('[X1] red'))}\nnot json\n`);
const r = judgeFile(dir, ['X1'], results);
assert.equal(r.status, 2, r.out);
assert.match(r.out, /line 2 of .* is not JSON/, r.out);
assert.doesNotMatch(r.out, /^red:/m, r.out);
});
});
/** Write fixture test files into `dir`, run them under the red reporter, return the JSONL path. */
function report(dir, files) {
for (const [name, body] of Object.entries(files)) writeFileSync(join(dir, name), body);
@ -206,7 +282,7 @@ test('red reporter + judge: an [X1] test that fails on assert.equal(1, 2) -> exi
});
});
test('red reporter + judge: a { skip: true } and a { todo: true } [X1] test carry skip and todo in their lines -> exit 1', () => {
test('[E23f] red reporter + judge: a { skip: true } and a { todo: true } [X1] test carry skip and todo in their lines -> exit 1', () => {
inTmp((dir) => {
const out = report(dir, {
'x1.test.mjs': `${HEAD}test('[X1] red', () => { assert.equal(1, 2); });\n`
@ -219,6 +295,8 @@ test('red reporter + judge: a { skip: true } and a { todo: true } [X1] test carr
assert.equal(byName('[X1] todo').length, 1, readFileSync(out, 'utf8'));
assert.deepEqual([byName('[X1] skipped')[0].skip, byName('[X1] skipped')[0].todo], [true, false]);
assert.deepEqual([byName('[X1] todo')[0].skip, byName('[X1] todo')[0].todo], [false, true]);
const hits = byName('[X1] red');
assert.ok(hits.length > 0, `the reporter wrote no [X1] red line: ${readFileSync(out, 'utf8')}`);
assert.deepEqual([byName('[X1] red')[0].skip, byName('[X1] red')[0].todo], [false, false]);
const r = judgeFile(dir, ['X1'], out);
assert.equal(r.status, 1, r.out);