fix(red-proof): the judge skips other rounds' ids and reports an unreadable tag
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
be0b631fed
commit
cde4c1d3ed
2 changed files with 105 additions and 7 deletions
|
|
@ -18,7 +18,10 @@
|
|||
// broken: not-assertion [id] {name} (...) it failed, but not on an assertion
|
||||
// broken: untagged-failure {name} (...) a failing event with no tag
|
||||
// broken: unlisted-tag [tag] {name} a tag that is not in the list
|
||||
// broken: unreadable-tag {name} a name opens with [ but holds no tag
|
||||
// broken: mutation-not-red [id] --mutation: no assertion failure
|
||||
// A tag on --other-ids (another round's id) is never judged; an id on both
|
||||
// lists is a usage error (exit 2).
|
||||
// All red: exit 0 and "red: [id] {n} test(s)" per id.
|
||||
|
||||
import { test } from 'node:test';
|
||||
|
|
@ -164,6 +167,79 @@ test('red judge: --mutation X1 with one [X1] assertion failure and one [X1] Type
|
|||
assert.match(r.out, /^broken: not-assertion \[X1\] \[X1\] crash \(ERR_TEST_FAILURE TypeError\)$/m, r.out);
|
||||
});
|
||||
|
||||
/** Run the judge with an --other-ids list written next to the id list, and planted events. */
|
||||
function judgeOther(ids, otherIds, events, extra = []) {
|
||||
return inTmp((dir) => {
|
||||
const results = join(dir, 'results.jsonl');
|
||||
writeFileSync(results, events.map((e) => JSON.stringify(e)).join('\n') + (events.length ? '\n' : ''));
|
||||
writeFileSync(join(dir, 'other-ids'), otherIds.join('\n') + '\n');
|
||||
return judgeFile(dir, ids, results, ['--other-ids', join(dir, 'other-ids'), ...extra]);
|
||||
});
|
||||
}
|
||||
|
||||
/** A string as a literal inside a RegExp. */
|
||||
const lit = (s) => s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
||||
|
||||
test('[RS] judge: a tag on --other-ids is not judged whatever its result', () => {
|
||||
const other = [
|
||||
passing('[X2] passes'),
|
||||
ev('[X2] red'),
|
||||
ev('[X2] crash', { code: 'ERR_MODULE_NOT_FOUND', cname: 'Error' }),
|
||||
ev('[X2] skipped', { ok: true, skip: true, code: null, cname: null, failureType: null }),
|
||||
ev('[X2] someday', { todo: true }),
|
||||
];
|
||||
const r = judgeOther(['X1'], ['X2'], [ev('[X1] red'), ...other]);
|
||||
assert.equal(r.status, 0, r.out);
|
||||
assert.match(r.out, /^red: \[X1\] 1 test\(s\)$/m, r.out);
|
||||
assert.doesNotMatch(r.out, /^broken:/m, r.out);
|
||||
assert.doesNotMatch(r.out, /X2/, 'an other-round tag must not be named');
|
||||
const m = judgeOther(['X1'], ['X2'], [ev('[X1] red'), ...other], ['--mutation', 'X1']);
|
||||
assert.equal(m.status, 0, m.out);
|
||||
assert.match(m.out, /^red: \[X1\] 1 test\(s\)$/m, m.out);
|
||||
assert.doesNotMatch(m.out, /^broken:/m, m.out);
|
||||
});
|
||||
|
||||
test('[RS] judge: an id on both lists is a usage error', () => {
|
||||
const r = judgeOther(['X1', 'X2'], ['X2'], [ev('[X1] red'), ev('[X2] red')]);
|
||||
assert.equal(r.status, 2, r.out);
|
||||
assert.match(r.out, /id X2 is on both --ids and --other-ids/, r.out);
|
||||
assert.doesNotMatch(r.out, /^red:/m, r.out);
|
||||
});
|
||||
|
||||
test('[E23a] judge: a name that opens with [ but has no readable tag is broken', () => {
|
||||
for (const name of ['[X 1] typo', '[] empty', '[#5 2] spaced']) {
|
||||
const p = judge(['X1'], [ev('[X1] red'), passing(name)]);
|
||||
assert.equal(p.status, 1, `${name} passing: ${p.out}`);
|
||||
assert.match(p.out, new RegExp(`^broken: unreadable-tag ${lit(name)}$`, 'm'), p.out);
|
||||
const f = judge(['X1'], [ev('[X1] red'), ev(name)]);
|
||||
assert.equal(f.status, 1, `${name} failing: ${f.out}`);
|
||||
assert.match(f.out, new RegExp(`^broken: unreadable-tag ${lit(name)}$`, 'm'), f.out);
|
||||
}
|
||||
});
|
||||
|
||||
test('[E23d] judge: a missing --ids file exits 2', () => {
|
||||
inTmp((dir) => {
|
||||
const results = join(dir, 'results.jsonl');
|
||||
writeFileSync(results, `${JSON.stringify(ev('[X1] red'))}\n`);
|
||||
const r = spawnSync(process.execPath, [JUDGE, '--ids', join(dir, 'no-such-ids'), '--results', results],
|
||||
{ encoding: 'utf8', env: childEnv() });
|
||||
assert.equal(r.status, 2, `${r.stdout}${r.stderr}`);
|
||||
assert.match(r.stderr, /cannot read the ids file/, r.stderr);
|
||||
assert.doesNotMatch(r.stdout, /^red:/m, r.stdout);
|
||||
});
|
||||
});
|
||||
|
||||
test('[E23d] judge: a non-JSON results line exits 2', () => {
|
||||
inTmp((dir) => {
|
||||
const results = join(dir, 'results.jsonl');
|
||||
writeFileSync(results, `${JSON.stringify(ev('[X1] red'))}\nnot json\n`);
|
||||
const r = judgeFile(dir, ['X1'], results);
|
||||
assert.equal(r.status, 2, r.out);
|
||||
assert.match(r.out, /line 2 of .* is not JSON/, r.out);
|
||||
assert.doesNotMatch(r.out, /^red:/m, r.out);
|
||||
});
|
||||
});
|
||||
|
||||
/** Write fixture test files into `dir`, run them under the red reporter, return the JSONL path. */
|
||||
function report(dir, files) {
|
||||
for (const [name, body] of Object.entries(files)) writeFileSync(join(dir, name), body);
|
||||
|
|
@ -206,7 +282,7 @@ test('red reporter + judge: an [X1] test that fails on assert.equal(1, 2) -> exi
|
|||
});
|
||||
});
|
||||
|
||||
test('red reporter + judge: a { skip: true } and a { todo: true } [X1] test carry skip and todo in their lines -> exit 1', () => {
|
||||
test('[E23f] red reporter + judge: a { skip: true } and a { todo: true } [X1] test carry skip and todo in their lines -> exit 1', () => {
|
||||
inTmp((dir) => {
|
||||
const out = report(dir, {
|
||||
'x1.test.mjs': `${HEAD}test('[X1] red', () => { assert.equal(1, 2); });\n`
|
||||
|
|
@ -219,6 +295,8 @@ test('red reporter + judge: a { skip: true } and a { todo: true } [X1] test carr
|
|||
assert.equal(byName('[X1] todo').length, 1, readFileSync(out, 'utf8'));
|
||||
assert.deepEqual([byName('[X1] skipped')[0].skip, byName('[X1] skipped')[0].todo], [true, false]);
|
||||
assert.deepEqual([byName('[X1] todo')[0].skip, byName('[X1] todo')[0].todo], [false, true]);
|
||||
const hits = byName('[X1] red');
|
||||
assert.ok(hits.length > 0, `the reporter wrote no [X1] red line: ${readFileSync(out, 'utf8')}`);
|
||||
assert.deepEqual([byName('[X1] red')[0].skip, byName('[X1] red')[0].todo], [false, false]);
|
||||
const r = judgeFile(dir, ['X1'], out);
|
||||
assert.equal(r.status, 1, r.out);
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue