306 lines
15 KiB
JavaScript
306 lines
15 KiB
JavaScript
// tests/lib/red-judge.test.mjs
|
|
// The red judge of fix round 4-5a (tests/helpers/red-judge.mjs) and its
|
|
// reporter (tests/helpers/red-reporter.mjs). The round's red script
|
|
// (tests/fix-round-4-5a-red.sh) runs every tagged test at ab6b32c and lets the
|
|
// judge decide; a judge that always exits 0 would prove nothing. So the judge
|
|
// is fed planted JSON lines with the tags [X1] and [X2] (never a real id) and
|
|
// must exit 1 on each broken rule, naming it on one line.
|
|
//
|
|
// The judge runs only as a CLI through spawnSync and is never imported: each
|
|
// test asserts the exit code and the line the judge prints, so every test
|
|
// fails on an assertion while the judge does not exist.
|
|
//
|
|
// Rule lines (one per broken rule, exit 1):
|
|
// broken: no-test [id] a listed id has no test
|
|
// broken: passed [id] {name} a tagged test passed
|
|
// broken: skipped [id] {name} a tagged test was skipped
|
|
// broken: todo [id] {name} a tagged test is a todo
|
|
// broken: not-assertion [id] {name} (...) it failed, but not on an assertion
|
|
// broken: untagged-failure {name} (...) a failing event with no tag
|
|
// broken: unlisted-tag [tag] {name} a tag that is not in the list
|
|
// broken: unreadable-tag {name} a name opens with [ but holds no tag
|
|
// broken: mutation-not-red [id] --mutation: no assertion failure
|
|
// A tag on --other-ids (another round's id) is never judged; an id on both
|
|
// lists is a usage error (exit 2).
|
|
// All red: exit 0 and "red: [id] {n} test(s)" per id.
|
|
|
|
import { test } from 'node:test';
|
|
import { strict as assert } from 'node:assert';
|
|
import { spawnSync } from 'node:child_process';
|
|
import { existsSync, mkdtempSync, readFileSync, realpathSync, rmSync, writeFileSync } from 'node:fs';
|
|
import { tmpdir } from 'node:os';
|
|
import { dirname, join } from 'node:path';
|
|
import { fileURLToPath } from 'node:url';
|
|
import { childEnv } from '../helpers/child-env.mjs';
|
|
|
|
const ROOT = join(dirname(fileURLToPath(import.meta.url)), '..', '..');
|
|
const JUDGE = join(ROOT, 'tests', 'helpers', 'red-judge.mjs');
|
|
const REPORTER = join(ROOT, 'tests', 'helpers', 'red-reporter.mjs');
|
|
|
|
/** A temp directory outside tests/, removed after `fn` returns. */
|
|
function inTmp(fn) {
|
|
const dir = realpathSync(mkdtempSync(join(tmpdir(), 'red-judge-test-')));
|
|
try {
|
|
return fn(dir);
|
|
} finally {
|
|
rmSync(dir, { recursive: true, force: true });
|
|
}
|
|
}
|
|
|
|
/** One planted reporter line. Defaults: a top-level test that failed on an assertion. */
|
|
function ev(name, over = {}) {
|
|
return {
|
|
name, file: '/planted/x.test.mjs', nesting: 0, ok: false, skip: false, todo: false,
|
|
code: 'ERR_ASSERTION', cname: 'AssertionError', failureType: 'testCodeFailure', ...over,
|
|
};
|
|
}
|
|
const passing = (name) => ev(name, { ok: true, code: null, cname: null, failureType: null });
|
|
|
|
/** The judge on a results file, the id list written into `dir`, with extra flags. */
|
|
function judgeFile(dir, ids, results, extra = []) {
|
|
writeFileSync(join(dir, 'ids'), ids.join('\n') + '\n');
|
|
const r = spawnSync(process.execPath, [JUDGE, '--ids', join(dir, 'ids'), '--results', results, ...extra],
|
|
{ encoding: 'utf8', env: childEnv() });
|
|
return { status: r.status, out: `${r.stdout}${r.stderr}` };
|
|
}
|
|
|
|
/** Run the judge on an id list and planted events, with extra flags. */
|
|
function judge(ids, events, extra = []) {
|
|
return inTmp((dir) => {
|
|
const results = join(dir, 'results.jsonl');
|
|
writeFileSync(results, events.map((e) => JSON.stringify(e)).join('\n') + (events.length ? '\n' : ''));
|
|
return judgeFile(dir, ids, results, extra);
|
|
});
|
|
}
|
|
|
|
test('red judge: a tagged test that passes -> exit 1, "broken: passed [X1]"', () => {
|
|
const r = judge(['X1'], [passing('[X1] holds')]);
|
|
assert.equal(r.status, 1, r.out);
|
|
assert.match(r.out, /^broken: passed \[X1\] \[X1\] holds$/m, r.out);
|
|
});
|
|
|
|
test('red judge: a tagged test that fails with ERR_MODULE_NOT_FOUND -> exit 1, "broken: not-assertion [X1]" (a crash is not red)', () => {
|
|
const r = judge(['X1'], [ev('[X1] loads', { code: 'ERR_MODULE_NOT_FOUND', cname: 'Error' })]);
|
|
assert.equal(r.status, 1, r.out);
|
|
assert.match(r.out, /^broken: not-assertion \[X1\] \[X1\] loads \(ERR_MODULE_NOT_FOUND Error\)$/m, r.out);
|
|
});
|
|
|
|
test('red judge: a listed id with no test -> exit 1, "broken: no-test [X2]"', () => {
|
|
const r = judge(['X1', 'X2'], [ev('[X1] red')]);
|
|
assert.equal(r.status, 1, r.out);
|
|
assert.match(r.out, /^broken: no-test \[X2\]$/m, r.out);
|
|
assert.doesNotMatch(r.out, /^broken: .*\[X1\]/m, 'X1 is red and must not be named');
|
|
});
|
|
|
|
test('red judge: a tag that is not in the list -> exit 1, "broken: unlisted-tag [X2]"', () => {
|
|
const r = judge(['X1'], [ev('[X1] red'), ev('[X2] stray')]);
|
|
assert.equal(r.status, 1, r.out);
|
|
assert.match(r.out, /^broken: unlisted-tag \[X2\] \[X2\] stray$/m, r.out);
|
|
});
|
|
|
|
test('red judge: an untagged failing event (a file that failed to load) -> exit 1, "broken: untagged-failure"', () => {
|
|
const r = judge(['X1'], [ev('[X1] red'), ev('x.test.mjs', { code: null, cname: null })]);
|
|
assert.equal(r.status, 1, r.out);
|
|
assert.match(r.out, /^broken: untagged-failure x\.test\.mjs \(\/planted\/x\.test\.mjs\)$/m, r.out);
|
|
});
|
|
|
|
test('red judge: an untagged aggregate (failureType subtestsFailed) next to a tagged assertion failure is not counted -> exit 0', () => {
|
|
const r = judge(['X1'], [
|
|
ev('[X1] red', { nesting: 1 }),
|
|
ev('group', { code: 'ERR_TEST_FAILURE', cname: null, failureType: 'subtestsFailed' }),
|
|
]);
|
|
assert.equal(r.status, 0, r.out);
|
|
assert.match(r.out, /^red: \[X1\] 1 test\(s\)$/m, r.out);
|
|
assert.doesNotMatch(r.out, /^broken:/m, r.out);
|
|
});
|
|
|
|
test('red judge: a skipped tagged test -> exit 1, "broken: skipped [X1]"', () => {
|
|
const r = judge(['X1'], [ev('[X1] red'), ev('[X1] later', { ok: true, skip: true, code: null, cname: null, failureType: null })]);
|
|
assert.equal(r.status, 1, r.out);
|
|
assert.match(r.out, /^broken: skipped \[X1\] \[X1\] later$/m, r.out);
|
|
});
|
|
|
|
test('red judge: every id covered by assertion failures -> exit 0, one "red: [id] {n} test(s)" line per id', () => {
|
|
const r = judge(['X1', 'X2'], [ev('[X1] one'), ev('[X1] two'), ev('[X2] three'), passing('untagged keep')]);
|
|
assert.equal(r.status, 0, r.out);
|
|
assert.match(r.out, /^red: \[X1\] 2 test\(s\)$/m, r.out);
|
|
assert.match(r.out, /^red: \[X2\] 1 test\(s\)$/m, r.out);
|
|
assert.doesNotMatch(r.out, /^broken:/m, r.out);
|
|
});
|
|
|
|
test('red judge: --pass-ids X1 with [X1] passing -> exit 0, the other ids still red', () => {
|
|
const r = judge(['X1', 'X2'], [passing('[X1] holds'), ev('[X2] red')], ['--pass-ids', 'X1']);
|
|
assert.equal(r.status, 0, r.out);
|
|
assert.match(r.out, /^red: \[X2\] 1 test\(s\)$/m, r.out);
|
|
assert.doesNotMatch(r.out, /^broken:/m, r.out);
|
|
});
|
|
|
|
test('red judge: --pass-ids X1 exempts only X1: a passing [X2] next to a passing [X1] -> exit 1, "broken: passed [X2]"', () => {
|
|
const r = judge(['X1', 'X2'], [passing('[X1] holds'), passing('[X2] holds too')], ['--pass-ids', 'X1']);
|
|
assert.equal(r.status, 1, r.out);
|
|
assert.match(r.out, /^broken: passed \[X2\] \[X2\] .+$/m, r.out);
|
|
assert.doesNotMatch(r.out, /^broken: .*\[X1\]/m, 'X1 is in --pass-ids and must not be named');
|
|
});
|
|
|
|
test('red judge: a todo tagged test next to a red one -> exit 1, "broken: todo [X1]"', () => {
|
|
const r = judge(['X1'], [ev('[X1] red'), ev('[X1] someday', { todo: true })]);
|
|
assert.equal(r.status, 1, r.out);
|
|
assert.match(r.out, /^broken: todo \[X1\] \[X1\] someday$/m, r.out);
|
|
});
|
|
|
|
test('red judge: --mutation X1 with no [X1] assertion failure -> exit 1, "broken: mutation-not-red [X1]"', () => {
|
|
const r = judge(['X1'], [passing('[X1] holds')], ['--mutation', 'X1']);
|
|
assert.equal(r.status, 1, r.out);
|
|
assert.match(r.out, /^broken: mutation-not-red \[X1\]$/m, r.out);
|
|
});
|
|
|
|
test('red judge: --mutation X1 with one [X1] assertion failure and one [X1] pass -> exit 0, "red: [X1] 1 test(s)"', () => {
|
|
const r = judge(['X1'], [passing('[X1] holds'), ev('[X1] red')], ['--mutation', 'X1']);
|
|
assert.equal(r.status, 0, r.out);
|
|
assert.match(r.out, /^red: \[X1\] 1 test\(s\)$/m, r.out);
|
|
assert.doesNotMatch(r.out, /^broken:/m, r.out);
|
|
});
|
|
|
|
test('red judge: --mutation X1 with one [X1] assertion failure and one [X1] TypeError crash -> exit 1, "broken: not-assertion [X1]"', () => {
|
|
const r = judge(['X1'], [ev('[X1] red'), ev('[X1] crash', { code: 'ERR_TEST_FAILURE', cname: 'TypeError' })], ['--mutation', 'X1']);
|
|
assert.equal(r.status, 1, r.out);
|
|
assert.match(r.out, /^broken: not-assertion \[X1\] \[X1\] crash \(ERR_TEST_FAILURE TypeError\)$/m, r.out);
|
|
});
|
|
|
|
/** Run the judge with an --other-ids list written next to the id list, and planted events. */
|
|
function judgeOther(ids, otherIds, events, extra = []) {
|
|
return inTmp((dir) => {
|
|
const results = join(dir, 'results.jsonl');
|
|
writeFileSync(results, events.map((e) => JSON.stringify(e)).join('\n') + (events.length ? '\n' : ''));
|
|
writeFileSync(join(dir, 'other-ids'), otherIds.join('\n') + '\n');
|
|
return judgeFile(dir, ids, results, ['--other-ids', join(dir, 'other-ids'), ...extra]);
|
|
});
|
|
}
|
|
|
|
/** A string as a literal inside a RegExp. */
|
|
const lit = (s) => s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
|
|
|
test('[RS] judge: a tag on --other-ids is not judged whatever its result', () => {
|
|
const other = [
|
|
passing('[X2] passes'),
|
|
ev('[X2] red'),
|
|
ev('[X2] crash', { code: 'ERR_MODULE_NOT_FOUND', cname: 'Error' }),
|
|
ev('[X2] skipped', { ok: true, skip: true, code: null, cname: null, failureType: null }),
|
|
ev('[X2] someday', { todo: true }),
|
|
];
|
|
const r = judgeOther(['X1'], ['X2'], [ev('[X1] red'), ...other]);
|
|
assert.equal(r.status, 0, r.out);
|
|
assert.match(r.out, /^red: \[X1\] 1 test\(s\)$/m, r.out);
|
|
assert.doesNotMatch(r.out, /^broken:/m, r.out);
|
|
assert.doesNotMatch(r.out, /X2/, 'an other-round tag must not be named');
|
|
const m = judgeOther(['X1'], ['X2'], [ev('[X1] red'), ...other], ['--mutation', 'X1']);
|
|
assert.equal(m.status, 0, m.out);
|
|
assert.match(m.out, /^red: \[X1\] 1 test\(s\)$/m, m.out);
|
|
assert.doesNotMatch(m.out, /^broken:/m, m.out);
|
|
});
|
|
|
|
test('[RS] judge: an id on both lists is a usage error', () => {
|
|
const r = judgeOther(['X1', 'X2'], ['X2'], [ev('[X1] red'), ev('[X2] red')]);
|
|
assert.equal(r.status, 2, r.out);
|
|
assert.match(r.out, /id X2 is on both --ids and --other-ids/, r.out);
|
|
assert.doesNotMatch(r.out, /^red:/m, r.out);
|
|
});
|
|
|
|
test('[E23a] judge: a name that opens with [ but has no readable tag is broken', () => {
|
|
for (const name of ['[X 1] typo', '[] empty', '[#5 2] spaced']) {
|
|
const p = judge(['X1'], [ev('[X1] red'), passing(name)]);
|
|
assert.equal(p.status, 1, `${name} passing: ${p.out}`);
|
|
assert.match(p.out, new RegExp(`^broken: unreadable-tag ${lit(name)}$`, 'm'), p.out);
|
|
const f = judge(['X1'], [ev('[X1] red'), ev(name)]);
|
|
assert.equal(f.status, 1, `${name} failing: ${f.out}`);
|
|
assert.match(f.out, new RegExp(`^broken: unreadable-tag ${lit(name)}$`, 'm'), f.out);
|
|
}
|
|
});
|
|
|
|
test('[E23d] judge: a missing --ids file exits 2', () => {
|
|
inTmp((dir) => {
|
|
const results = join(dir, 'results.jsonl');
|
|
writeFileSync(results, `${JSON.stringify(ev('[X1] red'))}\n`);
|
|
const r = spawnSync(process.execPath, [JUDGE, '--ids', join(dir, 'no-such-ids'), '--results', results],
|
|
{ encoding: 'utf8', env: childEnv() });
|
|
assert.equal(r.status, 2, `${r.stdout}${r.stderr}`);
|
|
assert.match(r.stderr, /cannot read the ids file/, r.stderr);
|
|
assert.doesNotMatch(r.stdout, /^red:/m, r.stdout);
|
|
});
|
|
});
|
|
|
|
test('[E23d] judge: a non-JSON results line exits 2', () => {
|
|
inTmp((dir) => {
|
|
const results = join(dir, 'results.jsonl');
|
|
writeFileSync(results, `${JSON.stringify(ev('[X1] red'))}\nnot json\n`);
|
|
const r = judgeFile(dir, ['X1'], results);
|
|
assert.equal(r.status, 2, r.out);
|
|
assert.match(r.out, /line 2 of .* is not JSON/, r.out);
|
|
assert.doesNotMatch(r.out, /^red:/m, r.out);
|
|
});
|
|
});
|
|
|
|
/** Write fixture test files into `dir`, run them under the red reporter, return the JSONL path. */
|
|
function report(dir, files) {
|
|
for (const [name, body] of Object.entries(files)) writeFileSync(join(dir, name), body);
|
|
const out = join(dir, 'out.jsonl');
|
|
spawnSync(process.execPath, ['--test', `--test-reporter=${REPORTER}`, `--test-reporter-destination=${out}`,
|
|
...Object.keys(files)], { cwd: dir, encoding: 'utf8', env: childEnv({ NODE_TEST_CONTEXT: undefined }) });
|
|
assert.ok(existsSync(out), 'the red reporter wrote no results file');
|
|
return out;
|
|
}
|
|
|
|
const HEAD = "import { test } from 'node:test';\nimport { strict as assert } from 'node:assert';\n";
|
|
|
|
test('red reporter + judge: a passing [X1] test and an [X2] test that throws a TypeError, run by node --test -> exit 1', () => {
|
|
inTmp((dir) => {
|
|
const out = report(dir, {
|
|
'x1.test.mjs': `${HEAD}test('[X1] holds', () => { assert.equal(1, 1); });\n`,
|
|
'x2.test.mjs': `${HEAD}test('[X2] throws', () => { throw new TypeError('boom'); });\n`,
|
|
});
|
|
const r = judgeFile(dir, ['X1', 'X2'], out);
|
|
assert.equal(r.status, 1, r.out);
|
|
assert.match(r.out, /^broken: passed \[X1\] \[X1\] holds$/m, r.out);
|
|
assert.match(r.out, /^broken: not-assertion \[X2\] \[X2\] throws \(ERR_TEST_FAILURE TypeError\)$/m, r.out);
|
|
});
|
|
});
|
|
|
|
test('red reporter + judge: an [X1] test that fails on assert.equal(1, 2) -> exit 0, and the line carries the reporter fields', () => {
|
|
inTmp((dir) => {
|
|
const out = report(dir, { 'x1.test.mjs': `${HEAD}test('[X1] red', () => { assert.equal(1, 2); });\n` });
|
|
const lines = readFileSync(out, 'utf8').split('\n').filter(Boolean).map((l) => JSON.parse(l));
|
|
const x1 = lines.filter((l) => l.name === '[X1] red');
|
|
assert.equal(x1.length, 1, readFileSync(out, 'utf8'));
|
|
assert.deepEqual(Object.keys(x1[0]).sort(),
|
|
['cname', 'code', 'failureType', 'file', 'name', 'nesting', 'ok', 'skip', 'todo']);
|
|
assert.deepEqual([x1[0].ok, x1[0].code, x1[0].cname, x1[0].failureType, x1[0].nesting],
|
|
[false, 'ERR_ASSERTION', 'AssertionError', 'testCodeFailure', 0]);
|
|
assert.equal(x1[0].file, join(dir, 'x1.test.mjs'));
|
|
const r = judgeFile(dir, ['X1'], out);
|
|
assert.equal(r.status, 0, r.out);
|
|
assert.match(r.out, /^red: \[X1\] 1 test\(s\)$/m, r.out);
|
|
});
|
|
});
|
|
|
|
test('[E23f] red reporter + judge: a { skip: true } and a { todo: true } [X1] test carry skip and todo in their lines -> exit 1', () => {
|
|
inTmp((dir) => {
|
|
const out = report(dir, {
|
|
'x1.test.mjs': `${HEAD}test('[X1] red', () => { assert.equal(1, 2); });\n`
|
|
+ `test('[X1] skipped', { skip: true }, () => {});\n`
|
|
+ `test('[X1] todo', { todo: true }, () => { assert.equal(1, 2); });\n`,
|
|
});
|
|
const lines = readFileSync(out, 'utf8').split('\n').filter(Boolean).map((l) => JSON.parse(l));
|
|
const byName = (n) => lines.filter((l) => l.name === n);
|
|
assert.equal(byName('[X1] skipped').length, 1, readFileSync(out, 'utf8'));
|
|
assert.equal(byName('[X1] todo').length, 1, readFileSync(out, 'utf8'));
|
|
assert.deepEqual([byName('[X1] skipped')[0].skip, byName('[X1] skipped')[0].todo], [true, false]);
|
|
assert.deepEqual([byName('[X1] todo')[0].skip, byName('[X1] todo')[0].todo], [false, true]);
|
|
const hits = byName('[X1] red');
|
|
assert.ok(hits.length > 0, `the reporter wrote no [X1] red line: ${readFileSync(out, 'utf8')}`);
|
|
assert.deepEqual([byName('[X1] red')[0].skip, byName('[X1] red')[0].todo], [false, false]);
|
|
const r = judgeFile(dir, ['X1'], out);
|
|
assert.equal(r.status, 1, r.out);
|
|
assert.match(r.out, /^broken: skipped \[X1\] \[X1\] skipped$/m, r.out);
|
|
assert.match(r.out, /^broken: todo \[X1\] \[X1\] todo$/m, r.out);
|
|
});
|
|
});
|