test(red): prove every id of round 4.5a red at ab6b32c, with a judge that is tested to go red
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
7869c4cbae
commit
5476a36782
SSH key fingerprint:
SHA256:JakMjO6FTBBzN0Bhfj9saOoEjaFxlSdYuZQQpM/lF9Q
5 changed files with 629 additions and 0 deletions
167
tests/fix-round-4-5a-red.sh
Normal file
167
tests/fix-round-4-5a-red.sh
Normal file
|
|
@ -0,0 +1,167 @@
|
|||
#!/usr/bin/env bash
|
||||
# tests/fix-round-4-5a-red.sh
|
||||
# The red proof of fix round 4-5a (brief: the last Success Criterion). Every
|
||||
# id of the round has tests whose names start with "[id]"; each of them must
|
||||
# fail on an assertion at the round's start point ab6b32c.
|
||||
# 1. Clone the repo into a temp directory outside it (git clone --shared
|
||||
# --no-checkout, then checkout --detach BASE, as
|
||||
# lib/verification/red-proof.mjs does; nothing registers in
|
||||
# .git/worktrees).
|
||||
# 2. Replace the clone's tests/ with the working tree's whole tests/ (tar),
|
||||
# helpers and fixtures included, so a new helper is no crash.
|
||||
# 3. Run the round's eight test files there, only the tagged tests
|
||||
# (--test-name-pattern per id), under tests/helpers/red-reporter.mjs, and
|
||||
# let tests/helpers/red-judge.mjs decide (--pass-ids DJ: DJ's defect is a
|
||||
# missing test, its sentences exist at ab6b32c).
|
||||
# 4. DJ by mutation: A deletes the copy-through span in
|
||||
# commands/trekreview.md, B deletes both spans in
|
||||
# agents/review-coordinator.md; each span must match exactly once. Each
|
||||
# run is judged with --mutation DJ; the file is restored with cp after.
|
||||
# Exit 0 all red, 1 a rule broken (the judge names it), 2 a setup failure
|
||||
# (clone, checkout, node or git missing); never 0 on a setup failure.
|
||||
# bash 3.2 compatible, ASCII only, no arrays.
|
||||
|
||||
set -u
|
||||
set -f
|
||||
|
||||
cd "$(dirname "$0")/.." || { echo "red script: SETUP FAIL - cannot cd to the repo root" >&2; exit 2; }
|
||||
ROOT="$(pwd -P)"
|
||||
|
||||
BASE=ab6b32c9304534b77c7ca2b36e7c993682c9efde
|
||||
IDS='#52 #47 #48 #46 #2 #26 #33 #1 #30 #28 R6 D67 #15 #12 #66 #102 #57 #67 #113 #111 #24 #23 F9 F4a F4b F4c WF CL ORD DJ'
|
||||
FILES='tests/lib/fix-round-4-5a-prose.test.mjs tests/lib/verdict-gate.test.mjs tests/lib/park-attempt-files.test.mjs tests/lib/autonomy-gate.test.mjs tests/lib/arg-parser.test.mjs tests/scripts/trekreview-armb-workflow.test.mjs tests/lib/coordinator-contract.test.mjs tests/validators/next-session-prompt-validator.test.mjs'
|
||||
JUDGE="$ROOT/tests/helpers/red-judge.mjs"
|
||||
|
||||
unset GIT_DIR GIT_WORK_TREE GIT_INDEX_FILE NODE_TEST_CONTEXT CLAUDE_CODE_SESSION_ID
|
||||
|
||||
setup_fail() {
|
||||
echo "red script: SETUP FAIL - $1" >&2
|
||||
exit 2
|
||||
}
|
||||
|
||||
command -v node >/dev/null 2>&1 || setup_fail "node is not on PATH"
|
||||
command -v git >/dev/null 2>&1 || setup_fail "git is not on PATH"
|
||||
command -v tar >/dev/null 2>&1 || setup_fail "tar is not on PATH"
|
||||
[ -f "$JUDGE" ] || setup_fail "missing $JUDGE"
|
||||
|
||||
TMP="$(mktemp -d "${TMPDIR:-/tmp}/fix-round-4-5a-red.XXXXXX")" || setup_fail "mktemp -d failed"
|
||||
case "$TMP" in
|
||||
''|/) setup_fail "mktemp -d gave an unusable directory '$TMP'" ;;
|
||||
esac
|
||||
TMP_REAL="$(cd "$TMP" && pwd -P)" || setup_fail "cannot enter $TMP"
|
||||
case "$TMP_REAL/" in
|
||||
"$ROOT"/*) rmdir "$TMP" 2>/dev/null; setup_fail "the temp directory $TMP_REAL is inside the repo" ;;
|
||||
esac
|
||||
trap 'rm -rf "$TMP"' EXIT
|
||||
trap 'rm -rf "$TMP"; exit 2' INT TERM HUP
|
||||
|
||||
TREE="$TMP/tree"
|
||||
|
||||
# ---- 1. the start point ----
|
||||
|
||||
git clone --quiet --shared --no-checkout "$ROOT" "$TREE" || setup_fail "could not clone $ROOT"
|
||||
git -C "$TREE" checkout --quiet --detach "$BASE" || setup_fail "could not check out $BASE"
|
||||
|
||||
# ---- 2. the working tree's whole tests/ ----
|
||||
|
||||
tar -cf "$TMP/tests.tar" tests || setup_fail "could not tar tests/"
|
||||
rm -rf "$TREE/tests" || setup_fail "could not remove the start point's tests/"
|
||||
tar -xf "$TMP/tests.tar" -C "$TREE" || setup_fail "could not copy tests/ into the start point"
|
||||
|
||||
for f in $FILES; do
|
||||
[ -f "$TREE/$f" ] || setup_fail "missing test file $f"
|
||||
done
|
||||
|
||||
for id in $IDS; do
|
||||
printf '%s\n' "$id"
|
||||
done > "$TMP/ids" || setup_fail "could not write $TMP/ids"
|
||||
|
||||
# run_tagged <results> <file...>: the tagged tests of the files, run in the
|
||||
# start point under the red reporter. The positional parameters carry the
|
||||
# name patterns (no arrays in bash 3.2).
|
||||
run_tagged() {
|
||||
out="$1"
|
||||
shift
|
||||
(cd "$TREE" && node --import ./tests/helpers/no-session-id.mjs --test \
|
||||
--test-reporter=./tests/helpers/red-reporter.mjs \
|
||||
--test-reporter-destination="$out" "$@") >/dev/null 2>"$out.stderr"
|
||||
[ -s "$out" ] || { cat "$out.stderr" >&2; setup_fail "the test run wrote no results to $out"; }
|
||||
}
|
||||
|
||||
# judge <args...>: the red judge; 2 (usage or input) is a setup failure.
|
||||
judge() {
|
||||
node "$JUDGE" "$@"
|
||||
rc=$?
|
||||
[ "$rc" -eq 2 ] && setup_fail "the red judge could not run (exit 2)"
|
||||
return "$rc"
|
||||
}
|
||||
|
||||
STATUS=0
|
||||
|
||||
# ---- 3. every id at the start point ----
|
||||
|
||||
set --
|
||||
for id in $IDS; do
|
||||
set -- "$@" "--test-name-pattern=\\[$id\\]"
|
||||
done
|
||||
for f in $FILES; do
|
||||
set -- "$@" "$f"
|
||||
done
|
||||
|
||||
echo "red script: the tagged tests at $BASE"
|
||||
run_tagged "$TMP/base.jsonl" "$@"
|
||||
judge --ids "$TMP/ids" --results "$TMP/base.jsonl" --pass-ids DJ || STATUS=1
|
||||
|
||||
# ---- 4. DJ under its two mutations ----
|
||||
|
||||
cat > "$TMP/mutate.mjs" <<'EOF'
|
||||
// mutate.mjs <file> <A|B>: delete DJ's copy-through span(s); each span must
|
||||
// match exactly once, else exit 1 and leave the file as it was.
|
||||
import { readFileSync, writeFileSync } from 'node:fs';
|
||||
const [file, which] = process.argv.slice(2);
|
||||
const SPANS = {
|
||||
A: [
|
||||
[/, and pass each reviewer's\s+`### Declined to Judge` lines verbatim from its saved\s+`\{project_dir\}\/\.review-out-\{reviewer\}\.md`: the coordinator copies them into\s+`## Declined to Judge`, so nothing a reviewer set aside disappears\./g, '.'],
|
||||
],
|
||||
B: [
|
||||
[/^- \*\*Reviewers' Declined to Judge lines\*\* \u2014 each reviewer's own\s+`### Declined to Judge` list, verbatim\. You copy them into\s+`## Declined to Judge` \(body section 9\); you do not rule on them\.\n/gm, ''],
|
||||
[/, then every\s+line a reviewer wrote under its own `### Declined to Judge`, copied as\s+`\{reviewer\}: \{line\}` \(they are not findings: they stay out of the JSON\s+`declined_to_judge` array and do not change the verdict\)/g, ''],
|
||||
],
|
||||
};
|
||||
if (!SPANS[which]) { console.error(`mutate: unknown mutation ${which}`); process.exit(1); }
|
||||
let text = readFileSync(file, 'utf8');
|
||||
for (const [re, by] of SPANS[which]) {
|
||||
const n = (text.match(re) || []).length;
|
||||
if (n !== 1) { console.error(`mutate ${which}: ${re.source.slice(0, 60)}... matched ${n} times in ${file}, not once`); process.exit(1); }
|
||||
text = text.replace(re, by);
|
||||
}
|
||||
writeFileSync(file, text);
|
||||
console.log(`mutate ${which}: ${SPANS[which].length} span(s) deleted in ${file}`);
|
||||
EOF
|
||||
[ -s "$TMP/mutate.mjs" ] || setup_fail "could not write $TMP/mutate.mjs"
|
||||
|
||||
# mutation <A|B> <repo-relative file> <what>: apply, run the DJ tests, judge, restore.
|
||||
mutation() {
|
||||
m="$1"
|
||||
rel="$2"
|
||||
echo "red script: DJ mutation $m ($3)"
|
||||
cp "$TREE/$rel" "$TMP/mutation-$m.orig" || setup_fail "could not save $rel"
|
||||
if node "$TMP/mutate.mjs" "$TREE/$rel" "$m"; then
|
||||
run_tagged "$TMP/mutation-$m.jsonl" "--test-name-pattern=\\[DJ\\]" tests/lib/fix-round-4-5a-prose.test.mjs
|
||||
judge --ids "$TMP/ids" --results "$TMP/mutation-$m.jsonl" --mutation DJ || STATUS=1
|
||||
else
|
||||
echo "broken: mutation-not-applied [DJ] $m on $rel"
|
||||
STATUS=1
|
||||
fi
|
||||
cp "$TMP/mutation-$m.orig" "$TREE/$rel" || setup_fail "could not restore $rel"
|
||||
}
|
||||
|
||||
mutation A commands/trekreview.md "the copy-through span in commands/trekreview.md deleted"
|
||||
mutation B agents/review-coordinator.md "both spans in agents/review-coordinator.md deleted"
|
||||
|
||||
if [ "$STATUS" -eq 0 ]; then
|
||||
echo "red script: ok - every id red at $BASE, DJ under both mutations"
|
||||
else
|
||||
echo "red script: FAIL - a rule is broken (see the broken: lines above)" >&2
|
||||
fi
|
||||
exit "$STATUS"
|
||||
145
tests/helpers/red-judge.mjs
Normal file
145
tests/helpers/red-judge.mjs
Normal file
|
|
@ -0,0 +1,145 @@
|
|||
#!/usr/bin/env node
|
||||
// tests/helpers/red-judge.mjs
|
||||
// The judge of the red script of fix round 4-5a (tests/fix-round-4-5a-red.sh).
|
||||
// It reads the JSON lines tests/helpers/red-reporter.mjs wrote for a run at
|
||||
// ab6b32c and decides whether every id of the round was seen red there: each
|
||||
// tagged test (a name that starts with "[id]") failed on an assertion.
|
||||
//
|
||||
// node tests/helpers/red-judge.mjs --ids <file> --results <jsonl>
|
||||
// [--pass-ids <id>[,<id>...]] [--mutation <id>]
|
||||
//
|
||||
// Plain run, one line per broken rule and exit 1 when:
|
||||
// broken: no-test [id] a listed id has no test
|
||||
// broken: passed [id] {name} a tagged test passed (unless its
|
||||
// id is in --pass-ids)
|
||||
// broken: skipped [id] {name} a tagged test was skipped
|
||||
// broken: todo [id] {name} a tagged test is a todo
|
||||
// broken: not-assertion [id] {name} ({code} {cname})
|
||||
// it failed on anything but
|
||||
// ERR_ASSERTION / AssertionError
|
||||
// (a crash is not red)
|
||||
// broken: untagged-failure {name} ({file}) a failing event with no tag (a
|
||||
// file that failed to load)
|
||||
// broken: unlisted-tag [tag] {name} a tag that is not in the list
|
||||
// An aggregate (a suite or file event with failureType subtestsFailed: it
|
||||
// failed only because a child failed) is not counted.
|
||||
// --mutation <id>: only that id's tests are judged (plus untagged failures
|
||||
// and unlisted tags); exit 1 with "broken: mutation-not-red [id]" unless at
|
||||
// least one of them fails on an assertion, and a "broken:" line for any of
|
||||
// them that fails otherwise, is skipped or is a todo. A passing one is fine.
|
||||
// Otherwise exit 0, printing "red: [id] {n} test(s)" per id (and
|
||||
// "pass: [id] {n} test(s) (--pass-ids)" for a passing --pass-ids id).
|
||||
// Exit 2: a usage error or an unreadable input (never 0).
|
||||
|
||||
import { readFileSync } from 'node:fs';
|
||||
|
||||
/** Print a usage problem and exit 2. */
|
||||
function usage(msg) {
|
||||
process.stderr.write(`red-judge: ${msg}\n` +
|
||||
'usage: red-judge.mjs --ids <file> --results <jsonl> [--pass-ids <id>[,<id>...]] [--mutation <id>]\n');
|
||||
process.exit(2);
|
||||
}
|
||||
|
||||
/** --flag value pairs; every flag takes one value. */
|
||||
function parse(argv) {
|
||||
const known = new Set(['--ids', '--results', '--pass-ids', '--mutation']);
|
||||
const o = {};
|
||||
for (let i = 0; i < argv.length; i += 2) {
|
||||
if (!known.has(argv[i])) usage(`unknown argument ${argv[i]}`);
|
||||
if (i + 1 >= argv.length || argv[i + 1].startsWith('--')) usage(`${argv[i]} needs a value`);
|
||||
o[argv[i]] = argv[i + 1];
|
||||
}
|
||||
if (!o['--ids'] || !o['--results']) usage('--ids and --results are required');
|
||||
return o;
|
||||
}
|
||||
|
||||
/** A file's text, or exit 2. */
|
||||
function readText(path, what) {
|
||||
try {
|
||||
return readFileSync(path, 'utf8');
|
||||
} catch (e) {
|
||||
return usage(`cannot read the ${what} file ${path}: ${e.code || e.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
/** The id a test name starts with, in brackets, or null. */
|
||||
function tagOf(name) {
|
||||
const m = /^\[([^\]\s]+)\]/.exec(String(name));
|
||||
return m ? m[1] : null;
|
||||
}
|
||||
|
||||
/** One line of output: whitespace collapsed. */
|
||||
const one = (s) => String(s).replace(/\s+/g, ' ').trim();
|
||||
|
||||
const opts = parse(process.argv.slice(2));
|
||||
const ids = readText(opts['--ids'], 'ids').split('\n').map((l) => l.trim()).filter(Boolean);
|
||||
if (ids.length === 0) usage(`the ids file ${opts['--ids']} lists no id`);
|
||||
const listed = new Set(ids);
|
||||
const passIds = new Set((opts['--pass-ids'] || '').split(',').map((s) => s.trim()).filter(Boolean));
|
||||
const mutation = opts['--mutation'] || null;
|
||||
if (mutation && !listed.has(mutation)) usage(`--mutation ${mutation} is not in the ids file`);
|
||||
|
||||
const events = [];
|
||||
readText(opts['--results'], 'results').split('\n').forEach((line, i) => {
|
||||
if (!line.trim()) return;
|
||||
try {
|
||||
events.push(JSON.parse(line));
|
||||
} catch {
|
||||
usage(`line ${i + 1} of ${opts['--results']} is not JSON`);
|
||||
}
|
||||
});
|
||||
|
||||
const broken = [];
|
||||
const seen = new Map();
|
||||
const red = new Map();
|
||||
const passed = new Map();
|
||||
const bump = (m, k) => m.set(k, (m.get(k) || 0) + 1);
|
||||
|
||||
for (const e of events) {
|
||||
if (e.ok === false && e.failureType === 'subtestsFailed') continue;
|
||||
const name = one(e.name);
|
||||
const tag = tagOf(name);
|
||||
if (!tag) {
|
||||
if (e.ok === false) broken.push(`broken: untagged-failure ${name} (${e.file ?? 'no file'})`);
|
||||
continue;
|
||||
}
|
||||
if (!listed.has(tag)) {
|
||||
broken.push(`broken: unlisted-tag [${tag}] ${name}`);
|
||||
continue;
|
||||
}
|
||||
if (mutation && tag !== mutation) continue;
|
||||
bump(seen, tag);
|
||||
if (e.skip) {
|
||||
broken.push(`broken: skipped [${tag}] ${name}`);
|
||||
} else if (e.todo) {
|
||||
broken.push(`broken: todo [${tag}] ${name}`);
|
||||
} else if (e.ok === true) {
|
||||
if (mutation || passIds.has(tag)) bump(passed, tag);
|
||||
else broken.push(`broken: passed [${tag}] ${name}`);
|
||||
} else if (e.code === 'ERR_ASSERTION' && e.cname === 'AssertionError') {
|
||||
bump(red, tag);
|
||||
} else {
|
||||
broken.push(`broken: not-assertion [${tag}] ${name} (${e.code ?? 'no code'} ${e.cname ?? 'no name'})`);
|
||||
}
|
||||
}
|
||||
|
||||
const judged = mutation ? [mutation] : ids;
|
||||
for (const id of judged) {
|
||||
if (mutation) {
|
||||
if (!red.get(id)) broken.push(`broken: mutation-not-red [${id}]`);
|
||||
} else if (!seen.get(id)) {
|
||||
broken.push(`broken: no-test [${id}]`);
|
||||
}
|
||||
}
|
||||
|
||||
if (broken.length > 0) {
|
||||
process.stdout.write(`${broken.join('\n')}\n`);
|
||||
process.exit(1);
|
||||
}
|
||||
const out = [];
|
||||
for (const id of judged) {
|
||||
if (red.get(id)) out.push(`red: [${id}] ${red.get(id)} test(s)`);
|
||||
if (!mutation && passed.get(id)) out.push(`pass: [${id}] ${passed.get(id)} test(s) (--pass-ids)`);
|
||||
}
|
||||
process.stdout.write(`${out.join('\n')}\n`);
|
||||
process.exit(0);
|
||||
39
tests/helpers/red-reporter.mjs
Normal file
39
tests/helpers/red-reporter.mjs
Normal file
|
|
@ -0,0 +1,39 @@
|
|||
// tests/helpers/red-reporter.mjs
|
||||
// A node:test custom reporter for the red script of fix round 4-5a
|
||||
// (tests/fix-round-4-5a-red.sh). It writes one JSON line per test:pass and
|
||||
// test:fail event, for tests/helpers/red-judge.mjs to read:
|
||||
// {name, file, nesting, ok, skip, todo, code, cname, failureType}
|
||||
// `code` and `cname` come from the error's cause (the error node:test reports
|
||||
// is its own ERR_TEST_FAILURE wrapper): ERR_ASSERTION / AssertionError for an
|
||||
// assertion. A cause with no code (a thrown TypeError) gives the wrapper's
|
||||
// code; a cause that is not an object (a file that failed to load, a suite
|
||||
// whose subtests failed) gives a null `cname`. A pass gives null for both.
|
||||
// `failureType` is node:test's own (testCodeFailure, subtestsFailed, ...).
|
||||
// Use: node --test --test-reporter=./tests/helpers/red-reporter.mjs
|
||||
// --test-reporter-destination=<file> <files>
|
||||
|
||||
/**
|
||||
* @param {AsyncIterable<{type: string, data: any}>} source - node:test events
|
||||
* @returns {AsyncGenerator<string>} one JSON line per pass or fail
|
||||
*/
|
||||
export default async function* redReporter(source) {
|
||||
for await (const event of source) {
|
||||
if (event.type !== 'test:pass' && event.type !== 'test:fail') continue;
|
||||
const d = event.data || {};
|
||||
const error = d.details && d.details.error;
|
||||
const cause = error && error.cause;
|
||||
const causeObj = cause !== null && typeof cause === 'object' ? cause : null;
|
||||
const line = {
|
||||
name: d.name,
|
||||
file: d.file ?? null,
|
||||
nesting: d.nesting,
|
||||
ok: event.type === 'test:pass',
|
||||
skip: Boolean(d.skip),
|
||||
todo: Boolean(d.todo),
|
||||
code: causeObj && causeObj.code != null ? String(causeObj.code) : (error && error.code != null ? String(error.code) : null),
|
||||
cname: causeObj && causeObj.name != null ? String(causeObj.name) : null,
|
||||
failureType: error && error.failureType !== undefined ? error.failureType : null,
|
||||
};
|
||||
yield `${JSON.stringify(line)}\n`;
|
||||
}
|
||||
}
|
||||
|
|
@ -1437,3 +1437,49 @@ test('e2e script: it names each of the four CLIs it runs', () => {
|
|||
assert.ok(text.includes(cli), `the e2e script does not run ${cli}`);
|
||||
}
|
||||
});
|
||||
|
||||
// --- red script: tests/fix-round-4-5a-red.sh --------------------------------
|
||||
// The round's red proof (the brief's last criterion). Static pins only: the
|
||||
// script runs itself as the step's Regression:. Its judge is tested to go red
|
||||
// in tests/lib/red-judge.test.mjs. The 30 ids are the brief's (brief.md:134
|
||||
// and the 4.5a rows), written here as a constant so the list in the script
|
||||
// cannot shrink without this test noticing.
|
||||
|
||||
const RED_SCRIPT = 'tests/fix-round-4-5a-red.sh';
|
||||
const RED_IDS = ['#52', '#47', '#48', '#46', '#2', '#26', '#33', '#1', '#30', '#28', 'R6', 'D67', '#15', '#12', '#66',
|
||||
'#102', '#57', '#67', '#113', '#111', '#24', '#23', 'F9', 'F4a', 'F4b', 'F4c', 'WF', 'CL', 'ORD', 'DJ'];
|
||||
|
||||
test('red script: tests/fix-round-4-5a-red.sh exists, is ASCII only, passes /bin/bash -n and uses no bash 4 form', () => {
|
||||
assert.ok(existsSync(join(ROOT, RED_SCRIPT)), `missing file ${RED_SCRIPT}`);
|
||||
const buf = readFileSync(join(ROOT, RED_SCRIPT));
|
||||
assert.equal(buf.findIndex((b) => b > 127), -1, 'the red script is not ASCII only');
|
||||
const r = spawnSync('/bin/bash', ['-n', join(ROOT, RED_SCRIPT)], { encoding: 'utf8', env: childEnv() });
|
||||
assert.equal(r.status, 0, r.stderr);
|
||||
const text = buf.toString('utf8');
|
||||
assert.match(text, /^set -u$/m, 'the red script does not set -u');
|
||||
for (const bash4 of [/\bdeclare -A\b/, /\bmapfile\b/, /\breadarray\b/, /\|&/, /^\s*[A-Za-z_]+=\(/m, /\$\{[A-Za-z_]+,,\}/]) {
|
||||
assert.doesNotMatch(text, bash4, `the red script uses ${bash4}, which bash 3.2 lacks or the round forbids`);
|
||||
}
|
||||
});
|
||||
|
||||
test('red script: an EXIT trap removes $TMP, and INT, TERM and HUP remove it too', () => {
|
||||
const text = read(RED_SCRIPT);
|
||||
assert.match(text, /^TMP="\$\(mktemp -d [^\n]*\)"/m, 'the red script does not make $TMP with mktemp -d');
|
||||
assert.match(text, /^trap 'rm -rf "\$TMP"' EXIT$/m, 'no EXIT trap removes $TMP');
|
||||
assert.match(text, /^trap '[^'\n]*rm -rf "\$TMP"[^'\n]*' INT TERM HUP$/m, 'no INT/TERM/HUP trap removes $TMP');
|
||||
});
|
||||
|
||||
test('red script: BASE is the full ab6b32c sha', () => {
|
||||
const text = read(RED_SCRIPT);
|
||||
const lines = text.split('\n').filter((l) => /^BASE=/.test(l));
|
||||
assert.deepEqual(lines, ['BASE=ab6b32c9304534b77c7ca2b36e7c993682c9efde'], 'BASE is not the one full ab6b32c sha');
|
||||
});
|
||||
|
||||
test('red script: the id list is the round\'s 30 ids, in the brief\'s order', () => {
|
||||
const text = read(RED_SCRIPT);
|
||||
const lines = text.split('\n').filter((l) => /^IDS=/.test(l));
|
||||
assert.equal(lines.length, 1, `expected one IDS= line; found ${lines.length}`);
|
||||
const m = lines[0].match(/^IDS='([^']*)'$/);
|
||||
assert.ok(m, `the IDS= line is not one single-quoted list: ${lines[0]}`);
|
||||
assert.deepEqual(m[1].trim().split(/\s+/), RED_IDS);
|
||||
});
|
||||
|
|
|
|||
232
tests/lib/red-judge.test.mjs
Normal file
232
tests/lib/red-judge.test.mjs
Normal file
|
|
@ -0,0 +1,232 @@
|
|||
// tests/lib/red-judge.test.mjs
|
||||
// The red judge of fix round 4-5a (tests/helpers/red-judge.mjs) and its
|
||||
// reporter (tests/helpers/red-reporter.mjs). The round's red script
|
||||
// (tests/fix-round-4-5a-red.sh) runs every tagged test at ab6b32c and lets the
|
||||
// judge decide; a judge that always exits 0 would prove nothing. So the judge
|
||||
// is fed planted JSON lines with the tags [X1] and [X2] (never a real id) and
|
||||
// must exit 1 on each broken rule, naming it on one line.
|
||||
//
|
||||
// The judge runs only as a CLI through spawnSync and is never imported: each
|
||||
// test asserts the exit code and the line the judge prints, so every test
|
||||
// fails on an assertion while the judge does not exist.
|
||||
//
|
||||
// Rule lines (one per broken rule, exit 1):
|
||||
// broken: no-test [id] a listed id has no test
|
||||
// broken: passed [id] {name} a tagged test passed
|
||||
// broken: skipped [id] {name} a tagged test was skipped
|
||||
// broken: todo [id] {name} a tagged test is a todo
|
||||
// broken: not-assertion [id] {name} (...) it failed, but not on an assertion
|
||||
// broken: untagged-failure {name} (...) a failing event with no tag
|
||||
// broken: unlisted-tag [tag] {name} a tag that is not in the list
|
||||
// broken: mutation-not-red [id] --mutation: no assertion failure
|
||||
// All red: exit 0 and "red: [id] {n} test(s)" per id.
|
||||
|
||||
import { test } from 'node:test';
|
||||
import { strict as assert } from 'node:assert';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { existsSync, mkdtempSync, readFileSync, realpathSync, rmSync, writeFileSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { dirname, join } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { childEnv } from '../helpers/child-env.mjs';
|
||||
|
||||
const ROOT = join(dirname(fileURLToPath(import.meta.url)), '..', '..');
|
||||
const JUDGE = join(ROOT, 'tests', 'helpers', 'red-judge.mjs');
|
||||
const REPORTER = join(ROOT, 'tests', 'helpers', 'red-reporter.mjs');
|
||||
|
||||
/** A temp directory outside tests/, removed after `fn` returns. */
|
||||
function inTmp(fn) {
|
||||
const dir = realpathSync(mkdtempSync(join(tmpdir(), 'red-judge-test-')));
|
||||
try {
|
||||
return fn(dir);
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
/** One planted reporter line. Defaults: a top-level test that failed on an assertion. */
|
||||
function ev(name, over = {}) {
|
||||
return {
|
||||
name, file: '/planted/x.test.mjs', nesting: 0, ok: false, skip: false, todo: false,
|
||||
code: 'ERR_ASSERTION', cname: 'AssertionError', failureType: 'testCodeFailure', ...over,
|
||||
};
|
||||
}
|
||||
const passing = (name) => ev(name, { ok: true, code: null, cname: null, failureType: null });
|
||||
|
||||
/** Run the judge on an id list and planted events, with extra flags. */
|
||||
function judge(ids, events, extra = []) {
|
||||
return inTmp((dir) => {
|
||||
const idsFile = join(dir, 'ids');
|
||||
const results = join(dir, 'results.jsonl');
|
||||
writeFileSync(idsFile, ids.join('\n') + '\n');
|
||||
writeFileSync(results, events.map((e) => JSON.stringify(e)).join('\n') + (events.length ? '\n' : ''));
|
||||
const r = spawnSync(process.execPath, [JUDGE, '--ids', idsFile, '--results', results, ...extra],
|
||||
{ encoding: 'utf8', env: childEnv() });
|
||||
return { status: r.status, out: `${r.stdout}${r.stderr}` };
|
||||
});
|
||||
}
|
||||
|
||||
test('red judge: a tagged test that passes -> exit 1, "broken: passed [X1]"', () => {
|
||||
const r = judge(['X1'], [passing('[X1] holds')]);
|
||||
assert.equal(r.status, 1, r.out);
|
||||
assert.match(r.out, /^broken: passed \[X1\] \[X1\] holds$/m, r.out);
|
||||
});
|
||||
|
||||
test('red judge: a tagged test that fails with ERR_MODULE_NOT_FOUND -> exit 1, "broken: not-assertion [X1]" (a crash is not red)', () => {
|
||||
const r = judge(['X1'], [ev('[X1] loads', { code: 'ERR_MODULE_NOT_FOUND', cname: 'Error' })]);
|
||||
assert.equal(r.status, 1, r.out);
|
||||
assert.match(r.out, /^broken: not-assertion \[X1\] \[X1\] loads \(ERR_MODULE_NOT_FOUND Error\)$/m, r.out);
|
||||
});
|
||||
|
||||
test('red judge: a listed id with no test -> exit 1, "broken: no-test [X2]"', () => {
|
||||
const r = judge(['X1', 'X2'], [ev('[X1] red')]);
|
||||
assert.equal(r.status, 1, r.out);
|
||||
assert.match(r.out, /^broken: no-test \[X2\]$/m, r.out);
|
||||
assert.doesNotMatch(r.out, /^broken: .*\[X1\]/m, 'X1 is red and must not be named');
|
||||
});
|
||||
|
||||
test('red judge: a tag that is not in the list -> exit 1, "broken: unlisted-tag [X2]"', () => {
|
||||
const r = judge(['X1'], [ev('[X1] red'), ev('[X2] stray')]);
|
||||
assert.equal(r.status, 1, r.out);
|
||||
assert.match(r.out, /^broken: unlisted-tag \[X2\] \[X2\] stray$/m, r.out);
|
||||
});
|
||||
|
||||
test('red judge: an untagged failing event (a file that failed to load) -> exit 1, "broken: untagged-failure"', () => {
|
||||
const r = judge(['X1'], [ev('[X1] red'), ev('x.test.mjs', { code: null, cname: null })]);
|
||||
assert.equal(r.status, 1, r.out);
|
||||
assert.match(r.out, /^broken: untagged-failure x\.test\.mjs \(\/planted\/x\.test\.mjs\)$/m, r.out);
|
||||
});
|
||||
|
||||
test('red judge: an untagged aggregate (failureType subtestsFailed) next to a tagged assertion failure is not counted -> exit 0', () => {
|
||||
const r = judge(['X1'], [
|
||||
ev('[X1] red', { nesting: 1 }),
|
||||
ev('group', { code: 'ERR_TEST_FAILURE', cname: null, failureType: 'subtestsFailed' }),
|
||||
]);
|
||||
assert.equal(r.status, 0, r.out);
|
||||
assert.match(r.out, /^red: \[X1\] 1 test\(s\)$/m, r.out);
|
||||
assert.doesNotMatch(r.out, /^broken:/m, r.out);
|
||||
});
|
||||
|
||||
test('red judge: a skipped tagged test -> exit 1, "broken: skipped [X1]"', () => {
|
||||
const r = judge(['X1'], [ev('[X1] red'), ev('[X1] later', { ok: true, skip: true, code: null, cname: null, failureType: null })]);
|
||||
assert.equal(r.status, 1, r.out);
|
||||
assert.match(r.out, /^broken: skipped \[X1\] \[X1\] later$/m, r.out);
|
||||
});
|
||||
|
||||
test('red judge: every id covered by assertion failures -> exit 0, one "red: [id] {n} test(s)" line per id', () => {
|
||||
const r = judge(['X1', 'X2'], [ev('[X1] one'), ev('[X1] two'), ev('[X2] three'), passing('untagged keep')]);
|
||||
assert.equal(r.status, 0, r.out);
|
||||
assert.match(r.out, /^red: \[X1\] 2 test\(s\)$/m, r.out);
|
||||
assert.match(r.out, /^red: \[X2\] 1 test\(s\)$/m, r.out);
|
||||
assert.doesNotMatch(r.out, /^broken:/m, r.out);
|
||||
});
|
||||
|
||||
test('red judge: --pass-ids X1 with [X1] passing -> exit 0, the other ids still red', () => {
|
||||
const r = judge(['X1', 'X2'], [passing('[X1] holds'), ev('[X2] red')], ['--pass-ids', 'X1']);
|
||||
assert.equal(r.status, 0, r.out);
|
||||
assert.match(r.out, /^red: \[X2\] 1 test\(s\)$/m, r.out);
|
||||
assert.doesNotMatch(r.out, /^broken:/m, r.out);
|
||||
});
|
||||
|
||||
test('red judge: --pass-ids X1 exempts only X1: a passing [X2] next to a passing [X1] -> exit 1, "broken: passed [X2]"', () => {
|
||||
const r = judge(['X1', 'X2'], [passing('[X1] holds'), passing('[X2] holds too')], ['--pass-ids', 'X1']);
|
||||
assert.equal(r.status, 1, r.out);
|
||||
assert.match(r.out, /^broken: passed \[X2\] \[X2\] .+$/m, r.out);
|
||||
assert.doesNotMatch(r.out, /^broken: .*\[X1\]/m, 'X1 is in --pass-ids and must not be named');
|
||||
});
|
||||
|
||||
test('red judge: a todo tagged test next to a red one -> exit 1, "broken: todo [X1]"', () => {
|
||||
const r = judge(['X1'], [ev('[X1] red'), ev('[X1] someday', { todo: true })]);
|
||||
assert.equal(r.status, 1, r.out);
|
||||
assert.match(r.out, /^broken: todo \[X1\] \[X1\] someday$/m, r.out);
|
||||
});
|
||||
|
||||
test('red judge: --mutation X1 with no [X1] assertion failure -> exit 1, "broken: mutation-not-red [X1]"', () => {
|
||||
const r = judge(['X1'], [passing('[X1] holds')], ['--mutation', 'X1']);
|
||||
assert.equal(r.status, 1, r.out);
|
||||
assert.match(r.out, /^broken: mutation-not-red \[X1\]$/m, r.out);
|
||||
});
|
||||
|
||||
test('red judge: --mutation X1 with one [X1] assertion failure and one [X1] pass -> exit 0, "red: [X1] 1 test(s)"', () => {
|
||||
const r = judge(['X1'], [passing('[X1] holds'), ev('[X1] red')], ['--mutation', 'X1']);
|
||||
assert.equal(r.status, 0, r.out);
|
||||
assert.match(r.out, /^red: \[X1\] 1 test\(s\)$/m, r.out);
|
||||
assert.doesNotMatch(r.out, /^broken:/m, r.out);
|
||||
});
|
||||
|
||||
test('red judge: --mutation X1 with one [X1] assertion failure and one [X1] TypeError crash -> exit 1, "broken: not-assertion [X1]"', () => {
|
||||
const r = judge(['X1'], [ev('[X1] red'), ev('[X1] crash', { code: 'ERR_TEST_FAILURE', cname: 'TypeError' })], ['--mutation', 'X1']);
|
||||
assert.equal(r.status, 1, r.out);
|
||||
assert.match(r.out, /^broken: not-assertion \[X1\] \[X1\] crash \(ERR_TEST_FAILURE TypeError\)$/m, r.out);
|
||||
});
|
||||
|
||||
/** Write fixture test files into `dir`, run them under the red reporter, return the JSONL path. */
|
||||
function report(dir, files) {
|
||||
for (const [name, body] of Object.entries(files)) writeFileSync(join(dir, name), body);
|
||||
const out = join(dir, 'out.jsonl');
|
||||
spawnSync(process.execPath, ['--test', `--test-reporter=${REPORTER}`, `--test-reporter-destination=${out}`,
|
||||
...Object.keys(files)], { cwd: dir, encoding: 'utf8', env: childEnv({ NODE_TEST_CONTEXT: undefined }) });
|
||||
assert.ok(existsSync(out), 'the red reporter wrote no results file');
|
||||
return out;
|
||||
}
|
||||
|
||||
/** The judge on a results file the reporter wrote. */
|
||||
function judgeFile(dir, ids, results) {
|
||||
writeFileSync(join(dir, 'ids'), ids.join('\n') + '\n');
|
||||
const r = spawnSync(process.execPath, [JUDGE, '--ids', join(dir, 'ids'), '--results', results],
|
||||
{ encoding: 'utf8', env: childEnv() });
|
||||
return { status: r.status, out: `${r.stdout}${r.stderr}` };
|
||||
}
|
||||
|
||||
const HEAD = "import { test } from 'node:test';\nimport { strict as assert } from 'node:assert';\n";
|
||||
|
||||
test('red reporter + judge: a passing [X1] test and an [X2] test that throws a TypeError, run by node --test -> exit 1', () => {
|
||||
inTmp((dir) => {
|
||||
const out = report(dir, {
|
||||
'x1.test.mjs': `${HEAD}test('[X1] holds', () => { assert.equal(1, 1); });\n`,
|
||||
'x2.test.mjs': `${HEAD}test('[X2] throws', () => { throw new TypeError('boom'); });\n`,
|
||||
});
|
||||
const r = judgeFile(dir, ['X1', 'X2'], out);
|
||||
assert.equal(r.status, 1, r.out);
|
||||
assert.match(r.out, /^broken: passed \[X1\] \[X1\] holds$/m, r.out);
|
||||
assert.match(r.out, /^broken: not-assertion \[X2\] \[X2\] throws \(ERR_TEST_FAILURE TypeError\)$/m, r.out);
|
||||
});
|
||||
});
|
||||
|
||||
test('red reporter + judge: an [X1] test that fails on assert.equal(1, 2) -> exit 0, and the line carries the reporter fields', () => {
|
||||
inTmp((dir) => {
|
||||
const out = report(dir, { 'x1.test.mjs': `${HEAD}test('[X1] red', () => { assert.equal(1, 2); });\n` });
|
||||
const lines = readFileSync(out, 'utf8').split('\n').filter(Boolean).map((l) => JSON.parse(l));
|
||||
const x1 = lines.filter((l) => l.name === '[X1] red');
|
||||
assert.equal(x1.length, 1, readFileSync(out, 'utf8'));
|
||||
assert.deepEqual(Object.keys(x1[0]).sort(),
|
||||
['cname', 'code', 'failureType', 'file', 'name', 'nesting', 'ok', 'skip', 'todo']);
|
||||
assert.deepEqual([x1[0].ok, x1[0].code, x1[0].cname, x1[0].failureType, x1[0].nesting],
|
||||
[false, 'ERR_ASSERTION', 'AssertionError', 'testCodeFailure', 0]);
|
||||
assert.equal(x1[0].file, join(dir, 'x1.test.mjs'));
|
||||
const r = judgeFile(dir, ['X1'], out);
|
||||
assert.equal(r.status, 0, r.out);
|
||||
assert.match(r.out, /^red: \[X1\] 1 test\(s\)$/m, r.out);
|
||||
});
|
||||
});
|
||||
|
||||
test('red reporter + judge: a { skip: true } and a { todo: true } [X1] test carry skip and todo in their lines -> exit 1', () => {
|
||||
inTmp((dir) => {
|
||||
const out = report(dir, {
|
||||
'x1.test.mjs': `${HEAD}test('[X1] red', () => { assert.equal(1, 2); });\n`
|
||||
+ `test('[X1] skipped', { skip: true }, () => {});\n`
|
||||
+ `test('[X1] todo', { todo: true }, () => { assert.equal(1, 2); });\n`,
|
||||
});
|
||||
const lines = readFileSync(out, 'utf8').split('\n').filter(Boolean).map((l) => JSON.parse(l));
|
||||
const byName = (n) => lines.filter((l) => l.name === n);
|
||||
assert.equal(byName('[X1] skipped').length, 1, readFileSync(out, 'utf8'));
|
||||
assert.equal(byName('[X1] todo').length, 1, readFileSync(out, 'utf8'));
|
||||
assert.deepEqual([byName('[X1] skipped')[0].skip, byName('[X1] skipped')[0].todo], [true, false]);
|
||||
assert.deepEqual([byName('[X1] todo')[0].skip, byName('[X1] todo')[0].todo], [false, true]);
|
||||
assert.deepEqual([byName('[X1] red')[0].skip, byName('[X1] red')[0].todo], [false, false]);
|
||||
const r = judgeFile(dir, ['X1'], out);
|
||||
assert.equal(r.status, 1, r.out);
|
||||
assert.match(r.out, /^broken: skipped \[X1\] \[X1\] skipped$/m, r.out);
|
||||
assert.match(r.out, /^broken: todo \[X1\] \[X1\] todo$/m, r.out);
|
||||
});
|
||||
});
|
||||
Loading…
Add table
Add a link
Reference in a new issue