test(red-proof): fix round 4.5b part 4 red proof and regression run
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
8b9b9439cf
commit
1f5b42556e
SSH key fingerprint:
SHA256:JakMjO6FTBBzN0Bhfj9saOoEjaFxlSdYuZQQpM/lF9Q
4 changed files with 299 additions and 0 deletions
52
tests/fix-round-4-5b-part-4-mutations.mjs
Normal file
52
tests/fix-round-4-5b-part-4-mutations.mjs
Normal file
|
|
@ -0,0 +1,52 @@
|
|||
// tests/fix-round-4-5b-part-4-mutations.mjs
|
||||
// The mutations of fix round 4-5b, part 4 (decision 90), for the shared red
|
||||
// script (tests/helpers/red-round.sh, through tests/helpers/red-mutate.mjs).
|
||||
// An id named here has tests that pass at the round's start point 591f3c4 and
|
||||
// is proven red under its mutations instead:
|
||||
// - CTB: the catch of the shell reading already blocks at 591f3c4 (the
|
||||
// round adds only its test, through a preload); the mutation makes the
|
||||
// catch let the command through, in 591f3c4's hook.
|
||||
// - NVR: the [TOK] title and part 2's case name are already reworded in the
|
||||
// working tree's tests/ that the script copies into the clone; each
|
||||
// mutation puts the old "never" wording back into one of the two files.
|
||||
// Every span must match exactly once in the clone (591f3c4's code with the
|
||||
// working tree's tests/). Data only; never matched by `npm test`.
|
||||
|
||||
export const MUTATIONS = [
|
||||
{
|
||||
key: 'CTB-catch',
|
||||
id: 'CTB',
|
||||
what: 'the catch of the shell reading lets the command through',
|
||||
spans: [
|
||||
{
|
||||
file: 'hooks/scripts/pre-bash-executor.mjs',
|
||||
find: 'blockArgs(true, true);',
|
||||
replace: 'void 0;',
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
key: 'NVR-title',
|
||||
id: 'NVR',
|
||||
what: 'the [TOK] title claims never again',
|
||||
spans: [
|
||||
{
|
||||
file: 'tests/lib/arg-parser-tokenize.test.mjs',
|
||||
find: '[TOK] a quoted value stops at a quote that starts a word, and',
|
||||
replace: '[TOK] a quoted value never swallows a later flag, and',
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
key: 'NVR-e2e',
|
||||
id: 'NVR',
|
||||
what: "part 2's case name claims never again",
|
||||
spans: [
|
||||
{
|
||||
file: 'tests/e2e-fix-round-4-5b-part-2.sh',
|
||||
find: "a quoted value stops at the quote that starts --since's value",
|
||||
replace: 'a quoted value never swallows --since',
|
||||
},
|
||||
],
|
||||
},
|
||||
];
|
||||
57
tests/fix-round-4-5b-part-4-red.sh
Normal file
57
tests/fix-round-4-5b-part-4-red.sh
Normal file
|
|
@ -0,0 +1,57 @@
|
|||
#!/usr/bin/env bash
|
||||
# tests/fix-round-4-5b-part-4-red.sh
|
||||
# The red proof of fix round 4-5b, part 4 (plan
|
||||
# .claude/projects/2026-10-31-fix-round-4-5b-part-4, decision 90), as a
|
||||
# wrapper over the shared red script tests/helpers/red-round.sh. Every id of
|
||||
# the round has tests whose names start with "[id]"; CSG, SLT, ESC and KI4
|
||||
# must each fail on an assertion at the round's start point 591f3c4, and CTB
|
||||
# and NVR, whose tests pass there, are proven red under their mutations.
|
||||
# This wrapper passes the shared script:
|
||||
# --base BASE, --ids IDS (the round's 6 ids), the round's FILES (the
|
||||
# three tagged test files; the untagged reader pins stay out);
|
||||
# --other-ids: the ids of fix round 4-5a, of 4-5b part 1, of part 2 and of
|
||||
# part 3, read with sed from the IDS= lines of tests/fix-round-4-5a-red.sh,
|
||||
# tests/fix-round-4-5b-red.sh, tests/fix-round-4-5b-part-2-red.sh and
|
||||
# tests/fix-round-4-5b-part-3-red.sh and passed as ONE value, in that
|
||||
# order (the shared script keeps only the last --other-ids it is given);
|
||||
# a tag on that list is not judged;
|
||||
# --mutations tests/fix-round-4-5b-part-4-mutations.mjs (CTB-catch,
|
||||
# NVR-title, NVR-e2e).
|
||||
# No --require-sha256: the round has no SHA-256 test.
|
||||
# Exit 0 all red, 1 a rule broken (the judge names it), 2 a setup failure
|
||||
# (also: no ids on one of the four older IDS= lines); never 0 on a setup
|
||||
# failure.
|
||||
# bash 3.2 compatible, ASCII only, no arrays.
|
||||
|
||||
set -u
|
||||
set -f
|
||||
|
||||
cd "$(dirname "$0")/.." || { echo "red script: setup failure: cannot cd to the repo root" >&2; exit 2; }
|
||||
ROOT="$(pwd -P)"
|
||||
|
||||
BASE=591f3c4793c751ae3c12fae403b9735341cd7591
|
||||
IDS='CSG SLT ESC CTB KI4 NVR'
|
||||
FILES='tests/hooks/bash-guard-args-gate.test.mjs tests/hooks/bash-guard-false-stops.test.mjs tests/lib/fix-round-4-5b-part-4-prose.test.mjs'
|
||||
|
||||
OTHER_A=$(sed -n "s/^IDS='\(.*\)'\$/\1/p" "$ROOT/tests/fix-round-4-5a-red.sh")
|
||||
if [ -z "$OTHER_A" ]; then
|
||||
echo "red script: setup failure: no ids on the IDS= line of $ROOT/tests/fix-round-4-5a-red.sh" >&2
|
||||
exit 2
|
||||
fi
|
||||
OTHER_B=$(sed -n "s/^IDS='\(.*\)'\$/\1/p" "$ROOT/tests/fix-round-4-5b-red.sh")
|
||||
if [ -z "$OTHER_B" ]; then
|
||||
echo "red script: setup failure: no ids on the IDS= line of $ROOT/tests/fix-round-4-5b-red.sh" >&2
|
||||
exit 2
|
||||
fi
|
||||
OTHER_C=$(sed -n "s/^IDS='\(.*\)'\$/\1/p" "$ROOT/tests/fix-round-4-5b-part-2-red.sh")
|
||||
if [ -z "$OTHER_C" ]; then
|
||||
echo "red script: setup failure: no ids on the IDS= line of $ROOT/tests/fix-round-4-5b-part-2-red.sh" >&2
|
||||
exit 2
|
||||
fi
|
||||
OTHER_D=$(sed -n "s/^IDS='\(.*\)'\$/\1/p" "$ROOT/tests/fix-round-4-5b-part-3-red.sh")
|
||||
if [ -z "$OTHER_D" ]; then
|
||||
echo "red script: setup failure: no ids on the IDS= line of $ROOT/tests/fix-round-4-5b-part-3-red.sh" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
exec /bin/bash "$ROOT/tests/helpers/red-round.sh" --base "$BASE" --ids "$IDS" --other-ids "$OTHER_A $OTHER_B $OTHER_C $OTHER_D" --mutations "$ROOT/tests/fix-round-4-5b-part-4-mutations.mjs" $FILES
|
||||
51
tests/fix-round-4-5b-part-4-regression.sh
Normal file
51
tests/fix-round-4-5b-part-4-regression.sh
Normal file
|
|
@ -0,0 +1,51 @@
|
|||
#!/usr/bin/env bash
|
||||
# tests/fix-round-4-5b-part-4-regression.sh
|
||||
# The Regression: of fix round 4-5b, part 4 (decision 90): the whole suite
|
||||
# (npm test), part 4's end to end (tests/e2e-fix-round-4-5b-part-4.sh),
|
||||
# part 3's (tests/e2e-fix-round-4-5b-part-3.sh), part 2's
|
||||
# (tests/e2e-fix-round-4-5b-part-2.sh), part 1's (tests/e2e-fix-round-4-5b.sh)
|
||||
# and part 4's red proof (tests/fix-round-4-5b-part-4-red.sh). The 4-5a,
|
||||
# part-1, part-2 and part-3 red proofs are not run here: one Regression
|
||||
# command has the criteria runner's 15-minute cap; they run as their own
|
||||
# Verification lines.
|
||||
# All six run, in that order; the script prints each exit and exits with the
|
||||
# first non-zero status, or 0 when all six exit 0.
|
||||
# bash 3.2 compatible, ASCII only, no arrays.
|
||||
|
||||
set -u
|
||||
|
||||
cd "$(dirname "$0")/.." || { echo "regression: cannot cd to the repo root" >&2; exit 2; }
|
||||
|
||||
FIRST=0
|
||||
|
||||
npm test
|
||||
rc=$?
|
||||
echo "regression: npm test exited $rc"
|
||||
[ "$FIRST" -eq 0 ] && FIRST=$rc
|
||||
|
||||
/bin/bash tests/e2e-fix-round-4-5b-part-4.sh
|
||||
rc=$?
|
||||
echo "regression: tests/e2e-fix-round-4-5b-part-4.sh exited $rc"
|
||||
[ "$FIRST" -eq 0 ] && FIRST=$rc
|
||||
|
||||
/bin/bash tests/e2e-fix-round-4-5b-part-3.sh
|
||||
rc=$?
|
||||
echo "regression: tests/e2e-fix-round-4-5b-part-3.sh exited $rc"
|
||||
[ "$FIRST" -eq 0 ] && FIRST=$rc
|
||||
|
||||
/bin/bash tests/e2e-fix-round-4-5b-part-2.sh
|
||||
rc=$?
|
||||
echo "regression: tests/e2e-fix-round-4-5b-part-2.sh exited $rc"
|
||||
[ "$FIRST" -eq 0 ] && FIRST=$rc
|
||||
|
||||
/bin/bash tests/e2e-fix-round-4-5b.sh
|
||||
rc=$?
|
||||
echo "regression: tests/e2e-fix-round-4-5b.sh exited $rc"
|
||||
[ "$FIRST" -eq 0 ] && FIRST=$rc
|
||||
|
||||
/bin/bash tests/fix-round-4-5b-part-4-red.sh
|
||||
rc=$?
|
||||
echo "regression: tests/fix-round-4-5b-part-4-red.sh exited $rc"
|
||||
[ "$FIRST" -eq 0 ] && FIRST=$rc
|
||||
|
||||
exit "$FIRST"
|
||||
139
tests/lib/fix-round-4-5b-part-4-scripts.test.mjs
Normal file
139
tests/lib/fix-round-4-5b-part-4-scripts.test.mjs
Normal file
|
|
@ -0,0 +1,139 @@
|
|||
// tests/lib/fix-round-4-5b-part-4-scripts.test.mjs
|
||||
// The form of fix round 4-5b part 4's shell scripts and mutation module (plan
|
||||
// .claude/projects/2026-10-31-fix-round-4-5b-part-4, Step 4d): the red
|
||||
// wrapper, the regression script, the end-to-end run and the mutations.
|
||||
// Untagged on purpose and listed in no red wrapper's FILES: these checks are
|
||||
// not a part-4 fix.
|
||||
// - each script exists, is ASCII only, passes /bin/bash -n and uses no
|
||||
// bash 4 form;
|
||||
// - the red wrapper hands the shared red script exactly one --other-ids,
|
||||
// whose words are the 4-5a IDS= words, then part 1's, then part 2's, then
|
||||
// part 3's, and --ids CSG SLT ESC CTB KI4 NVR (run from a temp root next
|
||||
// to a stub red-round.sh that prints its arguments, the method of
|
||||
// tests/lib/red-script.test.mjs);
|
||||
// - with the 4-5a, the part-1, the part-2 or the part-3 wrapper lacking its
|
||||
// IDS= line, the red wrapper exits 2 with a setup failure and never runs
|
||||
// the shared script;
|
||||
// - red-mutate.mjs list prints exactly the three part-4 mutations.
|
||||
|
||||
import { test } from 'node:test';
|
||||
import { strict as assert } from 'node:assert';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { dirname, join } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { childEnv } from '../helpers/child-env.mjs';
|
||||
|
||||
const ROOT = join(dirname(fileURLToPath(import.meta.url)), '..', '..');
|
||||
|
||||
const RED = 'tests/fix-round-4-5b-part-4-red.sh';
|
||||
const REGRESSION = 'tests/fix-round-4-5b-part-4-regression.sh';
|
||||
const E2E = 'tests/e2e-fix-round-4-5b-part-4.sh';
|
||||
const MUTATIONS = 'tests/fix-round-4-5b-part-4-mutations.mjs';
|
||||
const WRAP_A = 'tests/fix-round-4-5a-red.sh';
|
||||
const WRAP_B = 'tests/fix-round-4-5b-red.sh';
|
||||
const WRAP_C = 'tests/fix-round-4-5b-part-2-red.sh';
|
||||
const WRAP_D = 'tests/fix-round-4-5b-part-3-red.sh';
|
||||
const PART4_IDS = ['CSG', 'SLT', 'ESC', 'CTB', 'KI4', 'NVR'];
|
||||
|
||||
/** A repo file's bytes; the file must exist. */
|
||||
function bytes(rel) {
|
||||
assert.ok(existsSync(join(ROOT, rel)), `missing file ${rel}`);
|
||||
return readFileSync(join(ROOT, rel));
|
||||
}
|
||||
|
||||
/** The words of a wrapper's one IDS='...' line. */
|
||||
function idsOf(rel) {
|
||||
const lines = bytes(rel).toString('utf8').split('\n').filter((l) => l.startsWith('IDS='));
|
||||
assert.equal(lines.length, 1, `expected one IDS= line in ${rel}; found ${lines.length}`);
|
||||
const m = lines[0].match(/^IDS='([^']*)'$/);
|
||||
assert.ok(m, `the IDS= line of ${rel} is not one single-quoted list: ${lines[0]}`);
|
||||
return m[1].split(/\s+/).filter(Boolean);
|
||||
}
|
||||
|
||||
test('part-4 scripts: each exists, is ASCII only, passes /bin/bash -n and uses no bash 4 form', () => {
|
||||
for (const rel of [RED, REGRESSION, E2E]) {
|
||||
const buf = bytes(rel);
|
||||
assert.equal(buf.findIndex((b) => b > 127), -1, `${rel} is not ASCII only`);
|
||||
const r = spawnSync('/bin/bash', ['-n', join(ROOT, rel)], { encoding: 'utf8', env: childEnv() });
|
||||
assert.equal(r.status, 0, `${rel} does not pass /bin/bash -n: ${r.stderr}`);
|
||||
const text = buf.toString('utf8');
|
||||
for (const bash4 of [/\bdeclare -A\b/, /\bmapfile\b/, /\breadarray\b/, /\|&/, /^\s*[A-Za-z_]+=\(/m,
|
||||
/\$\{[A-Za-z_]+,,\}/, /\$\{[A-Za-z_]+\^\^\}/]) {
|
||||
assert.doesNotMatch(text, bash4, `${rel} uses ${bash4}, which bash 3.2 lacks`);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
/** A stand-in for the shared script: one ARG: line per argument, exit 0. */
|
||||
const STUB = `#!/bin/bash
|
||||
for a in "$@"; do printf 'ARG:%s\\n' "$a"; done
|
||||
exit 0
|
||||
`;
|
||||
|
||||
/** A temp root with the part-4 wrapper, the four older wrappers and the stub as tests/helpers/red-round.sh. */
|
||||
function stubRoot(base) {
|
||||
const dir = join(base, 'root');
|
||||
mkdirSync(join(dir, 'tests', 'helpers'), { recursive: true });
|
||||
for (const rel of [RED, WRAP_A, WRAP_B, WRAP_C, WRAP_D]) writeFileSync(join(dir, rel), bytes(rel));
|
||||
writeFileSync(join(dir, 'tests', 'helpers', 'red-round.sh'), STUB, { mode: 0o755 });
|
||||
return dir;
|
||||
}
|
||||
|
||||
/** The part-4 wrapper run from a temp root: status, output and the arguments the stub saw. */
|
||||
function runWrapper(dir) {
|
||||
const r = spawnSync('/bin/bash', [join(dir, RED)], { cwd: tmpdir(), encoding: 'utf8', timeout: 30000, env: childEnv() });
|
||||
const argv = (r.stdout || '').split('\n').filter((l) => l.startsWith('ARG:')).map((l) => l.slice(4));
|
||||
return { status: r.status, out: `${r.stdout}${r.stderr}`, stderr: r.stderr || '', argv };
|
||||
}
|
||||
|
||||
test('part-4 scripts: the red wrapper passes one --other-ids, the 4-5a, part-1, part-2 and part-3 ids in order, and its own six ids', () => {
|
||||
const text = bytes(RED).toString('utf8');
|
||||
const execLines = text.split('\n').filter((l) => /^exec /.test(l));
|
||||
assert.equal(execLines.length, 1, `${RED} must hold one exec line; found ${execLines.length}`);
|
||||
assert.equal(execLines[0].split('--other-ids').length - 1, 1, `${RED}: the exec line must pass --other-ids exactly once: ${execLines[0]}`);
|
||||
|
||||
const base = mkdtempSync(join(tmpdir(), 'part4-red-wrap-'));
|
||||
try {
|
||||
const r = runWrapper(stubRoot(base));
|
||||
assert.equal(r.status, 0, `${RED} did not run the shared script:\n${r.out}`);
|
||||
const at = r.argv.map((a, i) => (a === '--other-ids' ? i : -1)).filter((i) => i >= 0);
|
||||
assert.equal(at.length, 1, `${RED} handed the shared script --other-ids ${at.length} times:\n${r.out}`);
|
||||
const other = (r.argv[at[0] + 1] || '').split(/\s+/).filter(Boolean);
|
||||
assert.deepEqual(other, [...idsOf(WRAP_A), ...idsOf(WRAP_B), ...idsOf(WRAP_C), ...idsOf(WRAP_D)],
|
||||
`${RED}: --other-ids is not the IDS= words of ${WRAP_A}, then ${WRAP_B}, then ${WRAP_C}, then ${WRAP_D}\n${r.out}`);
|
||||
const idsAt = r.argv.indexOf('--ids');
|
||||
assert.ok(idsAt >= 0, `${RED} passed no --ids:\n${r.out}`);
|
||||
assert.deepEqual(r.argv[idsAt + 1].split(/\s+/).filter(Boolean), PART4_IDS, `${RED}: --ids is not ${PART4_IDS.join(' ')}\n${r.out}`);
|
||||
} finally {
|
||||
rmSync(base, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('part-4 scripts: the red wrapper exits 2 with a setup failure when an older wrapper has no IDS= line', () => {
|
||||
for (const missing of [WRAP_A, WRAP_B, WRAP_C, WRAP_D]) {
|
||||
const base = mkdtempSync(join(tmpdir(), 'part4-red-noids-'));
|
||||
try {
|
||||
const dir = stubRoot(base);
|
||||
const text = bytes(missing).toString('utf8');
|
||||
writeFileSync(join(dir, missing), text.split('\n').filter((l) => !l.startsWith('IDS=')).join('\n'));
|
||||
const r = runWrapper(dir);
|
||||
assert.equal(r.status, 2, `${RED}: ${missing} without an IDS= line is not exit 2\n${r.out}`);
|
||||
assert.match(r.stderr, /setup failure/, `${RED}: no setup failure line when ${missing} has no IDS= line\n${r.out}`);
|
||||
assert.deepEqual(r.argv, [], `${RED} ran the shared script though ${missing} has no IDS= line\n${r.out}`);
|
||||
} finally {
|
||||
rmSync(base, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
test('part-4 scripts: red-mutate.mjs list prints exactly the three part-4 mutations', () => {
|
||||
assert.ok(existsSync(join(ROOT, MUTATIONS)), `missing file ${MUTATIONS}`);
|
||||
const r = spawnSync(process.execPath, [join(ROOT, 'tests', 'helpers', 'red-mutate.mjs'), 'list', MUTATIONS],
|
||||
{ cwd: ROOT, encoding: 'utf8', timeout: 30000, env: childEnv() });
|
||||
assert.equal(r.status, 0, `red-mutate.mjs list ${MUTATIONS} exited ${r.status}:\n${r.stdout}${r.stderr}`);
|
||||
const lines = (r.stdout || '').split('\n').filter((l) => l !== '');
|
||||
assert.deepEqual(lines, ['CTB-catch CTB', 'NVR-title NVR', 'NVR-e2e NVR'],
|
||||
`red-mutate.mjs list ${MUTATIONS} does not print exactly the three part-4 mutations:\n${r.stdout}${r.stderr}`);
|
||||
});
|
||||
Loading…
Add table
Add a link
Reference in a new issue