open/voyage
1
0
Fork 0
Code Issues Releases 13 Activity

test(red-proof): fix round 4.5b2 part A red proof and regression run

Browse source 
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Kjell Tore Guttormsen 2026-11-02 10:34:40 +01:00
commit 63fd33daaf
Signed by: ktg
SSH key fingerprint: SHA256:JakMjO6FTBBzN0Bhfj9saOoEjaFxlSdYuZQQpM/lF9Q
4 changed files with 364 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

62
tests/fix-round-4-5b2-a-mutations.mjs Normal file
View file

@ -0,0 +1,62 @@
// tests/fix-round-4-5b2-a-mutations.mjs
// The mutations of fix round 4.5b2, part A (plan
// .claude/projects/2026-10-31-guard-4-5b2-part-a, Step 5, 5a; decisions 91,
// 107 and 123), for the shared red script (tests/helpers/red-round.sh,
// through tests/helpers/red-mutate.mjs).
// An id named here has tests that pass at the round's start point e2d6bec and
// is proven red under its mutations instead:
// - RXT: the try already holds the regex reading's escape limit at e2d6bec
// (the round adds only its test, which forces an error there through a
// preload); the mutation opens the try after the regex reading, in
// e2d6bec's hook, so that error escapes it.
// - CNT: the first-<< search already takes a line continuation between the
// two < at e2d6bec; the mutation drops that branch, in e2d6bec's hook.
// - HDR: the false-stops header is corrected in the working tree's tests/
// that the script copies into the clone; the mutation puts the wrong red
// base back.
// Every span must match exactly once in the clone (e2d6bec's code with the
// working tree's tests/). Data only; never matched by `npm test`.
export const MUTATIONS = [
{
key: 'RXT-try',
id: 'RXT',
what: 'the try opens after the regex reading',
spans: [
{
file: 'hooks/scripts/pre-bash-executor.mjs',
find: ' let argsVerdict = null;\n try {\n',
replace: ' let argsVerdict = null;\n',
},
{
file: 'hooks/scripts/pre-bash-executor.mjs',
find: ' let bodyFrom = Infinity;\n',
replace: ' try {\n let bodyFrom = Infinity;\n',
},
],
},
{
key: 'CNT-regex',
id: 'CNT',
what: 'the first-<< search loses its continuation branch',
spans: [
{
file: 'hooks/scripts/pre-bash-executor.mjs',
find: 'const h = /<(?:\\\\\\n)*</.exec(text);',
replace: 'const h = /<</.exec(text);',
},
],
},
{
key: 'HDR-header',
id: 'HDR',
what: 'the false-stops header names the wrong red base again',
spans: [
{
file: 'tests/hooks/bash-guard-false-stops.test.mjs',
find: 'loads at 591f3c4',
replace: 'loads at 12bca8b',
},
],
},
];

68
tests/fix-round-4-5b2-a-red.sh Normal file
View file

@ -0,0 +1,68 @@
#!/usr/bin/env bash
# tests/fix-round-4-5b2-a-red.sh
# The red proof of fix round 4.5b2, part A (plan
# .claude/projects/2026-10-31-guard-4-5b2-part-a/plan.md, Step 5, 5b;
# decisions 91, 107 and 123), as a wrapper over the shared red script
# tests/helpers/red-round.sh. Every id of the round has tests whose names
# start with "[id]"; SFX, ESR, EVL, KIA and SIS must each fail on an assertion
# at the round's start point e2d6bec, and RXT, CNT and HDR, whose tests pass
# there, are proven red under their mutations.
# This wrapper passes the shared script:
# --base BASE, the full sha of e2d6bec;
# --ids IDS (the round's 8 ids);
# --other-ids: the ids of fix round 4-5a, of 4-5b part 1, of part 2, of
# part 3 and of part 4, read with sed from the IDS= lines of
# tests/fix-round-4-5a-red.sh, tests/fix-round-4-5b-red.sh,
# tests/fix-round-4-5b-part-2-red.sh, tests/fix-round-4-5b-part-3-red.sh
# and tests/fix-round-4-5b-part-4-red.sh and passed as ONE value, in that
# order (the shared script keeps only the last --other-ids it is given);
# a tag on that list is not judged;
# --mutations tests/fix-round-4-5b2-a-mutations.mjs (RXT-try, CNT-regex,
# HDR-header);
# the round's FILES: the two tagged test files. The keep file
# (tests/hooks/bash-guard-4-5b2-a-known.test.mjs) holds at e2d6bec by
# design and its tag is no red-proof id; the corpus test and the scripts
# test are untagged; none of the three is listed.
# No --require-sha256: the round has no SHA-256 test.
# Exit 0 all red, 1 a rule broken (the judge names it), 2 a setup failure
# (also: no ids on one of the five older IDS= lines); never 0 on a setup
# failure.
# bash 3.2 compatible, ASCII only, no arrays.
set -u
set -f
cd "$(dirname "$0")/.." || { echo "red script: setup failure: cannot cd to the repo root" >&2; exit 2; }
ROOT="$(pwd -P)"
BASE=e2d6beccb7ea38cb8d596b6a35bcad0232ae3ca1
IDS='RXT CNT SFX ESR EVL KIA HDR SIS'
FILES='tests/hooks/bash-guard-4-5b2-a.test.mjs tests/lib/fix-round-4-5b2-a-prose.test.mjs'
OTHER_A=$(sed -n "s/^IDS='\(.*\)'\$/\1/p" "$ROOT/tests/fix-round-4-5a-red.sh")
if [ -z "$OTHER_A" ]; then
echo "red script: setup failure: no ids on the IDS= line of $ROOT/tests/fix-round-4-5a-red.sh" >&2
exit 2
fi
OTHER_B=$(sed -n "s/^IDS='\(.*\)'\$/\1/p" "$ROOT/tests/fix-round-4-5b-red.sh")
if [ -z "$OTHER_B" ]; then
echo "red script: setup failure: no ids on the IDS= line of $ROOT/tests/fix-round-4-5b-red.sh" >&2
exit 2
fi
OTHER_C=$(sed -n "s/^IDS='\(.*\)'\$/\1/p" "$ROOT/tests/fix-round-4-5b-part-2-red.sh")
if [ -z "$OTHER_C" ]; then
echo "red script: setup failure: no ids on the IDS= line of $ROOT/tests/fix-round-4-5b-part-2-red.sh" >&2
exit 2
fi
OTHER_D=$(sed -n "s/^IDS='\(.*\)'\$/\1/p" "$ROOT/tests/fix-round-4-5b-part-3-red.sh")
if [ -z "$OTHER_D" ]; then
echo "red script: setup failure: no ids on the IDS= line of $ROOT/tests/fix-round-4-5b-part-3-red.sh" >&2
exit 2
fi
OTHER_E=$(sed -n "s/^IDS='\(.*\)'\$/\1/p" "$ROOT/tests/fix-round-4-5b-part-4-red.sh")
if [ -z "$OTHER_E" ]; then
echo "red script: setup failure: no ids on the IDS= line of $ROOT/tests/fix-round-4-5b-part-4-red.sh" >&2
exit 2
fi
exec /bin/bash "$ROOT/tests/helpers/red-round.sh" --base "$BASE" --ids "$IDS" --other-ids "$OTHER_A $OTHER_B $OTHER_C $OTHER_D $OTHER_E" --mutations "$ROOT/tests/fix-round-4-5b2-a-mutations.mjs" $FILES

72
tests/fix-round-4-5b2-a-regression.sh Normal file
View file

@ -0,0 +1,72 @@
#!/usr/bin/env bash
# tests/fix-round-4-5b2-a-regression.sh
# The Regression: of fix round 4.5b2, part A (plan
# .claude/projects/2026-10-31-guard-4-5b2-part-a/plan.md, Step 5, 5c;
# decisions 91, 107 and 123): the whole suite (npm test, which runs the keep
# file and the corpus test too), the round's bypass corpus
# (tests/guard-bypass-corpus-4-5b2-a.sh), the round's end to end
# (tests/e2e-fix-round-4-5b2-a.sh), part 4's, part 3's, part 2's and part 1's
# end to end (tests/e2e-fix-round-4-5b-part-4.sh,
# tests/e2e-fix-round-4-5b-part-3.sh, tests/e2e-fix-round-4-5b-part-2.sh,
# tests/e2e-fix-round-4-5b.sh), the round's red proof
# (tests/fix-round-4-5b2-a-red.sh) and part 4's red proof
# (tests/fix-round-4-5b-part-4-red.sh, which lists the false-stops file this
# round edited). The 4-5a, part-1, part-2 and part-3 red proofs are not run
# here: one Regression command has the criteria runner's 15-minute cap; they
# run as their own Verification lines.
# All nine run, in that order; the script prints each exit and exits with the
# first non-zero status, or 0 when all nine exit 0.
# bash 3.2 compatible, ASCII only, no arrays.
set -u
cd "$(dirname "$0")/.." || { echo "regression: cannot cd to the repo root" >&2; exit 2; }
FIRST=0
npm test
rc=$?
echo "regression: npm test exited $rc"
[ "$FIRST" -eq 0 ] && FIRST=$rc
/bin/bash tests/guard-bypass-corpus-4-5b2-a.sh
rc=$?
echo "regression: tests/guard-bypass-corpus-4-5b2-a.sh exited $rc"
[ "$FIRST" -eq 0 ] && FIRST=$rc
/bin/bash tests/e2e-fix-round-4-5b2-a.sh
rc=$?
echo "regression: tests/e2e-fix-round-4-5b2-a.sh exited $rc"
[ "$FIRST" -eq 0 ] && FIRST=$rc
/bin/bash tests/e2e-fix-round-4-5b-part-4.sh
rc=$?
echo "regression: tests/e2e-fix-round-4-5b-part-4.sh exited $rc"
[ "$FIRST" -eq 0 ] && FIRST=$rc
/bin/bash tests/e2e-fix-round-4-5b-part-3.sh
rc=$?
echo "regression: tests/e2e-fix-round-4-5b-part-3.sh exited $rc"
[ "$FIRST" -eq 0 ] && FIRST=$rc
/bin/bash tests/e2e-fix-round-4-5b-part-2.sh
rc=$?
echo "regression: tests/e2e-fix-round-4-5b-part-2.sh exited $rc"
[ "$FIRST" -eq 0 ] && FIRST=$rc
/bin/bash tests/e2e-fix-round-4-5b.sh
rc=$?
echo "regression: tests/e2e-fix-round-4-5b.sh exited $rc"
[ "$FIRST" -eq 0 ] && FIRST=$rc
/bin/bash tests/fix-round-4-5b2-a-red.sh
rc=$?
echo "regression: tests/fix-round-4-5b2-a-red.sh exited $rc"
[ "$FIRST" -eq 0 ] && FIRST=$rc
/bin/bash tests/fix-round-4-5b-part-4-red.sh
rc=$?
echo "regression: tests/fix-round-4-5b-part-4-red.sh exited $rc"
[ "$FIRST" -eq 0 ] && FIRST=$rc
exit "$FIRST"

162
tests/lib/fix-round-4-5b2-a-scripts.test.mjs Normal file
View file

@ -0,0 +1,162 @@
// tests/lib/fix-round-4-5b2-a-scripts.test.mjs
// The form of fix round 4.5b2 part A's shell scripts and mutation module (plan
// .claude/projects/2026-10-31-guard-4-5b2-part-a, Step 5, 5d): the red
// wrapper, the regression script, the bypass corpus, the end-to-end run and
// the mutations.
// Untagged on purpose and listed in no red wrapper's FILES: these checks are
// not a part-A fix.
// - each script exists, is ASCII only, passes /bin/bash -n and uses no
// bash 4 form;
// - the red wrapper hands the shared red script exactly one --other-ids,
// whose words are the 4-5a IDS= words, then part 1's, part 2's, part 3's
// and part 4's, --ids RXT CNT SFX ESR EVL KIA HDR SIS, exactly the two
// tagged test files, and no argument naming the keep file or its tag (run
// from a temp root next to a stub red-round.sh that prints its arguments,
// the method of tests/lib/red-script.test.mjs);
// - with the 4-5a, the part-1, the part-2, the part-3 or the part-4 wrapper
// lacking its IDS= line, the red wrapper exits 2 with a setup failure and
// never runs the shared script;
// - red-mutate.mjs list prints exactly the three part-A mutations.
import { test } from 'node:test';
import { strict as assert } from 'node:assert';
import { spawnSync } from 'node:child_process';
import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { dirname, join } from 'node:path';
import { fileURLToPath } from 'node:url';
import { childEnv } from '../helpers/child-env.mjs';
const ROOT = join(dirname(fileURLToPath(import.meta.url)), '..', '..');
const RED = 'tests/fix-round-4-5b2-a-red.sh';
const REGRESSION = 'tests/fix-round-4-5b2-a-regression.sh';
const CORPUS = 'tests/guard-bypass-corpus-4-5b2-a.sh';
const E2E = 'tests/e2e-fix-round-4-5b2-a.sh';
const MUTATIONS = 'tests/fix-round-4-5b2-a-mutations.mjs';
const WRAP_A = 'tests/fix-round-4-5a-red.sh';
const WRAP_B = 'tests/fix-round-4-5b-red.sh';
const WRAP_C = 'tests/fix-round-4-5b-part-2-red.sh';
const WRAP_D = 'tests/fix-round-4-5b-part-3-red.sh';
const WRAP_E = 'tests/fix-round-4-5b-part-4-red.sh';
const OLDER = [WRAP_A, WRAP_B, WRAP_C, WRAP_D, WRAP_E];
const PARTA_IDS = ['RXT', 'CNT', 'SFX', 'ESR', 'EVL', 'KIA', 'HDR', 'SIS'];
const PARTA_FILES = ['tests/hooks/bash-guard-4-5b2-a.test.mjs', 'tests/lib/fix-round-4-5b2-a-prose.test.mjs'];
/** The shared script's options that take a value; every other argument after them is a test file. */
const VALUE_OPTIONS = ['--base', '--ids', '--other-ids', '--mutations'];
/** A repo file's bytes; the file must exist. */
function bytes(rel) {
assert.ok(existsSync(join(ROOT, rel)), `missing file ${rel}`);
return readFileSync(join(ROOT, rel));
}
/** The words of a wrapper's one IDS='...' line. */
function idsOf(rel) {
const lines = bytes(rel).toString('utf8').split('\n').filter((l) => l.startsWith('IDS='));
assert.equal(lines.length, 1, `expected one IDS= line in ${rel}; found ${lines.length}`);
const m = lines[0].match(/^IDS='([^']*)'$/);
assert.ok(m, `the IDS= line of ${rel} is not one single-quoted list: ${lines[0]}`);
return m[1].split(/\s+/).filter(Boolean);
}
test('part-A scripts: each exists, is ASCII only, passes /bin/bash -n and uses no bash 4 form', () => {
for (const rel of [RED, REGRESSION, CORPUS, E2E]) {
const buf = bytes(rel);
assert.equal(buf.findIndex((b) => b > 127), -1, `${rel} is not ASCII only`);
const r = spawnSync('/bin/bash', ['-n', join(ROOT, rel)], { encoding: 'utf8', env: childEnv() });
assert.equal(r.status, 0, `${rel} does not pass /bin/bash -n: ${r.stderr}`);
const text = buf.toString('utf8');
for (const bash4 of [/\bdeclare -A\b/, /\bmapfile\b/, /\breadarray\b/, /\|&/]) {
assert.doesNotMatch(text, bash4, `${rel} uses ${bash4}, which bash 3.2 lacks`);
}
}
});
/** A stand-in for the shared script: one ARG: line per argument, exit 0. */
const STUB = `#!/bin/bash
for a in "$@"; do printf 'ARG:%s\\n' "$a"; done
exit 0
`;
/** A temp root with the part-A wrapper, the five older wrappers and the stub as tests/helpers/red-round.sh. */
function stubRoot(base) {
const dir = join(base, 'root');
mkdirSync(join(dir, 'tests', 'helpers'), { recursive: true });
for (const rel of [RED, ...OLDER]) writeFileSync(join(dir, rel), bytes(rel));
writeFileSync(join(dir, 'tests', 'helpers', 'red-round.sh'), STUB, { mode: 0o755 });
return dir;
}
/** The part-A wrapper run from a temp root: status, output and the arguments the stub saw. */
function runWrapper(dir) {
const r = spawnSync('/bin/bash', [join(dir, RED)], { cwd: tmpdir(), encoding: 'utf8', timeout: 30000, env: childEnv() });
const argv = (r.stdout || '').split('\n').filter((l) => l.startsWith('ARG:')).map((l) => l.slice(4));
return { status: r.status, out: `${r.stdout}${r.stderr}`, stderr: r.stderr || '', argv };
}
/** The arguments that are neither an option nor an option's value: the test files. */
function filesOf(argv) {
const files = [];
for (let i = 0; i < argv.length; i += 1) {
if (VALUE_OPTIONS.includes(argv[i])) { i += 1; continue; }
if (argv[i].startsWith('--')) continue;
files.push(argv[i]);
}
return files;
}
test('part-A scripts: the red wrapper passes one --other-ids with the five older rounds\' ids in order, its own eight ids and its two tagged files', () => {
const text = bytes(RED).toString('utf8');
const execLines = text.split('\n').filter((l) => /^exec /.test(l));
assert.equal(execLines.length, 1, `${RED} must hold one exec line; found ${execLines.length}`);
assert.equal(execLines[0].split('--other-ids').length - 1, 1, `${RED}: the exec line must pass --other-ids exactly once: ${execLines[0]}`);
const base = mkdtempSync(join(tmpdir(), 'partA-red-wrap-'));
try {
const r = runWrapper(stubRoot(base));
assert.equal(r.status, 0, `${RED} did not run the shared script:\n${r.out}`);
const at = r.argv.map((a, i) => (a === '--other-ids' ? i : -1)).filter((i) => i >= 0);
assert.equal(at.length, 1, `${RED} handed the shared script --other-ids ${at.length} times:\n${r.out}`);
const other = (r.argv[at[0] + 1] || '').split(/\s+/).filter(Boolean);
assert.deepEqual(other, OLDER.flatMap((rel) => idsOf(rel)),
`${RED}: --other-ids is not the IDS= words of ${OLDER.join(', then ')}\n${r.out}`);
const idsAt = r.argv.indexOf('--ids');
assert.ok(idsAt >= 0, `${RED} passed no --ids:\n${r.out}`);
assert.deepEqual(r.argv[idsAt + 1].split(/\s+/).filter(Boolean), PARTA_IDS, `${RED}: --ids is not ${PARTA_IDS.join(' ')}\n${r.out}`);
assert.deepEqual(filesOf(r.argv), PARTA_FILES, `${RED}: the test files are not exactly ${PARTA_FILES.join(' and ')}\n${r.out}`);
for (const a of r.argv) {
assert.ok(!a.includes('-known') && !a.includes('KNA'), `${RED} passed an argument naming the keep file or its tag: ${a}\n${r.out}`);
}
} finally {
rmSync(base, { recursive: true, force: true });
}
});
test('part-A scripts: the red wrapper exits 2 with a setup failure when an older wrapper has no IDS= line', () => {
for (const missing of OLDER) {
const base = mkdtempSync(join(tmpdir(), 'partA-red-noids-'));
try {
const dir = stubRoot(base);
const text = bytes(missing).toString('utf8');
writeFileSync(join(dir, missing), text.split('\n').filter((l) => !l.startsWith('IDS=')).join('\n'));
const r = runWrapper(dir);
assert.equal(r.status, 2, `${RED}: ${missing} without an IDS= line is not exit 2\n${r.out}`);
assert.match(r.stderr, /setup failure/, `${RED}: no setup failure line when ${missing} has no IDS= line\n${r.out}`);
assert.ok(r.stderr.includes(missing), `${RED}: the setup failure line does not name ${missing}\n${r.out}`);
assert.deepEqual(r.argv, [], `${RED} ran the shared script though ${missing} has no IDS= line\n${r.out}`);
} finally {
rmSync(base, { recursive: true, force: true });
}
}
});
test('part-A scripts: red-mutate.mjs list prints exactly the three part-A mutations', () => {
assert.ok(existsSync(join(ROOT, MUTATIONS)), `missing file ${MUTATIONS}`);
const r = spawnSync(process.execPath, [join(ROOT, 'tests', 'helpers', 'red-mutate.mjs'), 'list', MUTATIONS],
{ cwd: ROOT, encoding: 'utf8', timeout: 30000, env: childEnv() });
assert.equal(r.status, 0, `red-mutate.mjs list ${MUTATIONS} exited ${r.status}:\n${r.stdout}${r.stderr}`);
const lines = (r.stdout || '').split('\n').filter((l) => l !== '');
assert.deepEqual(lines, ['RXT-try RXT', 'CNT-regex CNT', 'HDR-header HDR'],
`red-mutate.mjs list ${MUTATIONS} does not print exactly the three part-A mutations:\n${r.stdout}${r.stderr}`);
});