refactor(red-proof): the 4.5a script is a wrapper over the shared one; add the 4.5b wrapper
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
2dbfbaf8ca
commit
7566b71674
SSH key fingerprint:
SHA256:JakMjO6FTBBzN0Bhfj9saOoEjaFxlSdYuZQQpM/lF9Q
7 changed files with 388 additions and 154 deletions
40
tests/fix-round-4-5a-mutations.mjs
Normal file
40
tests/fix-round-4-5a-mutations.mjs
Normal file
|
|
@ -0,0 +1,40 @@
|
|||
// tests/fix-round-4-5a-mutations.mjs
|
||||
// The mutations of fix round 4-5a, for the shared red script
|
||||
// (tests/helpers/red-round.sh, through tests/helpers/red-mutate.mjs). DJ's
|
||||
// defect was a missing test: its sentences already stood at ab6b32c, so its
|
||||
// pins are proven red by deleting the copy-through spans. A deletes the span
|
||||
// in commands/trekreview.md, B both spans in agents/review-coordinator.md;
|
||||
// every span must match exactly once. Moved verbatim from the heredoc the
|
||||
// 4-5a red script used to write. Data only; never matched by `npm test`.
|
||||
|
||||
export const MUTATIONS = [
|
||||
{
|
||||
key: 'DJ-A',
|
||||
id: 'DJ',
|
||||
what: 'the copy-through span in commands/trekreview.md deleted',
|
||||
spans: [
|
||||
{
|
||||
file: 'commands/trekreview.md',
|
||||
find: /, and pass each reviewer's\s+`### Declined to Judge` lines verbatim from its saved\s+`\{project_dir\}\/\.review-out-\{reviewer\}\.md`: the coordinator copies them into\s+`## Declined to Judge`, so nothing a reviewer set aside disappears\./g,
|
||||
replace: '.',
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
key: 'DJ-B',
|
||||
id: 'DJ',
|
||||
what: 'both spans in agents/review-coordinator.md deleted',
|
||||
spans: [
|
||||
{
|
||||
file: 'agents/review-coordinator.md',
|
||||
find: /^- \*\*Reviewers' Declined to Judge lines\*\* \u2014 each reviewer's own\s+`### Declined to Judge` list, verbatim\. You copy them into\s+`## Declined to Judge` \(body section 9\); you do not rule on them\.\n/gm,
|
||||
replace: '',
|
||||
},
|
||||
{
|
||||
file: 'agents/review-coordinator.md',
|
||||
find: /, then every\s+line a reviewer wrote under its own `### Declined to Judge`, copied as\s+`\{reviewer\}: \{line\}` \(they are not findings: they stay out of the JSON\s+`declined_to_judge` array and do not change the verdict\)/g,
|
||||
replace: '',
|
||||
},
|
||||
],
|
||||
},
|
||||
];
|
||||
|
|
@ -1,167 +1,40 @@
|
|||
#!/usr/bin/env bash
|
||||
# tests/fix-round-4-5a-red.sh
|
||||
# The red proof of fix round 4-5a (brief: the last Success Criterion). Every
|
||||
# id of the round has tests whose names start with "[id]"; each of them must
|
||||
# fail on an assertion at the round's start point ab6b32c.
|
||||
# 1. Clone the repo into a temp directory outside it (git clone --shared
|
||||
# --no-checkout, then checkout --detach BASE, as
|
||||
# lib/verification/red-proof.mjs does; nothing registers in
|
||||
# .git/worktrees).
|
||||
# 2. Replace the clone's tests/ with the working tree's whole tests/ (tar),
|
||||
# helpers and fixtures included, so a new helper is no crash.
|
||||
# 3. Run the round's eight test files there, only the tagged tests
|
||||
# (--test-name-pattern per id), under tests/helpers/red-reporter.mjs, and
|
||||
# let tests/helpers/red-judge.mjs decide (--pass-ids DJ: DJ's defect is a
|
||||
# missing test, its sentences exist at ab6b32c).
|
||||
# 4. DJ by mutation: A deletes the copy-through span in
|
||||
# commands/trekreview.md, B deletes both spans in
|
||||
# agents/review-coordinator.md; each span must match exactly once. Each
|
||||
# run is judged with --mutation DJ; the file is restored with cp after.
|
||||
# The red proof of fix round 4-5a (brief: the last Success Criterion), as a
|
||||
# wrapper over the shared red script tests/helpers/red-round.sh. Every id of
|
||||
# the round has tests whose names start with "[id]"; each of them must fail on
|
||||
# an assertion at the round's start point ab6b32c, or, for DJ, under its
|
||||
# mutations.
|
||||
# This wrapper passes the shared script:
|
||||
# --base BASE, --ids IDS (the round's 30 ids), the round's FILES;
|
||||
# --other-ids: the ids of fix round 4-5b, read with sed from the IDS= line
|
||||
# of tests/fix-round-4-5b-red.sh (a tag on that list is not judged);
|
||||
# --mutations tests/fix-round-4-5a-mutations.mjs: DJ-A deletes the
|
||||
# copy-through span in commands/trekreview.md, DJ-B both spans in
|
||||
# agents/review-coordinator.md (DJ's defect is a missing test, its
|
||||
# sentences exist at ab6b32c, so DJ is a pass-id at the start point).
|
||||
# The shared script clones the start point outside the repo, puts the working
|
||||
# tree's tests/ in it, runs every tagged test of the files once, lets
|
||||
# tests/helpers/red-judge.mjs decide, then applies each mutation, runs DJ's
|
||||
# tests, judges and restores.
|
||||
# Exit 0 all red, 1 a rule broken (the judge names it), 2 a setup failure
|
||||
# (clone, checkout, node or git missing); never 0 on a setup failure.
|
||||
# (also: no ids on the 4-5b IDS= line); never 0 on a setup failure.
|
||||
# bash 3.2 compatible, ASCII only, no arrays.
|
||||
|
||||
set -u
|
||||
set -f
|
||||
|
||||
cd "$(dirname "$0")/.." || { echo "red script: SETUP FAIL - cannot cd to the repo root" >&2; exit 2; }
|
||||
cd "$(dirname "$0")/.." || { echo "red script: setup failure: cannot cd to the repo root" >&2; exit 2; }
|
||||
ROOT="$(pwd -P)"
|
||||
|
||||
BASE=ab6b32c9304534b77c7ca2b36e7c993682c9efde
|
||||
IDS='#52 #47 #48 #46 #2 #26 #33 #1 #30 #28 R6 D67 #15 #12 #66 #102 #57 #67 #113 #111 #24 #23 F9 F4a F4b F4c WF CL ORD DJ'
|
||||
FILES='tests/lib/fix-round-4-5a-prose.test.mjs tests/lib/verdict-gate.test.mjs tests/lib/park-attempt-files.test.mjs tests/lib/autonomy-gate.test.mjs tests/lib/arg-parser.test.mjs tests/scripts/trekreview-armb-workflow.test.mjs tests/lib/coordinator-contract.test.mjs tests/validators/next-session-prompt-validator.test.mjs'
|
||||
JUDGE="$ROOT/tests/helpers/red-judge.mjs"
|
||||
|
||||
unset GIT_DIR GIT_WORK_TREE GIT_INDEX_FILE NODE_TEST_CONTEXT CLAUDE_CODE_SESSION_ID
|
||||
|
||||
setup_fail() {
|
||||
echo "red script: SETUP FAIL - $1" >&2
|
||||
OTHER=$(sed -n "s/^IDS='\(.*\)'\$/\1/p" "$ROOT/tests/fix-round-4-5b-red.sh")
|
||||
if [ -z "$OTHER" ]; then
|
||||
echo "red script: setup failure: no ids on the IDS= line of $ROOT/tests/fix-round-4-5b-red.sh" >&2
|
||||
exit 2
|
||||
}
|
||||
|
||||
command -v node >/dev/null 2>&1 || setup_fail "node is not on PATH"
|
||||
command -v git >/dev/null 2>&1 || setup_fail "git is not on PATH"
|
||||
command -v tar >/dev/null 2>&1 || setup_fail "tar is not on PATH"
|
||||
[ -f "$JUDGE" ] || setup_fail "missing $JUDGE"
|
||||
|
||||
TMP="$(mktemp -d "${TMPDIR:-/tmp}/fix-round-4-5a-red.XXXXXX")" || setup_fail "mktemp -d failed"
|
||||
case "$TMP" in
|
||||
''|/) setup_fail "mktemp -d gave an unusable directory '$TMP'" ;;
|
||||
esac
|
||||
TMP_REAL="$(cd "$TMP" && pwd -P)" || setup_fail "cannot enter $TMP"
|
||||
case "$TMP_REAL/" in
|
||||
"$ROOT"/*) rmdir "$TMP" 2>/dev/null; setup_fail "the temp directory $TMP_REAL is inside the repo" ;;
|
||||
esac
|
||||
trap 'rm -rf "$TMP"' EXIT
|
||||
trap 'rm -rf "$TMP"; exit 2' INT TERM HUP
|
||||
|
||||
TREE="$TMP/tree"
|
||||
|
||||
# ---- 1. the start point ----
|
||||
|
||||
git clone --quiet --shared --no-checkout "$ROOT" "$TREE" || setup_fail "could not clone $ROOT"
|
||||
git -C "$TREE" checkout --quiet --detach "$BASE" || setup_fail "could not check out $BASE"
|
||||
|
||||
# ---- 2. the working tree's whole tests/ ----
|
||||
|
||||
tar -cf "$TMP/tests.tar" tests || setup_fail "could not tar tests/"
|
||||
rm -rf "$TREE/tests" || setup_fail "could not remove the start point's tests/"
|
||||
tar -xf "$TMP/tests.tar" -C "$TREE" || setup_fail "could not copy tests/ into the start point"
|
||||
|
||||
for f in $FILES; do
|
||||
[ -f "$TREE/$f" ] || setup_fail "missing test file $f"
|
||||
done
|
||||
|
||||
for id in $IDS; do
|
||||
printf '%s\n' "$id"
|
||||
done > "$TMP/ids" || setup_fail "could not write $TMP/ids"
|
||||
|
||||
# run_tagged <results> <file...>: the tagged tests of the files, run in the
|
||||
# start point under the red reporter. The positional parameters carry the
|
||||
# name patterns (no arrays in bash 3.2).
|
||||
run_tagged() {
|
||||
out="$1"
|
||||
shift
|
||||
(cd "$TREE" && node --import ./tests/helpers/no-session-id.mjs --test \
|
||||
--test-reporter=./tests/helpers/red-reporter.mjs \
|
||||
--test-reporter-destination="$out" "$@") >/dev/null 2>"$out.stderr"
|
||||
[ -s "$out" ] || { cat "$out.stderr" >&2; setup_fail "the test run wrote no results to $out"; }
|
||||
}
|
||||
|
||||
# judge <args...>: the red judge; 2 (usage or input) is a setup failure.
|
||||
judge() {
|
||||
node "$JUDGE" "$@"
|
||||
rc=$?
|
||||
[ "$rc" -eq 2 ] && setup_fail "the red judge could not run (exit 2)"
|
||||
return "$rc"
|
||||
}
|
||||
|
||||
STATUS=0
|
||||
|
||||
# ---- 3. every id at the start point ----
|
||||
|
||||
set --
|
||||
for id in $IDS; do
|
||||
set -- "$@" "--test-name-pattern=\\[$id\\]"
|
||||
done
|
||||
for f in $FILES; do
|
||||
set -- "$@" "$f"
|
||||
done
|
||||
|
||||
echo "red script: the tagged tests at $BASE"
|
||||
run_tagged "$TMP/base.jsonl" "$@"
|
||||
judge --ids "$TMP/ids" --results "$TMP/base.jsonl" --pass-ids DJ || STATUS=1
|
||||
|
||||
# ---- 4. DJ under its two mutations ----
|
||||
|
||||
cat > "$TMP/mutate.mjs" <<'EOF'
|
||||
// mutate.mjs <file> <A|B>: delete DJ's copy-through span(s); each span must
|
||||
// match exactly once, else exit 1 and leave the file as it was.
|
||||
import { readFileSync, writeFileSync } from 'node:fs';
|
||||
const [file, which] = process.argv.slice(2);
|
||||
const SPANS = {
|
||||
A: [
|
||||
[/, and pass each reviewer's\s+`### Declined to Judge` lines verbatim from its saved\s+`\{project_dir\}\/\.review-out-\{reviewer\}\.md`: the coordinator copies them into\s+`## Declined to Judge`, so nothing a reviewer set aside disappears\./g, '.'],
|
||||
],
|
||||
B: [
|
||||
[/^- \*\*Reviewers' Declined to Judge lines\*\* \u2014 each reviewer's own\s+`### Declined to Judge` list, verbatim\. You copy them into\s+`## Declined to Judge` \(body section 9\); you do not rule on them\.\n/gm, ''],
|
||||
[/, then every\s+line a reviewer wrote under its own `### Declined to Judge`, copied as\s+`\{reviewer\}: \{line\}` \(they are not findings: they stay out of the JSON\s+`declined_to_judge` array and do not change the verdict\)/g, ''],
|
||||
],
|
||||
};
|
||||
if (!SPANS[which]) { console.error(`mutate: unknown mutation ${which}`); process.exit(1); }
|
||||
let text = readFileSync(file, 'utf8');
|
||||
for (const [re, by] of SPANS[which]) {
|
||||
const n = (text.match(re) || []).length;
|
||||
if (n !== 1) { console.error(`mutate ${which}: ${re.source.slice(0, 60)}... matched ${n} times in ${file}, not once`); process.exit(1); }
|
||||
text = text.replace(re, by);
|
||||
}
|
||||
writeFileSync(file, text);
|
||||
console.log(`mutate ${which}: ${SPANS[which].length} span(s) deleted in ${file}`);
|
||||
EOF
|
||||
[ -s "$TMP/mutate.mjs" ] || setup_fail "could not write $TMP/mutate.mjs"
|
||||
|
||||
# mutation <A|B> <repo-relative file> <what>: apply, run the DJ tests, judge, restore.
|
||||
mutation() {
|
||||
m="$1"
|
||||
rel="$2"
|
||||
echo "red script: DJ mutation $m ($3)"
|
||||
cp "$TREE/$rel" "$TMP/mutation-$m.orig" || setup_fail "could not save $rel"
|
||||
if node "$TMP/mutate.mjs" "$TREE/$rel" "$m"; then
|
||||
run_tagged "$TMP/mutation-$m.jsonl" "--test-name-pattern=\\[DJ\\]" tests/lib/fix-round-4-5a-prose.test.mjs
|
||||
judge --ids "$TMP/ids" --results "$TMP/mutation-$m.jsonl" --mutation DJ || STATUS=1
|
||||
else
|
||||
echo "broken: mutation-not-applied [DJ] $m on $rel"
|
||||
STATUS=1
|
||||
fi
|
||||
cp "$TMP/mutation-$m.orig" "$TREE/$rel" || setup_fail "could not restore $rel"
|
||||
}
|
||||
|
||||
mutation A commands/trekreview.md "the copy-through span in commands/trekreview.md deleted"
|
||||
mutation B agents/review-coordinator.md "both spans in agents/review-coordinator.md deleted"
|
||||
|
||||
if [ "$STATUS" -eq 0 ]; then
|
||||
echo "red script: ok - every id red at $BASE, DJ under both mutations"
|
||||
else
|
||||
echo "red script: FAIL - a rule is broken (see the broken: lines above)" >&2
|
||||
fi
|
||||
exit "$STATUS"
|
||||
|
||||
exec /bin/bash "$ROOT/tests/helpers/red-round.sh" --base "$BASE" --ids "$IDS" --other-ids "$OTHER" --mutations "$ROOT/tests/fix-round-4-5a-mutations.mjs" $FILES
|
||||
|
|
|
|||
76
tests/fix-round-4-5b-mutations.mjs
Normal file
76
tests/fix-round-4-5b-mutations.mjs
Normal file
|
|
@ -0,0 +1,76 @@
|
|||
// tests/fix-round-4-5b-mutations.mjs
|
||||
// The mutations of fix round 4-5b, for the shared red script
|
||||
// (tests/helpers/red-round.sh, through tests/helpers/red-mutate.mjs). An id
|
||||
// named here has tests that pass at the round's start point be0b631 (the
|
||||
// red-proof instruments are this round's own work, and they live in the
|
||||
// working tree's tests/ that the script copies into the clone); each is
|
||||
// proven red under its mutation instead. Every span must match exactly once
|
||||
// in the clone. Data only; never matched by `npm test`. Later steps append
|
||||
// entries.
|
||||
|
||||
export const MUTATIONS = [
|
||||
{
|
||||
key: 'E23a',
|
||||
id: 'E23a',
|
||||
what: 'the base run selects one --test-name-pattern per round id instead of one ^\\[ pattern',
|
||||
spans: [
|
||||
{
|
||||
file: 'tests/helpers/red-round.sh',
|
||||
find: String.raw`run_tagged "$TMP/base.jsonl" --test-timeout=300000 --test-force-exit '--test-name-pattern=^\['`,
|
||||
replace: String.raw`PATS=''
|
||||
for id in $IDS; do
|
||||
PATS="$PATS --test-name-pattern=\\[$id\\]"
|
||||
done
|
||||
run_tagged "$TMP/base.jsonl" --test-timeout=300000 --test-force-exit $PATS`,
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
key: 'E23b',
|
||||
id: 'E23b',
|
||||
what: 'the base run loses its --test-timeout',
|
||||
spans: [
|
||||
{
|
||||
file: 'tests/helpers/red-round.sh',
|
||||
find: 'run_tagged "$TMP/base.jsonl" --test-timeout=300000',
|
||||
replace: 'run_tagged "$TMP/base.jsonl"',
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
key: 'RS',
|
||||
id: 'RS',
|
||||
what: 'the clone checks out HEAD instead of the base',
|
||||
spans: [
|
||||
{
|
||||
file: 'tests/helpers/red-round.sh',
|
||||
find: 'checkout --quiet --detach "$BASE"',
|
||||
replace: 'checkout --quiet --detach HEAD',
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
key: 'E23d',
|
||||
id: 'E23d',
|
||||
what: 'the red judge exits 0 on a usage error',
|
||||
spans: [
|
||||
{
|
||||
file: 'tests/helpers/red-judge.mjs',
|
||||
find: 'process.exit(2);',
|
||||
replace: 'process.exit(0);',
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
key: 'E23f',
|
||||
id: 'E23f',
|
||||
what: 'the red reporter drops the result of a test named [X1] red...',
|
||||
spans: [
|
||||
{
|
||||
file: 'tests/helpers/red-reporter.mjs',
|
||||
find: 'const d = event.data || {};',
|
||||
replace: "const d = event.data || {};\n if (String(d.name).startsWith('[X1] red')) continue;",
|
||||
},
|
||||
],
|
||||
},
|
||||
];
|
||||
40
tests/fix-round-4-5b-red.sh
Normal file
40
tests/fix-round-4-5b-red.sh
Normal file
|
|
@ -0,0 +1,40 @@
|
|||
#!/usr/bin/env bash
|
||||
# tests/fix-round-4-5b-red.sh
|
||||
# The red proof of fix round 4-5b (brief: fixed boundary 4), as a wrapper over
|
||||
# the shared red script tests/helpers/red-round.sh. Every id of the round has
|
||||
# tests whose names start with "[id]"; each of them must fail on an assertion
|
||||
# at the round's start point be0b631, or, for an id a mutation names, under
|
||||
# that mutation.
|
||||
# This wrapper passes the shared script:
|
||||
# --base BASE, --ids IDS (the round's 39 ids), the round's FILES;
|
||||
# --other-ids: the ids of fix round 4-5a, read with sed from the IDS= line
|
||||
# of tests/fix-round-4-5a-red.sh (a tag on that list is not judged);
|
||||
# --mutations tests/fix-round-4-5b-mutations.mjs (its ids are pass-ids at
|
||||
# the start point and are proven red under their mutations);
|
||||
# --require-sha256: the round has SHA-256 tests, so a git that cannot
|
||||
# create a SHA-256 repository is a setup failure, not a silent skip.
|
||||
# The shared script clones the start point outside the repo, puts the working
|
||||
# tree's tests/ in it, runs every tagged test of the files once, lets
|
||||
# tests/helpers/red-judge.mjs decide, then applies each mutation, runs that
|
||||
# id's tests, judges and restores.
|
||||
# Exit 0 all red, 1 a rule broken (the judge names it), 2 a setup failure
|
||||
# (also: no ids on the 4-5a IDS= line); never 0 on a setup failure.
|
||||
# bash 3.2 compatible, ASCII only, no arrays.
|
||||
|
||||
set -u
|
||||
set -f
|
||||
|
||||
cd "$(dirname "$0")/.." || { echo "red script: setup failure: cannot cd to the repo root" >&2; exit 2; }
|
||||
ROOT="$(pwd -P)"
|
||||
|
||||
BASE=be0b631fed21942b6c0fdae570b58e4112f8242d
|
||||
IDS='M6 M11 M4 m4 M3 M7 E22c E22e M5 S3 E20 M9 M2 K3 M1 M1r E23a E23b E23d E23f M8 M10 DJ5 M13 K2 DJ1 DJ2 DJ4 E14 E16a E17a E17b E18a Da Db K4 ARG PTR RS'
|
||||
FILES='tests/lib/verdict-gate.test.mjs tests/lib/done-line.test.mjs tests/lib/park-attempt-files.test.mjs tests/scripts/trekreview-armb-workflow.test.mjs tests/lib/coordinator-contract.test.mjs tests/lib/red-judge.test.mjs tests/lib/red-script.test.mjs tests/lib/review-recurrence.test.mjs tests/lib/arg-parser.test.mjs tests/commands/trekendsession.test.mjs tests/validators/next-session-prompt-validator.test.mjs tests/lib/fix-round-4-5b-prose.test.mjs tests/lib/fix-round-4-5a-prose.test.mjs tests/scripts/release-check.test.mjs'
|
||||
|
||||
OTHER=$(sed -n "s/^IDS='\(.*\)'\$/\1/p" "$ROOT/tests/fix-round-4-5a-red.sh")
|
||||
if [ -z "$OTHER" ]; then
|
||||
echo "red script: setup failure: no ids on the IDS= line of $ROOT/tests/fix-round-4-5a-red.sh" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
exec /bin/bash "$ROOT/tests/helpers/red-round.sh" --base "$BASE" --ids "$IDS" --other-ids "$OTHER" --mutations "$ROOT/tests/fix-round-4-5b-mutations.mjs" --require-sha256 $FILES
|
||||
31
tests/fix-round-4-5b-regression.sh
Normal file
31
tests/fix-round-4-5b-regression.sh
Normal file
|
|
@ -0,0 +1,31 @@
|
|||
#!/usr/bin/env bash
|
||||
# tests/fix-round-4-5b-regression.sh
|
||||
# The Regression: of fix round 4-5b's steps: the whole suite (npm test), the
|
||||
# 4-5a red proof (tests/fix-round-4-5a-red.sh, a wrapper over the shared red
|
||||
# script) and the 4-5a end to end (tests/e2e-fix-round-4-5a.sh, which npm test
|
||||
# only pins statically). All three run; the script exits with the first
|
||||
# non-zero status, or 0 when all three exit 0.
|
||||
# bash 3.2 compatible, ASCII only, no arrays.
|
||||
|
||||
set -u
|
||||
|
||||
cd "$(dirname "$0")/.." || { echo "regression: cannot cd to the repo root" >&2; exit 2; }
|
||||
|
||||
FIRST=0
|
||||
|
||||
npm test
|
||||
rc=$?
|
||||
echo "regression: npm test exited $rc"
|
||||
[ "$FIRST" -eq 0 ] && FIRST=$rc
|
||||
|
||||
/bin/bash tests/fix-round-4-5a-red.sh
|
||||
rc=$?
|
||||
echo "regression: tests/fix-round-4-5a-red.sh exited $rc"
|
||||
[ "$FIRST" -eq 0 ] && FIRST=$rc
|
||||
|
||||
/bin/bash tests/e2e-fix-round-4-5a.sh
|
||||
rc=$?
|
||||
echo "regression: tests/e2e-fix-round-4-5a.sh exited $rc"
|
||||
[ "$FIRST" -eq 0 ] && FIRST=$rc
|
||||
|
||||
exit "$FIRST"
|
||||
|
|
@ -1608,9 +1608,12 @@ test('e2e script: it names each of the four CLIs it runs', () => {
|
|||
// script runs itself as the step's Regression:. Its judge is tested to go red
|
||||
// in tests/lib/red-judge.test.mjs. The 30 ids are the brief's (brief.md:134
|
||||
// and the 4.5a rows), written here as a constant so the list in the script
|
||||
// cannot shrink without this test noticing.
|
||||
// cannot shrink without this test noticing. Since fix round 4-5b the script is
|
||||
// a wrapper over the shared tests/helpers/red-round.sh, which makes and
|
||||
// removes the temp directory, so the trap pin reads that file.
|
||||
|
||||
const RED_SCRIPT = 'tests/fix-round-4-5a-red.sh';
|
||||
const RED_ROUND = 'tests/helpers/red-round.sh';
|
||||
const RED_IDS = ['#52', '#47', '#48', '#46', '#2', '#26', '#33', '#1', '#30', '#28', 'R6', 'D67', '#15', '#12', '#66',
|
||||
'#102', '#57', '#67', '#113', '#111', '#24', '#23', 'F9', 'F4a', 'F4b', 'F4c', 'WF', 'CL', 'ORD', 'DJ'];
|
||||
|
||||
|
|
@ -1628,7 +1631,7 @@ test('red script: tests/fix-round-4-5a-red.sh exists, is ASCII only, passes /bin
|
|||
});
|
||||
|
||||
test('red script: an EXIT trap removes $TMP, and INT, TERM and HUP remove it too', () => {
|
||||
const text = read(RED_SCRIPT);
|
||||
const text = read(RED_ROUND);
|
||||
assert.match(text, /^TMP="\$\(mktemp -d [^\n]*\)"/m, 'the red script does not make $TMP with mktemp -d');
|
||||
assert.match(text, /^trap 'rm -rf "\$TMP"' EXIT$/m, 'no EXIT trap removes $TMP');
|
||||
assert.match(text, /^trap '[^'\n]*rm -rf "\$TMP"[^'\n]*' INT TERM HUP$/m, 'no INT/TERM/HUP trap removes $TMP');
|
||||
|
|
|
|||
|
|
@ -19,6 +19,11 @@
|
|||
// Every test first asserts that the script exists, so each fails on an
|
||||
// assertion while it does not. The script and its runs need Node >= 20.14
|
||||
// (--test-force-exit); on an older Node each test is skipped.
|
||||
//
|
||||
// The two round wrappers (tests/fix-round-4-5a-red.sh, tests/fix-round-4-5b-red.sh)
|
||||
// are run from a scratch copy whose tests/helpers/red-round.sh is a stub that
|
||||
// prints its arguments, so what each wrapper hands the shared script is seen
|
||||
// without a clone. Their mutation modules are loaded with a dynamic import.
|
||||
|
||||
import { after, test } from 'node:test';
|
||||
import { strict as assert } from 'node:assert';
|
||||
|
|
@ -28,7 +33,7 @@ import {
|
|||
} from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { delimiter, dirname, join } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { fileURLToPath, pathToFileURL } from 'node:url';
|
||||
import { childEnv } from '../helpers/child-env.mjs';
|
||||
import { pathWith } from '../helpers/bin-path.mjs';
|
||||
|
||||
|
|
@ -339,3 +344,169 @@ test('[E23b] the base run carries --test-timeout and --test-force-exit', { skip:
|
|||
}
|
||||
assert.match(text, /^#.*Node >= 20\.14/m, 'the header does not say that Node >= 20.14 is needed');
|
||||
});
|
||||
|
||||
// --- the round wrappers ------------------------------------------------------
|
||||
// Expected values from the plan (Step 3) and the 4.5a brief, written out here.
|
||||
|
||||
const WRAP_A = {
|
||||
rel: 'tests/fix-round-4-5a-red.sh',
|
||||
base: 'ab6b32c9304534b77c7ca2b36e7c993682c9efde',
|
||||
mutations: 'tests/fix-round-4-5a-mutations.mjs',
|
||||
files: ['tests/lib/fix-round-4-5a-prose.test.mjs', 'tests/lib/verdict-gate.test.mjs', 'tests/lib/park-attempt-files.test.mjs',
|
||||
'tests/lib/autonomy-gate.test.mjs', 'tests/lib/arg-parser.test.mjs', 'tests/scripts/trekreview-armb-workflow.test.mjs',
|
||||
'tests/lib/coordinator-contract.test.mjs', 'tests/validators/next-session-prompt-validator.test.mjs'],
|
||||
};
|
||||
const WRAP_B = {
|
||||
rel: 'tests/fix-round-4-5b-red.sh',
|
||||
base: 'be0b631fed21942b6c0fdae570b58e4112f8242d',
|
||||
mutations: 'tests/fix-round-4-5b-mutations.mjs',
|
||||
files: ['tests/lib/verdict-gate.test.mjs', 'tests/lib/done-line.test.mjs', 'tests/lib/park-attempt-files.test.mjs',
|
||||
'tests/scripts/trekreview-armb-workflow.test.mjs', 'tests/lib/coordinator-contract.test.mjs', 'tests/lib/red-judge.test.mjs',
|
||||
'tests/lib/red-script.test.mjs', 'tests/lib/review-recurrence.test.mjs', 'tests/lib/arg-parser.test.mjs',
|
||||
'tests/commands/trekendsession.test.mjs', 'tests/validators/next-session-prompt-validator.test.mjs',
|
||||
'tests/lib/fix-round-4-5b-prose.test.mjs', 'tests/lib/fix-round-4-5a-prose.test.mjs', 'tests/scripts/release-check.test.mjs'],
|
||||
};
|
||||
const IDS_B = ['M6', 'M11', 'M4', 'm4', 'M3', 'M7', 'E22c', 'E22e', 'M5', 'S3', 'E20', 'M9', 'M2', 'K3', 'M1', 'M1r', 'E23a',
|
||||
'E23b', 'E23d', 'E23f', 'M8', 'M10', 'DJ5', 'M13', 'K2', 'DJ1', 'DJ2', 'DJ4', 'E14', 'E16a', 'E17a', 'E17b', 'E18a', 'Da',
|
||||
'Db', 'K4', 'ARG', 'PTR', 'RS'];
|
||||
|
||||
/** A wrapper's text; asserts first that it exists. */
|
||||
function wrapperText(w) {
|
||||
const p = join(ROOT, w.rel);
|
||||
assert.ok(existsSync(p), `the wrapper ${w.rel} does not exist`);
|
||||
return readFileSync(p, 'utf8');
|
||||
}
|
||||
|
||||
/** The words of a wrapper's one NAME='...' line. */
|
||||
function quotedList(text, name, rel) {
|
||||
const lines = text.split('\n').filter((l) => l.startsWith(`${name}=`));
|
||||
assert.equal(lines.length, 1, `expected one ${name}= line in ${rel}; found ${lines.length}`);
|
||||
const m = lines[0].match(new RegExp(`^${name}='([^']*)'$`));
|
||||
assert.ok(m, `the ${name}= line of ${rel} is not one single-quoted list: ${lines[0]}`);
|
||||
return m[1].split(/\s+/).filter(Boolean);
|
||||
}
|
||||
|
||||
/** A stand-in for the shared script: one ARG: line per argument, exit $RS_STUB_EXIT (default 0). */
|
||||
const STUB = `#!/bin/bash
|
||||
for a in "$@"; do printf 'ARG:%s\\n' "$a"; done
|
||||
exit "\${RS_STUB_EXIT:-0}"
|
||||
`;
|
||||
|
||||
/** A scratch root holding both wrappers and the stub as tests/helpers/red-round.sh. */
|
||||
function stubRoot() {
|
||||
const a = wrapperText(WRAP_A);
|
||||
const b = wrapperText(WRAP_B);
|
||||
const dir = join(tmp('red-script-wrap-'), 'root');
|
||||
mkdirSync(join(dir, 'tests', 'helpers'), { recursive: true });
|
||||
writeFileSync(join(dir, WRAP_A.rel), a);
|
||||
writeFileSync(join(dir, WRAP_B.rel), b);
|
||||
writeFileSync(join(dir, 'tests', 'helpers', 'red-round.sh'), STUB, { mode: 0o755 });
|
||||
return dir;
|
||||
}
|
||||
|
||||
/** A wrapper run from the scratch root: status, output and the arguments the stub saw. */
|
||||
function runWrapper(dir, w, over = {}) {
|
||||
const r = spawnSync('/bin/bash', [join(dir, w.rel)], { cwd: tmpdir(), encoding: 'utf8', timeout: CHILD_TIMEOUT, env: env(over) });
|
||||
const argv = r.stdout.split('\n').filter((l) => l.startsWith('ARG:')).map((l) => l.slice(4));
|
||||
return { status: r.status, out: `${r.stdout}${r.stderr}`, stderr: r.stderr, argv };
|
||||
}
|
||||
|
||||
/** The stub's arguments as options (each with a value but --require-sha256) and files. */
|
||||
function parseCall(argv) {
|
||||
const call = { flags: [], files: [] };
|
||||
let i = 0;
|
||||
while (i < argv.length && argv[i].startsWith('--')) {
|
||||
if (argv[i] === '--require-sha256') {
|
||||
call.flags.push(argv[i]);
|
||||
i += 1;
|
||||
} else {
|
||||
call[argv[i]] = argv[i + 1];
|
||||
i += 2;
|
||||
}
|
||||
}
|
||||
call.files = argv.slice(i);
|
||||
return call;
|
||||
}
|
||||
|
||||
test('[RS] both wrappers exec the shared script with their own BASE, IDS and FILES', () => {
|
||||
const dir = stubRoot();
|
||||
for (const w of [WRAP_A, WRAP_B]) {
|
||||
const text = wrapperText(w);
|
||||
assert.deepEqual(text.split('\n').filter((l) => /^BASE=/.test(l)), [`BASE=${w.base}`], `${w.rel}: BASE is not the one full ${w.base.slice(0, 7)} sha`);
|
||||
assert.deepEqual(quotedList(text, 'FILES', w.rel), w.files, `${w.rel}: the FILES= line is not the round's files`);
|
||||
const ids = quotedList(text, 'IDS', w.rel);
|
||||
const r = runWrapper(dir, w);
|
||||
assert.equal(r.status, 0, `${w.rel} did not run the shared script:\n${r.out}`);
|
||||
assert.ok(r.argv.length > 0, `${w.rel} handed the shared script no arguments:\n${r.out}`);
|
||||
const call = parseCall(r.argv);
|
||||
assert.equal(call['--base'], w.base, `${w.rel}: --base\n${r.out}`);
|
||||
assert.deepEqual((call['--ids'] || '').split(/\s+/).filter(Boolean), ids, `${w.rel}: --ids is not its IDS= line\n${r.out}`);
|
||||
assert.equal(call['--mutations'], join(dir, w.mutations), `${w.rel}: --mutations is not its own module by absolute path\n${r.out}`);
|
||||
assert.deepEqual(call.files, w.files, `${w.rel}: the test files are not its FILES= line\n${r.out}`);
|
||||
const failed = runWrapper(dir, w, { RS_STUB_EXIT: '1' });
|
||||
assert.equal(failed.status, 1, `${w.rel} does not end with the shared script's exit status:\n${failed.out}`);
|
||||
}
|
||||
});
|
||||
|
||||
test('[RS] each wrapper reads the other round\'s ids from its IDS line, no copied list', () => {
|
||||
const dir = stubRoot();
|
||||
for (const [w, other] of [[WRAP_A, WRAP_B], [WRAP_B, WRAP_A]]) {
|
||||
const otherText = wrapperText(other);
|
||||
const otherIds = quotedList(otherText, 'IDS', other.rel);
|
||||
const r = runWrapper(dir, w);
|
||||
assert.equal(r.status, 0, r.out);
|
||||
assert.deepEqual((parseCall(r.argv)['--other-ids'] || '').split(/\s+/).filter(Boolean), otherIds,
|
||||
`${w.rel}: --other-ids is not the IDS= line of ${other.rel}\n${r.out}`);
|
||||
|
||||
const otherLine = otherText.split('\n').find((l) => l.startsWith('IDS='));
|
||||
writeFileSync(join(dir, other.rel), otherText.replace(otherLine, "IDS='P1 P2'"));
|
||||
const planted = runWrapper(dir, w);
|
||||
assert.equal(planted.status, 0, planted.out);
|
||||
assert.equal(parseCall(planted.argv)['--other-ids'], 'P1 P2', `${w.rel} does not read ${other.rel}'s IDS= line when it runs\n${planted.out}`);
|
||||
|
||||
writeFileSync(join(dir, other.rel), otherText.replace(otherLine, "IDS=''"));
|
||||
const empty = runWrapper(dir, w);
|
||||
assert.equal(empty.status, 2, `${w.rel}: an empty IDS= line in ${other.rel} is not a setup failure\n${empty.out}`);
|
||||
assert.deepEqual(empty.argv, [], `${w.rel} ran the shared script with no other-round ids\n${empty.out}`);
|
||||
assert.match(empty.stderr, /setup failure/, empty.out);
|
||||
writeFileSync(join(dir, other.rel), otherText);
|
||||
|
||||
assert.ok(!wrapperText(w).includes(otherIds.join(' ')), `${w.rel} holds a copy of ${other.rel}'s id list`);
|
||||
}
|
||||
});
|
||||
|
||||
test('[RS] the 4.5b IDS line holds exactly the 39 ids', () => {
|
||||
const ids = quotedList(wrapperText(WRAP_B), 'IDS', WRAP_B.rel);
|
||||
assert.equal(IDS_B.length, 39, 'control: the expected list is not 39 ids');
|
||||
assert.deepEqual(ids, IDS_B);
|
||||
assert.equal(new Set(ids).size, ids.length, 'an id is on the 4.5b IDS= line twice');
|
||||
});
|
||||
|
||||
test('[RS] only the 4.5b wrapper requires SHA-256', () => {
|
||||
const dir = stubRoot();
|
||||
const a = runWrapper(dir, WRAP_A);
|
||||
const b = runWrapper(dir, WRAP_B);
|
||||
assert.equal(a.status, 0, a.out);
|
||||
assert.equal(b.status, 0, b.out);
|
||||
assert.deepEqual(parseCall(a.argv).flags, [], `${WRAP_A.rel} passes a flag it should not:\n${a.out}`);
|
||||
assert.deepEqual(parseCall(b.argv).flags, ['--require-sha256'], `${WRAP_B.rel} does not pass --require-sha256:\n${b.out}`);
|
||||
});
|
||||
|
||||
test('[RS] every mutation module loads, keys are unique and every id is on its wrapper\'s IDS line', async () => {
|
||||
const expectKeys = { [WRAP_A.rel]: ['DJ-A', 'DJ-B'], [WRAP_B.rel]: ['E23a', 'E23b', 'RS', 'E23d', 'E23f'] };
|
||||
for (const w of [WRAP_A, WRAP_B]) {
|
||||
const ids = quotedList(wrapperText(w), 'IDS', w.rel);
|
||||
const p = join(ROOT, w.mutations);
|
||||
assert.ok(existsSync(p), `the mutation module ${w.mutations} does not exist`);
|
||||
const mod = await import(pathToFileURL(p).href);
|
||||
assert.ok(Array.isArray(mod.MUTATIONS) && mod.MUTATIONS.length > 0, `${w.mutations} exports no non-empty MUTATIONS array`);
|
||||
const keys = mod.MUTATIONS.map((m) => m.key);
|
||||
assert.equal(new Set(keys).size, keys.length, `${w.mutations}: a key is used twice: ${keys.join(' ')}`);
|
||||
for (const k of expectKeys[w.rel]) assert.ok(keys.includes(k), `${w.mutations} has no entry ${k}`);
|
||||
for (const m of mod.MUTATIONS) {
|
||||
assert.ok(ids.includes(m.id), `${w.mutations}: ${m.key} names id ${m.id}, which is not on the IDS= line of ${w.rel}`);
|
||||
assert.ok(Array.isArray(m.spans) && m.spans.length > 0, `${w.mutations}: ${m.key} has no spans`);
|
||||
for (const s of m.spans) assert.ok(typeof s.file === 'string' && s.file !== '' && !s.file.startsWith('/'), `${w.mutations}: ${m.key} has a span file that is not a relative path`);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue