#!/usr/bin/env bash # tests/fix-round-4-5a-red.sh # The red proof of fix round 4-5a (brief: the last Success Criterion). Every # id of the round has tests whose names start with "[id]"; each of them must # fail on an assertion at the round's start point ab6b32c. # 1. Clone the repo into a temp directory outside it (git clone --shared # --no-checkout, then checkout --detach BASE, as # lib/verification/red-proof.mjs does; nothing registers in # .git/worktrees). # 2. Replace the clone's tests/ with the working tree's whole tests/ (tar), # helpers and fixtures included, so a new helper is no crash. # 3. Run the round's eight test files there, only the tagged tests # (--test-name-pattern per id), under tests/helpers/red-reporter.mjs, and # let tests/helpers/red-judge.mjs decide (--pass-ids DJ: DJ's defect is a # missing test, its sentences exist at ab6b32c). # 4. DJ by mutation: A deletes the copy-through span in # commands/trekreview.md, B deletes both spans in # agents/review-coordinator.md; each span must match exactly once. Each # run is judged with --mutation DJ; the file is restored with cp after. # Exit 0 all red, 1 a rule broken (the judge names it), 2 a setup failure # (clone, checkout, node or git missing); never 0 on a setup failure. # bash 3.2 compatible, ASCII only, no arrays. set -u set -f cd "$(dirname "$0")/.." || { echo "red script: SETUP FAIL - cannot cd to the repo root" >&2; exit 2; } ROOT="$(pwd -P)" BASE=ab6b32c9304534b77c7ca2b36e7c993682c9efde IDS='#52 #47 #48 #46 #2 #26 #33 #1 #30 #28 R6 D67 #15 #12 #66 #102 #57 #67 #113 #111 #24 #23 F9 F4a F4b F4c WF CL ORD DJ' FILES='tests/lib/fix-round-4-5a-prose.test.mjs tests/lib/verdict-gate.test.mjs tests/lib/park-attempt-files.test.mjs tests/lib/autonomy-gate.test.mjs tests/lib/arg-parser.test.mjs tests/scripts/trekreview-armb-workflow.test.mjs tests/lib/coordinator-contract.test.mjs tests/validators/next-session-prompt-validator.test.mjs' JUDGE="$ROOT/tests/helpers/red-judge.mjs" unset GIT_DIR GIT_WORK_TREE GIT_INDEX_FILE NODE_TEST_CONTEXT CLAUDE_CODE_SESSION_ID setup_fail() { echo "red script: SETUP FAIL - $1" >&2 exit 2 } command -v node >/dev/null 2>&1 || setup_fail "node is not on PATH" command -v git >/dev/null 2>&1 || setup_fail "git is not on PATH" command -v tar >/dev/null 2>&1 || setup_fail "tar is not on PATH" [ -f "$JUDGE" ] || setup_fail "missing $JUDGE" TMP="$(mktemp -d "${TMPDIR:-/tmp}/fix-round-4-5a-red.XXXXXX")" || setup_fail "mktemp -d failed" case "$TMP" in ''|/) setup_fail "mktemp -d gave an unusable directory '$TMP'" ;; esac TMP_REAL="$(cd "$TMP" && pwd -P)" || setup_fail "cannot enter $TMP" case "$TMP_REAL/" in "$ROOT"/*) rmdir "$TMP" 2>/dev/null; setup_fail "the temp directory $TMP_REAL is inside the repo" ;; esac trap 'rm -rf "$TMP"' EXIT trap 'rm -rf "$TMP"; exit 2' INT TERM HUP TREE="$TMP/tree" # ---- 1. the start point ---- git clone --quiet --shared --no-checkout "$ROOT" "$TREE" || setup_fail "could not clone $ROOT" git -C "$TREE" checkout --quiet --detach "$BASE" || setup_fail "could not check out $BASE" # ---- 2. the working tree's whole tests/ ---- tar -cf "$TMP/tests.tar" tests || setup_fail "could not tar tests/" rm -rf "$TREE/tests" || setup_fail "could not remove the start point's tests/" tar -xf "$TMP/tests.tar" -C "$TREE" || setup_fail "could not copy tests/ into the start point" for f in $FILES; do [ -f "$TREE/$f" ] || setup_fail "missing test file $f" done for id in $IDS; do printf '%s\n' "$id" done > "$TMP/ids" || setup_fail "could not write $TMP/ids" # run_tagged : the tagged tests of the files, run in the # start point under the red reporter. The positional parameters carry the # name patterns (no arrays in bash 3.2). run_tagged() { out="$1" shift (cd "$TREE" && node --import ./tests/helpers/no-session-id.mjs --test \ --test-reporter=./tests/helpers/red-reporter.mjs \ --test-reporter-destination="$out" "$@") >/dev/null 2>"$out.stderr" [ -s "$out" ] || { cat "$out.stderr" >&2; setup_fail "the test run wrote no results to $out"; } } # judge : the red judge; 2 (usage or input) is a setup failure. judge() { node "$JUDGE" "$@" rc=$? [ "$rc" -eq 2 ] && setup_fail "the red judge could not run (exit 2)" return "$rc" } STATUS=0 # ---- 3. every id at the start point ---- set -- for id in $IDS; do set -- "$@" "--test-name-pattern=\\[$id\\]" done for f in $FILES; do set -- "$@" "$f" done echo "red script: the tagged tests at $BASE" run_tagged "$TMP/base.jsonl" "$@" judge --ids "$TMP/ids" --results "$TMP/base.jsonl" --pass-ids DJ || STATUS=1 # ---- 4. DJ under its two mutations ---- cat > "$TMP/mutate.mjs" <<'EOF' // mutate.mjs : delete DJ's copy-through span(s); each span must // match exactly once, else exit 1 and leave the file as it was. import { readFileSync, writeFileSync } from 'node:fs'; const [file, which] = process.argv.slice(2); const SPANS = { A: [ [/, and pass each reviewer's\s+`### Declined to Judge` lines verbatim from its saved\s+`\{project_dir\}\/\.review-out-\{reviewer\}\.md`: the coordinator copies them into\s+`## Declined to Judge`, so nothing a reviewer set aside disappears\./g, '.'], ], B: [ [/^- \*\*Reviewers' Declined to Judge lines\*\* \u2014 each reviewer's own\s+`### Declined to Judge` list, verbatim\. You copy them into\s+`## Declined to Judge` \(body section 9\); you do not rule on them\.\n/gm, ''], [/, then every\s+line a reviewer wrote under its own `### Declined to Judge`, copied as\s+`\{reviewer\}: \{line\}` \(they are not findings: they stay out of the JSON\s+`declined_to_judge` array and do not change the verdict\)/g, ''], ], }; if (!SPANS[which]) { console.error(`mutate: unknown mutation ${which}`); process.exit(1); } let text = readFileSync(file, 'utf8'); for (const [re, by] of SPANS[which]) { const n = (text.match(re) || []).length; if (n !== 1) { console.error(`mutate ${which}: ${re.source.slice(0, 60)}... matched ${n} times in ${file}, not once`); process.exit(1); } text = text.replace(re, by); } writeFileSync(file, text); console.log(`mutate ${which}: ${SPANS[which].length} span(s) deleted in ${file}`); EOF [ -s "$TMP/mutate.mjs" ] || setup_fail "could not write $TMP/mutate.mjs" # mutation : apply, run the DJ tests, judge, restore. mutation() { m="$1" rel="$2" echo "red script: DJ mutation $m ($3)" cp "$TREE/$rel" "$TMP/mutation-$m.orig" || setup_fail "could not save $rel" if node "$TMP/mutate.mjs" "$TREE/$rel" "$m"; then run_tagged "$TMP/mutation-$m.jsonl" "--test-name-pattern=\\[DJ\\]" tests/lib/fix-round-4-5a-prose.test.mjs judge --ids "$TMP/ids" --results "$TMP/mutation-$m.jsonl" --mutation DJ || STATUS=1 else echo "broken: mutation-not-applied [DJ] $m on $rel" STATUS=1 fi cp "$TMP/mutation-$m.orig" "$TREE/$rel" || setup_fail "could not restore $rel" } mutation A commands/trekreview.md "the copy-through span in commands/trekreview.md deleted" mutation B agents/review-coordinator.md "both spans in agents/review-coordinator.md deleted" if [ "$STATUS" -eq 0 ]; then echo "red script: ok - every id red at $BASE, DJ under both mutations" else echo "red script: FAIL - a rule is broken (see the broken: lines above)" >&2 fi exit "$STATUS"