#!/usr/bin/env bash # tests/helpers/red-round.sh # The shared red script of the fix rounds. A round's wrapper # (tests/fix-round--red.sh) hands it the round's start point, ids, the # other rounds' ids, a mutation module and the round's test files: # # red-round.sh --base --ids ' ...' --other-ids ' ...' # --mutations [--require-sha256] ... # # Every id of the round has tests whose names start with "[id]"; each of them # must fail on an assertion at the start point, or, for an id a mutation # names, under that mutation. # 1. Clone the repo into a temp directory outside it (git clone --shared # --no-checkout, then checkout --detach of the base; nothing registers # in .git/worktrees). With --require-sha256 the local git must be able to # create a SHA-256 repository first. # 2. Replace the clone's tests/ with the working tree's whole tests/ (tar), # helpers and fixtures included, so a new helper is no crash. # 3. Run every tagged test of the files once (--test-name-pattern=^\[) # under the red reporter and let the red judge decide: --ids, --other-ids # (another round's tag is never judged), --pass-ids (the ids the # mutation module names: their defect is proven by the mutation). # 4. For each mutation of the module (tests/helpers/red-mutate.mjs list): # apply it to the clone (every span exactly once, else # "broken: mutation-not-applied [id] key"), run only that id's tests # (^\[id\]), judge with --mutation , restore. # The reporter, the judge, the mutate helper and the --import preload always # run from the working tree's tests/helpers/ by absolute path, so a mutation # of the clone's copy never edits the instrument that measures it. # Exit 0 all red, 1 a rule broken (the "broken:" lines name it), 2 a setup # failure ("red script: setup failure: "; a bad base, a missing file, # no SHA-256 git under --require-sha256, an instrument that could not run); # never 0 on a setup failure. The clone is removed on every exit. # Needs Node >= 20.14 for --test-force-exit; an older Node rejects the option # and the run fails closed (a setup failure). # bash 3.2 compatible, ASCII only, no arrays. set -u set -f setup_fail() { echo "red script: setup failure: $1" >&2 exit 2 } ROOT="$(cd "$(dirname "$0")/../.." && pwd -P)" || setup_fail "cannot find the repo root from $0" unset GIT_DIR GIT_WORK_TREE GIT_INDEX_FILE NODE_TEST_CONTEXT CLAUDE_CODE_SESSION_ID # ---- arguments ---- BASE='' IDS='' OTHER_IDS='' MUTATIONS='' REQUIRE_SHA256=0 HAVE_BASE=0 HAVE_IDS=0 HAVE_OTHER=0 HAVE_MUTATIONS=0 while [ "$#" -gt 0 ]; do case "$1" in --base) [ "$#" -ge 2 ] || setup_fail "--base needs a value"; BASE="$2"; HAVE_BASE=1; shift 2 ;; --ids) [ "$#" -ge 2 ] || setup_fail "--ids needs a value"; IDS="$2"; HAVE_IDS=1; shift 2 ;; --other-ids) [ "$#" -ge 2 ] || setup_fail "--other-ids needs a value"; OTHER_IDS="$2"; HAVE_OTHER=1; shift 2 ;; --mutations) [ "$#" -ge 2 ] || setup_fail "--mutations needs a value"; MUTATIONS="$2"; HAVE_MUTATIONS=1; shift 2 ;; --require-sha256) REQUIRE_SHA256=1; shift ;; --) shift; break ;; -*) setup_fail "unknown option $1" ;; *) break ;; esac done [ "$HAVE_BASE" -eq 1 ] && [ -n "$BASE" ] || setup_fail "--base is required" [ "$HAVE_IDS" -eq 1 ] && [ -n "$IDS" ] || setup_fail "--ids ' ...' is required" [ "$HAVE_OTHER" -eq 1 ] || setup_fail "--other-ids ' ...' is required" [ "$HAVE_MUTATIONS" -eq 1 ] || setup_fail "--mutations is required" case "$MUTATIONS" in /*) ;; *) setup_fail "--mutations needs an absolute path, not '$MUTATIONS'" ;; esac [ -f "$MUTATIONS" ] || setup_fail "missing mutation module $MUTATIONS" [ "$#" -gt 0 ] || setup_fail "no test file given" command -v node >/dev/null 2>&1 || setup_fail "node is not on PATH" command -v git >/dev/null 2>&1 || setup_fail "git is not on PATH" command -v tar >/dev/null 2>&1 || setup_fail "tar is not on PATH" for f in no-session-id.mjs red-reporter.mjs red-judge.mjs red-mutate.mjs; do [ -f "$ROOT/tests/helpers/$f" ] || setup_fail "missing $ROOT/tests/helpers/$f" done # ---- the temp directory, outside the repo, removed on every exit ---- TMP="$(mktemp -d "${TMPDIR:-/tmp}/red-round.XXXXXX")" || setup_fail "mktemp -d failed" case "$TMP" in ''|/) setup_fail "mktemp -d gave an unusable directory '$TMP'" ;; esac TMP_REAL="$(cd "$TMP" && pwd -P)" || setup_fail "cannot enter $TMP" case "$TMP_REAL/" in "$ROOT"/*) rmdir "$TMP" 2>/dev/null; setup_fail "the temp directory $TMP_REAL is inside the repo" ;; esac trap 'rm -rf "$TMP"' EXIT trap 'rm -rf "$TMP"; exit 2' INT TERM HUP TREE="$TMP/tree" if [ "$REQUIRE_SHA256" -eq 1 ]; then git init --quiet --object-format=sha256 "$TMP/sha256-probe" >/dev/null 2>&1 \ || setup_fail "this git cannot create a SHA-256 repository (--require-sha256)" rm -rf "$TMP/sha256-probe" || setup_fail "could not remove $TMP/sha256-probe" fi # ---- 1. the start point ---- git clone --quiet --shared --no-checkout "$ROOT" "$TREE" || setup_fail "could not clone $ROOT" git -C "$TREE" checkout --quiet --detach "$BASE" || setup_fail "could not check out $BASE" # ---- 2. the working tree's whole tests/ ---- (cd "$ROOT" && tar -cf "$TMP/tests.tar" tests) || setup_fail "could not tar tests/" rm -rf "$TREE/tests" || setup_fail "could not remove the start point's tests/" tar -xf "$TMP/tests.tar" -C "$TREE" || setup_fail "could not copy tests/ into the start point" for f in "$@"; do [ -f "$TREE/$f" ] || setup_fail "missing test file $f" done for id in $IDS; do printf '%s\n' "$id" done > "$TMP/ids" || setup_fail "could not write $TMP/ids" : > "$TMP/other-ids" || setup_fail "could not write $TMP/other-ids" for id in $OTHER_IDS; do printf '%s\n' "$id" done >> "$TMP/other-ids" || setup_fail "could not write $TMP/other-ids" # ---- the mutations: " " per line; their ids are the pass-ids ---- node "$ROOT/tests/helpers/red-mutate.mjs" list "$MUTATIONS" > "$TMP/mutations" \ || setup_fail "red-mutate could not list $MUTATIONS" PASS_IDS='' while read -r key id rest <&3; do [ -n "$key" ] || continue case " $IDS " in *" $id "*) ;; *) setup_fail "mutation $key names id $id, which is not on --ids" ;; esac case ",$PASS_IDS," in *",$id,"*) ;; *) PASS_IDS="${PASS_IDS:+$PASS_IDS,}$id" ;; esac done 3< "$TMP/mutations" # run_tagged ... ...: a test run in the # start point under the red reporter. The positional parameters carry the # options and files (no arrays in bash 3.2). Node exits 0 (all passed) or 1 # (a test failed); anything else (a bad option, a crash) is a setup failure. run_tagged() { out="$1" shift (cd "$TREE" && node --import "$ROOT/tests/helpers/no-session-id.mjs" --test --test-reporter="$ROOT/tests/helpers/red-reporter.mjs" --test-reporter-destination="$out" "$@") /dev/null 2>"$out.stderr" rc=$? if [ "$rc" -ne 0 ] && [ "$rc" -ne 1 ]; then cat "$out.stderr" >&2 setup_fail "the test run exited $rc (needs Node >= 20.14)" fi [ -f "$out" ] || { cat "$out.stderr" >&2; setup_fail "the test run wrote no results to $out"; } } # judge : the red judge; 2 (usage or input) is a setup failure. judge() { node "$ROOT/tests/helpers/red-judge.mjs" "$@" /dev/null rc=$? if [ "$rc" -eq 0 ]; then run_tagged "$TMP/mutation-$N.jsonl" --test-timeout=300000 --test-force-exit "--test-name-pattern=^\\[$id\\]" "$@" judge --ids "$TMP/ids" --other-ids "$TMP/other-ids" --results "$TMP/mutation-$N.jsonl" --mutation "$id" || STATUS=1 node "$ROOT/tests/helpers/red-mutate.mjs" restore "$BK" "$TREE" /dev/null \ || setup_fail "could not restore the files of mutation $key" elif [ "$rc" -eq 1 ]; then echo "broken: mutation-not-applied [$id] $key" STATUS=1 else setup_fail "red-mutate could not apply $key (exit $rc)" fi done 3< "$TMP/mutations" if [ "$STATUS" -eq 0 ]; then echo "red script: ok - every id red at $BASE, $N mutation(s) red" else echo "red script: FAIL - a rule is broken (see the broken: lines above)" >&2 fi exit "$STATUS"