212 lines
8.7 KiB
Bash
212 lines
8.7 KiB
Bash
#!/usr/bin/env bash
|
|
# tests/helpers/red-round.sh
|
|
# The shared red script of the fix rounds. A round's wrapper
|
|
# (tests/fix-round-<n>-red.sh) hands it the round's start point, ids, the
|
|
# other rounds' ids, a mutation module and the round's test files:
|
|
#
|
|
# red-round.sh --base <sha> --ids '<id> ...' --other-ids '<id> ...'
|
|
# --mutations <module.mjs> [--require-sha256] <test file>...
|
|
#
|
|
# Every id of the round has tests whose names start with "[id]"; each of them
|
|
# must fail on an assertion at the start point, or, for an id a mutation
|
|
# names, under that mutation.
|
|
# 1. Clone the repo into a temp directory outside it (git clone --shared
|
|
# --no-checkout, then checkout --detach of the base; nothing registers
|
|
# in .git/worktrees). With --require-sha256 the local git must be able to
|
|
# create a SHA-256 repository first.
|
|
# 2. Replace the clone's tests/ with the working tree's whole tests/ (tar),
|
|
# helpers and fixtures included, so a new helper is no crash.
|
|
# 3. Run every tagged test of the files once (--test-name-pattern=^\[)
|
|
# under the red reporter and let the red judge decide: --ids, --other-ids
|
|
# (another round's tag is never judged), --pass-ids (the ids the
|
|
# mutation module names: their defect is proven by the mutation).
|
|
# 4. For each mutation of the module (tests/helpers/red-mutate.mjs list):
|
|
# apply it to the clone (every span exactly once, else
|
|
# "broken: mutation-not-applied [id] key"), run only that id's tests
|
|
# (^\[id\]), judge with --mutation <id>, restore.
|
|
# The reporter, the judge, the mutate helper and the --import preload always
|
|
# run from the working tree's tests/helpers/ by absolute path, so a mutation
|
|
# of the clone's copy never edits the instrument that measures it.
|
|
# Exit 0 all red, 1 a rule broken (the "broken:" lines name it), 2 a setup
|
|
# failure ("red script: setup failure: <reason>"; a bad base, a missing file,
|
|
# no SHA-256 git under --require-sha256, an instrument that could not run);
|
|
# never 0 on a setup failure. The clone is removed on every exit.
|
|
# Needs Node >= 20.14 for --test-force-exit; an older Node rejects the option
|
|
# and the run fails closed (a setup failure).
|
|
# bash 3.2 compatible, ASCII only, no arrays.
|
|
|
|
set -u
|
|
set -f
|
|
|
|
setup_fail() {
|
|
echo "red script: setup failure: $1" >&2
|
|
exit 2
|
|
}
|
|
|
|
ROOT="$(cd "$(dirname "$0")/../.." && pwd -P)" || setup_fail "cannot find the repo root from $0"
|
|
|
|
unset GIT_DIR GIT_WORK_TREE GIT_INDEX_FILE NODE_TEST_CONTEXT CLAUDE_CODE_SESSION_ID
|
|
|
|
# ---- arguments ----
|
|
|
|
BASE=''
|
|
IDS=''
|
|
OTHER_IDS=''
|
|
MUTATIONS=''
|
|
REQUIRE_SHA256=0
|
|
HAVE_BASE=0
|
|
HAVE_IDS=0
|
|
HAVE_OTHER=0
|
|
HAVE_MUTATIONS=0
|
|
while [ "$#" -gt 0 ]; do
|
|
case "$1" in
|
|
--base) [ "$#" -ge 2 ] || setup_fail "--base needs a value"; BASE="$2"; HAVE_BASE=1; shift 2 ;;
|
|
--ids) [ "$#" -ge 2 ] || setup_fail "--ids needs a value"; IDS="$2"; HAVE_IDS=1; shift 2 ;;
|
|
--other-ids) [ "$#" -ge 2 ] || setup_fail "--other-ids needs a value"; OTHER_IDS="$2"; HAVE_OTHER=1; shift 2 ;;
|
|
--mutations) [ "$#" -ge 2 ] || setup_fail "--mutations needs a value"; MUTATIONS="$2"; HAVE_MUTATIONS=1; shift 2 ;;
|
|
--require-sha256) REQUIRE_SHA256=1; shift ;;
|
|
--) shift; break ;;
|
|
-*) setup_fail "unknown option $1" ;;
|
|
*) break ;;
|
|
esac
|
|
done
|
|
[ "$HAVE_BASE" -eq 1 ] && [ -n "$BASE" ] || setup_fail "--base <sha> is required"
|
|
[ "$HAVE_IDS" -eq 1 ] && [ -n "$IDS" ] || setup_fail "--ids '<id> ...' is required"
|
|
[ "$HAVE_OTHER" -eq 1 ] || setup_fail "--other-ids '<id> ...' is required"
|
|
[ "$HAVE_MUTATIONS" -eq 1 ] || setup_fail "--mutations <module.mjs> is required"
|
|
case "$MUTATIONS" in
|
|
/*) ;;
|
|
*) setup_fail "--mutations needs an absolute path, not '$MUTATIONS'" ;;
|
|
esac
|
|
[ -f "$MUTATIONS" ] || setup_fail "missing mutation module $MUTATIONS"
|
|
[ "$#" -gt 0 ] || setup_fail "no test file given"
|
|
|
|
command -v node >/dev/null 2>&1 || setup_fail "node is not on PATH"
|
|
command -v git >/dev/null 2>&1 || setup_fail "git is not on PATH"
|
|
command -v tar >/dev/null 2>&1 || setup_fail "tar is not on PATH"
|
|
for f in no-session-id.mjs red-reporter.mjs red-judge.mjs red-mutate.mjs; do
|
|
[ -f "$ROOT/tests/helpers/$f" ] || setup_fail "missing $ROOT/tests/helpers/$f"
|
|
done
|
|
|
|
# ---- the temp directory, outside the repo, removed on every exit ----
|
|
|
|
TMP="$(mktemp -d "${TMPDIR:-/tmp}/red-round.XXXXXX")" || setup_fail "mktemp -d failed"
|
|
case "$TMP" in
|
|
''|/) setup_fail "mktemp -d gave an unusable directory '$TMP'" ;;
|
|
esac
|
|
TMP_REAL="$(cd "$TMP" && pwd -P)" || setup_fail "cannot enter $TMP"
|
|
case "$TMP_REAL/" in
|
|
"$ROOT"/*) rmdir "$TMP" 2>/dev/null; setup_fail "the temp directory $TMP_REAL is inside the repo" ;;
|
|
esac
|
|
trap 'rm -rf "$TMP"' EXIT
|
|
trap 'rm -rf "$TMP"; exit 2' INT TERM HUP
|
|
|
|
TREE="$TMP/tree"
|
|
|
|
if [ "$REQUIRE_SHA256" -eq 1 ]; then
|
|
git init --quiet --object-format=sha256 "$TMP/sha256-probe" >/dev/null 2>&1 \
|
|
|| setup_fail "this git cannot create a SHA-256 repository (--require-sha256)"
|
|
rm -rf "$TMP/sha256-probe" || setup_fail "could not remove $TMP/sha256-probe"
|
|
fi
|
|
|
|
# ---- 1. the start point ----
|
|
|
|
git clone --quiet --shared --no-checkout "$ROOT" "$TREE" || setup_fail "could not clone $ROOT"
|
|
git -C "$TREE" checkout --quiet --detach "$BASE" || setup_fail "could not check out $BASE"
|
|
|
|
# ---- 2. the working tree's whole tests/ ----
|
|
|
|
(cd "$ROOT" && tar -cf "$TMP/tests.tar" tests) || setup_fail "could not tar tests/"
|
|
rm -rf "$TREE/tests" || setup_fail "could not remove the start point's tests/"
|
|
tar -xf "$TMP/tests.tar" -C "$TREE" || setup_fail "could not copy tests/ into the start point"
|
|
|
|
for f in "$@"; do
|
|
[ -f "$TREE/$f" ] || setup_fail "missing test file $f"
|
|
done
|
|
|
|
for id in $IDS; do
|
|
printf '%s\n' "$id"
|
|
done > "$TMP/ids" || setup_fail "could not write $TMP/ids"
|
|
: > "$TMP/other-ids" || setup_fail "could not write $TMP/other-ids"
|
|
for id in $OTHER_IDS; do
|
|
printf '%s\n' "$id"
|
|
done >> "$TMP/other-ids" || setup_fail "could not write $TMP/other-ids"
|
|
|
|
# ---- the mutations: "<key> <id>" per line; their ids are the pass-ids ----
|
|
|
|
node "$ROOT/tests/helpers/red-mutate.mjs" list "$MUTATIONS" > "$TMP/mutations" \
|
|
|| setup_fail "red-mutate could not list $MUTATIONS"
|
|
PASS_IDS=''
|
|
while read -r key id rest <&3; do
|
|
[ -n "$key" ] || continue
|
|
case " $IDS " in
|
|
*" $id "*) ;;
|
|
*) setup_fail "mutation $key names id $id, which is not on --ids" ;;
|
|
esac
|
|
case ",$PASS_IDS," in
|
|
*",$id,"*) ;;
|
|
*) PASS_IDS="${PASS_IDS:+$PASS_IDS,}$id" ;;
|
|
esac
|
|
done 3< "$TMP/mutations"
|
|
|
|
# run_tagged <results> <node test option>... <file>...: a test run in the
|
|
# start point under the red reporter. The positional parameters carry the
|
|
# options and files (no arrays in bash 3.2). Node exits 0 (all passed) or 1
|
|
# (a test failed); anything else (a bad option, a crash) is a setup failure.
|
|
run_tagged() {
|
|
out="$1"
|
|
shift
|
|
(cd "$TREE" && node --import "$ROOT/tests/helpers/no-session-id.mjs" --test --test-reporter="$ROOT/tests/helpers/red-reporter.mjs" --test-reporter-destination="$out" "$@") </dev/null >/dev/null 2>"$out.stderr"
|
|
rc=$?
|
|
if [ "$rc" -ne 0 ] && [ "$rc" -ne 1 ]; then
|
|
cat "$out.stderr" >&2
|
|
setup_fail "the test run exited $rc (needs Node >= 20.14)"
|
|
fi
|
|
[ -f "$out" ] || { cat "$out.stderr" >&2; setup_fail "the test run wrote no results to $out"; }
|
|
}
|
|
|
|
# judge <args...>: the red judge; 2 (usage or input) is a setup failure.
|
|
judge() {
|
|
node "$ROOT/tests/helpers/red-judge.mjs" "$@" </dev/null
|
|
rc=$?
|
|
[ "$rc" -eq 2 ] && setup_fail "the red judge could not run (exit 2)"
|
|
return "$rc"
|
|
}
|
|
|
|
STATUS=0
|
|
|
|
# ---- 3. every tagged test at the start point, one run ----
|
|
|
|
echo "red script: the tagged tests at $BASE"
|
|
run_tagged "$TMP/base.jsonl" --test-timeout=300000 --test-force-exit '--test-name-pattern=^\[' "$@"
|
|
judge --ids "$TMP/ids" --other-ids "$TMP/other-ids" --pass-ids "$PASS_IDS" --results "$TMP/base.jsonl" || STATUS=1
|
|
|
|
# ---- 4. each mutation: apply, run its id's tests, judge, restore ----
|
|
|
|
N=0
|
|
while read -r key id rest <&3; do
|
|
[ -n "$key" ] || continue
|
|
N=$((N + 1))
|
|
BK="$TMP/backup-$N"
|
|
echo "red script: mutation $key [$id]"
|
|
node "$ROOT/tests/helpers/red-mutate.mjs" apply "$MUTATIONS" "$key" "$TREE" "$BK" </dev/null >/dev/null
|
|
rc=$?
|
|
if [ "$rc" -eq 0 ]; then
|
|
run_tagged "$TMP/mutation-$N.jsonl" --test-timeout=300000 --test-force-exit "--test-name-pattern=^\\[$id\\]" "$@"
|
|
judge --ids "$TMP/ids" --other-ids "$TMP/other-ids" --results "$TMP/mutation-$N.jsonl" --mutation "$id" || STATUS=1
|
|
node "$ROOT/tests/helpers/red-mutate.mjs" restore "$BK" "$TREE" </dev/null >/dev/null \
|
|
|| setup_fail "could not restore the files of mutation $key"
|
|
elif [ "$rc" -eq 1 ]; then
|
|
echo "broken: mutation-not-applied [$id] $key"
|
|
STATUS=1
|
|
else
|
|
setup_fail "red-mutate could not apply $key (exit $rc)"
|
|
fi
|
|
done 3< "$TMP/mutations"
|
|
|
|
if [ "$STATUS" -eq 0 ]; then
|
|
echo "red script: ok - every id red at $BASE, $N mutation(s) red"
|
|
else
|
|
echo "red script: FAIL - a rule is broken (see the broken: lines above)" >&2
|
|
fi
|
|
exit "$STATUS"
|