167 lines
7.2 KiB
Bash
167 lines
7.2 KiB
Bash
#!/usr/bin/env bash
|
|
# tests/fix-round-4-5a-red.sh
|
|
# The red proof of fix round 4-5a (brief: the last Success Criterion). Every
|
|
# id of the round has tests whose names start with "[id]"; each of them must
|
|
# fail on an assertion at the round's start point ab6b32c.
|
|
# 1. Clone the repo into a temp directory outside it (git clone --shared
|
|
# --no-checkout, then checkout --detach BASE, as
|
|
# lib/verification/red-proof.mjs does; nothing registers in
|
|
# .git/worktrees).
|
|
# 2. Replace the clone's tests/ with the working tree's whole tests/ (tar),
|
|
# helpers and fixtures included, so a new helper is no crash.
|
|
# 3. Run the round's eight test files there, only the tagged tests
|
|
# (--test-name-pattern per id), under tests/helpers/red-reporter.mjs, and
|
|
# let tests/helpers/red-judge.mjs decide (--pass-ids DJ: DJ's defect is a
|
|
# missing test, its sentences exist at ab6b32c).
|
|
# 4. DJ by mutation: A deletes the copy-through span in
|
|
# commands/trekreview.md, B deletes both spans in
|
|
# agents/review-coordinator.md; each span must match exactly once. Each
|
|
# run is judged with --mutation DJ; the file is restored with cp after.
|
|
# Exit 0 all red, 1 a rule broken (the judge names it), 2 a setup failure
|
|
# (clone, checkout, node or git missing); never 0 on a setup failure.
|
|
# bash 3.2 compatible, ASCII only, no arrays.
|
|
|
|
set -u
|
|
set -f
|
|
|
|
cd "$(dirname "$0")/.." || { echo "red script: SETUP FAIL - cannot cd to the repo root" >&2; exit 2; }
|
|
ROOT="$(pwd -P)"
|
|
|
|
BASE=ab6b32c9304534b77c7ca2b36e7c993682c9efde
|
|
IDS='#52 #47 #48 #46 #2 #26 #33 #1 #30 #28 R6 D67 #15 #12 #66 #102 #57 #67 #113 #111 #24 #23 F9 F4a F4b F4c WF CL ORD DJ'
|
|
FILES='tests/lib/fix-round-4-5a-prose.test.mjs tests/lib/verdict-gate.test.mjs tests/lib/park-attempt-files.test.mjs tests/lib/autonomy-gate.test.mjs tests/lib/arg-parser.test.mjs tests/scripts/trekreview-armb-workflow.test.mjs tests/lib/coordinator-contract.test.mjs tests/validators/next-session-prompt-validator.test.mjs'
|
|
JUDGE="$ROOT/tests/helpers/red-judge.mjs"
|
|
|
|
unset GIT_DIR GIT_WORK_TREE GIT_INDEX_FILE NODE_TEST_CONTEXT CLAUDE_CODE_SESSION_ID
|
|
|
|
setup_fail() {
|
|
echo "red script: SETUP FAIL - $1" >&2
|
|
exit 2
|
|
}
|
|
|
|
command -v node >/dev/null 2>&1 || setup_fail "node is not on PATH"
|
|
command -v git >/dev/null 2>&1 || setup_fail "git is not on PATH"
|
|
command -v tar >/dev/null 2>&1 || setup_fail "tar is not on PATH"
|
|
[ -f "$JUDGE" ] || setup_fail "missing $JUDGE"
|
|
|
|
TMP="$(mktemp -d "${TMPDIR:-/tmp}/fix-round-4-5a-red.XXXXXX")" || setup_fail "mktemp -d failed"
|
|
case "$TMP" in
|
|
''|/) setup_fail "mktemp -d gave an unusable directory '$TMP'" ;;
|
|
esac
|
|
TMP_REAL="$(cd "$TMP" && pwd -P)" || setup_fail "cannot enter $TMP"
|
|
case "$TMP_REAL/" in
|
|
"$ROOT"/*) rmdir "$TMP" 2>/dev/null; setup_fail "the temp directory $TMP_REAL is inside the repo" ;;
|
|
esac
|
|
trap 'rm -rf "$TMP"' EXIT
|
|
trap 'rm -rf "$TMP"; exit 2' INT TERM HUP
|
|
|
|
TREE="$TMP/tree"
|
|
|
|
# ---- 1. the start point ----
|
|
|
|
git clone --quiet --shared --no-checkout "$ROOT" "$TREE" || setup_fail "could not clone $ROOT"
|
|
git -C "$TREE" checkout --quiet --detach "$BASE" || setup_fail "could not check out $BASE"
|
|
|
|
# ---- 2. the working tree's whole tests/ ----
|
|
|
|
tar -cf "$TMP/tests.tar" tests || setup_fail "could not tar tests/"
|
|
rm -rf "$TREE/tests" || setup_fail "could not remove the start point's tests/"
|
|
tar -xf "$TMP/tests.tar" -C "$TREE" || setup_fail "could not copy tests/ into the start point"
|
|
|
|
for f in $FILES; do
|
|
[ -f "$TREE/$f" ] || setup_fail "missing test file $f"
|
|
done
|
|
|
|
for id in $IDS; do
|
|
printf '%s\n' "$id"
|
|
done > "$TMP/ids" || setup_fail "could not write $TMP/ids"
|
|
|
|
# run_tagged <results> <file...>: the tagged tests of the files, run in the
|
|
# start point under the red reporter. The positional parameters carry the
|
|
# name patterns (no arrays in bash 3.2).
|
|
run_tagged() {
|
|
out="$1"
|
|
shift
|
|
(cd "$TREE" && node --import ./tests/helpers/no-session-id.mjs --test \
|
|
--test-reporter=./tests/helpers/red-reporter.mjs \
|
|
--test-reporter-destination="$out" "$@") >/dev/null 2>"$out.stderr"
|
|
[ -s "$out" ] || { cat "$out.stderr" >&2; setup_fail "the test run wrote no results to $out"; }
|
|
}
|
|
|
|
# judge <args...>: the red judge; 2 (usage or input) is a setup failure.
|
|
judge() {
|
|
node "$JUDGE" "$@"
|
|
rc=$?
|
|
[ "$rc" -eq 2 ] && setup_fail "the red judge could not run (exit 2)"
|
|
return "$rc"
|
|
}
|
|
|
|
STATUS=0
|
|
|
|
# ---- 3. every id at the start point ----
|
|
|
|
set --
|
|
for id in $IDS; do
|
|
set -- "$@" "--test-name-pattern=\\[$id\\]"
|
|
done
|
|
for f in $FILES; do
|
|
set -- "$@" "$f"
|
|
done
|
|
|
|
echo "red script: the tagged tests at $BASE"
|
|
run_tagged "$TMP/base.jsonl" "$@"
|
|
judge --ids "$TMP/ids" --results "$TMP/base.jsonl" --pass-ids DJ || STATUS=1
|
|
|
|
# ---- 4. DJ under its two mutations ----
|
|
|
|
cat > "$TMP/mutate.mjs" <<'EOF'
|
|
// mutate.mjs <file> <A|B>: delete DJ's copy-through span(s); each span must
|
|
// match exactly once, else exit 1 and leave the file as it was.
|
|
import { readFileSync, writeFileSync } from 'node:fs';
|
|
const [file, which] = process.argv.slice(2);
|
|
const SPANS = {
|
|
A: [
|
|
[/, and pass each reviewer's\s+`### Declined to Judge` lines verbatim from its saved\s+`\{project_dir\}\/\.review-out-\{reviewer\}\.md`: the coordinator copies them into\s+`## Declined to Judge`, so nothing a reviewer set aside disappears\./g, '.'],
|
|
],
|
|
B: [
|
|
[/^- \*\*Reviewers' Declined to Judge lines\*\* \u2014 each reviewer's own\s+`### Declined to Judge` list, verbatim\. You copy them into\s+`## Declined to Judge` \(body section 9\); you do not rule on them\.\n/gm, ''],
|
|
[/, then every\s+line a reviewer wrote under its own `### Declined to Judge`, copied as\s+`\{reviewer\}: \{line\}` \(they are not findings: they stay out of the JSON\s+`declined_to_judge` array and do not change the verdict\)/g, ''],
|
|
],
|
|
};
|
|
if (!SPANS[which]) { console.error(`mutate: unknown mutation ${which}`); process.exit(1); }
|
|
let text = readFileSync(file, 'utf8');
|
|
for (const [re, by] of SPANS[which]) {
|
|
const n = (text.match(re) || []).length;
|
|
if (n !== 1) { console.error(`mutate ${which}: ${re.source.slice(0, 60)}... matched ${n} times in ${file}, not once`); process.exit(1); }
|
|
text = text.replace(re, by);
|
|
}
|
|
writeFileSync(file, text);
|
|
console.log(`mutate ${which}: ${SPANS[which].length} span(s) deleted in ${file}`);
|
|
EOF
|
|
[ -s "$TMP/mutate.mjs" ] || setup_fail "could not write $TMP/mutate.mjs"
|
|
|
|
# mutation <A|B> <repo-relative file> <what>: apply, run the DJ tests, judge, restore.
|
|
mutation() {
|
|
m="$1"
|
|
rel="$2"
|
|
echo "red script: DJ mutation $m ($3)"
|
|
cp "$TREE/$rel" "$TMP/mutation-$m.orig" || setup_fail "could not save $rel"
|
|
if node "$TMP/mutate.mjs" "$TREE/$rel" "$m"; then
|
|
run_tagged "$TMP/mutation-$m.jsonl" "--test-name-pattern=\\[DJ\\]" tests/lib/fix-round-4-5a-prose.test.mjs
|
|
judge --ids "$TMP/ids" --results "$TMP/mutation-$m.jsonl" --mutation DJ || STATUS=1
|
|
else
|
|
echo "broken: mutation-not-applied [DJ] $m on $rel"
|
|
STATUS=1
|
|
fi
|
|
cp "$TMP/mutation-$m.orig" "$TREE/$rel" || setup_fail "could not restore $rel"
|
|
}
|
|
|
|
mutation A commands/trekreview.md "the copy-through span in commands/trekreview.md deleted"
|
|
mutation B agents/review-coordinator.md "both spans in agents/review-coordinator.md deleted"
|
|
|
|
if [ "$STATUS" -eq 0 ]; then
|
|
echo "red script: ok - every id red at $BASE, DJ under both mutations"
|
|
else
|
|
echo "red script: FAIL - a rule is broken (see the broken: lines above)" >&2
|
|
fi
|
|
exit "$STATUS"
|