feat(red-proof): one red script that takes base, ids, mutations and files as arguments
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
cde4c1d3ed
commit
2dbfbaf8ca
SSH key fingerprint:
SHA256:JakMjO6FTBBzN0Bhfj9saOoEjaFxlSdYuZQQpM/lF9Q
3 changed files with 733 additions and 0 deletions
180
tests/helpers/red-mutate.mjs
Normal file
180
tests/helpers/red-mutate.mjs
Normal file
|
|
@ -0,0 +1,180 @@
|
|||
#!/usr/bin/env node
|
||||
// tests/helpers/red-mutate.mjs
|
||||
// The mutate helper of the shared red script (tests/helpers/red-round.sh).
|
||||
// A mutation module is a plain ES module (no ".test." in its name, so
|
||||
// `npm test` never runs it) that exports
|
||||
// MUTATIONS = [{ key, id, what, spans: [{ file, find, replace }] }]
|
||||
// key unique in the module; id one of the round's ids; file relative to the
|
||||
// tree; find a string or a RegExp with the g flag. A string find is replaced
|
||||
// literally; a RegExp find uses String.prototype.replace, $-patterns included.
|
||||
//
|
||||
// node tests/helpers/red-mutate.mjs list <module>
|
||||
// one "<key> <id>" line per entry, in module order.
|
||||
// node tests/helpers/red-mutate.mjs apply <module> <key> <tree> <backup-dir>
|
||||
// counts every span of the entry first (spans on one file count on the
|
||||
// text the earlier spans left); writes only when every span matched
|
||||
// exactly once: each touched file is backed up into <backup-dir> (which
|
||||
// must not exist yet), then written. Exit 1, nothing written and no
|
||||
// backup made, when a span matched 0 or 2+ times (one stderr line each).
|
||||
// node tests/helpers/red-mutate.mjs restore <backup-dir> <tree>
|
||||
// copies every backed-up file back into the tree and removes the backup.
|
||||
// Exit 2: a usage error, an unreadable module or file, a bad entry, a span
|
||||
// file outside the tree, a failed write or restore (never 0).
|
||||
|
||||
import { copyFileSync, existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
|
||||
import { dirname, isAbsolute, join, relative, resolve, sep } from 'node:path';
|
||||
import { pathToFileURL } from 'node:url';
|
||||
|
||||
const USAGE = 'usage: red-mutate.mjs list <module> | apply <module> <key> <tree> <backup-dir> | restore <backup-dir> <tree>';
|
||||
const MANIFEST = 'manifest.json';
|
||||
|
||||
/** Print a problem and exit 2. */
|
||||
function fail(msg) {
|
||||
process.stderr.write(`red-mutate: ${msg}\n${USAGE}\n`);
|
||||
process.exit(2);
|
||||
}
|
||||
|
||||
/** The module's MUTATIONS, checked; exit 2 on any defect. */
|
||||
async function load(modulePath) {
|
||||
let mod;
|
||||
try {
|
||||
mod = await import(pathToFileURL(resolve(modulePath)).href);
|
||||
} catch (e) {
|
||||
return fail(`cannot load the mutation module ${modulePath}: ${e.message}`);
|
||||
}
|
||||
const list = mod.MUTATIONS;
|
||||
if (!Array.isArray(list)) fail(`${modulePath} exports no MUTATIONS array`);
|
||||
const keys = new Set();
|
||||
list.forEach((m, i) => {
|
||||
const where = `MUTATIONS[${i}]`;
|
||||
if (!m || typeof m.key !== 'string' || !/^\S+$/.test(m.key)) fail(`${where}: key must be one word`);
|
||||
if (typeof m.id !== 'string' || !/^\S+$/.test(m.id)) fail(`${where} (${m.key}): id must be one word`);
|
||||
if (keys.has(m.key)) fail(`${where}: key ${m.key} is used twice`);
|
||||
keys.add(m.key);
|
||||
if (!Array.isArray(m.spans) || m.spans.length === 0) fail(`${where} (${m.key}): spans must be a non-empty array`);
|
||||
m.spans.forEach((s, j) => {
|
||||
const at = `${m.key} span ${j + 1}`;
|
||||
if (!s || typeof s.file !== 'string' || s.file === '' || isAbsolute(s.file)) fail(`${at}: file must be a relative path`);
|
||||
if (typeof s.replace !== 'string') fail(`${at}: replace must be a string`);
|
||||
if (s.find instanceof RegExp) {
|
||||
if (!s.find.global) fail(`${at}: a RegExp find needs the g flag`);
|
||||
} else if (typeof s.find !== 'string' || s.find === '') {
|
||||
fail(`${at}: find must be a non-empty string or a RegExp`);
|
||||
}
|
||||
});
|
||||
});
|
||||
return list;
|
||||
}
|
||||
|
||||
/** The absolute path of a tree-relative file, or exit 2 when it leaves the tree. */
|
||||
function inTree(tree, rel) {
|
||||
const root = resolve(tree);
|
||||
const abs = resolve(root, rel);
|
||||
const back = relative(root, abs);
|
||||
if (back === '' || back === '..' || back.startsWith(`..${sep}`) || isAbsolute(back)) fail(`${rel} is outside the tree ${tree}`);
|
||||
return abs;
|
||||
}
|
||||
|
||||
/** How often a find matches in a text. */
|
||||
function count(text, find) {
|
||||
if (typeof find === 'string') return text.split(find).length - 1;
|
||||
return [...text.matchAll(find)].length;
|
||||
}
|
||||
|
||||
/** The text with its one match replaced. */
|
||||
function replaceOnce(text, find, by) {
|
||||
if (typeof find === 'string') return text.replace(find, () => by);
|
||||
return text.replace(find, by);
|
||||
}
|
||||
|
||||
/** One line for a span's find. */
|
||||
const show = (find) => (typeof find === 'string' ? JSON.stringify(find) : String(find)).replace(/\s+/g, ' ').slice(0, 80);
|
||||
|
||||
async function list(argv) {
|
||||
if (argv.length !== 1) fail('list takes one argument');
|
||||
const entries = await load(argv[0]);
|
||||
process.stdout.write(entries.map((m) => `${m.key} ${m.id}\n`).join(''));
|
||||
}
|
||||
|
||||
async function apply(argv) {
|
||||
if (argv.length !== 4) fail('apply takes four arguments');
|
||||
const [modulePath, key, tree, backup] = argv;
|
||||
const entry = (await load(modulePath)).find((m) => m.key === key);
|
||||
if (!entry) fail(`no mutation with key ${key} in ${modulePath}`);
|
||||
if (!existsSync(tree)) fail(`the tree ${tree} does not exist`);
|
||||
if (existsSync(backup)) fail(`the backup directory ${backup} already exists (a mutation not restored?)`);
|
||||
|
||||
const texts = new Map();
|
||||
const originals = new Map();
|
||||
const misses = [];
|
||||
for (const span of entry.spans) {
|
||||
const abs = inTree(tree, span.file);
|
||||
if (!texts.has(span.file)) {
|
||||
if (!existsSync(abs)) {
|
||||
misses.push(`${key}: ${span.file} does not exist in the tree`);
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
const t = readFileSync(abs, 'utf8');
|
||||
texts.set(span.file, t);
|
||||
originals.set(span.file, t);
|
||||
} catch (e) {
|
||||
fail(`cannot read ${abs}: ${e.code || e.message}`);
|
||||
}
|
||||
}
|
||||
const text = texts.get(span.file);
|
||||
const n = count(text, span.find);
|
||||
if (n !== 1) {
|
||||
misses.push(`${key}: ${show(span.find)} matched ${n} times in ${span.file}, not once`);
|
||||
continue;
|
||||
}
|
||||
texts.set(span.file, replaceOnce(text, span.find, span.replace));
|
||||
}
|
||||
if (misses.length > 0) {
|
||||
process.stderr.write(`${misses.map((m) => `red-mutate: ${m}`).join('\n')}\n`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
try {
|
||||
mkdirSync(join(backup, 'files'), { recursive: true });
|
||||
for (const rel of texts.keys()) {
|
||||
const dest = join(backup, 'files', rel);
|
||||
mkdirSync(dirname(dest), { recursive: true });
|
||||
writeFileSync(dest, originals.get(rel));
|
||||
}
|
||||
writeFileSync(join(backup, MANIFEST), `${JSON.stringify([...texts.keys()])}\n`);
|
||||
} catch (e) {
|
||||
fail(`cannot write the backup in ${backup}: ${e.code || e.message}`);
|
||||
}
|
||||
try {
|
||||
for (const [rel, text] of texts) writeFileSync(inTree(tree, rel), text);
|
||||
} catch (e) {
|
||||
fail(`cannot write the mutated file: ${e.code || e.message} (restore from ${backup})`);
|
||||
}
|
||||
process.stdout.write(`red-mutate: ${key} [${entry.id}] applied, ${entry.spans.length} span(s) in ${texts.size} file(s)\n`);
|
||||
}
|
||||
|
||||
function restore(argv) {
|
||||
if (argv.length !== 2) fail('restore takes two arguments');
|
||||
const [backup, tree] = argv;
|
||||
let files;
|
||||
try {
|
||||
files = JSON.parse(readFileSync(join(backup, MANIFEST), 'utf8'));
|
||||
} catch (e) {
|
||||
return fail(`cannot read the backup manifest in ${backup}: ${e.code || e.message}`);
|
||||
}
|
||||
if (!Array.isArray(files)) fail(`the backup manifest in ${backup} is not a list`);
|
||||
try {
|
||||
for (const rel of files) copyFileSync(join(backup, 'files', rel), inTree(tree, rel));
|
||||
rmSync(backup, { recursive: true, force: true });
|
||||
} catch (e) {
|
||||
fail(`cannot restore from ${backup}: ${e.code || e.message}`);
|
||||
}
|
||||
process.stdout.write(`red-mutate: restored ${files.length} file(s)\n`);
|
||||
}
|
||||
|
||||
const [cmd, ...rest] = process.argv.slice(2);
|
||||
if (cmd === 'list') await list(rest);
|
||||
else if (cmd === 'apply') await apply(rest);
|
||||
else if (cmd === 'restore') restore(rest);
|
||||
else fail(cmd ? `unknown command ${cmd}` : 'no command');
|
||||
212
tests/helpers/red-round.sh
Normal file
212
tests/helpers/red-round.sh
Normal file
|
|
@ -0,0 +1,212 @@
|
|||
#!/usr/bin/env bash
|
||||
# tests/helpers/red-round.sh
|
||||
# The shared red script of the fix rounds. A round's wrapper
|
||||
# (tests/fix-round-<n>-red.sh) hands it the round's start point, ids, the
|
||||
# other rounds' ids, a mutation module and the round's test files:
|
||||
#
|
||||
# red-round.sh --base <sha> --ids '<id> ...' --other-ids '<id> ...'
|
||||
# --mutations <module.mjs> [--require-sha256] <test file>...
|
||||
#
|
||||
# Every id of the round has tests whose names start with "[id]"; each of them
|
||||
# must fail on an assertion at the start point, or, for an id a mutation
|
||||
# names, under that mutation.
|
||||
# 1. Clone the repo into a temp directory outside it (git clone --shared
|
||||
# --no-checkout, then checkout --detach of the base; nothing registers
|
||||
# in .git/worktrees). With --require-sha256 the local git must be able to
|
||||
# create a SHA-256 repository first.
|
||||
# 2. Replace the clone's tests/ with the working tree's whole tests/ (tar),
|
||||
# helpers and fixtures included, so a new helper is no crash.
|
||||
# 3. Run every tagged test of the files once (--test-name-pattern=^\[)
|
||||
# under the red reporter and let the red judge decide: --ids, --other-ids
|
||||
# (another round's tag is never judged), --pass-ids (the ids the
|
||||
# mutation module names: their defect is proven by the mutation).
|
||||
# 4. For each mutation of the module (tests/helpers/red-mutate.mjs list):
|
||||
# apply it to the clone (every span exactly once, else
|
||||
# "broken: mutation-not-applied [id] key"), run only that id's tests
|
||||
# (^\[id\]), judge with --mutation <id>, restore.
|
||||
# The reporter, the judge, the mutate helper and the --import preload always
|
||||
# run from the working tree's tests/helpers/ by absolute path, so a mutation
|
||||
# of the clone's copy never edits the instrument that measures it.
|
||||
# Exit 0 all red, 1 a rule broken (the "broken:" lines name it), 2 a setup
|
||||
# failure ("red script: setup failure: <reason>"; a bad base, a missing file,
|
||||
# no SHA-256 git under --require-sha256, an instrument that could not run);
|
||||
# never 0 on a setup failure. The clone is removed on every exit.
|
||||
# Needs Node >= 20.14 for --test-force-exit; an older Node rejects the option
|
||||
# and the run fails closed (a setup failure).
|
||||
# bash 3.2 compatible, ASCII only, no arrays.
|
||||
|
||||
set -u
|
||||
set -f
|
||||
|
||||
setup_fail() {
|
||||
echo "red script: setup failure: $1" >&2
|
||||
exit 2
|
||||
}
|
||||
|
||||
ROOT="$(cd "$(dirname "$0")/../.." && pwd -P)" || setup_fail "cannot find the repo root from $0"
|
||||
|
||||
unset GIT_DIR GIT_WORK_TREE GIT_INDEX_FILE NODE_TEST_CONTEXT CLAUDE_CODE_SESSION_ID
|
||||
|
||||
# ---- arguments ----
|
||||
|
||||
BASE=''
|
||||
IDS=''
|
||||
OTHER_IDS=''
|
||||
MUTATIONS=''
|
||||
REQUIRE_SHA256=0
|
||||
HAVE_BASE=0
|
||||
HAVE_IDS=0
|
||||
HAVE_OTHER=0
|
||||
HAVE_MUTATIONS=0
|
||||
while [ "$#" -gt 0 ]; do
|
||||
case "$1" in
|
||||
--base) [ "$#" -ge 2 ] || setup_fail "--base needs a value"; BASE="$2"; HAVE_BASE=1; shift 2 ;;
|
||||
--ids) [ "$#" -ge 2 ] || setup_fail "--ids needs a value"; IDS="$2"; HAVE_IDS=1; shift 2 ;;
|
||||
--other-ids) [ "$#" -ge 2 ] || setup_fail "--other-ids needs a value"; OTHER_IDS="$2"; HAVE_OTHER=1; shift 2 ;;
|
||||
--mutations) [ "$#" -ge 2 ] || setup_fail "--mutations needs a value"; MUTATIONS="$2"; HAVE_MUTATIONS=1; shift 2 ;;
|
||||
--require-sha256) REQUIRE_SHA256=1; shift ;;
|
||||
--) shift; break ;;
|
||||
-*) setup_fail "unknown option $1" ;;
|
||||
*) break ;;
|
||||
esac
|
||||
done
|
||||
[ "$HAVE_BASE" -eq 1 ] && [ -n "$BASE" ] || setup_fail "--base <sha> is required"
|
||||
[ "$HAVE_IDS" -eq 1 ] && [ -n "$IDS" ] || setup_fail "--ids '<id> ...' is required"
|
||||
[ "$HAVE_OTHER" -eq 1 ] || setup_fail "--other-ids '<id> ...' is required"
|
||||
[ "$HAVE_MUTATIONS" -eq 1 ] || setup_fail "--mutations <module.mjs> is required"
|
||||
case "$MUTATIONS" in
|
||||
/*) ;;
|
||||
*) setup_fail "--mutations needs an absolute path, not '$MUTATIONS'" ;;
|
||||
esac
|
||||
[ -f "$MUTATIONS" ] || setup_fail "missing mutation module $MUTATIONS"
|
||||
[ "$#" -gt 0 ] || setup_fail "no test file given"
|
||||
|
||||
command -v node >/dev/null 2>&1 || setup_fail "node is not on PATH"
|
||||
command -v git >/dev/null 2>&1 || setup_fail "git is not on PATH"
|
||||
command -v tar >/dev/null 2>&1 || setup_fail "tar is not on PATH"
|
||||
for f in no-session-id.mjs red-reporter.mjs red-judge.mjs red-mutate.mjs; do
|
||||
[ -f "$ROOT/tests/helpers/$f" ] || setup_fail "missing $ROOT/tests/helpers/$f"
|
||||
done
|
||||
|
||||
# ---- the temp directory, outside the repo, removed on every exit ----
|
||||
|
||||
TMP="$(mktemp -d "${TMPDIR:-/tmp}/red-round.XXXXXX")" || setup_fail "mktemp -d failed"
|
||||
case "$TMP" in
|
||||
''|/) setup_fail "mktemp -d gave an unusable directory '$TMP'" ;;
|
||||
esac
|
||||
TMP_REAL="$(cd "$TMP" && pwd -P)" || setup_fail "cannot enter $TMP"
|
||||
case "$TMP_REAL/" in
|
||||
"$ROOT"/*) rmdir "$TMP" 2>/dev/null; setup_fail "the temp directory $TMP_REAL is inside the repo" ;;
|
||||
esac
|
||||
trap 'rm -rf "$TMP"' EXIT
|
||||
trap 'rm -rf "$TMP"; exit 2' INT TERM HUP
|
||||
|
||||
TREE="$TMP/tree"
|
||||
|
||||
if [ "$REQUIRE_SHA256" -eq 1 ]; then
|
||||
git init --quiet --object-format=sha256 "$TMP/sha256-probe" >/dev/null 2>&1 \
|
||||
|| setup_fail "this git cannot create a SHA-256 repository (--require-sha256)"
|
||||
rm -rf "$TMP/sha256-probe" || setup_fail "could not remove $TMP/sha256-probe"
|
||||
fi
|
||||
|
||||
# ---- 1. the start point ----
|
||||
|
||||
git clone --quiet --shared --no-checkout "$ROOT" "$TREE" || setup_fail "could not clone $ROOT"
|
||||
git -C "$TREE" checkout --quiet --detach "$BASE" || setup_fail "could not check out $BASE"
|
||||
|
||||
# ---- 2. the working tree's whole tests/ ----
|
||||
|
||||
(cd "$ROOT" && tar -cf "$TMP/tests.tar" tests) || setup_fail "could not tar tests/"
|
||||
rm -rf "$TREE/tests" || setup_fail "could not remove the start point's tests/"
|
||||
tar -xf "$TMP/tests.tar" -C "$TREE" || setup_fail "could not copy tests/ into the start point"
|
||||
|
||||
for f in "$@"; do
|
||||
[ -f "$TREE/$f" ] || setup_fail "missing test file $f"
|
||||
done
|
||||
|
||||
for id in $IDS; do
|
||||
printf '%s\n' "$id"
|
||||
done > "$TMP/ids" || setup_fail "could not write $TMP/ids"
|
||||
: > "$TMP/other-ids" || setup_fail "could not write $TMP/other-ids"
|
||||
for id in $OTHER_IDS; do
|
||||
printf '%s\n' "$id"
|
||||
done >> "$TMP/other-ids" || setup_fail "could not write $TMP/other-ids"
|
||||
|
||||
# ---- the mutations: "<key> <id>" per line; their ids are the pass-ids ----
|
||||
|
||||
node "$ROOT/tests/helpers/red-mutate.mjs" list "$MUTATIONS" > "$TMP/mutations" \
|
||||
|| setup_fail "red-mutate could not list $MUTATIONS"
|
||||
PASS_IDS=''
|
||||
while read -r key id rest <&3; do
|
||||
[ -n "$key" ] || continue
|
||||
case " $IDS " in
|
||||
*" $id "*) ;;
|
||||
*) setup_fail "mutation $key names id $id, which is not on --ids" ;;
|
||||
esac
|
||||
case ",$PASS_IDS," in
|
||||
*",$id,"*) ;;
|
||||
*) PASS_IDS="${PASS_IDS:+$PASS_IDS,}$id" ;;
|
||||
esac
|
||||
done 3< "$TMP/mutations"
|
||||
|
||||
# run_tagged <results> <node test option>... <file>...: a test run in the
|
||||
# start point under the red reporter. The positional parameters carry the
|
||||
# options and files (no arrays in bash 3.2). Node exits 0 (all passed) or 1
|
||||
# (a test failed); anything else (a bad option, a crash) is a setup failure.
|
||||
run_tagged() {
|
||||
out="$1"
|
||||
shift
|
||||
(cd "$TREE" && node --import "$ROOT/tests/helpers/no-session-id.mjs" --test --test-reporter="$ROOT/tests/helpers/red-reporter.mjs" --test-reporter-destination="$out" "$@") </dev/null >/dev/null 2>"$out.stderr"
|
||||
rc=$?
|
||||
if [ "$rc" -ne 0 ] && [ "$rc" -ne 1 ]; then
|
||||
cat "$out.stderr" >&2
|
||||
setup_fail "the test run exited $rc (needs Node >= 20.14)"
|
||||
fi
|
||||
[ -f "$out" ] || { cat "$out.stderr" >&2; setup_fail "the test run wrote no results to $out"; }
|
||||
}
|
||||
|
||||
# judge <args...>: the red judge; 2 (usage or input) is a setup failure.
|
||||
judge() {
|
||||
node "$ROOT/tests/helpers/red-judge.mjs" "$@" </dev/null
|
||||
rc=$?
|
||||
[ "$rc" -eq 2 ] && setup_fail "the red judge could not run (exit 2)"
|
||||
return "$rc"
|
||||
}
|
||||
|
||||
STATUS=0
|
||||
|
||||
# ---- 3. every tagged test at the start point, one run ----
|
||||
|
||||
echo "red script: the tagged tests at $BASE"
|
||||
run_tagged "$TMP/base.jsonl" --test-timeout=300000 --test-force-exit '--test-name-pattern=^\[' "$@"
|
||||
judge --ids "$TMP/ids" --other-ids "$TMP/other-ids" --pass-ids "$PASS_IDS" --results "$TMP/base.jsonl" || STATUS=1
|
||||
|
||||
# ---- 4. each mutation: apply, run its id's tests, judge, restore ----
|
||||
|
||||
N=0
|
||||
while read -r key id rest <&3; do
|
||||
[ -n "$key" ] || continue
|
||||
N=$((N + 1))
|
||||
BK="$TMP/backup-$N"
|
||||
echo "red script: mutation $key [$id]"
|
||||
node "$ROOT/tests/helpers/red-mutate.mjs" apply "$MUTATIONS" "$key" "$TREE" "$BK" </dev/null >/dev/null
|
||||
rc=$?
|
||||
if [ "$rc" -eq 0 ]; then
|
||||
run_tagged "$TMP/mutation-$N.jsonl" --test-timeout=300000 --test-force-exit "--test-name-pattern=^\\[$id\\]" "$@"
|
||||
judge --ids "$TMP/ids" --other-ids "$TMP/other-ids" --results "$TMP/mutation-$N.jsonl" --mutation "$id" || STATUS=1
|
||||
node "$ROOT/tests/helpers/red-mutate.mjs" restore "$BK" "$TREE" </dev/null >/dev/null \
|
||||
|| setup_fail "could not restore the files of mutation $key"
|
||||
elif [ "$rc" -eq 1 ]; then
|
||||
echo "broken: mutation-not-applied [$id] $key"
|
||||
STATUS=1
|
||||
else
|
||||
setup_fail "red-mutate could not apply $key (exit $rc)"
|
||||
fi
|
||||
done 3< "$TMP/mutations"
|
||||
|
||||
if [ "$STATUS" -eq 0 ]; then
|
||||
echo "red script: ok - every id red at $BASE, $N mutation(s) red"
|
||||
else
|
||||
echo "red script: FAIL - a rule is broken (see the broken: lines above)" >&2
|
||||
fi
|
||||
exit "$STATUS"
|
||||
Loading…
Add table
Add a link
Reference in a new issue