feat(red-proof): one red script that takes base, ids, mutations and files as arguments
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
cde4c1d3ed
commit
2dbfbaf8ca
SSH key fingerprint:
SHA256:JakMjO6FTBBzN0Bhfj9saOoEjaFxlSdYuZQQpM/lF9Q
3 changed files with 733 additions and 0 deletions
180
tests/helpers/red-mutate.mjs
Normal file
180
tests/helpers/red-mutate.mjs
Normal file
|
|
@ -0,0 +1,180 @@
|
|||
#!/usr/bin/env node
|
||||
// tests/helpers/red-mutate.mjs
|
||||
// The mutate helper of the shared red script (tests/helpers/red-round.sh).
|
||||
// A mutation module is a plain ES module (no ".test." in its name, so
|
||||
// `npm test` never runs it) that exports
|
||||
// MUTATIONS = [{ key, id, what, spans: [{ file, find, replace }] }]
|
||||
// key unique in the module; id one of the round's ids; file relative to the
|
||||
// tree; find a string or a RegExp with the g flag. A string find is replaced
|
||||
// literally; a RegExp find uses String.prototype.replace, $-patterns included.
|
||||
//
|
||||
// node tests/helpers/red-mutate.mjs list <module>
|
||||
// one "<key> <id>" line per entry, in module order.
|
||||
// node tests/helpers/red-mutate.mjs apply <module> <key> <tree> <backup-dir>
|
||||
// counts every span of the entry first (spans on one file count on the
|
||||
// text the earlier spans left); writes only when every span matched
|
||||
// exactly once: each touched file is backed up into <backup-dir> (which
|
||||
// must not exist yet), then written. Exit 1, nothing written and no
|
||||
// backup made, when a span matched 0 or 2+ times (one stderr line each).
|
||||
// node tests/helpers/red-mutate.mjs restore <backup-dir> <tree>
|
||||
// copies every backed-up file back into the tree and removes the backup.
|
||||
// Exit 2: a usage error, an unreadable module or file, a bad entry, a span
|
||||
// file outside the tree, a failed write or restore (never 0).
|
||||
|
||||
import { copyFileSync, existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
|
||||
import { dirname, isAbsolute, join, relative, resolve, sep } from 'node:path';
|
||||
import { pathToFileURL } from 'node:url';
|
||||
|
||||
const USAGE = 'usage: red-mutate.mjs list <module> | apply <module> <key> <tree> <backup-dir> | restore <backup-dir> <tree>';
|
||||
const MANIFEST = 'manifest.json';
|
||||
|
||||
/** Print a problem and exit 2. */
|
||||
function fail(msg) {
|
||||
process.stderr.write(`red-mutate: ${msg}\n${USAGE}\n`);
|
||||
process.exit(2);
|
||||
}
|
||||
|
||||
/** The module's MUTATIONS, checked; exit 2 on any defect. */
|
||||
async function load(modulePath) {
|
||||
let mod;
|
||||
try {
|
||||
mod = await import(pathToFileURL(resolve(modulePath)).href);
|
||||
} catch (e) {
|
||||
return fail(`cannot load the mutation module ${modulePath}: ${e.message}`);
|
||||
}
|
||||
const list = mod.MUTATIONS;
|
||||
if (!Array.isArray(list)) fail(`${modulePath} exports no MUTATIONS array`);
|
||||
const keys = new Set();
|
||||
list.forEach((m, i) => {
|
||||
const where = `MUTATIONS[${i}]`;
|
||||
if (!m || typeof m.key !== 'string' || !/^\S+$/.test(m.key)) fail(`${where}: key must be one word`);
|
||||
if (typeof m.id !== 'string' || !/^\S+$/.test(m.id)) fail(`${where} (${m.key}): id must be one word`);
|
||||
if (keys.has(m.key)) fail(`${where}: key ${m.key} is used twice`);
|
||||
keys.add(m.key);
|
||||
if (!Array.isArray(m.spans) || m.spans.length === 0) fail(`${where} (${m.key}): spans must be a non-empty array`);
|
||||
m.spans.forEach((s, j) => {
|
||||
const at = `${m.key} span ${j + 1}`;
|
||||
if (!s || typeof s.file !== 'string' || s.file === '' || isAbsolute(s.file)) fail(`${at}: file must be a relative path`);
|
||||
if (typeof s.replace !== 'string') fail(`${at}: replace must be a string`);
|
||||
if (s.find instanceof RegExp) {
|
||||
if (!s.find.global) fail(`${at}: a RegExp find needs the g flag`);
|
||||
} else if (typeof s.find !== 'string' || s.find === '') {
|
||||
fail(`${at}: find must be a non-empty string or a RegExp`);
|
||||
}
|
||||
});
|
||||
});
|
||||
return list;
|
||||
}
|
||||
|
||||
/** The absolute path of a tree-relative file, or exit 2 when it leaves the tree. */
|
||||
function inTree(tree, rel) {
|
||||
const root = resolve(tree);
|
||||
const abs = resolve(root, rel);
|
||||
const back = relative(root, abs);
|
||||
if (back === '' || back === '..' || back.startsWith(`..${sep}`) || isAbsolute(back)) fail(`${rel} is outside the tree ${tree}`);
|
||||
return abs;
|
||||
}
|
||||
|
||||
/** How often a find matches in a text. */
|
||||
function count(text, find) {
|
||||
if (typeof find === 'string') return text.split(find).length - 1;
|
||||
return [...text.matchAll(find)].length;
|
||||
}
|
||||
|
||||
/** The text with its one match replaced. */
|
||||
function replaceOnce(text, find, by) {
|
||||
if (typeof find === 'string') return text.replace(find, () => by);
|
||||
return text.replace(find, by);
|
||||
}
|
||||
|
||||
/** One line for a span's find. */
|
||||
const show = (find) => (typeof find === 'string' ? JSON.stringify(find) : String(find)).replace(/\s+/g, ' ').slice(0, 80);
|
||||
|
||||
async function list(argv) {
|
||||
if (argv.length !== 1) fail('list takes one argument');
|
||||
const entries = await load(argv[0]);
|
||||
process.stdout.write(entries.map((m) => `${m.key} ${m.id}\n`).join(''));
|
||||
}
|
||||
|
||||
async function apply(argv) {
|
||||
if (argv.length !== 4) fail('apply takes four arguments');
|
||||
const [modulePath, key, tree, backup] = argv;
|
||||
const entry = (await load(modulePath)).find((m) => m.key === key);
|
||||
if (!entry) fail(`no mutation with key ${key} in ${modulePath}`);
|
||||
if (!existsSync(tree)) fail(`the tree ${tree} does not exist`);
|
||||
if (existsSync(backup)) fail(`the backup directory ${backup} already exists (a mutation not restored?)`);
|
||||
|
||||
const texts = new Map();
|
||||
const originals = new Map();
|
||||
const misses = [];
|
||||
for (const span of entry.spans) {
|
||||
const abs = inTree(tree, span.file);
|
||||
if (!texts.has(span.file)) {
|
||||
if (!existsSync(abs)) {
|
||||
misses.push(`${key}: ${span.file} does not exist in the tree`);
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
const t = readFileSync(abs, 'utf8');
|
||||
texts.set(span.file, t);
|
||||
originals.set(span.file, t);
|
||||
} catch (e) {
|
||||
fail(`cannot read ${abs}: ${e.code || e.message}`);
|
||||
}
|
||||
}
|
||||
const text = texts.get(span.file);
|
||||
const n = count(text, span.find);
|
||||
if (n !== 1) {
|
||||
misses.push(`${key}: ${show(span.find)} matched ${n} times in ${span.file}, not once`);
|
||||
continue;
|
||||
}
|
||||
texts.set(span.file, replaceOnce(text, span.find, span.replace));
|
||||
}
|
||||
if (misses.length > 0) {
|
||||
process.stderr.write(`${misses.map((m) => `red-mutate: ${m}`).join('\n')}\n`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
try {
|
||||
mkdirSync(join(backup, 'files'), { recursive: true });
|
||||
for (const rel of texts.keys()) {
|
||||
const dest = join(backup, 'files', rel);
|
||||
mkdirSync(dirname(dest), { recursive: true });
|
||||
writeFileSync(dest, originals.get(rel));
|
||||
}
|
||||
writeFileSync(join(backup, MANIFEST), `${JSON.stringify([...texts.keys()])}\n`);
|
||||
} catch (e) {
|
||||
fail(`cannot write the backup in ${backup}: ${e.code || e.message}`);
|
||||
}
|
||||
try {
|
||||
for (const [rel, text] of texts) writeFileSync(inTree(tree, rel), text);
|
||||
} catch (e) {
|
||||
fail(`cannot write the mutated file: ${e.code || e.message} (restore from ${backup})`);
|
||||
}
|
||||
process.stdout.write(`red-mutate: ${key} [${entry.id}] applied, ${entry.spans.length} span(s) in ${texts.size} file(s)\n`);
|
||||
}
|
||||
|
||||
function restore(argv) {
|
||||
if (argv.length !== 2) fail('restore takes two arguments');
|
||||
const [backup, tree] = argv;
|
||||
let files;
|
||||
try {
|
||||
files = JSON.parse(readFileSync(join(backup, MANIFEST), 'utf8'));
|
||||
} catch (e) {
|
||||
return fail(`cannot read the backup manifest in ${backup}: ${e.code || e.message}`);
|
||||
}
|
||||
if (!Array.isArray(files)) fail(`the backup manifest in ${backup} is not a list`);
|
||||
try {
|
||||
for (const rel of files) copyFileSync(join(backup, 'files', rel), inTree(tree, rel));
|
||||
rmSync(backup, { recursive: true, force: true });
|
||||
} catch (e) {
|
||||
fail(`cannot restore from ${backup}: ${e.code || e.message}`);
|
||||
}
|
||||
process.stdout.write(`red-mutate: restored ${files.length} file(s)\n`);
|
||||
}
|
||||
|
||||
const [cmd, ...rest] = process.argv.slice(2);
|
||||
if (cmd === 'list') await list(rest);
|
||||
else if (cmd === 'apply') await apply(rest);
|
||||
else if (cmd === 'restore') restore(rest);
|
||||
else fail(cmd ? `unknown command ${cmd}` : 'no command');
|
||||
212
tests/helpers/red-round.sh
Normal file
212
tests/helpers/red-round.sh
Normal file
|
|
@ -0,0 +1,212 @@
|
|||
#!/usr/bin/env bash
|
||||
# tests/helpers/red-round.sh
|
||||
# The shared red script of the fix rounds. A round's wrapper
|
||||
# (tests/fix-round-<n>-red.sh) hands it the round's start point, ids, the
|
||||
# other rounds' ids, a mutation module and the round's test files:
|
||||
#
|
||||
# red-round.sh --base <sha> --ids '<id> ...' --other-ids '<id> ...'
|
||||
# --mutations <module.mjs> [--require-sha256] <test file>...
|
||||
#
|
||||
# Every id of the round has tests whose names start with "[id]"; each of them
|
||||
# must fail on an assertion at the start point, or, for an id a mutation
|
||||
# names, under that mutation.
|
||||
# 1. Clone the repo into a temp directory outside it (git clone --shared
|
||||
# --no-checkout, then checkout --detach of the base; nothing registers
|
||||
# in .git/worktrees). With --require-sha256 the local git must be able to
|
||||
# create a SHA-256 repository first.
|
||||
# 2. Replace the clone's tests/ with the working tree's whole tests/ (tar),
|
||||
# helpers and fixtures included, so a new helper is no crash.
|
||||
# 3. Run every tagged test of the files once (--test-name-pattern=^\[)
|
||||
# under the red reporter and let the red judge decide: --ids, --other-ids
|
||||
# (another round's tag is never judged), --pass-ids (the ids the
|
||||
# mutation module names: their defect is proven by the mutation).
|
||||
# 4. For each mutation of the module (tests/helpers/red-mutate.mjs list):
|
||||
# apply it to the clone (every span exactly once, else
|
||||
# "broken: mutation-not-applied [id] key"), run only that id's tests
|
||||
# (^\[id\]), judge with --mutation <id>, restore.
|
||||
# The reporter, the judge, the mutate helper and the --import preload always
|
||||
# run from the working tree's tests/helpers/ by absolute path, so a mutation
|
||||
# of the clone's copy never edits the instrument that measures it.
|
||||
# Exit 0 all red, 1 a rule broken (the "broken:" lines name it), 2 a setup
|
||||
# failure ("red script: setup failure: <reason>"; a bad base, a missing file,
|
||||
# no SHA-256 git under --require-sha256, an instrument that could not run);
|
||||
# never 0 on a setup failure. The clone is removed on every exit.
|
||||
# Needs Node >= 20.14 for --test-force-exit; an older Node rejects the option
|
||||
# and the run fails closed (a setup failure).
|
||||
# bash 3.2 compatible, ASCII only, no arrays.
|
||||
|
||||
set -u
|
||||
set -f
|
||||
|
||||
setup_fail() {
|
||||
echo "red script: setup failure: $1" >&2
|
||||
exit 2
|
||||
}
|
||||
|
||||
ROOT="$(cd "$(dirname "$0")/../.." && pwd -P)" || setup_fail "cannot find the repo root from $0"
|
||||
|
||||
unset GIT_DIR GIT_WORK_TREE GIT_INDEX_FILE NODE_TEST_CONTEXT CLAUDE_CODE_SESSION_ID
|
||||
|
||||
# ---- arguments ----
|
||||
|
||||
BASE=''
|
||||
IDS=''
|
||||
OTHER_IDS=''
|
||||
MUTATIONS=''
|
||||
REQUIRE_SHA256=0
|
||||
HAVE_BASE=0
|
||||
HAVE_IDS=0
|
||||
HAVE_OTHER=0
|
||||
HAVE_MUTATIONS=0
|
||||
while [ "$#" -gt 0 ]; do
|
||||
case "$1" in
|
||||
--base) [ "$#" -ge 2 ] || setup_fail "--base needs a value"; BASE="$2"; HAVE_BASE=1; shift 2 ;;
|
||||
--ids) [ "$#" -ge 2 ] || setup_fail "--ids needs a value"; IDS="$2"; HAVE_IDS=1; shift 2 ;;
|
||||
--other-ids) [ "$#" -ge 2 ] || setup_fail "--other-ids needs a value"; OTHER_IDS="$2"; HAVE_OTHER=1; shift 2 ;;
|
||||
--mutations) [ "$#" -ge 2 ] || setup_fail "--mutations needs a value"; MUTATIONS="$2"; HAVE_MUTATIONS=1; shift 2 ;;
|
||||
--require-sha256) REQUIRE_SHA256=1; shift ;;
|
||||
--) shift; break ;;
|
||||
-*) setup_fail "unknown option $1" ;;
|
||||
*) break ;;
|
||||
esac
|
||||
done
|
||||
[ "$HAVE_BASE" -eq 1 ] && [ -n "$BASE" ] || setup_fail "--base <sha> is required"
|
||||
[ "$HAVE_IDS" -eq 1 ] && [ -n "$IDS" ] || setup_fail "--ids '<id> ...' is required"
|
||||
[ "$HAVE_OTHER" -eq 1 ] || setup_fail "--other-ids '<id> ...' is required"
|
||||
[ "$HAVE_MUTATIONS" -eq 1 ] || setup_fail "--mutations <module.mjs> is required"
|
||||
case "$MUTATIONS" in
|
||||
/*) ;;
|
||||
*) setup_fail "--mutations needs an absolute path, not '$MUTATIONS'" ;;
|
||||
esac
|
||||
[ -f "$MUTATIONS" ] || setup_fail "missing mutation module $MUTATIONS"
|
||||
[ "$#" -gt 0 ] || setup_fail "no test file given"
|
||||
|
||||
command -v node >/dev/null 2>&1 || setup_fail "node is not on PATH"
|
||||
command -v git >/dev/null 2>&1 || setup_fail "git is not on PATH"
|
||||
command -v tar >/dev/null 2>&1 || setup_fail "tar is not on PATH"
|
||||
for f in no-session-id.mjs red-reporter.mjs red-judge.mjs red-mutate.mjs; do
|
||||
[ -f "$ROOT/tests/helpers/$f" ] || setup_fail "missing $ROOT/tests/helpers/$f"
|
||||
done
|
||||
|
||||
# ---- the temp directory, outside the repo, removed on every exit ----
|
||||
|
||||
TMP="$(mktemp -d "${TMPDIR:-/tmp}/red-round.XXXXXX")" || setup_fail "mktemp -d failed"
|
||||
case "$TMP" in
|
||||
''|/) setup_fail "mktemp -d gave an unusable directory '$TMP'" ;;
|
||||
esac
|
||||
TMP_REAL="$(cd "$TMP" && pwd -P)" || setup_fail "cannot enter $TMP"
|
||||
case "$TMP_REAL/" in
|
||||
"$ROOT"/*) rmdir "$TMP" 2>/dev/null; setup_fail "the temp directory $TMP_REAL is inside the repo" ;;
|
||||
esac
|
||||
trap 'rm -rf "$TMP"' EXIT
|
||||
trap 'rm -rf "$TMP"; exit 2' INT TERM HUP
|
||||
|
||||
TREE="$TMP/tree"
|
||||
|
||||
if [ "$REQUIRE_SHA256" -eq 1 ]; then
|
||||
git init --quiet --object-format=sha256 "$TMP/sha256-probe" >/dev/null 2>&1 \
|
||||
|| setup_fail "this git cannot create a SHA-256 repository (--require-sha256)"
|
||||
rm -rf "$TMP/sha256-probe" || setup_fail "could not remove $TMP/sha256-probe"
|
||||
fi
|
||||
|
||||
# ---- 1. the start point ----
|
||||
|
||||
git clone --quiet --shared --no-checkout "$ROOT" "$TREE" || setup_fail "could not clone $ROOT"
|
||||
git -C "$TREE" checkout --quiet --detach "$BASE" || setup_fail "could not check out $BASE"
|
||||
|
||||
# ---- 2. the working tree's whole tests/ ----
|
||||
|
||||
(cd "$ROOT" && tar -cf "$TMP/tests.tar" tests) || setup_fail "could not tar tests/"
|
||||
rm -rf "$TREE/tests" || setup_fail "could not remove the start point's tests/"
|
||||
tar -xf "$TMP/tests.tar" -C "$TREE" || setup_fail "could not copy tests/ into the start point"
|
||||
|
||||
for f in "$@"; do
|
||||
[ -f "$TREE/$f" ] || setup_fail "missing test file $f"
|
||||
done
|
||||
|
||||
for id in $IDS; do
|
||||
printf '%s\n' "$id"
|
||||
done > "$TMP/ids" || setup_fail "could not write $TMP/ids"
|
||||
: > "$TMP/other-ids" || setup_fail "could not write $TMP/other-ids"
|
||||
for id in $OTHER_IDS; do
|
||||
printf '%s\n' "$id"
|
||||
done >> "$TMP/other-ids" || setup_fail "could not write $TMP/other-ids"
|
||||
|
||||
# ---- the mutations: "<key> <id>" per line; their ids are the pass-ids ----
|
||||
|
||||
node "$ROOT/tests/helpers/red-mutate.mjs" list "$MUTATIONS" > "$TMP/mutations" \
|
||||
|| setup_fail "red-mutate could not list $MUTATIONS"
|
||||
PASS_IDS=''
|
||||
while read -r key id rest <&3; do
|
||||
[ -n "$key" ] || continue
|
||||
case " $IDS " in
|
||||
*" $id "*) ;;
|
||||
*) setup_fail "mutation $key names id $id, which is not on --ids" ;;
|
||||
esac
|
||||
case ",$PASS_IDS," in
|
||||
*",$id,"*) ;;
|
||||
*) PASS_IDS="${PASS_IDS:+$PASS_IDS,}$id" ;;
|
||||
esac
|
||||
done 3< "$TMP/mutations"
|
||||
|
||||
# run_tagged <results> <node test option>... <file>...: a test run in the
|
||||
# start point under the red reporter. The positional parameters carry the
|
||||
# options and files (no arrays in bash 3.2). Node exits 0 (all passed) or 1
|
||||
# (a test failed); anything else (a bad option, a crash) is a setup failure.
|
||||
run_tagged() {
|
||||
out="$1"
|
||||
shift
|
||||
(cd "$TREE" && node --import "$ROOT/tests/helpers/no-session-id.mjs" --test --test-reporter="$ROOT/tests/helpers/red-reporter.mjs" --test-reporter-destination="$out" "$@") </dev/null >/dev/null 2>"$out.stderr"
|
||||
rc=$?
|
||||
if [ "$rc" -ne 0 ] && [ "$rc" -ne 1 ]; then
|
||||
cat "$out.stderr" >&2
|
||||
setup_fail "the test run exited $rc (needs Node >= 20.14)"
|
||||
fi
|
||||
[ -f "$out" ] || { cat "$out.stderr" >&2; setup_fail "the test run wrote no results to $out"; }
|
||||
}
|
||||
|
||||
# judge <args...>: the red judge; 2 (usage or input) is a setup failure.
|
||||
judge() {
|
||||
node "$ROOT/tests/helpers/red-judge.mjs" "$@" </dev/null
|
||||
rc=$?
|
||||
[ "$rc" -eq 2 ] && setup_fail "the red judge could not run (exit 2)"
|
||||
return "$rc"
|
||||
}
|
||||
|
||||
STATUS=0
|
||||
|
||||
# ---- 3. every tagged test at the start point, one run ----
|
||||
|
||||
echo "red script: the tagged tests at $BASE"
|
||||
run_tagged "$TMP/base.jsonl" --test-timeout=300000 --test-force-exit '--test-name-pattern=^\[' "$@"
|
||||
judge --ids "$TMP/ids" --other-ids "$TMP/other-ids" --pass-ids "$PASS_IDS" --results "$TMP/base.jsonl" || STATUS=1
|
||||
|
||||
# ---- 4. each mutation: apply, run its id's tests, judge, restore ----
|
||||
|
||||
N=0
|
||||
while read -r key id rest <&3; do
|
||||
[ -n "$key" ] || continue
|
||||
N=$((N + 1))
|
||||
BK="$TMP/backup-$N"
|
||||
echo "red script: mutation $key [$id]"
|
||||
node "$ROOT/tests/helpers/red-mutate.mjs" apply "$MUTATIONS" "$key" "$TREE" "$BK" </dev/null >/dev/null
|
||||
rc=$?
|
||||
if [ "$rc" -eq 0 ]; then
|
||||
run_tagged "$TMP/mutation-$N.jsonl" --test-timeout=300000 --test-force-exit "--test-name-pattern=^\\[$id\\]" "$@"
|
||||
judge --ids "$TMP/ids" --other-ids "$TMP/other-ids" --results "$TMP/mutation-$N.jsonl" --mutation "$id" || STATUS=1
|
||||
node "$ROOT/tests/helpers/red-mutate.mjs" restore "$BK" "$TREE" </dev/null >/dev/null \
|
||||
|| setup_fail "could not restore the files of mutation $key"
|
||||
elif [ "$rc" -eq 1 ]; then
|
||||
echo "broken: mutation-not-applied [$id] $key"
|
||||
STATUS=1
|
||||
else
|
||||
setup_fail "red-mutate could not apply $key (exit $rc)"
|
||||
fi
|
||||
done 3< "$TMP/mutations"
|
||||
|
||||
if [ "$STATUS" -eq 0 ]; then
|
||||
echo "red script: ok - every id red at $BASE, $N mutation(s) red"
|
||||
else
|
||||
echo "red script: FAIL - a rule is broken (see the broken: lines above)" >&2
|
||||
fi
|
||||
exit "$STATUS"
|
||||
341
tests/lib/red-script.test.mjs
Normal file
341
tests/lib/red-script.test.mjs
Normal file
|
|
@ -0,0 +1,341 @@
|
|||
// tests/lib/red-script.test.mjs
|
||||
// The shared red script (tests/helpers/red-round.sh) and its mutate helper
|
||||
// (tests/helpers/red-mutate.mjs). A round's wrapper hands the script a base
|
||||
// commit, the round's ids, the other rounds' ids, a mutation module and test
|
||||
// files; the script clones the base outside the repo, replaces the clone's
|
||||
// tests/ with the working tree's, runs every tagged test once
|
||||
// (--test-name-pattern=^\[), lets tests/helpers/red-judge.mjs decide, then
|
||||
// applies each named mutation, runs only that id's tests, judges and restores.
|
||||
// A script that always says "ok" would prove nothing, so it is run here on a
|
||||
// small fixture repo whose answer is known:
|
||||
// - lib/one.mjs returns 0 at BASE and 1 at HEAD: [X1] is red at BASE only;
|
||||
// - lib/two.mjs is right at both: [X2] passes at BASE and is red only under
|
||||
// the fixture's mutations (so it is a pass-id);
|
||||
// - tests/other.test.mjs holds a passing [Z9] (another round's id);
|
||||
// - tests/stray.test.mjs holds a mistyped [X 1] and an unlisted [Q9].
|
||||
// The fixture repo is seeded once in os.tmpdir() with the script and helpers
|
||||
// copied from this file's own ROOT/tests/helpers, and copied per test.
|
||||
//
|
||||
// Every test first asserts that the script exists, so each fails on an
|
||||
// assertion while it does not. The script and its runs need Node >= 20.14
|
||||
// (--test-force-exit); on an older Node each test is skipped.
|
||||
|
||||
import { after, test } from 'node:test';
|
||||
import { strict as assert } from 'node:assert';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import {
|
||||
cpSync, existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, realpathSync, rmSync, writeFileSync,
|
||||
} from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { delimiter, dirname, join } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { childEnv } from '../helpers/child-env.mjs';
|
||||
import { pathWith } from '../helpers/bin-path.mjs';
|
||||
|
||||
const ROOT = join(dirname(fileURLToPath(import.meta.url)), '..', '..');
|
||||
const HELPERS = join(ROOT, 'tests', 'helpers');
|
||||
const SCRIPT = join(HELPERS, 'red-round.sh');
|
||||
const INSTRUMENTS = ['red-round.sh', 'red-mutate.mjs', 'red-judge.mjs', 'red-reporter.mjs', 'no-session-id.mjs'];
|
||||
const CHILD_TIMEOUT = 120000;
|
||||
|
||||
const [NODE_MAJOR, NODE_MINOR] = process.versions.node.split('.').map(Number);
|
||||
const nodeTooOld = NODE_MAJOR < 20 || (NODE_MAJOR === 20 && NODE_MINOR < 14);
|
||||
|
||||
const cleanup = [];
|
||||
after(() => {
|
||||
for (const d of cleanup) rmSync(d, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
/** A new real-path temp directory, removed in after(). */
|
||||
function tmp(prefix) {
|
||||
const d = realpathSync(mkdtempSync(join(tmpdir(), prefix)));
|
||||
cleanup.push(d);
|
||||
return d;
|
||||
}
|
||||
|
||||
/** The environment of a git call or a script run: no inherited git or test-runner context. */
|
||||
const env = (over = {}) => childEnv({
|
||||
GIT_DIR: undefined, GIT_WORK_TREE: undefined, GIT_INDEX_FILE: undefined, NODE_TEST_CONTEXT: undefined, ...over,
|
||||
});
|
||||
|
||||
function git(cwd, ...args) {
|
||||
const r = spawnSync('git', args, { cwd, encoding: 'utf8', env: env() });
|
||||
if (r.status !== 0) throw new Error(`git ${args.join(' ')} failed in ${cwd}: ${r.stderr}`);
|
||||
return r.stdout.trim();
|
||||
}
|
||||
|
||||
/** Every test starts here: the script and its mutate helper exist. */
|
||||
function requireScript() {
|
||||
assert.ok(existsSync(SCRIPT), `the shared red script ${SCRIPT} does not exist`);
|
||||
assert.ok(existsSync(join(HELPERS, 'red-mutate.mjs')), 'tests/helpers/red-mutate.mjs does not exist');
|
||||
}
|
||||
|
||||
const X_TEST = `import { test } from 'node:test';
|
||||
import { strict as assert } from 'node:assert';
|
||||
import { appendFileSync } from 'node:fs';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { one } from '../lib/one.mjs';
|
||||
import { two } from '../lib/two.mjs';
|
||||
if (process.env.RS_SEEN_FILE) appendFileSync(process.env.RS_SEEN_FILE, fileURLToPath(import.meta.url) + '\\n');
|
||||
test('[X1] one is 1', () => { assert.equal(one(), 1); });
|
||||
test('[X2] two is 2', () => { assert.equal(two(), 2); });
|
||||
test('an untagged test', () => { assert.ok(true); });
|
||||
`;
|
||||
const OTHER_TEST = `import { test } from 'node:test';
|
||||
import { strict as assert } from 'node:assert';
|
||||
test('[Z9] another round test that passes', () => { assert.ok(true); });
|
||||
`;
|
||||
const STRAY_TEST = `import { test } from 'node:test';
|
||||
import { strict as assert } from 'node:assert';
|
||||
test('[X 1] a mistyped tag', () => { assert.equal(1, 2); });
|
||||
test('[Q9] an unlisted tag', () => { assert.equal(1, 2); });
|
||||
`;
|
||||
const TWO = '// two: the second value\nexport function two() { return 2; } // two\n';
|
||||
const MUTATIONS = `export const MUTATIONS = [
|
||||
{ key: 'X2-A', id: 'X2', what: 'two returns 3', spans: [{ file: 'lib/two.mjs', find: 'return 2;', replace: 'return 3;' }] },
|
||||
{ key: 'X2-B', id: 'X2', what: 'two returns 4 (finds return 2 only after X2-A was restored)', spans: [{ file: 'lib/two.mjs', find: /return 2;/g, replace: 'return 4;' }] },
|
||||
];
|
||||
`;
|
||||
|
||||
let seeded = null;
|
||||
|
||||
/** The fixture repo, seeded once: BASE (one() wrong), HEAD (one() right), helpers in the working tree. */
|
||||
function seed() {
|
||||
if (seeded) return seeded;
|
||||
const dir = join(tmp('red-script-seed-'), 'repo');
|
||||
mkdirSync(join(dir, 'lib'), { recursive: true });
|
||||
mkdirSync(join(dir, 'tests', 'helpers'), { recursive: true });
|
||||
git(dir, 'init', '-q');
|
||||
git(dir, 'config', 'user.name', 'Test');
|
||||
git(dir, 'config', 'user.email', 'test@example.invalid');
|
||||
git(dir, 'config', 'commit.gpgsign', 'false');
|
||||
git(dir, 'config', 'core.hooksPath', '');
|
||||
writeFileSync(join(dir, 'lib', 'one.mjs'), 'export function one() { return 0; }\n');
|
||||
writeFileSync(join(dir, 'lib', 'two.mjs'), TWO);
|
||||
writeFileSync(join(dir, 'tests', 'x.test.mjs'), X_TEST);
|
||||
writeFileSync(join(dir, 'tests', 'other.test.mjs'), OTHER_TEST);
|
||||
writeFileSync(join(dir, 'tests', 'stray.test.mjs'), STRAY_TEST);
|
||||
git(dir, 'add', 'lib', 'tests');
|
||||
git(dir, 'commit', '-q', '-m', 'base: one() is wrong');
|
||||
const base = git(dir, 'rev-parse', 'HEAD');
|
||||
writeFileSync(join(dir, 'lib', 'one.mjs'), 'export function one() { return 1; }\n');
|
||||
git(dir, 'add', 'lib/one.mjs');
|
||||
git(dir, 'commit', '-q', '-m', 'head: one() is right');
|
||||
for (const name of INSTRUMENTS) {
|
||||
if (existsSync(join(HELPERS, name))) cpSync(join(HELPERS, name), join(dir, 'tests', 'helpers', name));
|
||||
}
|
||||
writeFileSync(join(dir, 'mutations.mjs'), MUTATIONS);
|
||||
seeded = { dir, base };
|
||||
return seeded;
|
||||
}
|
||||
|
||||
/** A fresh copy of the seeded fixture repo. */
|
||||
function fixture() {
|
||||
const s = seed();
|
||||
const dir = join(tmp('red-script-fx-'), 'repo');
|
||||
cpSync(s.dir, dir, { recursive: true });
|
||||
return { dir, base: s.base, script: join(dir, 'tests', 'helpers', 'red-round.sh'), mutate: join(dir, 'tests', 'helpers', 'red-mutate.mjs') };
|
||||
}
|
||||
|
||||
/** The script on the fixture, from outside it. */
|
||||
function run(fx, args, over = {}) {
|
||||
const r = spawnSync('/bin/bash', [fx.script, ...args],
|
||||
{ cwd: tmpdir(), encoding: 'utf8', timeout: CHILD_TIMEOUT, env: env(over) });
|
||||
return { status: r.status, out: `${r.stdout}${r.stderr}`, stderr: r.stderr };
|
||||
}
|
||||
|
||||
/** The usual arguments: BASE, ids X1 X2, other id Z9, the fixture's mutations, then the files. */
|
||||
function args(fx, files, { ids = 'X1 X2', other = 'Z9', mutations = join(fx.dir, 'mutations.mjs') } = {}) {
|
||||
return ['--base', fx.base, '--ids', ids, '--other-ids', other, '--mutations', mutations, ...files];
|
||||
}
|
||||
|
||||
/** The mutate helper on the fixture. */
|
||||
function mutate(fx, ...rest) {
|
||||
const r = spawnSync(process.execPath, [fx.mutate, ...rest],
|
||||
{ encoding: 'utf8', timeout: CHILD_TIMEOUT, env: env() });
|
||||
return { status: r.status, out: `${r.stdout}${r.stderr}` };
|
||||
}
|
||||
|
||||
/** Every instrument path on the script's node lines: --import, --test-reporter= and the script right after node. */
|
||||
function instrumentScan(text) {
|
||||
const found = [];
|
||||
for (const line of text.replace(/\\\n/g, ' ').split('\n')) {
|
||||
if (/^\s*#/.test(line) || !/(^|[\s;(&|])node\s/.test(line)) continue;
|
||||
for (const m of line.matchAll(/(?:--import[=\s]+|--test-reporter=|\bnode\s+)("?)([^\s"']+\.mjs)\1/g)) found.push(m[2]);
|
||||
}
|
||||
return found;
|
||||
}
|
||||
const notFromWorkingTree = (paths) => paths.filter((p) => !p.startsWith('$ROOT/tests/helpers/'));
|
||||
|
||||
/** The script lines that call run_tagged (not its definition). */
|
||||
const runLines = (text) => text.split('\n').filter((l) => /^\s*run_tagged\s/.test(l));
|
||||
|
||||
test('[RS] a test red at BASE and green at HEAD exits 0', { skip: nodeTooOld && 'needs Node >= 20.14 for --test-force-exit' }, () => {
|
||||
requireScript();
|
||||
const fx = fixture();
|
||||
const r = run(fx, args(fx, ['tests/x.test.mjs']));
|
||||
assert.equal(r.status, 0, r.out);
|
||||
assert.match(r.out, /^red script: ok/m, r.out);
|
||||
assert.match(r.out, /^red: \[X1\] 1 test\(s\)$/m, r.out);
|
||||
assert.match(r.out, /^pass: \[X2\] 1 test\(s\) \(--pass-ids\)$/m, r.out);
|
||||
assert.equal((r.out.match(/^red: \[X2\] 1 test\(s\)$/gm) || []).length, 2, `both mutations X2-A and X2-B ran red (X2-B finds its span only after X2-A was restored)\n${r.out}`);
|
||||
assert.doesNotMatch(r.out, /^broken:/m, r.out);
|
||||
});
|
||||
|
||||
test('[RS] a test tagged with an other-round id is not judged', { skip: nodeTooOld && 'needs Node >= 20.14 for --test-force-exit' }, () => {
|
||||
requireScript();
|
||||
const fx = fixture();
|
||||
const files = ['tests/x.test.mjs', 'tests/other.test.mjs'];
|
||||
const r = run(fx, args(fx, files, { other: 'Z9' }));
|
||||
assert.equal(r.status, 0, r.out);
|
||||
assert.doesNotMatch(r.out, /\[Z9\]/, r.out);
|
||||
const control = run(fx, args(fx, files, { other: 'Z8' }));
|
||||
assert.equal(control.status, 1, `control: with Z9 not on --other-ids the run must fail\n${control.out}`);
|
||||
assert.match(control.out, /^broken: unlisted-tag \[Z9\]/m, control.out);
|
||||
});
|
||||
|
||||
test('[RS] a mutation that matches no span exits 1 and writes nothing', { skip: nodeTooOld && 'needs Node >= 20.14 for --test-force-exit' }, () => {
|
||||
requireScript();
|
||||
const fx = fixture();
|
||||
const mod = join(fx.dir, 'nomatch.mjs');
|
||||
writeFileSync(mod, `export const MUTATIONS = [
|
||||
{ key: 'X2-none', id: 'X2', what: 'a span that is not there', spans: [{ file: 'lib/two.mjs', find: 'return 99;', replace: 'return 3;' }] },
|
||||
{ key: 'X2-half', id: 'X2', what: 'one span matches, one does not', spans: [{ file: 'lib/one.mjs', find: 'return 1;', replace: 'return 5;' }, { file: 'lib/two.mjs', find: 'return 99;', replace: 'return 3;' }] },
|
||||
];
|
||||
`);
|
||||
const r = run(fx, args(fx, ['tests/x.test.mjs'], { mutations: mod }));
|
||||
assert.equal(r.status, 1, r.out);
|
||||
assert.match(r.out, /^broken: mutation-not-applied \[X2\] X2-none$/m, r.out);
|
||||
assert.doesNotMatch(r.out, /^red script: ok/m, r.out);
|
||||
|
||||
const oneBefore = readFileSync(join(fx.dir, 'lib', 'one.mjs'), 'utf8');
|
||||
const twoBefore = readFileSync(join(fx.dir, 'lib', 'two.mjs'), 'utf8');
|
||||
const backup = join(tmp('red-script-bk-'), 'backup');
|
||||
const m = mutate(fx, 'apply', mod, 'X2-half', fx.dir, backup);
|
||||
assert.equal(m.status, 1, m.out);
|
||||
assert.equal(readFileSync(join(fx.dir, 'lib', 'one.mjs'), 'utf8'), oneBefore, 'the span that matched was written although another did not match');
|
||||
assert.equal(readFileSync(join(fx.dir, 'lib', 'two.mjs'), 'utf8'), twoBefore);
|
||||
assert.equal(existsSync(backup), false, 'a mutation that was not applied wrote a backup');
|
||||
});
|
||||
|
||||
test('[RS] a span that matches twice exits 1 and writes nothing', { skip: nodeTooOld && 'needs Node >= 20.14 for --test-force-exit' }, () => {
|
||||
requireScript();
|
||||
const fx = fixture();
|
||||
const mod = join(fx.dir, 'twice.mjs');
|
||||
writeFileSync(mod, `export const MUTATIONS = [
|
||||
{ key: 'X2-twice', id: 'X2', what: 'a comment that is there twice', spans: [{ file: 'lib/two.mjs', find: '// two', replace: '' }] },
|
||||
{ key: 'X2-twice-re', id: 'X2', what: 'one span once, one twice', spans: [{ file: 'lib/one.mjs', find: 'return 1;', replace: 'return 5;' }, { file: 'lib/two.mjs', find: /\\/\\/ two/g, replace: '' }] },
|
||||
];
|
||||
`);
|
||||
const r = run(fx, args(fx, ['tests/x.test.mjs'], { mutations: mod }));
|
||||
assert.equal(r.status, 1, r.out);
|
||||
assert.match(r.out, /^broken: mutation-not-applied \[X2\] X2-twice$/m, r.out);
|
||||
assert.match(r.out, /^broken: mutation-not-applied \[X2\] X2-twice-re$/m, r.out);
|
||||
|
||||
const oneBefore = readFileSync(join(fx.dir, 'lib', 'one.mjs'), 'utf8');
|
||||
const backup = join(tmp('red-script-bk-'), 'backup');
|
||||
const m = mutate(fx, 'apply', mod, 'X2-twice-re', fx.dir, backup);
|
||||
assert.equal(m.status, 1, m.out);
|
||||
assert.equal(readFileSync(join(fx.dir, 'lib', 'one.mjs'), 'utf8'), oneBefore, 'the span that matched once was written although another matched twice');
|
||||
assert.equal(readFileSync(join(fx.dir, 'lib', 'two.mjs'), 'utf8'), TWO);
|
||||
assert.equal(existsSync(backup), false, 'a mutation that was not applied wrote a backup');
|
||||
});
|
||||
|
||||
test('[RS] a base that is no commit exits 2', { skip: nodeTooOld && 'needs Node >= 20.14 for --test-force-exit' }, () => {
|
||||
requireScript();
|
||||
const fx = fixture();
|
||||
const r = run(fx, ['--base', 'f'.repeat(40), '--ids', 'X1 X2', '--other-ids', 'Z9', '--mutations', join(fx.dir, 'mutations.mjs'), 'tests/x.test.mjs']);
|
||||
assert.equal(r.status, 2, r.out);
|
||||
assert.match(r.stderr, /^red script: setup failure: /m, r.out);
|
||||
assert.doesNotMatch(r.out, /^red script: ok/m, r.out);
|
||||
});
|
||||
|
||||
test('[RS] the clone is removed after the run', { skip: nodeTooOld && 'needs Node >= 20.14 for --test-force-exit' }, () => {
|
||||
requireScript();
|
||||
const fx = fixture();
|
||||
const td = tmp('red-script-tmpdir-');
|
||||
const seen = join(tmp('red-script-seen-'), 'seen');
|
||||
const r = run(fx, args(fx, ['tests/x.test.mjs']), { TMPDIR: td, RS_SEEN_FILE: seen });
|
||||
assert.equal(r.status, 0, r.out);
|
||||
const paths = existsSync(seen) ? readFileSync(seen, 'utf8').split('\n').filter(Boolean) : [];
|
||||
assert.ok(paths.length > 0, 'the test file never ran, so the clone was never seen');
|
||||
for (const p of paths) {
|
||||
assert.ok(p.startsWith(`${td}/`), `the test ran at ${p}, not in a clone under TMPDIR ${td}`);
|
||||
assert.equal(existsSync(p), false, `${p} is still there after the run`);
|
||||
}
|
||||
assert.deepEqual(readdirSync(td), [], 'the temp directory still holds files after the run');
|
||||
});
|
||||
|
||||
test('[RS] the instruments run from the working tree by absolute path', { skip: nodeTooOld && 'needs Node >= 20.14 for --test-force-exit' }, () => {
|
||||
requireScript();
|
||||
const text = readFileSync(SCRIPT, 'utf8');
|
||||
const paths = instrumentScan(text);
|
||||
for (const name of ['no-session-id.mjs', 'red-reporter.mjs', 'red-judge.mjs', 'red-mutate.mjs']) {
|
||||
assert.ok(paths.some((p) => p.endsWith(`/${name}`)), `no node line runs ${name}; found: ${paths.join(', ')}`);
|
||||
}
|
||||
assert.deepEqual(notFromWorkingTree(paths), [], 'an instrument is not run as $ROOT/tests/helpers/...');
|
||||
});
|
||||
|
||||
test('[RS] the instrument scan reports a planted ./tests/helpers/ line', { skip: nodeTooOld && 'needs Node >= 20.14 for --test-force-exit' }, () => {
|
||||
requireScript();
|
||||
const text = readFileSync(SCRIPT, 'utf8');
|
||||
const planted = text.replace(/--test-reporter=("?)\$ROOT\/tests\/helpers\/red-reporter\.mjs\1/, '--test-reporter=./tests/helpers/red-reporter.mjs');
|
||||
assert.notEqual(planted, text, 'control: the reporter path to plant over was not found');
|
||||
assert.deepEqual(notFromWorkingTree(instrumentScan(planted)), ['./tests/helpers/red-reporter.mjs'],
|
||||
'control: the scan does not report a planted relative reporter path');
|
||||
});
|
||||
|
||||
test('[RS] --require-sha256 with a git that cannot create a SHA-256 repository exits 2 with a setup-failure line', { skip: nodeTooOld && 'needs Node >= 20.14 for --test-force-exit' }, () => {
|
||||
requireScript();
|
||||
const fx = fixture();
|
||||
const real = pathWith(['git']);
|
||||
cleanup.push(real.path);
|
||||
const bin = tmp('red-script-fakegit-');
|
||||
const fake = join(bin, 'git');
|
||||
writeFileSync(fake, `#!/bin/sh
|
||||
case " $* " in
|
||||
*" init "*"--object-format=sha256"*) echo "fake git: no SHA-256 here" >&2; exit 128 ;;
|
||||
esac
|
||||
exec '${join(real.path, 'git')}' "$@"
|
||||
`, { mode: 0o755 });
|
||||
const PATH = `${bin}${delimiter}${process.env.PATH}`;
|
||||
const probe = spawnSync(fake, ['init', '--quiet', '--object-format=sha256', join(bin, 'probe')], { encoding: 'utf8', env: env({ PATH }) });
|
||||
assert.notEqual(probe.status, 0, 'control: the fake git created a SHA-256 repository');
|
||||
assert.equal(spawnSync(fake, ['--version'], { encoding: 'utf8', env: env({ PATH }) }).status, 0, 'control: the fake git does not run the real one');
|
||||
|
||||
const r = run(fx, [...args(fx, ['tests/x.test.mjs']).slice(0, 8), '--require-sha256', 'tests/x.test.mjs'], { PATH });
|
||||
assert.equal(r.status, 2, r.out);
|
||||
assert.match(r.stderr, /^red script: setup failure: .*SHA-256/m, r.out);
|
||||
assert.doesNotMatch(r.out, /^red script: ok/m, r.out);
|
||||
});
|
||||
|
||||
test('[E23a] a mistyped [X 1] and an unlisted [Q9] are reported and the run exits 1', { skip: nodeTooOld && 'needs Node >= 20.14 for --test-force-exit' }, () => {
|
||||
requireScript();
|
||||
const fx = fixture();
|
||||
const r = run(fx, args(fx, ['tests/x.test.mjs', 'tests/stray.test.mjs']));
|
||||
assert.equal(r.status, 1, r.out);
|
||||
assert.match(r.out, /^broken: unreadable-tag \[X 1\] a mistyped tag$/m, r.out);
|
||||
assert.match(r.out, /^broken: unlisted-tag \[Q9\] \[Q9\] an unlisted tag$/m, r.out);
|
||||
});
|
||||
|
||||
test('[E23a] the base run selects every tag with one ^\\[ pattern', { skip: nodeTooOld && 'needs Node >= 20.14 for --test-force-exit' }, () => {
|
||||
requireScript();
|
||||
const text = readFileSync(SCRIPT, 'utf8');
|
||||
const base = runLines(text).filter((l) => l.includes('run_tagged "$TMP/base.jsonl"'));
|
||||
assert.equal(base.length, 1, `not exactly one base-run line:\n${base.join('\n')}`);
|
||||
assert.ok(base[0].includes(` '--test-name-pattern=^\\['`), `the base run does not select with '--test-name-pattern=^\\[':\n${base[0]}`);
|
||||
assert.equal((base[0].match(/--test-name-pattern/g) || []).length, 1, `the base run carries more than one name pattern:\n${base[0]}`);
|
||||
});
|
||||
|
||||
test('[E23b] the base run carries --test-timeout and --test-force-exit', { skip: nodeTooOld && 'needs Node >= 20.14 for --test-force-exit' }, () => {
|
||||
requireScript();
|
||||
const text = readFileSync(SCRIPT, 'utf8');
|
||||
const lines = runLines(text);
|
||||
assert.ok(lines.some((l) => l.includes('run_tagged "$TMP/base.jsonl"')), 'no base-run line');
|
||||
assert.ok(lines.length >= 2, `the mutation run line is missing:\n${lines.join('\n')}`);
|
||||
for (const l of lines) {
|
||||
assert.match(l, / --test-timeout=[1-9]\d*(\s|$)/, `a run without --test-timeout:\n${l}`);
|
||||
assert.match(l, / --test-force-exit(\s|$)/, `a run without --test-force-exit:\n${l}`);
|
||||
}
|
||||
assert.match(text, /^#.*Node >= 20\.14/m, 'the header does not say that Node >= 20.14 is needed');
|
||||
});
|
||||
Loading…
Add table
Add a link
Reference in a new issue