// tests/lib/red-script.test.mjs // The shared red script (tests/helpers/red-round.sh) and its mutate helper // (tests/helpers/red-mutate.mjs). A round's wrapper hands the script a base // commit, the round's ids, the other rounds' ids, a mutation module and test // files; the script clones the base outside the repo, replaces the clone's // tests/ with the working tree's, runs every tagged test once // (--test-name-pattern=^\[), lets tests/helpers/red-judge.mjs decide, then // applies each named mutation, runs only that id's tests, judges and restores. // A script that always says "ok" would prove nothing, so it is run here on a // small fixture repo whose answer is known: // - lib/one.mjs returns 0 at BASE and 1 at HEAD: [X1] is red at BASE only; // - lib/two.mjs is right at both: [X2] passes at BASE and is red only under // the fixture's mutations (so it is a pass-id); // - tests/other.test.mjs holds a passing [Z9] (another round's id); // - tests/stray.test.mjs holds a mistyped [X 1] and an unlisted [Q9]. // The fixture repo is seeded once in os.tmpdir() with the script and helpers // copied from this file's own ROOT/tests/helpers, and copied per test. // // Every test first asserts that the script exists, so each fails on an // assertion while it does not. The script and its runs need Node >= 20.14 // (--test-force-exit); on an older Node each test is skipped. import { after, test } from 'node:test'; import { strict as assert } from 'node:assert'; import { spawnSync } from 'node:child_process'; import { cpSync, existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, realpathSync, rmSync, writeFileSync, } from 'node:fs'; import { tmpdir } from 'node:os'; import { delimiter, dirname, join } from 'node:path'; import { fileURLToPath } from 'node:url'; import { childEnv } from '../helpers/child-env.mjs'; import { pathWith } from '../helpers/bin-path.mjs'; const ROOT = join(dirname(fileURLToPath(import.meta.url)), '..', '..'); const HELPERS = join(ROOT, 'tests', 'helpers'); const SCRIPT = join(HELPERS, 'red-round.sh'); const INSTRUMENTS = ['red-round.sh', 'red-mutate.mjs', 'red-judge.mjs', 'red-reporter.mjs', 'no-session-id.mjs']; const CHILD_TIMEOUT = 120000; const [NODE_MAJOR, NODE_MINOR] = process.versions.node.split('.').map(Number); const nodeTooOld = NODE_MAJOR < 20 || (NODE_MAJOR === 20 && NODE_MINOR < 14); const cleanup = []; after(() => { for (const d of cleanup) rmSync(d, { recursive: true, force: true }); }); /** A new real-path temp directory, removed in after(). */ function tmp(prefix) { const d = realpathSync(mkdtempSync(join(tmpdir(), prefix))); cleanup.push(d); return d; } /** The environment of a git call or a script run: no inherited git or test-runner context. */ const env = (over = {}) => childEnv({ GIT_DIR: undefined, GIT_WORK_TREE: undefined, GIT_INDEX_FILE: undefined, NODE_TEST_CONTEXT: undefined, ...over, }); function git(cwd, ...args) { const r = spawnSync('git', args, { cwd, encoding: 'utf8', env: env() }); if (r.status !== 0) throw new Error(`git ${args.join(' ')} failed in ${cwd}: ${r.stderr}`); return r.stdout.trim(); } /** Every test starts here: the script and its mutate helper exist. */ function requireScript() { assert.ok(existsSync(SCRIPT), `the shared red script ${SCRIPT} does not exist`); assert.ok(existsSync(join(HELPERS, 'red-mutate.mjs')), 'tests/helpers/red-mutate.mjs does not exist'); } const X_TEST = `import { test } from 'node:test'; import { strict as assert } from 'node:assert'; import { appendFileSync } from 'node:fs'; import { fileURLToPath } from 'node:url'; import { one } from '../lib/one.mjs'; import { two } from '../lib/two.mjs'; if (process.env.RS_SEEN_FILE) appendFileSync(process.env.RS_SEEN_FILE, fileURLToPath(import.meta.url) + '\\n'); test('[X1] one is 1', () => { assert.equal(one(), 1); }); test('[X2] two is 2', () => { assert.equal(two(), 2); }); test('an untagged test', () => { assert.ok(true); }); `; const OTHER_TEST = `import { test } from 'node:test'; import { strict as assert } from 'node:assert'; test('[Z9] another round test that passes', () => { assert.ok(true); }); `; const STRAY_TEST = `import { test } from 'node:test'; import { strict as assert } from 'node:assert'; test('[X 1] a mistyped tag', () => { assert.equal(1, 2); }); test('[Q9] an unlisted tag', () => { assert.equal(1, 2); }); `; const TWO = '// two: the second value\nexport function two() { return 2; } // two\n'; const MUTATIONS = `export const MUTATIONS = [ { key: 'X2-A', id: 'X2', what: 'two returns 3', spans: [{ file: 'lib/two.mjs', find: 'return 2;', replace: 'return 3;' }] }, { key: 'X2-B', id: 'X2', what: 'two returns 4 (finds return 2 only after X2-A was restored)', spans: [{ file: 'lib/two.mjs', find: /return 2;/g, replace: 'return 4;' }] }, ]; `; let seeded = null; /** The fixture repo, seeded once: BASE (one() wrong), HEAD (one() right), helpers in the working tree. */ function seed() { if (seeded) return seeded; const dir = join(tmp('red-script-seed-'), 'repo'); mkdirSync(join(dir, 'lib'), { recursive: true }); mkdirSync(join(dir, 'tests', 'helpers'), { recursive: true }); git(dir, 'init', '-q'); git(dir, 'config', 'user.name', 'Test'); git(dir, 'config', 'user.email', 'test@example.invalid'); git(dir, 'config', 'commit.gpgsign', 'false'); git(dir, 'config', 'core.hooksPath', ''); writeFileSync(join(dir, 'lib', 'one.mjs'), 'export function one() { return 0; }\n'); writeFileSync(join(dir, 'lib', 'two.mjs'), TWO); writeFileSync(join(dir, 'tests', 'x.test.mjs'), X_TEST); writeFileSync(join(dir, 'tests', 'other.test.mjs'), OTHER_TEST); writeFileSync(join(dir, 'tests', 'stray.test.mjs'), STRAY_TEST); git(dir, 'add', 'lib', 'tests'); git(dir, 'commit', '-q', '-m', 'base: one() is wrong'); const base = git(dir, 'rev-parse', 'HEAD'); writeFileSync(join(dir, 'lib', 'one.mjs'), 'export function one() { return 1; }\n'); git(dir, 'add', 'lib/one.mjs'); git(dir, 'commit', '-q', '-m', 'head: one() is right'); for (const name of INSTRUMENTS) { if (existsSync(join(HELPERS, name))) cpSync(join(HELPERS, name), join(dir, 'tests', 'helpers', name)); } writeFileSync(join(dir, 'mutations.mjs'), MUTATIONS); seeded = { dir, base }; return seeded; } /** A fresh copy of the seeded fixture repo. */ function fixture() { const s = seed(); const dir = join(tmp('red-script-fx-'), 'repo'); cpSync(s.dir, dir, { recursive: true }); return { dir, base: s.base, script: join(dir, 'tests', 'helpers', 'red-round.sh'), mutate: join(dir, 'tests', 'helpers', 'red-mutate.mjs') }; } /** The script on the fixture, from outside it. */ function run(fx, args, over = {}) { const r = spawnSync('/bin/bash', [fx.script, ...args], { cwd: tmpdir(), encoding: 'utf8', timeout: CHILD_TIMEOUT, env: env(over) }); return { status: r.status, out: `${r.stdout}${r.stderr}`, stderr: r.stderr }; } /** The usual arguments: BASE, ids X1 X2, other id Z9, the fixture's mutations, then the files. */ function args(fx, files, { ids = 'X1 X2', other = 'Z9', mutations = join(fx.dir, 'mutations.mjs') } = {}) { return ['--base', fx.base, '--ids', ids, '--other-ids', other, '--mutations', mutations, ...files]; } /** The mutate helper on the fixture. */ function mutate(fx, ...rest) { const r = spawnSync(process.execPath, [fx.mutate, ...rest], { encoding: 'utf8', timeout: CHILD_TIMEOUT, env: env() }); return { status: r.status, out: `${r.stdout}${r.stderr}` }; } /** Every instrument path on the script's node lines: --import, --test-reporter= and the script right after node. */ function instrumentScan(text) { const found = []; for (const line of text.replace(/\\\n/g, ' ').split('\n')) { if (/^\s*#/.test(line) || !/(^|[\s;(&|])node\s/.test(line)) continue; for (const m of line.matchAll(/(?:--import[=\s]+|--test-reporter=|\bnode\s+)("?)([^\s"']+\.mjs)\1/g)) found.push(m[2]); } return found; } const notFromWorkingTree = (paths) => paths.filter((p) => !p.startsWith('$ROOT/tests/helpers/')); /** The script lines that call run_tagged (not its definition). */ const runLines = (text) => text.split('\n').filter((l) => /^\s*run_tagged\s/.test(l)); test('[RS] a test red at BASE and green at HEAD exits 0', { skip: nodeTooOld && 'needs Node >= 20.14 for --test-force-exit' }, () => { requireScript(); const fx = fixture(); const r = run(fx, args(fx, ['tests/x.test.mjs'])); assert.equal(r.status, 0, r.out); assert.match(r.out, /^red script: ok/m, r.out); assert.match(r.out, /^red: \[X1\] 1 test\(s\)$/m, r.out); assert.match(r.out, /^pass: \[X2\] 1 test\(s\) \(--pass-ids\)$/m, r.out); assert.equal((r.out.match(/^red: \[X2\] 1 test\(s\)$/gm) || []).length, 2, `both mutations X2-A and X2-B ran red (X2-B finds its span only after X2-A was restored)\n${r.out}`); assert.doesNotMatch(r.out, /^broken:/m, r.out); }); test('[RS] a test tagged with an other-round id is not judged', { skip: nodeTooOld && 'needs Node >= 20.14 for --test-force-exit' }, () => { requireScript(); const fx = fixture(); const files = ['tests/x.test.mjs', 'tests/other.test.mjs']; const r = run(fx, args(fx, files, { other: 'Z9' })); assert.equal(r.status, 0, r.out); assert.doesNotMatch(r.out, /\[Z9\]/, r.out); const control = run(fx, args(fx, files, { other: 'Z8' })); assert.equal(control.status, 1, `control: with Z9 not on --other-ids the run must fail\n${control.out}`); assert.match(control.out, /^broken: unlisted-tag \[Z9\]/m, control.out); }); test('[RS] a mutation that matches no span exits 1 and writes nothing', { skip: nodeTooOld && 'needs Node >= 20.14 for --test-force-exit' }, () => { requireScript(); const fx = fixture(); const mod = join(fx.dir, 'nomatch.mjs'); writeFileSync(mod, `export const MUTATIONS = [ { key: 'X2-none', id: 'X2', what: 'a span that is not there', spans: [{ file: 'lib/two.mjs', find: 'return 99;', replace: 'return 3;' }] }, { key: 'X2-half', id: 'X2', what: 'one span matches, one does not', spans: [{ file: 'lib/one.mjs', find: 'return 1;', replace: 'return 5;' }, { file: 'lib/two.mjs', find: 'return 99;', replace: 'return 3;' }] }, ]; `); const r = run(fx, args(fx, ['tests/x.test.mjs'], { mutations: mod })); assert.equal(r.status, 1, r.out); assert.match(r.out, /^broken: mutation-not-applied \[X2\] X2-none$/m, r.out); assert.doesNotMatch(r.out, /^red script: ok/m, r.out); const oneBefore = readFileSync(join(fx.dir, 'lib', 'one.mjs'), 'utf8'); const twoBefore = readFileSync(join(fx.dir, 'lib', 'two.mjs'), 'utf8'); const backup = join(tmp('red-script-bk-'), 'backup'); const m = mutate(fx, 'apply', mod, 'X2-half', fx.dir, backup); assert.equal(m.status, 1, m.out); assert.equal(readFileSync(join(fx.dir, 'lib', 'one.mjs'), 'utf8'), oneBefore, 'the span that matched was written although another did not match'); assert.equal(readFileSync(join(fx.dir, 'lib', 'two.mjs'), 'utf8'), twoBefore); assert.equal(existsSync(backup), false, 'a mutation that was not applied wrote a backup'); }); test('[RS] a span that matches twice exits 1 and writes nothing', { skip: nodeTooOld && 'needs Node >= 20.14 for --test-force-exit' }, () => { requireScript(); const fx = fixture(); const mod = join(fx.dir, 'twice.mjs'); writeFileSync(mod, `export const MUTATIONS = [ { key: 'X2-twice', id: 'X2', what: 'a comment that is there twice', spans: [{ file: 'lib/two.mjs', find: '// two', replace: '' }] }, { key: 'X2-twice-re', id: 'X2', what: 'one span once, one twice', spans: [{ file: 'lib/one.mjs', find: 'return 1;', replace: 'return 5;' }, { file: 'lib/two.mjs', find: /\\/\\/ two/g, replace: '' }] }, ]; `); const r = run(fx, args(fx, ['tests/x.test.mjs'], { mutations: mod })); assert.equal(r.status, 1, r.out); assert.match(r.out, /^broken: mutation-not-applied \[X2\] X2-twice$/m, r.out); assert.match(r.out, /^broken: mutation-not-applied \[X2\] X2-twice-re$/m, r.out); const oneBefore = readFileSync(join(fx.dir, 'lib', 'one.mjs'), 'utf8'); const backup = join(tmp('red-script-bk-'), 'backup'); const m = mutate(fx, 'apply', mod, 'X2-twice-re', fx.dir, backup); assert.equal(m.status, 1, m.out); assert.equal(readFileSync(join(fx.dir, 'lib', 'one.mjs'), 'utf8'), oneBefore, 'the span that matched once was written although another matched twice'); assert.equal(readFileSync(join(fx.dir, 'lib', 'two.mjs'), 'utf8'), TWO); assert.equal(existsSync(backup), false, 'a mutation that was not applied wrote a backup'); }); test('[RS] a base that is no commit exits 2', { skip: nodeTooOld && 'needs Node >= 20.14 for --test-force-exit' }, () => { requireScript(); const fx = fixture(); const r = run(fx, ['--base', 'f'.repeat(40), '--ids', 'X1 X2', '--other-ids', 'Z9', '--mutations', join(fx.dir, 'mutations.mjs'), 'tests/x.test.mjs']); assert.equal(r.status, 2, r.out); assert.match(r.stderr, /^red script: setup failure: /m, r.out); assert.doesNotMatch(r.out, /^red script: ok/m, r.out); }); test('[RS] the clone is removed after the run', { skip: nodeTooOld && 'needs Node >= 20.14 for --test-force-exit' }, () => { requireScript(); const fx = fixture(); const td = tmp('red-script-tmpdir-'); const seen = join(tmp('red-script-seen-'), 'seen'); const r = run(fx, args(fx, ['tests/x.test.mjs']), { TMPDIR: td, RS_SEEN_FILE: seen }); assert.equal(r.status, 0, r.out); const paths = existsSync(seen) ? readFileSync(seen, 'utf8').split('\n').filter(Boolean) : []; assert.ok(paths.length > 0, 'the test file never ran, so the clone was never seen'); for (const p of paths) { assert.ok(p.startsWith(`${td}/`), `the test ran at ${p}, not in a clone under TMPDIR ${td}`); assert.equal(existsSync(p), false, `${p} is still there after the run`); } assert.deepEqual(readdirSync(td), [], 'the temp directory still holds files after the run'); }); test('[RS] the instruments run from the working tree by absolute path', { skip: nodeTooOld && 'needs Node >= 20.14 for --test-force-exit' }, () => { requireScript(); const text = readFileSync(SCRIPT, 'utf8'); const paths = instrumentScan(text); for (const name of ['no-session-id.mjs', 'red-reporter.mjs', 'red-judge.mjs', 'red-mutate.mjs']) { assert.ok(paths.some((p) => p.endsWith(`/${name}`)), `no node line runs ${name}; found: ${paths.join(', ')}`); } assert.deepEqual(notFromWorkingTree(paths), [], 'an instrument is not run as $ROOT/tests/helpers/...'); }); test('[RS] the instrument scan reports a planted ./tests/helpers/ line', { skip: nodeTooOld && 'needs Node >= 20.14 for --test-force-exit' }, () => { requireScript(); const text = readFileSync(SCRIPT, 'utf8'); const planted = text.replace(/--test-reporter=("?)\$ROOT\/tests\/helpers\/red-reporter\.mjs\1/, '--test-reporter=./tests/helpers/red-reporter.mjs'); assert.notEqual(planted, text, 'control: the reporter path to plant over was not found'); assert.deepEqual(notFromWorkingTree(instrumentScan(planted)), ['./tests/helpers/red-reporter.mjs'], 'control: the scan does not report a planted relative reporter path'); }); test('[RS] --require-sha256 with a git that cannot create a SHA-256 repository exits 2 with a setup-failure line', { skip: nodeTooOld && 'needs Node >= 20.14 for --test-force-exit' }, () => { requireScript(); const fx = fixture(); const real = pathWith(['git']); cleanup.push(real.path); const bin = tmp('red-script-fakegit-'); const fake = join(bin, 'git'); writeFileSync(fake, `#!/bin/sh case " $* " in *" init "*"--object-format=sha256"*) echo "fake git: no SHA-256 here" >&2; exit 128 ;; esac exec '${join(real.path, 'git')}' "$@" `, { mode: 0o755 }); const PATH = `${bin}${delimiter}${process.env.PATH}`; const probe = spawnSync(fake, ['init', '--quiet', '--object-format=sha256', join(bin, 'probe')], { encoding: 'utf8', env: env({ PATH }) }); assert.notEqual(probe.status, 0, 'control: the fake git created a SHA-256 repository'); assert.equal(spawnSync(fake, ['--version'], { encoding: 'utf8', env: env({ PATH }) }).status, 0, 'control: the fake git does not run the real one'); const r = run(fx, [...args(fx, ['tests/x.test.mjs']).slice(0, 8), '--require-sha256', 'tests/x.test.mjs'], { PATH }); assert.equal(r.status, 2, r.out); assert.match(r.stderr, /^red script: setup failure: .*SHA-256/m, r.out); assert.doesNotMatch(r.out, /^red script: ok/m, r.out); }); test('[E23a] a mistyped [X 1] and an unlisted [Q9] are reported and the run exits 1', { skip: nodeTooOld && 'needs Node >= 20.14 for --test-force-exit' }, () => { requireScript(); const fx = fixture(); const r = run(fx, args(fx, ['tests/x.test.mjs', 'tests/stray.test.mjs'])); assert.equal(r.status, 1, r.out); assert.match(r.out, /^broken: unreadable-tag \[X 1\] a mistyped tag$/m, r.out); assert.match(r.out, /^broken: unlisted-tag \[Q9\] \[Q9\] an unlisted tag$/m, r.out); }); test('[E23a] the base run selects every tag with one ^\\[ pattern', { skip: nodeTooOld && 'needs Node >= 20.14 for --test-force-exit' }, () => { requireScript(); const text = readFileSync(SCRIPT, 'utf8'); const base = runLines(text).filter((l) => l.includes('run_tagged "$TMP/base.jsonl"')); assert.equal(base.length, 1, `not exactly one base-run line:\n${base.join('\n')}`); assert.ok(base[0].includes(` '--test-name-pattern=^\\['`), `the base run does not select with '--test-name-pattern=^\\[':\n${base[0]}`); assert.equal((base[0].match(/--test-name-pattern/g) || []).length, 1, `the base run carries more than one name pattern:\n${base[0]}`); }); test('[E23b] the base run carries --test-timeout and --test-force-exit', { skip: nodeTooOld && 'needs Node >= 20.14 for --test-force-exit' }, () => { requireScript(); const text = readFileSync(SCRIPT, 'utf8'); const lines = runLines(text); assert.ok(lines.some((l) => l.includes('run_tagged "$TMP/base.jsonl"')), 'no base-run line'); assert.ok(lines.length >= 2, `the mutation run line is missing:\n${lines.join('\n')}`); for (const l of lines) { assert.match(l, / --test-timeout=[1-9]\d*(\s|$)/, `a run without --test-timeout:\n${l}`); assert.match(l, / --test-force-exit(\s|$)/, `a run without --test-force-exit:\n${l}`); } assert.match(text, /^#.*Node >= 20\.14/m, 'the header does not say that Node >= 20.14 is needed'); });