// lib/verification/red-proof.mjs // Proof of red (P9 (1)). A step's test is run by a command: // // 1. against the START POINT (the step's recorded start_sha, with the step's // test files copied in): it must FAIL - otherwise the test proves nothing // about the change; // 2. against the RESULT (the working tree as the step left it): it must PASS; // 3. CONTROL: the start-point copy with the step's whole change laid on top // must PASS too. A copy that cannot go green even with the change failed // step 1 for an environment reason (a missing dependency, a path, a tool), // not because the change was missing - that is "not-run", never proven. // // `--red-only` runs step 1 alone: the gate the step helper passes BEFORE it // builds (and locks the test). The executor's own check afterwards is the full // run, and only the full run is authoritative. // // Exit 0 = proven, 1 = rejected (`not-red-at-base` or `not-green-at-result`), // 2 = could not run (`not-run`: no test files, an unknown start point, a command // outside the test-runner allowlist, a command that cannot start, a control run // that stays red). 2 is never a pass and never "proven". // // The start point is a temp `git clone --shared` of the repo checked out at // start_sha - a real git checkout, so a git-reading command behaves as it does // in the repo, and nothing in the working tree or the repo's worktree list is // touched. Every gitignored top-level entry of the working tree (node_modules, // .venv, a build cache) is symlinked in, so the copy has the same dependencies; // `--link a,b` adds deeper ones. Measured 2026-10-06 (review): with a bare // `git archive` copy, a test that needs node_modules "proved red" with no change. // Known limit, failing safe: a dependency that points back INTO the working // tree (an editable `pip install -e .`, a workspace symlink) makes the start // point see the new code, and a correct step reads `not-red-at-base`. // // Everything runs at the root of --cwd's repository: plan paths and commands // are root-relative, and `git archive`/`clone` paths from a subdirectory are // not (measured 2026-10-06). // // The command must be ONE test-runner invocation without shell operators // (criteria-runner.mjs `allowedCommand`), and it is screened through the // plugin's own PreToolUse denylist (`screenCommand`): a command spawned from // node never passes through the Bash tool, so the hook cannot fire by itself. // // When the output is TAP (`--test-reporter=tap`, pytest-tap, ...), the counts // `# tests N` / `# fail N` are reported per run. They show WHAT the red was: // "fail 1 of 1" on a node test file is usually a load error (an import the // start point lacks), not named failing tests - see // tests/fixtures/red-first/02243c6-always-allow-shim.mjs, re-measured through // this tool in tests/lib/red-proof.test.mjs. // // Borrowed: GitHub Spec Kit (MIT, github/spec-kit@2e941fe) - its test-first // rule is an instruction there; here a command proves it. import { cpSync, existsSync, lstatSync, mkdirSync, mkdtempSync, realpathSync, rmSync, symlinkSync } from 'node:fs'; import { spawnSync } from 'node:child_process'; import { tmpdir } from 'node:os'; import { dirname, isAbsolute, join, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; import { allowedCommand, screenCommand } from './criteria-runner.mjs'; export const DEFAULT_TIMEOUT_MS = 900_000; const TAIL = 2000; function tap(output) { const tests = String(output).match(/^# tests (\d+)$/m); const fail = String(output).match(/^# fail (\d+)$/m); return tests && fail ? { tests: Number(tests[1]), fail: Number(fail[1]) } : null; } // The command's exit code must be its own. Under `node --test` a child inherits // NODE_TEST_CONTEXT, and a nested `node --test` then reports to that parent and // exits 0 whatever its tests did (measured 2026-10-06: a red start point read as // green). So the variable never reaches the command. export function commandEnv(env = process.env) { const { NODE_TEST_CONTEXT, ...rest } = env; return rest; } export function repoRoot(dir) { const top = spawnSync('git', ['rev-parse', '--show-toplevel'], { cwd: dir, encoding: 'utf8' }); return top.status === 0 ? top.stdout.trim() : null; } function run(cmd, cwd, timeoutMs) { const r = spawnSync('/bin/sh', ['-c', cmd], { cwd, env: commandEnv(), encoding: 'utf8', timeout: timeoutMs, maxBuffer: 64 * 1024 * 1024 }); const output = `${r.stdout ?? ''}${r.stderr ?? ''}`; return { exit: r.status, signal: r.signal ?? null, tap: tap(output), tail: output.slice(-TAIL), spawnError: r.error ? r.error.message : null }; } const notRun = (reason, extra = {}) => ({ status: 'not-run', reason, ...extra }); // A shell that could not find or execute the command (126/127), a signal (the // timeout), or a spawn error is not a test verdict at all. function unrunnable(r) { if (r.spawnError) return `the command could not start: ${r.spawnError}`; if (r.exit === null) return `the command was killed (${r.signal ?? 'timeout'})`; if (r.exit === 126 || r.exit === 127) return `the shell could not run the command (exit ${r.exit})`; return null; } const git = (cwd, args) => spawnSync('git', args, { cwd, encoding: 'utf8', maxBuffer: 256 * 1024 * 1024 }); const lines = (s) => String(s ?? '').split('\n').filter(Boolean); function safeRel(p) { return p && !isAbsolute(p) && !p.split('/').includes('..'); } function copyIn(from, to, rel) { mkdirSync(dirname(join(to, rel)), { recursive: true }); rmSync(join(to, rel), { recursive: true, force: true }); cpSync(join(from, rel), join(to, rel), { recursive: true, verbatimSymlinks: true }); } /** Symlink every gitignored top-level entry of the working tree, plus `extra`, into `tree`. */ function linkDependencies(root, tree, extra) { const linked = []; const ignored = lines(git(root, ['ls-files', '--others', '--ignored', '--exclude-standard', '--directory']).stdout) .map((p) => p.replace(/\/$/, '').split('/')[0]); for (const l of new Set([...ignored, ...extra])) { if (!safeRel(l) || l === '.git') continue; if (existsSync(join(root, l)) && !existsSync(join(tree, l))) { mkdirSync(dirname(join(tree, l)), { recursive: true }); symlinkSync(join(root, l), join(tree, l)); linked.push(l); } } return linked; } /** Lay the working tree's change since `base` onto `tree` (tracked edits, deletions, untracked files). */ function applyResult(root, tree, base) { const status = lines(git(root, ['diff', '--name-status', '--no-renames', base]).stdout); for (const s of status) { const [code, rel] = s.split('\t'); if (!safeRel(rel)) continue; if (code.startsWith('D')) rmSync(join(tree, rel), { recursive: true, force: true }); else if (existsSync(join(root, rel)) || lstatSync(join(root, rel), { throwIfNoEntry: false })) copyIn(root, tree, rel); } for (const rel of lines(git(root, ['ls-files', '--others', '--exclude-standard']).stdout)) { if (safeRel(rel)) copyIn(root, tree, rel); } } /** * @param {{cwd: string, base: string, cmd: string, tests: string[], link?: string[], redOnly?: boolean, timeoutMs?: number}} o * @returns {{status: 'proven'|'rejected'|'not-run', reason: string, base?: object, result?: object, control?: object}} */ export function proveRed(o) { const given = resolve(o.cwd ?? process.cwd()); const timeoutMs = o.timeoutMs ?? DEFAULT_TIMEOUT_MS; const tests = (o.tests ?? []).filter(Boolean); if (!existsSync(given)) return notRun(`--cwd ${given} does not exist`); const cwd = repoRoot(given); if (!cwd) return notRun(`--cwd ${given} is not inside a git repository`); if (tests.length === 0) return notRun('no test files given: a step without its test cannot prove red'); for (const t of tests) { if (!safeRel(t)) return notRun(`test path must be repo-relative: ${t}`); if (!existsSync(join(cwd, t))) return notRun(`test file not found in the working tree: ${t}`); } for (const l of o.link ?? []) if (!safeRel(l)) return notRun(`--link must be repo-relative: ${l}`); const allowed = allowedCommand(o.cmd, { cwd }); if (!allowed.allowed) return notRun(`command not run: ${allowed.reason}`); const screen = screenCommand(o.cmd); if (!screen.allowed) return notRun(`command blocked: ${screen.rule}`); const sha = git(cwd, ['rev-parse', '--verify', '--quiet', `${o.base}^{commit}`]); if (sha.status !== 0) return notRun(`cannot resolve the start point ${o.base} in ${cwd}`); const baseSha = sha.stdout.trim(); // The result first: it is the working tree as it stands. `redOnly` is the // gate before building (the step helper's own check): the result is not run. let result; if (!o.redOnly) { result = run(o.cmd, cwd, timeoutMs); const resultBroken = unrunnable(result); if (resultBroken) return notRun(`at the result: ${resultBroken}`, { result }); } const tmp = mkdtempSync(join(tmpdir(), 'voyage-red-proof-')); try { const tree = join(tmp, 'tree'); const c = git(tmp, ['clone', '--quiet', '--shared', '--no-checkout', cwd, tree]); if (c.status !== 0) return notRun(`could not clone the start point: ${String(c.stderr).trim()}`, { result }); const co = git(tree, ['checkout', '--quiet', '--detach', baseSha]); if (co.status !== 0) return notRun(`could not check out ${baseSha}: ${String(co.stderr).trim()}`, { result }); for (const t of tests) copyIn(cwd, tree, t); const linked = linkDependencies(cwd, tree, o.link ?? []); const base = run(o.cmd, tree, timeoutMs); const baseBroken = unrunnable(base); if (baseBroken) return notRun(`at the start point: ${baseBroken}`, { base, result, linked }); const out = { base_sha: baseSha, linked, base, result }; if (base.exit === 0) return { status: 'rejected', reason: 'not-red-at-base', ...out }; if (o.redOnly) return { status: 'proven', reason: 'red at the start point (--red-only: the result was not run)', base_sha: baseSha, linked, base }; if (result.exit !== 0) return { status: 'rejected', reason: 'not-green-at-result', ...out }; // Control: the same copy with the whole change laid on. Red here means the // copy itself cannot run the test green, so its red proved nothing. applyResult(cwd, tree, baseSha); const control = run(o.cmd, tree, timeoutMs); if (control.exit !== 0) { return notRun('the control run (start-point copy + the change) stayed red: the start point failed for an environment reason, not for the missing change', { ...out, control }); } return { status: 'proven', reason: 'red at the start point, green at the result, green in the control', ...out, control }; } finally { rmSync(tmp, { recursive: true, force: true }); } } // --- CLI ------------------------------------------------------------------------ const USAGE = 'usage: red-proof.mjs --base --cmd "" --tests [--red-only] [--cwd ] [--link ] [--timeout ] [--json]'; const list = (v) => String(v).split(',').map((s) => s.trim()).filter(Boolean); export function main(argv) { const o = { cwd: process.cwd(), tests: [], link: [] }; let json = false; for (let i = 0; i < argv.length; i++) { const a = argv[i]; if (a === '--base' && argv[i + 1]) o.base = argv[++i]; else if (a === '--cmd' && argv[i + 1]) o.cmd = argv[++i]; else if (a === '--tests' && argv[i + 1]) o.tests = list(argv[++i]); else if (a === '--cwd' && argv[i + 1]) o.cwd = resolve(argv[++i]); else if (a === '--link' && argv[i + 1]) o.link = list(argv[++i]); else if (a === '--red-only') o.redOnly = true; else if (a === '--timeout' && argv[i + 1]) o.timeoutMs = Number(argv[++i]); else if (a === '--json') json = true; else { process.stderr.write(`red-proof: unknown argument ${a} (--tests takes ONE comma-separated list)\n${USAGE}\n`); return 2; } } if (!o.base || !o.cmd) { process.stderr.write(`red-proof: --base and --cmd are required\n${USAGE}\n`); return 2; } if (o.timeoutMs !== undefined && !Number.isFinite(o.timeoutMs)) { process.stderr.write(`red-proof: --timeout must be a number\n${USAGE}\n`); return 2; } let r; try { r = proveRed(o); } catch (e) { r = notRun(`red-proof crashed: ${e.message}`); } if (json) process.stdout.write(JSON.stringify(r, null, 2) + '\n'); else { const counts = (x) => (x?.tap ? ` (tap: ${x.tap.fail} of ${x.tap.tests} failed)` : ''); process.stdout.write(`red proof: ${r.status.toUpperCase()} - ${r.reason}\n`); if (r.base) process.stdout.write(` start point ${r.base_sha ?? o.base}: exit ${r.base.exit}${counts(r.base)}\n`); if (r.result) process.stdout.write(` result: exit ${r.result.exit}${counts(r.result)}\n`); if (r.control) process.stdout.write(` control: exit ${r.control.exit}${counts(r.control)}\n`); if (r.linked?.length) process.stdout.write(` linked into the start point: ${r.linked.join(', ')}\n`); } return r.status === 'proven' ? 0 : r.status === 'rejected' ? 1 : 2; } function isMain() { try { return realpathSync(process.argv[1]) === realpathSync(fileURLToPath(import.meta.url)); } catch { return false; } } if (process.argv[1] && isMain()) process.exitCode = main(process.argv.slice(2));