// lib/verification/red-proof.mjs // Proof of red (P9 (1)). A step's test is run by a command twice: // // 1. against the START POINT (the step's recorded start_sha, with the step's // test files copied in): it must FAIL - otherwise the test proves nothing // about the change; // 2. against the RESULT (the working tree as the step left it): it must PASS. // // `--red-only` runs step 1 alone: the gate the step helper passes BEFORE it // builds (and locks the test). The executor's own check afterwards is the full // two-sided run. // // Exit 0 = proven, 1 = rejected (`not-red-at-base` or `not-green-at-result`), // 2 = could not run (`not-run`: no test files, an unknown start point, a command // outside the test-runner allowlist, a command that cannot start). 2 is never a // pass and never "proven". // // The start point is materialized with `git archive ` into a temp // directory, never by checking anything out in the working tree: the step's // work is not touched, and no worktree is registered in the repo. Tracked files // only - pass `--link node_modules` (comma list) to symlink an untracked // dependency directory from the working tree into the start-point copy. // // The command must be ONE test-runner invocation without shell operators // (criteria-runner.mjs `allowedCommand`), and it is screened through the // plugin's own PreToolUse denylist (`screenCommand`): a command spawned from // node never passes through the Bash tool, so the hook cannot fire by itself. // // When the output is TAP (`--test-reporter=tap`, pytest-tap, ...), the counts // `# tests N` / `# fail N` are reported for both runs. They show WHAT the red // was: "fail 1 of 1" on a node test file is usually a load error (an import the // start point lacks), not named failing tests - see // tests/fixtures/red-first/02243c6-always-allow-shim.mjs, re-measured through // this tool in tests/lib/red-proof.test.mjs. // // Borrowed: GitHub Spec Kit (MIT, github/spec-kit@2e941fe) - its test-first // rule is an instruction there; here a command proves it. import { cpSync, existsSync, mkdirSync, mkdtempSync, realpathSync, rmSync, symlinkSync } from 'node:fs'; import { spawnSync } from 'node:child_process'; import { tmpdir } from 'node:os'; import { dirname, isAbsolute, join, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; import { allowedCommand, screenCommand } from './criteria-runner.mjs'; export const DEFAULT_TIMEOUT_MS = 900_000; const TAIL = 2000; function tap(output) { const tests = String(output).match(/^# tests (\d+)$/m); const fail = String(output).match(/^# fail (\d+)$/m); return tests && fail ? { tests: Number(tests[1]), fail: Number(fail[1]) } : null; } // The command's exit code must be its own. Under `node --test` a child inherits // NODE_TEST_CONTEXT, and a nested `node --test` then reports to that parent and // exits 0 whatever its tests did (measured 2026-10-06: a red start point read as // green). So the variable never reaches the command. export function commandEnv(env = process.env) { const { NODE_TEST_CONTEXT, ...rest } = env; return rest; } function run(cmd, cwd, timeoutMs) { const r = spawnSync('/bin/sh', ['-c', cmd], { cwd, env: commandEnv(), encoding: 'utf8', timeout: timeoutMs, maxBuffer: 64 * 1024 * 1024 }); const output = `${r.stdout ?? ''}${r.stderr ?? ''}`; return { exit: r.status, signal: r.signal ?? null, tap: tap(output), tail: output.slice(-TAIL), spawnError: r.error ? r.error.message : null }; } const notRun = (reason, extra = {}) => ({ status: 'not-run', reason, ...extra }); // A shell that could not find or execute the command (126/127), a signal (the // timeout), or a spawn error is not a test verdict at all. function unrunnable(r) { if (r.spawnError) return `the command could not start: ${r.spawnError}`; if (r.exit === null) return `the command was killed (${r.signal ?? 'timeout'})`; if (r.exit === 126 || r.exit === 127) return `the shell could not run the command (exit ${r.exit})`; return null; } /** * @param {{cwd: string, base: string, cmd: string, tests: string[], link?: string[], timeoutMs?: number}} o * @returns {{status: 'proven'|'rejected'|'not-run', reason: string, base?: object, result?: object}} */ export function repoRoot(dir) { const top = spawnSync('git', ['rev-parse', '--show-toplevel'], { cwd: dir, encoding: 'utf8' }); return top.status === 0 ? top.stdout.trim() : null; } export function proveRed(o) { const given = resolve(o.cwd ?? process.cwd()); const timeoutMs = o.timeoutMs ?? DEFAULT_TIMEOUT_MS; const tests = (o.tests ?? []).filter(Boolean); if (!existsSync(given)) return notRun(`--cwd ${given} does not exist`); // Plan paths and commands are relative to the repo root, and `git archive` // run from a subdirectory packs only that subdirectory (measured // 2026-10-06). So everything below runs at the root of --cwd's repo. const cwd = repoRoot(given); if (!cwd) return notRun(`--cwd ${given} is not inside a git repository`); if (tests.length === 0) return notRun('no test files given: a step without its test cannot prove red'); for (const t of tests) { if (isAbsolute(t) || t.split('/').includes('..')) return notRun(`test path must be repo-relative: ${t}`); if (!existsSync(join(cwd, t))) return notRun(`test file not found in the working tree: ${t}`); } const allowed = allowedCommand(o.cmd, { cwd }); if (!allowed.allowed) return notRun(`command not run: ${allowed.reason}`); const screen = screenCommand(o.cmd); if (!screen.allowed) return notRun(`command blocked: ${screen.rule}`); const sha = spawnSync('git', ['rev-parse', '--verify', '--quiet', `${o.base}^{commit}`], { cwd, encoding: 'utf8' }); if (sha.status !== 0) return notRun(`cannot resolve the start point ${o.base} in ${cwd}`); const baseSha = sha.stdout.trim(); // The result first: it is the working tree as it stands. `redOnly` is the // gate before building (the step helper's own check): the result is not run. let result; if (!o.redOnly) { result = run(o.cmd, cwd, timeoutMs); const resultBroken = unrunnable(result); if (resultBroken) return notRun(`at the result: ${resultBroken}`, { result }); } const tmp = mkdtempSync(join(tmpdir(), 'voyage-red-proof-')); try { const tree = join(tmp, 'tree'); mkdirSync(tree); const tar = join(tmp, 'start.tar'); const a = spawnSync('git', ['archive', '--format=tar', '-o', tar, baseSha], { cwd, encoding: 'utf8' }); if (a.status !== 0) return notRun(`git archive ${baseSha} failed: ${String(a.stderr).trim()}`, { result }); const x = spawnSync('tar', ['-xf', tar, '-C', tree], { encoding: 'utf8' }); if (x.status !== 0) return notRun(`tar could not unpack the start point: ${String(x.stderr).trim()}`, { result }); for (const t of tests) { mkdirSync(dirname(join(tree, t)), { recursive: true }); cpSync(join(cwd, t), join(tree, t)); } for (const l of o.link ?? []) { if (!l || isAbsolute(l) || l.split('/').includes('..')) return notRun(`--link must be repo-relative: ${l}`, { result }); if (existsSync(join(cwd, l)) && !existsSync(join(tree, l))) { mkdirSync(dirname(join(tree, l)), { recursive: true }); symlinkSync(join(cwd, l), join(tree, l)); } } const base = run(o.cmd, tree, timeoutMs); const baseBroken = unrunnable(base); if (baseBroken) return notRun(`at the start point: ${baseBroken}`, { base, result }); const out = { base_sha: baseSha, base, result }; if (base.exit === 0) return { status: 'rejected', reason: 'not-red-at-base', ...out }; if (o.redOnly) return { status: 'proven', reason: 'red at the start point (--red-only: the result was not run)', base_sha: baseSha, base }; if (result.exit !== 0) return { status: 'rejected', reason: 'not-green-at-result', ...out }; return { status: 'proven', reason: 'red at the start point, green at the result', ...out }; } finally { rmSync(tmp, { recursive: true, force: true }); } } // --- CLI ------------------------------------------------------------------------ const USAGE = 'usage: red-proof.mjs --base --cmd "" --tests [--red-only] [--cwd ] [--link ] [--timeout ] [--json]'; const list = (v) => String(v).split(',').map((s) => s.trim()).filter(Boolean); export function main(argv) { const o = { cwd: process.cwd(), tests: [], link: [] }; let json = false; for (let i = 0; i < argv.length; i++) { const a = argv[i]; if (a === '--base' && argv[i + 1]) o.base = argv[++i]; else if (a === '--cmd' && argv[i + 1]) o.cmd = argv[++i]; else if (a === '--tests' && argv[i + 1]) o.tests = list(argv[++i]); else if (a === '--cwd' && argv[i + 1]) o.cwd = resolve(argv[++i]); else if (a === '--link' && argv[i + 1]) o.link = list(argv[++i]); else if (a === '--red-only') o.redOnly = true; else if (a === '--timeout' && argv[i + 1]) o.timeoutMs = Number(argv[++i]); else if (a === '--json') json = true; else { process.stderr.write(`red-proof: unknown argument ${a}\n${USAGE}\n`); return 2; } } if (!o.base || !o.cmd) { process.stderr.write(`red-proof: --base and --cmd are required\n${USAGE}\n`); return 2; } if (o.timeoutMs !== undefined && !Number.isFinite(o.timeoutMs)) { process.stderr.write(`red-proof: --timeout must be a number\n${USAGE}\n`); return 2; } let r; try { r = proveRed(o); } catch (e) { r = notRun(`red-proof crashed: ${e.message}`); } if (json) process.stdout.write(JSON.stringify(r, null, 2) + '\n'); else { const counts = (x) => (x?.tap ? ` (tap: ${x.tap.fail} of ${x.tap.tests} failed)` : ''); process.stdout.write(`red proof: ${r.status.toUpperCase()} - ${r.reason}\n`); if (r.base) process.stdout.write(` start point ${r.base_sha ?? o.base}: exit ${r.base.exit}${counts(r.base)}\n`); if (r.result) process.stdout.write(` result: exit ${r.result.exit}${counts(r.result)}\n`); } return r.status === 'proven' ? 0 : r.status === 'rejected' ? 1 : 2; } function isMain() { try { return realpathSync(process.argv[1]) === realpathSync(fileURLToPath(import.meta.url)); } catch { return false; } } if (process.argv[1] && isMain()) process.exitCode = main(process.argv.slice(2));