// lib/verification/red-proof.mjs // Proof of red (P9 (1)). A step's test is run by a command: // // 1. against the START POINT (the step's recorded start_sha, with the step's // test files copied in): it must FAIL - otherwise the test proves nothing // about the change; // 2. against the RESULT (the working tree as the step left it): it must PASS; // 3. CONTROL: the start-point copy with the step's whole change laid on top // must PASS too. A copy that cannot go green even with the change failed // step 1 for an environment reason (a missing dependency, a path, a tool), // not because the change was missing - that is "not-run", never proven. // // `--red-only` runs step 1 alone: the gate the step helper passes BEFORE it // builds (and locks the test). The executor's own check afterwards is the full // run, and only the full run is authoritative. // // `--plan --step ` adds `tests_only` to a `not-red-at-base`: true // when every file in the step's `Files:` is a test and so is every file the // step changed since its start point, so the step can never be red and goes // up to the plan (see `testsOnly` below). // // Exit 0 = proven, 1 = rejected (`not-red-at-base` or `not-green-at-result`), // 2 = could not run (`not-run`: no test files, an unknown start point, a command // outside the test-runner allowlist, a command that cannot start, a control run // that stays red). 2 is never a pass and never "proven". // // The start point is a temp `git clone --shared` of the repo checked out at // start_sha - a real git checkout, so a git-reading command behaves as it does // in the repo, and nothing in the working tree or the repo's worktree list is // touched. Every gitignored top-level entry of the working tree (node_modules, // .venv, a build cache) is symlinked in, so the copy has the same dependencies; // `--link a,b` adds deeper ones. Measured 2026-10-06 (review): with a bare // `git archive` copy, a test that needs node_modules "proved red" with no change. // Known limit, failing safe: a dependency that points back INTO the working // tree (an editable `pip install -e .`, a workspace symlink) makes the start // point see the new code, and a correct step reads `not-red-at-base`. // // Everything runs at the root of --cwd's repository: plan paths and commands // are root-relative, and `git archive`/`clone` paths from a subdirectory are // not (measured 2026-10-06). // // The command must be ONE test-runner invocation without shell operators // (criteria-runner.mjs `allowedCommand`), and it is screened through the // plugin's own PreToolUse denylist (`screenCommand`): a command spawned from // node never passes through the Bash tool, so the hook cannot fire by itself. // It runs with no shell (`runCommand`): the words checked are the words run. // // When the output is TAP (`--test-reporter=tap`, pytest-tap, ...), the counts // `# tests N` / `# fail N` are reported per run. They show WHAT the red was: // "fail 1 of 1" on a node test file is usually a load error (an import the // start point lacks), not named failing tests - see // tests/fixtures/red-first/02243c6-always-allow-shim.mjs, re-measured through // this tool in tests/lib/red-proof.test.mjs. // // Borrowed: GitHub Spec Kit (MIT, github/spec-kit@2e941fe) - its test-first // rule is an instruction there; here a command proves it. import { cpSync, existsSync, lstatSync, mkdirSync, mkdtempSync, readFileSync, realpathSync, rmSync, symlinkSync } from 'node:fs'; import { spawnSync } from 'node:child_process'; import { tmpdir } from 'node:os'; import { dirname, isAbsolute, join, relative, resolve, sep } from 'node:path'; import { fileURLToPath } from 'node:url'; import { allowedCommand, runCommand, screenCommand } from './criteria-runner.mjs'; import { commandEnv } from '../util/command-env.mjs'; import { parseStepFields, sliceSteps, stepTestFile, stepTestsOnlyFromFields } from '../parsers/plan-schema.mjs'; export const DEFAULT_TIMEOUT_MS = 900_000; const TAIL = 2000; function tap(output) { const tests = String(output).match(/^# tests (\d+)$/m); const fail = String(output).match(/^# fail (\d+)$/m); return tests && fail ? { tests: Number(tests[1]), fail: Number(fail[1]) } : null; } // The command's exit code must be its own: NODE_TEST_CONTEXT never reaches it // (lib/util/command-env.mjs). export { commandEnv }; export function repoRoot(dir) { const top = spawnSync('git', ['rev-parse', '--show-toplevel'], { cwd: dir, encoding: 'utf8' }); return top.status === 0 ? top.stdout.trim() : null; } // A red that is the test file failing to LOAD (an import the start point // lacks) rather than a named test failing. Still a red — the change is what is // missing — but the result says which kind, in a field (`red_kind`), instead // of leaving it to a reader of `tap` (null without a TAP reporter: the // voyage-lekerepo trial's step 1, 2026-10-06). // Specific signatures only: a bare `SyntaxError` anywhere also matched an // assertion message («Missing expected exception (SyntaxError)») of a named // failing test (independent review, 2026-10-08). const LOAD_ERROR = /^\s*SyntaxError: |ERR_MODULE_NOT_FOUND|does not provide an export named|Cannot find module|ERR_REQUIRE_ESM|ModuleNotFoundError|ImportError: /m; // No shell: the words the screen checked are the words that run. function run(cmd, cwd, timeoutMs) { const r = runCommand(cmd, { cwd, timeoutMs }); const output = `${r.stdout ?? ''}${r.stderr ?? ''}`; return { exit: r.status, signal: r.signal ?? null, tap: tap(output), tail: output.slice(-TAIL), spawnError: r.error ? r.error.message : null, loadError: LOAD_ERROR.test(output) }; } /** What the red at the start point was: the test file failing to load, or a test failing. */ const redKind = (base) => (base.loadError ? 'load-error' : 'test-failure'); const notRun = (reason, extra = {}) => ({ status: 'not-run', reason, ...extra }); // With `--plan` and `--step`, a `not-red-at-base` also says whether the step // holds nothing but tests (plan-schema `stepTestsOnly`, the plan validator's // PLAN_STEP_TESTS_ONLY). Such a step can never be red: the behaviour its tests // cover already exists at its start point. Rewriting the test cannot fix that, so // the executor sends the step up to the plan instead of into fix rounds // (voyage-lekerepo trial 3, 2026-10-09). `null` = the plan or the step could // not be read; without `--plan` the field is absent and nothing new happens. // // The plan's `Files:` is prose, read by a reader that can misread it; the // step's own diff is a fact. So a step the plan calls tests-only is so only // when every file it changed since the start point is a test too (release // review of 6.3.0, round 6). Before building (`--red-only`) the diff holds the // step's test alone, so there the plan decides. The plan file itself and // Voyage's own files under .claude/ (progress, approvals) are not the step's change. function testsOnly(o, cwd, baseSha) { if (!o.plan) return {}; let text; try { text = readFileSync(o.plan, 'utf8'); } catch (e) { return { tests_only: null, tests_only_note: `cannot read the plan: ${e.message}` }; } const step = sliceSteps(text).find((s) => Number(s.n) === Number(o.step)); if (!step) return { tests_only: null, tests_only_note: `no "### Step ${o.step}:" in the plan` }; const fields = parseStepFields(step.body); if (!stepTestsOnlyFromFields(fields).testsOnly) return { tests_only: false }; const isTest = stepTestFile(fields); const given = new Set(o.tests ?? []); const changed = [ ...lines(git(cwd, ['diff', '--name-only', '--no-renames', baseSha]).stdout), ...lines(git(cwd, ['ls-files', '--others', '--exclude-standard']).stdout), ]; const planRel = relative(cwd, realpathSync(o.plan)).split(sep).join('/'); const other = [...new Set(changed)].filter((p) => p !== planRel && !p.startsWith('.claude/') && !given.has(p) && !isTest(p)); if (other.length === 0) return { tests_only: true }; return { tests_only: false, tests_only_note: `the plan's Files: names only tests, but the step's diff also changes ${other.slice(0, 5).join(', ')}` }; } // A program that could not start (a spawn error: not installed) or a signal // (the timeout) is not a test verdict at all. With no shell, an exit 126/127 // is the test program's own (a script whose tool the step has not built yet): // a red like any other, and the control run tells an environment fault apart // (release review of 6.3.0, round 6). function unrunnable(r) { if (r.spawnError) return `the command could not start: ${r.spawnError}`; if (r.exit === null) return `the command was killed (${r.signal ?? 'timeout'})`; return null; } const git = (cwd, args) => spawnSync('git', args, { cwd, encoding: 'utf8', maxBuffer: 256 * 1024 * 1024 }); const lines = (s) => String(s ?? '').split('\n').filter(Boolean); function safeRel(p) { return p && !isAbsolute(p) && !p.split('/').includes('..'); } function copyIn(from, to, rel) { mkdirSync(dirname(join(to, rel)), { recursive: true }); rmSync(join(to, rel), { recursive: true, force: true }); cpSync(join(from, rel), join(to, rel), { recursive: true, verbatimSymlinks: true }); } /** Symlink every gitignored top-level entry of the working tree, plus `extra`, into `tree`. */ function linkDependencies(root, tree, extra) { const linked = []; const ignored = lines(git(root, ['ls-files', '--others', '--ignored', '--exclude-standard', '--directory']).stdout) .map((p) => p.replace(/\/$/, '').split('/')[0]); for (const l of new Set([...ignored, ...extra])) { if (!safeRel(l) || l === '.git') continue; if (existsSync(join(root, l)) && !existsSync(join(tree, l))) { mkdirSync(dirname(join(tree, l)), { recursive: true }); symlinkSync(join(root, l), join(tree, l)); linked.push(l); } } return linked; } /** Lay the working tree's change since `base` onto `tree` (tracked edits, deletions, untracked files). */ function applyResult(root, tree, base) { const status = lines(git(root, ['diff', '--name-status', '--no-renames', base]).stdout); for (const s of status) { const [code, rel] = s.split('\t'); if (!safeRel(rel)) continue; if (code.startsWith('D')) rmSync(join(tree, rel), { recursive: true, force: true }); else if (existsSync(join(root, rel)) || lstatSync(join(root, rel), { throwIfNoEntry: false })) copyIn(root, tree, rel); } for (const rel of lines(git(root, ['ls-files', '--others', '--exclude-standard']).stdout)) { if (safeRel(rel)) copyIn(root, tree, rel); } } /** * @param {{cwd: string, base: string, cmd: string, tests: string[], link?: string[], redOnly?: boolean, timeoutMs?: number, plan?: string, step?: number}} o * @returns {{status: 'proven'|'rejected'|'not-run', reason: string, base?: object, result?: object, control?: object}} */ export function proveRed(o) { const given = resolve(o.cwd ?? process.cwd()); const timeoutMs = o.timeoutMs ?? DEFAULT_TIMEOUT_MS; const tests = (o.tests ?? []).filter(Boolean); if (!existsSync(given)) return notRun(`--cwd ${given} does not exist`); const cwd = repoRoot(given); if (!cwd) return notRun(`--cwd ${given} is not inside a git repository`); if (tests.length === 0) return notRun('no test files given: a step without its test cannot prove red'); for (const t of tests) { if (!safeRel(t)) return notRun(`test path must be repo-relative: ${t}`); if (!existsSync(join(cwd, t))) return notRun(`test file not found in the working tree: ${t}`); } for (const l of o.link ?? []) if (!safeRel(l)) return notRun(`--link must be repo-relative: ${l}`); const allowed = allowedCommand(o.cmd, { cwd }); if (!allowed.allowed) return notRun(`command not run: ${allowed.reason}`); const screen = screenCommand(o.cmd); if (!screen.allowed) return notRun(`command blocked: ${screen.rule}`); const sha = git(cwd, ['rev-parse', '--verify', '--quiet', `${o.base}^{commit}`]); if (sha.status !== 0) return notRun(`cannot resolve the start point ${o.base} in ${cwd}`); const baseSha = sha.stdout.trim(); // The result first: it is the working tree as it stands. `redOnly` is the // gate before building (the step helper's own check): the result is not run. let result; if (!o.redOnly) { result = run(o.cmd, cwd, timeoutMs); const resultBroken = unrunnable(result); if (resultBroken) return notRun(`at the result: ${resultBroken}`, { result }); } const tmp = mkdtempSync(join(tmpdir(), 'voyage-red-proof-')); try { const tree = join(tmp, 'tree'); const c = git(tmp, ['clone', '--quiet', '--shared', '--no-checkout', cwd, tree]); if (c.status !== 0) return notRun(`could not clone the start point: ${String(c.stderr).trim()}`, { result }); const co = git(tree, ['checkout', '--quiet', '--detach', baseSha]); if (co.status !== 0) return notRun(`could not check out ${baseSha}: ${String(co.stderr).trim()}`, { result }); for (const t of tests) copyIn(cwd, tree, t); const linked = linkDependencies(cwd, tree, o.link ?? []); const base = run(o.cmd, tree, timeoutMs); const baseBroken = unrunnable(base); if (baseBroken) return notRun(`at the start point: ${baseBroken}`, { base, result, linked }); const out = { base_sha: baseSha, linked, base, result }; if (base.exit === 0) return { status: 'rejected', reason: 'not-red-at-base', ...testsOnly(o, cwd, baseSha), ...out }; if (o.redOnly) return { status: 'proven', reason: 'red at the start point (--red-only: the result was not run)', red_kind: redKind(base), base_sha: baseSha, linked, base }; if (result.exit !== 0) return { status: 'rejected', reason: 'not-green-at-result', ...out }; // Control: the same copy with the whole change laid on. Red here means the // copy itself cannot run the test green, so its red proved nothing. applyResult(cwd, tree, baseSha); const control = run(o.cmd, tree, timeoutMs); if (control.exit !== 0) { return notRun('the control run (start-point copy + the change) stayed red: the start point failed for an environment reason, not for the missing change', { ...out, control }); } return { status: 'proven', reason: 'red at the start point, green at the result, green in the control', red_kind: redKind(base), ...out, control }; } finally { rmSync(tmp, { recursive: true, force: true }); } } // --- CLI ------------------------------------------------------------------------ const USAGE = 'usage: red-proof.mjs --base --cmd "" --tests [--plan --step ] [--red-only] [--cwd ] [--link ] [--timeout ] [--json]'; const list = (v) => String(v).split(',').map((s) => s.trim()).filter(Boolean); export function main(argv) { const o = { cwd: process.cwd(), tests: [], link: [] }; let json = false; for (let i = 0; i < argv.length; i++) { const a = argv[i]; if (a === '--base' && argv[i + 1]) o.base = argv[++i]; else if (a === '--cmd' && argv[i + 1]) o.cmd = argv[++i]; else if (a === '--tests' && argv[i + 1]) o.tests = list(argv[++i]); else if (a === '--cwd' && argv[i + 1]) o.cwd = resolve(argv[++i]); else if (a === '--link' && argv[i + 1]) o.link = list(argv[++i]); else if (a === '--plan' && argv[i + 1]) o.plan = resolve(argv[++i]); else if (a === '--step' && argv[i + 1]) o.step = Number(argv[++i]); else if (a === '--red-only') o.redOnly = true; else if (a === '--timeout' && argv[i + 1]) o.timeoutMs = Number(argv[++i]); else if (a === '--json') json = true; else { process.stderr.write(`red-proof: unknown argument ${a} (--tests takes ONE comma-separated list)\n${USAGE}\n`); return 2; } } if (!o.base || !o.cmd) { process.stderr.write(`red-proof: --base and --cmd are required\n${USAGE}\n`); return 2; } if (o.plan && !(Number.isInteger(o.step) && o.step > 0)) { process.stderr.write(`red-proof: --plan needs --step \n${USAGE}\n`); return 2; } if (o.step !== undefined && !o.plan) { process.stderr.write(`red-proof: --step needs --plan \n${USAGE}\n`); return 2; } if (o.timeoutMs !== undefined && !Number.isFinite(o.timeoutMs)) { process.stderr.write(`red-proof: --timeout must be a number\n${USAGE}\n`); return 2; } let r; try { r = proveRed(o); } catch (e) { r = notRun(`red-proof crashed: ${e.message}`); } if (json) process.stdout.write(JSON.stringify(r, null, 2) + '\n'); else { const counts = (x) => (x?.tap ? ` (tap: ${x.tap.fail} of ${x.tap.tests} failed)` : ''); process.stdout.write(`red proof: ${r.status.toUpperCase()} - ${r.reason}${r.red_kind ? ` (red: ${r.red_kind})` : ''}\n`); if (r.tests_only === true) process.stdout.write(' tests_only: true - every file in the step\'s Files: is a test, so it can never be red: up to the plan\n'); if (r.base) process.stdout.write(` start point ${r.base_sha ?? o.base}: exit ${r.base.exit}${counts(r.base)}\n`); if (r.result) process.stdout.write(` result: exit ${r.result.exit}${counts(r.result)}\n`); if (r.control) process.stdout.write(` control: exit ${r.control.exit}${counts(r.control)}\n`); if (r.linked?.length) process.stdout.write(` linked into the start point: ${r.linked.join(', ')}\n`); } return r.status === 'proven' ? 0 : r.status === 'rejected' ? 1 : 2; } function isMain() { try { return realpathSync(process.argv[1]) === realpathSync(fileURLToPath(import.meta.url)); } catch { return false; } } if (process.argv[1] && isMain()) process.exitCode = main(process.argv.slice(2));