voyage/lib/verification/red-proof.mjs

// lib/verification/red-proof.mjs
// Proof of red (P9 (1)). A step's test is run by a command:
//
//   1. against the START POINT (the step's recorded start_sha, with the step's
//      test files copied in): it must FAIL - otherwise the test proves nothing
//      about the change;
//   2. against the RESULT (the working tree as the step left it): it must PASS;
//   3. CONTROL: the start-point copy with the step's whole change laid on top
//      must PASS too. A copy that cannot go green even with the change failed
//      step 1 for an environment reason (a missing dependency, a path, a tool),
//      not because the change was missing - that is "not-run", never proven.
//
// `--red-only` runs step 1 alone: the gate the step helper passes BEFORE it
// builds (and locks the test). The executor's own check afterwards is the full
// run, and only the full run is authoritative.
//
// `--plan <plan.md> --step <N>` adds `tests_only` to a `not-red-at-base`: true
// when every file in the step's `Files:` is a test and so is every file the
// step changed since its start point, so the step can never be red and goes
// up to the plan (see `testsOnly` below).
//
// Exit 0 = proven, 1 = rejected (`not-red-at-base` or `not-green-at-result`),
// 2 = could not run (`not-run`: no test files, an unknown start point, a command
// outside the test-runner allowlist, a command that cannot start, a control run
// that stays red). 2 is never a pass and never "proven".
//
// The start point is a temp `git clone --shared` of the repo checked out at
// start_sha - a real git checkout, so a git-reading command behaves as it does
// in the repo, and nothing in the working tree or the repo's worktree list is
// touched. Every gitignored top-level entry of the working tree (node_modules,
// .venv, a build cache) is symlinked in, so the copy has the same dependencies;
// `--link a,b` adds deeper ones. Measured 2026-10-06 (review): with a bare
// `git archive` copy, a test that needs node_modules "proved red" with no change.
// Known limit, failing safe: a dependency that points back INTO the working
// tree (an editable `pip install -e .`, a workspace symlink) makes the start
// point see the new code, and a correct step reads `not-red-at-base`.
//
// Everything runs at the root of --cwd's repository: plan paths and commands
// are root-relative, and `git archive`/`clone` paths from a subdirectory are
// not (measured 2026-10-06).
//
// The command must be ONE test-runner invocation without shell operators
// (criteria-runner.mjs `allowedCommand`), and it is screened through the
// plugin's own PreToolUse denylist (`screenCommand`): a command spawned from
// node never passes through the Bash tool, so the hook cannot fire by itself.
// It runs with no shell (`runCommand`): the words checked are the words run.
//
// When the output is TAP (`--test-reporter=tap`, pytest-tap, ...), the counts
// `# tests N` / `# fail N` are reported per run. They show WHAT the red was:
// "fail 1 of 1" on a node test file is usually a load error (an import the
// start point lacks), not named failing tests - see
// tests/fixtures/red-first/02243c6-always-allow-shim.mjs, re-measured through
// this tool in tests/lib/red-proof.test.mjs.
//
// Borrowed: GitHub Spec Kit (MIT, github/spec-kit@2e941fe) - its test-first
// rule is an instruction there; here a command proves it.

import { cpSync, existsSync, lstatSync, mkdirSync, mkdtempSync, readFileSync, realpathSync, rmSync, symlinkSync } from 'node:fs';
import { spawnSync } from 'node:child_process';
import { tmpdir } from 'node:os';
import { dirname, isAbsolute, join, relative, resolve, sep } from 'node:path';
import { fileURLToPath } from 'node:url';
import { allowedCommand, runCommand, screenCommand } from './criteria-runner.mjs';
import { commandEnv } from '../util/command-env.mjs';
import { parseStepFields, sliceSteps, stepTestFile, stepTestsOnlyFromFields } from '../parsers/plan-schema.mjs';

export const DEFAULT_TIMEOUT_MS = 900_000;
const TAIL = 2000;

function tap(output) {
  const tests = String(output).match(/^# tests (\d+)$/m);
  const fail = String(output).match(/^# fail (\d+)$/m);
  return tests && fail ? { tests: Number(tests[1]), fail: Number(fail[1]) } : null;
}

// The command's exit code must be its own: NODE_TEST_CONTEXT never reaches it
// (lib/util/command-env.mjs).
export { commandEnv };

export function repoRoot(dir) {
  const top = spawnSync('git', ['rev-parse', '--show-toplevel'], { cwd: dir, encoding: 'utf8' });
  return top.status === 0 ? top.stdout.trim() : null;
}

// A red that is the test file failing to LOAD (an import the start point
// lacks) rather than a named test failing. Still a red — the change is what is
// missing — but the result says which kind, in a field (`red_kind`), instead
// of leaving it to a reader of `tap` (null without a TAP reporter: the
// voyage-lekerepo trial's step 1, 2026-10-06).
// Specific signatures only: a bare `SyntaxError` anywhere also matched an
// assertion message («Missing expected exception (SyntaxError)») of a named
// failing test (independent review, 2026-10-08).
const LOAD_ERROR = /^\s*SyntaxError: |ERR_MODULE_NOT_FOUND|does not provide an export named|Cannot find module|ERR_REQUIRE_ESM|ModuleNotFoundError|ImportError: /m;

// No shell: the words the screen checked are the words that run.
function run(cmd, cwd, timeoutMs) {
  const r = runCommand(cmd, { cwd, timeoutMs });
  const output = `${r.stdout ?? ''}${r.stderr ?? ''}`;
  return { exit: r.status, signal: r.signal ?? null, tap: tap(output), tail: output.slice(-TAIL), spawnError: r.error ? r.error.message : null, loadError: LOAD_ERROR.test(output) };
}

/** What the red at the start point was: the test file failing to load, or a test failing. */
const redKind = (base) => (base.loadError ? 'load-error' : 'test-failure');

const notRun = (reason, extra = {}) => ({ status: 'not-run', reason, ...extra });

// With `--plan` and `--step`, a `not-red-at-base` also says whether the step
// holds nothing but tests (plan-schema `stepTestsOnly`, the plan validator's
// PLAN_STEP_TESTS_ONLY). Such a step can never be red: the behaviour its tests
// cover already exists at its start point. Rewriting the test cannot fix that, so
// the executor sends the step up to the plan instead of into fix rounds
// (voyage-lekerepo trial 3, 2026-10-09). `null` = the plan or the step could
// not be read; without `--plan` the field is absent and nothing new happens.
//
// The plan's `Files:` is prose, read by a reader that can misread it; the
// step's own diff is a fact. So a step the plan calls tests-only is so only
// when every file it changed since the start point is a test too (release
// review of 6.3.0, round 6). Before building (`--red-only`) the diff holds the
// step's test alone, so there the plan decides. The plan file itself and
// Voyage's own files under .claude/ (progress, approvals) are not the step's change.
function testsOnly(o, cwd, baseSha) {
  if (!o.plan) return {};
  let text;
  try { text = readFileSync(o.plan, 'utf8'); }
  catch (e) { return { tests_only: null, tests_only_note: `cannot read the plan: ${e.message}` }; }
  const step = sliceSteps(text).find((s) => Number(s.n) === Number(o.step));
  if (!step) return { tests_only: null, tests_only_note: `no "### Step ${o.step}:" in the plan` };
  const fields = parseStepFields(step.body);
  if (!stepTestsOnlyFromFields(fields).testsOnly) return { tests_only: false };
  const isTest = stepTestFile(fields);
  const given = new Set(o.tests ?? []);
  const changed = [
    ...lines(git(cwd, ['diff', '--name-only', '--no-renames', baseSha]).stdout),
    ...lines(git(cwd, ['ls-files', '--others', '--exclude-standard']).stdout),
  ];
  const planRel = relative(cwd, realpathSync(o.plan)).split(sep).join('/');
  const other = [...new Set(changed)].filter((p) => p !== planRel && !p.startsWith('.claude/') && !given.has(p) && !isTest(p));
  if (other.length === 0) return { tests_only: true };
  return { tests_only: false, tests_only_note: `the plan's Files: names only tests, but the step's diff also changes ${other.slice(0, 5).join(', ')}` };
}

// A program that could not start (a spawn error: not installed) or a signal
// (the timeout) is not a test verdict at all. With no shell, an exit 126/127
// is the test program's own (a script whose tool the step has not built yet):
// a red like any other, and the control run tells an environment fault apart
// (release review of 6.3.0, round 6).
function unrunnable(r) {
  if (r.spawnError) return `the command could not start: ${r.spawnError}`;
  if (r.exit === null) return `the command was killed (${r.signal ?? 'timeout'})`;
  return null;
}

const git = (cwd, args) => spawnSync('git', args, { cwd, encoding: 'utf8', maxBuffer: 256 * 1024 * 1024 });
const lines = (s) => String(s ?? '').split('\n').filter(Boolean);

function safeRel(p) {
  return p && !isAbsolute(p) && !p.split('/').includes('..');
}

function copyIn(from, to, rel) {
  mkdirSync(dirname(join(to, rel)), { recursive: true });
  rmSync(join(to, rel), { recursive: true, force: true });
  cpSync(join(from, rel), join(to, rel), { recursive: true, verbatimSymlinks: true });
}

/** Symlink every gitignored top-level entry of the working tree, plus `extra`, into `tree`. */
function linkDependencies(root, tree, extra) {
  const linked = [];
  const ignored = lines(git(root, ['ls-files', '--others', '--ignored', '--exclude-standard', '--directory']).stdout)
    .map((p) => p.replace(/\/$/, '').split('/')[0]);
  for (const l of new Set([...ignored, ...extra])) {
    if (!safeRel(l) || l === '.git') continue;
    if (existsSync(join(root, l)) && !existsSync(join(tree, l))) {
      mkdirSync(dirname(join(tree, l)), { recursive: true });
      symlinkSync(join(root, l), join(tree, l));
      linked.push(l);
    }
  }
  return linked;
}

/** Lay the working tree's change since `base` onto `tree` (tracked edits, deletions, untracked files). */
function applyResult(root, tree, base) {
  const status = lines(git(root, ['diff', '--name-status', '--no-renames', base]).stdout);
  for (const s of status) {
    const [code, rel] = s.split('\t');
    if (!safeRel(rel)) continue;
    if (code.startsWith('D')) rmSync(join(tree, rel), { recursive: true, force: true });
    else if (existsSync(join(root, rel)) || lstatSync(join(root, rel), { throwIfNoEntry: false })) copyIn(root, tree, rel);
  }
  for (const rel of lines(git(root, ['ls-files', '--others', '--exclude-standard']).stdout)) {
    if (safeRel(rel)) copyIn(root, tree, rel);
  }
}

/**
 * @param {{cwd: string, base: string, cmd: string, tests: string[], link?: string[], redOnly?: boolean, timeoutMs?: number, plan?: string, step?: number}} o
 * @returns {{status: 'proven'|'rejected'|'not-run', reason: string, base?: object, result?: object, control?: object}}
 */
export function proveRed(o) {
  const given = resolve(o.cwd ?? process.cwd());
  const timeoutMs = o.timeoutMs ?? DEFAULT_TIMEOUT_MS;
  const tests = (o.tests ?? []).filter(Boolean);
  if (!existsSync(given)) return notRun(`--cwd ${given} does not exist`);
  const cwd = repoRoot(given);
  if (!cwd) return notRun(`--cwd ${given} is not inside a git repository`);
  if (tests.length === 0) return notRun('no test files given: a step without its test cannot prove red');
  for (const t of tests) {
    if (!safeRel(t)) return notRun(`test path must be repo-relative: ${t}`);
    if (!existsSync(join(cwd, t))) return notRun(`test file not found in the working tree: ${t}`);
  }
  for (const l of o.link ?? []) if (!safeRel(l)) return notRun(`--link must be repo-relative: ${l}`);
  const allowed = allowedCommand(o.cmd, { cwd });
  if (!allowed.allowed) return notRun(`command not run: ${allowed.reason}`);
  const screen = screenCommand(o.cmd);
  if (!screen.allowed) return notRun(`command blocked: ${screen.rule}`);
  const sha = git(cwd, ['rev-parse', '--verify', '--quiet', `${o.base}^{commit}`]);
  if (sha.status !== 0) return notRun(`cannot resolve the start point ${o.base} in ${cwd}`);
  const baseSha = sha.stdout.trim();

  // The result first: it is the working tree as it stands. `redOnly` is the
  // gate before building (the step helper's own check): the result is not run.
  let result;
  if (!o.redOnly) {
    result = run(o.cmd, cwd, timeoutMs);
    const resultBroken = unrunnable(result);
    if (resultBroken) return notRun(`at the result: ${resultBroken}`, { result });
  }

  const tmp = mkdtempSync(join(tmpdir(), 'voyage-red-proof-'));
  try {
    const tree = join(tmp, 'tree');
    const c = git(tmp, ['clone', '--quiet', '--shared', '--no-checkout', cwd, tree]);
    if (c.status !== 0) return notRun(`could not clone the start point: ${String(c.stderr).trim()}`, { result });
    const co = git(tree, ['checkout', '--quiet', '--detach', baseSha]);
    if (co.status !== 0) return notRun(`could not check out ${baseSha}: ${String(co.stderr).trim()}`, { result });
    for (const t of tests) copyIn(cwd, tree, t);
    const linked = linkDependencies(cwd, tree, o.link ?? []);

    const base = run(o.cmd, tree, timeoutMs);
    const baseBroken = unrunnable(base);
    if (baseBroken) return notRun(`at the start point: ${baseBroken}`, { base, result, linked });
    const out = { base_sha: baseSha, linked, base, result };
    if (base.exit === 0) return { status: 'rejected', reason: 'not-red-at-base', ...testsOnly(o, cwd, baseSha), ...out };
    if (o.redOnly) return { status: 'proven', reason: 'red at the start point (--red-only: the result was not run)', red_kind: redKind(base), base_sha: baseSha, linked, base };
    if (result.exit !== 0) return { status: 'rejected', reason: 'not-green-at-result', ...out };

    // Control: the same copy with the whole change laid on. Red here means the
    // copy itself cannot run the test green, so its red proved nothing.
    applyResult(cwd, tree, baseSha);
    const control = run(o.cmd, tree, timeoutMs);
    if (control.exit !== 0) {
      return notRun('the control run (start-point copy + the change) stayed red: the start point failed for an environment reason, not for the missing change', { ...out, control });
    }
    return { status: 'proven', reason: 'red at the start point, green at the result, green in the control', red_kind: redKind(base), ...out, control };
  } finally {
    rmSync(tmp, { recursive: true, force: true });
  }
}

// --- CLI ------------------------------------------------------------------------

const USAGE =
  'usage: red-proof.mjs --base <start_sha> --cmd "<test command>" --tests <a,b> [--plan <plan.md> --step <N>] [--red-only] [--cwd <dir>] [--link <dir,...>] [--timeout <ms>] [--json]';

const list = (v) => String(v).split(',').map((s) => s.trim()).filter(Boolean);

export function main(argv) {
  const o = { cwd: process.cwd(), tests: [], link: [] };
  let json = false;
  for (let i = 0; i < argv.length; i++) {
    const a = argv[i];
    if (a === '--base' && argv[i + 1]) o.base = argv[++i];
    else if (a === '--cmd' && argv[i + 1]) o.cmd = argv[++i];
    else if (a === '--tests' && argv[i + 1]) o.tests = list(argv[++i]);
    else if (a === '--cwd' && argv[i + 1]) o.cwd = resolve(argv[++i]);
    else if (a === '--link' && argv[i + 1]) o.link = list(argv[++i]);
    else if (a === '--plan' && argv[i + 1]) o.plan = resolve(argv[++i]);
    else if (a === '--step' && argv[i + 1]) o.step = Number(argv[++i]);
    else if (a === '--red-only') o.redOnly = true;
    else if (a === '--timeout' && argv[i + 1]) o.timeoutMs = Number(argv[++i]);
    else if (a === '--json') json = true;
    else { process.stderr.write(`red-proof: unknown argument ${a} (--tests takes ONE comma-separated list)\n${USAGE}\n`); return 2; }
  }
  if (!o.base || !o.cmd) { process.stderr.write(`red-proof: --base and --cmd are required\n${USAGE}\n`); return 2; }
  if (o.plan && !(Number.isInteger(o.step) && o.step > 0)) { process.stderr.write(`red-proof: --plan needs --step <N>\n${USAGE}\n`); return 2; }
  if (o.step !== undefined && !o.plan) { process.stderr.write(`red-proof: --step needs --plan <plan.md>\n${USAGE}\n`); return 2; }
  if (o.timeoutMs !== undefined && !Number.isFinite(o.timeoutMs)) { process.stderr.write(`red-proof: --timeout must be a number\n${USAGE}\n`); return 2; }
  let r;
  try { r = proveRed(o); }
  catch (e) { r = notRun(`red-proof crashed: ${e.message}`); }
  if (json) process.stdout.write(JSON.stringify(r, null, 2) + '\n');
  else {
    const counts = (x) => (x?.tap ? ` (tap: ${x.tap.fail} of ${x.tap.tests} failed)` : '');
    process.stdout.write(`red proof: ${r.status.toUpperCase()} - ${r.reason}${r.red_kind ? ` (red: ${r.red_kind})` : ''}\n`);
    if (r.tests_only === true) process.stdout.write('  tests_only: true - every file in the step\'s Files: is a test, so it can never be red: up to the plan\n');
    if (r.base) process.stdout.write(`  start point ${r.base_sha ?? o.base}: exit ${r.base.exit}${counts(r.base)}\n`);
    if (r.result) process.stdout.write(`  result: exit ${r.result.exit}${counts(r.result)}\n`);
    if (r.control) process.stdout.write(`  control: exit ${r.control.exit}${counts(r.control)}\n`);
    if (r.linked?.length) process.stdout.write(`  linked into the start point: ${r.linked.join(', ')}\n`);
  }
  return r.status === 'proven' ? 0 : r.status === 'rejected' ? 1 : 2;
}

function isMain() {
  try { return realpathSync(process.argv[1]) === realpathSync(fileURLToPath(import.meta.url)); }
  catch { return false; }
}

if (process.argv[1] && isMain()) process.exitCode = main(process.argv.slice(2));