voyage/lib/verification/red-proof.mjs

// lib/verification/red-proof.mjs
// Proof of red (P9 (1)). A step's test is run by a command twice:
//
//   1. against the START POINT (the step's recorded start_sha, with the step's
//      test files copied in): it must FAIL - otherwise the test proves nothing
//      about the change;
//   2. against the RESULT (the working tree as the step left it): it must PASS.
//
// `--red-only` runs step 1 alone: the gate the step helper passes BEFORE it
// builds (and locks the test). The executor's own check afterwards is the full
// two-sided run.
//
// Exit 0 = proven, 1 = rejected (`not-red-at-base` or `not-green-at-result`),
// 2 = could not run (`not-run`: no test files, an unknown start point, a command
// outside the test-runner allowlist, a command that cannot start). 2 is never a
// pass and never "proven".
//
// The start point is materialized with `git archive <start_sha>` into a temp
// directory, never by checking anything out in the working tree: the step's
// work is not touched, and no worktree is registered in the repo. Tracked files
// only - pass `--link node_modules` (comma list) to symlink an untracked
// dependency directory from the working tree into the start-point copy.
//
// The command must be ONE test-runner invocation without shell operators
// (criteria-runner.mjs `allowedCommand`), and it is screened through the
// plugin's own PreToolUse denylist (`screenCommand`): a command spawned from
// node never passes through the Bash tool, so the hook cannot fire by itself.
//
// When the output is TAP (`--test-reporter=tap`, pytest-tap, ...), the counts
// `# tests N` / `# fail N` are reported for both runs. They show WHAT the red
// was: "fail 1 of 1" on a node test file is usually a load error (an import the
// start point lacks), not named failing tests - see
// tests/fixtures/red-first/02243c6-always-allow-shim.mjs, re-measured through
// this tool in tests/lib/red-proof.test.mjs.
//
// Borrowed: GitHub Spec Kit (MIT, github/spec-kit@2e941fe) - its test-first
// rule is an instruction there; here a command proves it.

import { cpSync, existsSync, mkdirSync, mkdtempSync, realpathSync, rmSync, symlinkSync } from 'node:fs';
import { spawnSync } from 'node:child_process';
import { tmpdir } from 'node:os';
import { dirname, isAbsolute, join, resolve } from 'node:path';
import { fileURLToPath } from 'node:url';
import { allowedCommand, screenCommand } from './criteria-runner.mjs';

export const DEFAULT_TIMEOUT_MS = 900_000;
const TAIL = 2000;

function tap(output) {
  const tests = String(output).match(/^# tests (\d+)$/m);
  const fail = String(output).match(/^# fail (\d+)$/m);
  return tests && fail ? { tests: Number(tests[1]), fail: Number(fail[1]) } : null;
}

// The command's exit code must be its own. Under `node --test` a child inherits
// NODE_TEST_CONTEXT, and a nested `node --test` then reports to that parent and
// exits 0 whatever its tests did (measured 2026-10-06: a red start point read as
// green). So the variable never reaches the command.
export function commandEnv(env = process.env) {
  const { NODE_TEST_CONTEXT, ...rest } = env;
  return rest;
}

function run(cmd, cwd, timeoutMs) {
  const r = spawnSync('/bin/sh', ['-c', cmd], { cwd, env: commandEnv(), encoding: 'utf8', timeout: timeoutMs, maxBuffer: 64 * 1024 * 1024 });
  const output = `${r.stdout ?? ''}${r.stderr ?? ''}`;
  return { exit: r.status, signal: r.signal ?? null, tap: tap(output), tail: output.slice(-TAIL), spawnError: r.error ? r.error.message : null };
}

const notRun = (reason, extra = {}) => ({ status: 'not-run', reason, ...extra });

// A shell that could not find or execute the command (126/127), a signal (the
// timeout), or a spawn error is not a test verdict at all.
function unrunnable(r) {
  if (r.spawnError) return `the command could not start: ${r.spawnError}`;
  if (r.exit === null) return `the command was killed (${r.signal ?? 'timeout'})`;
  if (r.exit === 126 || r.exit === 127) return `the shell could not run the command (exit ${r.exit})`;
  return null;
}

/**
 * @param {{cwd: string, base: string, cmd: string, tests: string[], link?: string[], timeoutMs?: number}} o
 * @returns {{status: 'proven'|'rejected'|'not-run', reason: string, base?: object, result?: object}}
 */
export function repoRoot(dir) {
  const top = spawnSync('git', ['rev-parse', '--show-toplevel'], { cwd: dir, encoding: 'utf8' });
  return top.status === 0 ? top.stdout.trim() : null;
}

export function proveRed(o) {
  const given = resolve(o.cwd ?? process.cwd());
  const timeoutMs = o.timeoutMs ?? DEFAULT_TIMEOUT_MS;
  const tests = (o.tests ?? []).filter(Boolean);
  if (!existsSync(given)) return notRun(`--cwd ${given} does not exist`);
  // Plan paths and commands are relative to the repo root, and `git archive`
  // run from a subdirectory packs only that subdirectory (measured
  // 2026-10-06). So everything below runs at the root of --cwd's repo.
  const cwd = repoRoot(given);
  if (!cwd) return notRun(`--cwd ${given} is not inside a git repository`);
  if (tests.length === 0) return notRun('no test files given: a step without its test cannot prove red');
  for (const t of tests) {
    if (isAbsolute(t) || t.split('/').includes('..')) return notRun(`test path must be repo-relative: ${t}`);
    if (!existsSync(join(cwd, t))) return notRun(`test file not found in the working tree: ${t}`);
  }
  const allowed = allowedCommand(o.cmd, { cwd });
  if (!allowed.allowed) return notRun(`command not run: ${allowed.reason}`);
  const screen = screenCommand(o.cmd);
  if (!screen.allowed) return notRun(`command blocked: ${screen.rule}`);
  const sha = spawnSync('git', ['rev-parse', '--verify', '--quiet', `${o.base}^{commit}`], { cwd, encoding: 'utf8' });
  if (sha.status !== 0) return notRun(`cannot resolve the start point ${o.base} in ${cwd}`);
  const baseSha = sha.stdout.trim();

  // The result first: it is the working tree as it stands. `redOnly` is the
  // gate before building (the step helper's own check): the result is not run.
  let result;
  if (!o.redOnly) {
    result = run(o.cmd, cwd, timeoutMs);
    const resultBroken = unrunnable(result);
    if (resultBroken) return notRun(`at the result: ${resultBroken}`, { result });
  }

  const tmp = mkdtempSync(join(tmpdir(), 'voyage-red-proof-'));
  try {
    const tree = join(tmp, 'tree');
    mkdirSync(tree);
    const tar = join(tmp, 'start.tar');
    const a = spawnSync('git', ['archive', '--format=tar', '-o', tar, baseSha], { cwd, encoding: 'utf8' });
    if (a.status !== 0) return notRun(`git archive ${baseSha} failed: ${String(a.stderr).trim()}`, { result });
    const x = spawnSync('tar', ['-xf', tar, '-C', tree], { encoding: 'utf8' });
    if (x.status !== 0) return notRun(`tar could not unpack the start point: ${String(x.stderr).trim()}`, { result });
    for (const t of tests) {
      mkdirSync(dirname(join(tree, t)), { recursive: true });
      cpSync(join(cwd, t), join(tree, t));
    }
    for (const l of o.link ?? []) {
      if (!l || isAbsolute(l) || l.split('/').includes('..')) return notRun(`--link must be repo-relative: ${l}`, { result });
      if (existsSync(join(cwd, l)) && !existsSync(join(tree, l))) {
        mkdirSync(dirname(join(tree, l)), { recursive: true });
        symlinkSync(join(cwd, l), join(tree, l));
      }
    }
    const base = run(o.cmd, tree, timeoutMs);
    const baseBroken = unrunnable(base);
    if (baseBroken) return notRun(`at the start point: ${baseBroken}`, { base, result });

    const out = { base_sha: baseSha, base, result };
    if (base.exit === 0) return { status: 'rejected', reason: 'not-red-at-base', ...out };
    if (o.redOnly) return { status: 'proven', reason: 'red at the start point (--red-only: the result was not run)', base_sha: baseSha, base };
    if (result.exit !== 0) return { status: 'rejected', reason: 'not-green-at-result', ...out };
    return { status: 'proven', reason: 'red at the start point, green at the result', ...out };
  } finally {
    rmSync(tmp, { recursive: true, force: true });
  }
}

// --- CLI ------------------------------------------------------------------------

const USAGE =
  'usage: red-proof.mjs --base <start_sha> --cmd "<test command>" --tests <a,b> [--red-only] [--cwd <dir>] [--link <dir,...>] [--timeout <ms>] [--json]';

const list = (v) => String(v).split(',').map((s) => s.trim()).filter(Boolean);

export function main(argv) {
  const o = { cwd: process.cwd(), tests: [], link: [] };
  let json = false;
  for (let i = 0; i < argv.length; i++) {
    const a = argv[i];
    if (a === '--base' && argv[i + 1]) o.base = argv[++i];
    else if (a === '--cmd' && argv[i + 1]) o.cmd = argv[++i];
    else if (a === '--tests' && argv[i + 1]) o.tests = list(argv[++i]);
    else if (a === '--cwd' && argv[i + 1]) o.cwd = resolve(argv[++i]);
    else if (a === '--link' && argv[i + 1]) o.link = list(argv[++i]);
    else if (a === '--red-only') o.redOnly = true;
    else if (a === '--timeout' && argv[i + 1]) o.timeoutMs = Number(argv[++i]);
    else if (a === '--json') json = true;
    else { process.stderr.write(`red-proof: unknown argument ${a}\n${USAGE}\n`); return 2; }
  }
  if (!o.base || !o.cmd) { process.stderr.write(`red-proof: --base and --cmd are required\n${USAGE}\n`); return 2; }
  if (o.timeoutMs !== undefined && !Number.isFinite(o.timeoutMs)) { process.stderr.write(`red-proof: --timeout must be a number\n${USAGE}\n`); return 2; }
  let r;
  try { r = proveRed(o); }
  catch (e) { r = notRun(`red-proof crashed: ${e.message}`); }
  if (json) process.stdout.write(JSON.stringify(r, null, 2) + '\n');
  else {
    const counts = (x) => (x?.tap ? ` (tap: ${x.tap.fail} of ${x.tap.tests} failed)` : '');
    process.stdout.write(`red proof: ${r.status.toUpperCase()} - ${r.reason}\n`);
    if (r.base) process.stdout.write(`  start point ${r.base_sha ?? o.base}: exit ${r.base.exit}${counts(r.base)}\n`);
    if (r.result) process.stdout.write(`  result: exit ${r.result.exit}${counts(r.result)}\n`);
  }
  return r.status === 'proven' ? 0 : r.status === 'rejected' ? 1 : 2;
}

function isMain() {
  try { return realpathSync(process.argv[1]) === realpathSync(fileURLToPath(import.meta.url)); }
  catch { return false; }
}

if (process.argv[1] && isMain()) process.exitCode = main(process.argv.slice(2));