voyage/lib/verification/red-proof.mjs

// lib/verification/red-proof.mjs
// Proof of red (P9 (1)). A step's test is run by a command:
//
//   1. against the START POINT (the step's recorded start_sha, with the step's
//      test files copied in): it must FAIL - otherwise the test proves nothing
//      about the change;
//   2. against the RESULT (the working tree as the step left it): it must PASS;
//   3. CONTROL: the start-point copy with the step's whole change laid on top
//      must PASS too. A copy that cannot go green even with the change failed
//      step 1 for an environment reason (a missing dependency, a path, a tool),
//      not because the change was missing - that is "not-run", never proven.
//
// `--red-only` runs step 1 alone: the gate the step helper passes BEFORE it
// builds (and locks the test). The executor's own check afterwards is the full
// run, and only the full run is authoritative.
//
// `--plan <plan.md> --step <N>` adds `tests_only` to a `not-red-at-base`: true
// when every file in the step's `Files:` is a test, so the step can never be
// red and goes up to the plan. The plan's `Files:` alone decides (see
// `testsOnly` below).
//
// With `--plan <plan.md> --step <N>` the test list is also checked against the
// step's `Test first:` File line (plan-schema `testFirstFiles`: the leading
// backticked paths, never a path in the description - operator decision 20).
// No `--tests`: the plan's list is used, and that is the documented call. A
// `--tests` whose set differs from the plan's is `not-run` with reason
// `tests-mismatch` and the plan's list in `plan_tests` (review cf241526). A plan
// or step that cannot be read is `not-run` with a reason that starts
// `plan-unreadable`, whether or not `--tests` is given (operator decision 30,
// 2026-10-14). An empty plan list (a 1.7 step without `Test first:`, or below
// 1.8 a `File:` line without backticks) changes nothing here: the caller gives
// `--tests` (operator decision 41). Every result from the point the test list
// is settled on carries it in `tests` (the list it ran); the text output prints it on a `  tests:` line, and
// `plan_tests` on a `  plan_tests:` line, comma-joined as `--tests` takes it.
//
// Exit 0 = proven, 1 = rejected (`not-red-at-base` or `not-green-at-result`),
// 2 = could not run (`not-run`: no test files, a plan or step that cannot be
// read, an unknown start point, a command outside the test-runner allowlist, a
// command that cannot start, a control run that stays red). 2 is never a pass and never "proven".
//
// The start point is a temp `git clone --shared` of the repo checked out at
// start_sha - a real git checkout, so a git-reading command behaves as it does
// in the repo, and nothing in the working tree or the repo's worktree list is
// touched. Every gitignored top-level entry of the working tree (node_modules,
// .venv, a build cache) is symlinked in, so the copy has the same dependencies;
// `--link a,b` adds deeper ones. Measured 2026-10-06 (review): with a bare
// `git archive` copy, a test that needs node_modules "proved red" with no change.
// Known limit, failing safe: a dependency that points back INTO the working
// tree (an editable `pip install -e .`, a workspace symlink) makes the start
// point see the new code, and a correct step reads `not-red-at-base`.
//
// Everything runs at the root of --cwd's repository: plan paths and commands
// are root-relative, and `git archive`/`clone` paths from a subdirectory are
// not (measured 2026-10-06).
//
// The command must be ONE test-runner invocation without shell operators
// (criteria-runner.mjs `allowedCommand`), and it is screened through the
// plugin's own PreToolUse denylist (`screenCommand`): a command spawned from
// node never passes through the Bash tool, so the hook cannot fire by itself.
// It runs with no shell (`runCommand`): the words checked are the words run.
//
// When the output is TAP (`--test-reporter=tap`, pytest-tap, ...), the counts
// `# tests N` / `# fail N` are reported per run. They show WHAT the red was:
// "fail 1 of 1" on a node test file is usually a load error (an import the
// start point lacks), not named failing tests - see
// tests/fixtures/red-first/02243c6-always-allow-shim.mjs, re-measured through
// this tool in tests/lib/red-proof.test.mjs.
//
// Borrowed: GitHub Spec Kit (MIT, github/spec-kit@2e941fe) - its test-first
// rule is an instruction there; here a command proves it.

import { cpSync, existsSync, lstatSync, mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync } from 'node:fs';
import { spawnSync } from 'node:child_process';
import { tmpdir } from 'node:os';
import { dirname, isAbsolute, join, resolve } from 'node:path';
import { DEFAULT_TIMEOUT_MS, allowedCommand, runCommand, screenCommand, stoppedBy } from './criteria-runner.mjs';
import { commandEnv } from '../util/command-env.mjs';
import { extractPlanVersion, normPlanPath, parseStepFields, sliceSteps, stepTestsOnlyFromFields, testFirstFiles } from '../parsers/plan-schema.mjs';
import { isMain } from '../util/is-main.mjs';

// The one timeout, defined in criteria-runner.mjs (release review of 6.3.0, round 7).
export { DEFAULT_TIMEOUT_MS };
const TAIL = 2000;

function tap(output) {
  const tests = String(output).match(/^# tests (\d+)$/m);
  const fail = String(output).match(/^# fail (\d+)$/m);
  return tests && fail ? { tests: Number(tests[1]), fail: Number(fail[1]) } : null;
}

// The command's exit code must be its own: NODE_TEST_CONTEXT never reaches it
// (lib/util/command-env.mjs).
export { commandEnv };

export function repoRoot(dir) {
  const top = spawnSync('git', ['rev-parse', '--show-toplevel'], { cwd: dir, encoding: 'utf8' });
  return top.status === 0 ? top.stdout.trim() : null;
}

// A red that is the test file failing to LOAD (an import the start point
// lacks) rather than a named test failing. Still a red — the change is what is
// missing — but the result says which kind, in a field (`red_kind`), instead
// of leaving it to a reader of `tap` (null without a TAP reporter: the
// voyage-lekerepo trial's step 1, 2026-10-06).
// Specific signatures only: a bare `SyntaxError` anywhere also matched an
// assertion message («Missing expected exception (SyntaxError)») of a named
// failing test (independent review, 2026-10-08).
const LOAD_ERROR = /^\s*SyntaxError: |ERR_MODULE_NOT_FOUND|does not provide an export named|Cannot find module|ERR_REQUIRE_ESM|ModuleNotFoundError|ImportError: /m;

// No shell: the words the screen checked are the words that run.
function run(cmd, cwd, timeoutMs) {
  const r = runCommand(cmd, { cwd, timeoutMs });
  const output = `${r.stdout ?? ''}${r.stderr ?? ''}`;
  const stopped = stoppedBy(r, { timeoutMs });
  return { exit: r.status, signal: r.signal ?? null, tap: tap(output), tail: output.slice(-TAIL), stopped: stopped?.why ?? null, loadError: LOAD_ERROR.test(output) };
}

/** What the red at the start point was: the test file failing to load, or a test failing. */
const redKind = (base) => (base.loadError ? 'load-error' : 'test-failure');

const notRun = (reason, extra = {}) => ({ status: 'not-run', reason, ...extra });

// With `--plan` and `--step`, a `not-red-at-base` also says whether the step
// holds nothing but tests (plan-schema `stepTestsOnly`, the plan validator's
// PLAN_STEP_TESTS_ONLY). Such a step can never be red: the behaviour its tests
// cover already exists at its start point. Rewriting the test cannot fix that, so
// the executor sends the step up to the plan instead of into fix rounds
// (voyage-lekerepo trial 3, 2026-10-09). Without `--plan` the field is absent
// and nothing new happens; a plan or step that cannot be read never reaches here
// (`proveRed` returns `plan-unreadable` first).
//
// The plan's `Files:` alone decides (operator 2026-10-11): the step's diff is
// not read. In the documented flow the helper writes only the test before this
// gate, so the diff could never overrule the plan, and a change already in the
// tree when the step began (a dirty tree is allowed, trekexecute Sub-step 0)
// made a true tests-only step look like it built something (release review of
// 6.3.0, round 7). A path the step changed outside `Files:` is the
// controller's scope finding (trekexecute Sub-step C, Hard rule 2), not this check's.
function testsOnly(step) {
  if (!step) return {};
  return { tests_only: stepTestsOnlyFromFields(step.fields, step.version).testsOnly };
}

// With `--plan` and `--step`: the step's parsed fields and the plan's version,
// read once for both the test list and `testsOnly`. null without `--plan`;
// `{error}` when the plan or the step could not be read.
function planStep(o) {
  if (!o.plan) return null;
  let text;
  try { text = readFileSync(o.plan, 'utf8'); }
  catch (e) { return { error: `cannot read the plan: ${e.message}` }; }
  const step = sliceSteps(text).find((s) => Number(s.n) === Number(o.step));
  if (!step) return { error: `no "### Step ${o.step}:" in the plan` };
  return { fields: parseStepFields(step.body), version: extractPlanVersion(text) };
}

// A program that could not start (a spawn error: not installed), or one this
// run stopped (the timeout, output past the buffer, a signal), is not a test
// verdict at all (criteria-runner `stoppedBy`). With no shell, an exit 126/127
// is the test program's own (a script whose tool the step has not built yet):
// a red like any other, and the control run tells an environment fault apart
// (release review of 6.3.0, round 6).
function unrunnable(r) {
  return r.stopped ? `the command ${r.stopped}` : null;
}

const git = (cwd, args) => spawnSync('git', args, { cwd, encoding: 'utf8', maxBuffer: 256 * 1024 * 1024 });
const lines = (s) => String(s ?? '').split('\n').filter(Boolean);

function safeRel(p) {
  return p && !isAbsolute(p) && !p.split('/').includes('..');
}

function copyIn(from, to, rel) {
  mkdirSync(dirname(join(to, rel)), { recursive: true });
  rmSync(join(to, rel), { recursive: true, force: true });
  cpSync(join(from, rel), join(to, rel), { recursive: true, verbatimSymlinks: true });
}

/** Symlink every gitignored top-level entry of the working tree, plus `extra`, into `tree`. */
function linkDependencies(root, tree, extra) {
  const linked = [];
  const ignored = lines(git(root, ['ls-files', '--others', '--ignored', '--exclude-standard', '--directory']).stdout)
    .map((p) => p.replace(/\/$/, '').split('/')[0]);
  for (const l of new Set([...ignored, ...extra])) {
    if (!safeRel(l) || l === '.git') continue;
    if (existsSync(join(root, l)) && !existsSync(join(tree, l))) {
      mkdirSync(dirname(join(tree, l)), { recursive: true });
      symlinkSync(join(root, l), join(tree, l));
      linked.push(l);
    }
  }
  return linked;
}

/** Lay the working tree's change since `base` onto `tree` (tracked edits, deletions, untracked files). */
function applyResult(root, tree, base) {
  const status = lines(git(root, ['diff', '--name-status', '--no-renames', base]).stdout);
  for (const s of status) {
    const [code, rel] = s.split('\t');
    if (!safeRel(rel)) continue;
    if (code.startsWith('D')) rmSync(join(tree, rel), { recursive: true, force: true });
    else if (existsSync(join(root, rel)) || lstatSync(join(root, rel), { throwIfNoEntry: false })) copyIn(root, tree, rel);
  }
  for (const rel of lines(git(root, ['ls-files', '--others', '--exclude-standard']).stdout)) {
    if (safeRel(rel)) copyIn(root, tree, rel);
  }
}

/**
 * With `plan` and `step`: a plan or step that cannot be read is `not-run` with a
 * reason that starts `plan-unreadable`; no `tests` means the step's `Test first:`
 * File line is the list; a `tests` that is another set is `tests-mismatch` (the
 * plan's list in `plan_tests`); an empty plan list (a 1.7 step without
 * `Test first:`) changes nothing. Every result once the list is settled on
 * carries it in `tests` (the list it ran).
 * @param {{cwd: string, base: string, cmd: string, tests?: string[], link?: string[], redOnly?: boolean, timeoutMs?: number, plan?: string, step?: number}} o
 * @returns {{status: 'proven'|'rejected'|'not-run', reason: string, tests?: string[], plan_tests?: string[], base?: object, result?: object, control?: object}}
 */
export function proveRed(o) {
  const given = resolve(o.cwd ?? process.cwd());
  const timeoutMs = o.timeoutMs ?? DEFAULT_TIMEOUT_MS;
  let tests = (o.tests ?? []).filter(Boolean);
  if (!existsSync(given)) return notRun(`--cwd ${given} does not exist`);
  const cwd = repoRoot(given);
  if (!cwd) return notRun(`--cwd ${given} is not inside a git repository`);
  // The step's own test list (review cf241526): the step's `Test first:` File
  // line as testFirstFiles reads it (operator decision 20), taken when none is
  // given; a given list that is another set is refused. A plan or step that
  // cannot be read is refused with or without a given list (operator decision
  // 30); an empty plan list changes nothing.
  const step = planStep(o);
  if (step?.error) return notRun(`plan-unreadable: ${step.error}`);
  const fromPlan = step?.fields ? testFirstFiles(step.fields, step.version) : [];
  if (fromPlan.length > 0) {
    if (tests.length === 0) tests = fromPlan;
    else {
      const a = new Set(tests.map((t) => normPlanPath(String(t))));
      const b = new Set(fromPlan);
      if (a.size !== b.size || [...a].some((t) => !b.has(t))) {
        return notRun(`tests-mismatch: --tests names ${tests.join(',')}; the plan's Test first File line names ${fromPlan.join(',')}`, { plan_tests: fromPlan });
      }
    }
  }
  if (tests.length === 0) return notRun('no test files given: a step without its test cannot prove red');
  // From here on every result names the list it ran (`tests`, operator decision 30).
  const ran = (reason, extra = {}) => notRun(reason, { tests, ...extra });
  for (const t of tests) {
    if (!safeRel(t)) return ran(`test path must be repo-relative: ${t}`);
    if (!existsSync(join(cwd, t))) return ran(`test file not found in the working tree: ${t}`);
  }
  for (const l of o.link ?? []) if (!safeRel(l)) return ran(`--link must be repo-relative: ${l}`);
  const allowed = allowedCommand(o.cmd, { cwd });
  if (!allowed.allowed) return ran(`command not run: ${allowed.reason}`);
  const screen = screenCommand(o.cmd);
  if (!screen.allowed) return ran(`command blocked: ${screen.rule}`);
  const sha = git(cwd, ['rev-parse', '--verify', '--quiet', `${o.base}^{commit}`]);
  if (sha.status !== 0) return ran(`cannot resolve the start point ${o.base} in ${cwd}`);
  const baseSha = sha.stdout.trim();

  // The result first: it is the working tree as it stands. `redOnly` is the
  // gate before building (the step helper's own check): the result is not run.
  let result;
  if (!o.redOnly) {
    result = run(o.cmd, cwd, timeoutMs);
    const resultBroken = unrunnable(result);
    if (resultBroken) return ran(`at the result: ${resultBroken}`, { result });
  }

  const tmp = mkdtempSync(join(tmpdir(), 'voyage-red-proof-'));
  try {
    const tree = join(tmp, 'tree');
    const c = git(tmp, ['clone', '--quiet', '--shared', '--no-checkout', cwd, tree]);
    if (c.status !== 0) return ran(`could not clone the start point: ${String(c.stderr).trim()}`, { result });
    const co = git(tree, ['checkout', '--quiet', '--detach', baseSha]);
    if (co.status !== 0) return ran(`could not check out ${baseSha}: ${String(co.stderr).trim()}`, { result });
    for (const t of tests) copyIn(cwd, tree, t);
    const linked = linkDependencies(cwd, tree, o.link ?? []);

    const base = run(o.cmd, tree, timeoutMs);
    const baseBroken = unrunnable(base);
    if (baseBroken) return ran(`at the start point: ${baseBroken}`, { base, result, linked });
    const out = { tests, base_sha: baseSha, linked, base, result };
    if (base.exit === 0) return { status: 'rejected', reason: 'not-red-at-base', ...testsOnly(step), ...out };
    if (o.redOnly) return { status: 'proven', reason: 'red at the start point (--red-only: the result was not run)', red_kind: redKind(base), tests, base_sha: baseSha, linked, base };
    if (result.exit !== 0) return { status: 'rejected', reason: 'not-green-at-result', ...out };

    // Control: the same copy with the whole change laid on. Red here means the
    // copy itself cannot run the test green, so its red proved nothing.
    applyResult(cwd, tree, baseSha);
    const control = run(o.cmd, tree, timeoutMs);
    const controlBroken = unrunnable(control);
    if (controlBroken) return ran(`in the control run: ${controlBroken}`, { ...out, control });
    if (control.exit !== 0) {
      return ran('the control run (start-point copy + the change) stayed red: the start point failed for an environment reason, not for the missing change', { ...out, control });
    }
    return { status: 'proven', reason: 'red at the start point, green at the result, green in the control', red_kind: redKind(base), ...out, control };
  } finally {
    rmSync(tmp, { recursive: true, force: true });
  }
}

// --- CLI ------------------------------------------------------------------------

const USAGE =
  'usage: red-proof.mjs --base <start_sha> --cmd "<test command>" [--tests <a,b>] [--plan <plan.md> --step <N>] [--red-only] [--cwd <dir>] [--link <dir,...>] [--timeout <ms>] [--json]\n' +
  '  with --plan/--step the step\'s Test first File line gives the test list and the documented call passes no --tests; a --tests given with them must name the same set; a plan or step that cannot be read is not-run (plan-unreadable). Without --plan, or for a step whose Test first File line lists no test, --tests is required';

const list = (v) => String(v).split(',').map((s) => s.trim()).filter(Boolean);

export function main(argv) {
  const o = { cwd: process.cwd(), tests: [], link: [] };
  let json = false;
  for (let i = 0; i < argv.length; i++) {
    const a = argv[i];
    if (a === '--base' && argv[i + 1]) o.base = argv[++i];
    else if (a === '--cmd' && argv[i + 1]) o.cmd = argv[++i];
    else if (a === '--tests' && argv[i + 1]) o.tests = list(argv[++i]);
    else if (a === '--cwd' && argv[i + 1]) o.cwd = resolve(argv[++i]);
    else if (a === '--link' && argv[i + 1]) o.link = list(argv[++i]);
    else if (a === '--plan' && argv[i + 1]) o.plan = resolve(argv[++i]);
    else if (a === '--step' && argv[i + 1]) o.step = Number(argv[++i]);
    else if (a === '--red-only') o.redOnly = true;
    else if (a === '--timeout' && argv[i + 1]) o.timeoutMs = Number(argv[++i]);
    else if (a === '--json') json = true;
    else { process.stderr.write(`red-proof: unknown argument ${a} (--tests takes ONE comma-separated list)\n${USAGE}\n`); return 2; }
  }
  if (!o.base || !o.cmd) { process.stderr.write(`red-proof: --base and --cmd are required\n${USAGE}\n`); return 2; }
  if (o.plan && !(Number.isInteger(o.step) && o.step > 0)) { process.stderr.write(`red-proof: --plan needs --step <N>\n${USAGE}\n`); return 2; }
  if (o.step !== undefined && !o.plan) { process.stderr.write(`red-proof: --step needs --plan <plan.md>\n${USAGE}\n`); return 2; }
  if (o.timeoutMs !== undefined && !Number.isFinite(o.timeoutMs)) { process.stderr.write(`red-proof: --timeout must be a number\n${USAGE}\n`); return 2; }
  let r;
  try { r = proveRed(o); }
  catch (e) { r = notRun(`red-proof crashed: ${e.message}`); }
  if (json) process.stdout.write(JSON.stringify(r, null, 2) + '\n');
  else {
    const counts = (x) => (x?.tap ? ` (tap: ${x.tap.fail} of ${x.tap.tests} failed)` : '');
    process.stdout.write(`red proof: ${r.status.toUpperCase()} - ${r.reason}${r.red_kind ? ` (red: ${r.red_kind})` : ''}\n`);
    // The exact form --files and --tests take: one comma-joined list.
    if (r.tests) process.stdout.write(`  tests: ${r.tests.join(',')}\n`);
    if (r.plan_tests) process.stdout.write(`  plan_tests: ${r.plan_tests.join(',')}\n`);
    if (r.tests_only === true) process.stdout.write('  tests_only: true - every file in the step\'s Files: is a test, so it can never be red: up to the plan\n');
    if (r.base) process.stdout.write(`  start point ${r.base_sha ?? o.base}: exit ${r.base.exit}${counts(r.base)}\n`);
    if (r.result) process.stdout.write(`  result: exit ${r.result.exit}${counts(r.result)}\n`);
    if (r.control) process.stdout.write(`  control: exit ${r.control.exit}${counts(r.control)}\n`);
    if (r.linked?.length) process.stdout.write(`  linked into the start point: ${r.linked.join(', ')}\n`);
  }
  return r.status === 'proven' ? 0 : r.status === 'rejected' ? 1 : 2;
}

if (isMain(import.meta.url)) process.exitCode = main(process.argv.slice(2));