voyage/lib/verification/criteria-runner.mjs

#!/usr/bin/env node
// lib/verification/criteria-runner.mjs
// Runs the falsifiable checks a pipeline artifact DECLARES, and reports a
// verdict built from exit codes rather than from a reader's impression.
//
// Two artifacts declare such checks:
//   plan   `## Verification`     — `- [ ] `cmd` -> expected: ...`  (V1..Vn)
//   brief  `## Success Criteria` — `- text: `cmd` ...`             (SC1..SCn)
//
// Why this is code and not prose: `/trekexecute` must be able to FELL a
// single-session run on a criterion that does not hold, and `/trekreview` must
// hand `brief-conformance-reviewer` a real result instead of asking a
// Read/Glob/Grep agent to judge whether a command "passes". Neither is
// checkable while it lives only as an instruction.
//
// Fail-closed everywhere: a criterion that cannot run — placeholder text, no
// command, a screen that is unavailable — is NEVER reported as passed.
//
// Screening, two layers: every command is first screened against this file's
// own ALLOWLIST of test runners (npm test, node --test, pytest, `bash
// tests/<script>.sh`, a read-only git subcommand, …) — anything else is NOT
// RUN, never a failure — and what survives is then screened through the
// plugin's own PreToolUse denylist (hooks/scripts/pre-bash-executor.mjs) —
// catastrophe — before it runs.
//
// No shell. A command runs as its words (`runCommand`): the runner is started
// directly with the words the screen checked, so no quoting, glob, tilde,
// brace or variable can turn them into other words after the check. Five
// review rounds of 6.3.0 found one more way /bin/sh rewrote a checked word
// (quotes, tilde, a glob that reached `..`); with no shell there is nothing
// left to rewrite (operator 2026-10-11). The same function runs every check
// Voyage runs: Phase 7 criteria, `Regression:` (regression-check.mjs), proof
// of red (red-proof.mjs), the done line (done-line.mjs), the test lock
// (lib/util/test-lock.mjs).
// Commands spawned from here do not pass through the Bash tool, so that hook
// would not otherwise fire; the hook is invoked over its documented stdin
// protocol rather than copied, so there is exactly one denylist.
//
// Usage:
//   node lib/verification/criteria-runner.mjs --plan  <plan.md>  [--json] [--cwd D] [--timeout MS]
//   node lib/verification/criteria-runner.mjs --brief <brief.md> [--json|--evidence] [--cwd D] [--timeout MS]
// --evidence emits the markdown block /trekreview hands to the conformance
// reviewer; --json emits the whole report. They are two shapes of one run.
// Exit: 0 = ok, 1 = a criterion failed / was blocked / could not run, 2 = usage
// or read error (deliberately distinct from "a criterion failed").

import { readFileSync } from 'node:fs';
import { resolve, dirname, join } from 'node:path';
import { fileURLToPath } from 'node:url';
import { spawnSync } from 'node:child_process';
import { sliceSteps, parseStepFields } from '../parsers/plan-schema.mjs';
import { commandEnv } from '../util/command-env.mjs';
import { isMain } from '../util/is-main.mjs';

const REPO_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '..', '..');

export const PLAN_HEADING = '## Verification';
export const BRIEF_HEADING = '## Success Criteria';
export const DEFAULT_HOOK = join(REPO_ROOT, 'hooks', 'scripts', 'pre-bash-executor.mjs');
// The one timeout every check Voyage runs is stopped by (15 minutes): red
// proof, the done line, the test lock, Phase 7 criteria, regression-check and
// the release check's suite (release review of 6.3.0, round 7).
export const DEFAULT_TIMEOUT_MS = 900_000;
export const DEFAULT_MAX_OUTPUT = 4000;
const MAX_BUFFER = 64 * 1024 * 1024;

// --- parsing ----------------------------------------------------------------

// Fenced blocks are read TWICE over, for two opposite reasons: a `## ` heading
// inside a fence is quoted text and must not open a section (the repo's own
// examples/02-real-cli/REGENERATED.md quotes a whole plan outline inside one),
// while a shell-tagged fence INSIDE the section holds the commands themselves
// (examples/01-add-verbose-flag/plan.md writes its whole acceptance run that
// way). Reading only bullets made a correct plan parse to zero criteria, exit
// 1, and fell every single-session run — measured 2026-09-18.
const FENCE = /^\s*(?:```|~~~)(.*)$/;

// Languages whose fenced block is a run of commands. Deliberately a closed
// list: an untagged fence is far more often expected OUTPUT than input, and
// inventing a criterion from output is exactly the failure this file exists to
// prevent.
const SHELL_LANGS = new Set(['bash', 'sh', 'shell', 'zsh', 'console', 'shell-session']);

// One entry per line: its text, whether it sits inside a fence, that fence's
// info string, and whether it IS the fence marker.
function scanLines(markdown) {
  const out = [];
  let lang = null;
  for (const text of markdown.split('\n')) {
    const m = text.match(FENCE);
    if (m) {
      const open = lang === null;
      if (open) lang = m[1].trim().toLowerCase().split(/\s+/)[0];
      out.push({ text, fenced: true, lang: lang ?? '', marker: true });
      if (!open) lang = null;
      continue;
    }
    out.push({ text, fenced: lang !== null, lang: lang ?? '', marker: false });
  }
  return out;
}

// The lines from `heading` up to the next `## ` heading (exclusive), plus the
// 1-based line the heading itself is on — a section the runner cannot read has
// to be reported with a place to look. Headings inside a fence do not count on
// either end.
function sectionLines(markdown, heading) {
  const lines = scanLines(markdown);
  const start = lines.findIndex((l) => !l.fenced && l.text.trim() === heading);
  if (start === -1) return null;
  let end = lines.length;
  for (let i = start + 1; i < lines.length; i++) {
    if (!lines[i].fenced && lines[i].text.startsWith('## ')) { end = i; break; }
  }
  return { line: start + 1, lines: lines.slice(start + 1, end) };
}

// `- [ ] text`, `- [x] text`, `- text`, `* text`, `1. text` -> text
const BULLET = /^\s*(?:[-*]|\d+\.)\s+(?:\[[ xX]\]\s+)?(.*)$/;

const ENV_ASSIGN = /^[A-Za-z_][A-Za-z0-9_]*=\S*\s+/;

/**
 * Whether a backticked span is a command to RUN or something the sentence
 * merely NAMES. A plan's template puts the command first, but a brief's
 * criterion usually opens with the thing under discussion — a flag
 * (`--verbose`), a path (`tests/`) — and running those produced exit 2
 * ("invalid option") and exit 126 ("is a directory"), which the review rubric
 * reads as decisive: a BLOCKER invented out of prose. Measured 2026-09-18:
 * 3 of the 6 criteria in the repo's own example brief.
 *
 * Shape only, no filesystem lookup, so a span parses the same way everywhere.
 * And deliberately NO scanning on to a later span: "the first span that looks
 * like a command" invents commands out of prose — in that same example brief
 * it would have run `whoami` and `login`, two real binaries a sentence happens
 * to name. An absent measurement is the honest answer; a guessed one is not.
 */
export function looksLikeCommand(span) {
  let rest = String(span ?? '').trim();
  while (ENV_ASSIGN.test(rest)) rest = rest.replace(ENV_ASSIGN, '');
  const token = rest.split(/\s+/)[0] ?? '';
  if (token === '') return false;
  if (token.startsWith('-')) return false;                      // a flag
  if (token.endsWith('/')) return false;                        // a directory
  if (!/^[A-Za-z0-9_.@+~/-]+$/.test(token)) return false;       // quotes, braces, prose
  if (!/[A-Za-z0-9]/.test(token)) return false;
  // A lone relative path with a slash is a file the sentence names. An explicit
  // `./`, `../`, `/` or `~/` is an invocation and still runs.
  if (rest === token && token.includes('/') && !/^(\.{1,2}\/|\/|~\/)/.test(token)) return false;
  return true;
}

const COMMENT_NOT_PLAIN = 'a trailing comment with a quote, $, backtick or backslash (the span reaches a shell inside double quotes)';

// The first backtick-delimited span on the line is the command by convention —
// both templates put it first and a second span holds the expected output.
// A plan step's `Verify:` and `End-to-end:` fields follow the same convention.
export function firstCommand(text) {
  const m = text.match(/`([^`]+)`/);
  if (!m) return { command: null, reason: 'no-command' };
  const span = m[1].trim();
  const raw = dropComment(span);
  // Template placeholders (`{exact command}`) are not commands.
  if (span === '' || /^\{.*\}$/.test(raw)) return { command: null, reason: 'placeholder' };
  // A span that is only a comment names nothing to run (round 8).
  if (raw === '' || !looksLikeCommand(raw)) return { command: null, reason: 'not-a-command' };
  // The comment is never screened, but the executor hands the span as written
  // to red proof, the test lock and the done line inside double quotes
  // (`--cmd "{Verify command}"`): there a `"` ends the argument and a `$`, a
  // backtick or a backslash still means something (release review of 6.3.0,
  // round 8). A comment with one of them is refused, never dropped.
  if (/["$`\\]/.test(span.slice(raw.length))) return { command: null, reason: COMMENT_NOT_PLAIN };
  return { command: raw, reason: '' };
}

/**
 * A command without its trailing comment (`npm test   # all green`): a shell
 * drops it, and with no shell (runCommand) its words would reach the screen
 * and make an honest check NOT RUN (release review of 6.3.0, round 6). One
 * reading for every place a command comes from: a fence line, a backticked
 * span (`Verify:`, `Regression:`, `End-to-end:`, a `## Verification` bullet)
 * and a command passed as written (red proof, the done line, the test lock),
 * so the words screened are the words run (round 7). Only a `#` that opens a
 * word on the same line is a comment: `tests/a#b.test.mjs` is kept whole here,
 * and the allowlist then refuses the word, since a `#` is not a plain word
 * (round 8).
 */
export function dropComment(command) {
  return String(command ?? '').replace(/[ \t]+#[^\n]*$/, '').trim();
}

// A command line inside a shell-tagged fence. A console block may prefix the
// line with a `$` or `>` prompt; blank and comment-only lines declare nothing.
// A `#` root prompt is NOT stripped: it is indistinguishable from a comment,
// and running a comment is the worse of the two mistakes.
function fencedCommand(text) {
  const line = text.trim();
  if (line === '' || line.startsWith('#')) return null;
  return dropComment(line.replace(/^[$>]\s+/, ''));
}

function parseSection(markdown, heading, prefix) {
  const section = sectionLines(markdown, heading);
  if (section === null) return [];
  const lines = section.lines;
  const criteria = [];
  const push = (text, command, reason) =>
    criteria.push({ label: `${prefix}${criteria.length + 1}`, text, command, reason });
  // A bullet indented deeper than the section's shallowest bullet, carrying no
  // command of its own, is a sub-point of the criterion above it, never a
  // criterion of its own: one criterion with ten sub-points became SC4..SC13,
  // each `not-a-command` (voyage-lekerepo trial 2, 2026-10-08). The parent
  // counts them in `subpoints`. A sub-bullet that DOES carry a command stays a
  // criterion: folding it would leave its command unrun (independent review).
  const indentOf = (text) => text.match(/^\s*/)[0].replace(/\t/g, '    ').length;
  const bulletIndents = lines.filter((l) => !l.fenced && !l.marker && BULLET.test(l.text)).map((l) => indentOf(l.text));
  const baseIndent = bulletIndents.length ? Math.min(...bulletIndents) : 0;
  let lastBullet = null;
  for (const line of lines) {
    if (line.marker) continue;
    if (line.fenced) {
      if (!SHELL_LANGS.has(line.lang)) continue;
      const command = fencedCommand(line.text);
      if (command === null) continue;
      push(command, command, '');
      continue;
    }
    const bullet = line.text.match(BULLET);
    if (!bullet) continue;
    const nested = indentOf(line.text) > baseIndent;
    const text = bullet[1].trim();
    if (text === '') continue;
    const { command, reason } = firstCommand(text);
    if (nested && lastBullet && !command) {
      lastBullet.subpoints = (lastBullet.subpoints ?? 0) + 1;
      continue;
    }
    push(text, command, reason);
    if (!nested) lastBullet = criteria[criteria.length - 1];
  }
  return criteria;
}

/** Criteria declared in a plan's `## Verification` section, labelled V1..Vn. */
export function parsePlanVerification(markdown) {
  return parseSection(markdown, PLAN_HEADING, 'V');
}

/** Criteria declared in a brief's `## Success Criteria` section, labelled SC1..SCn. */
export function parseSuccessCriteria(markdown) {
  return parseSection(markdown, BRIEF_HEADING, 'SC');
}

// --- screening --------------------------------------------------------------

/**
 * Screen one command through the plugin's PreToolUse denylist.
 * Hook protocol: stdin JSON, exit 2 = block, exit 0 = allow. Any other exit
 * means the screen did not render a verdict — that denies, it never allows.
 */
export function screenCommand(command, opts = {}) {
  const hookPath = opts.hookPath ?? DEFAULT_HOOK;
  const r = spawnSync(process.execPath, [hookPath], {
    input: JSON.stringify({ tool_name: 'Bash', tool_input: { command: dropComment(command) } }),
    encoding: 'utf8',
    timeout: 30_000,
  });
  if (r.status === 0) return { allowed: true, rule: '' };
  const first = String(r.stderr || '').split('\n')[0].trim();
  if (r.status === 2) {
    return { allowed: false, rule: first.replace(/^\[voyage\]\s*BLOCKED:\s*/, '') || 'blocked by the executor denylist' };
  }
  return {
    allowed: false,
    rule: `screen unavailable (${hookPath} exited ${r.status === null ? 'on signal' : r.status}) — denied, never assumed safe`,
  };
}

// --- the allowlist ----------------------------------------------------------

// The executor denylist screens CATASTROPHE (rm -rf /, fork bombs, mkfs). It
// was never meant to screen an artifact that arrives from outside the repo,
// and /trekreview runs the commands a BRIEF declares. The first screen written
// for that was a denylist of writes; measured 2026-09-18, end to end through
// both layers, 5 of 11 named evasions still ran with real effect — `command`
// in front of the command name, an escaped `\rm` inside a shell fence,
// `find -delete`, `>|`, `tee`, a python one-liner — and 19 of 28 got past the
// refusal list on its own. A denylist in front of a shell is whack-a-mole:
// every quoting, aliasing and indirection form of /bin/sh is another mole.
//
// So the screen is an ALLOWLIST. A criterion runs only when the command is a
// known TEST RUNNER invocation and carries no shell operator at all. Anything
// else is NOT RUN, with the reason said out loud: it never reaches a shell,
// and — this is the other half — it is never reported as a failure either. An
// absent measurement is not a finding.
//
// What the allowlist deliberately does NOT do:
//   - It is not a sandbox. `npm test`, `npm run <script>` and `make test` run
//     whatever the repo's own package.json/Makefile says they run, including a
//     script that pushes. That is the repo's responsibility, not this file's.
//   - It is not an option parser. Each runner takes only plain words and a
//     short list of options that only read (RUNNER_ARGS); an option that
//     writes a file or loads code (`--test-reporter-destination`,
//     `--outputFile`, `--junitxml`, `--basetemp`, `--import`, `--config`,
//     `-o`, `-exec`) is on no list (release review of 6.3.0, round 3: rounds
//     1-2 closed this for git only).
//   - Plain words: the words reach the runner as written, so a word that
//     would mean something else in a shell — a quote, backslash, `$`, brace,
//     a leading `~` — is refused rather than passed on with a meaning its
//     author did not intend. A glob is passed on only to `node --test`, which
//     expands its own patterns (Node 21 and later); no other runner here
//     expands one, so elsewhere it is refused. A path that climbs out with
//     `..` or starts at `/` is refused: a check runs in the repo. A word that
//     starts with `@` is refused: pytest (8.2 and later) reads `@file` as a
//     file of more arguments, options the list would refuse.
//   - It rejects plenty of honest commands: an env prefix (`CI=1 npm test`), a
//     project's own binary (`./build/app --check`), anything piped. A check
//     that needs one of those is declared through a runner instead —
//     `bash tests/<script>.sh` is the documented way in.

// Anything that would let one command become two, or redirect a stream.
const SHELL_OPERATORS = /[;&|<>]|\$\(|`/;

// Read-only git subcommands: they report, they do not write a repository.
const READONLY_GIT = new Set(['status', 'log', 'diff', 'show', 'ls-files']);
// `--output`, `--output=<file>`, or an abbreviation of it git may accept.
function writesOutput(word) {
  const name = word.split('=')[0];
  return name.length >= 5 && '--output'.startsWith(name);
}

const deny = (reason) => ({ allowed: false, reason });
const permit = () => ({ allowed: true, reason: '' });

// A word that means the same with or without a shell (also git's refs like
// HEAD~1 and HEAD^, formats like %h, paths).
const PLAIN_WORD = /^[A-Za-z0-9_@%^~:.,=+/-]+$/;
// The same, with a glob in it: passed on only to `node --test`, which expands
// its own patterns from Node 21 on (sh would have expanded `.?` to `..`).
const GLOB_WORD = /^[A-Za-z0-9_./][A-Za-z0-9_@%^~:.,=+/*?-]*$/;
const NODE_GLOB_MAJOR = 21;
const runningNodeMajor = () => Number(process.versions.node.split('.')[0]);
// A path outside the repo: one that climbs with `..` or starts at `/`.
const leavesRepo = (w) => w.startsWith('/') || w.split('/').includes('..');
// The same for a git word, which may also name a path as `<rev>:<path>`.
const gitLeavesRepo = (w) => leavesRepo(w) || (w.includes(':') && leavesRepo(w.slice(w.indexOf(':') + 1)));

// After each runner's fixed words: the options it may take (`name` or
// `name=value`; `value` lists the ones whose value may be the next word, and
// `only` the ones whose value must be one of a few words: a built-in node test
// reporter only formats, a reporter path would load code),
// whether `--` may hand the rest to the repo's own script (`dashDash`: then
// only paths follow), and whether nothing may follow the fixed words at all
// (`noWords`: make reads a word as one more target, or `VAR=value` as an
// override). The lists hold options that only read.
const RUNNER_ARGS = {
  npm: { fixed: (t) => (t[1] === 'run' ? 3 : 2), opts: [], dashDash: true },
  node: {
    fixed: 2,
    opts: ['--test-name-pattern', '--test-skip-pattern', '--test-only', '--test-concurrency', '--test-timeout', '--test-reporter'],
    value: ['--test-reporter'],
    only: { '--test-reporter': ['spec', 'tap', 'dot', 'junit', 'lcov'] },
  },
  vitest: { fixed: 1, opts: ['--run', '--passWithNoTests', '--silent', '--bail'] },
  jest: { fixed: 1, opts: ['--ci', '--runInBand', '-i', '--silent', '--passWithNoTests', '--bail', '--verbose'] },
  pytest: { fixed: 1, opts: ['-q', '-qq', '-v', '-vv', '-x', '-s', '--quiet', '--verbose', '--exitfirst', '--tb', '--maxfail'] },
  cargo: { fixed: 2, opts: ['--release', '--workspace', '--all', '--lib', '--doc', '-q', '--quiet', '--no-fail-fast'] },
  go: { fixed: 2, opts: ['-v', '-short', '-race', '-failfast', '-cover', '-count', '-run'], value: ['-count', '-run'] },
  make: { fixed: 2, opts: [], noWords: true },
};
RUNNER_ARGS.python = RUNNER_ARGS.python3 = RUNNER_ARGS.uv = { ...RUNNER_ARGS.pytest, fixed: 3 };

function screenArgs(t, spec) {
  const fixed = typeof spec.fixed === 'function' ? spec.fixed(t) : spec.fixed;
  for (let i = fixed, paths = false; i < t.length; i++) {
    const w = t[i];
    // `make test install` runs the install target too: make takes nothing more.
    if (spec.noWords) return deny(`\`${t.slice(0, fixed).join(' ')}\` with more after it (${w})`);
    if (paths) {
      if (w.startsWith('-')) return deny(`an option after \`--\` (${w}): only paths may follow it`);
      continue;
    }
    if (w === '--' && spec.dashDash) { paths = true; continue; }
    if (w.startsWith('-')) {
      const name = w.split('=')[0];
      if (!spec.opts.includes(name)) return deny(`\`${t[0]}\` with an option that is not on its read-only list (${name})`);
      const inline = w.includes('=');
      const value = inline ? w.slice(name.length + 1) : spec.value?.includes(name) ? t[i + 1] : undefined;
      if (!inline && spec.value?.includes(name)) i++;
      if (spec.only?.[name] && !spec.only[name].includes(value)) {
        return deny(`\`${t[0]}\` with ${name} ${value ?? '(no value)'}: only ${spec.only[name].join('|')}`);
      }
      continue;
    }
  }
  return permit();
}

// `bash <path>` is the way a repo declares a check the allowlist has no name
// for. The path must live under tests/, be relative, and not climb out of it.
function isTestScript(path) {
  if (path === undefined) return false;
  if (!path.startsWith('tests/')) return false;
  return !path.split('/').includes('..');
}

function npmScripts(cwd) {
  try {
    const pkg = JSON.parse(readFileSync(join(cwd, 'package.json'), 'utf8'));
    return new Set(Object.keys(pkg.scripts ?? {}));
  } catch {
    return new Set();
  }
}

// One entry per runner: given the command's tokens (and the working tree the
// criteria run in), either allow it or say why not.
const RUNNERS = {
  npm: (t, cwd, writeTime) => {
    if (t[1] === 'test') return permit();
    if (t[1] !== 'run') return deny(`npm, but not \`npm test\` or \`npm run\` (\`npm ${t[1] ?? ''}\`)`);
    if (t[2] === undefined) return deny('npm run names no script');
    // When a brief or plan is written, the script may be one the plan adds.
    if (writeTime) return permit();
    return npmScripts(cwd).has(t[2])
      ? permit()
      : deny(`npm run names a script package.json does not declare (${t[2]})`);
  },
  node: (t) => (t[1] === '--test' ? permit() : deny('node, but not `node --test` — an arbitrary script is not a test runner')),
  vitest: () => permit(),
  jest: () => permit(),
  pytest: () => permit(),
  python: (t) => (t[1] === '-m' && t[2] === 'pytest' ? permit() : deny('python, but not `python -m pytest`')),
  python3: (t) => (t[1] === '-m' && t[2] === 'pytest' ? permit() : deny('python3, but not `python3 -m pytest`')),
  uv: (t) => (t[1] === 'run' && t[2] === 'pytest' ? permit() : deny('uv, but not `uv run pytest`')),
  cargo: (t) => (t[1] === 'test' ? permit() : deny('cargo, but not `cargo test`')),
  go: (t) => (t[1] === 'test' ? permit() : deny('go, but not `go test`')),
  make: (t) => (t[1] === 'test' ? permit() : deny('make, but not `make test`')),
  bash: (t) => (isTestScript(t[1]) ? permit() : deny('bash, but not a script under tests/')),
  // The system bash by its path (macOS: 3.2), so a check can pin the shell.
  '/bin/bash': (t) => (isTestScript(t[1]) ? permit() : deny('/bin/bash, but not a script under tests/')),
  git: (t) => {
    if (!READONLY_GIT.has(t[1])) return deny(`git, but not a read-only subcommand first (${[...READONLY_GIT].join('|')})`);
    // The words reach git as written (no shell), and only words that mean the
    // same with or without one run: no quote, backslash, `$`, glob or brace.
    // Then `--output` (and an abbreviation of it) is refused: it makes
    // diff/log/show write to a file (release review of 6.3.0, 2026-10-10: a
    // plan's check could overwrite any file, first plainly, then through
    // quoting while a shell still ran it).
    const odd = t.find((w) => !PLAIN_WORD.test(w));
    if (odd !== undefined) return deny(`git with a word that means something else in a shell (${odd}): only plain words run, as written`);
    if (t.some(writesOutput)) return deny('git with --output writes a file, so it is not a read-only check');
    // A check runs in the repo: no path that climbs out or starts at /, and no
    // --no-index, which makes git diff compare any two files (round 6).
    const out = t.slice(2).find((w) => w === '--no-index' || (!w.startsWith('-') && gitLeavesRepo(w)));
    if (out !== undefined) return deny(`git with a path outside the repo (${out}): a check runs in the repo`);
    return permit();
  },
};

/**
 * Whether a command is one the runner may execute. Returns `{allowed, reason}`.
 * `cwd` is the working tree the criteria run in — `npm run` is checked against
 * the package.json found there.
 */
export function allowedCommand(command, opts = {}) {
  return screenShape(command, opts, false);
}

/**
 * Whether a declared check could ever run, decided WITHOUT running it: the
 * same allowlist, used when a brief or a plan is written instead of only when
 * the run is judged. Measured in the voyage-lekerepo trial (2026-10-06): 11 of
 * 13 brief criteria were NOT RUN at review, and a plan criterion (`git grep`)
 * passed the validator, the plan gate and three critic rounds, then felled the
 * run in Phase 7. An `npm run` script is not looked up here: the plan may be
 * what adds it. Returns `{runnable, reason}`.
 */
export function runnableCheck(criterion, opts = {}) {
  if (!criterion?.command) return { runnable: false, reason: criterion?.reason || 'no command' };
  const r = screenShape(criterion.command, opts, true);
  return { runnable: r.allowed, reason: r.reason };
}

function screenShape(command, opts, writeTime) {
  const cwd = resolve(opts.cwd ?? process.cwd());
  const text = dropComment(command);
  if (text === '') return deny('the command is empty');
  if (/\n/.test(text)) return deny('a newline: one criterion declares one command');
  const operator = text.match(SHELL_OPERATORS);
  if (operator) return deny(`a shell operator (${operator[0]}) — one criterion declares one command, unpiped`);

  const tokens = commandWords(text);
  if (!Object.hasOwn(RUNNERS, tokens[0])) return deny(`\`${tokens[0]}\` is not a known test runner`);
  // A shell expands a tilde at the start of a word (and after = or : in an
  // assignment); no shell runs this, so the runner would get the ~ as written.
  const tilde = tokens.find((w) => /(^|[=:])~/.test(w));
  if (tilde !== undefined) return deny(`a word that means something else in a shell (${tilde}): a leading ~ is not expanded here`);
  const form = RUNNERS[tokens[0]](tokens, cwd, writeTime);
  if (!form.allowed || tokens[0] === 'git') return form;
  const words = tokens.slice(1);
  const nodeTest = tokens[0] === 'node' && tokens[1] === '--test';
  const nodeMajor = opts.nodeMajor ?? runningNodeMajor();
  const odd = words.find((w) => !PLAIN_WORD.test(w) && !(nodeTest && GLOB_WORD.test(w)));
  if (odd !== undefined) {
    return deny(GLOB_WORD.test(odd)
      ? `a glob (${odd}): no shell runs this, and \`${tokens[0]}\` does not expand one itself; name the files, or use \`node --test\``
      : `a word that means something else in a shell (${odd}): only plain words run, as written`);
  }
  const glob = nodeTest && nodeMajor < NODE_GLOB_MAJOR ? words.find((w) => !PLAIN_WORD.test(w)) : undefined;
  if (glob !== undefined) {
    return deny(`a glob (${glob}): node ${nodeMajor} does not expand one itself (Node ${NODE_GLOB_MAJOR} and later do), and no shell runs this; name the files`);
  }
  const at = words.find((w) => w.startsWith('@'));
  if (at !== undefined) return deny(`a word that starts with @ (${at}): pytest reads @file as a file of more arguments`);
  const out = words.find((w) => !w.startsWith('-') && leavesRepo(w));
  if (out !== undefined) return deny(`a path outside the repo (${out}): a check runs in the repo, with no .. and no absolute path`);
  return Object.hasOwn(RUNNER_ARGS, tokens[0]) ? screenArgs(tokens, RUNNER_ARGS[tokens[0]]) : permit();
}

/** The words of a command: what the screen checks is exactly what runs. */
export function commandWords(command) {
  return dropComment(command).split(/\s+/);
}

/**
 * Run a screened command with no shell: its first word is the program (found
 * on PATH), the rest are its arguments, as written. Without NODE_TEST_CONTEXT
 * (lib/util/command-env.mjs): started under `node --test`, a nested `node
 * --test` reported to the parent and exited 0 on a red test, a false green
 * (order 10, 2026-10-09). Returns spawnSync's result: `status` is null when
 * the program could not start (`error`) or was killed (`signal`).
 */
export function runCommand(command, { cwd, timeoutMs = DEFAULT_TIMEOUT_MS } = {}) {
  const [program, ...args] = commandWords(command);
  return spawnSync(program, args, { cwd, env: commandEnv(), encoding: 'utf8', timeout: timeoutMs, maxBuffer: MAX_BUFFER });
}

/**
 * Why a spawnSync run has no exit code of its own, or null when it has one.
 * spawnSync sets `error` AND `signal` together when it kills the program
 * itself: ETIMEDOUT for the timeout, ENOBUFS for output past `maxBuffer`. So
 * the error code is read first, and a bare `error` with no signal is a program
 * that never started (not installed). Red proof called a timeout "could not
 * start" and the release check called an overflow a timeout (release review of
 * 6.3.0, round 7). `started` is false only for a program that never ran.
 * @returns {{started: boolean, why: string}|null}
 */
export function stoppedBy(r, { timeoutMs = DEFAULT_TIMEOUT_MS, maxBuffer = MAX_BUFFER } = {}) {
  const signal = r.signal ?? 'SIGTERM';
  if (r.error?.code === 'ETIMEDOUT') return { started: true, why: `was killed (${signal}) after the ${Math.round(timeoutMs / 1000)} s timeout` };
  if (r.error?.code === 'ENOBUFS') {
    const size = maxBuffer >= 1024 * 1024 ? `${Math.round(maxBuffer / (1024 * 1024))} MB` : `${maxBuffer} bytes`;
    return { started: true, why: `was killed (${signal}): its output passed the ${size} that is read from it` };
  }
  if (r.signal) return { started: true, why: `was killed (${r.signal})` };
  if (r.error || r.status === null) return { started: false, why: `could not start: ${r.error?.message ?? 'no exit code'}` };
  return null;
}

// --- running ----------------------------------------------------------------

// A program that could not start (not installed: ENOENT) has no exit code and
// measured nothing: `startError` says so, and runCriteria reports it NOT RUN.
// One killed by the timeout did start, and a hang is a red (`failed`).
function defaultExec(command, { cwd, timeoutMs }) {
  const r = runCommand(command, { cwd, timeoutMs });
  const stopped = stoppedBy(r, { timeoutMs });
  if (stopped && !stopped.started) return { status: null, stdout: '', stderr: '', startError: r.error?.message ?? stopped.why };
  const why = stopped ? `${stopped.why}\n` : '';
  return { status: r.status, stdout: r.stdout ?? '', stderr: `${r.stderr ?? ''}${why}` };
}

function cap(text, max) {
  const t = String(text ?? '');
  return t.length <= max ? t : `${t.slice(0, max)}\n… [truncated: ${t.length} chars total]`;
}

/**
 * Run each criterion that has a command. Returns one result per criterion:
 *   passed     the command exited 0
 *   failed     the command exited non-zero (or was killed by the timeout)
 *   blocked    the executor denylist denied it — it never ran
 *   unrunnable there is nothing to run: no command (placeholder or prose-only
 *              criterion), a command outside the allowlist, or a runner that
 *              could not start (`startError`: not installed)
 */
export function runCriteria(criteria, opts = {}) {
  const exec = opts.exec ?? defaultExec;
  const screen = opts.screen ?? screenCommand;
  const allow = opts.allow ?? allowedCommand;
  const cwd = opts.cwd ?? process.cwd();
  const timeoutMs = opts.timeoutMs ?? DEFAULT_TIMEOUT_MS;
  const maxOutput = opts.maxOutput ?? DEFAULT_MAX_OUTPUT;
  const hookPath = opts.hookPath ?? DEFAULT_HOOK;

  return criteria.map((c) => {
    const base = { label: c.label, text: c.text, command: c.command };
    if (!c.command) {
      return { ...base, status: 'unrunnable', exitCode: null, output: `not runnable: ${c.reason || 'no command'}` };
    }
    const permission = allow(c.command, { cwd });
    if (!permission.allowed) {
      return {
        ...base,
        status: 'unrunnable',
        exitCode: null,
        output: `not runnable: outside the allowlist (${permission.reason})`,
      };
    }
    const verdict = screen(c.command, { hookPath });
    if (!verdict.allowed) {
      return { ...base, status: 'blocked', exitCode: null, output: `blocked: ${verdict.rule}` };
    }
    const r = exec(c.command, { cwd, timeoutMs });
    if (r.startError) {
      return { ...base, status: 'unrunnable', exitCode: null, output: `not runnable: the program could not start (${r.startError})`, startError: r.startError };
    }
    const output = cap(`${r.stdout ?? ''}${r.stderr ?? ''}`, maxOutput);
    return {
      ...base,
      status: r.status === 0 ? 'passed' : 'failed',
      exitCode: r.status,
      output,
    };
  });
}

/**
 * The verdict. `requireCommand` distinguishes the two callers: a plan promises
 * an exact command per criterion, so an unrunnable one is a defect in the plan;
 * a brief may hold criteria a human judges by reading, so there an unrunnable
 * criterion is reported and left to the reviewer.
 */
export function summarize(results, opts = {}) {
  const requireCommand = opts.requireCommand ?? true;
  const count = (s) => results.filter((r) => r.status === s).length;
  const summary = {
    total: results.length,
    passed: count('passed'),
    failed: count('failed'),
    blocked: count('blocked'),
    unrunnable: count('unrunnable'),
  };
  summary.ok =
    summary.failed === 0 &&
    summary.blocked === 0 &&
    (!requireCommand || (summary.unrunnable === 0 && summary.total > 0));
  return summary;
}

/** The commands a plan's steps declare in `End-to-end:` (first backticked span each). */
export function endToEndCommands(markdown) {
  return sliceSteps(String(markdown ?? '').replace(/\r\n?/g, '\n'))
    .map((s) => parseStepFields(s.body).get('End-to-end'))
    .filter(Boolean)
    .map((f) => firstCommand(f.text).command)
    .filter(Boolean);
}

const normalizeCommand = (c) => String(c).trim().replace(/\s+/g, ' ');
const SAYS_END_TO_END = /end to end|end-to-end|run the thing/i;

/**
 * Whether the run was verified end to end (P9 (6)). A passed criterion
 * verifies it when it runs a step's `End-to-end:` command — the field the plan
 * validator requires — or, the older form, when its text says it runs the
 * thing end to end. Searching the text alone missed a plan that had the field
 * and ran it green (voyage-lekerepo trial, 2026-10-06). A verification that
 * did not pass is never verified end to end.
 */
export function endToEndStatus(results, summary, e2eCommands = []) {
  const no = { status: 'not-verified', by: null, criterion: null };
  if (!summary.ok) return no;
  const passed = results.filter((r) => r.status === 'passed');
  const want = new Set(e2eCommands.map(normalizeCommand));
  const byField = passed.find((r) => r.command && want.has(normalizeCommand(r.command)));
  if (byField) return { status: 'verified', by: 'end-to-end-field', criterion: byField.label };
  const byText = passed.find((r) => SAYS_END_TO_END.test(r.text));
  if (byText) return { status: 'verified', by: 'criterion-text', criterion: byText.label };
  return no;
}

function report(kind, source, criteria, opts, error, e2eCommands = []) {
  const requireCommand = kind === 'plan';
  const results = error ? [] : runCriteria(criteria, opts);
  const summary = summarize(results, { requireCommand });
  const rep = {
    kind,
    source,
    heading: kind === 'plan' ? PLAN_HEADING : BRIEF_HEADING,
    results,
    summary,
    error: error ?? null,
  };
  if (kind === 'plan') rep.end_to_end = endToEndStatus(results, summary, e2eCommands);
  return rep;
}

/**
 * The section is THERE and nothing in it parsed as a check.
 *
 * "The section is empty" and "I cannot read this format" are different facts,
 * and reporting both as `0 of 0` collapsed them into one bare exit 1. Measured
 * 2026-09-18: nine ordinary shapes of a `## Verification` section — an untagged
 * fence, a ```text fence, a table, a heading with a suffix, an unclosed fence
 * earlier in the file — all came out that way, and the run was felled without
 * anyone being told which format the runner had failed to read.
 */
function noCriteria(source, heading, line) {
  return {
    code: 'NO_CRITERIA',
    message:
      `${source}:${line} — the "${heading}" section declares nothing the runner can read. `
      + 'It reads two forms: a bullet whose first backticked span is the command '
      + '(`- [ ] `npm test` -> expected: exit 0`), and a shell-tagged fence (```bash) '
      + 'whose lines are the commands. An untagged fence, a table or prose parses to nothing.',
  };
}

function readOrThrow(path) {
  try {
    return readFileSync(path, 'utf8');
  } catch (err) {
    const e = new Error(`cannot read ${path}: ${err.message}`);
    e.code = 'READ_ERROR';
    throw e;
  }
}

/**
 * Run a plan's `## Verification` criteria. A plan with no such section is NOT
 * ok: a plan that promises no end-to-end check cannot be verified, and that
 * must read as red rather than as nothing to do.
 */
export function runPlanVerification(planPath, opts = {}) {
  const md = readOrThrow(planPath);
  const section = sectionLines(md, PLAN_HEADING);
  if (section === null) {
    return report('plan', planPath, [], opts, {
      code: 'NO_VERIFICATION_SECTION',
      message: `${planPath} has no "${PLAN_HEADING}" section — nothing to verify, so the run cannot be called verified`,
    });
  }
  const criteria = parsePlanVerification(md);
  if (criteria.length === 0) {
    return report('plan', planPath, [], opts, noCriteria(planPath, PLAN_HEADING, section.line));
  }
  return report('plan', planPath, criteria, opts, null, endToEndCommands(md));
}

/**
 * Run a brief's `## Success Criteria` commands. Used by /trekreview to hand the
 * conformance reviewer a real result per criterion.
 */
export function runSuccessCriteriaChecks(briefPath, opts = {}) {
  const md = readOrThrow(briefPath);
  const section = sectionLines(md, BRIEF_HEADING);
  if (section === null) {
    return report('brief', briefPath, [], opts, {
      code: 'NO_SUCCESS_CRITERIA_SECTION',
      message: `${briefPath} has no "${BRIEF_HEADING}" section`,
    });
  }
  const criteria = parseSuccessCriteria(md);
  if (criteria.length === 0) {
    return report('brief', briefPath, [], opts, noCriteria(briefPath, BRIEF_HEADING, section.line));
  }
  return report('brief', briefPath, criteria, opts);
}

// --- rendering --------------------------------------------------------------

const MARK = { passed: 'PASS', failed: 'FAILED', blocked: 'BLOCKED', unrunnable: 'NOT RUN' };

export function render(rep) {
  const out = [];
  const s = rep.summary;
  out.push(`criteria-runner: ${s.ok ? 'OK' : 'NOT OK'} — ${rep.source} ${rep.heading}`);
  if (rep.error) out.push(`  error: ${rep.error.code} — ${rep.error.message}`);
  out.push(
    `  ${s.passed} passed · ${s.failed} failed · ${s.blocked} blocked · ${s.unrunnable} not run (of ${s.total})`
  );
  for (const r of rep.results) {
    const code = r.exitCode === null ? '' : ` (exit ${r.exitCode})`;
    out.push(`  [${MARK[r.status]}] ${r.label}: \`${r.command ?? r.text}\`${code}`);
    if (r.status !== 'passed' && r.output) {
      for (const line of r.output.split('\n').slice(0, 20)) out.push(`      ${line}`);
    }
  }
  if (rep.end_to_end) {
    const e = rep.end_to_end;
    out.push(e.status === 'verified'
      ? `  end to end: verified (${e.criterion}, ${e.by === 'end-to-end-field' ? "runs a step's End-to-end command" : 'its text says it runs the thing'})`
      : '  end to end: not verified end to end');
  }
  return out.join('\n');
}

/**
 * The evidence block /trekreview hands to `brief-conformance-reviewer`.
 *
 * The reviewer's tools are Read/Glob/Grep, so it cannot run a success
 * criterion's verification command. Building this block in code (rather than
 * letting the orchestrator narrate it) is what makes "passes" an exit code the
 * reviewer READS instead of a judgement it cannot make.
 */
export function formatCriteriaEvidence(rep) {
  const out = [];
  out.push('### Success-criteria check results (run for you — you did not run these)');
  out.push('');
  out.push(`Source: \`${rep.source}\` ${rep.heading}`);
  out.push('');
  if (rep.error) {
    out.push(`**${rep.error.code}** — ${rep.error.message}.`);
    out.push('');
    out.push('No criteria were run. Treat every criterion as NOT RUN.');
  } else {
    out.push('| Criterion | Command | Result | Exit | Evidence |');
    out.push('|---|---|---|---|---|');
    for (const r of rep.results) {
      const cmd = r.command ? `\`${cell(r.command)}\`` : '(none)';
      const exit = r.exitCode === null ? '—' : `exit ${r.exitCode}`;
      // The first line of output on every row, a PASS too: /trekreview tells
      // the reviewer it gets "the command, exit code and first line of output",
      // and a PASS row said "—" (voyage-lekerepo trial 2, 2026-10-08).
      const evidence = cell(firstLine(r.output)) || '—';
      out.push(`| ${r.label} | ${cmd} | ${MARK[r.status]} | ${exit} | ${evidence} |`);
    }
    const s = rep.summary;
    out.push('');
    out.push(
      `${s.passed} passed · ${s.failed} failed · ${s.blocked} blocked · ${s.unrunnable} not run (of ${s.total}).`
    );
  }
  out.push('');
  out.push(
    '**NOT RUN is not a pass.** You cannot run these commands yourself — never infer, assume or ' +
    'reconstruct a result for a criterion that has none. A criterion whose result is FAILED or ' +
    'BLOCKED is `Broken`; a criterion marked NOT RUN declared no command, a command outside ' +
    'the allowlist of test runners, or a runner that could not start, so it never ran — that is the ABSENCE of a ' +
    'measurement and never on its own a finding; a criterion with no result is judged on ' +
    'delivered code alone, and you say so in the Evidence column.'
  );
  return out.join('\n');
}

function firstLine(text) {
  return String(text ?? '').split('\n').map((l) => l.trim()).find((l) => l !== '') ?? '';
}

// Markdown table cells: no pipes, no newlines, bounded length.
function cell(text) {
  const t = String(text ?? '').replace(/\r?\n/g, ' ').replace(/\|/g, '\\|').trim();
  return t.length <= 120 ? t : `${t.slice(0, 120)}…`;
}

// --- CLI --------------------------------------------------------------------

const USAGE =
  'usage: criteria-runner.mjs (--plan <plan.md> | --brief <brief.md>) [--json | --evidence] [--cwd <dir>] [--timeout <ms>]';

export function main(argv) {
  let mode = null;
  let path = null;
  let json = false;
  let evidence = false;
  const opts = {};
  for (let i = 0; i < argv.length; i++) {
    const a = argv[i];
    if ((a === '--plan' || a === '--brief') && argv[i + 1]) {
      if (mode) { process.stderr.write(`criteria-runner: one mode at a time\n${USAGE}\n`); return 2; }
      mode = a.slice(2);
      path = argv[++i];
    } else if (a === '--json') json = true;
    else if (a === '--evidence') evidence = true;
    else if (a === '--cwd' && argv[i + 1]) opts.cwd = resolve(argv[++i]);
    else if (a === '--timeout' && argv[i + 1]) opts.timeoutMs = Number(argv[++i]);
    else { process.stderr.write(`criteria-runner: unknown argument ${a}\n${USAGE}\n`); return 2; }
  }
  if (!mode) { process.stderr.write(`criteria-runner: no artifact given\n${USAGE}\n`); return 2; }
  if (json && evidence) {
    process.stderr.write(`criteria-runner: --json and --evidence are two output shapes; pick one\n${USAGE}\n`);
    return 2;
  }
  if (opts.timeoutMs !== undefined && !Number.isFinite(opts.timeoutMs)) {
    process.stderr.write(`criteria-runner: --timeout must be a number\n${USAGE}\n`);
    return 2;
  }

  let rep;
  try {
    rep = mode === 'plan' ? runPlanVerification(path, opts) : runSuccessCriteriaChecks(path, opts);
  } catch (err) {
    process.stderr.write(`criteria-runner: ${err.message}\n`);
    return 2;
  }
  const text = json ? JSON.stringify(rep, null, 2) : evidence ? formatCriteriaEvidence(rep) : render(rep);
  process.stdout.write(text + '\n');
  return rep.summary.ok ? 0 : 1;
}

if (isMain(import.meta.url)) {
  process.exitCode = main(process.argv.slice(2));
}