voyage/lib/verification/criteria-runner.mjs

#!/usr/bin/env node
// lib/verification/criteria-runner.mjs
// Runs the falsifiable checks a pipeline artifact DECLARES, and reports a
// verdict built from exit codes rather than from a reader's impression.
//
// Two artifacts declare such checks:
//   plan   `## Verification`     — `- [ ] `cmd` -> expected: ...`  (V1..Vn)
//   brief  `## Success Criteria` — `- text: `cmd` ...`             (SC1..SCn)
//
// Why this is code and not prose: `/trekexecute` must be able to FELL a
// single-session run on a criterion that does not hold, and `/trekreview` must
// hand `brief-conformance-reviewer` a real result instead of asking a
// Read/Glob/Grep agent to judge whether a command "passes". Neither is
// checkable while it lives only as an instruction.
//
// Fail-closed everywhere: a criterion that cannot run — placeholder text, no
// command, a screen that is unavailable — is NEVER reported as passed.
//
// Screening: every command is screened through the plugin's own PreToolUse
// denylist (hooks/scripts/pre-bash-executor.mjs) before it reaches a shell.
// Commands spawned from here do not pass through the Bash tool, so that hook
// would not otherwise fire; the hook is invoked over its documented stdin
// protocol rather than copied, so there is exactly one denylist.
//
// Usage:
//   node lib/verification/criteria-runner.mjs --plan  <plan.md>  [--json] [--cwd D] [--timeout MS]
//   node lib/verification/criteria-runner.mjs --brief <brief.md> [--json|--evidence] [--cwd D] [--timeout MS]
// --evidence emits the markdown block /trekreview hands to the conformance
// reviewer; --json emits the whole report. They are two shapes of one run.
// Exit: 0 = ok, 1 = a criterion failed / was blocked / could not run, 2 = usage
// or read error (deliberately distinct from "a criterion failed").

import { readFileSync } from 'node:fs';
import { resolve, dirname, join } from 'node:path';
import { fileURLToPath } from 'node:url';
import { spawnSync } from 'node:child_process';

const REPO_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '..', '..');

export const PLAN_HEADING = '## Verification';
export const BRIEF_HEADING = '## Success Criteria';
export const DEFAULT_HOOK = join(REPO_ROOT, 'hooks', 'scripts', 'pre-bash-executor.mjs');
export const DEFAULT_TIMEOUT_MS = 900_000;
export const DEFAULT_MAX_OUTPUT = 4000;

// --- parsing ----------------------------------------------------------------

// Fenced blocks are read TWICE over, for two opposite reasons: a `## ` heading
// inside a fence is quoted text and must not open a section (the repo's own
// examples/02-real-cli/REGENERATED.md quotes a whole plan outline inside one),
// while a shell-tagged fence INSIDE the section holds the commands themselves
// (examples/01-add-verbose-flag/plan.md writes its whole acceptance run that
// way). Reading only bullets made a correct plan parse to zero criteria, exit
// 1, and fell every single-session run — measured 2026-09-18.
const FENCE = /^\s*(?:```|~~~)(.*)$/;

// Languages whose fenced block is a run of commands. Deliberately a closed
// list: an untagged fence is far more often expected OUTPUT than input, and
// inventing a criterion from output is exactly the failure this file exists to
// prevent.
const SHELL_LANGS = new Set(['bash', 'sh', 'shell', 'zsh', 'console', 'shell-session']);

// One entry per line: its text, whether it sits inside a fence, that fence's
// info string, and whether it IS the fence marker.
function scanLines(markdown) {
  const out = [];
  let lang = null;
  for (const text of markdown.split('\n')) {
    const m = text.match(FENCE);
    if (m) {
      const open = lang === null;
      if (open) lang = m[1].trim().toLowerCase().split(/\s+/)[0];
      out.push({ text, fenced: true, lang: lang ?? '', marker: true });
      if (!open) lang = null;
      continue;
    }
    out.push({ text, fenced: lang !== null, lang: lang ?? '', marker: false });
  }
  return out;
}

// The lines from `heading` up to the next `## ` heading (exclusive). Headings
// inside a fence do not count on either end.
function sectionLines(markdown, heading) {
  const lines = scanLines(markdown);
  const start = lines.findIndex((l) => !l.fenced && l.text.trim() === heading);
  if (start === -1) return null;
  let end = lines.length;
  for (let i = start + 1; i < lines.length; i++) {
    if (!lines[i].fenced && lines[i].text.startsWith('## ')) { end = i; break; }
  }
  return lines.slice(start + 1, end);
}

// `- [ ] text`, `- [x] text`, `- text`, `* text`, `1. text` -> text
const BULLET = /^\s*(?:[-*]|\d+\.)\s+(?:\[[ xX]\]\s+)?(.*)$/;

const ENV_ASSIGN = /^[A-Za-z_][A-Za-z0-9_]*=\S*\s+/;

/**
 * Whether a backticked span is a command to RUN or something the sentence
 * merely NAMES. A plan's template puts the command first, but a brief's
 * criterion usually opens with the thing under discussion — a flag
 * (`--verbose`), a path (`tests/`) — and running those produced exit 2
 * ("invalid option") and exit 126 ("is a directory"), which the review rubric
 * reads as decisive: a BLOCKER invented out of prose. Measured 2026-09-18:
 * 3 of the 6 criteria in the repo's own example brief.
 *
 * Shape only, no filesystem lookup, so a span parses the same way everywhere.
 * And deliberately NO scanning on to a later span: "the first span that looks
 * like a command" invents commands out of prose — in that same example brief
 * it would have run `whoami` and `login`, two real binaries a sentence happens
 * to name. An absent measurement is the honest answer; a guessed one is not.
 */
export function looksLikeCommand(span) {
  let rest = String(span ?? '').trim();
  while (ENV_ASSIGN.test(rest)) rest = rest.replace(ENV_ASSIGN, '');
  const token = rest.split(/\s+/)[0] ?? '';
  if (token === '') return false;
  if (token.startsWith('-')) return false;                      // a flag
  if (token.endsWith('/')) return false;                        // a directory
  if (!/^[A-Za-z0-9_.@+~/-]+$/.test(token)) return false;       // quotes, braces, prose
  if (!/[A-Za-z0-9]/.test(token)) return false;
  // A lone relative path with a slash is a file the sentence names. An explicit
  // `./`, `../`, `/` or `~/` is an invocation and still runs.
  if (rest === token && token.includes('/') && !/^(\.{1,2}\/|\/|~\/)/.test(token)) return false;
  return true;
}

// The first backtick-delimited span on the line is the command by convention —
// both templates put it first and a second span holds the expected output.
function firstCommand(text) {
  const m = text.match(/`([^`]+)`/);
  if (!m) return { command: null, reason: 'no-command' };
  const raw = m[1].trim();
  // Template placeholders (`{exact command}`) are not commands.
  if (raw === '' || /^\{.*\}$/.test(raw)) return { command: null, reason: 'placeholder' };
  if (!looksLikeCommand(raw)) return { command: null, reason: 'not-a-command' };
  return { command: raw, reason: '' };
}

// A command line inside a shell-tagged fence. A console block may prefix the
// line with a `$` or `>` prompt; blank and comment-only lines declare nothing.
// A `#` root prompt is NOT stripped: it is indistinguishable from a comment,
// and running a comment is the worse of the two mistakes.
function fencedCommand(text) {
  const line = text.trim();
  if (line === '' || line.startsWith('#')) return null;
  return line.replace(/^[$>]\s+/, '');
}

function parseSection(markdown, heading, prefix) {
  const lines = sectionLines(markdown, heading);
  if (lines === null) return [];
  const criteria = [];
  const push = (text, command, reason) =>
    criteria.push({ label: `${prefix}${criteria.length + 1}`, text, command, reason });
  for (const line of lines) {
    if (line.marker) continue;
    if (line.fenced) {
      if (!SHELL_LANGS.has(line.lang)) continue;
      const command = fencedCommand(line.text);
      if (command === null) continue;
      push(command, command, '');
      continue;
    }
    const bullet = line.text.match(BULLET);
    if (!bullet) continue;
    const text = bullet[1].trim();
    if (text === '') continue;
    const { command, reason } = firstCommand(text);
    push(text, command, reason);
  }
  return criteria;
}

/** Criteria declared in a plan's `## Verification` section, labelled V1..Vn. */
export function parsePlanVerification(markdown) {
  return parseSection(markdown, PLAN_HEADING, 'V');
}

/** Criteria declared in a brief's `## Success Criteria` section, labelled SC1..SCn. */
export function parseSuccessCriteria(markdown) {
  return parseSection(markdown, BRIEF_HEADING, 'SC');
}

// --- screening --------------------------------------------------------------

/**
 * Screen one command through the plugin's PreToolUse denylist.
 * Hook protocol: stdin JSON, exit 2 = block, exit 0 = allow. Any other exit
 * means the screen did not render a verdict — that denies, it never allows.
 */
export function screenCommand(command, opts = {}) {
  const hookPath = opts.hookPath ?? DEFAULT_HOOK;
  const r = spawnSync(process.execPath, [hookPath], {
    input: JSON.stringify({ tool_name: 'Bash', tool_input: { command } }),
    encoding: 'utf8',
    timeout: 30_000,
  });
  if (r.status === 0) return { allowed: true, rule: '' };
  const first = String(r.stderr || '').split('\n')[0].trim();
  if (r.status === 2) {
    return { allowed: false, rule: first.replace(/^\[voyage\]\s*BLOCKED:\s*/, '') || 'blocked by the executor denylist' };
  }
  return {
    allowed: false,
    rule: `screen unavailable (${hookPath} exited ${r.status === null ? 'on signal' : r.status}) — denied, never assumed safe`,
  };
}

// --- running ----------------------------------------------------------------

function defaultExec(command, { cwd, timeoutMs }) {
  const r = spawnSync('/bin/sh', ['-c', command], { cwd, encoding: 'utf8', timeout: timeoutMs });
  return { status: r.status, stdout: r.stdout ?? '', stderr: r.stderr ?? '' };
}

function cap(text, max) {
  const t = String(text ?? '');
  return t.length <= max ? t : `${t.slice(0, max)}\n… [truncated: ${t.length} chars total]`;
}

/**
 * Run each criterion that has a command. Returns one result per criterion:
 *   passed     the command exited 0
 *   failed     the command exited non-zero (or was killed by the timeout)
 *   blocked    the screen denied it — it never reached a shell
 *   unrunnable there is no command to run (placeholder or prose-only criterion)
 */
export function runCriteria(criteria, opts = {}) {
  const exec = opts.exec ?? defaultExec;
  const screen = opts.screen ?? screenCommand;
  const cwd = opts.cwd ?? process.cwd();
  const timeoutMs = opts.timeoutMs ?? DEFAULT_TIMEOUT_MS;
  const maxOutput = opts.maxOutput ?? DEFAULT_MAX_OUTPUT;
  const hookPath = opts.hookPath ?? DEFAULT_HOOK;

  return criteria.map((c) => {
    const base = { label: c.label, text: c.text, command: c.command };
    if (!c.command) {
      return { ...base, status: 'unrunnable', exitCode: null, output: `not runnable: ${c.reason || 'no command'}` };
    }
    const verdict = screen(c.command, { hookPath });
    if (!verdict.allowed) {
      return { ...base, status: 'blocked', exitCode: null, output: `blocked: ${verdict.rule}` };
    }
    const r = exec(c.command, { cwd, timeoutMs });
    const output = cap(`${r.stdout ?? ''}${r.stderr ?? ''}`, maxOutput);
    return {
      ...base,
      status: r.status === 0 ? 'passed' : 'failed',
      exitCode: r.status,
      output,
    };
  });
}

/**
 * The verdict. `requireCommand` distinguishes the two callers: a plan promises
 * an exact command per criterion, so an unrunnable one is a defect in the plan;
 * a brief may hold criteria a human judges by reading, so there an unrunnable
 * criterion is reported and left to the reviewer.
 */
export function summarize(results, opts = {}) {
  const requireCommand = opts.requireCommand ?? true;
  const count = (s) => results.filter((r) => r.status === s).length;
  const summary = {
    total: results.length,
    passed: count('passed'),
    failed: count('failed'),
    blocked: count('blocked'),
    unrunnable: count('unrunnable'),
  };
  summary.ok =
    summary.failed === 0 &&
    summary.blocked === 0 &&
    (!requireCommand || (summary.unrunnable === 0 && summary.total > 0));
  return summary;
}

function report(kind, source, criteria, opts, error) {
  const requireCommand = kind === 'plan';
  const results = error ? [] : runCriteria(criteria, opts);
  return {
    kind,
    source,
    heading: kind === 'plan' ? PLAN_HEADING : BRIEF_HEADING,
    results,
    summary: summarize(results, { requireCommand }),
    error: error ?? null,
  };
}

function readOrThrow(path) {
  try {
    return readFileSync(path, 'utf8');
  } catch (err) {
    const e = new Error(`cannot read ${path}: ${err.message}`);
    e.code = 'READ_ERROR';
    throw e;
  }
}

/**
 * Run a plan's `## Verification` criteria. A plan with no such section is NOT
 * ok: a plan that promises no end-to-end check cannot be verified, and that
 * must read as red rather than as nothing to do.
 */
export function runPlanVerification(planPath, opts = {}) {
  const md = readOrThrow(planPath);
  if (sectionLines(md, PLAN_HEADING) === null) {
    return report('plan', planPath, [], opts, {
      code: 'NO_VERIFICATION_SECTION',
      message: `${planPath} has no "${PLAN_HEADING}" section — nothing to verify, so the run cannot be called verified`,
    });
  }
  return report('plan', planPath, parsePlanVerification(md), opts);
}

/**
 * Run a brief's `## Success Criteria` commands. Used by /trekreview to hand the
 * conformance reviewer a real result per criterion.
 */
export function runSuccessCriteriaChecks(briefPath, opts = {}) {
  const md = readOrThrow(briefPath);
  if (sectionLines(md, BRIEF_HEADING) === null) {
    return report('brief', briefPath, [], opts, {
      code: 'NO_SUCCESS_CRITERIA_SECTION',
      message: `${briefPath} has no "${BRIEF_HEADING}" section`,
    });
  }
  return report('brief', briefPath, parseSuccessCriteria(md), opts);
}

// --- rendering --------------------------------------------------------------

const MARK = { passed: 'PASS', failed: 'FAILED', blocked: 'BLOCKED', unrunnable: 'NOT RUN' };

export function render(rep) {
  const out = [];
  const s = rep.summary;
  out.push(`criteria-runner: ${s.ok ? 'OK' : 'NOT OK'} — ${rep.source} ${rep.heading}`);
  if (rep.error) out.push(`  error: ${rep.error.code} — ${rep.error.message}`);
  out.push(
    `  ${s.passed} passed · ${s.failed} failed · ${s.blocked} blocked · ${s.unrunnable} not run (of ${s.total})`
  );
  for (const r of rep.results) {
    const code = r.exitCode === null ? '' : ` (exit ${r.exitCode})`;
    out.push(`  [${MARK[r.status]}] ${r.label}: \`${r.command ?? r.text}\`${code}`);
    if (r.status !== 'passed' && r.output) {
      for (const line of r.output.split('\n').slice(0, 20)) out.push(`      ${line}`);
    }
  }
  return out.join('\n');
}

/**
 * The evidence block /trekreview hands to `brief-conformance-reviewer`.
 *
 * The reviewer's tools are Read/Glob/Grep, so it cannot run a success
 * criterion's verification command. Building this block in code (rather than
 * letting the orchestrator narrate it) is what makes "passes" an exit code the
 * reviewer READS instead of a judgement it cannot make.
 */
export function formatCriteriaEvidence(rep) {
  const out = [];
  out.push('### Success-criteria check results (run for you — you did not run these)');
  out.push('');
  out.push(`Source: \`${rep.source}\` ${rep.heading}`);
  out.push('');
  if (rep.error) {
    out.push(`**${rep.error.code}** — ${rep.error.message}.`);
    out.push('');
    out.push('No criteria were run. Treat every criterion as NOT RUN.');
  } else {
    out.push('| Criterion | Command | Result | Exit | Evidence |');
    out.push('|---|---|---|---|---|');
    for (const r of rep.results) {
      const cmd = r.command ? `\`${cell(r.command)}\`` : '(none)';
      const exit = r.exitCode === null ? '—' : `exit ${r.exitCode}`;
      const evidence = r.status === 'passed' ? '—' : cell(firstLine(r.output)) || '—';
      out.push(`| ${r.label} | ${cmd} | ${MARK[r.status]} | ${exit} | ${evidence} |`);
    }
    const s = rep.summary;
    out.push('');
    out.push(
      `${s.passed} passed · ${s.failed} failed · ${s.blocked} blocked · ${s.unrunnable} not run (of ${s.total}).`
    );
  }
  out.push('');
  out.push(
    '**NOT RUN is not a pass.** You cannot run these commands yourself — never infer, assume or ' +
    'reconstruct a result for a criterion that has none. A criterion whose result is FAILED or ' +
    'BLOCKED is `Broken`; a criterion with no result is judged on delivered code alone, and you ' +
    'say so in the Evidence column.'
  );
  return out.join('\n');
}

function firstLine(text) {
  return String(text ?? '').split('\n').map((l) => l.trim()).find((l) => l !== '') ?? '';
}

// Markdown table cells: no pipes, no newlines, bounded length.
function cell(text) {
  const t = String(text ?? '').replace(/\r?\n/g, ' ').replace(/\|/g, '\\|').trim();
  return t.length <= 120 ? t : `${t.slice(0, 120)}…`;
}

// --- CLI --------------------------------------------------------------------

const USAGE =
  'usage: criteria-runner.mjs (--plan <plan.md> | --brief <brief.md>) [--json | --evidence] [--cwd <dir>] [--timeout <ms>]';

export function main(argv) {
  let mode = null;
  let path = null;
  let json = false;
  let evidence = false;
  const opts = {};
  for (let i = 0; i < argv.length; i++) {
    const a = argv[i];
    if ((a === '--plan' || a === '--brief') && argv[i + 1]) {
      if (mode) { process.stderr.write(`criteria-runner: one mode at a time\n${USAGE}\n`); return 2; }
      mode = a.slice(2);
      path = argv[++i];
    } else if (a === '--json') json = true;
    else if (a === '--evidence') evidence = true;
    else if (a === '--cwd' && argv[i + 1]) opts.cwd = resolve(argv[++i]);
    else if (a === '--timeout' && argv[i + 1]) opts.timeoutMs = Number(argv[++i]);
    else { process.stderr.write(`criteria-runner: unknown argument ${a}\n${USAGE}\n`); return 2; }
  }
  if (!mode) { process.stderr.write(`criteria-runner: no artifact given\n${USAGE}\n`); return 2; }
  if (json && evidence) {
    process.stderr.write(`criteria-runner: --json and --evidence are two output shapes; pick one\n${USAGE}\n`);
    return 2;
  }
  if (opts.timeoutMs !== undefined && !Number.isFinite(opts.timeoutMs)) {
    process.stderr.write(`criteria-runner: --timeout must be a number\n${USAGE}\n`);
    return 2;
  }

  let rep;
  try {
    rep = mode === 'plan' ? runPlanVerification(path, opts) : runSuccessCriteriaChecks(path, opts);
  } catch (err) {
    process.stderr.write(`criteria-runner: ${err.message}\n`);
    return 2;
  }
  const text = json ? JSON.stringify(rep, null, 2) : evidence ? formatCriteriaEvidence(rep) : render(rep);
  process.stdout.write(text + '\n');
  return rep.summary.ok ? 0 : 1;
}

if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
  process.exitCode = main(process.argv.slice(2));
}