#!/usr/bin/env node // lib/verification/criteria-runner.mjs // Runs the falsifiable checks a pipeline artifact DECLARES, and reports a // verdict built from exit codes rather than from a reader's impression. // // Two artifacts declare such checks: // plan `## Verification` — `- [ ] `cmd` -> expected: ...` (V1..Vn) // brief `## Success Criteria` — `- text: `cmd` ...` (SC1..SCn) // // Why this is code and not prose: `/trekexecute` must be able to FELL a // single-session run on a criterion that does not hold, and `/trekreview` must // hand `brief-conformance-reviewer` a real result instead of asking a // Read/Glob/Grep agent to judge whether a command "passes". Neither is // checkable while it lives only as an instruction. // // Fail-closed everywhere: a criterion that cannot run — placeholder text, no // command, a screen that is unavailable — is NEVER reported as passed. // // Screening, two layers: every command is first screened against this file's // own refusal list (writes — git push, recursive delete, pipe-to-shell, // redirection outside the working tree), then through the plugin's own // PreToolUse denylist (hooks/scripts/pre-bash-executor.mjs) — catastrophe — // before it reaches a shell. // Commands spawned from here do not pass through the Bash tool, so that hook // would not otherwise fire; the hook is invoked over its documented stdin // protocol rather than copied, so there is exactly one denylist. // // Usage: // node lib/verification/criteria-runner.mjs --plan [--json] [--cwd D] [--timeout MS] // node lib/verification/criteria-runner.mjs --brief [--json|--evidence] [--cwd D] [--timeout MS] // --evidence emits the markdown block /trekreview hands to the conformance // reviewer; --json emits the whole report. They are two shapes of one run. // Exit: 0 = ok, 1 = a criterion failed / was refused / blocked / could not run, 2 = usage // or read error (deliberately distinct from "a criterion failed"). import { readFileSync } from 'node:fs'; import { resolve, dirname, join, sep } from 'node:path'; import { homedir } from 'node:os'; import { fileURLToPath } from 'node:url'; import { spawnSync } from 'node:child_process'; const REPO_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '..', '..'); export const PLAN_HEADING = '## Verification'; export const BRIEF_HEADING = '## Success Criteria'; export const DEFAULT_HOOK = join(REPO_ROOT, 'hooks', 'scripts', 'pre-bash-executor.mjs'); export const DEFAULT_TIMEOUT_MS = 900_000; export const DEFAULT_MAX_OUTPUT = 4000; // --- parsing ---------------------------------------------------------------- // Fenced blocks are read TWICE over, for two opposite reasons: a `## ` heading // inside a fence is quoted text and must not open a section (the repo's own // examples/02-real-cli/REGENERATED.md quotes a whole plan outline inside one), // while a shell-tagged fence INSIDE the section holds the commands themselves // (examples/01-add-verbose-flag/plan.md writes its whole acceptance run that // way). Reading only bullets made a correct plan parse to zero criteria, exit // 1, and fell every single-session run — measured 2026-09-18. const FENCE = /^\s*(?:```|~~~)(.*)$/; // Languages whose fenced block is a run of commands. Deliberately a closed // list: an untagged fence is far more often expected OUTPUT than input, and // inventing a criterion from output is exactly the failure this file exists to // prevent. const SHELL_LANGS = new Set(['bash', 'sh', 'shell', 'zsh', 'console', 'shell-session']); // One entry per line: its text, whether it sits inside a fence, that fence's // info string, and whether it IS the fence marker. function scanLines(markdown) { const out = []; let lang = null; for (const text of markdown.split('\n')) { const m = text.match(FENCE); if (m) { const open = lang === null; if (open) lang = m[1].trim().toLowerCase().split(/\s+/)[0]; out.push({ text, fenced: true, lang: lang ?? '', marker: true }); if (!open) lang = null; continue; } out.push({ text, fenced: lang !== null, lang: lang ?? '', marker: false }); } return out; } // The lines from `heading` up to the next `## ` heading (exclusive). Headings // inside a fence do not count on either end. function sectionLines(markdown, heading) { const lines = scanLines(markdown); const start = lines.findIndex((l) => !l.fenced && l.text.trim() === heading); if (start === -1) return null; let end = lines.length; for (let i = start + 1; i < lines.length; i++) { if (!lines[i].fenced && lines[i].text.startsWith('## ')) { end = i; break; } } return lines.slice(start + 1, end); } // `- [ ] text`, `- [x] text`, `- text`, `* text`, `1. text` -> text const BULLET = /^\s*(?:[-*]|\d+\.)\s+(?:\[[ xX]\]\s+)?(.*)$/; const ENV_ASSIGN = /^[A-Za-z_][A-Za-z0-9_]*=\S*\s+/; /** * Whether a backticked span is a command to RUN or something the sentence * merely NAMES. A plan's template puts the command first, but a brief's * criterion usually opens with the thing under discussion — a flag * (`--verbose`), a path (`tests/`) — and running those produced exit 2 * ("invalid option") and exit 126 ("is a directory"), which the review rubric * reads as decisive: a BLOCKER invented out of prose. Measured 2026-09-18: * 3 of the 6 criteria in the repo's own example brief. * * Shape only, no filesystem lookup, so a span parses the same way everywhere. * And deliberately NO scanning on to a later span: "the first span that looks * like a command" invents commands out of prose — in that same example brief * it would have run `whoami` and `login`, two real binaries a sentence happens * to name. An absent measurement is the honest answer; a guessed one is not. */ export function looksLikeCommand(span) { let rest = String(span ?? '').trim(); while (ENV_ASSIGN.test(rest)) rest = rest.replace(ENV_ASSIGN, ''); const token = rest.split(/\s+/)[0] ?? ''; if (token === '') return false; if (token.startsWith('-')) return false; // a flag if (token.endsWith('/')) return false; // a directory if (!/^[A-Za-z0-9_.@+~/-]+$/.test(token)) return false; // quotes, braces, prose if (!/[A-Za-z0-9]/.test(token)) return false; // A lone relative path with a slash is a file the sentence names. An explicit // `./`, `../`, `/` or `~/` is an invocation and still runs. if (rest === token && token.includes('/') && !/^(\.{1,2}\/|\/|~\/)/.test(token)) return false; return true; } // The first backtick-delimited span on the line is the command by convention — // both templates put it first and a second span holds the expected output. function firstCommand(text) { const m = text.match(/`([^`]+)`/); if (!m) return { command: null, reason: 'no-command' }; const raw = m[1].trim(); // Template placeholders (`{exact command}`) are not commands. if (raw === '' || /^\{.*\}$/.test(raw)) return { command: null, reason: 'placeholder' }; if (!looksLikeCommand(raw)) return { command: null, reason: 'not-a-command' }; return { command: raw, reason: '' }; } // A command line inside a shell-tagged fence. A console block may prefix the // line with a `$` or `>` prompt; blank and comment-only lines declare nothing. // A `#` root prompt is NOT stripped: it is indistinguishable from a comment, // and running a comment is the worse of the two mistakes. function fencedCommand(text) { const line = text.trim(); if (line === '' || line.startsWith('#')) return null; return line.replace(/^[$>]\s+/, ''); } function parseSection(markdown, heading, prefix) { const lines = sectionLines(markdown, heading); if (lines === null) return []; const criteria = []; const push = (text, command, reason) => criteria.push({ label: `${prefix}${criteria.length + 1}`, text, command, reason }); for (const line of lines) { if (line.marker) continue; if (line.fenced) { if (!SHELL_LANGS.has(line.lang)) continue; const command = fencedCommand(line.text); if (command === null) continue; push(command, command, ''); continue; } const bullet = line.text.match(BULLET); if (!bullet) continue; const text = bullet[1].trim(); if (text === '') continue; const { command, reason } = firstCommand(text); push(text, command, reason); } return criteria; } /** Criteria declared in a plan's `## Verification` section, labelled V1..Vn. */ export function parsePlanVerification(markdown) { return parseSection(markdown, PLAN_HEADING, 'V'); } /** Criteria declared in a brief's `## Success Criteria` section, labelled SC1..SCn. */ export function parseSuccessCriteria(markdown) { return parseSection(markdown, BRIEF_HEADING, 'SC'); } // --- screening -------------------------------------------------------------- /** * Screen one command through the plugin's PreToolUse denylist. * Hook protocol: stdin JSON, exit 2 = block, exit 0 = allow. Any other exit * means the screen did not render a verdict — that denies, it never allows. */ export function screenCommand(command, opts = {}) { const hookPath = opts.hookPath ?? DEFAULT_HOOK; const r = spawnSync(process.execPath, [hookPath], { input: JSON.stringify({ tool_name: 'Bash', tool_input: { command } }), encoding: 'utf8', timeout: 30_000, }); if (r.status === 0) return { allowed: true, rule: '' }; const first = String(r.stderr || '').split('\n')[0].trim(); if (r.status === 2) { return { allowed: false, rule: first.replace(/^\[voyage\]\s*BLOCKED:\s*/, '') || 'blocked by the executor denylist' }; } return { allowed: false, rule: `screen unavailable (${hookPath} exited ${r.status === null ? 'on signal' : r.status}) — denied, never assumed safe`, }; } // --- refusing --------------------------------------------------------------- // The executor denylist screens CATASTROPHE (rm -rf /, fork bombs, mkfs). It // was never meant to screen an artifact that arrives from outside the repo, // and /trekreview now runs the commands a BRIEF declares. Measured 2026-09-18: // `git push origin main` and `rm -rf ` both ran; only the pipe-to-shell // was stopped. This list screens WRITES, and it is pinned by // tests/lib/criteria-runner.test.mjs so it cannot quietly shrink. // // A refusal is its own outcome, REFUSED_BY_POLICY: the command never reaches a // shell, and a refused criterion is never ok in either mode. // Segments of a command that each start at command position. const SEPARATORS = /(?:\|\||&&|[;|&\n])/; const ENV_PREFIX = /^[A-Za-z_][A-Za-z0-9_]*=\S*$/; // git's own options before the subcommand; these take a separate value. const GIT_VALUE_OPTS = new Set(['-C', '-c', '--git-dir', '--work-tree', '--namespace', '--exec-path']); // git subcommands that write somewhere this process does not own. const WRITING_GIT_SUBCOMMANDS = new Set(['push']); const RECURSIVE_FLAG = /^(?:-[A-Za-z]*r[A-Za-z]*|--recursive)$/; const PIPE_TO_SHELL = /\b(?:curl|wget|fetch)\b[^|]*\|\s*(?:sudo\s+)?(?:[\w./-]*\/)?(?:ba|z|k|da|a)?sh\b/; // `>` / `>>` and their target. `2>&1` and `>&2` duplicate a descriptor rather // than open a file, so `&` is excluded. const REDIRECT = /(?:^|[^>&\w])\d?>>?\s*(?!&)(?:"([^"]*)"|'([^']*)'|([^\s;|&<>]+))/g; const WRITABLE_DEVICES = new Set(['/dev/null', '/dev/stdout', '/dev/stderr', '/dev/tty']); const basenameOf = (token) => token.split('/').pop(); function segments(command) { return String(command).split(SEPARATORS).map((seg) => seg.trim()).filter(Boolean); } function tokensOf(segment) { const t = segment.split(/\s+/).filter(Boolean); while (t.length > 0 && ENV_PREFIX.test(t[0])) t.shift(); return t; } function gitSubcommand(tokens) { let i = 1; while (i < tokens.length) { const t = tokens[i]; if (!t.startsWith('-')) return t; if (GIT_VALUE_OPTS.has(t)) { i += 2; continue; } i += 1; } return null; } function redirectTargets(command) { const out = []; for (const m of String(command).matchAll(REDIRECT)) out.push(m[1] ?? m[2] ?? m[3]); return out.filter(Boolean); } /** * Whether a redirect target lands inside `cwd`. A target that cannot be * resolved statically — one carrying an unexpanded `$VAR` — is treated as * outside: the runner cannot know where it points, and guessing is how a * screen stops screening. */ function writesOutside(target, cwd) { if (WRITABLE_DEVICES.has(target)) return false; if (target.includes('$') || target.includes('`')) return true; const expanded = target.startsWith('~') ? join(homedir(), target.slice(1)) : target; const abs = resolve(cwd, expanded); return abs !== cwd && !abs.startsWith(cwd.endsWith(sep) ? cwd : cwd + sep); } /** * Screen one command against the refusal list. Returns `{refused, rule}`. * `cwd` is the working tree the criteria run in; writes outside it are refused. */ export function refuseCommand(command, opts = {}) { const cwd = resolve(opts.cwd ?? process.cwd()); const text = String(command ?? ''); const refuse = (rule) => ({ refused: true, rule }); if (PIPE_TO_SHELL.test(text)) return refuse('a download piped straight into a shell'); for (const segment of segments(text)) { const tokens = tokensOf(segment); if (tokens.length === 0) continue; const name = basenameOf(tokens[0]); if (name === 'git') { const sub = gitSubcommand(tokens); if (sub && WRITING_GIT_SUBCOMMANDS.has(sub)) { return refuse(`a remote-writing git subcommand (git ${sub})`); } } if (name === 'rm' && tokens.slice(1).some((t) => RECURSIVE_FLAG.test(t))) { return refuse('a recursive delete (rm -r)'); } } for (const target of redirectTargets(text)) { if (writesOutside(target, cwd)) { return refuse(`a write outside the working tree (${target})`); } } return { refused: false, rule: '' }; } // --- running ---------------------------------------------------------------- function defaultExec(command, { cwd, timeoutMs }) { const r = spawnSync('/bin/sh', ['-c', command], { cwd, encoding: 'utf8', timeout: timeoutMs }); return { status: r.status, stdout: r.stdout ?? '', stderr: r.stderr ?? '' }; } function cap(text, max) { const t = String(text ?? ''); return t.length <= max ? t : `${t.slice(0, max)}\n… [truncated: ${t.length} chars total]`; } /** * Run each criterion that has a command. Returns one result per criterion: * passed the command exited 0 * failed the command exited non-zero (or was killed by the timeout) * refused the refusal list denied it — it never reached a shell * blocked the executor denylist denied it — it never reached a shell * unrunnable there is no command to run (placeholder or prose-only criterion) */ export function runCriteria(criteria, opts = {}) { const exec = opts.exec ?? defaultExec; const screen = opts.screen ?? screenCommand; const refuse = opts.refuse ?? refuseCommand; const cwd = opts.cwd ?? process.cwd(); const timeoutMs = opts.timeoutMs ?? DEFAULT_TIMEOUT_MS; const maxOutput = opts.maxOutput ?? DEFAULT_MAX_OUTPUT; const hookPath = opts.hookPath ?? DEFAULT_HOOK; return criteria.map((c) => { const base = { label: c.label, text: c.text, command: c.command }; if (!c.command) { return { ...base, status: 'unrunnable', exitCode: null, output: `not runnable: ${c.reason || 'no command'}` }; } const refusal = refuse(c.command, { cwd }); if (refusal.refused) { return { ...base, status: 'refused', exitCode: null, output: `REFUSED_BY_POLICY: ${refusal.rule}` }; } const verdict = screen(c.command, { hookPath }); if (!verdict.allowed) { return { ...base, status: 'blocked', exitCode: null, output: `blocked: ${verdict.rule}` }; } const r = exec(c.command, { cwd, timeoutMs }); const output = cap(`${r.stdout ?? ''}${r.stderr ?? ''}`, maxOutput); return { ...base, status: r.status === 0 ? 'passed' : 'failed', exitCode: r.status, output, }; }); } /** * The verdict. `requireCommand` distinguishes the two callers: a plan promises * an exact command per criterion, so an unrunnable one is a defect in the plan; * a brief may hold criteria a human judges by reading, so there an unrunnable * criterion is reported and left to the reviewer. */ export function summarize(results, opts = {}) { const requireCommand = opts.requireCommand ?? true; const count = (s) => results.filter((r) => r.status === s).length; const summary = { total: results.length, passed: count('passed'), failed: count('failed'), blocked: count('blocked'), refused: count('refused'), unrunnable: count('unrunnable'), }; summary.ok = summary.failed === 0 && summary.blocked === 0 && summary.refused === 0 && (!requireCommand || (summary.unrunnable === 0 && summary.total > 0)); return summary; } function report(kind, source, criteria, opts, error) { const requireCommand = kind === 'plan'; const results = error ? [] : runCriteria(criteria, opts); return { kind, source, heading: kind === 'plan' ? PLAN_HEADING : BRIEF_HEADING, results, summary: summarize(results, { requireCommand }), error: error ?? null, }; } function readOrThrow(path) { try { return readFileSync(path, 'utf8'); } catch (err) { const e = new Error(`cannot read ${path}: ${err.message}`); e.code = 'READ_ERROR'; throw e; } } /** * Run a plan's `## Verification` criteria. A plan with no such section is NOT * ok: a plan that promises no end-to-end check cannot be verified, and that * must read as red rather than as nothing to do. */ export function runPlanVerification(planPath, opts = {}) { const md = readOrThrow(planPath); if (sectionLines(md, PLAN_HEADING) === null) { return report('plan', planPath, [], opts, { code: 'NO_VERIFICATION_SECTION', message: `${planPath} has no "${PLAN_HEADING}" section — nothing to verify, so the run cannot be called verified`, }); } return report('plan', planPath, parsePlanVerification(md), opts); } /** * Run a brief's `## Success Criteria` commands. Used by /trekreview to hand the * conformance reviewer a real result per criterion. */ export function runSuccessCriteriaChecks(briefPath, opts = {}) { const md = readOrThrow(briefPath); if (sectionLines(md, BRIEF_HEADING) === null) { return report('brief', briefPath, [], opts, { code: 'NO_SUCCESS_CRITERIA_SECTION', message: `${briefPath} has no "${BRIEF_HEADING}" section`, }); } return report('brief', briefPath, parseSuccessCriteria(md), opts); } // --- rendering -------------------------------------------------------------- const MARK = { passed: 'PASS', failed: 'FAILED', blocked: 'BLOCKED', refused: 'REFUSED', unrunnable: 'NOT RUN' }; export function render(rep) { const out = []; const s = rep.summary; out.push(`criteria-runner: ${s.ok ? 'OK' : 'NOT OK'} — ${rep.source} ${rep.heading}`); if (rep.error) out.push(` error: ${rep.error.code} — ${rep.error.message}`); out.push( ` ${s.passed} passed · ${s.failed} failed · ${s.blocked} blocked · ${s.refused} refused · ${s.unrunnable} not run (of ${s.total})` ); for (const r of rep.results) { const code = r.exitCode === null ? '' : ` (exit ${r.exitCode})`; out.push(` [${MARK[r.status]}] ${r.label}: \`${r.command ?? r.text}\`${code}`); if (r.status !== 'passed' && r.output) { for (const line of r.output.split('\n').slice(0, 20)) out.push(` ${line}`); } } return out.join('\n'); } /** * The evidence block /trekreview hands to `brief-conformance-reviewer`. * * The reviewer's tools are Read/Glob/Grep, so it cannot run a success * criterion's verification command. Building this block in code (rather than * letting the orchestrator narrate it) is what makes "passes" an exit code the * reviewer READS instead of a judgement it cannot make. */ export function formatCriteriaEvidence(rep) { const out = []; out.push('### Success-criteria check results (run for you — you did not run these)'); out.push(''); out.push(`Source: \`${rep.source}\` ${rep.heading}`); out.push(''); if (rep.error) { out.push(`**${rep.error.code}** — ${rep.error.message}.`); out.push(''); out.push('No criteria were run. Treat every criterion as NOT RUN.'); } else { out.push('| Criterion | Command | Result | Exit | Evidence |'); out.push('|---|---|---|---|---|'); for (const r of rep.results) { const cmd = r.command ? `\`${cell(r.command)}\`` : '(none)'; const exit = r.exitCode === null ? '—' : `exit ${r.exitCode}`; const evidence = r.status === 'passed' ? '—' : cell(firstLine(r.output)) || '—'; out.push(`| ${r.label} | ${cmd} | ${MARK[r.status]} | ${exit} | ${evidence} |`); } const s = rep.summary; out.push(''); out.push( `${s.passed} passed · ${s.failed} failed · ${s.blocked} blocked · ${s.refused} refused · ${s.unrunnable} not run (of ${s.total}).` ); } out.push(''); out.push( '**NOT RUN is not a pass.** You cannot run these commands yourself — never infer, assume or ' + 'reconstruct a result for a criterion that has none. A criterion whose result is FAILED or ' + 'BLOCKED is `Broken`; a criterion marked REFUSED was refused by policy before it reached a ' + 'shell, so like NOT RUN it is the ABSENCE of a measurement and never on its own a finding; ' + 'a criterion with no result is judged on delivered code alone, and you say so in the ' + 'Evidence column.' ); return out.join('\n'); } function firstLine(text) { return String(text ?? '').split('\n').map((l) => l.trim()).find((l) => l !== '') ?? ''; } // Markdown table cells: no pipes, no newlines, bounded length. function cell(text) { const t = String(text ?? '').replace(/\r?\n/g, ' ').replace(/\|/g, '\\|').trim(); return t.length <= 120 ? t : `${t.slice(0, 120)}…`; } // --- CLI -------------------------------------------------------------------- const USAGE = 'usage: criteria-runner.mjs (--plan | --brief ) [--json | --evidence] [--cwd ] [--timeout ]'; export function main(argv) { let mode = null; let path = null; let json = false; let evidence = false; const opts = {}; for (let i = 0; i < argv.length; i++) { const a = argv[i]; if ((a === '--plan' || a === '--brief') && argv[i + 1]) { if (mode) { process.stderr.write(`criteria-runner: one mode at a time\n${USAGE}\n`); return 2; } mode = a.slice(2); path = argv[++i]; } else if (a === '--json') json = true; else if (a === '--evidence') evidence = true; else if (a === '--cwd' && argv[i + 1]) opts.cwd = resolve(argv[++i]); else if (a === '--timeout' && argv[i + 1]) opts.timeoutMs = Number(argv[++i]); else { process.stderr.write(`criteria-runner: unknown argument ${a}\n${USAGE}\n`); return 2; } } if (!mode) { process.stderr.write(`criteria-runner: no artifact given\n${USAGE}\n`); return 2; } if (json && evidence) { process.stderr.write(`criteria-runner: --json and --evidence are two output shapes; pick one\n${USAGE}\n`); return 2; } if (opts.timeoutMs !== undefined && !Number.isFinite(opts.timeoutMs)) { process.stderr.write(`criteria-runner: --timeout must be a number\n${USAGE}\n`); return 2; } let rep; try { rep = mode === 'plan' ? runPlanVerification(path, opts) : runSuccessCriteriaChecks(path, opts); } catch (err) { process.stderr.write(`criteria-runner: ${err.message}\n`); return 2; } const text = json ? JSON.stringify(rep, null, 2) : evidence ? formatCriteriaEvidence(rep) : render(rep); process.stdout.write(text + '\n'); return rep.summary.ok ? 0 : 1; } if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { process.exitCode = main(process.argv.slice(2)); }