fix(execute): run the plan's Verification on the single-session path (D-03)
A trekplan's `## Verification` section is where the brief's success criteria land. Phase 7 opened with "**Skip for trekplans.**", and only the multi-session wave path (Phase 2.6 Step 3) ran master verification. A plan executed in ONE session therefore reported `completed` without ever running the criteria it was measured against — the executor's own belief was the only evidence. The check now exists as code, not as an instruction: - `lib/verification/criteria-runner.mjs` parses the criteria an artifact DECLARES (a plan's `## Verification`, a brief's `## Success Criteria`), runs each command, and returns a verdict built from exit codes. Fail-closed throughout: a placeholder, a prose-only criterion, or an unavailable screen is `unrunnable`/`blocked`, never `passed`. A plan with no `## Verification` section exits 1 — a plan that promises no end-to-end check cannot be reported as verified. - Every command is screened through the plugin's own PreToolUse denylist (`hooks/scripts/pre-bash-executor.mjs`) before it reaches a shell. Chose invoking that hook over its documented stdin protocol rather than copying its rules, because a command spawned from node never passes through the Bash tool and so the hook cannot fire by itself — this keeps exactly one denylist. - Phase 7 is now "Exit / verification check": session specs run the exit condition, trekplans run the criteria runner. Phase 4's entry-condition skip for trekplans stands — a plan carries no entry condition; the exit side is not symmetrical. - A failing criterion FELLS the run: `plan_verification.status != "passed"` forbids `result: completed`. That is clause 2 of the stop-signal contract, now enforced on the single-session path too. Red first: `tests/lib/criteria-runner.test.mjs` (26 tests) against two committed fixture plans, one of which declares a criterion that fails on purpose. The doc pin in `tests/lib/doc-consistency.test.mjs` guards the wiring — a capability no phase calls is the same defect wearing a lib/ file; verified red against the pre-fix Phase 7 (skip present, runner absent, no fell-the-run clause). Suite: 1108 (1106/0/2), up 27 from 1081. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
parent
8d1669ef51
commit
e55ca9fc89
SSH key fingerprint:
SHA256:JakMjO6FTBBzN0Bhfj9saOoEjaFxlSdYuZQQpM/lF9Q
6 changed files with 756 additions and 6 deletions
308
lib/verification/criteria-runner.mjs
Normal file
308
lib/verification/criteria-runner.mjs
Normal file
|
|
@ -0,0 +1,308 @@
|
|||
#!/usr/bin/env node
|
||||
// lib/verification/criteria-runner.mjs
|
||||
// Runs the falsifiable checks a pipeline artifact DECLARES, and reports a
|
||||
// verdict built from exit codes rather than from a reader's impression.
|
||||
//
|
||||
// Two artifacts declare such checks:
|
||||
// plan `## Verification` — `- [ ] `cmd` -> expected: ...` (V1..Vn)
|
||||
// brief `## Success Criteria` — `- text: `cmd` ...` (SC1..SCn)
|
||||
//
|
||||
// Why this is code and not prose: `/trekexecute` must be able to FELL a
|
||||
// single-session run on a criterion that does not hold, and `/trekreview` must
|
||||
// hand `brief-conformance-reviewer` a real result instead of asking a
|
||||
// Read/Glob/Grep agent to judge whether a command "passes". Neither is
|
||||
// checkable while it lives only as an instruction.
|
||||
//
|
||||
// Fail-closed everywhere: a criterion that cannot run — placeholder text, no
|
||||
// command, a screen that is unavailable — is NEVER reported as passed.
|
||||
//
|
||||
// Screening: every command is screened through the plugin's own PreToolUse
|
||||
// denylist (hooks/scripts/pre-bash-executor.mjs) before it reaches a shell.
|
||||
// Commands spawned from here do not pass through the Bash tool, so that hook
|
||||
// would not otherwise fire; the hook is invoked over its documented stdin
|
||||
// protocol rather than copied, so there is exactly one denylist.
|
||||
//
|
||||
// Usage:
|
||||
// node lib/verification/criteria-runner.mjs --plan <plan.md> [--json] [--cwd D] [--timeout MS]
|
||||
// node lib/verification/criteria-runner.mjs --brief <brief.md> [--json] [--cwd D] [--timeout MS]
|
||||
// Exit: 0 = ok, 1 = a criterion failed / was blocked / could not run, 2 = usage
|
||||
// or read error (deliberately distinct from "a criterion failed").
|
||||
|
||||
import { readFileSync } from 'node:fs';
|
||||
import { resolve, dirname, join } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
|
||||
const REPO_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '..', '..');
|
||||
|
||||
export const PLAN_HEADING = '## Verification';
|
||||
export const BRIEF_HEADING = '## Success Criteria';
|
||||
export const DEFAULT_HOOK = join(REPO_ROOT, 'hooks', 'scripts', 'pre-bash-executor.mjs');
|
||||
export const DEFAULT_TIMEOUT_MS = 900_000;
|
||||
export const DEFAULT_MAX_OUTPUT = 4000;
|
||||
|
||||
// --- parsing ----------------------------------------------------------------
|
||||
|
||||
// The lines from `heading` up to the next `## ` heading (exclusive).
|
||||
function sectionLines(markdown, heading) {
|
||||
const lines = markdown.split('\n');
|
||||
const start = lines.findIndex((l) => l.trim() === heading);
|
||||
if (start === -1) return null;
|
||||
let end = lines.length;
|
||||
for (let i = start + 1; i < lines.length; i++) {
|
||||
if (lines[i].startsWith('## ')) { end = i; break; }
|
||||
}
|
||||
return lines.slice(start + 1, end);
|
||||
}
|
||||
|
||||
// `- [ ] text`, `- [x] text`, `- text`, `* text`, `1. text` -> text
|
||||
const BULLET = /^\s*(?:[-*]|\d+\.)\s+(?:\[[ xX]\]\s+)?(.*)$/;
|
||||
|
||||
// The first backtick-delimited span on the line is the command by convention —
|
||||
// both templates put it first and a second span holds the expected output.
|
||||
function firstCommand(text) {
|
||||
const m = text.match(/`([^`]+)`/);
|
||||
if (!m) return { command: null, reason: 'no-command' };
|
||||
const raw = m[1].trim();
|
||||
// Template placeholders (`{exact command}`) are not commands.
|
||||
if (raw === '' || /^\{.*\}$/.test(raw)) return { command: null, reason: 'placeholder' };
|
||||
return { command: raw, reason: '' };
|
||||
}
|
||||
|
||||
function parseSection(markdown, heading, prefix) {
|
||||
const lines = sectionLines(markdown, heading);
|
||||
if (lines === null) return [];
|
||||
const criteria = [];
|
||||
for (const line of lines) {
|
||||
const bullet = line.match(BULLET);
|
||||
if (!bullet) continue;
|
||||
const text = bullet[1].trim();
|
||||
if (text === '') continue;
|
||||
const { command, reason } = firstCommand(text);
|
||||
criteria.push({ label: `${prefix}${criteria.length + 1}`, text, command, reason });
|
||||
}
|
||||
return criteria;
|
||||
}
|
||||
|
||||
/** Criteria declared in a plan's `## Verification` section, labelled V1..Vn. */
|
||||
export function parsePlanVerification(markdown) {
|
||||
return parseSection(markdown, PLAN_HEADING, 'V');
|
||||
}
|
||||
|
||||
/** Criteria declared in a brief's `## Success Criteria` section, labelled SC1..SCn. */
|
||||
export function parseSuccessCriteria(markdown) {
|
||||
return parseSection(markdown, BRIEF_HEADING, 'SC');
|
||||
}
|
||||
|
||||
// --- screening --------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Screen one command through the plugin's PreToolUse denylist.
|
||||
* Hook protocol: stdin JSON, exit 2 = block, exit 0 = allow. Any other exit
|
||||
* means the screen did not render a verdict — that denies, it never allows.
|
||||
*/
|
||||
export function screenCommand(command, opts = {}) {
|
||||
const hookPath = opts.hookPath ?? DEFAULT_HOOK;
|
||||
const r = spawnSync(process.execPath, [hookPath], {
|
||||
input: JSON.stringify({ tool_name: 'Bash', tool_input: { command } }),
|
||||
encoding: 'utf8',
|
||||
timeout: 30_000,
|
||||
});
|
||||
if (r.status === 0) return { allowed: true, rule: '' };
|
||||
const first = String(r.stderr || '').split('\n')[0].trim();
|
||||
if (r.status === 2) {
|
||||
return { allowed: false, rule: first.replace(/^\[voyage\]\s*BLOCKED:\s*/, '') || 'blocked by the executor denylist' };
|
||||
}
|
||||
return {
|
||||
allowed: false,
|
||||
rule: `screen unavailable (${hookPath} exited ${r.status === null ? 'on signal' : r.status}) — denied, never assumed safe`,
|
||||
};
|
||||
}
|
||||
|
||||
// --- running ----------------------------------------------------------------
|
||||
|
||||
function defaultExec(command, { cwd, timeoutMs }) {
|
||||
const r = spawnSync('/bin/sh', ['-c', command], { cwd, encoding: 'utf8', timeout: timeoutMs });
|
||||
return { status: r.status, stdout: r.stdout ?? '', stderr: r.stderr ?? '' };
|
||||
}
|
||||
|
||||
function cap(text, max) {
|
||||
const t = String(text ?? '');
|
||||
return t.length <= max ? t : `${t.slice(0, max)}\n… [truncated: ${t.length} chars total]`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Run each criterion that has a command. Returns one result per criterion:
|
||||
* passed the command exited 0
|
||||
* failed the command exited non-zero (or was killed by the timeout)
|
||||
* blocked the screen denied it — it never reached a shell
|
||||
* unrunnable there is no command to run (placeholder or prose-only criterion)
|
||||
*/
|
||||
export function runCriteria(criteria, opts = {}) {
|
||||
const exec = opts.exec ?? defaultExec;
|
||||
const screen = opts.screen ?? screenCommand;
|
||||
const cwd = opts.cwd ?? process.cwd();
|
||||
const timeoutMs = opts.timeoutMs ?? DEFAULT_TIMEOUT_MS;
|
||||
const maxOutput = opts.maxOutput ?? DEFAULT_MAX_OUTPUT;
|
||||
const hookPath = opts.hookPath ?? DEFAULT_HOOK;
|
||||
|
||||
return criteria.map((c) => {
|
||||
const base = { label: c.label, text: c.text, command: c.command };
|
||||
if (!c.command) {
|
||||
return { ...base, status: 'unrunnable', exitCode: null, output: `not runnable: ${c.reason || 'no command'}` };
|
||||
}
|
||||
const verdict = screen(c.command, { hookPath });
|
||||
if (!verdict.allowed) {
|
||||
return { ...base, status: 'blocked', exitCode: null, output: `blocked: ${verdict.rule}` };
|
||||
}
|
||||
const r = exec(c.command, { cwd, timeoutMs });
|
||||
const output = cap(`${r.stdout ?? ''}${r.stderr ?? ''}`, maxOutput);
|
||||
return {
|
||||
...base,
|
||||
status: r.status === 0 ? 'passed' : 'failed',
|
||||
exitCode: r.status,
|
||||
output,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* The verdict. `requireCommand` distinguishes the two callers: a plan promises
|
||||
* an exact command per criterion, so an unrunnable one is a defect in the plan;
|
||||
* a brief may hold criteria a human judges by reading, so there an unrunnable
|
||||
* criterion is reported and left to the reviewer.
|
||||
*/
|
||||
export function summarize(results, opts = {}) {
|
||||
const requireCommand = opts.requireCommand ?? true;
|
||||
const count = (s) => results.filter((r) => r.status === s).length;
|
||||
const summary = {
|
||||
total: results.length,
|
||||
passed: count('passed'),
|
||||
failed: count('failed'),
|
||||
blocked: count('blocked'),
|
||||
unrunnable: count('unrunnable'),
|
||||
};
|
||||
summary.ok =
|
||||
summary.failed === 0 &&
|
||||
summary.blocked === 0 &&
|
||||
(!requireCommand || (summary.unrunnable === 0 && summary.total > 0));
|
||||
return summary;
|
||||
}
|
||||
|
||||
function report(kind, source, criteria, opts, error) {
|
||||
const requireCommand = kind === 'plan';
|
||||
const results = error ? [] : runCriteria(criteria, opts);
|
||||
return {
|
||||
kind,
|
||||
source,
|
||||
heading: kind === 'plan' ? PLAN_HEADING : BRIEF_HEADING,
|
||||
results,
|
||||
summary: summarize(results, { requireCommand }),
|
||||
error: error ?? null,
|
||||
};
|
||||
}
|
||||
|
||||
function readOrThrow(path) {
|
||||
try {
|
||||
return readFileSync(path, 'utf8');
|
||||
} catch (err) {
|
||||
const e = new Error(`cannot read ${path}: ${err.message}`);
|
||||
e.code = 'READ_ERROR';
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Run a plan's `## Verification` criteria. A plan with no such section is NOT
|
||||
* ok: a plan that promises no end-to-end check cannot be verified, and that
|
||||
* must read as red rather than as nothing to do.
|
||||
*/
|
||||
export function runPlanVerification(planPath, opts = {}) {
|
||||
const md = readOrThrow(planPath);
|
||||
if (sectionLines(md, PLAN_HEADING) === null) {
|
||||
return report('plan', planPath, [], opts, {
|
||||
code: 'NO_VERIFICATION_SECTION',
|
||||
message: `${planPath} has no "${PLAN_HEADING}" section — nothing to verify, so the run cannot be called verified`,
|
||||
});
|
||||
}
|
||||
return report('plan', planPath, parsePlanVerification(md), opts);
|
||||
}
|
||||
|
||||
/**
|
||||
* Run a brief's `## Success Criteria` commands. Used by /trekreview to hand the
|
||||
* conformance reviewer a real result per criterion.
|
||||
*/
|
||||
export function runSuccessCriteriaChecks(briefPath, opts = {}) {
|
||||
const md = readOrThrow(briefPath);
|
||||
if (sectionLines(md, BRIEF_HEADING) === null) {
|
||||
return report('brief', briefPath, [], opts, {
|
||||
code: 'NO_SUCCESS_CRITERIA_SECTION',
|
||||
message: `${briefPath} has no "${BRIEF_HEADING}" section`,
|
||||
});
|
||||
}
|
||||
return report('brief', briefPath, parseSuccessCriteria(md), opts);
|
||||
}
|
||||
|
||||
// --- rendering --------------------------------------------------------------
|
||||
|
||||
const MARK = { passed: 'PASS', failed: 'FAILED', blocked: 'BLOCKED', unrunnable: 'NOT RUN' };
|
||||
|
||||
export function render(rep) {
|
||||
const out = [];
|
||||
const s = rep.summary;
|
||||
out.push(`criteria-runner: ${s.ok ? 'OK' : 'NOT OK'} — ${rep.source} ${rep.heading}`);
|
||||
if (rep.error) out.push(` error: ${rep.error.code} — ${rep.error.message}`);
|
||||
out.push(
|
||||
` ${s.passed} passed · ${s.failed} failed · ${s.blocked} blocked · ${s.unrunnable} not run (of ${s.total})`
|
||||
);
|
||||
for (const r of rep.results) {
|
||||
const code = r.exitCode === null ? '' : ` (exit ${r.exitCode})`;
|
||||
out.push(` [${MARK[r.status]}] ${r.label}: \`${r.command ?? r.text}\`${code}`);
|
||||
if (r.status !== 'passed' && r.output) {
|
||||
for (const line of r.output.split('\n').slice(0, 20)) out.push(` ${line}`);
|
||||
}
|
||||
}
|
||||
return out.join('\n');
|
||||
}
|
||||
|
||||
// --- CLI --------------------------------------------------------------------
|
||||
|
||||
const USAGE =
|
||||
'usage: criteria-runner.mjs (--plan <plan.md> | --brief <brief.md>) [--json] [--cwd <dir>] [--timeout <ms>]';
|
||||
|
||||
export function main(argv) {
|
||||
let mode = null;
|
||||
let path = null;
|
||||
let json = false;
|
||||
const opts = {};
|
||||
for (let i = 0; i < argv.length; i++) {
|
||||
const a = argv[i];
|
||||
if ((a === '--plan' || a === '--brief') && argv[i + 1]) {
|
||||
if (mode) { process.stderr.write(`criteria-runner: one mode at a time\n${USAGE}\n`); return 2; }
|
||||
mode = a.slice(2);
|
||||
path = argv[++i];
|
||||
} else if (a === '--json') json = true;
|
||||
else if (a === '--cwd' && argv[i + 1]) opts.cwd = resolve(argv[++i]);
|
||||
else if (a === '--timeout' && argv[i + 1]) opts.timeoutMs = Number(argv[++i]);
|
||||
else { process.stderr.write(`criteria-runner: unknown argument ${a}\n${USAGE}\n`); return 2; }
|
||||
}
|
||||
if (!mode) { process.stderr.write(`criteria-runner: no artifact given\n${USAGE}\n`); return 2; }
|
||||
if (opts.timeoutMs !== undefined && !Number.isFinite(opts.timeoutMs)) {
|
||||
process.stderr.write(`criteria-runner: --timeout must be a number\n${USAGE}\n`);
|
||||
return 2;
|
||||
}
|
||||
|
||||
let rep;
|
||||
try {
|
||||
rep = mode === 'plan' ? runPlanVerification(path, opts) : runSuccessCriteriaChecks(path, opts);
|
||||
} catch (err) {
|
||||
process.stderr.write(`criteria-runner: ${err.message}\n`);
|
||||
return 2;
|
||||
}
|
||||
process.stdout.write((json ? JSON.stringify(rep, null, 2) : render(rep)) + '\n');
|
||||
return rep.summary.ok ? 0 : 1;
|
||||
}
|
||||
|
||||
if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
|
||||
process.exitCode = main(process.argv.slice(2));
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue