fix(validate): a check that can never run is caught when the brief or plan is written

voyage-lekerepo end-to-end trial (2026-10-06), report points 2, 3, 4, 9:
11 of 13 brief criteria were NOT RUN at review with no warning in /trekbrief;
a plan `git grep` criterion passed validator, gate and three critic rounds,
then felled the run in Phase 7; the End-to-end field and Phase 7's text
search never met; a prose Verify left the test command undefined.

- criteria-runner: runnableCheck() screens a declared check without running
  it (same allowlist; an npm run script is not looked up at write time);
  /bin/bash tests/<script> allowed; the plan report carries end_to_end,
  verified when a passed criterion runs a step's End-to-end command (the
  older text form still counts).
- plan-validator 1.8: PLAN_VERIFICATION_NOT_RUNNABLE,
  PLAN_STEP_VERIFY_NOT_RUNNABLE, PLAN_END_TO_END_NOT_RUNNABLE,
  PLAN_END_TO_END_NOT_VERIFIED (the gate inherits them).
- brief-validator: BRIEF_SC_NOT_RUNNABLE warning; /trekbrief offers a
  tests/ script; brief-reviewer no longer proposes unrunnable commands.
- trekexecute: Phase 7 copies end_to_end from the runner; Verify's test
  command is its first backticked span.

Measured on the trial's own brief and plan: 11 warnings (the 11 NOT RUN),
and the plan stops on V5 git grep and the unlisted End-to-end command.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Kjell Tore Guttormsen 2026-10-08 06:42:33 +02:00
commit c56f852ab6
Signed by: ktg
SSH key fingerprint: SHA256:JakMjO6FTBBzN0Bhfj9saOoEjaFxlSdYuZQQpM/lF9Q
14 changed files with 422 additions and 38 deletions

View file

@ -38,6 +38,7 @@ import { readFileSync } from 'node:fs';
import { resolve, dirname, join } from 'node:path';
import { fileURLToPath } from 'node:url';
import { spawnSync } from 'node:child_process';
import { sliceSteps, parseStepFields } from '../parsers/plan-schema.mjs';
const REPO_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '..', '..');
@ -135,7 +136,8 @@ export function looksLikeCommand(span) {
// The first backtick-delimited span on the line is the command by convention —
// both templates put it first and a second span holds the expected output.
function firstCommand(text) {
// A plan step's `Verify:` and `End-to-end:` fields follow the same convention.
export function firstCommand(text) {
const m = text.match(/`([^`]+)`/);
if (!m) return { command: null, reason: 'no-command' };
const raw = m[1].trim();
@ -273,10 +275,12 @@ function npmScripts(cwd) {
// One entry per runner: given the command's tokens (and the working tree the
// criteria run in), either allow it or say why not.
const RUNNERS = {
npm: (t, cwd) => {
npm: (t, cwd, writeTime) => {
if (t[1] === 'test') return permit();
if (t[1] !== 'run') return deny(`npm, but not \`npm test\` or \`npm run\` (\`npm ${t[1] ?? ''}\`)`);
if (t[2] === undefined) return deny('npm run names no script');
// When a brief or plan is written, the script may be one the plan adds.
if (writeTime) return permit();
return npmScripts(cwd).has(t[2])
? permit()
: deny(`npm run names a script package.json does not declare (${t[2]})`);
@ -292,6 +296,8 @@ const RUNNERS = {
go: (t) => (t[1] === 'test' ? permit() : deny('go, but not `go test`')),
make: (t) => (t[1] === 'test' ? permit() : deny('make, but not `make test`')),
bash: (t) => (isTestScript(t[1]) ? permit() : deny('bash, but not a script under tests/')),
// The system bash by its path (macOS: 3.2), so a check can pin the shell.
'/bin/bash': (t) => (isTestScript(t[1]) ? permit() : deny('/bin/bash, but not a script under tests/')),
git: (t) =>
READONLY_GIT.has(t[1])
? permit()
@ -304,6 +310,25 @@ const RUNNERS = {
* the package.json found there.
*/
export function allowedCommand(command, opts = {}) {
return screenShape(command, opts, false);
}
/**
* Whether a declared check could ever run, decided WITHOUT running it: the
* same allowlist, used when a brief or a plan is written instead of only when
* the run is judged. Measured in the voyage-lekerepo trial (2026-10-06): 11 of
* 13 brief criteria were NOT RUN at review, and a plan criterion (`git grep`)
* passed the validator, the plan gate and three critic rounds, then felled the
* run in Phase 7. An `npm run` script is not looked up here: the plan may be
* what adds it. Returns `{runnable, reason}`.
*/
export function runnableCheck(criterion, opts = {}) {
if (!criterion?.command) return { runnable: false, reason: criterion?.reason || 'no command' };
const r = screenShape(criterion.command, opts, true);
return { runnable: r.allowed, reason: r.reason };
}
function screenShape(command, opts, writeTime) {
const cwd = resolve(opts.cwd ?? process.cwd());
const text = String(command ?? '');
if (text.trim() === '') return deny('the command is empty');
@ -312,9 +337,8 @@ export function allowedCommand(command, opts = {}) {
if (operator) return deny(`a shell operator (${operator[0]}) — one criterion declares one command, unpiped`);
const tokens = text.trim().split(/\s+/);
const runner = RUNNERS[tokens[0]];
if (!runner) return deny(`\`${tokens[0]}\` is not a known test runner`);
return runner(tokens, cwd);
if (!Object.hasOwn(RUNNERS, tokens[0])) return deny(`\`${tokens[0]}\` is not a known test runner`);
return RUNNERS[tokens[0]](tokens, cwd, writeTime);
}
// --- running ----------------------------------------------------------------
@ -398,17 +422,52 @@ export function summarize(results, opts = {}) {
return summary;
}
function report(kind, source, criteria, opts, error) {
/** The commands a plan's steps declare in `End-to-end:` (first backticked span each). */
export function endToEndCommands(markdown) {
return sliceSteps(String(markdown ?? '').replace(/\r\n?/g, '\n'))
.map((s) => parseStepFields(s.body).get('End-to-end'))
.filter(Boolean)
.map((f) => firstCommand(f.text).command)
.filter(Boolean);
}
const normalizeCommand = (c) => String(c).trim().replace(/\s+/g, ' ');
const SAYS_END_TO_END = /end to end|end-to-end|run the thing/i;
/**
* Whether the run was verified end to end (P9 (6)). A passed criterion
* verifies it when it runs a step's `End-to-end:` command — the field the plan
* validator requires — or, the older form, when its text says it runs the
* thing end to end. Searching the text alone missed a plan that had the field
* and ran it green (voyage-lekerepo trial, 2026-10-06). A verification that
* did not pass is never verified end to end.
*/
export function endToEndStatus(results, summary, e2eCommands = []) {
const no = { status: 'not-verified', by: null, criterion: null };
if (!summary.ok) return no;
const passed = results.filter((r) => r.status === 'passed');
const want = new Set(e2eCommands.map(normalizeCommand));
const byField = passed.find((r) => r.command && want.has(normalizeCommand(r.command)));
if (byField) return { status: 'verified', by: 'end-to-end-field', criterion: byField.label };
const byText = passed.find((r) => SAYS_END_TO_END.test(r.text));
if (byText) return { status: 'verified', by: 'criterion-text', criterion: byText.label };
return no;
}
function report(kind, source, criteria, opts, error, e2eCommands = []) {
const requireCommand = kind === 'plan';
const results = error ? [] : runCriteria(criteria, opts);
return {
const summary = summarize(results, { requireCommand });
const rep = {
kind,
source,
heading: kind === 'plan' ? PLAN_HEADING : BRIEF_HEADING,
results,
summary: summarize(results, { requireCommand }),
summary,
error: error ?? null,
};
if (kind === 'plan') rep.end_to_end = endToEndStatus(results, summary, e2eCommands);
return rep;
}
/**
@ -460,7 +519,7 @@ export function runPlanVerification(planPath, opts = {}) {
if (criteria.length === 0) {
return report('plan', planPath, [], opts, noCriteria(planPath, PLAN_HEADING, section.line));
}
return report('plan', planPath, criteria, opts);
return report('plan', planPath, criteria, opts, null, endToEndCommands(md));
}
/**
@ -502,6 +561,12 @@ export function render(rep) {
for (const line of r.output.split('\n').slice(0, 20)) out.push(` ${line}`);
}
}
if (rep.end_to_end) {
const e = rep.end_to_end;
out.push(e.status === 'verified'
? ` end to end: verified (${e.criterion}, ${e.by === 'end-to-end-field' ? "runs a step's End-to-end command" : 'its text says it runs the thing'})`
: ' end to end: not verified end to end');
}
return out.join('\n');
}