fix(validate): a check that can never run is caught when the brief or plan is written

voyage-lekerepo end-to-end trial (2026-10-06), report points 2, 3, 4, 9:
11 of 13 brief criteria were NOT RUN at review with no warning in /trekbrief;
a plan `git grep` criterion passed validator, gate and three critic rounds,
then felled the run in Phase 7; the End-to-end field and Phase 7's text
search never met; a prose Verify left the test command undefined.

- criteria-runner: runnableCheck() screens a declared check without running
  it (same allowlist; an npm run script is not looked up at write time);
  /bin/bash tests/<script> allowed; the plan report carries end_to_end,
  verified when a passed criterion runs a step's End-to-end command (the
  older text form still counts).
- plan-validator 1.8: PLAN_VERIFICATION_NOT_RUNNABLE,
  PLAN_STEP_VERIFY_NOT_RUNNABLE, PLAN_END_TO_END_NOT_RUNNABLE,
  PLAN_END_TO_END_NOT_VERIFIED (the gate inherits them).
- brief-validator: BRIEF_SC_NOT_RUNNABLE warning; /trekbrief offers a
  tests/ script; brief-reviewer no longer proposes unrunnable commands.
- trekexecute: Phase 7 copies end_to_end from the runner; Verify's test
  command is its first backticked span.

Measured on the trial's own brief and plan: 11 warnings (the 11 NOT RUN),
and the plan stops on V5 git grep and the unlisted End-to-end command.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Kjell Tore Guttormsen 2026-10-08 06:42:33 +02:00
commit c56f852ab6
Signed by: ktg
SSH key fingerprint: SHA256:JakMjO6FTBBzN0Bhfj9saOoEjaFxlSdYuZQQpM/lF9Q
14 changed files with 422 additions and 38 deletions

View file

@ -15,7 +15,12 @@
// - the optional step fields `Interface in:`, `Interface out:`,
// `Review focus:` are never empty when present;
// - `ceremony:` is small | bounded | architectural (absent = architectural);
// an architectural plan carries `## Design` (see checkDesign).
// an architectural plan carries `## Design` (see checkDesign);
// - every check the run will execute can run (checkRunnable): each
// `## Verification` criterion, each step's `Verify:` (its first backticked
// span is the step's test command) and each `End-to-end:` command pass the
// criteria runner's allowlist, and each `End-to-end:` command also stands
// under `## Verification`, where Phase 7 runs it.
// Strict mode reports these as errors; --soft downgrades them to warnings.
import { readFileSync, existsSync, realpathSync } from 'node:fs';
@ -28,6 +33,7 @@ import {
import { validateAllManifests } from '../parsers/manifest-yaml.mjs';
import { parseDocument } from '../util/frontmatter.mjs';
import { issue, fail } from '../util/result.mjs';
import { parsePlanVerification, firstCommand, runnableCheck } from '../verification/criteria-runner.mjs';
export const OPTIONAL_STEP_FIELDS = Object.freeze(['Interface in', 'Interface out', 'Review focus']);
export const DESIGN_SUBSECTIONS = Object.freeze([
@ -69,6 +75,56 @@ function checkSteps(sections) {
return out;
}
const WAY_IN = 'A check the allowlist has no name for (a pipe, `jq`, `grep`, the product\'s own binary) is ' +
'wrapped in a script under tests/ and declared as `bash tests/<script>.sh`.';
/**
* Every check the run will execute can run. Without this a plan passed the
* validator, the plan gate and three critic rounds with a `git grep` criterion
* the runner refuses, and Phase 7 felled a run whose steps had all passed
* (voyage-lekerepo trial, 2026-10-06).
*/
function checkRunnable(text, sections, opts) {
const out = [];
const criteria = parsePlanVerification(text);
for (const c of criteria) {
const r = runnableCheck(c, opts);
if (!r.runnable) {
out.push(issue('PLAN_VERIFICATION_NOT_RUNNABLE',
`## Verification ${c.label} can never run (${r.reason}): ${c.text}`,
`Phase 7 runs every criterion and fells the run on one it cannot run. ${WAY_IN}`));
}
}
const verified = new Set(criteria.filter(c => c.command).map(c => c.command.trim().replace(/\s+/g, ' ')));
for (const s of sections) {
const f = parseStepFields(s.body);
if (f.has('Verify')) {
const v = firstCommand(f.get('Verify').text);
const r = runnableCheck(v, opts);
if (!r.runnable) {
out.push(issue('PLAN_STEP_VERIFY_NOT_RUNNABLE',
`Step ${s.n}: \`Verify:\` does not open with one runnable test command (${r.reason})`,
'The first backticked span of `Verify:` is the step\'s test command: proof of red and the done line run it, ' +
`and nothing after it is run. ${WAY_IN}`));
}
}
if (f.has('End-to-end') && BACKTICKED.test(f.get('End-to-end').text)) {
const e = firstCommand(f.get('End-to-end').text);
const r = runnableCheck(e, opts);
if (!r.runnable) {
out.push(issue('PLAN_END_TO_END_NOT_RUNNABLE',
`Step ${s.n}: \`End-to-end:\` command can never run (${r.reason})`,
`Run the product from a script: \`bash tests/e2e.sh\`. ${WAY_IN}`));
} else if (!verified.has(e.command.trim().replace(/\s+/g, ' '))) {
out.push(issue('PLAN_END_TO_END_NOT_VERIFIED',
`Step ${s.n}: \`End-to-end:\` command \`${e.command}\` does not stand under ## Verification`,
'Phase 7 runs `## Verification`; list the same command there, or the result is "not verified end to end".'));
}
}
}
return out;
}
/** Table body rows (skips the header and the |---| separator). */
function tableRows(text) {
const rows = text.split('\n').filter(l => l.trim().startsWith('|'));
@ -176,7 +232,7 @@ export function validatePlanContent(rawText, opts = {}) {
const ceremony = extractPlanCeremony(text);
if (versionAtLeast(planVersion, '1.8')) {
const v18 = [...checkSteps(sections)];
const v18 = [...checkSteps(sections), ...checkRunnable(text, sections, { cwd: opts.cwd })];
if (!ceremony.valid) {
v18.push(issue('PLAN_CEREMONY_INVALID', `ceremony=${ceremony.raw}`,
`Allowed: ${CEREMONY_VALUES.join(' | ')} (absent = architectural).`));