fix(validate): a check that can never run is caught when the brief or plan is written
voyage-lekerepo end-to-end trial (2026-10-06), report points 2, 3, 4, 9: 11 of 13 brief criteria were NOT RUN at review with no warning in /trekbrief; a plan `git grep` criterion passed validator, gate and three critic rounds, then felled the run in Phase 7; the End-to-end field and Phase 7's text search never met; a prose Verify left the test command undefined. - criteria-runner: runnableCheck() screens a declared check without running it (same allowlist; an npm run script is not looked up at write time); /bin/bash tests/<script> allowed; the plan report carries end_to_end, verified when a passed criterion runs a step's End-to-end command (the older text form still counts). - plan-validator 1.8: PLAN_VERIFICATION_NOT_RUNNABLE, PLAN_STEP_VERIFY_NOT_RUNNABLE, PLAN_END_TO_END_NOT_RUNNABLE, PLAN_END_TO_END_NOT_VERIFIED (the gate inherits them). - brief-validator: BRIEF_SC_NOT_RUNNABLE warning; /trekbrief offers a tests/ script; brief-reviewer no longer proposes unrunnable commands. - trekexecute: Phase 7 copies end_to_end from the runner; Verify's test command is its first backticked span. Measured on the trial's own brief and plan: 11 warnings (the 11 NOT RUN), and the plan stops on V5 git grep and the unlisted End-to-end command. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
ca6ee8779a
commit
c56f852ab6
14 changed files with 422 additions and 38 deletions
|
|
@ -15,7 +15,12 @@
|
|||
// - the optional step fields `Interface in:`, `Interface out:`,
|
||||
// `Review focus:` are never empty when present;
|
||||
// - `ceremony:` is small | bounded | architectural (absent = architectural);
|
||||
// an architectural plan carries `## Design` (see checkDesign).
|
||||
// an architectural plan carries `## Design` (see checkDesign);
|
||||
// - every check the run will execute can run (checkRunnable): each
|
||||
// `## Verification` criterion, each step's `Verify:` (its first backticked
|
||||
// span is the step's test command) and each `End-to-end:` command pass the
|
||||
// criteria runner's allowlist, and each `End-to-end:` command also stands
|
||||
// under `## Verification`, where Phase 7 runs it.
|
||||
// Strict mode reports these as errors; --soft downgrades them to warnings.
|
||||
|
||||
import { readFileSync, existsSync, realpathSync } from 'node:fs';
|
||||
|
|
@ -28,6 +33,7 @@ import {
|
|||
import { validateAllManifests } from '../parsers/manifest-yaml.mjs';
|
||||
import { parseDocument } from '../util/frontmatter.mjs';
|
||||
import { issue, fail } from '../util/result.mjs';
|
||||
import { parsePlanVerification, firstCommand, runnableCheck } from '../verification/criteria-runner.mjs';
|
||||
|
||||
export const OPTIONAL_STEP_FIELDS = Object.freeze(['Interface in', 'Interface out', 'Review focus']);
|
||||
export const DESIGN_SUBSECTIONS = Object.freeze([
|
||||
|
|
@ -69,6 +75,56 @@ function checkSteps(sections) {
|
|||
return out;
|
||||
}
|
||||
|
||||
const WAY_IN = 'A check the allowlist has no name for (a pipe, `jq`, `grep`, the product\'s own binary) is ' +
|
||||
'wrapped in a script under tests/ and declared as `bash tests/<script>.sh`.';
|
||||
|
||||
/**
|
||||
* Every check the run will execute can run. Without this a plan passed the
|
||||
* validator, the plan gate and three critic rounds with a `git grep` criterion
|
||||
* the runner refuses, and Phase 7 felled a run whose steps had all passed
|
||||
* (voyage-lekerepo trial, 2026-10-06).
|
||||
*/
|
||||
function checkRunnable(text, sections, opts) {
|
||||
const out = [];
|
||||
const criteria = parsePlanVerification(text);
|
||||
for (const c of criteria) {
|
||||
const r = runnableCheck(c, opts);
|
||||
if (!r.runnable) {
|
||||
out.push(issue('PLAN_VERIFICATION_NOT_RUNNABLE',
|
||||
`## Verification ${c.label} can never run (${r.reason}): ${c.text}`,
|
||||
`Phase 7 runs every criterion and fells the run on one it cannot run. ${WAY_IN}`));
|
||||
}
|
||||
}
|
||||
const verified = new Set(criteria.filter(c => c.command).map(c => c.command.trim().replace(/\s+/g, ' ')));
|
||||
for (const s of sections) {
|
||||
const f = parseStepFields(s.body);
|
||||
if (f.has('Verify')) {
|
||||
const v = firstCommand(f.get('Verify').text);
|
||||
const r = runnableCheck(v, opts);
|
||||
if (!r.runnable) {
|
||||
out.push(issue('PLAN_STEP_VERIFY_NOT_RUNNABLE',
|
||||
`Step ${s.n}: \`Verify:\` does not open with one runnable test command (${r.reason})`,
|
||||
'The first backticked span of `Verify:` is the step\'s test command: proof of red and the done line run it, ' +
|
||||
`and nothing after it is run. ${WAY_IN}`));
|
||||
}
|
||||
}
|
||||
if (f.has('End-to-end') && BACKTICKED.test(f.get('End-to-end').text)) {
|
||||
const e = firstCommand(f.get('End-to-end').text);
|
||||
const r = runnableCheck(e, opts);
|
||||
if (!r.runnable) {
|
||||
out.push(issue('PLAN_END_TO_END_NOT_RUNNABLE',
|
||||
`Step ${s.n}: \`End-to-end:\` command can never run (${r.reason})`,
|
||||
`Run the product from a script: \`bash tests/e2e.sh\`. ${WAY_IN}`));
|
||||
} else if (!verified.has(e.command.trim().replace(/\s+/g, ' '))) {
|
||||
out.push(issue('PLAN_END_TO_END_NOT_VERIFIED',
|
||||
`Step ${s.n}: \`End-to-end:\` command \`${e.command}\` does not stand under ## Verification`,
|
||||
'Phase 7 runs `## Verification`; list the same command there, or the result is "not verified end to end".'));
|
||||
}
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Table body rows (skips the header and the |---| separator). */
|
||||
function tableRows(text) {
|
||||
const rows = text.split('\n').filter(l => l.trim().startsWith('|'));
|
||||
|
|
@ -176,7 +232,7 @@ export function validatePlanContent(rawText, opts = {}) {
|
|||
|
||||
const ceremony = extractPlanCeremony(text);
|
||||
if (versionAtLeast(planVersion, '1.8')) {
|
||||
const v18 = [...checkSteps(sections)];
|
||||
const v18 = [...checkSteps(sections), ...checkRunnable(text, sections, { cwd: opts.cwd })];
|
||||
if (!ceremony.valid) {
|
||||
v18.push(issue('PLAN_CEREMONY_INVALID', `ceremony=${ceremony.raw}`,
|
||||
`Allowed: ${CEREMONY_VALUES.join(' | ')} (absent = architectural).`));
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue