fix(verification): a section the runner cannot read SAYS so, and trekplan is pinned to the format it reads

Measured 2026-09-18: nine ordinary shapes of a plan's `## Verification`
section parsed to zero criteria - an untagged fence, a ```text fence, a
markdown table, `## Verification (acceptance)`, `## Verification:`,
`### Verification`, an unclosed fence earlier in the document. Every one came
out as `0 of 0`, NOT OK, exit 1, and Phase 7 then forbade `result: completed`
without anyone being told that the FORMAT, not the code, was the problem.
"The section is empty" and "I cannot read this format" are different facts.

Three changes, one hole:

- The runner reports `NO_CRITERIA` with a source line (`plan.md:NN`) when the
  section is there and nothing in it parsed, and names the two forms it does
  read. Same for a brief's `## Success Criteria`, so the evidence block the
  conformance reviewer gets says which of the two it is looking at rather than
  showing an empty table.
- Phase 7 says it out loud instead of failing silently: report the source line
  and the two forms, and say that the plan is what failed there, not the run.
- `/trekplan` now pins what it produces to what the runner reads: the heading
  is exactly `## Verification`, the criteria are a bullet whose first
  backticked span is the command or a shell-tagged fence, and the command must
  be one the allowlist runs. A doc-consistency test holds the writer and the
  reader together, so a runner that learns a new form must update the source.

Honest about the round trip: the two round-trip tests were GREEN on arrival -
the template already writes the bullet form the runner reads. What was missing
was not the format but the PIN: `/trekplan` mandated neither the heading string
nor the format, so a plan could satisfy the command's own instructions and
still parse to nothing. The tests now hold that.

Red first: 5 of the 7 new tests failed before the change (3 NO_CRITERIA, 2
doc-consistency); the 2 round-trip tests are guards, and said so above.
Suite 1161 (1159/0/2). Gate unchanged: defects 0 of 7, intact, exit 1.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Kjell Tore Guttormsen 2026-09-18 03:00:13 +02:00
commit f90e1cf02d
Signed by: ktg
SSH key fingerprint: SHA256:JakMjO6FTBBzN0Bhfj9saOoEjaFxlSdYuZQQpM/lF9Q
5 changed files with 195 additions and 9 deletions

View file

@ -83,8 +83,10 @@ function scanLines(markdown) {
return out;
}
// The lines from `heading` up to the next `## ` heading (exclusive). Headings
// inside a fence do not count on either end.
// The lines from `heading` up to the next `## ` heading (exclusive), plus the
// 1-based line the heading itself is on — a section the runner cannot read has
// to be reported with a place to look. Headings inside a fence do not count on
// either end.
function sectionLines(markdown, heading) {
const lines = scanLines(markdown);
const start = lines.findIndex((l) => !l.fenced && l.text.trim() === heading);
@ -93,7 +95,7 @@ function sectionLines(markdown, heading) {
for (let i = start + 1; i < lines.length; i++) {
if (!lines[i].fenced && lines[i].text.startsWith('## ')) { end = i; break; }
}
return lines.slice(start + 1, end);
return { line: start + 1, lines: lines.slice(start + 1, end) };
}
// `- [ ] text`, `- [x] text`, `- text`, `* text`, `1. text` -> text
@ -154,8 +156,9 @@ function fencedCommand(text) {
}
function parseSection(markdown, heading, prefix) {
const lines = sectionLines(markdown, heading);
if (lines === null) return [];
const section = sectionLines(markdown, heading);
if (section === null) return [];
const lines = section.lines;
const criteria = [];
const push = (text, command, reason) =>
criteria.push({ label: `${prefix}${criteria.length + 1}`, text, command, reason });
@ -408,6 +411,27 @@ function report(kind, source, criteria, opts, error) {
};
}
/**
* The section is THERE and nothing in it parsed as a check.
*
* "The section is empty" and "I cannot read this format" are different facts,
* and reporting both as `0 of 0` collapsed them into one bare exit 1. Measured
* 2026-09-18: nine ordinary shapes of a `## Verification` section — an untagged
* fence, a ```text fence, a table, a heading with a suffix, an unclosed fence
* earlier in the file — all came out that way, and the run was felled without
* anyone being told which format the runner had failed to read.
*/
function noCriteria(source, heading, line) {
return {
code: 'NO_CRITERIA',
message:
`${source}:${line} — the "${heading}" section declares nothing the runner can read. `
+ 'It reads two forms: a bullet whose first backticked span is the command '
+ '(`- [ ] `npm test` -> expected: exit 0`), and a shell-tagged fence (```bash) '
+ 'whose lines are the commands. An untagged fence, a table or prose parses to nothing.',
};
}
function readOrThrow(path) {
try {
return readFileSync(path, 'utf8');
@ -425,13 +449,18 @@ function readOrThrow(path) {
*/
export function runPlanVerification(planPath, opts = {}) {
const md = readOrThrow(planPath);
if (sectionLines(md, PLAN_HEADING) === null) {
const section = sectionLines(md, PLAN_HEADING);
if (section === null) {
return report('plan', planPath, [], opts, {
code: 'NO_VERIFICATION_SECTION',
message: `${planPath} has no "${PLAN_HEADING}" section — nothing to verify, so the run cannot be called verified`,
});
}
return report('plan', planPath, parsePlanVerification(md), opts);
const criteria = parsePlanVerification(md);
if (criteria.length === 0) {
return report('plan', planPath, [], opts, noCriteria(planPath, PLAN_HEADING, section.line));
}
return report('plan', planPath, criteria, opts);
}
/**
@ -440,13 +469,18 @@ export function runPlanVerification(planPath, opts = {}) {
*/
export function runSuccessCriteriaChecks(briefPath, opts = {}) {
const md = readOrThrow(briefPath);
if (sectionLines(md, BRIEF_HEADING) === null) {
const section = sectionLines(md, BRIEF_HEADING);
if (section === null) {
return report('brief', briefPath, [], opts, {
code: 'NO_SUCCESS_CRITERIA_SECTION',
message: `${briefPath} has no "${BRIEF_HEADING}" section`,
});
}
return report('brief', briefPath, parseSuccessCriteria(md), opts);
const criteria = parseSuccessCriteria(md);
if (criteria.length === 0) {
return report('brief', briefPath, [], opts, noCriteria(briefPath, BRIEF_HEADING, section.line));
}
return report('brief', briefPath, criteria, opts);
}
// --- rendering --------------------------------------------------------------