fix(verification): the criteria runner reads the repo's own plan format
The runner read bullet lines only. The repo's own example plan writes its
whole acceptance run as a fenced bash block, so `## Verification` parsed to
ZERO criteria, the runner exited 1, and Phase 7 forbade `result: completed`
- a correct plan felled every single-session run. Measured 2026-09-18 on
6cafb4c: none of the repo's plan artifacts exited 0.
Fences are now read twice over, for two opposite reasons:
- a `## ` heading INSIDE a fence is quoted text and no longer opens a
section. examples/02-real-cli/REGENERATED.md is a report that quotes a
plan outline inside one fence; it used to yield an empty section that
read as "0 of 0", and now yields the honest NO_VERIFICATION_SECTION.
- a shell-tagged fence inside the section holds the commands. The tag list
is closed (bash/sh/shell/zsh/console/shell-session): an untagged fence is
more often expected OUTPUT than input, and inventing a criterion from
output is the failure this file exists to prevent.
Blank and comment-only lines inside the block declare nothing. A `$`/`>`
console prompt is stripped; a `#` root prompt is NOT, because it cannot be
told from a comment and running a comment is the worse mistake.
Measured after the fix (parse only - one example names a fictional CLI):
examples/01 6 criteria, plan-template 2 (both placeholders, correctly NOT
RUN), the two runner fixtures 2 each, plan-run-C 1, REGENERATED.md 0 with
NO_VERIFICATION_SECTION.
Divergence from the order's premise, stated for the record: it said 2 of 3
example plans write `## Verification` as a fenced bash block. Ground truth
is 1 of 3 - REGENERATED.md has no section of its own at all - and the
example plan's block holds 6 command lines, not 5.
Red first: 4 of the 6 new tests failed before this change.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
parent
6cafb4c04c
commit
0013d292d6
2 changed files with 145 additions and 7 deletions
|
|
@ -45,14 +45,49 @@ export const DEFAULT_MAX_OUTPUT = 4000;
|
|||
|
||||
// --- parsing ----------------------------------------------------------------
|
||||
|
||||
// The lines from `heading` up to the next `## ` heading (exclusive).
|
||||
// Fenced blocks are read TWICE over, for two opposite reasons: a `## ` heading
|
||||
// inside a fence is quoted text and must not open a section (the repo's own
|
||||
// examples/02-real-cli/REGENERATED.md quotes a whole plan outline inside one),
|
||||
// while a shell-tagged fence INSIDE the section holds the commands themselves
|
||||
// (examples/01-add-verbose-flag/plan.md writes its whole acceptance run that
|
||||
// way). Reading only bullets made a correct plan parse to zero criteria, exit
|
||||
// 1, and fell every single-session run — measured 2026-09-18.
|
||||
const FENCE = /^\s*(?:```|~~~)(.*)$/;
|
||||
|
||||
// Languages whose fenced block is a run of commands. Deliberately a closed
|
||||
// list: an untagged fence is far more often expected OUTPUT than input, and
|
||||
// inventing a criterion from output is exactly the failure this file exists to
|
||||
// prevent.
|
||||
const SHELL_LANGS = new Set(['bash', 'sh', 'shell', 'zsh', 'console', 'shell-session']);
|
||||
|
||||
// One entry per line: its text, whether it sits inside a fence, that fence's
|
||||
// info string, and whether it IS the fence marker.
|
||||
function scanLines(markdown) {
|
||||
const out = [];
|
||||
let lang = null;
|
||||
for (const text of markdown.split('\n')) {
|
||||
const m = text.match(FENCE);
|
||||
if (m) {
|
||||
const open = lang === null;
|
||||
if (open) lang = m[1].trim().toLowerCase().split(/\s+/)[0];
|
||||
out.push({ text, fenced: true, lang: lang ?? '', marker: true });
|
||||
if (!open) lang = null;
|
||||
continue;
|
||||
}
|
||||
out.push({ text, fenced: lang !== null, lang: lang ?? '', marker: false });
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
// The lines from `heading` up to the next `## ` heading (exclusive). Headings
|
||||
// inside a fence do not count on either end.
|
||||
function sectionLines(markdown, heading) {
|
||||
const lines = markdown.split('\n');
|
||||
const start = lines.findIndex((l) => l.trim() === heading);
|
||||
const lines = scanLines(markdown);
|
||||
const start = lines.findIndex((l) => !l.fenced && l.text.trim() === heading);
|
||||
if (start === -1) return null;
|
||||
let end = lines.length;
|
||||
for (let i = start + 1; i < lines.length; i++) {
|
||||
if (lines[i].startsWith('## ')) { end = i; break; }
|
||||
if (!lines[i].fenced && lines[i].text.startsWith('## ')) { end = i; break; }
|
||||
}
|
||||
return lines.slice(start + 1, end);
|
||||
}
|
||||
|
|
@ -71,17 +106,37 @@ function firstCommand(text) {
|
|||
return { command: raw, reason: '' };
|
||||
}
|
||||
|
||||
// A command line inside a shell-tagged fence. A console block may prefix the
|
||||
// line with a `$` or `>` prompt; blank and comment-only lines declare nothing.
|
||||
// A `#` root prompt is NOT stripped: it is indistinguishable from a comment,
|
||||
// and running a comment is the worse of the two mistakes.
|
||||
function fencedCommand(text) {
|
||||
const line = text.trim();
|
||||
if (line === '' || line.startsWith('#')) return null;
|
||||
return line.replace(/^[$>]\s+/, '');
|
||||
}
|
||||
|
||||
function parseSection(markdown, heading, prefix) {
|
||||
const lines = sectionLines(markdown, heading);
|
||||
if (lines === null) return [];
|
||||
const criteria = [];
|
||||
const push = (text, command, reason) =>
|
||||
criteria.push({ label: `${prefix}${criteria.length + 1}`, text, command, reason });
|
||||
for (const line of lines) {
|
||||
const bullet = line.match(BULLET);
|
||||
if (line.marker) continue;
|
||||
if (line.fenced) {
|
||||
if (!SHELL_LANGS.has(line.lang)) continue;
|
||||
const command = fencedCommand(line.text);
|
||||
if (command === null) continue;
|
||||
push(command, command, '');
|
||||
continue;
|
||||
}
|
||||
const bullet = line.text.match(BULLET);
|
||||
if (!bullet) continue;
|
||||
const text = bullet[1].trim();
|
||||
if (text === '') continue;
|
||||
const { command, reason } = firstCommand(text);
|
||||
criteria.push({ label: `${prefix}${criteria.length + 1}`, text, command, reason });
|
||||
push(text, command, reason);
|
||||
}
|
||||
return criteria;
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue