fix(verification): the criteria runner reads the repo's own plan format

The runner read bullet lines only. The repo's own example plan writes its
whole acceptance run as a fenced bash block, so `## Verification` parsed to
ZERO criteria, the runner exited 1, and Phase 7 forbade `result: completed`
- a correct plan felled every single-session run. Measured 2026-09-18 on
6cafb4c: none of the repo's plan artifacts exited 0.

Fences are now read twice over, for two opposite reasons:

- a `## ` heading INSIDE a fence is quoted text and no longer opens a
  section. examples/02-real-cli/REGENERATED.md is a report that quotes a
  plan outline inside one fence; it used to yield an empty section that
  read as "0 of 0", and now yields the honest NO_VERIFICATION_SECTION.
- a shell-tagged fence inside the section holds the commands. The tag list
  is closed (bash/sh/shell/zsh/console/shell-session): an untagged fence is
  more often expected OUTPUT than input, and inventing a criterion from
  output is the failure this file exists to prevent.

Blank and comment-only lines inside the block declare nothing. A `$`/`>`
console prompt is stripped; a `#` root prompt is NOT, because it cannot be
told from a comment and running a comment is the worse mistake.

Measured after the fix (parse only - one example names a fictional CLI):
examples/01 6 criteria, plan-template 2 (both placeholders, correctly NOT
RUN), the two runner fixtures 2 each, plan-run-C 1, REGENERATED.md 0 with
NO_VERIFICATION_SECTION.

Divergence from the order's premise, stated for the record: it said 2 of 3
example plans write `## Verification` as a fenced bash block. Ground truth
is 1 of 3 - REGENERATED.md has no section of its own at all - and the
example plan's block holds 6 command lines, not 5.

Red first: 4 of the 6 new tests failed before this change.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Kjell Tore Guttormsen 2026-09-18 02:02:14 +02:00
commit 0013d292d6
Signed by: ktg
SSH key fingerprint: SHA256:JakMjO6FTBBzN0Bhfj9saOoEjaFxlSdYuZQQpM/lF9Q
2 changed files with 145 additions and 7 deletions

View file

@ -45,14 +45,49 @@ export const DEFAULT_MAX_OUTPUT = 4000;
// --- parsing ---------------------------------------------------------------- // --- parsing ----------------------------------------------------------------
// The lines from `heading` up to the next `## ` heading (exclusive). // Fenced blocks are read TWICE over, for two opposite reasons: a `## ` heading
// inside a fence is quoted text and must not open a section (the repo's own
// examples/02-real-cli/REGENERATED.md quotes a whole plan outline inside one),
// while a shell-tagged fence INSIDE the section holds the commands themselves
// (examples/01-add-verbose-flag/plan.md writes its whole acceptance run that
// way). Reading only bullets made a correct plan parse to zero criteria, exit
// 1, and fell every single-session run — measured 2026-09-18.
const FENCE = /^\s*(?:```|~~~)(.*)$/;
// Languages whose fenced block is a run of commands. Deliberately a closed
// list: an untagged fence is far more often expected OUTPUT than input, and
// inventing a criterion from output is exactly the failure this file exists to
// prevent.
const SHELL_LANGS = new Set(['bash', 'sh', 'shell', 'zsh', 'console', 'shell-session']);
// One entry per line: its text, whether it sits inside a fence, that fence's
// info string, and whether it IS the fence marker.
function scanLines(markdown) {
const out = [];
let lang = null;
for (const text of markdown.split('\n')) {
const m = text.match(FENCE);
if (m) {
const open = lang === null;
if (open) lang = m[1].trim().toLowerCase().split(/\s+/)[0];
out.push({ text, fenced: true, lang: lang ?? '', marker: true });
if (!open) lang = null;
continue;
}
out.push({ text, fenced: lang !== null, lang: lang ?? '', marker: false });
}
return out;
}
// The lines from `heading` up to the next `## ` heading (exclusive). Headings
// inside a fence do not count on either end.
function sectionLines(markdown, heading) { function sectionLines(markdown, heading) {
const lines = markdown.split('\n'); const lines = scanLines(markdown);
const start = lines.findIndex((l) => l.trim() === heading); const start = lines.findIndex((l) => !l.fenced && l.text.trim() === heading);
if (start === -1) return null; if (start === -1) return null;
let end = lines.length; let end = lines.length;
for (let i = start + 1; i < lines.length; i++) { for (let i = start + 1; i < lines.length; i++) {
if (lines[i].startsWith('## ')) { end = i; break; } if (!lines[i].fenced && lines[i].text.startsWith('## ')) { end = i; break; }
} }
return lines.slice(start + 1, end); return lines.slice(start + 1, end);
} }
@ -71,17 +106,37 @@ function firstCommand(text) {
return { command: raw, reason: '' }; return { command: raw, reason: '' };
} }
// A command line inside a shell-tagged fence. A console block may prefix the
// line with a `$` or `>` prompt; blank and comment-only lines declare nothing.
// A `#` root prompt is NOT stripped: it is indistinguishable from a comment,
// and running a comment is the worse of the two mistakes.
function fencedCommand(text) {
const line = text.trim();
if (line === '' || line.startsWith('#')) return null;
return line.replace(/^[$>]\s+/, '');
}
function parseSection(markdown, heading, prefix) { function parseSection(markdown, heading, prefix) {
const lines = sectionLines(markdown, heading); const lines = sectionLines(markdown, heading);
if (lines === null) return []; if (lines === null) return [];
const criteria = []; const criteria = [];
const push = (text, command, reason) =>
criteria.push({ label: `${prefix}${criteria.length + 1}`, text, command, reason });
for (const line of lines) { for (const line of lines) {
const bullet = line.match(BULLET); if (line.marker) continue;
if (line.fenced) {
if (!SHELL_LANGS.has(line.lang)) continue;
const command = fencedCommand(line.text);
if (command === null) continue;
push(command, command, '');
continue;
}
const bullet = line.text.match(BULLET);
if (!bullet) continue; if (!bullet) continue;
const text = bullet[1].trim(); const text = bullet[1].trim();
if (text === '') continue; if (text === '') continue;
const { command, reason } = firstCommand(text); const { command, reason } = firstCommand(text);
criteria.push({ label: `${prefix}${criteria.length + 1}`, text, command, reason }); push(text, command, reason);
} }
return criteria; return criteria;
} }

View file

@ -12,7 +12,7 @@ import { strict as assert } from 'node:assert';
import { spawnSync } from 'node:child_process'; import { spawnSync } from 'node:child_process';
import { join, dirname } from 'node:path'; import { join, dirname } from 'node:path';
import { fileURLToPath } from 'node:url'; import { fileURLToPath } from 'node:url';
import { mkdtempSync, writeFileSync } from 'node:fs'; import { mkdtempSync, writeFileSync, readFileSync } from 'node:fs';
import { tmpdir } from 'node:os'; import { tmpdir } from 'node:os';
import { import {
@ -350,3 +350,86 @@ test('CLI: --evidence and --json together exit 2 — one output shape at a time'
assert.equal(r.status, 2); assert.equal(r.status, 2);
assert.match(r.stderr, /usage/); assert.match(r.stderr, /usage/);
}); });
// --- the repo's OWN plan format --------------------------------------------
//
// D-03's runner read bullet lines only. The repo's own example plan writes its
// `## Verification` as a fenced bash block, so it parsed to ZERO criteria, the
// runner exited 1, and Phase 7 forbade `result: completed` — a correct plan
// felled every single-session run. Measured 2026-09-18 (PM checkpoint on
// 6cafb4c): 0 of the repo's plan artifacts exited 0.
const SHELL_BLOCK_PLAN = [
'# Plan',
'',
'## Verification',
'',
'Final acceptance run after step 3:',
'',
'```bash',
'npm test # all green',
'# a comment-only line is not a criterion',
'',
'node --test tests/lib/x.test.mjs',
'```',
'',
'## Estimated Scope',
'',
'```bash',
'not-a-criterion --outside-the-section',
'```',
].join('\n');
test('parsePlanVerification: a fenced shell block yields one criterion per command line', () => {
const criteria = parsePlanVerification(SHELL_BLOCK_PLAN);
assert.deepEqual(criteria.map((c) => c.label), ['V1', 'V2']);
assert.equal(criteria[0].command, 'npm test # all green');
assert.equal(criteria[1].command, 'node --test tests/lib/x.test.mjs');
});
test('parsePlanVerification: a fenced block and a bullet list declaring the same commands parse identically', () => {
const fenced = '## Verification\n\n```bash\nnpm test\nnode --test tests/lib/x.test.mjs\n```\n';
const bulleted = '## Verification\n\n- [ ] `npm test`\n- [ ] `node --test tests/lib/x.test.mjs`\n';
assert.deepEqual(
parsePlanVerification(fenced).map((c) => [c.label, c.command]),
parsePlanVerification(bulleted).map((c) => [c.label, c.command]),
);
});
test('parsePlanVerification: only a shell-tagged fence is read as commands', () => {
const md = '## Verification\n\n```json\n{"expected": "output"}\n```\n\n- [ ] `npm test`\n';
const criteria = parsePlanVerification(md);
assert.deepEqual(criteria.map((c) => c.command), ['npm test'],
'an untagged or non-shell fence holds expected output, not commands — never invent a criterion from it');
});
test('parsePlanVerification: a `## ` heading inside a fence is quoted text, not a section', () => {
const md = '# Report\n\n```\n# Plan\n## Verification\n## Plan-critic notes\n```\n\nNo real section here.\n';
assert.deepEqual(parsePlanVerification(md), []);
});
// Every plan artifact the repo ships must survive its own runner. This test
// parses; it never runs — one example plan names a fictional CLI on purpose.
const PLAN_ARTIFACTS = [
'examples/01-add-verbose-flag/plan.md',
'templates/plan-template.md',
'tests/fixtures/plan-verification-fails.md',
'tests/fixtures/plan-verification-passes.md',
'tests/synthetic/plan-run-C.md',
];
test("the repo's own plan artifacts each declare at least one criterion", () => {
for (const rel of PLAN_ARTIFACTS) {
const criteria = parsePlanVerification(readFileSync(join(ROOT, rel), 'utf8'));
assert.ok(criteria.length >= 1, `${rel} parsed to ${criteria.length} criteria`);
}
});
// examples/02-real-cli/REGENERATED.md is a REPORT that quotes a plan outline
// inside a fence; it has no `## Verification` of its own. The honest answer is
// NO_VERIFICATION_SECTION — not an empty section that reads as "0 of 0".
test('a report that merely quotes a plan outline has no ## Verification section', () => {
const report = runPlanVerification(join(ROOT, 'examples/02-real-cli/REGENERATED.md'));
assert.equal(report.error?.code, 'NO_VERIFICATION_SECTION');
assert.equal(report.summary.ok, false);
});