fix(verification): the criteria runner reads the repo's own plan format
The runner read bullet lines only. The repo's own example plan writes its
whole acceptance run as a fenced bash block, so `## Verification` parsed to
ZERO criteria, the runner exited 1, and Phase 7 forbade `result: completed`
- a correct plan felled every single-session run. Measured 2026-09-18 on
6cafb4c: none of the repo's plan artifacts exited 0.
Fences are now read twice over, for two opposite reasons:
- a `## ` heading INSIDE a fence is quoted text and no longer opens a
section. examples/02-real-cli/REGENERATED.md is a report that quotes a
plan outline inside one fence; it used to yield an empty section that
read as "0 of 0", and now yields the honest NO_VERIFICATION_SECTION.
- a shell-tagged fence inside the section holds the commands. The tag list
is closed (bash/sh/shell/zsh/console/shell-session): an untagged fence is
more often expected OUTPUT than input, and inventing a criterion from
output is the failure this file exists to prevent.
Blank and comment-only lines inside the block declare nothing. A `$`/`>`
console prompt is stripped; a `#` root prompt is NOT, because it cannot be
told from a comment and running a comment is the worse mistake.
Measured after the fix (parse only - one example names a fictional CLI):
examples/01 6 criteria, plan-template 2 (both placeholders, correctly NOT
RUN), the two runner fixtures 2 each, plan-run-C 1, REGENERATED.md 0 with
NO_VERIFICATION_SECTION.
Divergence from the order's premise, stated for the record: it said 2 of 3
example plans write `## Verification` as a fenced bash block. Ground truth
is 1 of 3 - REGENERATED.md has no section of its own at all - and the
example plan's block holds 6 command lines, not 5.
Red first: 4 of the 6 new tests failed before this change.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
parent
6cafb4c04c
commit
0013d292d6
2 changed files with 145 additions and 7 deletions
|
|
@ -12,7 +12,7 @@ import { strict as assert } from 'node:assert';
|
|||
import { spawnSync } from 'node:child_process';
|
||||
import { join, dirname } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { mkdtempSync, writeFileSync } from 'node:fs';
|
||||
import { mkdtempSync, writeFileSync, readFileSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
|
||||
import {
|
||||
|
|
@ -350,3 +350,86 @@ test('CLI: --evidence and --json together exit 2 — one output shape at a time'
|
|||
assert.equal(r.status, 2);
|
||||
assert.match(r.stderr, /usage/);
|
||||
});
|
||||
|
||||
// --- the repo's OWN plan format --------------------------------------------
|
||||
//
|
||||
// D-03's runner read bullet lines only. The repo's own example plan writes its
|
||||
// `## Verification` as a fenced bash block, so it parsed to ZERO criteria, the
|
||||
// runner exited 1, and Phase 7 forbade `result: completed` — a correct plan
|
||||
// felled every single-session run. Measured 2026-09-18 (PM checkpoint on
|
||||
// 6cafb4c): 0 of the repo's plan artifacts exited 0.
|
||||
|
||||
const SHELL_BLOCK_PLAN = [
|
||||
'# Plan',
|
||||
'',
|
||||
'## Verification',
|
||||
'',
|
||||
'Final acceptance run after step 3:',
|
||||
'',
|
||||
'```bash',
|
||||
'npm test # all green',
|
||||
'# a comment-only line is not a criterion',
|
||||
'',
|
||||
'node --test tests/lib/x.test.mjs',
|
||||
'```',
|
||||
'',
|
||||
'## Estimated Scope',
|
||||
'',
|
||||
'```bash',
|
||||
'not-a-criterion --outside-the-section',
|
||||
'```',
|
||||
].join('\n');
|
||||
|
||||
test('parsePlanVerification: a fenced shell block yields one criterion per command line', () => {
|
||||
const criteria = parsePlanVerification(SHELL_BLOCK_PLAN);
|
||||
assert.deepEqual(criteria.map((c) => c.label), ['V1', 'V2']);
|
||||
assert.equal(criteria[0].command, 'npm test # all green');
|
||||
assert.equal(criteria[1].command, 'node --test tests/lib/x.test.mjs');
|
||||
});
|
||||
|
||||
test('parsePlanVerification: a fenced block and a bullet list declaring the same commands parse identically', () => {
|
||||
const fenced = '## Verification\n\n```bash\nnpm test\nnode --test tests/lib/x.test.mjs\n```\n';
|
||||
const bulleted = '## Verification\n\n- [ ] `npm test`\n- [ ] `node --test tests/lib/x.test.mjs`\n';
|
||||
assert.deepEqual(
|
||||
parsePlanVerification(fenced).map((c) => [c.label, c.command]),
|
||||
parsePlanVerification(bulleted).map((c) => [c.label, c.command]),
|
||||
);
|
||||
});
|
||||
|
||||
test('parsePlanVerification: only a shell-tagged fence is read as commands', () => {
|
||||
const md = '## Verification\n\n```json\n{"expected": "output"}\n```\n\n- [ ] `npm test`\n';
|
||||
const criteria = parsePlanVerification(md);
|
||||
assert.deepEqual(criteria.map((c) => c.command), ['npm test'],
|
||||
'an untagged or non-shell fence holds expected output, not commands — never invent a criterion from it');
|
||||
});
|
||||
|
||||
test('parsePlanVerification: a `## ` heading inside a fence is quoted text, not a section', () => {
|
||||
const md = '# Report\n\n```\n# Plan\n## Verification\n## Plan-critic notes\n```\n\nNo real section here.\n';
|
||||
assert.deepEqual(parsePlanVerification(md), []);
|
||||
});
|
||||
|
||||
// Every plan artifact the repo ships must survive its own runner. This test
|
||||
// parses; it never runs — one example plan names a fictional CLI on purpose.
|
||||
const PLAN_ARTIFACTS = [
|
||||
'examples/01-add-verbose-flag/plan.md',
|
||||
'templates/plan-template.md',
|
||||
'tests/fixtures/plan-verification-fails.md',
|
||||
'tests/fixtures/plan-verification-passes.md',
|
||||
'tests/synthetic/plan-run-C.md',
|
||||
];
|
||||
|
||||
test("the repo's own plan artifacts each declare at least one criterion", () => {
|
||||
for (const rel of PLAN_ARTIFACTS) {
|
||||
const criteria = parsePlanVerification(readFileSync(join(ROOT, rel), 'utf8'));
|
||||
assert.ok(criteria.length >= 1, `${rel} parsed to ${criteria.length} criteria`);
|
||||
}
|
||||
});
|
||||
|
||||
// examples/02-real-cli/REGENERATED.md is a REPORT that quotes a plan outline
|
||||
// inside a fence; it has no `## Verification` of its own. The honest answer is
|
||||
// NO_VERIFICATION_SECTION — not an empty section that reads as "0 of 0".
|
||||
test('a report that merely quotes a plan outline has no ## Verification section', () => {
|
||||
const report = runPlanVerification(join(ROOT, 'examples/02-real-cli/REGENERATED.md'));
|
||||
assert.equal(report.error?.code, 'NO_VERIFICATION_SECTION');
|
||||
assert.equal(report.summary.ok, false);
|
||||
});
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue