fix(verification): a backticked span is only run when it IS a command
"The first backtick span is the command" is right for a plan, whose template
puts the command first, and wrong for a brief, whose criterion usually opens
by NAMING the thing under discussion. Measured 2026-09-18 on the repo's own
example brief: 5 of 6 criteria FAILED, 3 of them parse artifacts - `--verbose`
run as a command gave exit 2 ("invalid option"), `tests/` gave exit 126 ("is a
directory"). The rubric reads a FAILED result as decisive, so each one became
a BROKEN_SUCCESS_CRITERION BLOCKER about prose.
looksLikeCommand() screens the span by SHAPE only - no filesystem lookup, so a
span parses the same everywhere. Refused: a leading flag, a directory, a token
carrying quotes/braces/prose, and a lone relative path with a slash (an
explicit ./, ../, / or ~/ still runs, as do env-var prefixes). A refused span
is `unrunnable` with reason `not-a-command` - its own outcome, never FAILED,
and it never reaches a shell.
It deliberately does NOT scan on to a later span. "The first span that LOOKS
like a command" invents commands out of prose: in that same example brief it
would have run `whoami` and `login`, two real binaries a sentence happens to
name. An absent measurement is honest; a guessed one is not.
The shape check applies to prose spans only. Inside a shell-tagged fence the
author has already declared shell, so `[ -f x ] || exit 1` still runs.
The rubric follows: a NOT RUN result is never on its own a finding. The
Partial row now describes half-built DELIVERED CODE, and the reviewer gets a
table of the three reason strings - no-command, placeholder, not-a-command -
with what each says about the sentence rather than about the code.
Not covered, stated for the record: a multi-token span whose first token is a
non-executable file (`tests/golden/login.stdout --check`) still runs, and a
criterion whose command is real but whose binary is absent still reports the
shell's exit 127 - that is a true measurement of a missing binary, not a
parse artifact.
Red first: 4 runner tests + 1 doc-consistency pin failed before this change.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
parent
0013d292d6
commit
106dcb0091
4 changed files with 152 additions and 1 deletions
|
|
@ -433,3 +433,86 @@ test('a report that merely quotes a plan outline has no ## Verification section'
|
|||
assert.equal(report.error?.code, 'NO_VERIFICATION_SECTION');
|
||||
assert.equal(report.summary.ok, false);
|
||||
});
|
||||
|
||||
// --- a span is not a command just because it is in backticks ---------------
|
||||
//
|
||||
// "The first backtick span is the command" is right for a plan (the template
|
||||
// puts it first) and wrong for a brief, whose sentence usually OPENS with the
|
||||
// thing under discussion: a flag, a path. Running those produced `/bin/sh: --:
|
||||
// invalid option` (exit 2) and "is a directory" (exit 126), and the review
|
||||
// rubric reads a FAILED result as decisive -> a BLOCKER invented out of prose.
|
||||
// Measured 2026-09-18: 3 of the 6 criteria in the repo's own example brief.
|
||||
|
||||
const EXAMPLE_BRIEF = join(ROOT, 'examples', '01-add-verbose-flag', 'brief.md');
|
||||
|
||||
test('parseSuccessCriteria: a leading flag is not a command — the criterion is NOT RUN', () => {
|
||||
const md = '## Success Criteria\n\n- `--verbose` works in any position: `app --verbose run`\n';
|
||||
const [sc] = parseSuccessCriteria(md);
|
||||
assert.equal(sc.command, null);
|
||||
assert.equal(sc.reason, 'not-a-command');
|
||||
});
|
||||
|
||||
test('parseSuccessCriteria: a bare directory or file path is not a command', () => {
|
||||
const md = [
|
||||
'## Success Criteria',
|
||||
'',
|
||||
'- Existing tests in `tests/` continue to pass',
|
||||
'- The golden file `tests/golden/login.stdout` is unchanged',
|
||||
'',
|
||||
].join('\n');
|
||||
for (const sc of parseSuccessCriteria(md)) {
|
||||
assert.equal(sc.command, null, `${sc.label} must not be run as a command`);
|
||||
assert.equal(sc.reason, 'not-a-command');
|
||||
}
|
||||
});
|
||||
|
||||
test('parseSuccessCriteria: a real command, an explicit path and env prefixes still run', () => {
|
||||
const md = [
|
||||
'## Success Criteria',
|
||||
'',
|
||||
'- All tests pass: `npm test`',
|
||||
'- The build script runs: `./scripts/build.sh --ci`',
|
||||
'- Absolute paths run: `/usr/bin/true`',
|
||||
'- Env prefixes are not the command: `CI=1 npm test`',
|
||||
'',
|
||||
].join('\n');
|
||||
assert.deepEqual(
|
||||
parseSuccessCriteria(md).map((c) => c.command),
|
||||
['npm test', './scripts/build.sh --ci', '/usr/bin/true', 'CI=1 npm test'],
|
||||
);
|
||||
});
|
||||
|
||||
test("the repo's own example brief yields ZERO parse artifacts", () => {
|
||||
const criteria = parseSuccessCriteria(readFileSync(EXAMPLE_BRIEF, 'utf8'));
|
||||
assert.equal(criteria.length, 6);
|
||||
for (const c of criteria) {
|
||||
if (c.command === null) continue;
|
||||
assert.ok(!c.command.startsWith('-'), `${c.label} would run a flag: ${c.command}`);
|
||||
assert.ok(!/^\S+\/(\s|$)/.test(c.command), `${c.label} would run a path: ${c.command}`);
|
||||
}
|
||||
// SC3/SC4 open on a flag, SC6 on a directory: three criteria that used to be
|
||||
// FAILED with a parse exit code and are now an absent measurement.
|
||||
for (const label of ['SC3', 'SC4', 'SC6']) {
|
||||
const c = criteria.find((x) => x.label === label);
|
||||
assert.equal(c.command, null, `${label}: ${c.command}`);
|
||||
assert.equal(c.reason, 'not-a-command');
|
||||
}
|
||||
});
|
||||
|
||||
test('runCriteria: a not-a-command criterion is unrunnable and says why — never failed', () => {
|
||||
const criteria = parseSuccessCriteria('## Success Criteria\n\n- `--verbose` is accepted everywhere\n');
|
||||
const exec = execDouble({});
|
||||
const [r] = runCriteria(criteria, { exec, screen: allowAll });
|
||||
assert.equal(r.status, 'unrunnable');
|
||||
assert.notEqual(r.status, 'failed');
|
||||
assert.equal(r.exitCode, null);
|
||||
assert.match(r.output, /not-a-command/);
|
||||
assert.deepEqual(exec.calls, [], 'a span that is not a command must never reach a shell');
|
||||
});
|
||||
|
||||
test('a shell-tagged fence declares commands even when a line is not command-shaped', () => {
|
||||
const md = '## Verification\n\n```bash\n[ -f README.md ] || exit 1\n```\n';
|
||||
const [v] = parsePlanVerification(md);
|
||||
assert.equal(v.command, '[ -f README.md ] || exit 1',
|
||||
'inside an explicitly shell-tagged fence the author HAS declared shell; the shape check belongs to prose spans only');
|
||||
});
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue