fix(review): run the success-criteria commands and hand the reviewer the result (D-04)

The rubric required `brief-conformance-reviewer` to classify a Success
Criterion as Full only when "its verification command/test exists and passes".
Its tools are `Read`, `Glob`, `Grep`. It cannot run anything, so "passes" was
either guessed from the command's mere existence or quietly downgraded to
"exists" — a BLOCKER-tier rule key resting on an impression.

The reviewer stays read-only — a reviewer that executes the code it reviews is
not an independent reviewer. The command does the running instead:

- `/trekreview` Phase 4.5 runs the brief's `## Success Criteria` commands
  through `lib/verification/criteria-runner.mjs --brief --evidence` and captures
  the block as `sc_evidence_block`, pasted verbatim into the reviewer prompt in
  Phase 5. The exit code does not stop the review — a failing criterion is
  exactly what the review exists to find.
- `formatCriteriaEvidence` builds that block in code: one row per criterion with
  the command, the exit code and the first output line. Chose a code-built block
  over an orchestrator-written summary so the orchestrator cannot narrate a pass
  that never happened.
- The rubric now judges the supplied result: `PASS` supports Full, `FAILED` /
  `BLOCKED` is `Broken` with the exit code cited, and `NOT RUN` is the absence
  of a measurement — never evidence in either direction.
- Phase 4.5 is skipped in `quick` mode: that mode does not launch the
  conformance reviewer, so there is nobody to hand the result to.

Red first: seven tests in `tests/lib/criteria-runner.test.mjs` against a
committed brief fixture whose three criteria pass, fail, and are prose-only.
The two doc pins were verified red against the pre-fix files (rubric asked
"exists and passes"; no Phase 4.5; the block reached nobody).

Suite: 1117 (1115/0/2), up 9.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Kjell Tore Guttormsen 2026-09-18 01:34:18 +02:00
commit c23b009738
Signed by: ktg
SSH key fingerprint: SHA256:JakMjO6FTBBzN0Bhfj9saOoEjaFxlSdYuZQQpM/lF9Q
6 changed files with 274 additions and 8 deletions

View file

@ -24,7 +24,9 @@
//
// Usage:
// node lib/verification/criteria-runner.mjs --plan <plan.md> [--json] [--cwd D] [--timeout MS]
// node lib/verification/criteria-runner.mjs --brief <brief.md> [--json] [--cwd D] [--timeout MS]
// node lib/verification/criteria-runner.mjs --brief <brief.md> [--json|--evidence] [--cwd D] [--timeout MS]
// --evidence emits the markdown block /trekreview hands to the conformance
// reviewer; --json emits the whole report. They are two shapes of one run.
// Exit: 0 = ok, 1 = a criterion failed / was blocked / could not run, 2 = usage
// or read error (deliberately distinct from "a criterion failed").
@ -265,15 +267,69 @@ export function render(rep) {
return out.join('\n');
}
/**
* The evidence block /trekreview hands to `brief-conformance-reviewer`.
*
* The reviewer's tools are Read/Glob/Grep, so it cannot run a success
* criterion's verification command. Building this block in code (rather than
* letting the orchestrator narrate it) is what makes "passes" an exit code the
* reviewer READS instead of a judgement it cannot make.
*/
export function formatCriteriaEvidence(rep) {
const out = [];
out.push('### Success-criteria check results (run for you — you did not run these)');
out.push('');
out.push(`Source: \`${rep.source}\` ${rep.heading}`);
out.push('');
if (rep.error) {
out.push(`**${rep.error.code}** — ${rep.error.message}.`);
out.push('');
out.push('No criteria were run. Treat every criterion as NOT RUN.');
} else {
out.push('| Criterion | Command | Result | Exit | Evidence |');
out.push('|---|---|---|---|---|');
for (const r of rep.results) {
const cmd = r.command ? `\`${cell(r.command)}\`` : '(none)';
const exit = r.exitCode === null ? '—' : `exit ${r.exitCode}`;
const evidence = r.status === 'passed' ? '—' : cell(firstLine(r.output)) || '—';
out.push(`| ${r.label} | ${cmd} | ${MARK[r.status]} | ${exit} | ${evidence} |`);
}
const s = rep.summary;
out.push('');
out.push(
`${s.passed} passed · ${s.failed} failed · ${s.blocked} blocked · ${s.unrunnable} not run (of ${s.total}).`
);
}
out.push('');
out.push(
'**NOT RUN is not a pass.** You cannot run these commands yourself — never infer, assume or ' +
'reconstruct a result for a criterion that has none. A criterion whose result is FAILED or ' +
'BLOCKED is `Broken`; a criterion with no result is judged on delivered code alone, and you ' +
'say so in the Evidence column.'
);
return out.join('\n');
}
function firstLine(text) {
return String(text ?? '').split('\n').map((l) => l.trim()).find((l) => l !== '') ?? '';
}
// Markdown table cells: no pipes, no newlines, bounded length.
function cell(text) {
const t = String(text ?? '').replace(/\r?\n/g, ' ').replace(/\|/g, '\\|').trim();
return t.length <= 120 ? t : `${t.slice(0, 120)}…`;
}
// --- CLI --------------------------------------------------------------------
const USAGE =
'usage: criteria-runner.mjs (--plan <plan.md> | --brief <brief.md>) [--json] [--cwd <dir>] [--timeout <ms>]';
'usage: criteria-runner.mjs (--plan <plan.md> | --brief <brief.md>) [--json | --evidence] [--cwd <dir>] [--timeout <ms>]';
export function main(argv) {
let mode = null;
let path = null;
let json = false;
let evidence = false;
const opts = {};
for (let i = 0; i < argv.length; i++) {
const a = argv[i];
@ -282,11 +338,16 @@ export function main(argv) {
mode = a.slice(2);
path = argv[++i];
} else if (a === '--json') json = true;
else if (a === '--evidence') evidence = true;
else if (a === '--cwd' && argv[i + 1]) opts.cwd = resolve(argv[++i]);
else if (a === '--timeout' && argv[i + 1]) opts.timeoutMs = Number(argv[++i]);
else { process.stderr.write(`criteria-runner: unknown argument ${a}\n${USAGE}\n`); return 2; }
}
if (!mode) { process.stderr.write(`criteria-runner: no artifact given\n${USAGE}\n`); return 2; }
if (json && evidence) {
process.stderr.write(`criteria-runner: --json and --evidence are two output shapes; pick one\n${USAGE}\n`);
return 2;
}
if (opts.timeoutMs !== undefined && !Number.isFinite(opts.timeoutMs)) {
process.stderr.write(`criteria-runner: --timeout must be a number\n${USAGE}\n`);
return 2;
@ -299,7 +360,8 @@ export function main(argv) {
process.stderr.write(`criteria-runner: ${err.message}\n`);
return 2;
}
process.stdout.write((json ? JSON.stringify(rep, null, 2) : render(rep)) + '\n');
const text = json ? JSON.stringify(rep, null, 2) : evidence ? formatCriteriaEvidence(rep) : render(rep);
process.stdout.write(text + '\n');
return rep.summary.ok ? 0 : 1;
}