fix(review): run the success-criteria commands and hand the reviewer the result (D-04)
The rubric required `brief-conformance-reviewer` to classify a Success Criterion as Full only when "its verification command/test exists and passes". Its tools are `Read`, `Glob`, `Grep`. It cannot run anything, so "passes" was either guessed from the command's mere existence or quietly downgraded to "exists" — a BLOCKER-tier rule key resting on an impression. The reviewer stays read-only — a reviewer that executes the code it reviews is not an independent reviewer. The command does the running instead: - `/trekreview` Phase 4.5 runs the brief's `## Success Criteria` commands through `lib/verification/criteria-runner.mjs --brief --evidence` and captures the block as `sc_evidence_block`, pasted verbatim into the reviewer prompt in Phase 5. The exit code does not stop the review — a failing criterion is exactly what the review exists to find. - `formatCriteriaEvidence` builds that block in code: one row per criterion with the command, the exit code and the first output line. Chose a code-built block over an orchestrator-written summary so the orchestrator cannot narrate a pass that never happened. - The rubric now judges the supplied result: `PASS` supports Full, `FAILED` / `BLOCKED` is `Broken` with the exit code cited, and `NOT RUN` is the absence of a measurement — never evidence in either direction. - Phase 4.5 is skipped in `quick` mode: that mode does not launch the conformance reviewer, so there is nobody to hand the result to. Red first: seven tests in `tests/lib/criteria-runner.test.mjs` against a committed brief fixture whose three criteria pass, fail, and are prose-only. The two doc pins were verified red against the pre-fix files (rubric asked "exists and passes"; no Phase 4.5; the block reached nobody). Suite: 1117 (1115/0/2), up 9. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
parent
e55ca9fc89
commit
c23b009738
6 changed files with 274 additions and 8 deletions
|
|
@ -24,7 +24,9 @@
|
|||
//
|
||||
// Usage:
|
||||
// node lib/verification/criteria-runner.mjs --plan <plan.md> [--json] [--cwd D] [--timeout MS]
|
||||
// node lib/verification/criteria-runner.mjs --brief <brief.md> [--json] [--cwd D] [--timeout MS]
|
||||
// node lib/verification/criteria-runner.mjs --brief <brief.md> [--json|--evidence] [--cwd D] [--timeout MS]
|
||||
// --evidence emits the markdown block /trekreview hands to the conformance
|
||||
// reviewer; --json emits the whole report. They are two shapes of one run.
|
||||
// Exit: 0 = ok, 1 = a criterion failed / was blocked / could not run, 2 = usage
|
||||
// or read error (deliberately distinct from "a criterion failed").
|
||||
|
||||
|
|
@ -265,15 +267,69 @@ export function render(rep) {
|
|||
return out.join('\n');
|
||||
}
|
||||
|
||||
/**
|
||||
* The evidence block /trekreview hands to `brief-conformance-reviewer`.
|
||||
*
|
||||
* The reviewer's tools are Read/Glob/Grep, so it cannot run a success
|
||||
* criterion's verification command. Building this block in code (rather than
|
||||
* letting the orchestrator narrate it) is what makes "passes" an exit code the
|
||||
* reviewer READS instead of a judgement it cannot make.
|
||||
*/
|
||||
export function formatCriteriaEvidence(rep) {
|
||||
const out = [];
|
||||
out.push('### Success-criteria check results (run for you — you did not run these)');
|
||||
out.push('');
|
||||
out.push(`Source: \`${rep.source}\` ${rep.heading}`);
|
||||
out.push('');
|
||||
if (rep.error) {
|
||||
out.push(`**${rep.error.code}** — ${rep.error.message}.`);
|
||||
out.push('');
|
||||
out.push('No criteria were run. Treat every criterion as NOT RUN.');
|
||||
} else {
|
||||
out.push('| Criterion | Command | Result | Exit | Evidence |');
|
||||
out.push('|---|---|---|---|---|');
|
||||
for (const r of rep.results) {
|
||||
const cmd = r.command ? `\`${cell(r.command)}\`` : '(none)';
|
||||
const exit = r.exitCode === null ? '—' : `exit ${r.exitCode}`;
|
||||
const evidence = r.status === 'passed' ? '—' : cell(firstLine(r.output)) || '—';
|
||||
out.push(`| ${r.label} | ${cmd} | ${MARK[r.status]} | ${exit} | ${evidence} |`);
|
||||
}
|
||||
const s = rep.summary;
|
||||
out.push('');
|
||||
out.push(
|
||||
`${s.passed} passed · ${s.failed} failed · ${s.blocked} blocked · ${s.unrunnable} not run (of ${s.total}).`
|
||||
);
|
||||
}
|
||||
out.push('');
|
||||
out.push(
|
||||
'**NOT RUN is not a pass.** You cannot run these commands yourself — never infer, assume or ' +
|
||||
'reconstruct a result for a criterion that has none. A criterion whose result is FAILED or ' +
|
||||
'BLOCKED is `Broken`; a criterion with no result is judged on delivered code alone, and you ' +
|
||||
'say so in the Evidence column.'
|
||||
);
|
||||
return out.join('\n');
|
||||
}
|
||||
|
||||
function firstLine(text) {
|
||||
return String(text ?? '').split('\n').map((l) => l.trim()).find((l) => l !== '') ?? '';
|
||||
}
|
||||
|
||||
// Markdown table cells: no pipes, no newlines, bounded length.
|
||||
function cell(text) {
|
||||
const t = String(text ?? '').replace(/\r?\n/g, ' ').replace(/\|/g, '\\|').trim();
|
||||
return t.length <= 120 ? t : `${t.slice(0, 120)}…`;
|
||||
}
|
||||
|
||||
// --- CLI --------------------------------------------------------------------
|
||||
|
||||
const USAGE =
|
||||
'usage: criteria-runner.mjs (--plan <plan.md> | --brief <brief.md>) [--json] [--cwd <dir>] [--timeout <ms>]';
|
||||
'usage: criteria-runner.mjs (--plan <plan.md> | --brief <brief.md>) [--json | --evidence] [--cwd <dir>] [--timeout <ms>]';
|
||||
|
||||
export function main(argv) {
|
||||
let mode = null;
|
||||
let path = null;
|
||||
let json = false;
|
||||
let evidence = false;
|
||||
const opts = {};
|
||||
for (let i = 0; i < argv.length; i++) {
|
||||
const a = argv[i];
|
||||
|
|
@ -282,11 +338,16 @@ export function main(argv) {
|
|||
mode = a.slice(2);
|
||||
path = argv[++i];
|
||||
} else if (a === '--json') json = true;
|
||||
else if (a === '--evidence') evidence = true;
|
||||
else if (a === '--cwd' && argv[i + 1]) opts.cwd = resolve(argv[++i]);
|
||||
else if (a === '--timeout' && argv[i + 1]) opts.timeoutMs = Number(argv[++i]);
|
||||
else { process.stderr.write(`criteria-runner: unknown argument ${a}\n${USAGE}\n`); return 2; }
|
||||
}
|
||||
if (!mode) { process.stderr.write(`criteria-runner: no artifact given\n${USAGE}\n`); return 2; }
|
||||
if (json && evidence) {
|
||||
process.stderr.write(`criteria-runner: --json and --evidence are two output shapes; pick one\n${USAGE}\n`);
|
||||
return 2;
|
||||
}
|
||||
if (opts.timeoutMs !== undefined && !Number.isFinite(opts.timeoutMs)) {
|
||||
process.stderr.write(`criteria-runner: --timeout must be a number\n${USAGE}\n`);
|
||||
return 2;
|
||||
|
|
@ -299,7 +360,8 @@ export function main(argv) {
|
|||
process.stderr.write(`criteria-runner: ${err.message}\n`);
|
||||
return 2;
|
||||
}
|
||||
process.stdout.write((json ? JSON.stringify(rep, null, 2) : render(rep)) + '\n');
|
||||
const text = json ? JSON.stringify(rep, null, 2) : evidence ? formatCriteriaEvidence(rep) : render(rep);
|
||||
process.stdout.write(text + '\n');
|
||||
return rep.summary.ok ? 0 : 1;
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue