fix(review): run the success-criteria commands and hand the reviewer the result (D-04)

The rubric required `brief-conformance-reviewer` to classify a Success
Criterion as Full only when "its verification command/test exists and passes".
Its tools are `Read`, `Glob`, `Grep`. It cannot run anything, so "passes" was
either guessed from the command's mere existence or quietly downgraded to
"exists" — a BLOCKER-tier rule key resting on an impression.

The reviewer stays read-only — a reviewer that executes the code it reviews is
not an independent reviewer. The command does the running instead:

- `/trekreview` Phase 4.5 runs the brief's `## Success Criteria` commands
  through `lib/verification/criteria-runner.mjs --brief --evidence` and captures
  the block as `sc_evidence_block`, pasted verbatim into the reviewer prompt in
  Phase 5. The exit code does not stop the review — a failing criterion is
  exactly what the review exists to find.
- `formatCriteriaEvidence` builds that block in code: one row per criterion with
  the command, the exit code and the first output line. Chose a code-built block
  over an orchestrator-written summary so the orchestrator cannot narrate a pass
  that never happened.
- The rubric now judges the supplied result: `PASS` supports Full, `FAILED` /
  `BLOCKED` is `Broken` with the exit code cited, and `NOT RUN` is the absence
  of a measurement — never evidence in either direction.
- Phase 4.5 is skipped in `quick` mode: that mode does not launch the
  conformance reviewer, so there is nobody to hand the result to.

Red first: seven tests in `tests/lib/criteria-runner.test.mjs` against a
committed brief fixture whose three criteria pass, fail, and are prose-only.
The two doc pins were verified red against the pre-fix files (rubric asked
"exists and passes"; no Phase 4.5; the block reached nobody).

Suite: 1117 (1115/0/2), up 9.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

tests/fixtures/brief-success-criteria.md

@@ -0,0 +1,30 @@
+---
+task: criteria-runner fixture — a brief whose success criteria are mixed
+slug: criteria-runner-fixture-brief
+brief_version: "2.2"
+framing: new-direction
+---
+
+# Task Brief: criteria-runner fixture
+
+Fixture only. Consumed by `tests/lib/criteria-runner.test.mjs`. It is the
+falsifying case for D-04: a read-only reviewer cannot run these commands, so
+/trekreview runs them and hands over the result.
+
+## TL;DR
+
+Three success criteria: one passes, one fails on purpose, one is prose only.
+
+## Goal
+
+Exercise the brief side of the criteria runner.
+
+## Success Criteria
+
+- The runner reaches the shell: `true` exits 0
+- The failing case is visible: `false` exits 0 (FAILS on purpose — exit 1)
+- No new runtime dependencies are introduced
+
+## Non-Goals
+
+- Being executed by the pipeline. This file is a test fixture.

tests/lib/criteria-runner.test.mjs

@@ -22,6 +22,8 @@ import {
   runCriteria,
   summarize,
   runPlanVerification,
+  runSuccessCriteriaChecks,
+  formatCriteriaEvidence,
   render,
 } from '../../lib/verification/criteria-runner.mjs';
 
@@ -278,3 +280,73 @@ test('CLI: no mode flag exits 2 — it never guesses which artifact it was given
   assert.equal(r.status, 2);
   assert.match(r.stderr, /usage/);
 });
+
+// --- the brief path: evidence handed to a read-only reviewer ----------------
+//
+// D-04: brief-conformance-reviewer is asked to judge whether a Success
+// Criterion's verification command "exists and passes", but its tools are
+// Read/Glob/Grep — it cannot run anything. /trekreview runs the commands and
+// hands over the RESULT, and the block it hands over is built by code so the
+// orchestrator cannot narrate a pass that never happened.
+
+test('runSuccessCriteriaChecks: each criterion gets a real result, prose ones are NOT RUN', () => {
+  const report = runSuccessCriteriaChecks(join(FIX, 'brief-success-criteria.md'));
+  assert.equal(report.kind, 'brief');
+  assert.deepEqual(report.results.map((r) => r.label), ['SC1', 'SC2', 'SC3']);
+  assert.equal(report.results[0].status, 'passed');
+  assert.equal(report.results[1].status, 'failed');
+  assert.equal(report.results[1].exitCode, 1);
+  assert.equal(report.results[2].status, 'unrunnable');
+  assert.equal(report.summary.ok, false, 'a failing criterion is never ok, in either mode');
+});
+
+test('runSuccessCriteriaChecks: a brief with no Success Criteria section reports the code', () => {
+  const dir = mkdtempSync(join(tmpdir(), 'criteria-runner-'));
+  const p = join(dir, 'brief.md');
+  writeFileSync(p, '# Brief\n\n## Goal\n\nSomething.\n');
+  const report = runSuccessCriteriaChecks(p);
+  assert.equal(report.error.code, 'NO_SUCCESS_CRITERIA_SECTION');
+  assert.deepEqual(report.results, []);
+});
+
+test('formatCriteriaEvidence: one row per criterion, with command and exit code', () => {
+  const report = runSuccessCriteriaChecks(join(FIX, 'brief-success-criteria.md'));
+  const block = formatCriteriaEvidence(report);
+  for (const label of ['SC1', 'SC2', 'SC3']) assert.match(block, new RegExp(`\\| ${label} \\|`));
+  assert.match(block, /PASS/);
+  assert.match(block, /FAILED/);
+  assert.match(block, /NOT RUN/);
+  assert.match(block, /exit 1/);
+});
+
+test('formatCriteriaEvidence: the block forbids inferring a pass for a criterion with no result', () => {
+  const report = runSuccessCriteriaChecks(join(FIX, 'brief-success-criteria.md'));
+  const block = formatCriteriaEvidence(report);
+  assert.match(block, /NOT RUN/);
+  assert.match(
+    block, /never.*(infer|assume)/i,
+    'the evidence block must state in-band that NOT RUN is not a pass — the reviewer cannot re-run it',
+  );
+});
+
+test('formatCriteriaEvidence: an unreadable section still yields a block that says so', () => {
+  const dir = mkdtempSync(join(tmpdir(), 'criteria-runner-'));
+  const p = join(dir, 'brief.md');
+  writeFileSync(p, '# Brief\n\n## Goal\n\nSomething.\n');
+  const block = formatCriteriaEvidence(runSuccessCriteriaChecks(p));
+  assert.match(block, /NO_SUCCESS_CRITERIA_SECTION/);
+  assert.match(block, /no criteria were run/i);
+});
+
+test('CLI: --evidence emits the reviewer block, and --brief still exits 1 on a failure', () => {
+  const r = cli(['--brief', join(FIX, 'brief-success-criteria.md'), '--evidence']);
+  assert.equal(r.status, 1, r.stderr);
+  assert.match(r.stdout, /\| SC1 \|/);
+  assert.match(r.stdout, /NOT RUN/);
+});
+
+test('CLI: --evidence and --json together exit 2 — one output shape at a time', () => {
+  const r = cli(['--brief', join(FIX, 'brief-success-criteria.md'), '--evidence', '--json']);
+  assert.equal(r.status, 2);
+  assert.match(r.stderr, /usage/);
+});

tests/lib/doc-consistency.test.mjs

@@ -1852,3 +1852,30 @@ test('D-03: trekexecute Phase 7 runs the plan Verification on the single-session
     'a failing criterion must fell the run, not be noted in the final report',
   );
 });
+
+// End-state defect D-04: the rubric required brief-conformance-reviewer to decide whether a
+// Success Criterion's verification command "exists and passes" — with Read/Glob/Grep. The
+// command now runs the commands and hands over the result; the reviewer stays read-only.
+// Fix the SOURCE.
+test('D-04: the conformance reviewer judges a supplied result, and never runs a command', () => {
+  const a = read('agents/brief-conformance-reviewer.md');
+  assert.ok(
+    !/exists and passes/.test(a),
+    'the rubric may no longer ask a Read/Glob/Grep agent whether a command passes — that ask IS defect D-04',
+  );
+  assert.match(a, /^tools: \["Read", "Glob", "Grep"\]$/m, 'the reviewer stays read-only: no Bash');
+  assert.match(a, /You never run a command/, 'the reviewer must be told in-band that it does not execute');
+  assert.match(a, /Success-criteria check results/, 'the reviewer must document the supplied evidence as an input');
+});
+
+test('D-04: trekreview runs the success-criteria commands and hands the reviewer the result', () => {
+  const t = read('commands/trekreview.md');
+  const phase = (t.split("\n## Phase 4.5 — ")[1] || '').split('\n## ')[0];
+  assert.ok(phase.length > 0, 'trekreview.md must carry the phase that runs the success-criteria checks');
+  assert.match(phase, /criteria-runner\.mjs" \\\n  --brief .* --evidence/, 'the phase must invoke the runner in --evidence mode');
+  assert.match(phase, /sc_evidence_block/, 'the phase must name the captured block the reviewer prompt carries');
+  assert.match(
+    t.split('\n## Phase 5 — ')[1] || '', /sc_evidence_block/,
+    'the reviewer-launch phase must hand the block over — a block nobody passes is not evidence',
+  );
+});