fix(validate): a check that can never run is caught when the brief or plan is written

voyage-lekerepo end-to-end trial (2026-10-06), report points 2, 3, 4, 9:
11 of 13 brief criteria were NOT RUN at review with no warning in /trekbrief;
a plan `git grep` criterion passed validator, gate and three critic rounds,
then felled the run in Phase 7; the End-to-end field and Phase 7's text
search never met; a prose Verify left the test command undefined.

- criteria-runner: runnableCheck() screens a declared check without running
  it (same allowlist; an npm run script is not looked up at write time);
  /bin/bash tests/<script> allowed; the plan report carries end_to_end,
  verified when a passed criterion runs a step's End-to-end command (the
  older text form still counts).
- plan-validator 1.8: PLAN_VERIFICATION_NOT_RUNNABLE,
  PLAN_STEP_VERIFY_NOT_RUNNABLE, PLAN_END_TO_END_NOT_RUNNABLE,
  PLAN_END_TO_END_NOT_VERIFIED (the gate inherits them).
- brief-validator: BRIEF_SC_NOT_RUNNABLE warning; /trekbrief offers a
  tests/ script; brief-reviewer no longer proposes unrunnable commands.
- trekexecute: Phase 7 copies end_to_end from the runner; Verify's test
  command is its first backticked span.

Measured on the trial's own brief and plan: 11 warnings (the 11 NOT RUN),
and the plan stops on V5 git grep and the unlisted End-to-end command.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/fixtures/jsonl-schemas.md

@@ -22,7 +22,7 @@
 | trekbrief-stats | ts, task, slug, mode, interview_turns, review_iterations, brief_quality, research_topics, auto_research, auto_result, intent_approved, project_dir, order_id, order_claimed_at, order_reason | commands/trekbrief.md (orchestrator-emit Phase 7; order_* added by lib/stats/stats-append.mjs) | trekbrief.md:1124-1139 | profile, phase_models, profile_source | none |
 | trekresearch-stats | ts, question, mode, scope, engine, slug, project_dir, brief_path, dimensions, dimensions_baseline, dimensions_baseline_preserved, effort, conv_turns, empty_turns, unique_sources, agents_local, agents_external, gemini_used, confidence, contradictions, open_questions | commands/trekresearch.md (orchestrator-emit Stats tracking) | trekresearch.md:634-676 | profile, phase_models, parallel_agents, external_research_enabled, profile_source | none |
 | trekplan-stats | ts, task, mode, slug, brief_path, project_dir, codebase_size, codebase_files, agents_deployed, deep_dives, research_briefs_used, research_scout_used, critic_verdict, guardian_verdict, outcome | commands/trekplan.md (orchestrator-emit Phase 12) | trekplan.md:805-826 | profile, phase_models, parallel_agents, profile_source | none |
-| trekexecute-stats (Phase 9 record) | ts, plan, plan_type, mode, result, steps_total, steps_passed, steps_failed, steps_skipped, failed_at_step | lib/stats/stats-append.mjs (trekexecute.md Phase 9) | trekexecute.md:2325-2347 | profile, phase_models, profile_source | none |
+| trekexecute-stats (Phase 9 record) | ts, plan, plan_type, mode, result, steps_total, steps_passed, steps_failed, steps_skipped, failed_at_step | lib/stats/stats-append.mjs (trekexecute.md Phase 9) | trekexecute.md:2330-2352 | profile, phase_models, profile_source | none |
 | trekexecute-stats (autonomy events) | ts, event, known_event, payload | lib/stats/event-emit.mjs `emit()` | event-emit.mjs:78-99 | payload.profile, payload.phase_models, payload.profile_source | none |
 | trekexecute-stats (PostToolUse Bash) | ts, session_id, command_excerpt, duration_ms, success | hooks/scripts/post-bash-stats.mjs (Bash PostToolUse) | post-bash-stats.mjs:42-54 | none (hook is plugin-level, not profile-aware) | command_excerpt (CWE-212) |
 | trekreview-stats | ts, slug, verdict, counts (BLOCKER/MAJOR/MINOR/SUGGESTION), reviewed_files_count, mode, duration_ms | commands/trekreview.md (orchestrator-emit Phase 8) | trekreview.md:255 | profile, phase_models, profile_source | none |

tests/helpers/plan-v18-builder.mjs

@@ -99,7 +99,7 @@ export function buildPlan(o = {}) {
   const header = ceremony === null
     ? `> Generated by trekplan v6.1.1 on 2026-10-06 — \`plan_version: ${version}\``
     : `> Generated by trekplan v6.1.1 on 2026-10-06 — \`plan_version: ${version}\` · \`ceremony: ${ceremony}\``;
-  const stepOpts = o.steps ?? [{}, { endToEnd: 'node src/cli.mjs --demo' }];
+  const stepOpts = o.steps ?? [{}, { endToEnd: 'bash tests/e2e.sh' }];
   const parts = [
     '# Test plan',
     '',
@@ -123,6 +123,8 @@ export function buildPlan(o = {}) {
     '## Verification',
     '',
     '- [ ] `npm test` → expected: exit 0',
+    // Each End-to-end command also stands here: Phase 7 runs this section.
+    ...[...new Set(stepOpts.filter(s => s.endToEnd).map(s => s.endToEnd))].map(c => `- [ ] \`${c}\` → expected: exit 0`),
     '',
   );
   return parts.join('\n');

tests/validators/plan-gate.test.mjs

@@ -73,7 +73,7 @@ test('K3: no critic verdict → 1 (the plan has not been through the critic)', (
 });
 
 test('K3: an invalid plan → 1 even with a clean verdict', () => {
-  const bad = buildPlan({ steps: [{ testFirst: false }, { endToEnd: 'node x' }] });
+  const bad = buildPlan({ steps: [{ testFirst: false }, { endToEnd: 'bash tests/e2e.sh' }] });
   const g = gatePlan(rec(bad, dedupOut()).text);
   assert.equal(g.code, 1);
   assert.ok(g.reasons.some(x => x.code === 'PLAN_GATE_INVALID_PLAN'));

tests/validators/plan-validator-v18.test.mjs

@@ -31,7 +31,7 @@ test('a complete 1.8 architectural plan validates strict', () => {
 });
 
 test('1.8: a step without `Test first:` is rejected', () => {
-  const r = validatePlanContent(buildPlan({ steps: [{ testFirst: false }, { endToEnd: 'node src/cli.mjs' }] }));
+  const r = validatePlanContent(buildPlan({ steps: [{ testFirst: false }, { endToEnd: 'bash tests/e2e.sh' }] }));
   assert.equal(r.valid, false);
   assert.ok(codes(r).includes('PLAN_STEP_NO_TEST_FIRST'), codes(r).join(','));
   assert.match(r.errors.find(e => e.code === 'PLAN_STEP_NO_TEST_FIRST').message, /Step 1/);
@@ -57,10 +57,10 @@ test('1.8: `End-to-end:` without a backticked command does not count', () => {
 test('1.8: K4 fields are optional, but an empty one is rejected', () => {
   const withFields = buildPlan({ steps: [
     { interfaceOut: '`run1(): number` read by Step 2', reviewFocus: 'empty input — pinned by `tests/mod1.test.mjs`' },
-    { interfaceIn: '`run1()` from Step 1', endToEnd: 'node src/cli.mjs' },
+    { interfaceIn: '`run1()` from Step 1', endToEnd: 'bash tests/e2e.sh' },
   ] });
   assert.deepEqual(validatePlanContent(withFields).errors, []);
-  const empty = buildPlan({ steps: [{ interfaceIn: '' }, { endToEnd: 'node src/cli.mjs' }] });
+  const empty = buildPlan({ steps: [{ interfaceIn: '' }, { endToEnd: 'bash tests/e2e.sh' }] });
   assert.ok(codes(validatePlanContent(empty)).includes('PLAN_STEP_FIELD_EMPTY'));
 });
 
@@ -236,7 +236,7 @@ test('CLI --strict exits 1 on a 1.8 plan missing Test first; exits 0 through a s
   const dir = mkdtempSync(join(tmpdir(), 'plan v18 '));
   try {
     const bad = join(dir, 'bad.md');
-    writeFileSync(bad, buildPlan({ steps: [{ testFirst: false }, { endToEnd: 'node x' }] }));
+    writeFileSync(bad, buildPlan({ steps: [{ testFirst: false }, { endToEnd: 'bash tests/e2e.sh' }] }));
     let code = 0;
     try { execFileSync('node', [VALIDATOR, '--strict', '--json', bad], { stdio: 'pipe' }); } catch (e) { code = e.status; }
     assert.equal(code, 1);

tests/validators/runnable-checks.test.mjs

@@ -0,0 +1,197 @@
+// tests/validators/runnable-checks.test.mjs
+// A check that can never run is caught when the brief or the plan is written,
+// not when the run is judged (voyage-lekerepo end-to-end trial, 2026-10-06,
+// report points 2, 3, 4 and 9):
+//   - 11 of 13 brief success criteria were NOT RUN at review, and nothing in
+//     /trekbrief warned;
+//   - a plan whose `## Verification` held `git grep ...` passed the validator,
+//     the plan gate and three critic rounds, then felled the run in Phase 7;
+//   - the plan's `End-to-end:` field and Phase 7's end-to-end check did not
+//     meet: Phase 7 only searched the criterion TEXT for "end to end";
+//   - a step's `Verify:` was prose with several commands, and nothing said
+//     which one the executor runs.
+// The runner's own allowlist decides "can run" — one list, used early and late.
+
+import { test } from 'node:test';
+import { strict as assert } from 'node:assert';
+import { mkdtempSync, writeFileSync, rmSync } from 'node:fs';
+import { join } from 'node:path';
+import { tmpdir } from 'node:os';
+import { validatePlanContent } from '../../lib/validators/plan-validator.mjs';
+import { validateBriefContent } from '../../lib/validators/brief-validator.mjs';
+import {
+  allowedCommand, runnableCheck, endToEndCommands, runPlanVerification,
+} from '../../lib/verification/criteria-runner.mjs';
+import { buildPlan } from '../helpers/plan-v18-builder.mjs';
+
+const codes = (list) => list.map((e) => e.code);
+
+// --- the shared screen -------------------------------------------------------
+
+test('runner: `/bin/bash tests/<script>.sh` is the same declared check as `bash tests/<script>.sh`', () => {
+  assert.equal(allowedCommand('/bin/bash tests/e2e.sh').allowed, true);
+  assert.equal(allowedCommand('/bin/bash scripts/deploy.sh').allowed, false);
+});
+
+test('runner: at write time an `npm run` script may still be added by the plan itself', () => {
+  const empty = mkdtempSync(join(tmpdir(), 'voyage-runnable-'));
+  try {
+    assert.equal(allowedCommand('npm run e2e', { cwd: empty }).allowed, false);
+    assert.equal(runnableCheck({ command: 'npm run e2e' }, { cwd: empty }).runnable, true);
+  } finally {
+    rmSync(empty, { recursive: true, force: true });
+  }
+});
+
+test('runner: runnableCheck says why a check can never run', () => {
+  const r = runnableCheck({ command: 'git grep -q -e --max -- README.md' });
+  assert.equal(r.runnable, false);
+  assert.match(r.reason, /read-only/);
+  assert.equal(runnableCheck({ command: null, reason: 'no-command' }).runnable, false);
+  assert.equal(runnableCheck({ command: 'npm test' }).runnable, true);
+});
+
+// --- the plan (point 3, 4, 9) ------------------------------------------------
+
+test('plan 1.8: a `## Verification` criterion the runner can never run is an error, named', () => {
+  const plan = buildPlan().replace(
+    '- [ ] `npm test` → expected: exit 0',
+    '- [ ] `npm test` → expected: exit 0\n- [ ] `git grep -q -e --max -- README.md` → expected: exit 0',
+  );
+  const r = validatePlanContent(plan);
+  assert.equal(r.valid, false);
+  const e = r.errors.find((x) => x.code === 'PLAN_VERIFICATION_NOT_RUNNABLE');
+  assert.ok(e, codes(r.errors).join(','));
+  assert.match(e.message, /git grep/);
+  assert.match(e.hint, /bash tests\//);
+});
+
+test('plan 1.8: a `## Verification` bullet with no command is an error (a plan promises commands)', () => {
+  const plan = buildPlan().replace(
+    '- [ ] `npm test` → expected: exit 0',
+    '- [ ] `npm test` → expected: exit 0\n- [ ] the README mentions the new flag',
+  );
+  assert.ok(codes(validatePlanContent(plan).errors).includes('PLAN_VERIFICATION_NOT_RUNNABLE'));
+});
+
+test('plan 1.8: an `End-to-end:` command the runner can never run is an error', () => {
+  const r = validatePlanContent(buildPlan({ steps: [{}, { endToEnd: 'node src/cli.mjs --demo' }] }));
+  assert.ok(codes(r.errors).includes('PLAN_END_TO_END_NOT_RUNNABLE'), codes(r.errors).join(','));
+});
+
+test('plan 1.8: an `End-to-end:` command that does not also stand under `## Verification` is an error', () => {
+  const plan = buildPlan().replace('\n- [ ] `bash tests/e2e.sh` → expected: exit 0', '');
+  assert.ok(!plan.includes('- [ ] `bash tests/e2e.sh`'), 'fixture: the builder lists the end-to-end command');
+  const r = validatePlanContent(plan);
+  assert.ok(codes(r.errors).includes('PLAN_END_TO_END_NOT_VERIFIED'), codes(r.errors).join(','));
+});
+
+test('plan 1.8: a step whose `Verify:` does not open with one runnable test command is an error', () => {
+  const plan = buildPlan().replace(
+    '- **Verify:** `node --test tests/mod1.test.mjs` → expected: pass',
+    '- **Verify:** `node bin/slug.mjs 2>&1 | grep -q -- --max` → expected: exit 0',
+  );
+  const r = validatePlanContent(plan);
+  const e = r.errors.find((x) => x.code === 'PLAN_STEP_VERIFY_NOT_RUNNABLE');
+  assert.ok(e, codes(r.errors).join(','));
+  assert.match(e.message, /Step 1/);
+});
+
+test('plan 1.8: a step whose `Verify:` opens with a test command and then describes output is fine', () => {
+  const plan = buildPlan().replace(
+    '- **Verify:** `node --test tests/mod1.test.mjs` → expected: pass',
+    '- **Verify:** `node --test tests/mod1.test.mjs` → expected: `fail 0`',
+  );
+  assert.deepEqual(validatePlanContent(plan).errors, []);
+});
+
+test('plan 1.8, soft: the runnability findings are warnings', () => {
+  const r = validatePlanContent(buildPlan({ steps: [{}, { endToEnd: 'node src/cli.mjs' }] }), { strict: false });
+  assert.ok(codes(r.warnings).includes('PLAN_END_TO_END_NOT_RUNNABLE'));
+  assert.ok(!codes(r.errors).includes('PLAN_END_TO_END_NOT_RUNNABLE'));
+});
+
+test('plan 1.7: none of the runnability rules apply', () => {
+  const plan = buildPlan({ version: '1.7', steps: [{}, { endToEnd: 'node src/cli.mjs' }] })
+    .replace('- [ ] `npm test` → expected: exit 0', '- [ ] `git grep x` → expected: exit 0');
+  const r = validatePlanContent(plan);
+  for (const c of ['PLAN_VERIFICATION_NOT_RUNNABLE', 'PLAN_END_TO_END_NOT_RUNNABLE', 'PLAN_END_TO_END_NOT_VERIFIED', 'PLAN_STEP_VERIFY_NOT_RUNNABLE']) {
+    assert.ok(!codes(r.errors).includes(c) && !codes(r.warnings).includes(c), c);
+  }
+});
+
+// --- Phase 7 reads the End-to-end field (point 4) ----------------------------
+
+function withPlan(text, fn) {
+  const dir = mkdtempSync(join(tmpdir(), 'voyage-e2e-'));
+  try {
+    const p = join(dir, 'plan.md');
+    writeFileSync(p, text);
+    return fn(p);
+  } finally {
+    rmSync(dir, { recursive: true, force: true });
+  }
+}
+
+const passAll = { exec: () => ({ status: 0, stdout: '', stderr: '' }), screen: () => ({ allowed: true, rule: '' }) };
+
+test('runner: the plan\'s `End-to-end:` commands are read from its steps', () => {
+  assert.deepEqual(endToEndCommands(buildPlan()), ['bash tests/e2e.sh']);
+});
+
+test('runner: a passed criterion that runs the step\'s End-to-end command verifies end to end — no magic words needed', () => {
+  withPlan(buildPlan(), (p) => {
+    const rep = runPlanVerification(p, passAll);
+    assert.equal(rep.summary.ok, true);
+    assert.equal(rep.end_to_end.status, 'verified');
+    assert.equal(rep.end_to_end.by, 'end-to-end-field');
+  });
+});
+
+test('runner: the older text form still verifies (session specs, plans below 1.8)', () => {
+  const plan = buildPlan({ steps: [{}, {}] }).replace(
+    '- [ ] `npm test` → expected: exit 0',
+    '- [ ] `npm test` → expected: exit 0 — runs the CLI end to end',
+  );
+  withPlan(plan, (p) => {
+    assert.equal(runPlanVerification(p, passAll).end_to_end.status, 'verified');
+  });
+});
+
+test('runner: a failing verification is never verified end to end', () => {
+  withPlan(buildPlan(), (p) => {
+    const rep = runPlanVerification(p, { ...passAll, exec: (cmd) => ({ status: cmd === 'npm test' ? 1 : 0, stdout: '', stderr: '' }) });
+    assert.equal(rep.summary.ok, false);
+    assert.equal(rep.end_to_end.status, 'not-verified');
+  });
+});
+
+// --- the brief (point 2) -----------------------------------------------------
+
+function brief(criteria) {
+  return [
+    '---', 'type: trekbrief', 'brief_version: "2.2"', 'task: t', 'slug: s',
+    'research_topics: 0', 'research_status: skipped', 'framing: new-direction', 'phase_signals_partial: true', '---', '',
+    '## TL;DR', '', 'A toy.', '', '## Intent', '', 'Why.', '', '## Goal', '', 'What.', '',
+    '## Success Criteria', '', ...criteria, '',
+  ].join('\n');
+}
+
+test('brief: a success criterion whose command the runner can never run is a warning that names it and the way in', () => {
+  const r = validateBriefContent(brief([
+    '- `npm test` exits 0.',
+    '- `node bin/slug.mjs --max 9 "Hello Big World"` prints `hello-big`.',
+    '- The README explains the flag.',
+  ]));
+  assert.equal(r.valid, true, JSON.stringify(r.errors));
+  const w = r.warnings.filter((x) => x.code === 'BRIEF_SC_NOT_RUNNABLE');
+  assert.equal(w.length, 1, JSON.stringify(r.warnings));
+  assert.match(w[0].message, /SC2/);
+  assert.match(w[0].message, /node bin\/slug\.mjs/);
+  assert.match(w[0].hint, /bash tests\//);
+});
+
+test('brief: runnable and prose-only criteria raise no runnability warning', () => {
+  const r = validateBriefContent(brief(['- `npm test` exits 0.', '- `bash tests/e2e.sh` exits 0.', '- The output reads well.']));
+  assert.ok(!codes(r.warnings).includes('BRIEF_SC_NOT_RUNNABLE'), JSON.stringify(r.warnings));
+});