voyage/tests/validators/runnable-checks.test.mjs

// tests/validators/runnable-checks.test.mjs
// A check that can never run is caught when the brief or the plan is written,
// not when the run is judged (voyage-lekerepo end-to-end trial, 2026-10-06,
// report points 2, 3, 4 and 9):
//   - 11 of 13 brief success criteria were NOT RUN at review, and nothing in
//     /trekbrief warned;
//   - a plan whose `## Verification` held `git grep ...` passed the validator,
//     the plan gate and three critic rounds, then felled the run in Phase 7;
//   - the plan's `End-to-end:` field and Phase 7's end-to-end check did not
//     meet: Phase 7 only searched the criterion TEXT for "end to end";
//   - a step's `Verify:` was prose with several commands, and nothing said
//     which one the executor runs.
// The runner's own allowlist decides "can run" — one list, used early and late.

import { test } from 'node:test';
import { strict as assert } from 'node:assert';
import { mkdtempSync, writeFileSync, rmSync } from 'node:fs';
import { join } from 'node:path';
import { tmpdir } from 'node:os';
import { validatePlanContent } from '../../lib/validators/plan-validator.mjs';
import { validateBriefContent } from '../../lib/validators/brief-validator.mjs';
import {
  allowedCommand, runnableCheck, endToEndCommands, runPlanVerification,
} from '../../lib/verification/criteria-runner.mjs';
import { buildPlan } from '../helpers/plan-v18-builder.mjs';

const codes = (list) => list.map((e) => e.code);

// --- the shared screen -------------------------------------------------------

test('runner: `/bin/bash tests/<script>.sh` is the same declared check as `bash tests/<script>.sh`', () => {
  assert.equal(allowedCommand('/bin/bash tests/e2e.sh').allowed, true);
  assert.equal(allowedCommand('/bin/bash scripts/deploy.sh').allowed, false);
});

test('runner: at write time an `npm run` script may still be added by the plan itself', () => {
  const empty = mkdtempSync(join(tmpdir(), 'voyage-runnable-'));
  try {
    assert.equal(allowedCommand('npm run e2e', { cwd: empty }).allowed, false);
    assert.equal(runnableCheck({ command: 'npm run e2e' }, { cwd: empty }).runnable, true);
  } finally {
    rmSync(empty, { recursive: true, force: true });
  }
});

test('runner: runnableCheck says why a check can never run', () => {
  const r = runnableCheck({ command: 'git grep -q -e --max -- README.md' });
  assert.equal(r.runnable, false);
  assert.match(r.reason, /read-only/);
  assert.equal(runnableCheck({ command: null, reason: 'no-command' }).runnable, false);
  assert.equal(runnableCheck({ command: 'npm test' }).runnable, true);
});

// --- the plan (point 3, 4, 9) ------------------------------------------------

test('plan 1.8: a `## Verification` criterion the runner can never run is an error, named', () => {
  const plan = buildPlan().replace(
    '- [ ] `npm test` → expected: exit 0',
    '- [ ] `npm test` → expected: exit 0\n- [ ] `git grep -q -e --max -- README.md` → expected: exit 0',
  );
  const r = validatePlanContent(plan);
  assert.equal(r.valid, false);
  const e = r.errors.find((x) => x.code === 'PLAN_VERIFICATION_NOT_RUNNABLE');
  assert.ok(e, codes(r.errors).join(','));
  assert.match(e.message, /git grep/);
  assert.match(e.hint, /bash tests\//);
});

test('plan 1.8: a `## Verification` bullet with no command is an error (a plan promises commands)', () => {
  const plan = buildPlan().replace(
    '- [ ] `npm test` → expected: exit 0',
    '- [ ] `npm test` → expected: exit 0\n- [ ] the README mentions the new flag',
  );
  assert.ok(codes(validatePlanContent(plan).errors).includes('PLAN_VERIFICATION_NOT_RUNNABLE'));
});

test('plan 1.8: an `End-to-end:` command the runner can never run is an error', () => {
  const r = validatePlanContent(buildPlan({ steps: [{}, { endToEnd: 'node src/cli.mjs --demo' }] }));
  assert.ok(codes(r.errors).includes('PLAN_END_TO_END_NOT_RUNNABLE'), codes(r.errors).join(','));
});

test('plan 1.8: an `End-to-end:` command that does not also stand under `## Verification` is an error', () => {
  const plan = buildPlan().replace('\n- [ ] `bash tests/e2e.sh` → expected: exit 0', '');
  assert.ok(!plan.includes('- [ ] `bash tests/e2e.sh`'), 'fixture: the builder lists the end-to-end command');
  const r = validatePlanContent(plan);
  assert.ok(codes(r.errors).includes('PLAN_END_TO_END_NOT_VERIFIED'), codes(r.errors).join(','));
});

test('plan 1.8: a step whose `Verify:` does not open with one runnable test command is an error', () => {
  const plan = buildPlan().replace(
    '- **Verify:** `node --test tests/mod1.test.mjs` → expected: pass',
    '- **Verify:** `node bin/slug.mjs 2>&1 | grep -q -- --max` → expected: exit 0',
  );
  const r = validatePlanContent(plan);
  const e = r.errors.find((x) => x.code === 'PLAN_STEP_VERIFY_NOT_RUNNABLE');
  assert.ok(e, codes(r.errors).join(','));
  assert.match(e.message, /Step 1/);
});

test('plan 1.8: a step whose `Verify:` opens with a test command and then describes output is fine', () => {
  const plan = buildPlan().replace(
    '- **Verify:** `node --test tests/mod1.test.mjs` → expected: pass',
    '- **Verify:** `node --test tests/mod1.test.mjs` → expected: `fail 0`',
  );
  assert.deepEqual(validatePlanContent(plan).errors, []);
});

test('plan 1.8, soft: the runnability findings are warnings', () => {
  const r = validatePlanContent(buildPlan({ steps: [{}, { endToEnd: 'node src/cli.mjs' }] }), { strict: false });
  assert.ok(codes(r.warnings).includes('PLAN_END_TO_END_NOT_RUNNABLE'));
  assert.ok(!codes(r.errors).includes('PLAN_END_TO_END_NOT_RUNNABLE'));
});

test('plan 1.7: none of the runnability rules apply', () => {
  const plan = buildPlan({ version: '1.7', steps: [{}, { endToEnd: 'node src/cli.mjs' }] })
    .replace('- [ ] `npm test` → expected: exit 0', '- [ ] `git grep x` → expected: exit 0');
  const r = validatePlanContent(plan);
  for (const c of ['PLAN_VERIFICATION_NOT_RUNNABLE', 'PLAN_END_TO_END_NOT_RUNNABLE', 'PLAN_END_TO_END_NOT_VERIFIED', 'PLAN_STEP_VERIFY_NOT_RUNNABLE']) {
    assert.ok(!codes(r.errors).includes(c) && !codes(r.warnings).includes(c), c);
  }
});

// --- Phase 7 reads the End-to-end field (point 4) ----------------------------

function withPlan(text, fn) {
  const dir = mkdtempSync(join(tmpdir(), 'voyage-e2e-'));
  try {
    const p = join(dir, 'plan.md');
    writeFileSync(p, text);
    return fn(p);
  } finally {
    rmSync(dir, { recursive: true, force: true });
  }
}

const passAll = { exec: () => ({ status: 0, stdout: '', stderr: '' }), screen: () => ({ allowed: true, rule: '' }) };

test('runner: the plan\'s `End-to-end:` commands are read from its steps', () => {
  assert.deepEqual(endToEndCommands(buildPlan()), ['bash tests/e2e.sh']);
});

test('runner: a passed criterion that runs the step\'s End-to-end command verifies end to end — no magic words needed', () => {
  withPlan(buildPlan(), (p) => {
    const rep = runPlanVerification(p, passAll);
    assert.equal(rep.summary.ok, true);
    assert.equal(rep.end_to_end.status, 'verified');
    assert.equal(rep.end_to_end.by, 'end-to-end-field');
  });
});

test('runner: the older text form still verifies (session specs, plans below 1.8)', () => {
  const plan = buildPlan({ steps: [{}, {}] }).replace(
    '- [ ] `npm test` → expected: exit 0',
    '- [ ] `npm test` → expected: exit 0 — runs the CLI end to end',
  );
  withPlan(plan, (p) => {
    assert.equal(runPlanVerification(p, passAll).end_to_end.status, 'verified');
  });
});

test('runner: a failing verification is never verified end to end', () => {
  withPlan(buildPlan(), (p) => {
    const rep = runPlanVerification(p, { ...passAll, exec: (cmd) => ({ status: cmd === 'npm test' ? 1 : 0, stdout: '', stderr: '' }) });
    assert.equal(rep.summary.ok, false);
    assert.equal(rep.end_to_end.status, 'not-verified');
  });
});

// --- the brief (point 2) -----------------------------------------------------

function brief(criteria) {
  return [
    '---', 'type: trekbrief', 'brief_version: "2.2"', 'task: t', 'slug: s',
    'research_topics: 0', 'research_status: skipped', 'framing: new-direction', 'phase_signals_partial: true', '---', '',
    '## TL;DR', '', 'A toy.', '', '## Intent', '', 'Why.', '', '## Goal', '', 'What.', '',
    '## Success Criteria', '', ...criteria, '',
  ].join('\n');
}

test('brief: a success criterion whose command the runner can never run is a warning that names it and the way in', () => {
  const r = validateBriefContent(brief([
    '- `npm test` exits 0.',
    '- `node bin/slug.mjs --max 9 "Hello Big World"` prints `hello-big`.',
    '- The README explains the flag.',
  ]));
  assert.equal(r.valid, true, JSON.stringify(r.errors));
  const w = r.warnings.filter((x) => x.code === 'BRIEF_SC_NOT_RUNNABLE');
  assert.equal(w.length, 1, JSON.stringify(r.warnings));
  assert.match(w[0].message, /SC2/);
  assert.match(w[0].message, /node bin\/slug\.mjs/);
  assert.match(w[0].hint, /bash tests\//);
});

test('brief: a criterion whose first backticked span is not a command is a warning, as the runner reports it (trial 5, finding 1)', () => {
  const r = validateBriefContent(brief([
    '- The contract holds, now that the usage line names `--stdin` and the README too: `/bin/bash tests/contract.sh` exits 0',
    '- `npm test` exits 0.',
  ]));
  assert.equal(r.valid, true, JSON.stringify(r.errors));
  const w = r.warnings.filter((x) => x.code === 'BRIEF_SC_NOT_RUNNABLE');
  assert.equal(w.length, 1, JSON.stringify(r.warnings));
  assert.match(w[0].message, /SC1's first backticked span is `--stdin`/);
  assert.match(w[0].message, /\/bin\/bash tests\/contract\.sh/);
  assert.match(w[0].message, /not-a-command/);
  assert.match(w[0].hint, /first backticked span/);
});

test('brief: a prose criterion that names a flag or a path, with no command after it, stays unflagged (HANDOVER-CONTRACTS)', () => {
  const r = validateBriefContent(brief([
    '- The `--verbose` flag is documented.',
    '- `docs/install.md` explains the install steps.',
  ]));
  assert.ok(!codes(r.warnings).includes('BRIEF_SC_NOT_RUNNABLE'), JSON.stringify(r.warnings));
});

test('brief: runnable and prose-only criteria raise no runnability warning', () => {
  const r = validateBriefContent(brief(['- `npm test` exits 0.', '- `bash tests/e2e.sh` exits 0.', '- The output reads well.']));
  assert.ok(!codes(r.warnings).includes('BRIEF_SC_NOT_RUNNABLE'), JSON.stringify(r.warnings));
});