voyage/tests/lib/criteria-runner.test.mjs

// tests/lib/criteria-runner.test.mjs
// The criteria runner is what makes a declared check a RUN check: it parses the
// falsifiable criteria a plan (`## Verification`) or a brief (`## Success
// Criteria`) declares, screens each command through the executor's own
// PreToolUse denylist, runs it, and returns a verdict built from exit codes.
//
// Fail-closed is the whole point: a criterion that cannot run (placeholder, no
// command, screen unavailable) must never read as "passed".

import { test } from 'node:test';
import { strict as assert } from 'node:assert';
import { spawnSync } from 'node:child_process';
import { join, dirname } from 'node:path';
import { fileURLToPath } from 'node:url';
import { mkdtempSync, writeFileSync, readFileSync } from 'node:fs';
import { tmpdir } from 'node:os';

import {
  parsePlanVerification,
  parseSuccessCriteria,
  screenCommand,
  runCriteria,
  summarize,
  runPlanVerification,
  runSuccessCriteriaChecks,
  formatCriteriaEvidence,
  render,
} from '../../lib/verification/criteria-runner.mjs';

const HERE = dirname(fileURLToPath(import.meta.url));
const ROOT = join(HERE, '..', '..');
const CLI = join(ROOT, 'lib', 'verification', 'criteria-runner.mjs');
const FIX = join(ROOT, 'tests', 'fixtures');
const HOOK = join(ROOT, 'hooks', 'scripts', 'pre-bash-executor.mjs');

// An exec double: maps a command string to {status, stdout, stderr}.
function execDouble(table) {
  const calls = [];
  const exec = (command) => {
    calls.push(command);
    return table[command] ?? { status: 127, stdout: '', stderr: 'not in table' };
  };
  exec.calls = calls;
  return exec;
}

// A screen double that allows everything, so exec behaviour can be tested alone.
const allowAll = () => ({ allowed: true, rule: '' });

// --- parsing ---------------------------------------------------------------

test('parsePlanVerification: checkbox bullets become V1..Vn with their command', () => {
  const md = [
    '# Plan',
    '',
    '## Verification',
    '',
    '- [ ] `npm test` -> expected: exit 0',
    '- [x] `node --test tests/lib/x.test.mjs` -> expected: 3 passing',
    '',
    '## Estimated Scope',
    '',
    '- [ ] `not-a-criterion` (outside the section)',
  ].join('\n');

  const criteria = parsePlanVerification(md);
  assert.equal(criteria.length, 2);
  assert.deepEqual(criteria.map((c) => c.label), ['V1', 'V2']);
  assert.equal(criteria[0].command, 'npm test');
  assert.equal(criteria[1].command, 'node --test tests/lib/x.test.mjs');
  assert.match(criteria[0].text, /expected: exit 0/);
});

test('parsePlanVerification: a template placeholder is unrunnable, not a command', () => {
  const md = '## Verification\n\n- [ ] `{exact command}` -> expected: `{exact output}`\n';
  const criteria = parsePlanVerification(md);
  assert.equal(criteria.length, 1);
  assert.equal(criteria[0].command, null);
  assert.equal(criteria[0].reason, 'placeholder');
});

test('parsePlanVerification: a missing section yields no criteria', () => {
  assert.deepEqual(parsePlanVerification('# Plan\n\n## Steps\n\n- do a thing\n'), []);
});

test('parseSuccessCriteria: bullets become SC1..SCn and take the FIRST backticked span', () => {
  const md = [
    '## Success Criteria',
    '',
    '- All existing tests pass: `npm test` exits 0',
    '- Endpoint returns 200: `curl -s localhost:3000/health` -> `"ok"`',
    '- No new runtime dependencies are introduced',
    '',
    '## Research Plan',
  ].join('\n');

  const criteria = parseSuccessCriteria(md);
  assert.deepEqual(criteria.map((c) => c.label), ['SC1', 'SC2', 'SC3']);
  assert.equal(criteria[0].command, 'npm test');
  assert.equal(criteria[1].command, 'curl -s localhost:3000/health');
  assert.equal(criteria[2].command, null);
  assert.equal(criteria[2].reason, 'no-command');
});

// --- screening -------------------------------------------------------------

test('screenCommand: the real executor denylist blocks a catastrophic command', () => {
  const verdict = screenCommand('rm -rf ~', { hookPath: HOOK });
  assert.equal(verdict.allowed, false);
  assert.match(verdict.rule, /rm -rf|destruction/i);
});

test('screenCommand: an ordinary command passes the real denylist', () => {
  assert.equal(screenCommand('npm test', { hookPath: HOOK }).allowed, true);
});

test('screenCommand: an unavailable screen denies (fail-closed, never silently allows)', () => {
  const verdict = screenCommand('npm test', { hookPath: join(ROOT, 'hooks', 'scripts', 'no-such-hook.mjs') });
  assert.equal(verdict.allowed, false);
  assert.match(verdict.rule, /screen unavailable/i);
});

// --- running ---------------------------------------------------------------

test('runCriteria: exit 0 passes, a non-zero exit fails, output is captured', () => {
  const criteria = parsePlanVerification(
    '## Verification\n\n- [ ] `good`\n- [ ] `bad`\n'
  );
  const exec = execDouble({
    good: { status: 0, stdout: 'all green\n', stderr: '' },
    bad: { status: 1, stdout: '', stderr: '1 failing\n' },
  });
  const results = runCriteria(criteria, { exec, screen: allowAll });

  assert.deepEqual(results.map((r) => r.status), ['passed', 'failed']);
  assert.equal(results[0].exitCode, 0);
  assert.equal(results[1].exitCode, 1);
  assert.match(results[1].output, /1 failing/);
  assert.deepEqual(exec.calls, ['good', 'bad']);
});

test('runCriteria: a blocked command is marked blocked and is NEVER executed', () => {
  const criteria = parsePlanVerification('## Verification\n\n- [ ] `rm -rf ~`\n');
  const exec = execDouble({});
  const results = runCriteria(criteria, {
    exec,
    screen: () => ({ allowed: false, rule: 'Filesystem root/home destruction' }),
  });

  assert.equal(results[0].status, 'blocked');
  assert.equal(results[0].exitCode, null);
  assert.match(results[0].output, /Filesystem root\/home destruction/);
  assert.deepEqual(exec.calls, [], 'a blocked command must not reach the shell');
});

test('runCriteria: a criterion with no command is unrunnable, not passed', () => {
  const criteria = parseSuccessCriteria('## Success Criteria\n\n- No new dependencies\n');
  const results = runCriteria(criteria, { exec: execDouble({}), screen: allowAll });
  assert.equal(results[0].status, 'unrunnable');
  assert.equal(results[0].exitCode, null);
});

test('runCriteria: output is capped so a verbose command cannot flood a prompt', () => {
  const criteria = parsePlanVerification('## Verification\n\n- [ ] `loud`\n');
  const exec = execDouble({ loud: { status: 0, stdout: 'x'.repeat(10000), stderr: '' } });
  const results = runCriteria(criteria, { exec, screen: allowAll, maxOutput: 200 });
  assert.ok(results[0].output.length < 400, `capped, got ${results[0].output.length}`);
  assert.match(results[0].output, /truncated/);
});

// --- the verdict -----------------------------------------------------------

test('summarize: one failing criterion makes the run NOT ok', () => {
  const results = [
    { status: 'passed' }, { status: 'failed' }, { status: 'passed' },
  ];
  const s = summarize(results, { requireCommand: true });
  assert.equal(s.ok, false);
  assert.equal(s.failed, 1);
  assert.equal(s.passed, 2);
  assert.equal(s.total, 3);
});

test('summarize: in plan mode an unrunnable criterion makes the run NOT ok', () => {
  const s = summarize([{ status: 'passed' }, { status: 'unrunnable' }], { requireCommand: true });
  assert.equal(s.ok, false);
  assert.equal(s.unrunnable, 1);
});

test('summarize: in brief mode an unrunnable criterion is reported, not failed', () => {
  const s = summarize([{ status: 'passed' }, { status: 'unrunnable' }], { requireCommand: false });
  assert.equal(s.ok, true);
  assert.equal(s.unrunnable, 1);
});

test('summarize: a blocked criterion is never ok, in either mode', () => {
  for (const requireCommand of [true, false]) {
    assert.equal(summarize([{ status: 'blocked' }], { requireCommand }).ok, false);
  }
});

test('summarize: zero criteria is NOT ok in plan mode (a plan that promises nothing)', () => {
  assert.equal(summarize([], { requireCommand: true }).ok, false);
});

// --- the single-session path, end to end -----------------------------------

test('runPlanVerification: a plan whose success criterion FAILS fells the run', () => {
  const report = runPlanVerification(join(FIX, 'plan-verification-fails.md'));
  assert.equal(report.kind, 'plan');
  assert.equal(report.summary.ok, false);
  assert.equal(report.summary.failed, 1);
  const failed = report.results.find((r) => r.status === 'failed');
  assert.ok(failed, 'the failing criterion is reported by label');
  assert.match(failed.label, /^V\d+$/);
});

test('runPlanVerification: a plan whose criteria all pass is ok', () => {
  const report = runPlanVerification(join(FIX, 'plan-verification-passes.md'));
  assert.equal(report.summary.ok, true);
  assert.equal(report.summary.failed, 0);
  assert.equal(report.summary.total, 2);
});

test('runPlanVerification: a plan with no ## Verification section is NOT ok', () => {
  const dir = mkdtempSync(join(tmpdir(), 'criteria-runner-'));
  const p = join(dir, 'plan.md');
  writeFileSync(p, '# Plan\n\n## Steps\n\n- do a thing\n');
  const report = runPlanVerification(p);
  assert.equal(report.summary.ok, false);
  assert.equal(report.error.code, 'NO_VERIFICATION_SECTION');
});

test('render: the report names every non-passing criterion and its exit code', () => {
  const report = runPlanVerification(join(FIX, 'plan-verification-fails.md'));
  const text = render(report);
  assert.match(text, /FAILED/);
  assert.match(text, /exit 1/);
});

// --- the CLI ---------------------------------------------------------------

function cli(args) {
  return spawnSync(process.execPath, [CLI, ...args], { encoding: 'utf8', cwd: ROOT });
}

test('CLI: --plan exits 1 when a criterion fails', () => {
  const r = cli(['--plan', join(FIX, 'plan-verification-fails.md')]);
  assert.equal(r.status, 1, r.stderr);
  assert.match(r.stdout, /FAILED/);
});

test('CLI: --plan exits 0 when every criterion passes', () => {
  const r = cli(['--plan', join(FIX, 'plan-verification-passes.md')]);
  assert.equal(r.status, 0, r.stderr + r.stdout);
});

test('CLI: --json emits a parseable report with the summary', () => {
  const r = cli(['--plan', join(FIX, 'plan-verification-fails.md'), '--json']);
  assert.equal(r.status, 1);
  const out = JSON.parse(r.stdout);
  assert.equal(out.summary.ok, false);
  assert.equal(out.results.length, out.summary.total);
});

test('CLI: a missing file exits 2 — a read error is not a failed criterion', () => {
  const r = cli(['--plan', join(FIX, 'no-such-plan.md')]);
  assert.equal(r.status, 2);
  assert.match(r.stderr, /criteria-runner/);
});

test('CLI: an unknown argument exits 2 with usage', () => {
  const r = cli(['--nope']);
  assert.equal(r.status, 2);
  assert.match(r.stderr, /usage/);
});

test('CLI: no mode flag exits 2 — it never guesses which artifact it was given', () => {
  const r = cli([]);
  assert.equal(r.status, 2);
  assert.match(r.stderr, /usage/);
});

// --- the brief path: evidence handed to a read-only reviewer ----------------
//
// D-04: brief-conformance-reviewer is asked to judge whether a Success
// Criterion's verification command "exists and passes", but its tools are
// Read/Glob/Grep — it cannot run anything. /trekreview runs the commands and
// hands over the RESULT, and the block it hands over is built by code so the
// orchestrator cannot narrate a pass that never happened.

test('runSuccessCriteriaChecks: each criterion gets a real result, prose ones are NOT RUN', () => {
  const report = runSuccessCriteriaChecks(join(FIX, 'brief-success-criteria.md'));
  assert.equal(report.kind, 'brief');
  assert.deepEqual(report.results.map((r) => r.label), ['SC1', 'SC2', 'SC3']);
  assert.equal(report.results[0].status, 'passed');
  assert.equal(report.results[1].status, 'failed');
  assert.equal(report.results[1].exitCode, 1);
  assert.equal(report.results[2].status, 'unrunnable');
  assert.equal(report.summary.ok, false, 'a failing criterion is never ok, in either mode');
});

test('runSuccessCriteriaChecks: a brief with no Success Criteria section reports the code', () => {
  const dir = mkdtempSync(join(tmpdir(), 'criteria-runner-'));
  const p = join(dir, 'brief.md');
  writeFileSync(p, '# Brief\n\n## Goal\n\nSomething.\n');
  const report = runSuccessCriteriaChecks(p);
  assert.equal(report.error.code, 'NO_SUCCESS_CRITERIA_SECTION');
  assert.deepEqual(report.results, []);
});

test('formatCriteriaEvidence: one row per criterion, with command and exit code', () => {
  const report = runSuccessCriteriaChecks(join(FIX, 'brief-success-criteria.md'));
  const block = formatCriteriaEvidence(report);
  for (const label of ['SC1', 'SC2', 'SC3']) assert.match(block, new RegExp(`\\| ${label} \\|`));
  assert.match(block, /PASS/);
  assert.match(block, /FAILED/);
  assert.match(block, /NOT RUN/);
  assert.match(block, /exit 1/);
});

test('formatCriteriaEvidence: the block forbids inferring a pass for a criterion with no result', () => {
  const report = runSuccessCriteriaChecks(join(FIX, 'brief-success-criteria.md'));
  const block = formatCriteriaEvidence(report);
  assert.match(block, /NOT RUN/);
  assert.match(
    block, /never.*(infer|assume)/i,
    'the evidence block must state in-band that NOT RUN is not a pass — the reviewer cannot re-run it',
  );
});

test('formatCriteriaEvidence: an unreadable section still yields a block that says so', () => {
  const dir = mkdtempSync(join(tmpdir(), 'criteria-runner-'));
  const p = join(dir, 'brief.md');
  writeFileSync(p, '# Brief\n\n## Goal\n\nSomething.\n');
  const block = formatCriteriaEvidence(runSuccessCriteriaChecks(p));
  assert.match(block, /NO_SUCCESS_CRITERIA_SECTION/);
  assert.match(block, /no criteria were run/i);
});

test('CLI: --evidence emits the reviewer block, and --brief still exits 1 on a failure', () => {
  const r = cli(['--brief', join(FIX, 'brief-success-criteria.md'), '--evidence']);
  assert.equal(r.status, 1, r.stderr);
  assert.match(r.stdout, /\| SC1 \|/);
  assert.match(r.stdout, /NOT RUN/);
});

test('CLI: --evidence and --json together exit 2 — one output shape at a time', () => {
  const r = cli(['--brief', join(FIX, 'brief-success-criteria.md'), '--evidence', '--json']);
  assert.equal(r.status, 2);
  assert.match(r.stderr, /usage/);
});

// --- the repo's OWN plan format --------------------------------------------
//
// D-03's runner read bullet lines only. The repo's own example plan writes its
// `## Verification` as a fenced bash block, so it parsed to ZERO criteria, the
// runner exited 1, and Phase 7 forbade `result: completed` — a correct plan
// felled every single-session run. Measured 2026-09-18 (PM checkpoint on
// 6cafb4c): 0 of the repo's plan artifacts exited 0.

const SHELL_BLOCK_PLAN = [
  '# Plan',
  '',
  '## Verification',
  '',
  'Final acceptance run after step 3:',
  '',
  '```bash',
  'npm test                      # all green',
  '# a comment-only line is not a criterion',
  '',
  'node --test tests/lib/x.test.mjs',
  '```',
  '',
  '## Estimated Scope',
  '',
  '```bash',
  'not-a-criterion --outside-the-section',
  '```',
].join('\n');

test('parsePlanVerification: a fenced shell block yields one criterion per command line', () => {
  const criteria = parsePlanVerification(SHELL_BLOCK_PLAN);
  assert.deepEqual(criteria.map((c) => c.label), ['V1', 'V2']);
  assert.equal(criteria[0].command, 'npm test                      # all green');
  assert.equal(criteria[1].command, 'node --test tests/lib/x.test.mjs');
});

test('parsePlanVerification: a fenced block and a bullet list declaring the same commands parse identically', () => {
  const fenced = '## Verification\n\n```bash\nnpm test\nnode --test tests/lib/x.test.mjs\n```\n';
  const bulleted = '## Verification\n\n- [ ] `npm test`\n- [ ] `node --test tests/lib/x.test.mjs`\n';
  assert.deepEqual(
    parsePlanVerification(fenced).map((c) => [c.label, c.command]),
    parsePlanVerification(bulleted).map((c) => [c.label, c.command]),
  );
});

test('parsePlanVerification: only a shell-tagged fence is read as commands', () => {
  const md = '## Verification\n\n```json\n{"expected": "output"}\n```\n\n- [ ] `npm test`\n';
  const criteria = parsePlanVerification(md);
  assert.deepEqual(criteria.map((c) => c.command), ['npm test'],
    'an untagged or non-shell fence holds expected output, not commands — never invent a criterion from it');
});

test('parsePlanVerification: a `## ` heading inside a fence is quoted text, not a section', () => {
  const md = '# Report\n\n```\n# Plan\n## Verification\n## Plan-critic notes\n```\n\nNo real section here.\n';
  assert.deepEqual(parsePlanVerification(md), []);
});

// Every plan artifact the repo ships must survive its own runner. This test
// parses; it never runs — one example plan names a fictional CLI on purpose.
const PLAN_ARTIFACTS = [
  'examples/01-add-verbose-flag/plan.md',
  'templates/plan-template.md',
  'tests/fixtures/plan-verification-fails.md',
  'tests/fixtures/plan-verification-passes.md',
  'tests/synthetic/plan-run-C.md',
];

test("the repo's own plan artifacts each declare at least one criterion", () => {
  for (const rel of PLAN_ARTIFACTS) {
    const criteria = parsePlanVerification(readFileSync(join(ROOT, rel), 'utf8'));
    assert.ok(criteria.length >= 1, `${rel} parsed to ${criteria.length} criteria`);
  }
});

// examples/02-real-cli/REGENERATED.md is a REPORT that quotes a plan outline
// inside a fence; it has no `## Verification` of its own. The honest answer is
// NO_VERIFICATION_SECTION — not an empty section that reads as "0 of 0".
test('a report that merely quotes a plan outline has no ## Verification section', () => {
  const report = runPlanVerification(join(ROOT, 'examples/02-real-cli/REGENERATED.md'));
  assert.equal(report.error?.code, 'NO_VERIFICATION_SECTION');
  assert.equal(report.summary.ok, false);
});

// --- a span is not a command just because it is in backticks ---------------
//
// "The first backtick span is the command" is right for a plan (the template
// puts it first) and wrong for a brief, whose sentence usually OPENS with the
// thing under discussion: a flag, a path. Running those produced `/bin/sh: --:
// invalid option` (exit 2) and "is a directory" (exit 126), and the review
// rubric reads a FAILED result as decisive -> a BLOCKER invented out of prose.
// Measured 2026-09-18: 3 of the 6 criteria in the repo's own example brief.

const EXAMPLE_BRIEF = join(ROOT, 'examples', '01-add-verbose-flag', 'brief.md');

test('parseSuccessCriteria: a leading flag is not a command — the criterion is NOT RUN', () => {
  const md = '## Success Criteria\n\n- `--verbose` works in any position: `app --verbose run`\n';
  const [sc] = parseSuccessCriteria(md);
  assert.equal(sc.command, null);
  assert.equal(sc.reason, 'not-a-command');
});

test('parseSuccessCriteria: a bare directory or file path is not a command', () => {
  const md = [
    '## Success Criteria',
    '',
    '- Existing tests in `tests/` continue to pass',
    '- The golden file `tests/golden/login.stdout` is unchanged',
    '',
  ].join('\n');
  for (const sc of parseSuccessCriteria(md)) {
    assert.equal(sc.command, null, `${sc.label} must not be run as a command`);
    assert.equal(sc.reason, 'not-a-command');
  }
});

test('parseSuccessCriteria: a real command, an explicit path and env prefixes still run', () => {
  const md = [
    '## Success Criteria',
    '',
    '- All tests pass: `npm test`',
    '- The build script runs: `./scripts/build.sh --ci`',
    '- Absolute paths run: `/usr/bin/true`',
    '- Env prefixes are not the command: `CI=1 npm test`',
    '',
  ].join('\n');
  assert.deepEqual(
    parseSuccessCriteria(md).map((c) => c.command),
    ['npm test', './scripts/build.sh --ci', '/usr/bin/true', 'CI=1 npm test'],
  );
});

test("the repo's own example brief yields ZERO parse artifacts", () => {
  const criteria = parseSuccessCriteria(readFileSync(EXAMPLE_BRIEF, 'utf8'));
  assert.equal(criteria.length, 6);
  for (const c of criteria) {
    if (c.command === null) continue;
    assert.ok(!c.command.startsWith('-'), `${c.label} would run a flag: ${c.command}`);
    assert.ok(!/^\S+\/(\s|$)/.test(c.command), `${c.label} would run a path: ${c.command}`);
  }
  // SC3/SC4 open on a flag, SC6 on a directory: three criteria that used to be
  // FAILED with a parse exit code and are now an absent measurement.
  for (const label of ['SC3', 'SC4', 'SC6']) {
    const c = criteria.find((x) => x.label === label);
    assert.equal(c.command, null, `${label}: ${c.command}`);
    assert.equal(c.reason, 'not-a-command');
  }
});

test('runCriteria: a not-a-command criterion is unrunnable and says why — never failed', () => {
  const criteria = parseSuccessCriteria('## Success Criteria\n\n- `--verbose` is accepted everywhere\n');
  const exec = execDouble({});
  const [r] = runCriteria(criteria, { exec, screen: allowAll });
  assert.equal(r.status, 'unrunnable');
  assert.notEqual(r.status, 'failed');
  assert.equal(r.exitCode, null);
  assert.match(r.output, /not-a-command/);
  assert.deepEqual(exec.calls, [], 'a span that is not a command must never reach a shell');
});

test('a shell-tagged fence declares commands even when a line is not command-shaped', () => {
  const md = '## Verification\n\n```bash\n[ -f README.md ] || exit 1\n```\n';
  const [v] = parsePlanVerification(md);
  assert.equal(v.command, '[ -f README.md ] || exit 1',
    'inside an explicitly shell-tagged fence the author HAS declared shell; the shape check belongs to prose spans only');
});