voyage/tests/lib/proevesett.test.mjs

// tests/lib/proevesett.test.mjs
// Structure of the plugin-eval suite under evals/ (docs/proevesett.md). The
// suite itself runs headless children and costs money; these tests pin what
// can be checked offline: that each grader can see what it claims to see.
//
//   - the new case for veikart steg 1's gate exists, and its fixture really
//     reaches the intent gate (passes the brief validator, has no marker)
//   - review-requires-project: the error line is written by CODE (the arg
//     parser), and the grader reads the parser's code from the trace
//   - grader weaknesses the PM re-measurement found (2026-09-23):
//     PASS/FAIL as raw substrings, no-error-code missing REVIEW_WRONG_TYPE,
//     no-write blind to Bash writes
//
// Trace facts this relies on (measured on the PM's recorded traces): the
// expanded command prose is NOT in the trace; tool output is, JSON-escaped
// once (\"code\": \"REVIEW_BAD_FINDING_ID\").

import { test } from 'node:test';
import { strict as assert } from 'node:assert';
import { spawnSync } from 'node:child_process';
import { existsSync, mkdtempSync, readdirSync, readFileSync, rmSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { dirname, join } from 'node:path';
import { fileURLToPath } from 'node:url';

const HERE = dirname(fileURLToPath(import.meta.url));
const ROOT = join(HERE, '..', '..');
const EVALS = join(ROOT, 'evals');
const read = (rel) => readFileSync(join(ROOT, rel), 'utf8');

/** Minimal frontmatter reader for grader files: key: value, quoted or not. */
function grader(rel) {
  const text = read(rel);
  const m = /^---\n([\s\S]*?)\n---/.exec(text);
  assert.ok(m, `${rel}: no frontmatter`);
  const out = {};
  for (const line of m[1].split('\n')) {
    const kv = /^(\w+):\s*(.*)$/.exec(line);
    if (!kv) continue;
    let v = kv[2].trim();
    if (v.startsWith("'") && v.endsWith("'")) v = v.slice(1, -1).replace(/''/g, "'");
    else if (v.startsWith('"') && v.endsWith('"')) v = JSON.parse(v);
    out[kv[1]] = v;
  }
  return out;
}

const cases = () => readdirSync(EVALS).filter((d) => existsSync(join(EVALS, d, 'prompt.md'))).sort();
const graders = (c) => readdirSync(join(EVALS, c, 'graders')).filter((f) => f.endsWith('.md')).sort();

/** How a tool's JSON output appears inside a trace line (escaped once). */
const inTrace = (obj, indent) => JSON.stringify(JSON.stringify(obj, null, indent));

/** How a Bash tool input appears to tool_used's input_match (JSON-encoded input). */
const bashInput = (command) => JSON.stringify({ command, description: 'Validate the review -> report' });

const COMMAND_PROSE = readdirSync(join(ROOT, 'commands')).map((f) => read(join('commands', f))).join('\n');

// ---- the new case: veikart steg 1's intent gate ----

const GATE_CASE = 'plan-halts-without-intent-approval';

test('proevesett: a case exists for the steg 1 intent gate, with its graders', () => {
  assert.ok(cases().includes(GATE_CASE), `evals/${GATE_CASE} missing`);
  assert.deepEqual(graders(GATE_CASE), ['gate-ran.md', 'names-code.md', 'no-agent.md', 'no-plan-file.md']);
  const prompt = read(`evals/${GATE_CASE}/prompt.md`);
  assert.match(prompt, /^\/voyage:trekplan --project proj$/m);
  assert.match(prompt, /^runs: 1$/m);
  const g = grader(`evals/${GATE_CASE}/graders/gate-ran.md`);
  assert.equal(g.type, 'tool_used');
  assert.equal(g.tool, 'Bash');
  assert.match('node /p/lib/validators/intent-approval.mjs --check --json "proj/brief.md"', new RegExp(g.input_match));
  const n = grader(`evals/${GATE_CASE}/graders/names-code.md`);
  assert.equal(n.target, 'trace');
  const re = new RegExp(n.pattern);
  assert.match(inTrace({ valid: false, errors: [{ code: 'BRIEF_INTENT_NOT_APPROVED' }] }, 2), re);
  assert.doesNotMatch(COMMAND_PROSE, re, 'the pattern must not be satisfiable by quoting the command prose');
  const noAgent = grader(`evals/${GATE_CASE}/graders/no-agent.md`);
  assert.deepEqual([noAgent.type, noAgent.tool, noAgent.min, noAgent.max], ['tool_used', 'Agent', '0', '0']);
  const noPlan = grader(`evals/${GATE_CASE}/graders/no-plan-file.md`);
  assert.deepEqual([noPlan.type, noPlan.path, noPlan.exists], ['file_exists', '**/plan.md', 'false']);
});

test('proevesett: the gate case fixture passes the brief validator and stops at the intent gate', () => {
  const ws = mkdtempSync(join(tmpdir(), 'proevesett-'));
  try {
    const s = spawnSync('bash', [join(EVALS, GATE_CASE, 'scaffold.sh')], { cwd: ws, encoding: 'utf8' });
    assert.equal(s.status, 0, s.stderr);
    const brief = join(ws, 'proj', 'brief.md');
    const v = spawnSync('node', [join(ROOT, 'lib/validators/brief-validator.mjs'), '--soft', '--json', brief], { encoding: 'utf8' });
    assert.equal(v.status, 0, `brief-validator must pass so the run reaches the intent gate: ${v.stdout} ${v.stderr}`);
    const g = spawnSync('node', [join(ROOT, 'lib/validators/intent-approval.mjs'), '--check', '--json', brief], { encoding: 'utf8' });
    assert.equal(g.status, 1);
    assert.deepEqual(JSON.parse(g.stdout).errors.map((e) => e.code), ['BRIEF_INTENT_NOT_APPROVED']);
  } finally { rmSync(ws, { recursive: true, force: true }); }
});

// ---- case 5: the missing-flag error comes from code ----

test('review-requires-project: the arg parser, run as trekreview.md spells it, prints the error (exit 1)', () => {
  const all = read('commands/trekreview.md').split('\n');
  const at = all.map((l, i) =>
    (/^\s*node \$\{CLAUDE_PLUGIN_ROOT\}\/lib\/parsers\/arg-parser\.mjs --command trekreview\b/.test(l) ? i : -1)).filter((i) => i >= 0);
  assert.equal(at.length, 1, 'trekreview.md must carry exactly one arg-parser line');
  const line = all[at[0]].trim();
  assert.match(line, / --stdin <<'VOYAGE_ARGS_EOF_7F3C9E'$/, 'the parser must read the arguments on stdin from the quoted heredoc');
  const block = all.slice(at[0], at[0] + 3).map((l) => l.trim());
  assert.deepEqual(block.slice(1), ['$ARGUMENTS', 'VOYAGE_ARGS_EOF_7F3C9E'],
    'the parser must receive $ARGUMENTS as the heredoc body (the Bash tool has no "$@")');
  const run = (args) => spawnSync('bash', ['-c',
    block.join('\n').split('${CLAUDE_PLUGIN_ROOT}').join(ROOT).split('$ARGUMENTS').join(args)], { encoding: 'utf8' });
  const none = run('');
  assert.equal(none.status, 1, `missing --project must exit 1: ${none.stdout} ${none.stderr}`);
  assert.deepEqual(JSON.parse(none.stdout).errors.map((e) => e.code), ['ARG_REQUIRED_MISSING']);
  assert.match(none.stderr, /^Error: --project <dir> is required\.$/m);
  assert.match(none.stderr, /^Usage: \/trekreview --project <dir>/m);
  const ok = run('--project proj --validate');
  assert.equal(ok.status, 0, ok.stdout + ok.stderr);
  assert.equal(JSON.parse(ok.stdout).flags['--project'], 'proj');
});

test('review-requires-project: the prose does not compose the error itself; the grader reads the parser code', () => {
  const prose = read('commands/trekreview.md');
  assert.doesNotMatch(prose, /Error: --project <dir> is required/, 'the model must relay the parser, not compose the line');
  assert.doesNotMatch(prose, /ARG_REQUIRED_MISSING/);
  const gs = graders('review-requires-project');
  assert.ok(gs.includes('parser-ran.md') && gs.includes('names-missing-project.md'), gs.join(','));
  const n = grader('evals/review-requires-project/graders/names-missing-project.md');
  assert.equal(n.target, 'trace');
  const re = new RegExp(n.pattern);
  assert.match(inTrace({ errors: [{ code: 'ARG_REQUIRED_MISSING' }] }), re);
  assert.doesNotMatch(COMMAND_PROSE, re);
});

// ---- grader weaknesses ----

test('known-negative no-error-code sees EVERY review-validator code (incl. REVIEW_WRONG_TYPE), bound to the output form', () => {
  const src = read('lib/validators/review-validator.mjs');
  const codes = [...new Set([...src.matchAll(/issue\('(REVIEW_[A-Z_]+)'/g)].map((m) => m[1]))];
  assert.ok(codes.includes('REVIEW_WRONG_TYPE') && codes.includes('REVIEW_VERSION_FORMAT'), codes.join(','));
  const g = grader('evals/review-validate-passes-clean-review/graders/no-error-code.md');
  assert.equal(g.target, 'trace');
  assert.equal(g.match, 'not_contains');
  const re = new RegExp(g.pattern);
  for (const code of codes) {
    assert.match(inTrace({ valid: false, errors: [{ code }] }, 2), re, `${code} must be caught`);
  }
  assert.doesNotMatch(inTrace({ valid: true, errors: [], warnings: [] }, 2), re);
  assert.doesNotMatch(COMMAND_PROSE, re);
});

test('says-pass / says-fail are not raw substrings: a negation does not pass them', () => {
  const pass = new RegExp(grader('evals/review-validate-passes-clean-review/graders/says-pass.md').pattern);
  const fail = new RegExp(grader('evals/review-validate-flags-bad-finding-id/graders/says-fail.md').pattern);
  for (const s of ['review-validator: PASS proj/review.md', 'Result: **PASS**', 'PASS — review.md is valid']) assert.match(s, pass);
  for (const s of ['The review did not PASS.', 'NOT PASS', 'PASSED_WITH_WARNINGS']) assert.doesNotMatch(s, pass);
  for (const s of ['review-validator: FAIL proj/review.md', 'Result: **FAIL**']) assert.match(s, fail);
  for (const s of ['It did not FAIL.', 'NOT FAIL', 'FAILOVER']) assert.doesNotMatch(s, fail);
  // The verdict word, not one spelling of it: v1's raw substring took these,
  // v2's word boundary dropped them (a correct reply would fail the case).
  for (const s of ['Validation FAILED', '**FAILED**: proj/review.md']) assert.match(s, fail);
  for (const s of ['Validation PASSED', '**PASSED**: proj/review.md']) assert.match(s, pass);
  for (const s of ['It did not FAILED.', 'FAILEDX']) assert.doesNotMatch(s, fail);
});

test('says-pass / says-fail take the verdict in any letter case, and still refuse the wrong one', () => {
  const pass = new RegExp(grader('evals/review-validate-passes-clean-review/graders/says-pass.md').pattern);
  const fail = new RegExp(grader('evals/review-validate-flags-bad-finding-id/graders/says-fail.md').pattern);
  // Case-8 replies that failed says-pass in the PM re-measurement of ea66a98
  // (2 of 12): correct, validator exit 0, but the verdict only in lower case.
  const lowerPass = [
    '`proj/review.md` passed the strict schema check, with 0 errors and 0 warnings.',
    '`proj/review.md` passed the strict schema validator: 0 errors and 0 warnings.',
    '`proj/review.md` passes the strict schema check, with no errors and no warnings.',
    'Validation passed.',
  ];
  for (const s of lowerPass) assert.match(s, pass);
  for (const s of ['review.md failed the strict schema check.', 'Validation fails: 1 error.', 'Result: failed']) assert.match(s, fail);
  // The wrong verdict, in any case, is still refused.
  for (const s of ['not passed', 'Not passed.', 'The review did not pass.', 'It does not pass the strict validator.',
    'It doesn’t pass.', "It doesn't pass.", 'failed', 'Validation failed.', 'bypass', 'passage']) {
    assert.doesNotMatch(s, pass, s);
  }
  for (const s of ['not failed', 'It did not fail.', "It didn't fail.", 'passed', 'failover', 'failure']) {
    assert.doesNotMatch(s, fail, s);
  }
  // Each case's real reply (as recorded) must not satisfy the other case's
  // grader: case 6 replies say "did not pass", case 8 replies never say fail.
  const case6 = '**FAIL**: `proj/review.md` did not pass the strict schema check (the validator exited with code 1).';
  const case6b = '**FAIL:** `proj/review.md` does not pass the strict validator. The validator exited with code 1.';
  for (const s of [case6, case6b]) { assert.doesNotMatch(s, pass, s); assert.match(s, fail, s); }
  for (const s of lowerPass) assert.doesNotMatch(s, fail, s);
});

test('says-pass / says-fail refuse a reply whose capitalised verdict is the opposite one', () => {
  const pass = new RegExp(grader('evals/review-validate-passes-clean-review/graders/says-pass.md').pattern);
  const fail = new RegExp(grader('evals/review-validate-flags-bad-finding-id/graders/says-fail.md').pattern);
  // A correct case-6 reply recorded on 687710a: the verdict is FAIL, and a
  // side clause says "passed". Lower case alone would let it satisfy says-pass.
  const mixedFail = '**FAIL**: `proj/review.md` did not pass the strict validator (exit 1).\n\n'
    + 'The first entry is `F-001`. The other checks passed.';
  assert.match(mixedFail, fail);
  assert.doesNotMatch(mixedFail, pass);
  assert.doesNotMatch('Result: FAILED\nEverything else passed.', pass);
  assert.doesNotMatch('**PASS**: proj/review.md is valid.\nNothing failed.', fail);
  assert.doesNotMatch('**PASSED**\n`review.md` fails nothing.', fail);
  // The opposite word in lower case, or negated in capitals, does not refuse.
  assert.match('**PASS**: nothing failed.', pass);
  assert.match('It did not FAIL: `proj/review.md` passed.', pass);
  assert.match('It did not PASS: `proj/review.md` failed.', fail);
  // Multi-line: the refusal looks at the whole reply, not the line of the verdict.
  assert.doesNotMatch('`proj/review.md` passed one check.\n\n**FAIL**', pass);
});

test('every case with a no-write grader also has no-bash-write, and it sees a write through Bash', () => {
  const withNoWrite = cases().filter((c) => graders(c).includes('no-write.md'));
  assert.ok(withNoWrite.length >= 5, withNoWrite.join(','));
  for (const c of withNoWrite) {
    assert.ok(graders(c).includes('no-bash-write.md'), `${c}: no-write sees only the Write tool`);
    const g = grader(`evals/${c}/graders/no-bash-write.md`);
    assert.deepEqual([g.type, g.tool, g.min, g.max], ['tool_used', 'Bash', '0', '0'], c);
    const re = new RegExp(g.input_match);
    for (const cmd of [
      "cat > proj/review.md <<'EOF'\nx\nEOF",
      'echo x >> proj/review.md',
      'printf x>proj/plan.md',
      'node -e 1 | tee proj/out.txt',
      "sed -i '' 's/a/b/' proj/review.md",
      'rm proj/review.md',
    ]) assert.match(bashInput(cmd), re, `${c}: must see ${cmd}`);
    for (const cmd of [
      'node /p/lib/validators/review-validator.mjs --json "proj/review.md"',
      'node /p/lib/validators/review-validator.mjs --json proj/review.md 2>&1',
      'ls proj 2>/dev/null',
      'node /p/lib/parsers/arg-parser.mjs --command trekreview -- ',
      'cat proj/review.md | head -5',
    ]) assert.doesNotMatch(bashInput(cmd), re, `${c}: must not flag ${cmd}`);
  }
});