test(proevesett): red — no case for the intent gate, case 5 composed in prose, weak graders

Offline structure tests for evals/ (the suite itself needs headless runs):
- no case reaches the steg 1 intent gate (PM: 0 traces touched it). The
  new case's scaffold is in: its brief passes brief-validator --soft
  (2.1 WITH phase_signals) and --check gives BRIEF_INTENT_NOT_APPROVED —
  that test is green, so the fixture reaches the gate.
- review-requires-project: trekreview.md composes 'Error: --project <dir>
  is required.' in prose (8/10 in the PM run, backticks broke the regex),
  and its arg-parser line passes "$@", which the Bash tool never has.
- no-error-code misses REVIEW_WRONG_TYPE; PASS/FAIL graders are raw
  substrings; no-write cannot see a write through Bash.
7 tests, 6 red.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

evals/plan-halts-without-intent-approval/case.yaml

@@ -0,0 +1,4 @@
+schema_version: "1.1"
+name: plan-halts-without-intent-approval
+context:
+  scaffold_script: scaffold.sh

evals/plan-halts-without-intent-approval/scaffold.sh

@@ -0,0 +1,45 @@
+#!/bin/bash
+# Seeds the empty run workspace with a project folder whose brief passes the
+# sequencing gate (brief_version 2.1 WITH phase_signals) but carries no
+# intent_approved_hash, so /trekplan must stop at the intent-approval gate.
+# Self-contained on purpose: the run cannot read the eval directory.
+set -eu
+mkdir -p proj
+cat > proj/brief.md <<'BRIEF'
+---
+type: trekbrief
+brief_version: "2.1"
+created: 2026-09-23
+task: "Eval fixture: add a greeting helper"
+slug: eval-greeting-intent
+project_dir: proj/
+research_topics: 0
+research_status: skipped
+auto_research: false
+interview_turns: 1
+source: fixture
+phase_signals:
+  - phase: research
+    effort: low
+  - phase: plan
+    effort: standard
+  - phase: execute
+    effort: standard
+  - phase: review
+    effort: standard
+---
+
+# Task: add a greeting helper
+
+## Intent
+
+Eval fixture for the Voyage proevesett. Not a real task.
+
+## Goal
+
+A function `greet(name)` returns `Hello, <name>!`.
+
+## Success Criteria
+
+- `greet("Ada")` returns `Hello, Ada!`.
+BRIEF

tests/lib/proevesett.test.mjs

@@ -0,0 +1,182 @@
+// tests/lib/proevesett.test.mjs
+// Structure of the plugin-eval suite under evals/ (docs/proevesett.md). The
+// suite itself runs headless children and costs money; these tests pin what
+// can be checked offline: that each grader can see what it claims to see.
+//
+//   - the new case for veikart steg 1's gate exists, and its fixture really
+//     reaches the intent gate (passes the sequencing gate, has no marker)
+//   - review-requires-project: the error line is written by CODE (the arg
+//     parser), and the grader reads the parser's code from the trace
+//   - grader weaknesses the PM re-measurement found (2026-09-23):
+//     PASS/FAIL as raw substrings, no-error-code missing REVIEW_WRONG_TYPE,
+//     no-write blind to Bash writes
+//
+// Trace facts this relies on (measured on the PM's recorded traces): the
+// expanded command prose is NOT in the trace; tool output is, JSON-escaped
+// once (\"code\": \"REVIEW_BAD_FINDING_ID\").
+
+import { test } from 'node:test';
+import { strict as assert } from 'node:assert';
+import { spawnSync } from 'node:child_process';
+import { existsSync, mkdtempSync, readdirSync, readFileSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { dirname, join } from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+const HERE = dirname(fileURLToPath(import.meta.url));
+const ROOT = join(HERE, '..', '..');
+const EVALS = join(ROOT, 'evals');
+const read = (rel) => readFileSync(join(ROOT, rel), 'utf8');
+
+/** Minimal frontmatter reader for grader files: key: value, quoted or not. */
+function grader(rel) {
+  const text = read(rel);
+  const m = /^---\n([\s\S]*?)\n---/.exec(text);
+  assert.ok(m, `${rel}: no frontmatter`);
+  const out = {};
+  for (const line of m[1].split('\n')) {
+    const kv = /^(\w+):\s*(.*)$/.exec(line);
+    if (!kv) continue;
+    let v = kv[2].trim();
+    if (v.startsWith("'") && v.endsWith("'")) v = v.slice(1, -1).replace(/''/g, "'");
+    else if (v.startsWith('"') && v.endsWith('"')) v = JSON.parse(v);
+    out[kv[1]] = v;
+  }
+  return out;
+}
+
+const cases = () => readdirSync(EVALS).filter((d) => existsSync(join(EVALS, d, 'prompt.md'))).sort();
+const graders = (c) => readdirSync(join(EVALS, c, 'graders')).filter((f) => f.endsWith('.md')).sort();
+
+/** How a tool's JSON output appears inside a trace line (escaped once). */
+const inTrace = (obj, indent) => JSON.stringify(JSON.stringify(obj, null, indent));
+
+/** How a Bash tool input appears to tool_used's input_match (JSON-encoded input). */
+const bashInput = (command) => JSON.stringify({ command, description: 'Validate the review -> report' });
+
+const COMMAND_PROSE = readdirSync(join(ROOT, 'commands')).map((f) => read(join('commands', f))).join('\n');
+
+// ---- the new case: veikart steg 1's intent gate ----
+
+const GATE_CASE = 'plan-halts-without-intent-approval';
+
+test('proevesett: a case exists for the steg 1 intent gate, with its graders', () => {
+  assert.ok(cases().includes(GATE_CASE), `evals/${GATE_CASE} missing`);
+  assert.deepEqual(graders(GATE_CASE), ['gate-ran.md', 'names-code.md', 'no-agent.md', 'no-plan-file.md']);
+  const prompt = read(`evals/${GATE_CASE}/prompt.md`);
+  assert.match(prompt, /^\/voyage:trekplan --project proj$/m);
+  assert.match(prompt, /^runs: 1$/m);
+  const g = grader(`evals/${GATE_CASE}/graders/gate-ran.md`);
+  assert.equal(g.type, 'tool_used');
+  assert.equal(g.tool, 'Bash');
+  assert.match('node /p/lib/validators/intent-approval.mjs --check --json "proj/brief.md"', new RegExp(g.input_match));
+  const n = grader(`evals/${GATE_CASE}/graders/names-code.md`);
+  assert.equal(n.target, 'trace');
+  const re = new RegExp(n.pattern);
+  assert.match(inTrace({ valid: false, errors: [{ code: 'BRIEF_INTENT_NOT_APPROVED' }] }, 2), re);
+  assert.doesNotMatch(COMMAND_PROSE, re, 'the pattern must not be satisfiable by quoting the command prose');
+  assert.doesNotMatch(inTrace({ valid: false, errors: [{ code: 'BRIEF_V51_MISSING_SIGNALS' }] }, 2), re);
+  const noAgent = grader(`evals/${GATE_CASE}/graders/no-agent.md`);
+  assert.deepEqual([noAgent.type, noAgent.tool, noAgent.min, noAgent.max], ['tool_used', 'Agent', '0', '0']);
+  const noPlan = grader(`evals/${GATE_CASE}/graders/no-plan-file.md`);
+  assert.deepEqual([noPlan.type, noPlan.path, noPlan.exists], ['file_exists', '**/plan.md', 'false']);
+});
+
+test('proevesett: the gate case fixture passes the sequencing gate and stops at the intent gate', () => {
+  const ws = mkdtempSync(join(tmpdir(), 'proevesett-'));
+  try {
+    const s = spawnSync('bash', [join(EVALS, GATE_CASE, 'scaffold.sh')], { cwd: ws, encoding: 'utf8' });
+    assert.equal(s.status, 0, s.stderr);
+    const brief = join(ws, 'proj', 'brief.md');
+    const v = spawnSync('node', [join(ROOT, 'lib/validators/brief-validator.mjs'), '--soft', '--json', brief], { encoding: 'utf8' });
+    assert.equal(v.status, 0, `brief-validator must pass so the run reaches the intent gate: ${v.stdout} ${v.stderr}`);
+    const g = spawnSync('node', [join(ROOT, 'lib/validators/intent-approval.mjs'), '--check', '--json', brief], { encoding: 'utf8' });
+    assert.equal(g.status, 1);
+    assert.deepEqual(JSON.parse(g.stdout).errors.map((e) => e.code), ['BRIEF_INTENT_NOT_APPROVED']);
+  } finally { rmSync(ws, { recursive: true, force: true }); }
+});
+
+// ---- case 5: the missing-flag error comes from code ----
+
+test('review-requires-project: the arg parser, run as trekreview.md spells it, prints the error (exit 1)', () => {
+  const lines = read('commands/trekreview.md').split('\n').filter((l) =>
+    /^\s*node \$\{CLAUDE_PLUGIN_ROOT\}\/lib\/parsers\/arg-parser\.mjs --command trekreview\b/.test(l));
+  assert.equal(lines.length, 1, 'trekreview.md must carry exactly one arg-parser line');
+  const line = lines[0].trim();
+  assert.match(line, /\$ARGUMENTS/, 'the parser must receive $ARGUMENTS (the Bash tool has no "$@")');
+  const run = (args) => spawnSync('bash', ['-c',
+    line.split('${CLAUDE_PLUGIN_ROOT}').join(ROOT).split('$ARGUMENTS').join(args)], { encoding: 'utf8' });
+  const none = run('');
+  assert.equal(none.status, 1, `missing --project must exit 1: ${none.stdout} ${none.stderr}`);
+  assert.deepEqual(JSON.parse(none.stdout).errors.map((e) => e.code), ['ARG_REQUIRED_MISSING']);
+  assert.match(none.stderr, /^Error: --project <dir> is required\.$/m);
+  assert.match(none.stderr, /^Usage: \/trekreview --project <dir>/m);
+  const ok = run('--project proj --validate');
+  assert.equal(ok.status, 0, ok.stdout + ok.stderr);
+  assert.equal(JSON.parse(ok.stdout).flags['--project'], 'proj');
+});
+
+test('review-requires-project: the prose does not compose the error itself; the grader reads the parser code', () => {
+  const prose = read('commands/trekreview.md');
+  assert.doesNotMatch(prose, /Error: --project <dir> is required/, 'the model must relay the parser, not compose the line');
+  assert.doesNotMatch(prose, /ARG_REQUIRED_MISSING/);
+  const gs = graders('review-requires-project');
+  assert.ok(gs.includes('parser-ran.md') && gs.includes('names-missing-project.md'), gs.join(','));
+  const n = grader('evals/review-requires-project/graders/names-missing-project.md');
+  assert.equal(n.target, 'trace');
+  const re = new RegExp(n.pattern);
+  assert.match(inTrace({ errors: [{ code: 'ARG_REQUIRED_MISSING' }] }), re);
+  assert.doesNotMatch(COMMAND_PROSE, re);
+});
+
+// ---- grader weaknesses ----
+
+test('known-negative no-error-code sees EVERY review-validator code (incl. REVIEW_WRONG_TYPE), bound to the output form', () => {
+  const src = read('lib/validators/review-validator.mjs');
+  const codes = [...new Set([...src.matchAll(/issue\('(REVIEW_[A-Z_]+)'/g)].map((m) => m[1]))];
+  assert.ok(codes.includes('REVIEW_WRONG_TYPE') && codes.includes('REVIEW_VERSION_FORMAT'), codes.join(','));
+  const g = grader('evals/review-validate-passes-clean-review/graders/no-error-code.md');
+  assert.equal(g.target, 'trace');
+  assert.equal(g.match, 'not_contains');
+  const re = new RegExp(g.pattern);
+  for (const code of codes) {
+    assert.match(inTrace({ valid: false, errors: [{ code }] }, 2), re, `${code} must be caught`);
+  }
+  assert.doesNotMatch(inTrace({ valid: true, errors: [], warnings: [] }, 2), re);
+  assert.doesNotMatch(COMMAND_PROSE, re);
+});
+
+test('says-pass / says-fail are not raw substrings: a negation does not pass them', () => {
+  const pass = new RegExp(grader('evals/review-validate-passes-clean-review/graders/says-pass.md').pattern);
+  const fail = new RegExp(grader('evals/review-validate-flags-bad-finding-id/graders/says-fail.md').pattern);
+  for (const s of ['review-validator: PASS proj/review.md', 'Result: **PASS**', 'PASS — review.md is valid']) assert.match(s, pass);
+  for (const s of ['The review did not PASS.', 'NOT PASS', 'PASSED_WITH_WARNINGS']) assert.doesNotMatch(s, pass);
+  for (const s of ['review-validator: FAIL proj/review.md', 'Result: **FAIL**']) assert.match(s, fail);
+  for (const s of ['It did not FAIL.', 'NOT FAIL', 'FAILOVER']) assert.doesNotMatch(s, fail);
+});
+
+test('every case with a no-write grader also has no-bash-write, and it sees a write through Bash', () => {
+  const withNoWrite = cases().filter((c) => graders(c).includes('no-write.md'));
+  assert.ok(withNoWrite.length >= 5, withNoWrite.join(','));
+  for (const c of withNoWrite) {
+    assert.ok(graders(c).includes('no-bash-write.md'), `${c}: no-write sees only the Write tool`);
+    const g = grader(`evals/${c}/graders/no-bash-write.md`);
+    assert.deepEqual([g.type, g.tool, g.min, g.max], ['tool_used', 'Bash', '0', '0'], c);
+    const re = new RegExp(g.input_match);
+    for (const cmd of [
+      "cat > proj/review.md <<'EOF'\nx\nEOF",
+      'echo x >> proj/review.md',
+      'printf x>proj/plan.md',
+      'node -e 1 | tee proj/out.txt',
+      "sed -i '' 's/a/b/' proj/review.md",
+      'rm proj/review.md',
+    ]) assert.match(bashInput(cmd), re, `${c}: must see ${cmd}`);
+    for (const cmd of [
+      'node /p/lib/validators/review-validator.mjs --json "proj/review.md"',
+      'node /p/lib/validators/review-validator.mjs --json proj/review.md 2>&1',
+      'ls proj 2>/dev/null',
+      'node /p/lib/parsers/arg-parser.mjs --command trekreview -- ',
+      'cat proj/review.md | head -5',
+    ]) assert.doesNotMatch(bashInput(cmd), re, `${c}: must not flag ${cmd}`);
+  }
+});