voyage/tests/lib/skills-criteria-prose-pins.test.mjs

// tests/lib/skills-criteria-prose-pins.test.mjs
// Wave 3, order 7c: the `writing-criteria` skill and its wiring (contract S3,
// and S4 on the two agents order 7c owns).
//
// What is pinned, and why:
//   • the skill's form is not pinned here: the form check
//     (lib/skills/skill-lint.mjs) and the routing test (lib/skills/routing.mjs)
//     run over every skill in `npm test` and are the authority (contract S1);
//   • every example the skill marks `Runs:` is accepted by the criteria
//     runner, every `Refused:` is refused, and every `Never red:` is accepted
//     (the runner cannot see that it never fails; the skill says so) — so the
//     skill can never teach a criterion the runner refuses;
//   • brief-reviewer and plan-critic preload both review skills, and
//     /trekplan names writing-criteria where the plan is written and both
//     skills where it starts its three reviewers;
//   • the criteria text the skill replaced is gone from the agents, the
//     templates and /trekplan (measured 17.10 on 31c77fd; one copy left);
//   • every runnable example in the two templates is accepted by the runner.
//
// The behaviour itself (writing runnable criteria; a reviewer naming one that
// cannot go red) was measured with helpers in the authoring session and is not
// something a model-free test can see. The eval cases under
// evals/skill-writing-criteria-* carry it for the operator's own runs.
// Counted as doc pins by lib/util/test-census.mjs (file name *-prose-pins).

import { test } from 'node:test';
import { strict as assert } from 'node:assert';
import { existsSync, readdirSync, readFileSync } from 'node:fs';
import { dirname, join } from 'node:path';
import { fileURLToPath } from 'node:url';
import { firstCommand, runnableCheck } from '../../lib/verification/criteria-runner.mjs';
import { readFrontmatterFields } from '../../lib/skills/skill-frontmatter.mjs';

const HERE = dirname(fileURLToPath(import.meta.url));
const ROOT = join(HERE, '..', '..');
const read = (rel) => readFileSync(join(ROOT, rel), 'utf8');

const SKILL = 'skills/writing-criteria/SKILL.md';
const BOTH = ['voyage:writing-criteria', 'voyage:reviewing-adversarially'];

function section(text, startRe, endRe) {
  const start = text.search(startRe);
  assert.ok(start >= 0, `section ${startRe} not found`);
  const rest = text.slice(start);
  const end = rest.slice(1).search(endRe);
  return end < 0 ? rest : rest.slice(0, end + 1);
}

const runs = (span) => runnableCheck(firstCommand(`\`${span}\``), { cwd: ROOT }).runnable;

/**
 * The examples a text marks, and each one that the runner judges otherwise
 * than its mark says. `Runs:` and `Never red:` must be accepted, `Refused:`
 * refused.
 */
export function markedExamples(text) {
  const out = [];
  for (const m of text.matchAll(/^- (Runs|Refused|Never red): `([^`]+)`/gm)) {
    const runnable = runs(m[2]);
    const want = m[1] !== 'Refused';
    out.push({ mark: m[1], command: m[2], runnable, wrong: runnable !== want });
  }
  return out;
}

// --- the skill's examples go through the criteria runner ---------------------

test('writing-criteria: every Runs:/Never red: example is accepted and every Refused: example refused', () => {
  const ex = markedExamples(read(SKILL));
  const count = (mark) => ex.filter((e) => e.mark === mark).length;
  assert.ok(count('Runs') >= 5, `need >= 5 Runs: examples, have ${count('Runs')}`);
  assert.ok(count('Refused') >= 5, `need >= 5 Refused: examples, have ${count('Refused')}`);
  assert.ok(count('Never red') >= 1, 'the skill shows at least one runnable check that can never go red');
  assert.deepEqual(ex.filter((e) => e.wrong), [], 'an example marked otherwise than the runner judges it');
});

test('writing-criteria: the example check is red on a stub that marks a pipe as runnable', () => {
  const stub = '- Runs: `npm test`\n- Runs: `node bin/app.mjs list --json | jq length`\n- Refused: `npm test && npm run lint`\n';
  const wrong = markedExamples(stub).filter((e) => e.wrong).map((e) => e.command);
  assert.deepEqual(wrong, ['node bin/app.mjs list --json | jq length']);
  // And the other way round: a runnable command marked refused is caught too.
  assert.deepEqual(markedExamples('- Refused: `npm test`\n').filter((e) => e.wrong).length, 1);
});

test('writing-criteria: the measured red-baseline failures are named in the skill', () => {
  const text = read(SKILL);
  assert.match(text, /\*\*One criterion, one command\.\*\*/);
  assert.match(text, /\*\*The command is the first backticked span on the line\.\*\*/);
  assert.match(text, /\*\*A check that needs more is a script\.\*\*/);
  assert.match(text, /\*\*It must be able to go red\.\*\*/);
  assert.match(text, /`bash tests\/<name>\.sh`/);
});

test('writing-criteria: "Enforced by" names the runner and the validators, and says go-red is prose only', () => {
  const enforced = section(read(SKILL), /^## Enforced by/m, /^## /m);
  assert.match(enforced, /lib\/verification\/criteria-runner\.mjs/);
  assert.match(enforced, /deterministic/);
  assert.match(enforced, /lib\/validators\/plan-validator\.mjs/);
  assert.match(enforced, /lib\/validators\/brief-validator\.mjs/);
  assert.match(enforced, /prose only: that a criterion can go red and that it tests what the brief asks for/);
  assert.match(enforced, /`brief-reviewer`\s+and `plan-critic` judge that by reading/);
});

test('writing-criteria: credits say it is Voyage\'s own content, nothing borrowed', () => {
  const credits = section(read(SKILL), /^## Credits/m, /^## /m);
  assert.match(credits, /Voyage's own content; nothing is borrowed/);
  assert.doesNotMatch(credits, /\bMIT\b|@[0-9a-f]{7}/);
});

// --- wiring: two agents preload it, /trekplan names it -----------------------

test('brief-reviewer and plan-critic preload writing-criteria and reviewing-adversarially (S3 + S4), tools unchanged', () => {
  for (const a of ['brief-reviewer', 'plan-critic']) {
    const fm = readFrontmatterFields(read(`agents/${a}.md`)).fields;
    assert.deepEqual(fm.skills?.value, BOTH, `agents/${a}.md skills:`);
    assert.deepEqual(fm.tools.value, ['Read', 'Glob', 'Grep'], `agents/${a}.md keeps its read-only tools`);
  }
});

test('/trekplan names writing-criteria where the plan is written', () => {
  const p8 = section(read('commands/trekplan.md'), /^## Phase 8 — Deep planning/m, /^## Phase 9/m);
  assert.match(p8, /voyage:writing-criteria/);
});

test('/trekplan names both skills where it starts brief-reviewer, plan-critic and scope-guardian', () => {
  const cmd = read('commands/trekplan.md');
  const p4b = section(cmd, /^## Phase 4b — Brief review/m, /^## Phase 5/m);
  const p9 = section(cmd, /^Both reviewers are asked for/m, /^After both complete/m);
  // The plugin-qualified names stand where the agents are started ...
  for (const [name, text] of [['brief-reviewer', p4b], ['plan-critic and scope-guardian', p9]]) {
    for (const s of BOTH) assert.ok(text.includes(s), `/trekplan must name ${s} where it starts ${name}`);
  }
  // ... and each of the three prompts asks for both skills itself.
  const prompts = [
    ['brief-reviewer', p4b.split('Prompt:')[1]],
    ['plan-critic', p9.split('**plan-critic**')[1].split('**scope-guardian**')[0]],
    ['scope-guardian', p9.split('**scope-guardian**')[1]],
  ];
  for (const [name, text] of prompts) {
    assert.match(text, /Follow the writing-criteria skill[^(]*\([^)]*go\s+red[^)]*\)/, `${name}: its prompt asks for writing-criteria, with the rule`);
    assert.match(text, /the\s+reviewing-adversarially\s+skill/, `${name}: its prompt asks for reviewing-adversarially`);
  }
});

// --- the copied text is gone ------------------------------------------------

// Measured 17.10 on 31c77fd: these sentences carried the runner's rules for
// writing a criterion. Each now lives in the skill; a copy coming back is drift.
const MOVED = [
  ['agents/brief-reviewer.md', '**Runnable at review.** `/trekreview` runs a criterion\'s command only when it'],
  ['agents/brief-reviewer.md', 'Anything else — the product\'s own binary (`node bin/x.mjs …`), `jq`, `grep`,'],
  ['agents/brief-reviewer.md', 'Never PROPOSE such a command as a'],
  ['agents/plan-critic.md', '(K10: one backticked command the criteria runner allows,'],
  ['templates/trekbrief-template.md', 'One command per criterion, from the runner\'s allowlist'],
  ['templates/trekbrief-template.md', 'in plain words (it runs with no shell: no quote, `$` or glob outside `node --test`).'],
  ['templates/plan-template.md', 'runner allows, no pipe and no `&&` — usually `npm test` — that the executor'],
  ['templates/plan-template.md', '`{a command the criteria runner allows that runs the product the way a user does'],
  ['commands/trekplan.md', 'The command must also be one the criteria runner is allowed to run: a test'],
  ['commands/trekplan.md', 'grep`) — with no pipe, redirect or `&&`. It runs with no shell: its words'],
  ['commands/trekplan.md', 'takes a short list of read-only options, and nothing may follow'],
];

test('the criteria text that moved into the skill is no longer copied into the agents, templates and /trekplan', () => {
  const back = MOVED.filter(([f, s]) => read(f).includes(s));
  assert.deepEqual(back, [], 'copied criteria text is back — it belongs to the skill');
});

test('brief-reviewer and plan-critic keep their own role and point at the preloaded skill', () => {
  assert.match(read('agents/brief-reviewer.md'), /preloaded\s+`writing-criteria` skill/);
  assert.match(read('agents/brief-reviewer.md'), /BRIEF_SC_NOT_RUNNABLE/, 'flagging an unrunnable criterion stays its job');
  const d8 = section(read('agents/plan-critic.md'), /^### 8\. Verification gaps/m, /^### 9\./m);
  assert.match(d8, /preloaded\s+`writing-criteria` skill/);
  assert.match(d8, /can never go red/, 'a criterion that cannot go red is a finding');
});

// --- the templates' examples run --------------------------------------------

// A backticked span inside a criterion line that is no placeholder (`{…}`) is
// an example the template teaches: the runner must accept it.
function templateExamples(text) {
  const out = [];
  for (const line of text.split('\n')) {
    if (!/^\s*-\s|\*\*(Verify|Regression|End-to-end):\*\*/.test(line)) continue;
    for (const m of line.matchAll(/`([^`]+)`/g)) {
      const span = m[1].trim();
      if (/^\{.*\}$/.test(span) || !firstCommand(`\`${span}\``).command) continue;
      out.push(span);
    }
  }
  return out;
}

test('every runnable example in the brief template\'s Success Criteria is accepted by the criteria runner', () => {
  const sc = section(read('templates/trekbrief-template.md'), /^## Success Criteria/m, /^## /m);
  const ex = templateExamples(sc);
  assert.ok(ex.length >= 3, `found ${ex.length}: ${JSON.stringify(ex)}`);
  assert.deepEqual(ex.filter((s) => !runs(s)), []);
});

test('every runnable example in the plan template\'s steps and Verification is accepted by the criteria runner', () => {
  const t = read('templates/plan-template.md');
  const steps = section(t, /^## Implementation Plan/m, /^### Manifest — objective completion predicate/m);
  const ver = section(t, /^## Verification/m, /^## /m);
  const ex = [...templateExamples(steps), ...templateExamples(ver)]
    .filter((s) => !/^(git add|path\/|`)/.test(s) && !/\.(ts|mjs|js|md)$/.test(s));
  // Measured 17.10: every check in the template was a placeholder; `Regression:`
  // now shows the real command it names ("usually `npm test`").
  assert.ok(ex.includes('npm test'), `found ${ex.length}: ${JSON.stringify(ex)}`);
  assert.deepEqual(ex.filter((s) => !runs(s)), []);
});

// --- the eval cases ---------------------------------------------------------

test('writing-criteria eval cases: 3 fire, 2 stay quiet (test-first and running the app), 1 effect case', () => {
  const dirs = readdirSync(join(ROOT, 'evals')).filter((d) => d.startsWith('skill-writing-criteria-'));
  const fires = dirs.filter((d) => /-fires-\d$/.test(d));
  const quiet = dirs.filter((d) => /-quiet-\d$/.test(d));
  const effect = dirs.filter((d) => d.endsWith('-effect'));
  assert.equal(fires.length, 3);
  assert.equal(quiet.length, 2);
  assert.equal(effect.length, 1);
  for (const d of dirs) assert.ok(existsSync(join(ROOT, 'evals', d, 'prompt.md')), `${d}/prompt.md`);
  const owners = quiet.map((d) => read(`evals/${d}/prompt.md`).match(/owner=([\w:-]+)/)?.[1]).sort();
  assert.ok(owners.includes('test-first'), 'one quiet case belongs to test-first ("write a test")');
  const quietText = quiet.map((d) => read(`evals/${d}/prompt.md`)).join('\n');
  assert.match(quietText, /\b(start|run)\b[^\n]*\bapp\b/i, 'one quiet case is "run the app" (Claude Code\'s run)');
});