voyage/tests/lib/p10-model-free.test.mjs

// tests/lib/p10-model-free.test.mjs
// P10's model-free part (wave 4, order A1). Every case in evals/ names the
// event it came from (`from=`), not only the skill cases; the routing check
// covers every case and says how many it checked for `from=` and how many it
// routed; and the release gate that would read the last test-set run is
// written down as exempted with a reason, not built.

import { test } from 'node:test';
import { strict as assert } from 'node:assert';
import { spawnSync } from 'node:child_process';
import { mkdtempSync, mkdirSync, writeFileSync, readdirSync, readFileSync, existsSync, rmSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { dirname, join } from 'node:path';
import { fileURLToPath } from 'node:url';
import { readFrontmatterFields } from '../../lib/skills/skill-frontmatter.mjs';
import { checkRouting } from '../../lib/skills/routing.mjs';

const ROOT = join(dirname(fileURLToPath(import.meta.url)), '..', '..');
const CLI = join(ROOT, 'lib', 'skills', 'routing.mjs');

const tagsOf = (file) => {
  const f = readFrontmatterFields(readFileSync(file, 'utf8')).fields.tags;
  return f && Array.isArray(f.value) ? f.value.map(String) : [];
};

test('P10: every case in evals/ names the event it came from (from=)', () => {
  const dirs = readdirSync(join(ROOT, 'evals')).filter((d) => existsSync(join(ROOT, 'evals', d, 'prompt.md'))).sort();
  assert.ok(dirs.length >= 45, `${dirs.length} cases`);
  const missing = dirs.filter((d) => !tagsOf(join(ROOT, 'evals', d, 'prompt.md')).some((t) => /^from=\S+$/.test(t)));
  assert.deepEqual(missing, [], `${missing.length} of ${dirs.length} cases have no from=: ${missing.join(', ')}`);
});

/** A plugin root with one skill, its 3/2/1 cases, and the extra cases given. */
function stubRoot(extra) {
  const root = mkdtempSync(join(tmpdir(), 'p10-model-free-'));
  mkdirSync(join(root, 'skills', 'tidy-imports'), { recursive: true });
  writeFileSync(join(root, 'skills', 'tidy-imports', 'SKILL.md'),
    '---\nname: tidy-imports\ndescription: Sorts the import lines of a source file. Use when imports are out of order.\n---\n');
  mkdirSync(join(root, 'commands'), { recursive: true });
  writeFileSync(join(root, 'commands', 'rotate-logs.md'), '---\nname: rotate-logs\ndescription: Compresses and deletes old log files on disk.\n---\n');
  const cases = {
    p1: ['skill-trigger, skill=tidy-imports, from=test', 'sort the import lines'],
    p2: ['skill-trigger, skill=tidy-imports, from=test', 'import lines are out of order'],
    p3: ['skill-trigger, skill=tidy-imports, from=test', 'sort imports in the source file'],
    n1: ['skill-no-trigger, skill=tidy-imports, owner=rotate-logs, from=test', 'delete old log files'],
    n2: ['skill-no-trigger, skill=tidy-imports, owner=rotate-logs, from=test', 'compress the logs on disk'],
    e1: ['skill-effect, skill=tidy-imports, from=test', 'x'],
    ...extra,
  };
  for (const [n, [tags, prompt]] of Object.entries(cases)) {
    mkdirSync(join(root, 'evals', n), { recursive: true });
    writeFileSync(join(root, 'evals', n, 'prompt.md'), `---\ndescription: "case ${n}"\ntags: [${tags}]\nruns: 1\n---\n\n${prompt}\n`);
  }
  return root;
}

test('P10 (R1): a case with no skill tag and no from= is a finding of the routing check', () => {
  const root = stubRoot({ 'plain-gate': ['proevesett', '/voyage:trekplan'], 'plain-sourced': ['proevesett, from=test', '/voyage:trekplan'] });
  try {
    const r = checkRouting(root);
    const src = r.errors.filter((e) => e.code === 'ROUTING_CASE_NO_SOURCE').map((e) => e.case);
    assert.deepEqual(src, ['plain-gate'], JSON.stringify(r.errors));
    assert.ok(!r.errors.some((e) => e.case === 'plain-sourced'), 'a sourced case with no skill tag is not routed');
  } finally { rmSync(root, { recursive: true, force: true }); }
});

test('P10 (R1): the report shows the denominators: cases checked for from= and cases routed', () => {
  const root = stubRoot({ 'plain-sourced': ['proevesett, from=test', '/voyage:trekplan'] });
  try {
    const r = checkRouting(root);
    assert.deepEqual([r.stats.sourced, r.stats.cases], [7, 6], JSON.stringify(r.stats));
    const out = spawnSync('node', [CLI, '--root', root], { encoding: 'utf8' });
    assert.match(out.stdout, /7 cases checked for from=, 6 routed/, out.stdout + out.stderr);
  } finally { rmSync(root, { recursive: true, force: true }); }
});

// Found by its words, not its heading: the section title is set at the release.
test('P10: the CHANGELOG says the eval gate is exempted, with the reason', () => {
  const md = readFileSync(join(ROOT, 'CHANGELOG.md'), 'utf8');
  assert.ok(/^- [^\n]*eval gate \(P10\)[^\n]*exempted[^\n]*test set runs again/m.test(md),
    'a CHANGELOG line must say the eval gate (P10) is exempted until the test set runs again');
});