voyage/tests/lib/skill-routing.test.mjs

// tests/lib/skill-routing.test.mjs
// The model-free routing test (P10, "Bygg nå"): lib/skills/routing.mjs ranks
// the descriptions of every skill, the 7 commands and the 21 spawnable agents
// against prompts, with no model. A collision between two descriptions fails;
// each skill needs 3 prompts where it should fire (top 3) and 2 where it
// should not, and for those a named owner must rank higher. The thresholds are
// pinned here: they are never lowered.

import { test } from 'node:test';
import { strict as assert } from 'node:assert';
import { spawnSync } from 'node:child_process';
import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { dirname, join } from 'node:path';
import { fileURLToPath } from 'node:url';
import {
  THRESHOLDS, tokenize, buildCorpus, rank, collisions, loadEntries, loadCases, checkRouting,
} from '../../lib/skills/routing.mjs';

const HERE = dirname(fileURLToPath(import.meta.url));
const ROOT = join(HERE, '..', '..');
const CLI = join(ROOT, 'lib', 'skills', 'routing.mjs');

const skillMd = (name, description) =>
  `---\nname: ${name}\ndescription: ${description}\n---\n\n# ${name}\n`;

/** Plugin-shaped temp root: skills, commands, agents, evals. */
function plugin({ skills = {}, commands = {}, agents = {}, cases = {} }) {
  const root = mkdtempSync(join(tmpdir(), 'skill-routing-'));
  for (const [n, d] of Object.entries(skills)) {
    mkdirSync(join(root, 'skills', n), { recursive: true });
    writeFileSync(join(root, 'skills', n, 'SKILL.md'), skillMd(n, d));
  }
  mkdirSync(join(root, 'commands'), { recursive: true });
  for (const [n, d] of Object.entries(commands)) {
    writeFileSync(join(root, 'commands', `${n}.md`), `---\nname: ${n}\ndescription: ${d}\n---\n`);
  }
  mkdirSync(join(root, 'agents'), { recursive: true });
  for (const [n, d] of Object.entries(agents)) {
    writeFileSync(join(root, 'agents', `${n}.md`), `---\nname: ${n}\ndescription: |\n  ${d}\nmodel: opus\n---\n`);
  }
  for (const [n, { tags, prompt }] of Object.entries(cases)) {
    mkdirSync(join(root, 'evals', n, 'graders'), { recursive: true });
    writeFileSync(join(root, 'evals', n, 'prompt.md'),
      `---\ndescription: "case ${n}"\ntags: [${tags.join(', ')}]\nruns: 1\n---\n\n${prompt}\n`);
  }
  return root;
}

function withPlugin(spec, fn) {
  const root = plugin(spec);
  try { return fn(root); } finally { rmSync(root, { recursive: true, force: true }); }
}

const codes = (r) => r.errors.map((e) => e.code);

// ---- the test that fails first (ordre 7a) ----

test('routing: two test skills with near-identical descriptions are red (collision)', () => {
  withPlugin({
    skills: {
      'tidy-imports': 'Sorts and groups the import lines of a source file. Use when imports are out of order.',
      'order-imports': 'Sorts and groups the import lines in a source file. Use when the imports are out of order.',
    },
  }, (root) => {
    const r = checkRouting(root);
    assert.ok(codes(r).includes('ROUTING_COLLISION'), JSON.stringify(r.errors));
    const c = r.errors.find((e) => e.code === 'ROUTING_COLLISION');
    assert.deepEqual([c.a, c.b].sort(), ['skill:order-imports', 'skill:tidy-imports']);
  });
});

test('routing: distinct descriptions do not collide (known negative)', () => {
  const corpus = buildCorpus([
    { id: 'skill:tidy-imports', name: 'tidy-imports', description: 'Sorts the import lines of a source file. Use when imports are out of order.' },
    { id: 'skill:rotate-logs', name: 'rotate-logs', description: 'Compresses and deletes old log files on disk. Use when the log directory grows.' },
  ]);
  assert.deepEqual(collisions(corpus).filter((c) => c.level === 'error'), []);
});

// ---- the pieces ----

test('routing: thresholds are pinned and never lowered', () => {
  assert.deepEqual({ ...THRESHOLDS }, {
    COLLISION_ERROR: 0.75,
    COLLISION_WARN: 0.5,
    POSITIVE_TOP_K: 3,
    MIN_POSITIVE: 3,
    MIN_NEGATIVE: 2,
    MIN_EFFECT: 1,
    MIN_RANK1_RATE: 1,
  });
  assert.ok(Object.isFrozen(THRESHOLDS));
});

test('routing: the tokenizer folds case, drops stop words and light-stems', () => {
  assert.deepEqual(tokenize('Use when the Tests are FAILING'), tokenize('test fail'));
  assert.deepEqual(tokenize('a an the of'), []);
});

test('routing: rank puts the matching description first, and a no-overlap prompt scores 0', () => {
  const corpus = buildCorpus([
    { id: 'skill:tidy-imports', name: 'tidy-imports', description: 'Sorts the import lines of a source file.' },
    { id: 'skill:rotate-logs', name: 'rotate-logs', description: 'Compresses and deletes old log files on disk.' },
  ]);
  assert.equal(rank('the log files fill the disk', corpus)[0].id, 'skill:rotate-logs');
  assert.ok(rank('zebra quantum', corpus).every((r) => r.score === 0));
});

test('routing: the reader takes block-scalar agent descriptions and skips orchestrator reference docs', () => {
  withPlugin({
    agents: { 'plan-critic': 'Use this agent when a plan needs adversarial review.' },
    commands: { trekplan: 'Deep implementation planning from a task brief.' },
  }, (root) => {
    writeFileSync(join(root, 'agents', 'planning-orchestrator.md'),
      '---\nname: planning-orchestrator\ndescription: Reference document, not a spawnable capability.\n---\n');
    const ids = loadEntries(root).map((e) => e.id).sort();
    assert.deepEqual(ids, ['agent:plan-critic', 'command:trekplan']);
    assert.equal(loadEntries(root).find((e) => e.id === 'agent:plan-critic').description.trim(),
      'Use this agent when a plan needs adversarial review.');
  });
});

test('routing: cases come from evals/ tags; a skill below 3 / 2 / 1 is red', () => {
  withPlugin({
    skills: { 'tidy-imports': 'Sorts the import lines of a source file. Use when imports are out of order.' },
    cases: {
      'skill-tidy-imports-fires-1': { tags: ['skill-trigger', 'skill=tidy-imports', 'from=test'], prompt: 'sort the import lines in main.js' },
    },
  }, (root) => {
    const cases = loadCases(root);
    assert.equal(cases.length, 1);
    assert.deepEqual([cases[0].kind, cases[0].skill, cases[0].from], ['trigger', 'tidy-imports', 'test']);
    assert.ok(codes(checkRouting(root)).includes('ROUTING_BELOW_MINIMUM'));
  });
});

test('routing: a positive outside the top 3 is red; a negative won by the skill itself is red', () => {
  const spec = {
    skills: { 'tidy-imports': 'Sorts the import lines of a source file. Use when imports are out of order.' },
    commands: {
      'rotate-logs': 'Compresses and deletes old log files on disk.',
      'scan-ports': 'Lists the open network ports of a host.',
      'count-words': 'Counts the words of a text document.',
    },
    cases: {
      'p1': { tags: ['skill-trigger', 'skill=tidy-imports', 'from=test'], prompt: 'sort the import lines' },
      'p2': { tags: ['skill-trigger', 'skill=tidy-imports', 'from=test'], prompt: 'import lines are out of order' },
      'p3': { tags: ['skill-trigger', 'skill=tidy-imports', 'from=test'], prompt: 'compress the old logs on disk and list ports and count words' },
      'n1': { tags: ['skill-no-trigger', 'skill=tidy-imports', 'owner=rotate-logs', 'from=test'], prompt: 'delete old log files' },
      'n2': { tags: ['skill-no-trigger', 'skill=tidy-imports', 'owner=rotate-logs', 'from=test'], prompt: 'sort the import lines of the log file' },
      'e1': { tags: ['skill-effect', 'skill=tidy-imports', 'from=test'], prompt: 'x' },
    },
  };
  withPlugin(spec, (root) => {
    const r = checkRouting(root);
    assert.ok(r.errors.some((e) => e.code === 'ROUTING_POSITIVE_MISSED' && e.case === 'p3'), JSON.stringify(r.errors));
    assert.ok(r.errors.some((e) => e.code === 'ROUTING_NEGATIVE_WON' && e.case === 'n2'), JSON.stringify(r.errors));
    assert.ok(!r.errors.some((e) => e.case === 'n1'), JSON.stringify(r.errors));
  });
});

test('routing: a negative must name an owner that exists and outranks the skill', () => {
  withPlugin({
    skills: { 'tidy-imports': 'Sorts the import lines of a source file. Use when imports are out of order.' },
    cases: {
      'n1': { tags: ['skill-no-trigger', 'skill=tidy-imports', 'from=test'], prompt: 'delete old log files' },
      'n2': { tags: ['skill-no-trigger', 'skill=tidy-imports', 'owner=nobody', 'from=test'], prompt: 'delete old log files' },
    },
  }, (root) => {
    const r = checkRouting(root);
    assert.ok(r.errors.some((e) => e.code === 'ROUTING_NEGATIVE_NO_OWNER' && e.case === 'n1'), JSON.stringify(r.errors));
    assert.ok(r.errors.some((e) => e.code === 'ROUTING_UNKNOWN_OWNER' && e.case === 'n2'), JSON.stringify(r.errors));
  });
});

// ---- R2: an owner from Claude Code's own skills (wave 4, order A2) ----

const R2_SPEC = (prompt, owner = 'run') => ({
  skills: { 'tidy-imports': 'Sorts the import lines of a source file. Use when imports are out of order.' },
  commands: { 'rotate-logs': 'Compresses and deletes old log files on disk.' },
  cases: { n1: { tags: ['skill-no-trigger', 'skill=tidy-imports', `owner=${owner}`, 'from=test'], prompt } },
});

test('routing (R2): owner=run, a built-in Claude Code skill, is no unknown owner, and the report says what was checked', () => {
  withPlugin(R2_SPEC('start the app and click through the new screen'), (root) => {
    const r = checkRouting(root);
    assert.ok(!r.errors.some((e) => e.case === 'n1'), JSON.stringify(r.errors));
    const note = (r.notes || []).find((n) => n.case === 'n1');
    assert.ok(note, JSON.stringify(r));
    assert.match(note.message, /owner outside the corpus: only "not taken" checked/);
    const out = spawnSync('node', [CLI, '--root', root], { encoding: 'utf8' });
    assert.match(out.stdout, /note ROUTING_OWNER_OUTSIDE_CORPUS: n1: owner=run [^\n]*owner outside the corpus: only "not taken" checked/, out.stdout);
  });
});

test('routing (R2): a skill that ranks first is still red when the owner is run', () => {
  withPlugin(R2_SPEC('sort the import lines of the source file'), (root) => {
    const r = checkRouting(root);
    assert.ok(r.errors.some((e) => e.code === 'ROUTING_NEGATIVE_WON' && e.case === 'n1'), JSON.stringify(r.errors));
  });
});

test('routing (R2): an owner neither in the corpus nor on the built-in list is still unknown', () => {
  withPlugin(R2_SPEC('start the app and click through the new screen', 'nosuchthing'), (root) => {
    const r = checkRouting(root);
    assert.ok(r.errors.some((e) => e.code === 'ROUTING_UNKNOWN_OWNER' && e.case === 'n1'), JSON.stringify(r.errors));
  });
});

test('routing: every case names the event it came from (P10)', () => {
  withPlugin({
    skills: { 'tidy-imports': 'Sorts the import lines of a source file. Use when imports are out of order.' },
    cases: { 'p1': { tags: ['skill-trigger', 'skill=tidy-imports'], prompt: 'sort the import lines' } },
  }, (root) => assert.ok(codes(checkRouting(root)).includes('ROUTING_CASE_NO_SOURCE')));
});

test('routing CLI: 0 clean, 1 findings, 2 could not run', () => {
  withPlugin({
    skills: {
      'tidy-imports': 'Sorts and groups the import lines of a source file. Use when imports are out of order.',
      'order-imports': 'Sorts and groups the import lines in a source file. Use when the imports are out of order.',
    },
  }, (root) => {
    const b = spawnSync('node', [CLI, '--root', root, '--json'], { encoding: 'utf8' });
    assert.equal(b.status, 1, b.stdout + b.stderr);
    assert.ok(JSON.parse(b.stdout).errors.length > 0);
    const n = spawnSync('node', [CLI, '--root', join(root, 'nope')], { encoding: 'utf8' });
    assert.equal(n.status, 2, n.stdout + n.stderr);
  });
});

// ---- Voyage's own catalogue ----

test('routing: Voyage\'s catalogue holds every skill, the 7 commands and the 21 spawnable agents', () => {
  const entries = loadEntries(ROOT);
  const by = (k) => entries.filter((e) => e.kind === k);
  assert.equal(by('command').length, 7, by('command').map((e) => e.name).join(','));
  assert.equal(by('agent').length, 21, by('agent').map((e) => e.name).join(','));
  assert.ok(by('skill').length >= 2, by('skill').map((e) => e.name).join(','));
  for (const e of entries) assert.ok(e.description.trim().length > 20, `${e.id}: description read as "${e.description}"`);
  assert.equal(new Set(entries.map((e) => e.name)).size, entries.length, 'names must be unique across kinds');
});

test('routing: Voyage\'s catalogue is green: no collision, every case routes, rank-1 floor held', () => {
  const r = checkRouting(ROOT);
  assert.deepEqual(r.errors, [], JSON.stringify(r.errors, null, 2));
  assert.ok(r.stats.positives >= 6 && r.stats.negatives >= 4, JSON.stringify(r.stats));
});