voyage/lib/skills/routing.mjs

#!/usr/bin/env node
// lib/skills/routing.mjs
// The model-free routing test (praksisanalyse P10, "Bygg naa"; contract S1).
// Ranks the descriptions of every skill, command and spawnable agent against
// prompts, with no model, so a description that would steal or lose a prompt
// is caught in `npm test` instead of in a user's session.
//
//   - A collision between two descriptions (cosine >= COLLISION_ERROR) fails.
//   - Each skill has at least MIN_POSITIVE prompts where it should fire (it must
//     rank in the top POSITIVE_TOP_K with a score > 0), MIN_NEGATIVE where it
//     should not (it must not rank first, and a named owner must rank above it),
//     and MIN_EFFECT effect cases.
//   - The share of positives where the skill ranks first is held at or above
//     MIN_RANK1_RATE.
// The thresholds are pinned by tests/lib/skill-routing.test.mjs. They are
// never lowered: a failing case is fixed in the description, never in a number.
//
// The cases are the plugin-eval cases themselves (evals/<case>/prompt.md):
// one source for the operator's model-run eval and this model-free check.
// A case is tagged in its prompt.md frontmatter:
//   tags: [skill-trigger | skill-no-trigger | skill-effect, skill=<name>,
//          owner=<command|agent|skill name> (negatives), from=<event>]
// An owner may also be one of Claude Code's own skills on BUILTIN_OWNERS (R2);
// it stands outside the corpus, so only "the skill does not rank first" is checked.
// `from=` names the event the case came from (P10, R1): it is required on every
// case in evals/, and a case with no skill tag (the fixed test set) is not
// routed, only checked for `from=`.
//
// What this cannot see: how Claude Code really chooses. A ranking over word
// overlap is a proxy; the operator's eval run is the measurement.
//
// CLI: node lib/skills/routing.mjs [--root <plugin root>] [--json]
//   exit 0 = green · 1 = findings · 2 = could not run
//
// Adapted from addyosmani/agent-skills @9d0c60d, scripts/run-evals.js (tier 2:
// stemmed TF-IDF cosine, name weighted twice, top-3 positives, owner-ranked
// negatives, collision and rank-1 floors) - MIT, Copyright (c) 2025 Addy
// Osmani. Reimplemented, no code copied; the reader here takes multi-line
// (`description: |`) descriptions, which the original reads as "|".

import { existsSync, readdirSync, readFileSync, statSync } from 'node:fs';
import { dirname, join, resolve } from 'node:path';
import { fileURLToPath } from 'node:url';
import { descriptionOf, readFrontmatterFields } from './skill-frontmatter.mjs';
import { isMain } from '../util/is-main.mjs';

export const THRESHOLDS = Object.freeze({
  COLLISION_ERROR: 0.75,
  COLLISION_WARN: 0.5,
  POSITIVE_TOP_K: 3,
  MIN_POSITIVE: 3,
  MIN_NEGATIVE: 2,
  MIN_EFFECT: 1,
  MIN_RANK1_RATE: 1,
});

// ---- text ----

const STOP = new Set([
  'a', 'an', 'and', 'any', 'are', 'as', 'at', 'be', 'by', 'can', 'do', 'does', 'for', 'from', 'has', 'have',
  'in', 'into', 'is', 'it', 'its', 'of', 'on', 'or', 'so', 'than', 'that', 'the', 'their', 'them', 'then',
  'there', 'these', 'this', 'those', 'to', 'use', 'used', 'uses', 'using', 'was', 'were', 'what', 'when',
  'which', 'while', 'who', 'will', 'with', 'you', 'your', 'we', 'our', 'me', 'my', 'i', 'not', 'no', 'one',
  'all', 'each', 'every', 'only', 'also', 'just', 'via', 'per', 'agent', 'skill', 'command', 'should',
]);

function stem(t) {
  for (const suf of ['ations', 'ation', 'ings', 'ing', 'ers', 'ies', 'ied', 'ed', 'es', 'er', 'ly']) {
    if (t.length > suf.length + 3 && t.endsWith(suf)) {
      t = suf === 'ies' || suf === 'ied' ? `${t.slice(0, -suf.length)}y` : t.slice(0, -suf.length);
      break;
    }
  }
  if (t.length > 3 && t.endsWith('s') && !t.endsWith('ss')) t = t.slice(0, -1);
  if (t.length > 4 && t.endsWith('e')) t = t.slice(0, -1);
  if (t.length > 4 && t[t.length - 1] === t[t.length - 2] && !'aeiou'.includes(t[t.length - 1])) t = t.slice(0, -1);
  return t;
}

/** Lower-case word tokens, stop words dropped, light-stemmed. */
export function tokenize(text) {
  return String(text)
    .toLowerCase()
    .replace(/[^a-z0-9\s-]/g, ' ')
    .split(/[\s-]+/)
    .filter((t) => t.length > 2 && !STOP.has(t))
    .map(stem)
    .filter((t) => t.length > 2 && !STOP.has(t));
}

function termFreq(tokens) {
  const tf = new Map();
  for (const t of tokens) tf.set(t, (tf.get(t) || 0) + 1);
  return tf;
}

/**
 * @param {{id: string, name: string, description: string}[]} entries
 * @returns {{ docs: Map<string, Map<string, number>>, idf: (t: string) => number, entries: object[] }}
 */
export function buildCorpus(entries) {
  const docs = new Map();
  for (const e of entries) {
    const nameTokens = tokenize(e.name.replace(/[-:]/g, ' '));
    docs.set(e.id, termFreq([...nameTokens, ...nameTokens, ...tokenize(e.description)]));
  }
  const df = new Map();
  for (const tf of docs.values()) for (const term of tf.keys()) df.set(term, (df.get(term) || 0) + 1);
  const n = docs.size;
  const idf = (term) => Math.log(1 + n / (1 + (df.get(term) || 0)));
  // The corpus never changes: its vectors are built once, not per prompt or pair.
  const vecs = new Map([...docs].map(([id, tf]) => [id, vec(tf, idf)]));
  return { docs, idf, vecs, entries };
}

function vec(tf, idf) {
  const v = new Map();
  for (const [term, f] of tf) v.set(term, f * idf(term));
  return v;
}

function cosine(a, b) {
  let dot = 0; let na = 0; let nb = 0;
  for (const [t, w] of a) { na += w * w; const bw = b.get(t); if (bw) dot += w * bw; }
  for (const w of b.values()) nb += w * w;
  return na && nb ? dot / (Math.sqrt(na) * Math.sqrt(nb)) : 0;
}

/** Every entry scored against a prompt, best first (ties by id, so the order is stable). */
export function rank(prompt, corpus) {
  const pv = vec(termFreq(tokenize(prompt)), corpus.idf);
  const out = [];
  for (const [id, v] of corpus.vecs) out.push({ id, score: cosine(pv, v) });
  return out.sort((a, b) => b.score - a.score || a.id.localeCompare(b.id));
}

/** Pairs of descriptions too alike: level "error" at COLLISION_ERROR, "warn" at COLLISION_WARN. */
export function collisions(corpus, t = THRESHOLDS) {
  const ids = [...corpus.docs.keys()];
  const out = [];
  for (let i = 0; i < ids.length; i++) {
    for (let j = i + 1; j < ids.length; j++) {
      const sim = cosine(corpus.vecs.get(ids[i]), corpus.vecs.get(ids[j]));
      if (sim >= t.COLLISION_ERROR) out.push({ a: ids[i], b: ids[j], sim, level: 'error' });
      else if (sim >= t.COLLISION_WARN) out.push({ a: ids[i], b: ids[j], sim, level: 'warn' });
    }
  }
  return out;
}

// ---- loading ----

/** Orchestrator reference docs are not spawnable capabilities (CLAUDE.md, Inventory). */
const isReferenceDoc = (name, description) => /-orchestrator$/.test(name) || /^Reference document, not a spawnable/.test(description.trim());

/** Skills, commands and spawnable agents of a plugin root. */
export function loadEntries(root) {
  const entries = [];
  const skillsDir = join(root, 'skills');
  if (existsSync(skillsDir)) {
    for (const n of readdirSync(skillsDir).sort()) {
      const f = join(skillsDir, n, 'SKILL.md');
      if (!existsSync(f)) continue;
      const text = readFileSync(f, 'utf8');
      const fm = readFrontmatterFields(text).fields;
      const name = fm.name ? String(fm.name.value) : n;
      entries.push({ id: `skill:${name}`, kind: 'skill', name, description: descriptionOf(text) });
    }
  }
  for (const [kind, dir] of [['command', 'commands'], ['agent', 'agents']]) {
    const d = join(root, dir);
    if (!existsSync(d)) continue;
    for (const file of readdirSync(d).filter((x) => x.endsWith('.md')).sort()) {
      const text = readFileSync(join(d, file), 'utf8');
      const fm = readFrontmatterFields(text).fields;
      const name = fm.name ? String(fm.name.value) : file.replace(/\.md$/, '');
      const description = descriptionOf(text);
      if (kind === 'agent' && isReferenceDoc(name, description)) continue;
      entries.push({ id: `${kind}:${name}`, kind, name, description });
    }
  }
  return entries;
}

const KIND_TAGS = { 'skill-trigger': 'trigger', 'skill-no-trigger': 'no-trigger', 'skill-effect': 'effect' };

/**
 * Every eval case of a plugin root (evals/<case>/prompt.md), routed or not.
 * `kind` is null for a case with no skill tag (the fixed test set): it is not
 * routed, only checked for `from=` (P10, R1).
 */
export function loadAllCases(root) {
  const d = join(root, 'evals');
  if (!existsSync(d)) return [];
  const out = [];
  for (const name of readdirSync(d).sort()) {
    const f = join(d, name, 'prompt.md');
    if (!existsSync(f) || !statSync(join(d, name)).isDirectory()) continue;
    const text = readFileSync(f, 'utf8');
    const tagsField = readFrontmatterFields(text).fields.tags;
    const tags = tagsField && Array.isArray(tagsField.value) ? tagsField.value.map(String) : [];
    const kindTag = tags.find((t) => Object.hasOwn(KIND_TAGS, t));
    const val = (k) => { const t = tags.find((x) => x.startsWith(`${k}=`)); return t ? t.slice(k.length + 1) : null; };
    const body = text.replace(/^---\n[\s\S]*?\n---\n?/, '').trim();
    out.push({ case: name, kind: kindTag ? KIND_TAGS[kindTag] : null, skill: val('skill'), owner: val('owner'), from: val('from'), prompt: body });
  }
  return out;
}

/** The routing-tagged eval cases of a plugin root. */
export function loadCases(root) {
  return loadAllCases(root).filter((c) => c.kind);
}

// ---- the check ----

/**
 * Claude Code's own skills a negative case may name as its owner (R2). They
 * stand outside the corpus, so for such an owner only "not taken" is checked:
 * the skill under test must not rank first. Kept short and fixed.
 *   run - the bundled "Launch and drive your app to see a change working"
 *         skill, seen in Claude Code 2.1.303 (code.claude.com/docs/en/skills,
 *         "Run and verify your app").
 */
export const BUILTIN_OWNERS = Object.freeze(['run']);

/**
 * Run the whole routing check over a plugin root.
 * @returns {{ errors: object[], warnings: object[], stats: object }}
 */
export function checkRouting(root, t = THRESHOLDS) {
  const entries = loadEntries(root);
  const corpus = buildCorpus(entries);
  const skills = entries.filter((e) => e.kind === 'skill');
  // A skill is looked up among skills only: an agent or command with the same
  // name must not take its place. An owner may be any kind; a skill wins a tie.
  const skillByName = new Map(skills.map((e) => [e.name, e]));
  const byName = new Map([...entries.filter((e) => e.kind !== 'skill'), ...skills].map((e) => [e.name, e]));
  const all = loadAllCases(root);
  const cases = all.filter((c) => c.kind);
  const errors = [];
  const warnings = [];
  const notes = [];
  let positives = 0; let rank1 = 0; let negatives = 0;

  for (const c of collisions(corpus, t)) {
    const entry = { code: c.level === 'error' ? 'ROUTING_COLLISION' : 'ROUTING_OVERLAP', a: c.a, b: c.b, sim: Number(c.sim.toFixed(3)),
      message: `${c.a} and ${c.b}: descriptions ${(c.sim * 100).toFixed(0)}% alike` };
    (c.level === 'error' ? errors : warnings).push(entry);
  }

  for (const c of all) {
    if (!c.from) errors.push({ code: 'ROUTING_CASE_NO_SOURCE', case: c.case, message: `${c.case}: tag the event the case came from (from=...)` });
  }

  for (const c of cases) {
    const self = c.skill && skillByName.get(c.skill);
    if (!self || self.kind !== 'skill') {
      errors.push({ code: 'ROUTING_UNKNOWN_SKILL', case: c.case, message: `${c.case}: skill=${c.skill} is not a skill in skills/` });
      continue;
    }
    if (c.kind === 'effect') continue;
    const ranking = rank(c.prompt, corpus);
    const at = ranking.findIndex((r) => r.id === self.id);
    const hit = ranking[at];
    const top = ranking.slice(0, 3).map((r) => `${r.id} ${r.score.toFixed(2)}`).join(', ');
    if (c.kind === 'trigger') {
      positives++;
      if (at === 0 && hit.score > 0) rank1++;
      if (!(at < t.POSITIVE_TOP_K && hit.score > 0)) {
        errors.push({ code: 'ROUTING_POSITIVE_MISSED', case: c.case, rank: at + 1,
          message: `${c.case}: ${self.id} ranks #${at + 1} (score ${hit.score.toFixed(2)}); needs top ${t.POSITIVE_TOP_K}. Top: ${top}` });
      }
    } else {
      negatives++;
      if (ranking[0].id === self.id && ranking[0].score > 0) {
        errors.push({ code: 'ROUTING_NEGATIVE_WON', case: c.case, message: `${c.case}: ${self.id} ranks first on a prompt it must not take. Top: ${top}` });
      }
      if (!c.owner) {
        errors.push({ code: 'ROUTING_NEGATIVE_NO_OWNER', case: c.case, message: `${c.case}: a negative case names the owner that should take the prompt (owner=...)` });
      } else if (!byName.has(c.owner) && BUILTIN_OWNERS.includes(c.owner)) {
        notes.push({ code: 'ROUTING_OWNER_OUTSIDE_CORPUS', case: c.case,
          message: `${c.case}: owner=${c.owner} is Claude Code's own; owner outside the corpus: only "not taken" checked` });
      } else if (!byName.has(c.owner)) {
        errors.push({ code: 'ROUTING_UNKNOWN_OWNER', case: c.case, message: `${c.case}: owner=${c.owner} is no skill, command or spawnable agent` });
      } else {
        const ownerAt = ranking.findIndex((r) => r.id === byName.get(c.owner).id);
        if (ranking[ownerAt].score === 0 || ownerAt > at) {
          errors.push({ code: 'ROUTING_OWNER_OUTRANKED', case: c.case,
            message: `${c.case}: owner ${c.owner} (#${ownerAt + 1}) must rank above ${self.id} (#${at + 1}). Top: ${top}` });
        }
      }
    }
  }

  for (const s of skills) {
    const mine = cases.filter((c) => c.skill === s.name);
    const n = (k) => mine.filter((c) => c.kind === k).length;
    if (n('trigger') < t.MIN_POSITIVE || n('no-trigger') < t.MIN_NEGATIVE || n('effect') < t.MIN_EFFECT) {
      errors.push({ code: 'ROUTING_BELOW_MINIMUM', skill: s.name,
        message: `${s.name}: ${n('trigger')} trigger / ${n('no-trigger')} no-trigger / ${n('effect')} effect cases; needs ${t.MIN_POSITIVE} / ${t.MIN_NEGATIVE} / ${t.MIN_EFFECT}` });
    }
  }

  const rate = positives ? rank1 / positives : 1;
  if (positives && rate < t.MIN_RANK1_RATE) {
    errors.push({ code: 'ROUTING_RANK1_FLOOR', message: `${rank1} of ${positives} positives rank their skill first (${(rate * 100).toFixed(0)}%); the floor is ${(t.MIN_RANK1_RATE * 100).toFixed(0)}%` });
  }
  return { errors, warnings, notes, stats: { entries: entries.length, skills: skills.length, sourced: all.length, cases: cases.length, positives, negatives, rank1 } };
}

// ---- CLI ----

if (isMain(import.meta.url)) {
  const args = process.argv.slice(2);
  const at = args.indexOf('--root');
  const root = resolve(at >= 0 ? args[at + 1] || '' : join(dirname(fileURLToPath(import.meta.url)), '..', '..'));
  if (!existsSync(root) || !statSync(root).isDirectory()) {
    process.stderr.write(`routing: not measured: ${root} is not a directory\n`);
    process.exit(2);
  }
  const r = checkRouting(root);
  if (args.includes('--json')) process.stdout.write(`${JSON.stringify(r, null, 2)}\n`);
  else {
    for (const e of r.errors) process.stdout.write(`FAIL ${e.code}: ${e.message}\n`);
    for (const w of r.warnings) process.stdout.write(`warn ${w.code}: ${w.message}\n`);
    for (const n of r.notes) process.stdout.write(`note ${n.code}: ${n.message}\n`);
    process.stdout.write(`${r.stats.entries} descriptions, ${r.stats.sourced} cases checked for from=, ${r.stats.cases} routed, ${r.stats.rank1}/${r.stats.positives} positives first; ${r.errors.length} finding(s)\n`);
  }
  process.exit(r.errors.length ? 1 : 0);
}