open/voyage
1
0
Fork 0
Code Issues Releases 13 Activity

feat(routing): every eval case names its event; the P10 eval gate is exempted with its reason (A1)

Browse source 
P10's model-free part (wave 4, order A1, decision 63). The routing test now
checks every case in evals/ for from=, not only the skill-tagged ones (R1):
a case with no skill tag is not routed, only checked, and the report gives
both counts ("45 cases checked for from=, 37 routed"). The eight test-set
cases get from=proevesett-eval-suite (d311f3b) or
from=proevesett-intent-gate (3892a83), the commits that added them.

The release gate that reads the last test-set run is written into the
runbook as exempted with its reason (it waits for the fixed test set to run
again) and still says where it goes (scripts/release-check.mjs).

review-630's prototype-tag stub gets from=test: under R1 an unrouted case is
still checked for from=, and the stub tests that a prototype name is never
routed, which stands.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Kjell Tore Guttormsen 2026-10-19 18:15:00 +02:00
commit cf08df9d55
Signed by: ktg
SSH key fingerprint: SHA256:JakMjO6FTBBzN0Bhfj9saOoEjaFxlSdYuZQQpM/lF9Q
17 changed files with 145 additions and 26 deletions
Download patch file Download diff file Expand all files Collapse all files

32
lib/skills/routing.mjs
View file

@ -20,7 +20,9 @@
// A case is tagged in its prompt.md frontmatter:
// tags: [skill-trigger | skill-no-trigger | skill-effect, skill=<name>,
// owner=<command|agent|skill name> (negatives), from=<event>]
// `from=` names the event the case came from (P10: every case is marked).
// `from=` names the event the case came from (P10, R1): it is required on every
// case in evals/, and a case with no skill tag (the fixed test set) is not
// routed, only checked for `from=`.
//
// What this cannot see: how Claude Code really chooses. A ranking over word
// overlap is a proxy; the operator's eval run is the measurement.
@ -180,8 +182,12 @@ export function loadEntries(root) {
const KIND_TAGS = { 'skill-trigger': 'trigger', 'skill-no-trigger': 'no-trigger', 'skill-effect': 'effect' };
/** The routing-tagged eval cases of a plugin root (evals/<case>/prompt.md). */
export function loadCases(root) {
/**
* Every eval case of a plugin root (evals/<case>/prompt.md), routed or not.
* `kind` is null for a case with no skill tag (the fixed test set): it is not
* routed, only checked for `from=` (P10, R1).
*/
export function loadAllCases(root) {
const d = join(root, 'evals');
if (!existsSync(d)) return [];
const out = [];
@ -192,14 +198,18 @@ export function loadCases(root) {
const tagsField = readFrontmatterFields(text).fields.tags;
const tags = tagsField && Array.isArray(tagsField.value) ? tagsField.value.map(String) : [];
const kindTag = tags.find((t) => Object.hasOwn(KIND_TAGS, t));
if (!kindTag) continue;
const val = (k) => { const t = tags.find((x) => x.startsWith(`${k}=`)); return t ? t.slice(k.length + 1) : null; };
const body = text.replace(/^---\n[\s\S]*?\n---\n?/, '').trim();
out.push({ case: name, kind: KIND_TAGS[kindTag], skill: val('skill'), owner: val('owner'), from: val('from'), prompt: body });
out.push({ case: name, kind: kindTag ? KIND_TAGS[kindTag] : null, skill: val('skill'), owner: val('owner'), from: val('from'), prompt: body });
}
return out;
}
/** The routing-tagged eval cases of a plugin root. */
export function loadCases(root) {
return loadAllCases(root).filter((c) => c.kind);
}
// ---- the check ----
/**
@ -214,7 +224,8 @@ export function checkRouting(root, t = THRESHOLDS) {
// name must not take its place. An owner may be any kind; a skill wins a tie.
const skillByName = new Map(skills.map((e) => [e.name, e]));
const byName = new Map([...entries.filter((e) => e.kind !== 'skill'), ...skills].map((e) => [e.name, e]));
const cases = loadCases(root);
const all = loadAllCases(root);
const cases = all.filter((c) => c.kind);
const errors = [];
const warnings = [];
let positives = 0; let rank1 = 0; let negatives = 0;
@ -225,8 +236,11 @@ export function checkRouting(root, t = THRESHOLDS) {
(c.level === 'error' ? errors : warnings).push(entry);
}
for (const c of cases) {
for (const c of all) {
if (!c.from) errors.push({ code: 'ROUTING_CASE_NO_SOURCE', case: c.case, message: `${c.case}: tag the event the case came from (from=...)` });
}
for (const c of cases) {
const self = c.skill && skillByName.get(c.skill);
if (!self || self.kind !== 'skill') {
errors.push({ code: 'ROUTING_UNKNOWN_SKILL', case: c.case, message: `${c.case}: skill=${c.skill} is not a skill in skills/` });
@ -276,7 +290,7 @@ export function checkRouting(root, t = THRESHOLDS) {
if (positives && rate < t.MIN_RANK1_RATE) {
errors.push({ code: 'ROUTING_RANK1_FLOOR', message: `${rank1} of ${positives} positives rank their skill first (${(rate * 100).toFixed(0)}%); the floor is ${(t.MIN_RANK1_RATE * 100).toFixed(0)}%` });
}
return { errors, warnings, stats: { entries: entries.length, skills: skills.length, cases: cases.length, positives, negatives, rank1 } };
return { errors, warnings, stats: { entries: entries.length, skills: skills.length, sourced: all.length, cases: cases.length, positives, negatives, rank1 } };
}
// ---- CLI ----
@ -294,7 +308,7 @@ if (isMain(import.meta.url)) {
else {
for (const e of r.errors) process.stdout.write(`FAIL ${e.code}: ${e.message}\n`);
for (const w of r.warnings) process.stdout.write(`warn ${w.code}: ${w.message}\n`);
process.stdout.write(`${r.stats.entries} descriptions, ${r.stats.cases} cases, ${r.stats.rank1}/${r.stats.positives} positives first; ${r.errors.length} finding(s)\n`);
process.stdout.write(`${r.stats.entries} descriptions, ${r.stats.sourced} cases checked for from=, ${r.stats.cases} routed, ${r.stats.rank1}/${r.stats.positives} positives first; ${r.errors.length} finding(s)\n`);
}
process.exit(r.errors.length ? 1 : 0);
}