feat(skills): skill workshop, test-first and debugging preloaded into the step helper
The workshop (ordre 7a, contracts S1 and S2): a form check (lib/skills/skill-lint.mjs) that follows Claude Code's own skill fields, and a model-free routing test (lib/skills/routing.mjs) over every skill, the 7 commands and the 21 spawnable agents, reading the eval cases themselves (evals/skill-*, 3 fire / 2 quiet / 1 effect, tagged with the event they came from). Thresholds pinned, never lowered. The two skills were written against failures measured first in helpers without them: red only as an import crash, the whole suite run from a hand-made file list instead of the repo's own command, and an unreproduced fix shipped under authority pressure. step-implementer preloads both through its skills: line; no tool list change. Credits: addyosmani/agent-skills @9d0c60d, obra/superpowers @8ca22db, mattpocock/skills @c55ee46 (MIT), Anthropic skill-creator (Apache-2.0). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
4ee711a6ac
commit
cc832fd430
SSH key fingerprint:
SHA256:JakMjO6FTBBzN0Bhfj9saOoEjaFxlSdYuZQQpM/lF9Q
41 changed files with 1996 additions and 0 deletions
298
lib/skills/routing.mjs
Normal file
298
lib/skills/routing.mjs
Normal file
|
|
@ -0,0 +1,298 @@
|
|||
#!/usr/bin/env node
|
||||
// lib/skills/routing.mjs
|
||||
// The model-free routing test (praksisanalyse P10, "Bygg naa"; contract S1).
|
||||
// Ranks the descriptions of every skill, command and spawnable agent against
|
||||
// prompts, with no model, so a description that would steal or lose a prompt
|
||||
// is caught in `npm test` instead of in a user's session.
|
||||
//
|
||||
// - A collision between two descriptions (cosine >= COLLISION_ERROR) fails.
|
||||
// - Each skill has at least MIN_POSITIVE prompts where it should fire (it must
|
||||
// rank in the top POSITIVE_TOP_K with a score > 0), MIN_NEGATIVE where it
|
||||
// should not (it must not rank first, and a named owner must rank above it),
|
||||
// and MIN_EFFECT effect cases.
|
||||
// - The share of positives where the skill ranks first is held at or above
|
||||
// MIN_RANK1_RATE.
|
||||
// The thresholds are pinned by tests/lib/skill-routing.test.mjs. They are
|
||||
// never lowered: a failing case is fixed in the description, never in a number.
|
||||
//
|
||||
// The cases are the plugin-eval cases themselves (evals/<case>/prompt.md):
|
||||
// one source for the operator's model-run eval and this model-free check.
|
||||
// A case is tagged in its prompt.md frontmatter:
|
||||
// tags: [skill-trigger | skill-no-trigger | skill-effect, skill=<name>,
|
||||
// owner=<command|agent|skill name> (negatives), from=<event>]
|
||||
// `from=` names the event the case came from (P10: every case is marked).
|
||||
//
|
||||
// What this cannot see: how Claude Code really chooses. A ranking over word
|
||||
// overlap is a proxy; the operator's eval run is the measurement.
|
||||
//
|
||||
// CLI: node lib/skills/routing.mjs [--root <plugin root>] [--json]
|
||||
// exit 0 = green · 1 = findings · 2 = could not run
|
||||
//
|
||||
// Adapted from addyosmani/agent-skills @9d0c60d, scripts/run-evals.js (tier 2:
|
||||
// stemmed TF-IDF cosine, name weighted twice, top-3 positives, owner-ranked
|
||||
// negatives, collision and rank-1 floors) - MIT, Copyright (c) 2025 Addy
|
||||
// Osmani. Reimplemented, no code copied; the reader here takes multi-line
|
||||
// (`description: |`) descriptions, which the original reads as "|".
|
||||
|
||||
import { existsSync, readdirSync, readFileSync, realpathSync, statSync } from 'node:fs';
|
||||
import { dirname, join, resolve } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { descriptionOf, readFrontmatterFields } from './skill-frontmatter.mjs';
|
||||
|
||||
export const THRESHOLDS = Object.freeze({
|
||||
COLLISION_ERROR: 0.75,
|
||||
COLLISION_WARN: 0.5,
|
||||
POSITIVE_TOP_K: 3,
|
||||
MIN_POSITIVE: 3,
|
||||
MIN_NEGATIVE: 2,
|
||||
MIN_EFFECT: 1,
|
||||
MIN_RANK1_RATE: 1,
|
||||
});
|
||||
|
||||
// ---- text ----
|
||||
|
||||
const STOP = new Set([
|
||||
'a', 'an', 'and', 'any', 'are', 'as', 'at', 'be', 'by', 'can', 'do', 'does', 'for', 'from', 'has', 'have',
|
||||
'in', 'into', 'is', 'it', 'its', 'of', 'on', 'or', 'so', 'than', 'that', 'the', 'their', 'them', 'then',
|
||||
'there', 'these', 'this', 'those', 'to', 'use', 'used', 'uses', 'using', 'was', 'were', 'what', 'when',
|
||||
'which', 'while', 'who', 'will', 'with', 'you', 'your', 'we', 'our', 'me', 'my', 'i', 'not', 'no', 'one',
|
||||
'all', 'each', 'every', 'only', 'also', 'just', 'via', 'per', 'agent', 'skill', 'command', 'should',
|
||||
]);
|
||||
|
||||
function stem(t) {
|
||||
for (const suf of ['ations', 'ation', 'ings', 'ing', 'ers', 'ies', 'ied', 'ed', 'es', 'er', 'ly']) {
|
||||
if (t.length > suf.length + 3 && t.endsWith(suf)) {
|
||||
t = suf === 'ies' || suf === 'ied' ? `${t.slice(0, -suf.length)}y` : t.slice(0, -suf.length);
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (t.length > 3 && t.endsWith('s') && !t.endsWith('ss')) t = t.slice(0, -1);
|
||||
if (t.length > 4 && t.endsWith('e')) t = t.slice(0, -1);
|
||||
if (t.length > 4 && t[t.length - 1] === t[t.length - 2] && !'aeiou'.includes(t[t.length - 1])) t = t.slice(0, -1);
|
||||
return t;
|
||||
}
|
||||
|
||||
/** Lower-case word tokens, stop words dropped, light-stemmed. */
|
||||
export function tokenize(text) {
|
||||
return String(text)
|
||||
.toLowerCase()
|
||||
.replace(/[^a-z0-9\s-]/g, ' ')
|
||||
.split(/[\s-]+/)
|
||||
.filter((t) => t.length > 2 && !STOP.has(t))
|
||||
.map(stem)
|
||||
.filter((t) => t.length > 2 && !STOP.has(t));
|
||||
}
|
||||
|
||||
function termFreq(tokens) {
|
||||
const tf = new Map();
|
||||
for (const t of tokens) tf.set(t, (tf.get(t) || 0) + 1);
|
||||
return tf;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {{id: string, name: string, description: string}[]} entries
|
||||
* @returns {{ docs: Map<string, Map<string, number>>, idf: (t: string) => number, entries: object[] }}
|
||||
*/
|
||||
export function buildCorpus(entries) {
|
||||
const docs = new Map();
|
||||
for (const e of entries) {
|
||||
const nameTokens = tokenize(e.name.replace(/[-:]/g, ' '));
|
||||
docs.set(e.id, termFreq([...nameTokens, ...nameTokens, ...tokenize(e.description)]));
|
||||
}
|
||||
const df = new Map();
|
||||
for (const tf of docs.values()) for (const term of tf.keys()) df.set(term, (df.get(term) || 0) + 1);
|
||||
const n = docs.size;
|
||||
return { docs, idf: (term) => Math.log(1 + n / (1 + (df.get(term) || 0))), entries };
|
||||
}
|
||||
|
||||
function vec(tf, idf) {
|
||||
const v = new Map();
|
||||
for (const [term, f] of tf) v.set(term, f * idf(term));
|
||||
return v;
|
||||
}
|
||||
|
||||
function cosine(a, b) {
|
||||
let dot = 0; let na = 0; let nb = 0;
|
||||
for (const [t, w] of a) { na += w * w; const bw = b.get(t); if (bw) dot += w * bw; }
|
||||
for (const w of b.values()) nb += w * w;
|
||||
return na && nb ? dot / (Math.sqrt(na) * Math.sqrt(nb)) : 0;
|
||||
}
|
||||
|
||||
/** Every entry scored against a prompt, best first (ties by id, so the order is stable). */
|
||||
export function rank(prompt, corpus) {
|
||||
const pv = vec(termFreq(tokenize(prompt)), corpus.idf);
|
||||
const out = [];
|
||||
for (const [id, tf] of corpus.docs) out.push({ id, score: cosine(pv, vec(tf, corpus.idf)) });
|
||||
return out.sort((a, b) => b.score - a.score || a.id.localeCompare(b.id));
|
||||
}
|
||||
|
||||
/** Pairs of descriptions too alike: level "error" at COLLISION_ERROR, "warn" at COLLISION_WARN. */
|
||||
export function collisions(corpus, t = THRESHOLDS) {
|
||||
const ids = [...corpus.docs.keys()];
|
||||
const out = [];
|
||||
for (let i = 0; i < ids.length; i++) {
|
||||
for (let j = i + 1; j < ids.length; j++) {
|
||||
const sim = cosine(vec(corpus.docs.get(ids[i]), corpus.idf), vec(corpus.docs.get(ids[j]), corpus.idf));
|
||||
if (sim >= t.COLLISION_ERROR) out.push({ a: ids[i], b: ids[j], sim, level: 'error' });
|
||||
else if (sim >= t.COLLISION_WARN) out.push({ a: ids[i], b: ids[j], sim, level: 'warn' });
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
// ---- loading ----
|
||||
|
||||
/** Orchestrator reference docs are not spawnable capabilities (CLAUDE.md, Inventory). */
|
||||
const isReferenceDoc = (name, description) => /-orchestrator$/.test(name) || /^Reference document, not a spawnable/.test(description.trim());
|
||||
|
||||
/** Skills, commands and spawnable agents of a plugin root. */
|
||||
export function loadEntries(root) {
|
||||
const entries = [];
|
||||
const skillsDir = join(root, 'skills');
|
||||
if (existsSync(skillsDir)) {
|
||||
for (const n of readdirSync(skillsDir).sort()) {
|
||||
const f = join(skillsDir, n, 'SKILL.md');
|
||||
if (!existsSync(f)) continue;
|
||||
const text = readFileSync(f, 'utf8');
|
||||
const fm = readFrontmatterFields(text).fields;
|
||||
const name = fm.name ? String(fm.name.value) : n;
|
||||
entries.push({ id: `skill:${name}`, kind: 'skill', name, description: descriptionOf(text) });
|
||||
}
|
||||
}
|
||||
for (const [kind, dir] of [['command', 'commands'], ['agent', 'agents']]) {
|
||||
const d = join(root, dir);
|
||||
if (!existsSync(d)) continue;
|
||||
for (const file of readdirSync(d).filter((x) => x.endsWith('.md')).sort()) {
|
||||
const text = readFileSync(join(d, file), 'utf8');
|
||||
const fm = readFrontmatterFields(text).fields;
|
||||
const name = fm.name ? String(fm.name.value) : file.replace(/\.md$/, '');
|
||||
const description = descriptionOf(text);
|
||||
if (kind === 'agent' && isReferenceDoc(name, description)) continue;
|
||||
entries.push({ id: `${kind}:${name}`, kind, name, description });
|
||||
}
|
||||
}
|
||||
return entries;
|
||||
}
|
||||
|
||||
const KIND_TAGS = { 'skill-trigger': 'trigger', 'skill-no-trigger': 'no-trigger', 'skill-effect': 'effect' };
|
||||
|
||||
/** The routing-tagged eval cases of a plugin root (evals/<case>/prompt.md). */
|
||||
export function loadCases(root) {
|
||||
const d = join(root, 'evals');
|
||||
if (!existsSync(d)) return [];
|
||||
const out = [];
|
||||
for (const name of readdirSync(d).sort()) {
|
||||
const f = join(d, name, 'prompt.md');
|
||||
if (!existsSync(f) || !statSync(join(d, name)).isDirectory()) continue;
|
||||
const text = readFileSync(f, 'utf8');
|
||||
const tagsField = readFrontmatterFields(text).fields.tags;
|
||||
const tags = tagsField && Array.isArray(tagsField.value) ? tagsField.value.map(String) : [];
|
||||
const kindTag = tags.find((t) => t in KIND_TAGS);
|
||||
if (!kindTag) continue;
|
||||
const val = (k) => { const t = tags.find((x) => x.startsWith(`${k}=`)); return t ? t.slice(k.length + 1) : null; };
|
||||
const body = text.replace(/^---\n[\s\S]*?\n---\n?/, '').trim();
|
||||
out.push({ case: name, kind: KIND_TAGS[kindTag], skill: val('skill'), owner: val('owner'), from: val('from'), prompt: body });
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
// ---- the check ----
|
||||
|
||||
/**
|
||||
* Run the whole routing check over a plugin root.
|
||||
* @returns {{ errors: object[], warnings: object[], stats: object }}
|
||||
*/
|
||||
export function checkRouting(root, t = THRESHOLDS) {
|
||||
const entries = loadEntries(root);
|
||||
const corpus = buildCorpus(entries);
|
||||
const byName = new Map(entries.map((e) => [e.name, e]));
|
||||
const skills = entries.filter((e) => e.kind === 'skill');
|
||||
const cases = loadCases(root);
|
||||
const errors = [];
|
||||
const warnings = [];
|
||||
let positives = 0; let rank1 = 0; let negatives = 0;
|
||||
|
||||
for (const c of collisions(corpus, t)) {
|
||||
const entry = { code: c.level === 'error' ? 'ROUTING_COLLISION' : 'ROUTING_OVERLAP', a: c.a, b: c.b, sim: Number(c.sim.toFixed(3)),
|
||||
message: `${c.a} and ${c.b}: descriptions ${(c.sim * 100).toFixed(0)}% alike` };
|
||||
(c.level === 'error' ? errors : warnings).push(entry);
|
||||
}
|
||||
|
||||
for (const c of cases) {
|
||||
if (!c.from) errors.push({ code: 'ROUTING_CASE_NO_SOURCE', case: c.case, message: `${c.case}: tag the event the case came from (from=...)` });
|
||||
const self = c.skill && byName.get(c.skill);
|
||||
if (!self || self.kind !== 'skill') {
|
||||
errors.push({ code: 'ROUTING_UNKNOWN_SKILL', case: c.case, message: `${c.case}: skill=${c.skill} is not a skill in skills/` });
|
||||
continue;
|
||||
}
|
||||
if (c.kind === 'effect') continue;
|
||||
const ranking = rank(c.prompt, corpus);
|
||||
const at = ranking.findIndex((r) => r.id === self.id);
|
||||
const hit = ranking[at];
|
||||
const top = ranking.slice(0, 3).map((r) => `${r.id} ${r.score.toFixed(2)}`).join(', ');
|
||||
if (c.kind === 'trigger') {
|
||||
positives++;
|
||||
if (at === 0 && hit.score > 0) rank1++;
|
||||
if (!(at < t.POSITIVE_TOP_K && hit.score > 0)) {
|
||||
errors.push({ code: 'ROUTING_POSITIVE_MISSED', case: c.case, rank: at + 1,
|
||||
message: `${c.case}: ${self.id} ranks #${at + 1} (score ${hit.score.toFixed(2)}); needs top ${t.POSITIVE_TOP_K}. Top: ${top}` });
|
||||
}
|
||||
} else {
|
||||
negatives++;
|
||||
if (ranking[0].id === self.id && ranking[0].score > 0) {
|
||||
errors.push({ code: 'ROUTING_NEGATIVE_WON', case: c.case, message: `${c.case}: ${self.id} ranks first on a prompt it must not take. Top: ${top}` });
|
||||
}
|
||||
if (!c.owner) {
|
||||
errors.push({ code: 'ROUTING_NEGATIVE_NO_OWNER', case: c.case, message: `${c.case}: a negative case names the owner that should take the prompt (owner=...)` });
|
||||
} else if (!byName.has(c.owner)) {
|
||||
errors.push({ code: 'ROUTING_UNKNOWN_OWNER', case: c.case, message: `${c.case}: owner=${c.owner} is no skill, command or spawnable agent` });
|
||||
} else {
|
||||
const ownerAt = ranking.findIndex((r) => r.id === byName.get(c.owner).id);
|
||||
if (ranking[ownerAt].score === 0 || ownerAt > at) {
|
||||
errors.push({ code: 'ROUTING_OWNER_OUTRANKED', case: c.case,
|
||||
message: `${c.case}: owner ${c.owner} (#${ownerAt + 1}) must rank above ${self.id} (#${at + 1}). Top: ${top}` });
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (const s of skills) {
|
||||
const mine = cases.filter((c) => c.skill === s.name);
|
||||
const n = (k) => mine.filter((c) => c.kind === k).length;
|
||||
if (n('trigger') < t.MIN_POSITIVE || n('no-trigger') < t.MIN_NEGATIVE || n('effect') < t.MIN_EFFECT) {
|
||||
errors.push({ code: 'ROUTING_BELOW_MINIMUM', skill: s.name,
|
||||
message: `${s.name}: ${n('trigger')} trigger / ${n('no-trigger')} no-trigger / ${n('effect')} effect cases; needs ${t.MIN_POSITIVE} / ${t.MIN_NEGATIVE} / ${t.MIN_EFFECT}` });
|
||||
}
|
||||
}
|
||||
|
||||
const rate = positives ? rank1 / positives : 1;
|
||||
if (positives && rate < t.MIN_RANK1_RATE) {
|
||||
errors.push({ code: 'ROUTING_RANK1_FLOOR', message: `${rank1} of ${positives} positives rank their skill first (${(rate * 100).toFixed(0)}%); the floor is ${(t.MIN_RANK1_RATE * 100).toFixed(0)}%` });
|
||||
}
|
||||
return { errors, warnings, stats: { entries: entries.length, skills: skills.length, cases: cases.length, positives, negatives, rank1 } };
|
||||
}
|
||||
|
||||
// ---- CLI ----
|
||||
|
||||
function isMain() {
|
||||
if (!process.argv[1]) return false;
|
||||
try { return realpathSync(fileURLToPath(import.meta.url)) === realpathSync(process.argv[1]); } catch { return false; }
|
||||
}
|
||||
|
||||
if (isMain()) {
|
||||
const args = process.argv.slice(2);
|
||||
const at = args.indexOf('--root');
|
||||
const root = resolve(at >= 0 ? args[at + 1] || '' : join(dirname(fileURLToPath(import.meta.url)), '..', '..'));
|
||||
if (!existsSync(root) || !statSync(root).isDirectory()) {
|
||||
process.stderr.write(`routing: not measured: ${root} is not a directory\n`);
|
||||
process.exit(2);
|
||||
}
|
||||
const r = checkRouting(root);
|
||||
if (args.includes('--json')) process.stdout.write(`${JSON.stringify(r, null, 2)}\n`);
|
||||
else {
|
||||
for (const e of r.errors) process.stdout.write(`FAIL ${e.code}: ${e.message}\n`);
|
||||
for (const w of r.warnings) process.stdout.write(`warn ${w.code}: ${w.message}\n`);
|
||||
process.stdout.write(`${r.stats.entries} descriptions, ${r.stats.cases} cases, ${r.stats.rank1}/${r.stats.positives} positives first; ${r.errors.length} finding(s)\n`);
|
||||
}
|
||||
process.exit(r.errors.length ? 1 : 0);
|
||||
}
|
||||
143
lib/skills/skill-frontmatter.mjs
Normal file
143
lib/skills/skill-frontmatter.mjs
Normal file
|
|
@ -0,0 +1,143 @@
|
|||
// lib/skills/skill-frontmatter.mjs
|
||||
// Frontmatter reader for skill, command and agent files: the fields the skill
|
||||
// workshop needs, with the line each one is on and the YAML traps it hits.
|
||||
//
|
||||
// Why not lib/util/frontmatter.mjs: that parser deliberately rejects
|
||||
// multi-line strings, and Voyage's agents write `description: |`. This reader
|
||||
// takes plain, quoted and block-scalar (`|`, `>`, with `-`/`+`) values, flow
|
||||
// lists, and indented lists or maps (kept as raw text). It is not a YAML
|
||||
// parser: it reads the subset a SKILL.md uses and reports what would break a
|
||||
// real one. Claude Code loads a skill whose YAML does not parse "with no
|
||||
// fields set", so the description is gone and the skill cannot match; a trap
|
||||
// here is an error, not a style note.
|
||||
|
||||
const KEY_RE = /^([A-Za-z0-9_-]+):(.*)$/;
|
||||
const BLOCK_RE = /^([|>])([+-]?)\s*$/;
|
||||
|
||||
/**
|
||||
* Split a file into its frontmatter lines and body.
|
||||
* @returns {{ ok: boolean, fmLines: string[], body: string, bodyStartLine: number }}
|
||||
*/
|
||||
export function splitFile(text) {
|
||||
const src = String(text).replace(/^/, '');
|
||||
const lines = src.split(/\r?\n/);
|
||||
if (lines[0] !== '---') return { ok: false, fmLines: [], body: src, bodyStartLine: 1 };
|
||||
const end = lines.indexOf('---', 1);
|
||||
if (end < 0) return { ok: false, fmLines: [], body: src, bodyStartLine: 1 };
|
||||
return { ok: true, fmLines: lines.slice(1, end), body: lines.slice(end + 1).join('\n'), bodyStartLine: end + 2 };
|
||||
}
|
||||
|
||||
/** The YAML traps in one plain (unquoted, single-line) scalar value. */
|
||||
export function plainScalarHazards(value) {
|
||||
const out = [];
|
||||
if (/:\s/.test(value) || /:$/.test(value)) out.push('unquoted ": " (YAML reads a mapping and the line fails)');
|
||||
if (/\s#/.test(value)) out.push('" #" starts a YAML comment: the rest of the value is cut');
|
||||
if (/^[`@%*&!|>,?{]/.test(value)) out.push(`a value may not start with "${value[0]}" unquoted`);
|
||||
if (/^\[/.test(value) && !/\]$/.test(value)) out.push('an unclosed flow list');
|
||||
if (/^-\s/.test(value)) out.push('"- " at the start reads as a list item');
|
||||
return out;
|
||||
}
|
||||
|
||||
function unquote(v) {
|
||||
if (v.startsWith('"')) {
|
||||
if (v.length < 2 || !v.endsWith('"') || v.endsWith('\\"')) return { error: 'unclosed double quote' };
|
||||
try { return { value: JSON.parse(v) }; } catch { return { error: 'bad escape in a double-quoted value' }; }
|
||||
}
|
||||
if (v.startsWith("'")) {
|
||||
if (v.length < 2 || !v.endsWith("'")) return { error: 'unclosed single quote' };
|
||||
return { value: v.slice(1, -1).replace(/''/g, "'") };
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function blockValue(lines, style, chomp) {
|
||||
const nonBlank = lines.filter((l) => l.trim() !== '');
|
||||
const indent = nonBlank.length ? Math.min(...nonBlank.map((l) => l.match(/^ */)[0].length)) : 0;
|
||||
const stripped = lines.map((l) => l.slice(indent));
|
||||
while (stripped.length && stripped[stripped.length - 1].trim() === '') stripped.pop();
|
||||
let text;
|
||||
if (style === '|') text = stripped.join('\n');
|
||||
else {
|
||||
// Folded: lines join with a space; a blank line is a newline.
|
||||
text = '';
|
||||
for (const l of stripped) {
|
||||
if (l === '') text += '\n';
|
||||
else text += (text === '' || text.endsWith('\n') ? '' : ' ') + l;
|
||||
}
|
||||
}
|
||||
if (chomp === '-') return text;
|
||||
return `${text}\n`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Read the frontmatter of a skill/command/agent file.
|
||||
* @returns {{ ok: boolean, fields: Object<string, {value: any, line: number, style: string}>,
|
||||
* hazards: {line: number, message: string}[], error?: string }}
|
||||
*/
|
||||
export function readFrontmatterFields(text) {
|
||||
const split = splitFile(text);
|
||||
if (!split.ok) return { ok: false, fields: {}, hazards: [], error: 'no frontmatter: the file must start with a "---" line and close it' };
|
||||
const { fmLines } = split;
|
||||
const fields = {};
|
||||
const hazards = [];
|
||||
let i = 0;
|
||||
while (i < fmLines.length) {
|
||||
const line = fmLines[i];
|
||||
const lineNo = i + 2; // line 1 is the opening ---
|
||||
if (/\t/.test(line)) hazards.push({ line: lineNo, message: 'a tab in frontmatter: YAML forbids tabs for indentation' });
|
||||
if (line.trim() === '' || line.trimStart().startsWith('#')) { i++; continue; }
|
||||
if (/^\s/.test(line)) { hazards.push({ line: lineNo, message: 'an indented line that belongs to no key' }); i++; continue; }
|
||||
const m = KEY_RE.exec(line);
|
||||
if (!m) { hazards.push({ line: lineNo, message: `not a "key: value" line: ${line.slice(0, 60)}` }); i++; continue; }
|
||||
const key = m[1];
|
||||
const rest = m[2].replace(/^[ \t]+/, '');
|
||||
if (m[2] !== '' && !/^[ \t]/.test(m[2])) hazards.push({ line: lineNo, message: `"${key}:" needs a space after the colon` });
|
||||
if (key in fields) hazards.push({ line: lineNo, message: `duplicate key "${key}"` });
|
||||
|
||||
// Collect the indented continuation lines (block scalar, list or map).
|
||||
let j = i + 1;
|
||||
const cont = [];
|
||||
while (j < fmLines.length && (fmLines[j].trim() === '' || /^\s/.test(fmLines[j]))) { cont.push(fmLines[j]); j++; }
|
||||
while (cont.length && cont[cont.length - 1].trim() === '') { cont.pop(); }
|
||||
|
||||
const block = BLOCK_RE.exec(rest);
|
||||
if (block) {
|
||||
fields[key] = { value: blockValue(cont, block[1], block[2]), line: lineNo, style: 'block' };
|
||||
} else if (rest === '') {
|
||||
const items = cont.filter((l) => l.trim() !== '');
|
||||
if (items.length && items.every((l) => /^\s+-\s/.test(l))) {
|
||||
fields[key] = { value: items.map((l) => l.replace(/^\s+-\s+/, '').trim()), line: lineNo, style: 'list' };
|
||||
} else {
|
||||
fields[key] = { value: items.join('\n'), line: lineNo, style: items.length ? 'map' : 'empty' };
|
||||
}
|
||||
} else {
|
||||
if (cont.length) hazards.push({ line: lineNo, message: `"${key}" has a value and indented lines under it` });
|
||||
const q = unquote(rest);
|
||||
if (q && q.error) {
|
||||
hazards.push({ line: lineNo, message: q.error });
|
||||
fields[key] = { value: rest, line: lineNo, style: 'broken' };
|
||||
} else if (q) {
|
||||
fields[key] = { value: q.value, line: lineNo, style: 'quoted' };
|
||||
} else if (/^\[.*\]$/.test(rest)) {
|
||||
const inner = rest.slice(1, -1).trim();
|
||||
fields[key] = {
|
||||
value: inner === '' ? [] : inner.split(',').map((s) => { const t = s.trim(); const u = unquote(t); return u && !u.error ? u.value : t; }),
|
||||
line: lineNo, style: 'flow',
|
||||
};
|
||||
} else {
|
||||
for (const h of plainScalarHazards(rest)) hazards.push({ line: lineNo, message: `${key}: ${h}` });
|
||||
fields[key] = { value: rest, line: lineNo, style: 'plain' };
|
||||
}
|
||||
}
|
||||
i = j;
|
||||
}
|
||||
return { ok: hazards.length === 0, fields, hazards };
|
||||
}
|
||||
|
||||
/** The description of a file as one string ('' when absent). */
|
||||
export function descriptionOf(text) {
|
||||
const r = readFrontmatterFields(text);
|
||||
const d = r.fields.description;
|
||||
if (!d) return '';
|
||||
return Array.isArray(d.value) ? d.value.join(' ') : String(d.value);
|
||||
}
|
||||
281
lib/skills/skill-lint.mjs
Normal file
281
lib/skills/skill-lint.mjs
Normal file
|
|
@ -0,0 +1,281 @@
|
|||
#!/usr/bin/env node
|
||||
// lib/skills/skill-lint.mjs
|
||||
// The skill form check (contract S1). Model-free; `npm test` runs it over
|
||||
// every skills/<name>/SKILL.md through tests/lib/skill-lint.test.mjs.
|
||||
//
|
||||
// It follows Claude Code's own skill fields (code.claude.com/docs/en/skills.md,
|
||||
// "Frontmatter reference"), not the narrower portable spec: Voyage needs
|
||||
// fields such as `user-invocable` that the portable spec rejects.
|
||||
//
|
||||
// Rules (error codes):
|
||||
// SKILL_NO_FRONTMATTER no leading --- block
|
||||
// SKILL_YAML_HAZARD a line real YAML would misread (unquoted ": ",
|
||||
// " #", tab, unclosed quote, leading indicator)
|
||||
// SKILL_UNKNOWN_FIELD a field Claude Code does not know (it would be
|
||||
// ignored silently)
|
||||
// SKILL_NAME_MISMATCH / _FORM name = directory name; kebab-case, <= 64,
|
||||
// no "anthropic" / "claude"
|
||||
// SKILL_DESC_MISSING / _TOO_LONG description present, <= 1 024 characters
|
||||
// SKILL_DESC_NO_USE_WHEN the description says "Use when ..."
|
||||
// SKILL_DESC_NOT_THIRD_PERSON no I / we / you in the description
|
||||
// SKILL_MODEL_INVOCATION_DISABLED disable-model-invocation set true: the skill
|
||||
// could not be preloaded into a helper nor
|
||||
// called by a command ("commands are the trunk")
|
||||
// SKILL_TOO_LONG SKILL.md is 500 lines or more
|
||||
// SKILL_MISSING_HEADING / _ORDER the fixed headings, in order
|
||||
// SKILL_ENFORCED_BY_VAGUE each "Enforced by" bullet names a check
|
||||
// (a backticked path) or says "prose only"
|
||||
// SKILL_REF_MISSING a linked or backticked file does not exist
|
||||
// SKILL_SHELL_INJECTION !`cmd` or a ```! block: Voyage advises users
|
||||
// to switch skill shell execution off
|
||||
// SKILL_SELF_EXEMPTION a skill that exempts itself
|
||||
//
|
||||
// Exemptions live in EXEMPTIONS below, with a reason, never in a skill.
|
||||
//
|
||||
// CLI: node lib/skills/skill-lint.mjs [--root <plugin root>] [--json]
|
||||
// exit 0 = every skill passes · 1 = findings · 2 = could not run
|
||||
//
|
||||
// Adapted from addyosmani/agent-skills @9d0c60d, scripts/lib/skill-lint.js
|
||||
// (rules, held-in-the-linter exemptions, fenced-code stripping) and
|
||||
// docs/skill-anatomy.md - MIT, Copyright (c) 2025 Addy Osmani. Reimplemented,
|
||||
// no code copied.
|
||||
|
||||
import { existsSync, readdirSync, readFileSync, realpathSync, statSync } from 'node:fs';
|
||||
import { dirname, join, resolve } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { readFrontmatterFields, splitFile } from './skill-frontmatter.mjs';
|
||||
|
||||
/** Claude Code's skill frontmatter fields (skills.md, Frontmatter reference). */
|
||||
export const KNOWN_FIELDS = new Set([
|
||||
'name', 'description', 'when_to_use', 'argument-hint', 'arguments',
|
||||
'disable-model-invocation', 'user-invocable', 'allowed-tools', 'disallowed-tools',
|
||||
'model', 'effort', 'context', 'agent', 'background', 'hooks', 'paths', 'shell',
|
||||
'metadata', 'license', 'compatibility',
|
||||
]);
|
||||
|
||||
/** The fixed headings of a Voyage skill (S1), in this order. */
|
||||
export const REQUIRED_HEADINGS = ['Overview', 'When to use', 'Process', 'Red flags', 'Enforced by', 'Credits'];
|
||||
|
||||
export const MAX_DESCRIPTION = 1024;
|
||||
export const MAX_LINES = 500;
|
||||
|
||||
/**
|
||||
* Exemptions, held here and only here: skill name -> { codes, reason }.
|
||||
* Empty by design; an entry needs a reason a reviewer can judge.
|
||||
* @type {Map<string, {codes: string[], reason: string}>}
|
||||
*/
|
||||
export const EXEMPTIONS = new Map();
|
||||
|
||||
const TRUE_WORDS = /^(true|yes|on|1)$/i;
|
||||
const FIRST_SECOND_PERSON = /\b(you|your|yours|you're|we|we're|our|ours|us|me|my|mine)\b/i;
|
||||
const FIRST_PERSON_I = /(^|[^A-Za-z.])(I|I'm|I'll|I've)(?![A-Za-z.])/;
|
||||
const PATH_PREFIXES = ['lib/', 'hooks/', 'agents/', 'commands/', 'templates/', 'docs/', 'skills/', 'scripts/'];
|
||||
|
||||
function err(code, message, line) { return { code, message, ...(line ? { line } : {}) }; }
|
||||
|
||||
/** Body lines with fenced code blocks blanked (their headings and links do not count). */
|
||||
function unfenced(body) {
|
||||
let inFence = false;
|
||||
return body.split('\n').map((l) => {
|
||||
if (/^\s*(```|~~~)/.test(l)) { inFence = !inFence; return ''; }
|
||||
return inFence ? '' : l;
|
||||
});
|
||||
}
|
||||
|
||||
/** The lines under one `## heading`, up to the next `## `. */
|
||||
function section(lines, heading) {
|
||||
const start = lines.findIndex((l) => l.trim() === `## ${heading}`);
|
||||
if (start < 0) return null;
|
||||
const out = [];
|
||||
for (let i = start + 1; i < lines.length && !/^##\s/.test(lines[i]); i++) out.push(lines[i]);
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* Lint one skill directory.
|
||||
* @param {string} skillDir path to skills/<name>
|
||||
* @param {{pluginRoot?: string}} [opts] root that backticked plugin paths resolve against
|
||||
* @returns {{ name: string, errors: object[], warnings: object[] }}
|
||||
*/
|
||||
export function lintSkill(skillDir, opts = {}) {
|
||||
const dirName = skillDir.split(/[\\/]/).filter(Boolean).pop();
|
||||
const pluginRoot = opts.pluginRoot || resolve(skillDir, '..', '..');
|
||||
const file = join(skillDir, 'SKILL.md');
|
||||
const errors = [];
|
||||
const warnings = [];
|
||||
if (!existsSync(file)) return { name: dirName, errors: [err('SKILL_NO_FRONTMATTER', 'SKILL.md is missing')], warnings };
|
||||
const text = readFileSync(file, 'utf8');
|
||||
|
||||
// Frontmatter
|
||||
const fm = readFrontmatterFields(text);
|
||||
if (fm.error) errors.push(err('SKILL_NO_FRONTMATTER', fm.error, 1));
|
||||
for (const h of fm.hazards) errors.push(err('SKILL_YAML_HAZARD', h.message, h.line));
|
||||
const f = fm.fields;
|
||||
for (const [key, v] of Object.entries(f)) {
|
||||
if (!KNOWN_FIELDS.has(key)) errors.push(err('SKILL_UNKNOWN_FIELD', `"${key}" is not a Claude Code skill field; it would be ignored`, v.line));
|
||||
}
|
||||
|
||||
// Name
|
||||
const name = f.name ? String(f.name.value) : dirName;
|
||||
if (f.name && name !== dirName) errors.push(err('SKILL_NAME_MISMATCH', `name "${name}" must equal the directory name "${dirName}"`, f.name.line));
|
||||
if (!/^[a-z0-9]+(-[a-z0-9]+)*$/.test(name) || name.length > 64 || /anthropic|claude/.test(name)) {
|
||||
errors.push(err('SKILL_NAME_FORM', `name "${name}": kebab-case, at most 64 characters, without "anthropic" or "claude"`, f.name && f.name.line));
|
||||
}
|
||||
|
||||
// Description
|
||||
const d = f.description;
|
||||
const desc = d ? String(Array.isArray(d.value) ? d.value.join(' ') : d.value).trim() : '';
|
||||
if (!desc) errors.push(err('SKILL_DESC_MISSING', 'a description is required: it is how Claude decides to use the skill'));
|
||||
else {
|
||||
if (desc.length > MAX_DESCRIPTION) errors.push(err('SKILL_DESC_TOO_LONG', `description is ${desc.length} characters (max ${MAX_DESCRIPTION})`, d.line));
|
||||
if (!/\bUse when\b/.test(desc)) errors.push(err('SKILL_DESC_NO_USE_WHEN', 'the description must say when to use the skill: "Use when ..."', d.line));
|
||||
if (FIRST_SECOND_PERSON.test(desc) || FIRST_PERSON_I.test(desc)) {
|
||||
errors.push(err('SKILL_DESC_NOT_THIRD_PERSON', 'write the description in the third person (no I / we / you)', d.line));
|
||||
}
|
||||
}
|
||||
|
||||
// Invocation: no Voyage skill is closed to the model.
|
||||
const dmi = f['disable-model-invocation'];
|
||||
if (dmi && TRUE_WORDS.test(String(dmi.value).trim())) {
|
||||
errors.push(err('SKILL_MODEL_INVOCATION_DISABLED',
|
||||
'disable-model-invocation is true: no helper could preload the skill and no command could call it', dmi.line));
|
||||
}
|
||||
|
||||
// Self-declared exemption (frontmatter or body)
|
||||
if (/lint-exempt/i.test(text)) errors.push(err('SKILL_SELF_EXEMPTION', 'a skill may not exempt itself; exemptions live in EXEMPTIONS in lib/skills/skill-lint.mjs'));
|
||||
|
||||
// Size
|
||||
const lineCount = text.split('\n').length;
|
||||
if (lineCount >= MAX_LINES) errors.push(err('SKILL_TOO_LONG', `SKILL.md has ${lineCount} lines (must stay under ${MAX_LINES}); move detail to a linked file`));
|
||||
|
||||
// Body: headings, enforcement, references, shell injection
|
||||
const split = splitFile(text);
|
||||
const bodyLines = unfenced(split.ok ? split.body : text);
|
||||
const headings = bodyLines.filter((l) => /^##\s/.test(l)).map((l) => l.replace(/^##\s+/, '').trim());
|
||||
const positions = [];
|
||||
for (const h of REQUIRED_HEADINGS) {
|
||||
const at = headings.indexOf(h);
|
||||
if (at < 0) errors.push(err('SKILL_MISSING_HEADING', `missing "## ${h}"`));
|
||||
else positions.push(at);
|
||||
}
|
||||
if (positions.some((p, i) => i > 0 && p < positions[i - 1])) {
|
||||
errors.push(err('SKILL_HEADING_ORDER', `the headings must come in this order: ${REQUIRED_HEADINGS.join(', ')}`));
|
||||
}
|
||||
|
||||
const enforced = section(bodyLines, 'Enforced by');
|
||||
if (enforced) {
|
||||
const bullets = enforced.filter((l) => /^\s*[-*]\s/.test(l));
|
||||
if (bullets.length === 0) errors.push(err('SKILL_ENFORCED_BY_VAGUE', '"Enforced by" lists no check: name one, or say "prose only"'));
|
||||
for (const b of bullets) {
|
||||
if (!/`[^`]+`/.test(b) && !/prose only/i.test(b)) {
|
||||
errors.push(err('SKILL_ENFORCED_BY_VAGUE', `"${b.trim().slice(0, 70)}": name the check (a backticked path) or say "prose only"`));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const fullBody = split.ok ? split.body : text;
|
||||
if (/!`[^`]+`/.test(fullBody) || /^\s*```!/m.test(fullBody)) {
|
||||
errors.push(err('SKILL_SHELL_INJECTION', 'shell injection (!`cmd` or a ```! block) runs on load; Voyage advises users to turn it off, so a skill may not depend on it'));
|
||||
}
|
||||
|
||||
// A credit bullet that names another repository at a commit ("owner/repo
|
||||
// @abc1234, `skills/tdd/SKILL.md`") cites a path in THAT repository; its
|
||||
// backticked paths are not looked up here. Every other line is.
|
||||
const external = new Set();
|
||||
for (let i = 0; i < bodyLines.length; i++) {
|
||||
if (!/^\s*[-*]\s/.test(bodyLines[i])) continue;
|
||||
let j = i + 1;
|
||||
while (j < bodyLines.length && /^\s{2,}\S/.test(bodyLines[j]) && !/^\s*[-*]\s/.test(bodyLines[j])) j++;
|
||||
if (/[\w.-]+\/[\w.-]+ @[0-9a-f]{7,40}\b/.test(bodyLines.slice(i, j).join(' '))) for (let k = i; k < j; k++) external.add(k);
|
||||
}
|
||||
const refs = new Set();
|
||||
for (const [i, l] of bodyLines.entries()) {
|
||||
for (const m of l.matchAll(/\]\(([^)\s#]+)(#[^)]*)?\)/g)) {
|
||||
if (!/^[a-z]+:/.test(m[1])) refs.add(JSON.stringify(['skill', m[1]]));
|
||||
}
|
||||
if (external.has(i)) continue;
|
||||
for (const m of l.matchAll(/`([^`\s]+)`/g)) {
|
||||
const p = m[1].replace(/^\$\{CLAUDE_PLUGIN_ROOT\}\//, '');
|
||||
if (PATH_PREFIXES.some((x) => p.startsWith(x)) && /\.[a-z]+$/.test(p) && !/[{}*<>]/.test(p)) refs.add(JSON.stringify(['plugin', p]));
|
||||
}
|
||||
}
|
||||
for (const r of refs) {
|
||||
const [base, p] = JSON.parse(r);
|
||||
const abs = base === 'skill' ? join(skillDir, p) : join(pluginRoot, p);
|
||||
if (!existsSync(abs)) errors.push(err('SKILL_REF_MISSING', `${p} does not exist (${base === 'skill' ? 'relative to the skill' : 'relative to the plugin root'})`));
|
||||
}
|
||||
|
||||
// Exemptions held in the check
|
||||
const ex = EXEMPTIONS.get(name);
|
||||
const kept = ex ? errors.filter((e) => !ex.codes.includes(e.code)) : errors;
|
||||
return { name, errors: kept, warnings };
|
||||
}
|
||||
|
||||
/** Lint every skills/<name>/ under a plugin root. */
|
||||
export function lintSkillsRoot(pluginRoot) {
|
||||
const dir = join(pluginRoot, 'skills');
|
||||
const skills = {};
|
||||
if (!existsSync(dir)) return { skills };
|
||||
for (const name of readdirSync(dir).sort()) {
|
||||
const p = join(dir, name);
|
||||
if (!statSync(p).isDirectory()) continue;
|
||||
skills[name] = lintSkill(p, { pluginRoot });
|
||||
}
|
||||
return { skills };
|
||||
}
|
||||
|
||||
// ---- CLI ----
|
||||
|
||||
function isMain() {
|
||||
if (!process.argv[1]) return false;
|
||||
try { return realpathSync(fileURLToPath(import.meta.url)) === realpathSync(process.argv[1]); } catch { return false; }
|
||||
}
|
||||
|
||||
if (isMain()) {
|
||||
const args = process.argv.slice(2);
|
||||
const at = args.indexOf('--root');
|
||||
const root = resolve(at >= 0 ? args[at + 1] || '' : join(dirname(fileURLToPath(import.meta.url)), '..', '..'));
|
||||
const json = args.includes('--json');
|
||||
if (!existsSync(root) || !existsSync(join(root, 'skills'))) {
|
||||
process.stderr.write(`skill-lint: not measured: no skills/ directory under ${root}\n`);
|
||||
process.exit(2);
|
||||
}
|
||||
const r = lintSkillsRoot(root);
|
||||
const bad = Object.values(r.skills).filter((s) => s.errors.length);
|
||||
if (json) process.stdout.write(`${JSON.stringify(r, null, 2)}\n`);
|
||||
else {
|
||||
for (const s of Object.values(r.skills)) {
|
||||
process.stdout.write(`${s.errors.length ? 'FAIL' : 'ok '} ${s.name}\n`);
|
||||
for (const e of s.errors) process.stdout.write(` ${e.code}${e.line ? ` (line ${e.line})` : ''}: ${e.message}\n`);
|
||||
}
|
||||
process.stdout.write(`${Object.keys(r.skills).length} skill(s), ${bad.length} with findings\n`);
|
||||
}
|
||||
process.exit(bad.length ? 1 : 0);
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Which commands and agents ask for each skill ("the commands are the trunk").
|
||||
* Asked for = an agent's `skills:` list holds `voyage:<name>` or `<name>`, or a
|
||||
* command or agent file names `voyage:<name>` in its text. A bare word in prose
|
||||
* is not a request. Returns { <skill>: ['agents/x.md', 'commands/y.md', ...] }.
|
||||
*/
|
||||
export function findSkillUsers(pluginRoot) {
|
||||
const dir = join(pluginRoot, 'skills');
|
||||
const names = existsSync(dir) ? readdirSync(dir).filter((n) => existsSync(join(dir, n, 'SKILL.md'))).sort() : [];
|
||||
const users = Object.fromEntries(names.map((n) => [n, []]));
|
||||
for (const sub of ['agents', 'commands']) {
|
||||
const d = join(pluginRoot, sub);
|
||||
if (!existsSync(d)) continue;
|
||||
for (const f of readdirSync(d).filter((x) => x.endsWith('.md')).sort()) {
|
||||
const text = readFileSync(join(d, f), 'utf8');
|
||||
const listed = readFrontmatterFields(text).fields.skills;
|
||||
const preload = listed && Array.isArray(listed.value) ? listed.value.map((v) => String(v).replace(/^voyage:/, '')) : [];
|
||||
for (const n of names) {
|
||||
const named = new RegExp(`(^|[^A-Za-z0-9_-])voyage:${n}(?![A-Za-z0-9_-])`).test(text);
|
||||
if (preload.includes(n) || named) users[n].push(`${sub}/${f}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
return users;
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue