feat(skills): skill workshop, test-first and debugging preloaded into the step helper
The workshop (ordre 7a, contracts S1 and S2): a form check (lib/skills/skill-lint.mjs) that follows Claude Code's own skill fields, and a model-free routing test (lib/skills/routing.mjs) over every skill, the 7 commands and the 21 spawnable agents, reading the eval cases themselves (evals/skill-*, 3 fire / 2 quiet / 1 effect, tagged with the event they came from). Thresholds pinned, never lowered. The two skills were written against failures measured first in helpers without them: red only as an import crash, the whole suite run from a hand-made file list instead of the repo's own command, and an unreproduced fix shipped under authority pressure. step-implementer preloads both through its skills: line; no tool list change. Credits: addyosmani/agent-skills @9d0c60d, obra/superpowers @8ca22db, mattpocock/skills @c55ee46 (MIT), Anthropic skill-creator (Apache-2.0). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
4ee711a6ac
commit
cc832fd430
SSH key fingerprint:
SHA256:JakMjO6FTBBzN0Bhfj9saOoEjaFxlSdYuZQQpM/lF9Q
41 changed files with 1996 additions and 0 deletions
251
tests/lib/skill-lint.test.mjs
Normal file
251
tests/lib/skill-lint.test.mjs
Normal file
|
|
@ -0,0 +1,251 @@
|
|||
// tests/lib/skill-lint.test.mjs
|
||||
// The skill form check (S1): lib/skills/skill-lint.mjs. Model-free, part of
|
||||
// `npm test`. Each rule is shown biting on a skill built for it (known
|
||||
// positive) and passing a skill that keeps the form (known negative); the last
|
||||
// tests run the check over the skills Voyage ships.
|
||||
|
||||
import { test } from 'node:test';
|
||||
import { strict as assert } from 'node:assert';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { dirname, join } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { lintSkill, lintSkillsRoot, REQUIRED_HEADINGS, EXEMPTIONS } from '../../lib/skills/skill-lint.mjs';
|
||||
import { readFrontmatterFields } from '../../lib/skills/skill-frontmatter.mjs';
|
||||
|
||||
const HERE = dirname(fileURLToPath(import.meta.url));
|
||||
const ROOT = join(HERE, '..', '..');
|
||||
const CLI = join(ROOT, 'lib', 'skills', 'skill-lint.mjs');
|
||||
|
||||
const BODY = [
|
||||
'# Example', '',
|
||||
'## Overview', 'One line.', '',
|
||||
'## When to use', '- when the example applies', '- not when it does not', '',
|
||||
'## Process', '1. Do the thing.', '',
|
||||
'## Red flags', '- skipping the thing', '',
|
||||
'## Enforced by', '- prose only: nothing checks this.', '',
|
||||
'## Credits', '- Voyage\'s own content.', '',
|
||||
].join('\n');
|
||||
|
||||
const GOOD_FM = [
|
||||
'---',
|
||||
'name: good-skill',
|
||||
'description: Checks an example the same way every time. Use when an example needs checking.',
|
||||
'---',
|
||||
].join('\n');
|
||||
|
||||
/** A plugin-shaped temp root with skills/<name>/SKILL.md; returns the root. */
|
||||
function plugin(skills) {
|
||||
const root = mkdtempSync(join(tmpdir(), 'skill-lint-'));
|
||||
for (const [name, text] of Object.entries(skills)) {
|
||||
mkdirSync(join(root, 'skills', name), { recursive: true });
|
||||
writeFileSync(join(root, 'skills', name, 'SKILL.md'), text);
|
||||
}
|
||||
return root;
|
||||
}
|
||||
|
||||
const codes = (r) => r.errors.map((e) => e.code).sort();
|
||||
|
||||
function withSkill(name, text, fn) {
|
||||
const root = plugin({ [name]: text });
|
||||
try { return fn(lintSkill(join(root, 'skills', name), { pluginRoot: root }), root); }
|
||||
finally { rmSync(root, { recursive: true, force: true }); }
|
||||
}
|
||||
|
||||
// ---- the test that fails first (ordre 7a) ----
|
||||
|
||||
test('skill-lint: a skill without "Use when" and with an unquoted colon is red', () => {
|
||||
const text = [
|
||||
'---',
|
||||
'name: bad-skill',
|
||||
'description: Checks things: all of them, every time.',
|
||||
'---',
|
||||
BODY,
|
||||
].join('\n');
|
||||
withSkill('bad-skill', text, (r) => {
|
||||
assert.ok(codes(r).includes('SKILL_YAML_HAZARD'), codes(r).join(','));
|
||||
assert.ok(codes(r).includes('SKILL_DESC_NO_USE_WHEN'), codes(r).join(','));
|
||||
});
|
||||
});
|
||||
|
||||
test('skill-lint: a skill that keeps the form is green (known negative)', () => {
|
||||
withSkill('good-skill', `${GOOD_FM}\n${BODY}`, (r) => assert.deepEqual(r.errors, []));
|
||||
});
|
||||
|
||||
// ---- one rule at a time ----
|
||||
|
||||
test('skill-lint: name must equal the directory name and be kebab-case', () => {
|
||||
withSkill('other-dir', `${GOOD_FM}\n${BODY}`, (r) => assert.ok(codes(r).includes('SKILL_NAME_MISMATCH')));
|
||||
const fm = GOOD_FM.replace('name: good-skill', 'name: Good_Skill');
|
||||
withSkill('Good_Skill', `${fm}\n${BODY}`, (r) => assert.ok(codes(r).includes('SKILL_NAME_FORM')));
|
||||
const reserved = GOOD_FM.replace('name: good-skill', 'name: claude-helper');
|
||||
withSkill('claude-helper', `${reserved}\n${BODY}`, (r) => assert.ok(codes(r).includes('SKILL_NAME_FORM')));
|
||||
});
|
||||
|
||||
test('skill-lint: description is required, at most 1 024 characters', () => {
|
||||
const none = GOOD_FM.replace(/^description:.*$/m, 'license: MIT');
|
||||
withSkill('good-skill', `${none}\n${BODY}`, (r) => assert.ok(codes(r).includes('SKILL_DESC_MISSING')));
|
||||
const long = `Checks an example. Use when ${'an example needs checking and '.repeat(40)}done.`;
|
||||
assert.ok(long.length > 1024);
|
||||
const fm = GOOD_FM.replace(/^description:.*$/m, `description: ${long}`);
|
||||
withSkill('good-skill', `${fm}\n${BODY}`, (r) => assert.ok(codes(r).includes('SKILL_DESC_TOO_LONG')));
|
||||
const exact = `Use when x. ${'y'.repeat(1024 - 12)}`;
|
||||
assert.equal(exact.length, 1024);
|
||||
const fm2 = GOOD_FM.replace(/^description:.*$/m, `description: ${exact}`);
|
||||
withSkill('good-skill', `${fm2}\n${BODY}`, (r) => assert.ok(!codes(r).includes('SKILL_DESC_TOO_LONG')));
|
||||
});
|
||||
|
||||
test('skill-lint: description is in the third person', () => {
|
||||
for (const d of [
|
||||
'I check an example. Use when an example needs checking.',
|
||||
'Checks your example. Use when an example needs checking.',
|
||||
'Use when you need an example checked.',
|
||||
'We check examples. Use when an example needs checking.',
|
||||
]) {
|
||||
const fm = GOOD_FM.replace(/^description:.*$/m, `description: ${d}`);
|
||||
withSkill('good-skill', `${fm}\n${BODY}`, (r) => assert.ok(codes(r).includes('SKILL_DESC_NOT_THIRD_PERSON'), d));
|
||||
}
|
||||
const ok = GOOD_FM.replace(/^description:.*$/m, 'description: Checks an example, i.e. one case. Use when the user asks for a check.');
|
||||
withSkill('good-skill', `${ok}\n${BODY}`, (r) => assert.ok(!codes(r).includes('SKILL_DESC_NOT_THIRD_PERSON')));
|
||||
});
|
||||
|
||||
test('skill-lint: YAML traps are caught: unquoted colon, comment marker, tab, unclosed quote, leading indicator', () => {
|
||||
for (const line of [
|
||||
'description: Use when x: y happens.',
|
||||
'description: Use when x happens #1.',
|
||||
'description:\tUse when x happens.',
|
||||
'description: "Use when x happens.',
|
||||
'description: `Use` when x happens.',
|
||||
'description: @Use when x happens.',
|
||||
]) {
|
||||
const fm = GOOD_FM.replace(/^description:.*$/m, line);
|
||||
withSkill('good-skill', `${fm}\n${BODY}`, (r) => assert.ok(codes(r).includes('SKILL_YAML_HAZARD'), line));
|
||||
}
|
||||
// Quoted, the colon is safe.
|
||||
const quoted = GOOD_FM.replace(/^description:.*$/m, 'description: "Checks x: y. Use when x happens."');
|
||||
withSkill('good-skill', `${quoted}\n${BODY}`, (r) => assert.deepEqual(r.errors, []));
|
||||
});
|
||||
|
||||
test('skill-lint: a multi-line block-scalar description is read whole', () => {
|
||||
const fm = ['---', 'name: good-skill', 'description: |', ' Checks an example the same way: every time.',
|
||||
' Use when an example needs checking.', '---'].join('\n');
|
||||
withSkill('good-skill', `${fm}\n${BODY}`, (r) => assert.deepEqual(r.errors, []));
|
||||
const f = readFrontmatterFields(`${fm}\n${BODY}`);
|
||||
assert.equal(f.fields.description.value, 'Checks an example the same way: every time.\nUse when an example needs checking.\n');
|
||||
const folded = readFrontmatterFields(['---', 'description: >-', ' one', ' two', '---', ''].join('\n'));
|
||||
assert.equal(folded.fields.description.value, 'one two');
|
||||
});
|
||||
|
||||
test('skill-lint: disable-model-invocation in any true spelling is refused', () => {
|
||||
for (const v of ['true', 'True', 'yes', 'on', '1']) {
|
||||
const fm = GOOD_FM.replace('---\nname', `---\ndisable-model-invocation: ${v}\nname`);
|
||||
withSkill('good-skill', `${fm}\n${BODY}`, (r) => assert.ok(codes(r).includes('SKILL_MODEL_INVOCATION_DISABLED'), v));
|
||||
}
|
||||
const off = GOOD_FM.replace('---\nname', '---\ndisable-model-invocation: false\nname');
|
||||
withSkill('good-skill', `${off}\n${BODY}`, (r) => assert.deepEqual(r.errors, []));
|
||||
});
|
||||
|
||||
test('skill-lint: only Claude Code\'s own frontmatter fields are accepted', () => {
|
||||
const fm = GOOD_FM.replace('---\nname', '---\ntrigger: always\nname');
|
||||
withSkill('good-skill', `${fm}\n${BODY}`, (r) => assert.ok(codes(r).includes('SKILL_UNKNOWN_FIELD')));
|
||||
const known = GOOD_FM.replace('---\nname', '---\nlicense: MIT\nuser-invocable: true\nname');
|
||||
withSkill('good-skill', `${known}\n${BODY}`, (r) => assert.deepEqual(r.errors, []));
|
||||
});
|
||||
|
||||
test('skill-lint: SKILL.md must stay under 500 lines', () => {
|
||||
const pad = Array.from({ length: 500 }, (_, i) => `- line ${i}`).join('\n');
|
||||
withSkill('good-skill', `${GOOD_FM}\n${BODY}\n${pad}\n`, (r) => assert.ok(codes(r).includes('SKILL_TOO_LONG')));
|
||||
});
|
||||
|
||||
test('skill-lint: the fixed headings must all be there, in order', () => {
|
||||
assert.deepEqual(REQUIRED_HEADINGS, ['Overview', 'When to use', 'Process', 'Red flags', 'Enforced by', 'Credits']);
|
||||
withSkill('good-skill', `${GOOD_FM}\n${BODY.replace('## Red flags', '## Warnings')}`,
|
||||
(r) => assert.ok(codes(r).includes('SKILL_MISSING_HEADING')));
|
||||
const swapped = BODY.replace('## Overview\nOne line.', '## Process\n1. x.').replace('## Process\n1. Do the thing.', '## Overview\nOne line.');
|
||||
withSkill('good-skill', `${GOOD_FM}\n${swapped}`, (r) => assert.ok(codes(r).includes('SKILL_HEADING_ORDER')));
|
||||
// A heading inside a fenced code block does not count.
|
||||
const fenced = BODY.replace('## Credits', '```\n## Credits\n```');
|
||||
withSkill('good-skill', `${GOOD_FM}\n${fenced}`, (r) => assert.ok(codes(r).includes('SKILL_MISSING_HEADING')));
|
||||
});
|
||||
|
||||
test('skill-lint: "Enforced by" names a check that exists, or says prose only', () => {
|
||||
const none = BODY.replace('- prose only: nothing checks this.', '- something, somewhere.');
|
||||
withSkill('good-skill', `${GOOD_FM}\n${none}`, (r) => assert.ok(codes(r).includes('SKILL_ENFORCED_BY_VAGUE')));
|
||||
const real = BODY.replace('- prose only: nothing checks this.', '- `lib/check.mjs` stops it.');
|
||||
const root = plugin({ 'good-skill': `${GOOD_FM}\n${real}` });
|
||||
try {
|
||||
assert.ok(codes(lintSkill(join(root, 'skills', 'good-skill'), { pluginRoot: root })).includes('SKILL_REF_MISSING'));
|
||||
mkdirSync(join(root, 'lib'));
|
||||
writeFileSync(join(root, 'lib', 'check.mjs'), '');
|
||||
assert.deepEqual(lintSkill(join(root, 'skills', 'good-skill'), { pluginRoot: root }).errors, []);
|
||||
} finally { rmSync(root, { recursive: true, force: true }); }
|
||||
});
|
||||
|
||||
test('skill-lint: files the skill points to must exist', () => {
|
||||
const linked = BODY.replace('One line.', 'One line. See [the detail](reference/detail.md).');
|
||||
const root = plugin({ 'good-skill': `${GOOD_FM}\n${linked}` });
|
||||
try {
|
||||
assert.ok(codes(lintSkill(join(root, 'skills', 'good-skill'), { pluginRoot: root })).includes('SKILL_REF_MISSING'));
|
||||
mkdirSync(join(root, 'skills', 'good-skill', 'reference'));
|
||||
writeFileSync(join(root, 'skills', 'good-skill', 'reference', 'detail.md'), '# detail\n');
|
||||
assert.deepEqual(lintSkill(join(root, 'skills', 'good-skill'), { pluginRoot: root }).errors, []);
|
||||
} finally { rmSync(root, { recursive: true, force: true }); }
|
||||
});
|
||||
|
||||
test('skill-lint: paths under "Credits" name the source repository, not this plugin', () => {
|
||||
const credited = BODY.replace("- Voyage's own content.", '- owner/repo @abc1234, `skills/tdd/SKILL.md`: MIT.');
|
||||
withSkill('good-skill', `${GOOD_FM}\n${credited}`, (r) => assert.deepEqual(r.errors, []));
|
||||
// The same path anywhere else must exist.
|
||||
const elsewhere = BODY.replace('One line.', 'One line, see `skills/tdd/SKILL.md`.');
|
||||
withSkill('good-skill', `${GOOD_FM}\n${elsewhere}`, (r) => assert.ok(codes(r).includes('SKILL_REF_MISSING')));
|
||||
});
|
||||
|
||||
test('skill-lint: no shell injection (inline or fenced), since Voyage advises turning it off', () => {
|
||||
withSkill('good-skill', `${GOOD_FM}\n${BODY.replace('One line.', 'Status: !`git status`')}`,
|
||||
(r) => assert.ok(codes(r).includes('SKILL_SHELL_INJECTION')));
|
||||
withSkill('good-skill', `${GOOD_FM}\n${BODY.replace('One line.', '```!\ngit status\n```')}`,
|
||||
(r) => assert.ok(codes(r).includes('SKILL_SHELL_INJECTION')));
|
||||
// A plain fenced bash block is fine; Claude reads it, nothing runs it on load.
|
||||
withSkill('good-skill', `${GOOD_FM}\n${BODY.replace('One line.', '```bash\nnpm test\n```')}`,
|
||||
(r) => assert.deepEqual(r.errors, []));
|
||||
});
|
||||
|
||||
test('skill-lint: an exemption lives in the check, never in the skill', () => {
|
||||
assert.ok(EXEMPTIONS instanceof Map);
|
||||
for (const [name, entry] of EXEMPTIONS) {
|
||||
assert.ok(Array.isArray(entry.codes) && entry.codes.length > 0, `${name}: exempt codes`);
|
||||
assert.ok(typeof entry.reason === 'string' && entry.reason.length > 10, `${name}: a reason`);
|
||||
}
|
||||
const self = `${GOOD_FM}\n<!-- skill-lint-exempt: SKILL_MISSING_HEADING -->\n${BODY}`;
|
||||
withSkill('good-skill', self, (r) => assert.ok(codes(r).includes('SKILL_SELF_EXEMPTION')));
|
||||
const meta = GOOD_FM.replace('---\nname', '---\nmetadata:\n lint-exempt: true\nname');
|
||||
withSkill('good-skill', `${meta}\n${BODY}`, (r) => assert.ok(codes(r).includes('SKILL_SELF_EXEMPTION')));
|
||||
});
|
||||
|
||||
test('skill-lint CLI: 0 clean, 1 findings, 2 could not run', () => {
|
||||
const good = plugin({ 'good-skill': `${GOOD_FM}\n${BODY}` });
|
||||
const bad = plugin({ 'bad-skill': `---\nname: bad-skill\ndescription: Checks: things.\n---\n${BODY}` });
|
||||
try {
|
||||
const g = spawnSync('node', [CLI, '--root', good], { encoding: 'utf8' });
|
||||
assert.equal(g.status, 0, g.stdout + g.stderr);
|
||||
const b = spawnSync('node', [CLI, '--root', bad, '--json'], { encoding: 'utf8' });
|
||||
assert.equal(b.status, 1, b.stdout + b.stderr);
|
||||
assert.ok(JSON.parse(b.stdout).skills['bad-skill'].errors.length > 0);
|
||||
const n = spawnSync('node', [CLI, '--root', join(good, 'nope')], { encoding: 'utf8' });
|
||||
assert.equal(n.status, 2, n.stdout + n.stderr);
|
||||
} finally {
|
||||
rmSync(good, { recursive: true, force: true });
|
||||
rmSync(bad, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
// ---- the skills Voyage ships ----
|
||||
|
||||
test('skill-lint: every skill Voyage ships passes the form check', () => {
|
||||
const r = lintSkillsRoot(ROOT);
|
||||
assert.ok(Object.keys(r.skills).length >= 2, `found ${Object.keys(r.skills).join(',')}`);
|
||||
for (const [name, res] of Object.entries(r.skills)) {
|
||||
assert.deepEqual(res.errors, [], `${name}: ${JSON.stringify(res.errors, null, 2)}`);
|
||||
}
|
||||
});
|
||||
215
tests/lib/skill-routing.test.mjs
Normal file
215
tests/lib/skill-routing.test.mjs
Normal file
|
|
@ -0,0 +1,215 @@
|
|||
// tests/lib/skill-routing.test.mjs
|
||||
// The model-free routing test (P10, "Bygg nå"): lib/skills/routing.mjs ranks
|
||||
// the descriptions of every skill, the 7 commands and the 21 spawnable agents
|
||||
// against prompts, with no model. A collision between two descriptions fails;
|
||||
// each skill needs 3 prompts where it should fire (top 3) and 2 where it
|
||||
// should not, and for those a named owner must rank higher. The thresholds are
|
||||
// pinned here: they are never lowered.
|
||||
|
||||
import { test } from 'node:test';
|
||||
import { strict as assert } from 'node:assert';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { dirname, join } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import {
|
||||
THRESHOLDS, tokenize, buildCorpus, rank, collisions, loadEntries, loadCases, checkRouting,
|
||||
} from '../../lib/skills/routing.mjs';
|
||||
|
||||
const HERE = dirname(fileURLToPath(import.meta.url));
|
||||
const ROOT = join(HERE, '..', '..');
|
||||
const CLI = join(ROOT, 'lib', 'skills', 'routing.mjs');
|
||||
|
||||
const skillMd = (name, description) =>
|
||||
`---\nname: ${name}\ndescription: ${description}\n---\n\n# ${name}\n`;
|
||||
|
||||
/** Plugin-shaped temp root: skills, commands, agents, evals. */
|
||||
function plugin({ skills = {}, commands = {}, agents = {}, cases = {} }) {
|
||||
const root = mkdtempSync(join(tmpdir(), 'skill-routing-'));
|
||||
for (const [n, d] of Object.entries(skills)) {
|
||||
mkdirSync(join(root, 'skills', n), { recursive: true });
|
||||
writeFileSync(join(root, 'skills', n, 'SKILL.md'), skillMd(n, d));
|
||||
}
|
||||
mkdirSync(join(root, 'commands'), { recursive: true });
|
||||
for (const [n, d] of Object.entries(commands)) {
|
||||
writeFileSync(join(root, 'commands', `${n}.md`), `---\nname: ${n}\ndescription: ${d}\n---\n`);
|
||||
}
|
||||
mkdirSync(join(root, 'agents'), { recursive: true });
|
||||
for (const [n, d] of Object.entries(agents)) {
|
||||
writeFileSync(join(root, 'agents', `${n}.md`), `---\nname: ${n}\ndescription: |\n ${d}\nmodel: opus\n---\n`);
|
||||
}
|
||||
for (const [n, { tags, prompt }] of Object.entries(cases)) {
|
||||
mkdirSync(join(root, 'evals', n, 'graders'), { recursive: true });
|
||||
writeFileSync(join(root, 'evals', n, 'prompt.md'),
|
||||
`---\ndescription: "case ${n}"\ntags: [${tags.join(', ')}]\nruns: 1\n---\n\n${prompt}\n`);
|
||||
}
|
||||
return root;
|
||||
}
|
||||
|
||||
function withPlugin(spec, fn) {
|
||||
const root = plugin(spec);
|
||||
try { return fn(root); } finally { rmSync(root, { recursive: true, force: true }); }
|
||||
}
|
||||
|
||||
const codes = (r) => r.errors.map((e) => e.code);
|
||||
|
||||
// ---- the test that fails first (ordre 7a) ----
|
||||
|
||||
test('routing: two test skills with near-identical descriptions are red (collision)', () => {
|
||||
withPlugin({
|
||||
skills: {
|
||||
'tidy-imports': 'Sorts and groups the import lines of a source file. Use when imports are out of order.',
|
||||
'order-imports': 'Sorts and groups the import lines in a source file. Use when the imports are out of order.',
|
||||
},
|
||||
}, (root) => {
|
||||
const r = checkRouting(root);
|
||||
assert.ok(codes(r).includes('ROUTING_COLLISION'), JSON.stringify(r.errors));
|
||||
const c = r.errors.find((e) => e.code === 'ROUTING_COLLISION');
|
||||
assert.deepEqual([c.a, c.b].sort(), ['skill:order-imports', 'skill:tidy-imports']);
|
||||
});
|
||||
});
|
||||
|
||||
test('routing: distinct descriptions do not collide (known negative)', () => {
|
||||
const corpus = buildCorpus([
|
||||
{ id: 'skill:tidy-imports', name: 'tidy-imports', description: 'Sorts the import lines of a source file. Use when imports are out of order.' },
|
||||
{ id: 'skill:rotate-logs', name: 'rotate-logs', description: 'Compresses and deletes old log files on disk. Use when the log directory grows.' },
|
||||
]);
|
||||
assert.deepEqual(collisions(corpus).filter((c) => c.level === 'error'), []);
|
||||
});
|
||||
|
||||
// ---- the pieces ----
|
||||
|
||||
test('routing: thresholds are pinned and never lowered', () => {
|
||||
assert.deepEqual({ ...THRESHOLDS }, {
|
||||
COLLISION_ERROR: 0.75,
|
||||
COLLISION_WARN: 0.5,
|
||||
POSITIVE_TOP_K: 3,
|
||||
MIN_POSITIVE: 3,
|
||||
MIN_NEGATIVE: 2,
|
||||
MIN_EFFECT: 1,
|
||||
MIN_RANK1_RATE: 1,
|
||||
});
|
||||
assert.ok(Object.isFrozen(THRESHOLDS));
|
||||
});
|
||||
|
||||
test('routing: the tokenizer folds case, drops stop words and light-stems', () => {
|
||||
assert.deepEqual(tokenize('Use when the Tests are FAILING'), tokenize('test fail'));
|
||||
assert.deepEqual(tokenize('a an the of'), []);
|
||||
});
|
||||
|
||||
test('routing: rank puts the matching description first, and a no-overlap prompt scores 0', () => {
|
||||
const corpus = buildCorpus([
|
||||
{ id: 'skill:tidy-imports', name: 'tidy-imports', description: 'Sorts the import lines of a source file.' },
|
||||
{ id: 'skill:rotate-logs', name: 'rotate-logs', description: 'Compresses and deletes old log files on disk.' },
|
||||
]);
|
||||
assert.equal(rank('the log files fill the disk', corpus)[0].id, 'skill:rotate-logs');
|
||||
assert.ok(rank('zebra quantum', corpus).every((r) => r.score === 0));
|
||||
});
|
||||
|
||||
test('routing: the reader takes block-scalar agent descriptions and skips orchestrator reference docs', () => {
|
||||
withPlugin({
|
||||
agents: { 'plan-critic': 'Use this agent when a plan needs adversarial review.' },
|
||||
commands: { trekplan: 'Deep implementation planning from a task brief.' },
|
||||
}, (root) => {
|
||||
writeFileSync(join(root, 'agents', 'planning-orchestrator.md'),
|
||||
'---\nname: planning-orchestrator\ndescription: Reference document, not a spawnable capability.\n---\n');
|
||||
const ids = loadEntries(root).map((e) => e.id).sort();
|
||||
assert.deepEqual(ids, ['agent:plan-critic', 'command:trekplan']);
|
||||
assert.equal(loadEntries(root).find((e) => e.id === 'agent:plan-critic').description.trim(),
|
||||
'Use this agent when a plan needs adversarial review.');
|
||||
});
|
||||
});
|
||||
|
||||
test('routing: cases come from evals/ tags; a skill below 3 / 2 / 1 is red', () => {
|
||||
withPlugin({
|
||||
skills: { 'tidy-imports': 'Sorts the import lines of a source file. Use when imports are out of order.' },
|
||||
cases: {
|
||||
'skill-tidy-imports-fires-1': { tags: ['skill-trigger', 'skill=tidy-imports', 'from=test'], prompt: 'sort the import lines in main.js' },
|
||||
},
|
||||
}, (root) => {
|
||||
const cases = loadCases(root);
|
||||
assert.equal(cases.length, 1);
|
||||
assert.deepEqual([cases[0].kind, cases[0].skill, cases[0].from], ['trigger', 'tidy-imports', 'test']);
|
||||
assert.ok(codes(checkRouting(root)).includes('ROUTING_BELOW_MINIMUM'));
|
||||
});
|
||||
});
|
||||
|
||||
test('routing: a positive outside the top 3 is red; a negative won by the skill itself is red', () => {
|
||||
const spec = {
|
||||
skills: { 'tidy-imports': 'Sorts the import lines of a source file. Use when imports are out of order.' },
|
||||
commands: {
|
||||
'rotate-logs': 'Compresses and deletes old log files on disk.',
|
||||
'scan-ports': 'Lists the open network ports of a host.',
|
||||
'count-words': 'Counts the words of a text document.',
|
||||
},
|
||||
cases: {
|
||||
'p1': { tags: ['skill-trigger', 'skill=tidy-imports', 'from=test'], prompt: 'sort the import lines' },
|
||||
'p2': { tags: ['skill-trigger', 'skill=tidy-imports', 'from=test'], prompt: 'import lines are out of order' },
|
||||
'p3': { tags: ['skill-trigger', 'skill=tidy-imports', 'from=test'], prompt: 'compress the old logs on disk and list ports and count words' },
|
||||
'n1': { tags: ['skill-no-trigger', 'skill=tidy-imports', 'owner=rotate-logs', 'from=test'], prompt: 'delete old log files' },
|
||||
'n2': { tags: ['skill-no-trigger', 'skill=tidy-imports', 'owner=rotate-logs', 'from=test'], prompt: 'sort the import lines of the log file' },
|
||||
'e1': { tags: ['skill-effect', 'skill=tidy-imports', 'from=test'], prompt: 'x' },
|
||||
},
|
||||
};
|
||||
withPlugin(spec, (root) => {
|
||||
const r = checkRouting(root);
|
||||
assert.ok(r.errors.some((e) => e.code === 'ROUTING_POSITIVE_MISSED' && e.case === 'p3'), JSON.stringify(r.errors));
|
||||
assert.ok(r.errors.some((e) => e.code === 'ROUTING_NEGATIVE_WON' && e.case === 'n2'), JSON.stringify(r.errors));
|
||||
assert.ok(!r.errors.some((e) => e.case === 'n1'), JSON.stringify(r.errors));
|
||||
});
|
||||
});
|
||||
|
||||
test('routing: a negative must name an owner that exists and outranks the skill', () => {
|
||||
withPlugin({
|
||||
skills: { 'tidy-imports': 'Sorts the import lines of a source file. Use when imports are out of order.' },
|
||||
cases: {
|
||||
'n1': { tags: ['skill-no-trigger', 'skill=tidy-imports', 'from=test'], prompt: 'delete old log files' },
|
||||
'n2': { tags: ['skill-no-trigger', 'skill=tidy-imports', 'owner=nobody', 'from=test'], prompt: 'delete old log files' },
|
||||
},
|
||||
}, (root) => {
|
||||
const r = checkRouting(root);
|
||||
assert.ok(r.errors.some((e) => e.code === 'ROUTING_NEGATIVE_NO_OWNER' && e.case === 'n1'), JSON.stringify(r.errors));
|
||||
assert.ok(r.errors.some((e) => e.code === 'ROUTING_UNKNOWN_OWNER' && e.case === 'n2'), JSON.stringify(r.errors));
|
||||
});
|
||||
});
|
||||
|
||||
test('routing: every case names the event it came from (P10)', () => {
|
||||
withPlugin({
|
||||
skills: { 'tidy-imports': 'Sorts the import lines of a source file. Use when imports are out of order.' },
|
||||
cases: { 'p1': { tags: ['skill-trigger', 'skill=tidy-imports'], prompt: 'sort the import lines' } },
|
||||
}, (root) => assert.ok(codes(checkRouting(root)).includes('ROUTING_CASE_NO_SOURCE')));
|
||||
});
|
||||
|
||||
test('routing CLI: 0 clean, 1 findings, 2 could not run', () => {
|
||||
withPlugin({
|
||||
skills: {
|
||||
'tidy-imports': 'Sorts and groups the import lines of a source file. Use when imports are out of order.',
|
||||
'order-imports': 'Sorts and groups the import lines in a source file. Use when the imports are out of order.',
|
||||
},
|
||||
}, (root) => {
|
||||
const b = spawnSync('node', [CLI, '--root', root, '--json'], { encoding: 'utf8' });
|
||||
assert.equal(b.status, 1, b.stdout + b.stderr);
|
||||
assert.ok(JSON.parse(b.stdout).errors.length > 0);
|
||||
const n = spawnSync('node', [CLI, '--root', join(root, 'nope')], { encoding: 'utf8' });
|
||||
assert.equal(n.status, 2, n.stdout + n.stderr);
|
||||
});
|
||||
});
|
||||
|
||||
// ---- Voyage's own catalogue ----
|
||||
|
||||
test('routing: Voyage\'s catalogue holds every skill, the 7 commands and the 21 spawnable agents', () => {
|
||||
const entries = loadEntries(ROOT);
|
||||
const by = (k) => entries.filter((e) => e.kind === k);
|
||||
assert.equal(by('command').length, 7, by('command').map((e) => e.name).join(','));
|
||||
assert.equal(by('agent').length, 21, by('agent').map((e) => e.name).join(','));
|
||||
assert.ok(by('skill').length >= 2, by('skill').map((e) => e.name).join(','));
|
||||
for (const e of entries) assert.ok(e.description.trim().length > 20, `${e.id}: description read as "${e.description}"`);
|
||||
assert.equal(new Set(entries.map((e) => e.name)).size, entries.length, 'names must be unique across kinds');
|
||||
});
|
||||
|
||||
test('routing: Voyage\'s catalogue is green: no collision, every case routes, rank-1 floor held', () => {
|
||||
const r = checkRouting(ROOT);
|
||||
assert.deepEqual(r.errors, [], JSON.stringify(r.errors, null, 2));
|
||||
assert.ok(r.stats.positives >= 6 && r.stats.negatives >= 4, JSON.stringify(r.stats));
|
||||
});
|
||||
150
tests/lib/skills-workshop-prose-pins.test.mjs
Normal file
150
tests/lib/skills-workshop-prose-pins.test.mjs
Normal file
|
|
@ -0,0 +1,150 @@
|
|||
// tests/lib/skills-workshop-prose-pins.test.mjs
|
||||
// Pins for the skill workshop (ordre 7a, contracts S1 and S2): every skill is
|
||||
// asked for by a command or an agent ("the commands are the trunk"), none is
|
||||
// closed to the model, the step helper preloads test-first and debugging, each
|
||||
// skill names what enforces it and credits what it borrowed, the eval cases
|
||||
// keep the 3 / 2 / 1 form, and the docs carry the ## Skills section.
|
||||
// Counted as doc pins by lib/util/test-census.mjs (file name *-prose-pins).
|
||||
|
||||
import { test } from 'node:test';
|
||||
import { strict as assert } from 'node:assert';
|
||||
import { existsSync, mkdtempSync, mkdirSync, readdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { dirname, join } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { findSkillUsers } from '../../lib/skills/skill-lint.mjs';
|
||||
import { readFrontmatterFields } from '../../lib/skills/skill-frontmatter.mjs';
|
||||
import { loadCases } from '../../lib/skills/routing.mjs';
|
||||
|
||||
const HERE = dirname(fileURLToPath(import.meta.url));
|
||||
const ROOT = join(HERE, '..', '..');
|
||||
const read = (rel) => readFileSync(join(ROOT, rel), 'utf8');
|
||||
const SKILLS = readdirSync(join(ROOT, 'skills')).filter((d) => existsSync(join(ROOT, 'skills', d, 'SKILL.md'))).sort();
|
||||
|
||||
test('pin: Voyage ships the test-first and debugging skills (S2)', () => {
|
||||
for (const s of ['test-first', 'debugging']) assert.ok(SKILLS.includes(s), `skills/${s}/SKILL.md missing`);
|
||||
});
|
||||
|
||||
// ---- "the commands are the trunk": a skill nobody asks for is red ----
|
||||
|
||||
test('pin: findSkillUsers sees a skills: line and a voyage:<name> mention, and reports an unused skill', () => {
|
||||
const root = mkdtempSync(join(tmpdir(), 'skill-users-'));
|
||||
try {
|
||||
for (const s of ['used-by-agent', 'used-by-command', 'unused']) {
|
||||
mkdirSync(join(root, 'skills', s), { recursive: true });
|
||||
writeFileSync(join(root, 'skills', s, 'SKILL.md'), `---\nname: ${s}\ndescription: x\n---\n`);
|
||||
}
|
||||
mkdirSync(join(root, 'agents'));
|
||||
mkdirSync(join(root, 'commands'));
|
||||
writeFileSync(join(root, 'agents', 'a.md'), '---\nname: a\nskills:\n - voyage:used-by-agent\n---\nbody\n');
|
||||
writeFileSync(join(root, 'commands', 'c.md'), '---\nname: c\n---\nCall the Skill tool with `voyage:used-by-command`.\n');
|
||||
const users = findSkillUsers(root);
|
||||
assert.deepEqual(users['used-by-agent'], ['agents/a.md']);
|
||||
assert.deepEqual(users['used-by-command'], ['commands/c.md']);
|
||||
assert.deepEqual(users.unused, []);
|
||||
// A bare word in prose is not a request for the skill.
|
||||
writeFileSync(join(root, 'commands', 'd.md'), '---\nname: d\n---\nThis mentions unused in passing.\n');
|
||||
assert.deepEqual(findSkillUsers(root).unused, []);
|
||||
} finally { rmSync(root, { recursive: true, force: true }); }
|
||||
});
|
||||
|
||||
test('pin: every skill in skills/ is asked for by at least one command or agent', () => {
|
||||
const users = findSkillUsers(ROOT);
|
||||
for (const s of SKILLS) assert.ok(users[s] && users[s].length > 0, `skills/${s}: no command or agent names it`);
|
||||
});
|
||||
|
||||
test('pin: no Voyage skill sets disable-model-invocation (it would block preloading and command calls)', () => {
|
||||
for (const s of SKILLS) {
|
||||
const f = readFrontmatterFields(read(`skills/${s}/SKILL.md`)).fields['disable-model-invocation'];
|
||||
assert.ok(!f || !/^(true|yes|on|1)$/i.test(String(f.value).trim()), `skills/${s}: disable-model-invocation is set`);
|
||||
}
|
||||
});
|
||||
|
||||
// ---- S2: the step helper preloads both ----
|
||||
|
||||
test('pin: step-implementer preloads voyage:test-first and voyage:debugging, without the Skill tool', () => {
|
||||
const fm = readFrontmatterFields(read('agents/step-implementer.md')).fields;
|
||||
assert.deepEqual(fm.skills.value, ['voyage:test-first', 'voyage:debugging']);
|
||||
// Preloading uses the skills field, not the Skill tool (sub-agents.md,
|
||||
// "Preload skills into subagents"): the tool list stays as it was.
|
||||
assert.deepEqual(fm.tools.value, ['Read', 'Write', 'Edit', 'Bash', 'Glob', 'Grep']);
|
||||
});
|
||||
|
||||
// ---- each skill says what enforces it, and credits what it borrowed ----
|
||||
|
||||
test('pin: debugging says plainly that the root-cause claim is prose only', () => {
|
||||
const t = read('skills/debugging/SKILL.md');
|
||||
const enforced = t.split('## Enforced by')[1].split('## Credits')[0];
|
||||
assert.match(enforced, /prose only: the claim that the root cause was found/);
|
||||
assert.match(enforced, /lib\/verification\/red-proof\.mjs/);
|
||||
});
|
||||
|
||||
test('pin: test-first says the proof of red counts a crash as red, so the assertion red is prose only', () => {
|
||||
const enforced = read('skills/test-first/SKILL.md').split('## Enforced by')[1].split('## Credits')[0];
|
||||
assert.match(enforced, /a crash also passes it/);
|
||||
for (const p of ['lib/verification/red-proof.mjs', 'lib/util/test-lock.mjs', 'lib/verification/test-integrity.mjs', 'lib/verification/done-line.mjs']) {
|
||||
assert.ok(enforced.includes(p), `test-first must name ${p}`);
|
||||
}
|
||||
});
|
||||
|
||||
test('pin: each skill credits its sources with commit and licence', () => {
|
||||
const want = {
|
||||
'test-first': ['obra/superpowers @8ca22db', 'addyosmani/agent-skills @9d0c60d', 'mattpocock/skills @c55ee46'],
|
||||
debugging: ['obra/superpowers @8ca22db', 'addyosmani/agent-skills @9d0c60d', 'mattpocock/skills @c55ee46'],
|
||||
};
|
||||
for (const [s, sources] of Object.entries(want)) {
|
||||
const credits = read(`skills/${s}/SKILL.md`).split('## Credits')[1];
|
||||
for (const src of sources) assert.ok(credits.includes(src), `${s}: credit ${src}`);
|
||||
assert.equal((credits.match(/\bMIT\b/g) || []).length, sources.length, `${s}: one licence per source`);
|
||||
}
|
||||
});
|
||||
|
||||
// ---- S1: the eval case form (3 should fire / 2 should not / 1 effect) ----
|
||||
|
||||
test('pin: each skill has 3 trigger, 2 no-trigger and 1 effect case, and the graders read the Skill tool', () => {
|
||||
const cases = loadCases(ROOT);
|
||||
for (const s of SKILLS) {
|
||||
const mine = cases.filter((c) => c.skill === s);
|
||||
const n = (k) => mine.filter((c) => c.kind === k);
|
||||
assert.ok(n('trigger').length >= 3 && n('no-trigger').length >= 2 && n('effect').length >= 1, `${s}: ${mine.length} cases`);
|
||||
for (const c of [...n('trigger'), ...n('no-trigger')]) {
|
||||
const dir = join(ROOT, 'evals', c.case, 'graders');
|
||||
const g = readdirSync(dir).map((f) => readFrontmatterFields(readFileSync(join(dir, f), 'utf8')).fields);
|
||||
const skillGrader = g.find((x) => x.tool && x.tool.value === 'Skill');
|
||||
assert.ok(skillGrader, `${c.case}: needs a tool_used Skill grader`);
|
||||
const re = new RegExp(skillGrader.input_match.value);
|
||||
assert.match(JSON.stringify({ skill: `voyage:${s}` }), re, `${c.case}: must see voyage:${s}`);
|
||||
assert.match(JSON.stringify({ skill: s }), re, `${c.case}: must see the bare name`);
|
||||
for (const other of SKILLS.filter((o) => o !== s)) assert.doesNotMatch(JSON.stringify({ skill: `voyage:${other}` }), re);
|
||||
if (c.kind === 'no-trigger') {
|
||||
assert.deepEqual([skillGrader.min.value, skillGrader.max.value, skillGrader.arm.value], ['0', '0', 'both'], c.case);
|
||||
}
|
||||
}
|
||||
for (const c of n('effect')) {
|
||||
const desc = readFrontmatterFields(read(`evals/${c.case}/prompt.md`)).fields.description.value;
|
||||
assert.match(desc, /judge model explicitly \(never Haiku\)/, `${c.case}: the judge model is the operator's explicit choice`);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
// ---- docs: ## Skills, the authoring guide ----
|
||||
|
||||
test('pin: README and CLAUDE.md carry a ## Skills section naming both skills and the workshop checks', () => {
|
||||
for (const f of ['README.md', 'CLAUDE.md']) {
|
||||
const t = read(f);
|
||||
const sec = t.split(/^## Skills$/m)[1];
|
||||
assert.ok(sec, `${f}: ## Skills missing`);
|
||||
const body = sec.split(/^## /m)[0];
|
||||
for (const s of ['test-first', 'debugging', 'lib/skills/skill-lint.mjs', 'lib/skills/routing.mjs', 'docs/skills-authoring.md']) {
|
||||
assert.ok(body.includes(s), `${f} ## Skills must name ${s}`);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
test('pin: docs/skills-authoring.md teaches the red baseline and stays public-safe', () => {
|
||||
const t = read('docs/skills-authoring.md');
|
||||
for (const s of ['## Red baseline first', 'control', 'lib/skills/skill-lint.mjs', 'lib/skills/routing.mjs', 'skill-trigger', 'skill-no-trigger', 'skill-effect', 'from=']) {
|
||||
assert.ok(t.includes(s), `skills-authoring.md must cover ${s}`);
|
||||
}
|
||||
assert.doesNotMatch(t, /\/Users\/|\/private\/tmp|\.local\.md/, 'no local paths in a public doc');
|
||||
});
|
||||
Loading…
Add table
Add a link
Reference in a new issue