// tests/lib/skills-workshop-prose-pins.test.mjs // Pins for the skill workshop (ordre 7a, contracts S1 and S2): every skill is // asked for by a command or an agent ("the commands are the trunk"), none is // closed to the model, the step helper preloads test-first and debugging, each // skill names what enforces it and credits what it borrowed, the eval cases // keep the 3 / 2 / 1 form, and the docs carry the ## Skills section. // Counted as doc pins by lib/util/test-census.mjs (file name *-prose-pins). import { test } from 'node:test'; import { strict as assert } from 'node:assert'; import { existsSync, mkdtempSync, mkdirSync, readdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { dirname, join } from 'node:path'; import { fileURLToPath } from 'node:url'; import { findSkillUsers } from '../../lib/skills/skill-lint.mjs'; import { readFrontmatterFields } from '../../lib/skills/skill-frontmatter.mjs'; import { loadCases } from '../../lib/skills/routing.mjs'; const HERE = dirname(fileURLToPath(import.meta.url)); const ROOT = join(HERE, '..', '..'); const read = (rel) => readFileSync(join(ROOT, rel), 'utf8'); const SKILLS = readdirSync(join(ROOT, 'skills')).filter((d) => existsSync(join(ROOT, 'skills', d, 'SKILL.md'))).sort(); test('pin: Voyage ships the test-first and debugging skills (S2)', () => { for (const s of ['test-first', 'debugging']) assert.ok(SKILLS.includes(s), `skills/${s}/SKILL.md missing`); }); // ---- "the commands are the trunk": a skill nobody asks for is red ---- test('pin: findSkillUsers sees a skills: line and a voyage: mention, and reports an unused skill', () => { const root = mkdtempSync(join(tmpdir(), 'skill-users-')); try { for (const s of ['used-by-agent', 'used-by-command', 'unused']) { mkdirSync(join(root, 'skills', s), { recursive: true }); writeFileSync(join(root, 'skills', s, 'SKILL.md'), `---\nname: ${s}\ndescription: x\n---\n`); } mkdirSync(join(root, 'agents')); mkdirSync(join(root, 'commands')); writeFileSync(join(root, 'agents', 'a.md'), '---\nname: a\nskills:\n - voyage:used-by-agent\n---\nbody\n'); writeFileSync(join(root, 'commands', 'c.md'), '---\nname: c\n---\nCall the Skill tool with `voyage:used-by-command`.\n'); const users = findSkillUsers(root); assert.deepEqual(users['used-by-agent'], ['agents/a.md']); assert.deepEqual(users['used-by-command'], ['commands/c.md']); assert.deepEqual(users.unused, []); // A bare word in prose is not a request for the skill. writeFileSync(join(root, 'commands', 'd.md'), '---\nname: d\n---\nThis mentions unused in passing.\n'); assert.deepEqual(findSkillUsers(root).unused, []); } finally { rmSync(root, { recursive: true, force: true }); } }); test('pin: every skill in skills/ is asked for by at least one command or agent', () => { const users = findSkillUsers(ROOT); for (const s of SKILLS) assert.ok(users[s] && users[s].length > 0, `skills/${s}: no command or agent names it`); }); test('pin: no Voyage skill sets disable-model-invocation (it would block preloading and command calls)', () => { for (const s of SKILLS) { const f = readFrontmatterFields(read(`skills/${s}/SKILL.md`)).fields['disable-model-invocation']; assert.ok(!f || !/^(true|yes|on|1)$/i.test(String(f.value).trim()), `skills/${s}: disable-model-invocation is set`); } }); // ---- S2: the step helper preloads both ---- test('pin: step-implementer preloads voyage:test-first and voyage:debugging, without the Skill tool', () => { const fm = readFrontmatterFields(read('agents/step-implementer.md')).fields; assert.deepEqual(fm.skills.value, ['voyage:test-first', 'voyage:debugging']); // Preloading uses the skills field, not the Skill tool (sub-agents.md, // "Preload skills into subagents"): the tool list stays as it was. assert.deepEqual(fm.tools.value, ['Read', 'Write', 'Edit', 'Bash', 'Glob', 'Grep']); }); // ---- each skill says what enforces it, and credits what it borrowed ---- test('pin: debugging says plainly that the root-cause claim is prose only', () => { const t = read('skills/debugging/SKILL.md'); const enforced = t.split('## Enforced by')[1].split('## Credits')[0]; assert.match(enforced, /prose only: the claim that the root cause was found/); assert.match(enforced, /lib\/verification\/red-proof\.mjs/); }); test('pin: test-first says the proof of red counts a crash as red, so the assertion red is prose only', () => { const enforced = read('skills/test-first/SKILL.md').split('## Enforced by')[1].split('## Credits')[0]; assert.match(enforced, /a crash also passes it/); for (const p of ['lib/verification/red-proof.mjs', 'lib/util/test-lock.mjs', 'lib/verification/test-integrity.mjs', 'lib/verification/done-line.mjs']) { assert.ok(enforced.includes(p), `test-first must name ${p}`); } }); test('pin: each skill credits its sources with commit and licence', () => { const want = { 'test-first': ['obra/superpowers @8ca22db', 'addyosmani/agent-skills @9d0c60d', 'mattpocock/skills @c55ee46'], debugging: ['obra/superpowers @8ca22db', 'addyosmani/agent-skills @9d0c60d', 'mattpocock/skills @c55ee46'], }; for (const [s, sources] of Object.entries(want)) { const credits = read(`skills/${s}/SKILL.md`).split('## Credits')[1]; for (const src of sources) assert.ok(credits.includes(src), `${s}: credit ${src}`); assert.equal((credits.match(/\bMIT\b/g) || []).length, sources.length, `${s}: one licence per source`); } }); // ---- S1: the eval case form (3 should fire / 2 should not / 1 effect) ---- test('pin: each skill has 3 trigger, 2 no-trigger and 1 effect case, and the graders read the Skill tool', () => { const cases = loadCases(ROOT); for (const s of SKILLS) { const mine = cases.filter((c) => c.skill === s); const n = (k) => mine.filter((c) => c.kind === k); assert.ok(n('trigger').length >= 3 && n('no-trigger').length >= 2 && n('effect').length >= 1, `${s}: ${mine.length} cases`); for (const c of [...n('trigger'), ...n('no-trigger')]) { const dir = join(ROOT, 'evals', c.case, 'graders'); const g = readdirSync(dir).map((f) => readFrontmatterFields(readFileSync(join(dir, f), 'utf8')).fields); const skillGrader = g.find((x) => x.tool && x.tool.value === 'Skill'); assert.ok(skillGrader, `${c.case}: needs a tool_used Skill grader`); const re = new RegExp(skillGrader.input_match.value); assert.match(JSON.stringify({ skill: `voyage:${s}` }), re, `${c.case}: must see voyage:${s}`); assert.match(JSON.stringify({ skill: s }), re, `${c.case}: must see the bare name`); for (const other of SKILLS.filter((o) => o !== s)) assert.doesNotMatch(JSON.stringify({ skill: `voyage:${other}` }), re); if (c.kind === 'no-trigger') { assert.deepEqual([skillGrader.min.value, skillGrader.max.value, skillGrader.arm.value], ['0', '0', 'both'], c.case); } } for (const c of n('effect')) { const desc = readFrontmatterFields(read(`evals/${c.case}/prompt.md`)).fields.description.value; assert.match(desc, /judge model explicitly \(never Haiku\)/, `${c.case}: the judge model is the operator's explicit choice`); } } }); // ---- docs: ## Skills, the authoring guide ---- test('pin: README and CLAUDE.md carry a ## Skills section naming both skills and the workshop checks', () => { for (const f of ['README.md', 'CLAUDE.md']) { const t = read(f); const sec = t.split(/^## Skills$/m)[1]; assert.ok(sec, `${f}: ## Skills missing`); const body = sec.split(/^## /m)[0]; for (const s of ['test-first', 'debugging', 'lib/skills/skill-lint.mjs', 'lib/skills/routing.mjs', 'docs/skills-authoring.md']) { assert.ok(body.includes(s), `${f} ## Skills must name ${s}`); } } }); test('pin: docs/skills-authoring.md teaches the red baseline and stays public-safe', () => { const t = read('docs/skills-authoring.md'); for (const s of ['## Red baseline first', 'control', 'lib/skills/skill-lint.mjs', 'lib/skills/routing.mjs', 'skill-trigger', 'skill-no-trigger', 'skill-effect', 'from=']) { assert.ok(t.includes(s), `skills-authoring.md must cover ${s}`); } assert.doesNotMatch(t, /\/Users\/|\/private\/tmp|\.local\.md/, 'no local paths in a public doc'); });