// tests/kb-eval/test-skill-ops-create.test.mjs // Spor B / B3 (Sesjon 20): create_skill — the CONSTRUCTIVE op (dvelende, last B3). // // merge/sanitize/retire are DESTRUCTIVE: their guardrail proves "no curated // value LOST" (preservation). create_skill is the mirror: its guardrail proves // "no sub-bar skill BORN" (quality floor). The binding gate-criterion is that // the generated scaffold passes eval.mjs's OWN deterministic rubric from birth // (K2/K3/K5/K6/refCountConsistency + K10 sibling-non-overlap) — so this file // re-runs the very eval checkers against the scaffold to prove it independently. // // Two layers, like every other B3 op: // (A) pure core — planCreateSkill(name, ctx): mutates nothing, reads no disk, // returns { entry, scaffold, guardrail }. Refuses (ok=false) on a failing // criterion, a sibling overlap >= threshold, or a name collision. // (B) impure shell — runCreatePlan(...) gated dry-run (0 writes under skills/) // + applyApprovedAction(...) create branch (writes the scaffold on --apply). // // Every disk scenario runs against a tmpdir fixture — NEVER the real skills/. import { test } from 'node:test'; import assert from 'node:assert/strict'; import { mkdtempSync, rmSync, mkdirSync, writeFileSync, readFileSync, existsSync, readdirSync, statSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { planCreateSkill, runCreatePlan, applyApprovedAction, TEST_DOMAIN_SPEC, CREATE_OPERATION, } from '../../scripts/kb-eval/lib/skill-ops.mjs'; import { loadDecisions, saveDecisions, recordAction, actionKey, isActionDecided, } from '../../scripts/kb-update/lib/decisions-io.mjs'; import { applyCategoryAdditions, applyCategoryReassignments } from '../../scripts/kb-update/lib/taxonomy.mjs'; import { splitFrontmatter, extractDescription, checkK2, checkK3, checkK5K6, checkRefConsistency, } from '../../scripts/kb-eval/eval.mjs'; // =========================================================================== // (A) Pure core — planCreateSkill // =========================================================================== const NAME = 'create-self-test'; test('planCreateSkill — guardrail OK for a clean test-domain scaffold; entry is a PENDING create_skill', () => { const { entry, guardrail, scaffold } = planCreateSkill(NAME, { ...TEST_DOMAIN_SPEC, decided_at: '2026-06-20' }); assert.equal(guardrail.ok, true, 'a born-compliant scaffold passes the constructive guardrail'); assert.deepEqual(guardrail.failed, [], 'no failing criteria'); assert.equal(guardrail.nameAvailable, true); assert.equal(entry.operation_type, CREATE_OPERATION); assert.equal(entry.status, 'pending', 'planner only proposes — never approved'); assert.deepEqual(entry.targets, { name: NAME }); assert.equal(entry.decided_at, '2026-06-20'); // scaffold carries the SKILL.md + ref files + taxonomy additions assert.equal(scaffold.skillMd.path, `skills/${NAME}/SKILL.md`); assert.equal(scaffold.files.length, 3, 'TEST_DOMAIN_SPEC = 2 core + 1 patterns'); assert.deepEqual( scaffold.files.map((f) => f.path).sort(), [ `skills/${NAME}/references/core/mechanism.md`, `skills/${NAME}/references/core/overview.md`, `skills/${NAME}/references/patterns/routing.md`, ], ); assert.deepEqual(scaffold.taxonomyAdditions, [ { category: 'core', skill: NAME }, { category: 'patterns', skill: NAME }, ]); }); test('planCreateSkill — THE gate-criterion: the scaffold passes eval.mjs deterministic K1–K10 from birth', () => { const { scaffold } = planCreateSkill(NAME, { ...TEST_DOMAIN_SPEC }); // Independently re-derive the verdict via eval.mjs's OWN checkers (not the // planner's guardrail) against the generated SKILL.md. const { frontmatter, body } = splitFrontmatter(scaffold.skillMd.content); const description = extractDescription(frontmatter); const totalRefFiles = scaffold.files.length; // actual per-folder counts derived from the scaffold files (what eval reads off disk) const perFolder = {}; for (const f of scaffold.files) { const cat = f.path.split('/references/')[1].split('/')[0]; perFolder[cat] = (perFolder[cat] || 0) + 1; } const k2 = checkK2(description); const k3 = checkK3(body); const { K5, K6 } = checkK5K6(body, totalRefFiles); const rc = checkRefConsistency(body, perFolder); assert.equal(k2.pass, true, 'K2 description format'); assert.equal(k3.pass, true, 'K3 body <= 500 lines'); assert.equal(K5.pass, true, `K5 progressive disclosure (ratio ${K5.namedRatio})`); assert.equal(K6.pass, true, 'K6 routing table has named start files'); assert.equal(rc.consistent, true, `refCountConsistency: ${JSON.stringify(rc.mismatches)}`); }); test('planCreateSkill — guardrail FAILS when the description fails K2 (no use-when form, < 3 quoted phrases)', () => { const { guardrail } = planCreateSkill(NAME, { ...TEST_DOMAIN_SPEC, description: 'A short blurb about a thing with no required format.', }); assert.equal(guardrail.ok, false, 'a sub-bar description must not be born compliant'); assert.ok(guardrail.failed.includes('K2'), 'K2 surfaced as the failing criterion'); assert.equal(guardrail.k2.pass, false); }); test('planCreateSkill — guardrail FAILS on a name collision (skill already exists)', () => { const { guardrail } = planCreateSkill(NAME, { ...TEST_DOMAIN_SPEC, existingSkills: ['ms-ai-engineering', NAME, 'ms-ai-security'], }); assert.equal(guardrail.ok, false, 'cannot create over an existing skill'); assert.equal(guardrail.nameAvailable, false); assert.ok(guardrail.failed.includes('name')); }); test('planCreateSkill — guardrail FAILS when the new description overlaps a sibling at/above K10 threshold', () => { // A sibling that shares a distinctive token with the new description, plus a // tiny threshold so ANY overlap trips K10 — proves the sibling-non-overlap gate. const ctx = { ...TEST_DOMAIN_SPEC, description: 'Use this skill for the widget gadget domain. Triggers on: "widget a", "widget b", "widget c".', existingDescriptions: { 'sibling-skill': 'A sibling that also owns the widget surface entirely.' }, promptSet: {}, threshold: 0.0001, }; const { guardrail } = planCreateSkill(NAME, ctx); assert.equal(guardrail.ok, false, 'an over-overlapping description must be refused'); assert.equal(guardrail.k10.pass, false); assert.ok(guardrail.k10.maxCombined > 0, 'shares a distinctive token with the sibling'); assert.ok(guardrail.failed.includes('K10')); }); test('planCreateSkill — a lone skill (no siblings) trivially passes K10', () => { const { guardrail } = planCreateSkill(NAME, { ...TEST_DOMAIN_SPEC, existingDescriptions: {}, promptSet: {} }); assert.equal(guardrail.k10.pass, true); assert.equal(guardrail.k10.maxCombined, 0); assert.equal(guardrail.k10.worstSibling, null); }); test('planCreateSkill — refCountConsistency holds: the routing table cites the scaffold counts exactly', () => { const { guardrail, scaffold } = planCreateSkill(NAME, { ...TEST_DOMAIN_SPEC }); assert.equal(guardrail.refConsistency.consistent, true); // the body must cite `references/core/` | 2 and `references/patterns/` | 1 assert.match(scaffold.skillMd.content, /`references\/core\/`\s*\|\s*2/); assert.match(scaffold.skillMd.content, /`references\/patterns\/`\s*\|\s*1/); }); // =========================================================================== // (B) Impure shell — runCreatePlan (read-only existing skills + gated write) // =========================================================================== function withSkillsFixture(fn, { seedExisting = [] } = {}) { const root = mkdtempSync(join(tmpdir(), 'skillops-create-')); const skillsDir = join(root, 'skills'); const dataDir = join(root, 'data'); mkdirSync(skillsDir, { recursive: true }); mkdirSync(dataDir, { recursive: true }); // Seed any pre-existing skill dirs (for collision tests). for (const s of seedExisting) { mkdirSync(join(skillsDir, s), { recursive: true }); writeFileSync(join(skillsDir, s, 'SKILL.md'), `---\nname: ${s}\ndescription: existing\n---\n# ${s}\n`); } // Minimal taxonomy so applyCategoryAdditions has something to load. writeFileSync(join(dataDir, 'domain-taxonomy.json'), JSON.stringify({ category_skill: {} }, null, 2) + '\n'); try { return fn({ root, skillsDir, dataDir }); } finally { rmSync(root, { recursive: true, force: true }); } } function snapshot(dir) { const out = {}; const walk = (d) => { for (const e of readdirSync(d, { withFileTypes: true })) { const p = join(d, e.name); if (e.isDirectory()) walk(p); else { const s = statSync(p); out[p] = `${s.size}:${s.mtimeMs}`; } } }; walk(dir); return out; } test('runCreatePlan --write — records a PENDING entry and touches NOTHING under skills/', () => { withSkillsFixture(({ skillsDir, dataDir }) => { const before = snapshot(skillsDir); const { entry, guardrail } = runCreatePlan(NAME, { skillsDir, dataDir, write: true, decided_at: '2026-06-20', ...TEST_DOMAIN_SPEC }); assert.equal(entry.status, 'pending'); assert.equal(guardrail.ok, true); const led = loadDecisions(dataDir); assert.equal(isActionDecided(led, actionKey(entry)), true); assert.equal(led.actions[actionKey(entry)].status, 'pending'); assert.equal(actionKey(entry), `create_skill:${NAME}`); // THE CONSTRUCTIVE INVARIANT: skills/ is byte-for-byte unchanged by a dry-run. assert.deepEqual(snapshot(skillsDir), before, 'dry-run must not write under skills/'); }, { seedExisting: ['ms-ai-engineering'] }); }); test('runCreatePlan without --write — pure preview, no ledger file created', () => { withSkillsFixture(({ skillsDir, dataDir }) => { const { entry } = runCreatePlan(NAME, { skillsDir, dataDir, write: false, ...TEST_DOMAIN_SPEC }); assert.equal(entry.status, 'pending'); assert.equal(existsSync(join(dataDir, 'decisions.json')), false, 'no write without --write'); }); }); test('runCreatePlan — refuses a name that already exists on disk (read-only collision check)', () => { withSkillsFixture(({ skillsDir, dataDir }) => { const { guardrail } = runCreatePlan('ms-ai-security', { skillsDir, dataDir, write: false, ...TEST_DOMAIN_SPEC }); assert.equal(guardrail.ok, false); assert.equal(guardrail.nameAvailable, false); }, { seedExisting: ['ms-ai-security'] }); }); // =========================================================================== // (C) Impure executor — applyApprovedAction create (writes the scaffold) // =========================================================================== // Seed an operator-approved create entry exactly as the gate would after a // reviewed `plan-skill-op create … --write` dry-run. function approveCreate(name, spec, { skillsDir, dataDir }) { const { entry } = runCreatePlan(name, { skillsDir, dataDir, write: false, decided_at: '2026-06-20', ...spec }); const approved = { ...entry, status: 'approved' }; saveDecisions(recordAction(loadDecisions(dataDir), approved), dataDir); return approved; } test('applyApprovedAction create — writes the scaffold, persists taxonomy, flips the ledger; the created skill passes eval K1–K10', () => { withSkillsFixture(({ root, skillsDir, dataDir }) => { const approved = approveCreate(NAME, TEST_DOMAIN_SPEC, { skillsDir, dataDir }); const res = applyApprovedAction(approved, { pluginRoot: root, skillsDir, dataDir, taxonomyDataDir: dataDir, createSpec: { description: TEST_DOMAIN_SPEC.description, categories: TEST_DOMAIN_SPEC.categories }, apply: true, decided_at: '2026-06-21', }); assert.equal(res.applied, true); assert.equal(res.revalidate.ok, true); // SKILL.md + all 3 reference files exist on disk. assert.ok(existsSync(join(skillsDir, NAME, 'SKILL.md')), 'SKILL.md written'); assert.ok(existsSync(join(skillsDir, NAME, 'references/core/overview.md')), 'core/overview.md written'); assert.ok(existsSync(join(skillsDir, NAME, 'references/core/mechanism.md')), 'core/mechanism.md written'); assert.ok(existsSync(join(skillsDir, NAME, 'references/patterns/routing.md')), 'patterns/routing.md written'); // Taxonomy ownership persisted additively. const tax = JSON.parse(readFileSync(join(dataDir, 'domain-taxonomy.json'), 'utf8')); assert.equal(tax.category_skill.core, NAME); assert.equal(tax.category_skill.patterns, NAME); // Ledger flipped approved -> applied. assert.equal(loadDecisions(dataDir).actions[actionKey(approved)].status, 'applied'); // INDEPENDENT proof: the on-disk created skill passes the deterministic rubric. const content = readFileSync(join(skillsDir, NAME, 'SKILL.md'), 'utf8'); const { frontmatter, body } = splitFrontmatter(content); const description = extractDescription(frontmatter); const perFolder = { core: 2, patterns: 1 }; assert.equal(checkK2(description).pass, true); assert.equal(checkK3(body).pass, true); const { K5, K6 } = checkK5K6(body, 3); assert.equal(K5.pass, true); assert.equal(K6.pass, true); assert.equal(checkRefConsistency(body, perFolder).consistent, true); }); }); test('applyApprovedAction create — preview (apply:false) writes nothing and leaves the ledger at approved', () => { withSkillsFixture(({ root, skillsDir, dataDir }) => { const approved = approveCreate(NAME, TEST_DOMAIN_SPEC, { skillsDir, dataDir }); const res = applyApprovedAction(approved, { pluginRoot: root, skillsDir, dataDir, taxonomyDataDir: dataDir, createSpec: { description: TEST_DOMAIN_SPEC.description, categories: TEST_DOMAIN_SPEC.categories }, apply: false, }); assert.equal(res.applied, false); assert.equal(res.preview, true); assert.equal(res.revalidate.ok, true, 'name still free -> revalidation would pass'); assert.equal(existsSync(join(skillsDir, NAME)), false, 'preview creates nothing'); assert.equal(loadDecisions(dataDir).actions[actionKey(approved)].status, 'approved'); }); }); test('applyApprovedAction create — DRIFT (name taken after approval) aborts with ZERO mutation', () => { withSkillsFixture(({ root, skillsDir, dataDir }) => { const approved = approveCreate(NAME, TEST_DOMAIN_SPEC, { skillsDir, dataDir }); // Disk drifts: a skill with the target name now exists -> fresh guardrail // nameAvailable=false -> guardrail differs from the approved snapshot. mkdirSync(join(skillsDir, NAME), { recursive: true }); writeFileSync(join(skillsDir, NAME, 'SKILL.md'), '---\nname: squatter\ndescription: x\n---\n# squatter\n'); const res = applyApprovedAction(approved, { pluginRoot: root, skillsDir, dataDir, taxonomyDataDir: dataDir, createSpec: { description: TEST_DOMAIN_SPEC.description, categories: TEST_DOMAIN_SPEC.categories }, apply: true, }); assert.equal(res.applied, false, 'a name collision after approval must abort'); assert.equal(res.revalidate.ok, false); // The pre-existing (squatter) SKILL.md is untouched; no references/ created. assert.equal(existsSync(join(skillsDir, NAME, 'references')), false, 'no scaffold written on abort'); assert.equal(loadDecisions(dataDir).actions[actionKey(approved)].status, 'approved', 'ledger left at approved'); }); }); test('applyApprovedAction create — refuses a non-approved (pending) entry with zero mutation', () => { withSkillsFixture(({ root, skillsDir, dataDir }) => { const { entry } = runCreatePlan(NAME, { skillsDir, dataDir, write: false, ...TEST_DOMAIN_SPEC }); saveDecisions(recordAction(loadDecisions(dataDir), entry), dataDir); // pending, not approved const res = applyApprovedAction(entry, { pluginRoot: root, skillsDir, dataDir, taxonomyDataDir: dataDir, createSpec: { description: TEST_DOMAIN_SPEC.description, categories: TEST_DOMAIN_SPEC.categories }, apply: true, }); assert.equal(res.applied, false); assert.match(res.revalidate.reason, /approved/i); assert.equal(existsSync(join(skillsDir, NAME)), false, 'no mutation'); }); }); // =========================================================================== // (D) Taxonomy helper — applyCategoryAdditions (additive, never clobbers) // =========================================================================== test('applyCategoryAdditions — adds new category ownership, no-ops on an already-owned category', () => { const tax = { category_skill: { existing: 'owner-skill' } }; const next = applyCategoryAdditions(tax, [ { category: 'brand-new', skill: 'new-skill' }, { category: 'existing', skill: 'new-skill' }, // must NOT clobber owner-skill ]); assert.equal(next.category_skill['brand-new'], 'new-skill', 'new category added'); assert.equal(next.category_skill.existing, 'owner-skill', 'existing owner never clobbered'); // pure: original untouched assert.equal(tax.category_skill['brand-new'], undefined, 'input not mutated'); }); test('applyCategoryAdditions — disjoint from applyCategoryReassignments (additions vs repoint stay separate ops)', () => { const tax = { category_skill: { a: 's1' } }; const added = applyCategoryAdditions(tax, [{ category: 'b', skill: 's2' }]); const repointed = applyCategoryReassignments(added, [{ category: 'a', from: 's1', to: 's2' }]); assert.deepEqual(repointed.category_skill, { a: 's2', b: 's2' }); });