ms-ai-architect/tests/kb-eval/test-skill-ops-create.test.mjs

// tests/kb-eval/test-skill-ops-create.test.mjs
// Spor B / B3 (Sesjon 20): create_skill — the CONSTRUCTIVE op (dvelende, last B3).
//
// merge/sanitize/retire are DESTRUCTIVE: their guardrail proves "no curated
// value LOST" (preservation). create_skill is the mirror: its guardrail proves
// "no sub-bar skill BORN" (quality floor). The binding gate-criterion is that
// the generated scaffold passes eval.mjs's OWN deterministic rubric from birth
// (K2/K3/K5/K6/refCountConsistency + K10 sibling-non-overlap) — so this file
// re-runs the very eval checkers against the scaffold to prove it independently.
//
// Two layers, like every other B3 op:
//  (A) pure core   — planCreateSkill(name, ctx): mutates nothing, reads no disk,
//      returns { entry, scaffold, guardrail }. Refuses (ok=false) on a failing
//      criterion, a sibling overlap >= threshold, or a name collision.
//  (B) impure shell — runCreatePlan(...) gated dry-run (0 writes under skills/)
//      + applyApprovedAction(...) create branch (writes the scaffold on --apply).
//
// Every disk scenario runs against a tmpdir fixture — NEVER the real skills/.

import { test } from 'node:test';
import assert from 'node:assert/strict';
import { mkdtempSync, rmSync, mkdirSync, writeFileSync, readFileSync, existsSync, readdirSync, statSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import {
  planCreateSkill,
  runCreatePlan,
  applyApprovedAction,
  TEST_DOMAIN_SPEC,
  CREATE_OPERATION,
} from '../../scripts/kb-eval/lib/skill-ops.mjs';
import {
  loadDecisions,
  saveDecisions,
  recordAction,
  actionKey,
  isActionDecided,
} from '../../scripts/kb-update/lib/decisions-io.mjs';
import { applyCategoryAdditions, applyCategoryReassignments } from '../../scripts/kb-update/lib/taxonomy.mjs';
import {
  splitFrontmatter,
  extractDescription,
  checkK2,
  checkK3,
  checkK5K6,
  checkRefConsistency,
} from '../../scripts/kb-eval/eval.mjs';

// ===========================================================================
// (A) Pure core — planCreateSkill
// ===========================================================================

const NAME = 'create-self-test';

test('planCreateSkill — guardrail OK for a clean test-domain scaffold; entry is a PENDING create_skill', () => {
  const { entry, guardrail, scaffold } = planCreateSkill(NAME, { ...TEST_DOMAIN_SPEC, decided_at: '2026-06-20' });
  assert.equal(guardrail.ok, true, 'a born-compliant scaffold passes the constructive guardrail');
  assert.deepEqual(guardrail.failed, [], 'no failing criteria');
  assert.equal(guardrail.nameAvailable, true);
  assert.equal(entry.operation_type, CREATE_OPERATION);
  assert.equal(entry.status, 'pending', 'planner only proposes — never approved');
  assert.deepEqual(entry.targets, { name: NAME });
  assert.equal(entry.decided_at, '2026-06-20');
  // scaffold carries the SKILL.md + ref files + taxonomy additions
  assert.equal(scaffold.skillMd.path, `skills/${NAME}/SKILL.md`);
  assert.equal(scaffold.files.length, 3, 'TEST_DOMAIN_SPEC = 2 core + 1 patterns');
  assert.deepEqual(
    scaffold.files.map((f) => f.path).sort(),
    [
      `skills/${NAME}/references/core/mechanism.md`,
      `skills/${NAME}/references/core/overview.md`,
      `skills/${NAME}/references/patterns/routing.md`,
    ],
  );
  assert.deepEqual(scaffold.taxonomyAdditions, [
    { category: 'core', skill: NAME },
    { category: 'patterns', skill: NAME },
  ]);
});

test('planCreateSkill — THE gate-criterion: the scaffold passes eval.mjs deterministic K1–K10 from birth', () => {
  const { scaffold } = planCreateSkill(NAME, { ...TEST_DOMAIN_SPEC });
  // Independently re-derive the verdict via eval.mjs's OWN checkers (not the
  // planner's guardrail) against the generated SKILL.md.
  const { frontmatter, body } = splitFrontmatter(scaffold.skillMd.content);
  const description = extractDescription(frontmatter);
  const totalRefFiles = scaffold.files.length;
  // actual per-folder counts derived from the scaffold files (what eval reads off disk)
  const perFolder = {};
  for (const f of scaffold.files) {
    const cat = f.path.split('/references/')[1].split('/')[0];
    perFolder[cat] = (perFolder[cat] || 0) + 1;
  }
  const k2 = checkK2(description);
  const k3 = checkK3(body);
  const { K5, K6 } = checkK5K6(body, totalRefFiles);
  const rc = checkRefConsistency(body, perFolder);

  assert.equal(k2.pass, true, 'K2 description format');
  assert.equal(k3.pass, true, 'K3 body <= 500 lines');
  assert.equal(K5.pass, true, `K5 progressive disclosure (ratio ${K5.namedRatio})`);
  assert.equal(K6.pass, true, 'K6 routing table has named start files');
  assert.equal(rc.consistent, true, `refCountConsistency: ${JSON.stringify(rc.mismatches)}`);
});

test('planCreateSkill — guardrail FAILS when the description fails K2 (no use-when form, < 3 quoted phrases)', () => {
  const { guardrail } = planCreateSkill(NAME, {
    ...TEST_DOMAIN_SPEC,
    description: 'A short blurb about a thing with no required format.',
  });
  assert.equal(guardrail.ok, false, 'a sub-bar description must not be born compliant');
  assert.ok(guardrail.failed.includes('K2'), 'K2 surfaced as the failing criterion');
  assert.equal(guardrail.k2.pass, false);
});

test('planCreateSkill — guardrail FAILS on a name collision (skill already exists)', () => {
  const { guardrail } = planCreateSkill(NAME, {
    ...TEST_DOMAIN_SPEC,
    existingSkills: ['ms-ai-engineering', NAME, 'ms-ai-security'],
  });
  assert.equal(guardrail.ok, false, 'cannot create over an existing skill');
  assert.equal(guardrail.nameAvailable, false);
  assert.ok(guardrail.failed.includes('name'));
});

test('planCreateSkill — guardrail FAILS when the new description overlaps a sibling at/above K10 threshold', () => {
  // A sibling that shares a distinctive token with the new description, plus a
  // tiny threshold so ANY overlap trips K10 — proves the sibling-non-overlap gate.
  const ctx = {
    ...TEST_DOMAIN_SPEC,
    description: 'Use this skill for the widget gadget domain. Triggers on: "widget a", "widget b", "widget c".',
    existingDescriptions: { 'sibling-skill': 'A sibling that also owns the widget surface entirely.' },
    promptSet: {},
    threshold: 0.0001,
  };
  const { guardrail } = planCreateSkill(NAME, ctx);
  assert.equal(guardrail.ok, false, 'an over-overlapping description must be refused');
  assert.equal(guardrail.k10.pass, false);
  assert.ok(guardrail.k10.maxCombined > 0, 'shares a distinctive token with the sibling');
  assert.ok(guardrail.failed.includes('K10'));
});

test('planCreateSkill — a lone skill (no siblings) trivially passes K10', () => {
  const { guardrail } = planCreateSkill(NAME, { ...TEST_DOMAIN_SPEC, existingDescriptions: {}, promptSet: {} });
  assert.equal(guardrail.k10.pass, true);
  assert.equal(guardrail.k10.maxCombined, 0);
  assert.equal(guardrail.k10.worstSibling, null);
});

test('planCreateSkill — refCountConsistency holds: the routing table cites the scaffold counts exactly', () => {
  const { guardrail, scaffold } = planCreateSkill(NAME, { ...TEST_DOMAIN_SPEC });
  assert.equal(guardrail.refConsistency.consistent, true);
  // the body must cite `references/core/` | 2 and `references/patterns/` | 1
  assert.match(scaffold.skillMd.content, /`references\/core\/`\s*\|\s*2/);
  assert.match(scaffold.skillMd.content, /`references\/patterns\/`\s*\|\s*1/);
});

// ===========================================================================
// (B) Impure shell — runCreatePlan (read-only existing skills + gated write)
// ===========================================================================

function withSkillsFixture(fn, { seedExisting = [] } = {}) {
  const root = mkdtempSync(join(tmpdir(), 'skillops-create-'));
  const skillsDir = join(root, 'skills');
  const dataDir = join(root, 'data');
  mkdirSync(skillsDir, { recursive: true });
  mkdirSync(dataDir, { recursive: true });
  // Seed any pre-existing skill dirs (for collision tests).
  for (const s of seedExisting) {
    mkdirSync(join(skillsDir, s), { recursive: true });
    writeFileSync(join(skillsDir, s, 'SKILL.md'), `---\nname: ${s}\ndescription: existing\n---\n# ${s}\n`);
  }
  // Minimal taxonomy so applyCategoryAdditions has something to load.
  writeFileSync(join(dataDir, 'domain-taxonomy.json'), JSON.stringify({ category_skill: {} }, null, 2) + '\n');
  try {
    return fn({ root, skillsDir, dataDir });
  } finally {
    rmSync(root, { recursive: true, force: true });
  }
}

function snapshot(dir) {
  const out = {};
  const walk = (d) => {
    for (const e of readdirSync(d, { withFileTypes: true })) {
      const p = join(d, e.name);
      if (e.isDirectory()) walk(p);
      else { const s = statSync(p); out[p] = `${s.size}:${s.mtimeMs}`; }
    }
  };
  walk(dir);
  return out;
}

test('runCreatePlan --write — records a PENDING entry and touches NOTHING under skills/', () => {
  withSkillsFixture(({ skillsDir, dataDir }) => {
    const before = snapshot(skillsDir);
    const { entry, guardrail } = runCreatePlan(NAME, { skillsDir, dataDir, write: true, decided_at: '2026-06-20', ...TEST_DOMAIN_SPEC });

    assert.equal(entry.status, 'pending');
    assert.equal(guardrail.ok, true);

    const led = loadDecisions(dataDir);
    assert.equal(isActionDecided(led, actionKey(entry)), true);
    assert.equal(led.actions[actionKey(entry)].status, 'pending');
    assert.equal(actionKey(entry), `create_skill:${NAME}`);

    // THE CONSTRUCTIVE INVARIANT: skills/ is byte-for-byte unchanged by a dry-run.
    assert.deepEqual(snapshot(skillsDir), before, 'dry-run must not write under skills/');
  }, { seedExisting: ['ms-ai-engineering'] });
});

test('runCreatePlan without --write — pure preview, no ledger file created', () => {
  withSkillsFixture(({ skillsDir, dataDir }) => {
    const { entry } = runCreatePlan(NAME, { skillsDir, dataDir, write: false, ...TEST_DOMAIN_SPEC });
    assert.equal(entry.status, 'pending');
    assert.equal(existsSync(join(dataDir, 'decisions.json')), false, 'no write without --write');
  });
});

test('runCreatePlan — refuses a name that already exists on disk (read-only collision check)', () => {
  withSkillsFixture(({ skillsDir, dataDir }) => {
    const { guardrail } = runCreatePlan('ms-ai-security', { skillsDir, dataDir, write: false, ...TEST_DOMAIN_SPEC });
    assert.equal(guardrail.ok, false);
    assert.equal(guardrail.nameAvailable, false);
  }, { seedExisting: ['ms-ai-security'] });
});

// ===========================================================================
// (C) Impure executor — applyApprovedAction create (writes the scaffold)
// ===========================================================================

// Seed an operator-approved create entry exactly as the gate would after a
// reviewed `plan-skill-op create … --write` dry-run.
function approveCreate(name, spec, { skillsDir, dataDir }) {
  const { entry } = runCreatePlan(name, { skillsDir, dataDir, write: false, decided_at: '2026-06-20', ...spec });
  const approved = { ...entry, status: 'approved' };
  saveDecisions(recordAction(loadDecisions(dataDir), approved), dataDir);
  return approved;
}

test('applyApprovedAction create — writes the scaffold, persists taxonomy, flips the ledger; the created skill passes eval K1–K10', () => {
  withSkillsFixture(({ root, skillsDir, dataDir }) => {
    const approved = approveCreate(NAME, TEST_DOMAIN_SPEC, { skillsDir, dataDir });
    const res = applyApprovedAction(approved, {
      pluginRoot: root, skillsDir, dataDir, taxonomyDataDir: dataDir,
      createSpec: { description: TEST_DOMAIN_SPEC.description, categories: TEST_DOMAIN_SPEC.categories },
      apply: true, decided_at: '2026-06-21',
    });

    assert.equal(res.applied, true);
    assert.equal(res.revalidate.ok, true);

    // SKILL.md + all 3 reference files exist on disk.
    assert.ok(existsSync(join(skillsDir, NAME, 'SKILL.md')), 'SKILL.md written');
    assert.ok(existsSync(join(skillsDir, NAME, 'references/core/overview.md')), 'core/overview.md written');
    assert.ok(existsSync(join(skillsDir, NAME, 'references/core/mechanism.md')), 'core/mechanism.md written');
    assert.ok(existsSync(join(skillsDir, NAME, 'references/patterns/routing.md')), 'patterns/routing.md written');

    // Taxonomy ownership persisted additively.
    const tax = JSON.parse(readFileSync(join(dataDir, 'domain-taxonomy.json'), 'utf8'));
    assert.equal(tax.category_skill.core, NAME);
    assert.equal(tax.category_skill.patterns, NAME);

    // Ledger flipped approved -> applied.
    assert.equal(loadDecisions(dataDir).actions[actionKey(approved)].status, 'applied');

    // INDEPENDENT proof: the on-disk created skill passes the deterministic rubric.
    const content = readFileSync(join(skillsDir, NAME, 'SKILL.md'), 'utf8');
    const { frontmatter, body } = splitFrontmatter(content);
    const description = extractDescription(frontmatter);
    const perFolder = { core: 2, patterns: 1 };
    assert.equal(checkK2(description).pass, true);
    assert.equal(checkK3(body).pass, true);
    const { K5, K6 } = checkK5K6(body, 3);
    assert.equal(K5.pass, true);
    assert.equal(K6.pass, true);
    assert.equal(checkRefConsistency(body, perFolder).consistent, true);
  });
});

test('applyApprovedAction create — preview (apply:false) writes nothing and leaves the ledger at approved', () => {
  withSkillsFixture(({ root, skillsDir, dataDir }) => {
    const approved = approveCreate(NAME, TEST_DOMAIN_SPEC, { skillsDir, dataDir });
    const res = applyApprovedAction(approved, {
      pluginRoot: root, skillsDir, dataDir, taxonomyDataDir: dataDir,
      createSpec: { description: TEST_DOMAIN_SPEC.description, categories: TEST_DOMAIN_SPEC.categories },
      apply: false,
    });
    assert.equal(res.applied, false);
    assert.equal(res.preview, true);
    assert.equal(res.revalidate.ok, true, 'name still free -> revalidation would pass');
    assert.equal(existsSync(join(skillsDir, NAME)), false, 'preview creates nothing');
    assert.equal(loadDecisions(dataDir).actions[actionKey(approved)].status, 'approved');
  });
});

test('applyApprovedAction create — DRIFT (name taken after approval) aborts with ZERO mutation', () => {
  withSkillsFixture(({ root, skillsDir, dataDir }) => {
    const approved = approveCreate(NAME, TEST_DOMAIN_SPEC, { skillsDir, dataDir });
    // Disk drifts: a skill with the target name now exists -> fresh guardrail
    // nameAvailable=false -> guardrail differs from the approved snapshot.
    mkdirSync(join(skillsDir, NAME), { recursive: true });
    writeFileSync(join(skillsDir, NAME, 'SKILL.md'), '---\nname: squatter\ndescription: x\n---\n# squatter\n');

    const res = applyApprovedAction(approved, {
      pluginRoot: root, skillsDir, dataDir, taxonomyDataDir: dataDir,
      createSpec: { description: TEST_DOMAIN_SPEC.description, categories: TEST_DOMAIN_SPEC.categories },
      apply: true,
    });
    assert.equal(res.applied, false, 'a name collision after approval must abort');
    assert.equal(res.revalidate.ok, false);
    // The pre-existing (squatter) SKILL.md is untouched; no references/ created.
    assert.equal(existsSync(join(skillsDir, NAME, 'references')), false, 'no scaffold written on abort');
    assert.equal(loadDecisions(dataDir).actions[actionKey(approved)].status, 'approved', 'ledger left at approved');
  });
});

test('applyApprovedAction create — refuses a non-approved (pending) entry with zero mutation', () => {
  withSkillsFixture(({ root, skillsDir, dataDir }) => {
    const { entry } = runCreatePlan(NAME, { skillsDir, dataDir, write: false, ...TEST_DOMAIN_SPEC });
    saveDecisions(recordAction(loadDecisions(dataDir), entry), dataDir); // pending, not approved

    const res = applyApprovedAction(entry, {
      pluginRoot: root, skillsDir, dataDir, taxonomyDataDir: dataDir,
      createSpec: { description: TEST_DOMAIN_SPEC.description, categories: TEST_DOMAIN_SPEC.categories },
      apply: true,
    });
    assert.equal(res.applied, false);
    assert.match(res.revalidate.reason, /approved/i);
    assert.equal(existsSync(join(skillsDir, NAME)), false, 'no mutation');
  });
});

// ===========================================================================
// (D) Taxonomy helper — applyCategoryAdditions (additive, never clobbers)
// ===========================================================================

test('applyCategoryAdditions — adds new category ownership, no-ops on an already-owned category', () => {
  const tax = { category_skill: { existing: 'owner-skill' } };
  const next = applyCategoryAdditions(tax, [
    { category: 'brand-new', skill: 'new-skill' },
    { category: 'existing', skill: 'new-skill' }, // must NOT clobber owner-skill
  ]);
  assert.equal(next.category_skill['brand-new'], 'new-skill', 'new category added');
  assert.equal(next.category_skill.existing, 'owner-skill', 'existing owner never clobbered');
  // pure: original untouched
  assert.equal(tax.category_skill['brand-new'], undefined, 'input not mutated');
});

test('applyCategoryAdditions — disjoint from applyCategoryReassignments (additions vs repoint stay separate ops)', () => {
  const tax = { category_skill: { a: 's1' } };
  const added = applyCategoryAdditions(tax, [{ category: 'b', skill: 's2' }]);
  const repointed = applyCategoryReassignments(added, [{ category: 'a', from: 's1', to: 's2' }]);
  assert.deepEqual(repointed.category_skill, { a: 's2', b: 's2' });
});