From e47fc9bd591a0ce3bb673f616b9811c5052f7a26 Mon Sep 17 00:00:00 2001 From: Kjell Tore Guttormsen Date: Sat, 20 Jun 2026 21:37:19 +0200 Subject: [PATCH] =?UTF-8?q?feat(ms-ai-architect):=20Sesjon=2020=20?= =?UTF-8?q?=E2=80=94=20B3=20create=5Fskill=20(dvelende,=20konstruktiv=20op?= =?UTF-8?q?)=20[skip-docs]?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit B3 + Spor B (krav 5+6) KOMPLETT. create_skill er speilbildet av de destruktive ops: merge/saner/retire beviser "ingen kuratert verdi TAPT"; create beviser "ingen sub-bar skill FØDT" (kvalitetsgulv). Ingen ny domene-skill shippet (svar 3) — mekanismen bevist, ikke brukt. - planCreateSkill (ren): genererer scaffold + selv-validerer mot eval.mjs sine EGNE checkers (K2/K3/K5/K6/refTall + K10) — rubrikken har én kilde. Guardrail ok iff alle deterministiske K passerer + K10 --- scripts/kb-eval/apply-skill-op.mjs | 67 +++- scripts/kb-eval/lib/skill-ops.mjs | 282 ++++++++++++++- scripts/kb-eval/plan-skill-op.mjs | 44 ++- scripts/kb-update/lib/taxonomy.mjs | 20 ++ tests/kb-eval/test-skill-ops-create.test.mjs | 356 +++++++++++++++++++ 5 files changed, 758 insertions(+), 11 deletions(-) create mode 100644 tests/kb-eval/test-skill-ops-create.test.mjs diff --git a/scripts/kb-eval/apply-skill-op.mjs b/scripts/kb-eval/apply-skill-op.mjs index d4badc5..6cdd5ac 100644 --- a/scripts/kb-eval/apply-skill-op.mjs +++ b/scripts/kb-eval/apply-skill-op.mjs @@ -21,26 +21,59 @@ // node scripts/kb-eval/apply-skill-op.mjs retire [--apply] [--date YYYY-MM-DD] // node scripts/kb-eval/apply-skill-op.mjs merge [--apply] [--date YYYY-MM-DD] +import { readFileSync, readdirSync, existsSync } from 'node:fs'; import { dirname, join } from 'node:path'; import { fileURLToPath } from 'node:url'; +import { splitFrontmatter, extractDescription } from './eval.mjs'; import { loadTaxonomy } from '../kb-update/lib/taxonomy.mjs'; import { loadDecisions, listActions, actionKey } from '../kb-update/lib/decisions-io.mjs'; -import { applyApprovedAction, SANITIZE_OPERATION, RETIRE_OPERATION, MERGE_OPERATION } from './lib/skill-ops.mjs'; +import { + applyApprovedAction, + SANITIZE_OPERATION, + RETIRE_OPERATION, + MERGE_OPERATION, + CREATE_OPERATION, + TEST_DOMAIN_SPEC, +} from './lib/skill-ops.mjs'; const __dirname = dirname(fileURLToPath(import.meta.url)); const PLUGIN_ROOT = join(__dirname, '..', '..'); const SKILLS_DIR = join(PLUGIN_ROOT, 'skills'); const AGENTS_DIR = join(PLUGIN_ROOT, 'agents'); const LEDGER_DATA_DIR = join(__dirname, '..', 'kb-update', 'data'); +const PROMPTS_FILE = join(__dirname, 'data', 'k1-trigger-prompts.json'); -const OP_BY_VERB = { sanitize: SANITIZE_OPERATION, retire: RETIRE_OPERATION, merge: MERGE_OPERATION }; +const OP_BY_VERB = { + sanitize: SANITIZE_OPERATION, + retire: RETIRE_OPERATION, + merge: MERGE_OPERATION, + create: CREATE_OPERATION, +}; + +/** Read the SKILL.md descriptions from disk (read-only) — create's K10 siblings. */ +function loadDescriptions() { + const out = {}; + for (const e of readdirSync(SKILLS_DIR, { withFileTypes: true })) { + if (!e.isDirectory()) continue; + const md = join(SKILLS_DIR, e.name, 'SKILL.md'); + if (!existsSync(md)) continue; + out[e.name] = extractDescription(splitFrontmatter(readFileSync(md, 'utf8')).frontmatter); + } + return out; +} + +/** Read the curated K1 trigger-prompt set (boundary-tension source for K10). */ +function loadPromptSet() { + return existsSync(PROMPTS_FILE) ? JSON.parse(readFileSync(PROMPTS_FILE, 'utf8')) : {}; +} const USAGE = 'Usage:\n' + ' node scripts/kb-eval/apply-skill-op.mjs list\n' + ' node scripts/kb-eval/apply-skill-op.mjs sanitize [--apply] [--date YYYY-MM-DD]\n' + ' node scripts/kb-eval/apply-skill-op.mjs retire [--apply] [--date YYYY-MM-DD]\n' + - ' node scripts/kb-eval/apply-skill-op.mjs merge [--apply] [--date YYYY-MM-DD]'; + ' node scripts/kb-eval/apply-skill-op.mjs merge [--apply] [--date YYYY-MM-DD]\n' + + ' node scripts/kb-eval/apply-skill-op.mjs create [--apply] [--date YYYY-MM-DD]'; /** List approved actions awaiting apply (operator's apply queue). */ function listApproved() { @@ -51,7 +84,7 @@ function listApproved() { if (!approved.length) console.log(' (ingen — kjør plan-skill-op … --write og sett status:approved i decisions.json)'); for (const a of approved) { const t = a.targets ?? {}; - const tgt = t.skill ?? `${t.absorbed} -> ${t.absorber}`; + const tgt = t.skill ?? t.name ?? `${t.absorbed} -> ${t.absorber}`; console.log(` • ${a.operation_type}: ${tgt} [guardrail ${a.guardrail?.ok ? 'OK' : 'FAILED'}]`); } if (applied.length) console.log(`\nAllerede anvendt (${applied.length}): ${applied.map((a) => actionKey(a)).join(', ')}`); @@ -73,6 +106,12 @@ function printResult(res, verb, label, apply) { `${d.taxonomyReassignments?.length ?? 0} kategori(er), arkivert absorbed-SKILL.md. ` + `Kjør på nytt med --apply for å eksekvere.`, ); + } else if (verb === 'create') { + const sc = res.plan?.scaffold ?? {}; + console.log( + `\n Ville opprettet skills/${label}/ — 1 SKILL.md + ${sc.files?.length ?? 0} ref-fil(er), ` + + `${sc.taxonomyAdditions?.length ?? 0} taksonomi-tillegg. Kjør på nytt med --apply for å eksekvere.`, + ); } else { const n = verb === 'sanitize' ? (d.removals?.length ?? 0) : (d.archiveMoves?.length ?? 0); console.log(`\n Ville arkivert+fjernet ${n} fil(er). Kjør på nytt med --apply for å eksekvere.`); @@ -87,6 +126,9 @@ function printResult(res, verb, label, apply) { console.log(` flyttet: ${rep.refMoves} ref-fil(er) -> absorber`); console.log(` taksonomi: ${rep.taxonomyReassignments} kategori(er) reassignet + persistert`); if (rep.archivedSkillMd) console.log(` arkivert: ${rep.archivedSkillMd}`); + } else if (rep.op === CREATE_OPERATION) { + console.log(` opprettet: ${rep.filesWritten} fil(er) under ${rep.createdDir} (1 SKILL.md + ${rep.filesWritten - 1} ref-filer)`); + console.log(` taksonomi: ${rep.taxonomyAdditions} tillegg (additivt)`); } else { console.log(` arkivert: ${rep.moves} fil(er) -> archive/`); } @@ -111,7 +153,8 @@ function main() { process.exit(2); } - // Resolve targets + ledger key per op arity (merge takes two skills). + // Resolve targets + ledger key per op arity (merge takes two skills; create + // is keyed on its new name; sanitize/retire on their one skill). let targets, label; if (op === MERGE_OPERATION) { if (positional.length < 3) { @@ -120,6 +163,13 @@ function main() { } targets = { absorber: positional[1], absorbed: positional[2] }; label = `${targets.absorbed} -> ${targets.absorber}`; + } else if (op === CREATE_OPERATION) { + if (positional.length < 2) { + console.error(USAGE); + process.exit(2); + } + targets = { name: positional[1] }; + label = positional[1]; } else { if (positional.length < 2) { console.error(USAGE); @@ -141,8 +191,10 @@ function main() { process.exit(1); } - // retire + merge both consult the taxonomy (owned-category awareness / repoint). + // retire + merge both consult the taxonomy (owned-category awareness / repoint); + // create re-supplies its scaffold spec + K10 siblings for drift-safe revalidation. const needsTaxonomy = op === RETIRE_OPERATION || op === MERGE_OPERATION; + const isCreate = op === CREATE_OPERATION; const res = applyApprovedAction(entry, { pluginRoot: PLUGIN_ROOT, skillsDir: SKILLS_DIR, @@ -152,6 +204,9 @@ function main() { apply, decided_at: apply ? decided_at : null, categorySkill: needsTaxonomy ? loadTaxonomy(LEDGER_DATA_DIR).category_skill : undefined, + createSpec: isCreate ? { description: TEST_DOMAIN_SPEC.description, categories: TEST_DOMAIN_SPEC.categories } : undefined, + existingDescriptions: isCreate ? loadDescriptions() : undefined, + promptSet: isCreate ? loadPromptSet() : undefined, }); printResult(res, verb, label, apply); } diff --git a/scripts/kb-eval/lib/skill-ops.mjs b/scripts/kb-eval/lib/skill-ops.mjs index 69cf9c1..c985f53 100644 --- a/scripts/kb-eval/lib/skill-ops.mjs +++ b/scripts/kb-eval/lib/skill-ops.mjs @@ -12,7 +12,7 @@ // preserved (ref-file count-invariant + set-equality) and refuse (ok=false) // when two reference files would clobber on the same target path. -import { readdirSync, readFileSync, existsSync, statSync, mkdirSync, renameSync, rmSync } from 'node:fs'; +import { readdirSync, readFileSync, writeFileSync, existsSync, statSync, mkdirSync, renameSync, rmSync } from 'node:fs'; import { join, relative, basename, dirname, sep } from 'node:path'; import { isDeepStrictEqual } from 'node:util'; import { @@ -22,11 +22,30 @@ import { actionKey, setActionStatus, } from '../../kb-update/lib/decisions-io.mjs'; -import { loadTaxonomy, saveTaxonomy, applyCategoryReassignments } from '../../kb-update/lib/taxonomy.mjs'; +import { + loadTaxonomy, + saveTaxonomy, + applyCategoryReassignments, + applyCategoryAdditions, +} from '../../kb-update/lib/taxonomy.mjs'; +import { + splitFrontmatter, + extractDescription, + checkK2, + checkK3, + checkK5K6, + checkRefConsistency, +} from '../eval.mjs'; +import { + computeOverlapFromInputs, + perSkillSiblingOverlap, + K10_OVERLAP_THRESHOLD, +} from './sibling-overlap.mjs'; export const MERGE_OPERATION = 'merge_skills'; export const SANITIZE_OPERATION = 'sanitize_skill'; export const RETIRE_OPERATION = 'retire_skill'; +export const CREATE_OPERATION = 'create_skill'; // --------------------------------------------------------------------------- // Pure core — planMergeSkills @@ -275,6 +294,192 @@ export function planRetireSkill(skill, ctx = {}) { return { entry, diff, guardrail }; } +// --------------------------------------------------------------------------- +// Pure core — planCreateSkill (CONSTRUCTIVE: born compliant or refused) +// --------------------------------------------------------------------------- + +/** + * Canonical "dvelende" scaffold spec. create_skill ships no new domain skill + * (operator answer 3 — the mechanism is proven, not exercised on production): + * this throwaway test-domain spec is what both CLIs and the tests scaffold to + * demonstrate a skill that passes the deterministic rubric from birth. + */ +export const TEST_DOMAIN_SPEC = { + description: + 'Use this skill when validating the create_skill scaffolding mechanism against a throwaway test domain. ' + + 'Triggers on: "create-skill self-test", "scaffold validation", "dvelende create probe".', + categories: [ + { + category: 'core', + files: [ + { file: 'overview.md', title: 'Test-domene oversikt', source: 'internal://create-skill-self-test' }, + { file: 'mechanism.md', title: 'Scaffold-mekanikk', source: 'internal://create-skill-self-test' }, + ], + }, + { + category: 'patterns', + files: [ + { file: 'routing.md', title: 'Progressiv disclosure-mønster', source: 'internal://create-skill-self-test' }, + ], + }, + ], +}; + +/** Title-case a kebab/snake skill name for the SKILL.md heading. */ +function titleCase(name) { + return name + .split(/[-_]/) + .filter(Boolean) + .map((w) => w[0].toUpperCase() + w.slice(1)) + .join(' '); +} + +/** + * Generate a SKILL.md that satisfies eval.mjs's deterministic rubric: a + * use-when/quoted description (K2), a short body (K3), a routing table that + * cites per-folder counts (refCountConsistency) AND names every reference file + * (K5 namedRatio + K6 named start files). + */ +function buildSkillMd(name, description, categories) { + const tableRows = categories + .map((c) => `| \`references/${c.category}/\` | ${c.files.length} |`) + .join('\n'); + const startHere = categories + .flatMap((c) => c.files.map((f) => `- \`references/${c.category}/${f.file}\` — ${f.title ?? f.file}`)) + .join('\n'); + return ( + `---\n` + + `name: ${name}\n` + + `description: ${description}\n` + + `---\n\n` + + `> **INSTRUKSJON:** Test-domene-skill generert av create_skill-mekanismen (dvelende). ` + + `Bruk kunnskapsbasen i \`references/\` for detaljert veiledning.\n\n` + + `# ${titleCase(name)}\n\n` + + `Scaffold generert av create_skill — født compliant mot K1–K10 (deterministisk delmengde). ` + + `Dette er et test-domene; ingen ny domene-skill shippes.\n\n` + + `## Referansekart\n\n` + + `| Område | Antall filer |\n` + + `|--------|--------------|\n` + + `${tableRows}\n\n` + + `### Start her\n` + + `${startHere}\n` + ); +} + +/** Minimal curated-shaped reference body (Source-header hygiene; content is not + * part of the deterministic gate — eval reads SKILL.md, not ref bodies). */ +function buildRefBody(f) { + return ( + `# ${f.title ?? f.file}\n\n` + + `**Source:** ${f.source ?? 'internal://create-skill-self-test'}\n\n` + + `Generert referansefil for create_skill test-domene (dvelende). ` + + `Erstattes av kuratert innhold dersom en ekte domene-skill noen gang opprettes.\n` + ); +} + +/** + * Plan a new-skill scaffold. Pure; mutates nothing; reads no disk. The + * CONSTRUCTIVE guardrail is the mirror of the destructive ones: merge/sanitize/ + * retire prove "no curated value LOST" (preservation); create proves "no + * sub-bar skill BORN" (quality floor). It refuses (ok=false) when the generated + * scaffold would fail any deterministic criterion eval.mjs enforces (K2/K3/K5/ + * K6/refCountConsistency), when the new description overlaps a sibling at/above + * the K10 threshold, or when the name already belongs to a skill. The verdict is + * re-derived from eval.mjs's OWN checkers — the rubric has a single source. + * + * @param {string} name new skill directory name (kebab-case) + * @param {{ description?: string, + * categories?: Array<{ category: string, files: Array<{file: string, title?: string, source?: string}> }>, + * existingSkills?: string[], existingDescriptions?: Record, + * promptSet?: object, threshold?: number, decided_at?: string|null, note?: string }} [ctx] + * @returns {{ entry: object, scaffold: object, guardrail: object }} + */ +export function planCreateSkill(name, ctx = {}) { + if (!name) throw new Error('planCreateSkill: name is required'); + const description = ctx.description ?? ''; + const categories = ctx.categories ?? []; + const existingSkills = ctx.existingSkills ?? []; + const existingDescriptions = ctx.existingDescriptions ?? {}; + const promptSet = ctx.promptSet ?? {}; + const threshold = ctx.threshold ?? K10_OVERLAP_THRESHOLD; + + // Build the scaffold artifacts. + const skillMdContent = buildSkillMd(name, description, categories); + const files = categories.flatMap((c) => + c.files.map((f) => ({ + path: `skills/${name}/references/${c.category}/${f.file}`, + content: buildRefBody(f), + })), + ); + const scaffold = { + skillMd: { path: `skills/${name}/SKILL.md`, content: skillMdContent }, + files, + taxonomyAdditions: categories.map((c) => ({ category: c.category, skill: name })), + }; + + // CONSTRUCTIVE guardrail — re-run eval.mjs's deterministic checkers against the + // generated SKILL.md (single source of truth for the rubric). + const { frontmatter, body } = splitFrontmatter(skillMdContent); + const parsedDescription = extractDescription(frontmatter); + const perFolder = {}; + for (const c of categories) perFolder[c.category] = c.files.length; + const totalRefFiles = files.length; + + const k2 = checkK2(parsedDescription); + const k3 = checkK3(body); + const { K5: k5, K6: k6 } = checkK5K6(body, totalRefFiles); + const refConsistency = checkRefConsistency(body, perFolder); + + // K10 — sibling-scope-non-overlap of the NEW description vs the existing skills + // (cross-skill; same overlap core as eval's attachSiblingOverlap). + const descriptionsBySkill = { ...existingDescriptions, [name]: parsedDescription }; + const overlap = computeOverlapFromInputs(descriptionsBySkill, promptSet); + const k10map = perSkillSiblingOverlap(overlap.pairs, { threshold }); + const k10 = k10map[name] ?? { maxCombined: 0, worstSibling: null, pass: true, threshold }; + + const nameAvailable = !existingSkills.includes(name); + + const failed = []; + if (!nameAvailable) failed.push('name'); + if (!k2.pass) failed.push('K2'); + if (!k3.pass) failed.push('K3'); + if (!k5.pass) failed.push('K5'); + if (!k6.pass) failed.push('K6'); + if (!refConsistency.consistent) failed.push('refConsistency'); + if (!k10.pass) failed.push('K10'); + + const guardrail = { + method: + 'constructive (mirror of the destructive guardrails): no sub-bar skill is born. The scaffold must pass ' + + "eval.mjs's deterministic rubric from birth (K2/K3/K5/K6/refCountConsistency), must not overlap a sibling " + + 'at/above the K10 threshold, and the name must be free. ok iff failed === [].', + nameAvailable, + k2, + k3, + k5, + k6, + refConsistency, + k10, + failed, + ok: failed.length === 0, + }; + + const entry = { + operation_type: CREATE_OPERATION, + status: 'pending', + decided_at: ctx.decided_at ?? null, // never Date.now() in a pure lib + targets: { name }, + guardrail, + note: + ctx.note ?? + `Dry-run: create ${name} — scaffold ${ + guardrail.ok ? 'passes K1–K10 (deterministic) — born compliant' : 'FAILS rubric (' + failed.join(',') + ')' + }. Applies nothing.`, + }; + + return { entry, scaffold, guardrail }; +} + // --------------------------------------------------------------------------- // Impure shell — read-only disk load + gated ledger write // --------------------------------------------------------------------------- @@ -398,6 +603,35 @@ export function runRetirePlan(skill, opts) { return result; } +/** + * Drive a create dry-run: read existing skill NAMES read-only (name-collision + * check), plan the scaffold, and (only with write:true) record the PENDING entry + * in decisions.json. Writes NOTHING under skills/ — the constructive apply is a + * separate approved step. The scaffold content comes from the caller (the CLI's + * TEST_DOMAIN_SPEC); K10 siblings come from the caller's existing descriptions. + * @param {string} name + * @param {{ skillsDir: string, dataDir: string, write?: boolean, decided_at?: string|null, + * description?: string, categories?: object[], + * existingDescriptions?: Record, promptSet?: object, threshold?: number }} opts + * @returns {{ entry: object, scaffold: object, guardrail: object }} + */ +export function runCreatePlan(name, opts) { + const { + skillsDir, dataDir, write = false, decided_at = null, + description, categories, existingDescriptions, promptSet, threshold, + } = opts; + const existingSkills = existsSync(skillsDir) + ? readdirSync(skillsDir, { withFileTypes: true }).filter((e) => e.isDirectory()).map((e) => e.name) + : []; + const result = planCreateSkill(name, { + description, categories, existingSkills, existingDescriptions, promptSet, threshold, decided_at, + }); + if (write) { + saveDecisions(recordAction(loadDecisions(dataDir), result.entry), dataDir); + } + return result; +} + // =========================================================================== // Apply-path (Sesjon 18–19) — the DESTRUCTIVE frontier. An operator-approved // ledger entry is executed into a real skills/ mutation. Single-skill ops @@ -469,6 +703,7 @@ export function applyApprovedAction(approvedEntry, opts) { const { pluginRoot, skillsDir, agentsDir, dataDir, taxonomyDataDir, apply = false, decided_at = null, categorySkill, + createSpec, existingDescriptions, promptSet, } = opts; const op = approvedEntry?.operation_type; const target = approvedEntry?.targets ?? {}; @@ -481,8 +716,17 @@ export function applyApprovedAction(approvedEntry, opts) { fresh = runRetirePlan(target.skill, { skillsDir, dataDir, write: false, categorySkill }); } else if (op === MERGE_OPERATION) { fresh = runMergePlan(target.absorber, target.absorbed, { skillsDir, dataDir, write: false, categorySkill }); + } else if (op === CREATE_OPERATION) { + // create has no prior on-disk state to re-derive: the scaffold spec is + // re-supplied by the caller; the disk-dependent part is name-availability. + const spec = createSpec ?? {}; + fresh = runCreatePlan(target.name, { + skillsDir, dataDir, write: false, decided_at: null, + description: spec.description, categories: spec.categories, + existingDescriptions, promptSet, + }); } else { - throw new Error(`applyApprovedAction: unsupported operation_type "${op}" — applies sanitize/retire/merge only (create_skill staged later)`); + throw new Error(`applyApprovedAction: unsupported operation_type "${op}" — applies sanitize/retire/merge/create only`); } // 2. Idempotent revalidation against fresh disk — refuse on any failure. @@ -491,6 +735,38 @@ export function applyApprovedAction(approvedEntry, opts) { return { applied: false, preview: !apply, revalidate, plan: fresh }; } + // 3a-create — CONSTRUCTIVE: write the born-compliant scaffold + persist + // taxonomy ownership additively. No archival (nothing pre-existed); the + // guardrail's nameAvailable made real is that skills/ is created fresh. + if (op === CREATE_OPERATION) { + const allFiles = [fresh.scaffold.skillMd, ...fresh.scaffold.files]; + for (const f of allFiles) { + const abs = join(pluginRoot, f.path); + mkdirSync(dirname(abs), { recursive: true }); + writeFileSync(abs, f.content); + } + // Persist taxonomy ownership for the new categories (additive — never + // clobbers a sibling's owned category; that would be a merge, not a create). + const taxDir = taxonomyDataDir ?? dataDir; + const additions = fresh.scaffold.taxonomyAdditions; + if (additions.length) { + saveTaxonomy(applyCategoryAdditions(loadTaxonomy(taxDir), additions), taxDir); + } + // Flip the ledger entry approved -> applied (gated write; audit record kept). + saveDecisions(setActionStatus(loadDecisions(dataDir), actionKey(approvedEntry), 'applied', decided_at), dataDir); + return { + applied: true, + revalidate, + report: { + op, target, + filesWritten: allFiles.length, + taxonomyAdditions: additions.length, + createdDir: `skills/${target.name}`, + ledgerStatus: 'applied', + }, + }; + } + // 3a. merge — curated RELOCATION (not archival) + taxonomy persist + shell archive. if (op === MERGE_OPERATION) { const { absorber, absorbed } = target; diff --git a/scripts/kb-eval/plan-skill-op.mjs b/scripts/kb-eval/plan-skill-op.mjs index 73dd3cf..fd798b2 100644 --- a/scripts/kb-eval/plan-skill-op.mjs +++ b/scripts/kb-eval/plan-skill-op.mjs @@ -17,7 +17,7 @@ import { join, dirname } from 'node:path'; import { fileURLToPath } from 'node:url'; import { splitFrontmatter, extractDescription } from './eval.mjs'; import { loadTaxonomy } from '../kb-update/lib/taxonomy.mjs'; -import { runMergePlan, runSanitizePlan, runRetirePlan } from './lib/skill-ops.mjs'; +import { runMergePlan, runSanitizePlan, runRetirePlan, runCreatePlan, TEST_DOMAIN_SPEC } from './lib/skill-ops.mjs'; const __dirname = dirname(fileURLToPath(import.meta.url)); const PLUGIN_ROOT = join(__dirname, '..', '..'); @@ -25,6 +25,7 @@ const SKILLS_DIR = join(PLUGIN_ROOT, 'skills'); const AGENTS_DIR = join(PLUGIN_ROOT, 'agents'); const LEDGER_DATA_DIR = join(__dirname, '..', 'kb-update', 'data'); const TAX_DATA_DIR = LEDGER_DATA_DIR; +const PROMPTS_FILE = join(__dirname, 'data', 'k1-trigger-prompts.json'); /** Read the SKILL.md descriptions from disk (read-only). */ function loadDescriptions() { @@ -38,6 +39,11 @@ function loadDescriptions() { return out; } +/** Read the curated K1 trigger-prompt set (boundary-tension source for K10). */ +function loadPromptSet() { + return existsSync(PROMPTS_FILE) ? JSON.parse(readFileSync(PROMPTS_FILE, 'utf8')) : {}; +} + function printPlan({ entry, diff, guardrail }) { const g = guardrail; console.log(`\nmerge_skills — DRY-RUN: ${entry.targets.absorbed} -> ${entry.targets.absorber}\n`); @@ -89,11 +95,33 @@ function printRetirePlan({ entry, diff, guardrail }) { console.log(`\nStatus: ${entry.status} (forslag — anvender INGENTING; krever operatør-godkjenning + apply-sesjon).\n`); } +function printCreatePlan({ entry, scaffold, guardrail }) { + const g = guardrail; + console.log(`\ncreate_skill — DRY-RUN: ${entry.targets.name}\n`); + console.log(`Guardrail (konstruktiv — født compliant ELLER nektet):`); + console.log( + ` navn-ledig=${g.nameAvailable} K2=${g.k2.pass} K3=${g.k3.pass} K5=${g.k5.pass} ` + + `K6=${g.k6.pass} refTall=${g.refConsistency.consistent} K10=${g.k10.pass}`, + ); + console.log(` feilede=${g.failed.length ? g.failed.join(', ') : '0'}`); + console.log(` => ${g.ok ? 'OK (scaffold passerer K1–K10 deterministisk — trygt å opprette)' : 'FAILED (sub-bar skill — blokkert)'}\n`); + console.log(`Scaffold:`); + console.log(` ${scaffold.skillMd.path}`); + console.log(` ref-filer: ${scaffold.files.length}`); + for (const f of scaffold.files) console.log(` • ${f.path}`); + if (scaffold.taxonomyAdditions.length) { + console.log(` taksonomi-tillegg (additivt):`); + for (const t of scaffold.taxonomyAdditions) console.log(` • ${t.category} -> ${t.skill}`); + } + console.log(`\nStatus: ${entry.status} (forslag — anvender INGENTING; krever operatør-godkjenning + apply-sesjon).\n`); +} + const USAGE = 'Usage:\n' + ' node scripts/kb-eval/plan-skill-op.mjs merge [--json] [--write [--date YYYY-MM-DD]]\n' + ' node scripts/kb-eval/plan-skill-op.mjs sanitize [--json] [--write [--date YYYY-MM-DD]]\n' + - ' node scripts/kb-eval/plan-skill-op.mjs retire [--hard] [--json] [--write [--date YYYY-MM-DD]]'; + ' node scripts/kb-eval/plan-skill-op.mjs retire [--hard] [--json] [--write [--date YYYY-MM-DD]]\n' + + ' node scripts/kb-eval/plan-skill-op.mjs create [--json] [--write [--date YYYY-MM-DD]]'; function main() { const args = process.argv.slice(2); @@ -142,6 +170,18 @@ function main() { }); printer = printRetirePlan; label = 'retire_skill'; + } else if (op === 'create' && positional.length >= 2) { + result = runCreatePlan(positional[1], { + skillsDir: SKILLS_DIR, + dataDir: LEDGER_DATA_DIR, + write: doWrite, + decided_at, + ...TEST_DOMAIN_SPEC, + existingDescriptions: loadDescriptions(), + promptSet: loadPromptSet(), + }); + printer = printCreatePlan; + label = 'create_skill'; } else { console.error(USAGE); process.exit(2); diff --git a/scripts/kb-update/lib/taxonomy.mjs b/scripts/kb-update/lib/taxonomy.mjs index 4c5bb99..9c92c2d 100644 --- a/scripts/kb-update/lib/taxonomy.mjs +++ b/scripts/kb-update/lib/taxonomy.mjs @@ -56,6 +56,26 @@ export function applyCategoryReassignments(tax, reassignments) { return { ...tax, category_skill: next }; } +/** + * Add category ownership for a NEW skill (create_skill, Sesjon 20). Pure — + * returns a new taxonomy, mutates nothing. For each { category, skill }, the + * category is assigned to `skill` ONLY when it is currently UNOWNED (absent): + * an existing owner is never clobbered (a create must not steal a sibling's + * category — that would be a merge, a different gated op). Additive counterpart + * to applyCategoryReassignments: reassign repoints an owned category, add claims + * an unowned one. The same gated apply-path is the only writer. + * @param {object} tax + * @param {Array<{category: string, skill: string}>} additions + * @returns {object} new taxonomy + */ +export function applyCategoryAdditions(tax, additions) { + const next = { ...tax.category_skill }; + for (const { category, skill } of additions) { + if (next[category] === undefined) next[category] = skill; + } + return { ...tax, category_skill: next }; +} + /** * Target sitemap child-name prefixes to scan/poll. * @param {object} tax diff --git a/tests/kb-eval/test-skill-ops-create.test.mjs b/tests/kb-eval/test-skill-ops-create.test.mjs new file mode 100644 index 0000000..59e2803 --- /dev/null +++ b/tests/kb-eval/test-skill-ops-create.test.mjs @@ -0,0 +1,356 @@ +// tests/kb-eval/test-skill-ops-create.test.mjs +// Spor B / B3 (Sesjon 20): create_skill — the CONSTRUCTIVE op (dvelende, last B3). +// +// merge/sanitize/retire are DESTRUCTIVE: their guardrail proves "no curated +// value LOST" (preservation). create_skill is the mirror: its guardrail proves +// "no sub-bar skill BORN" (quality floor). The binding gate-criterion is that +// the generated scaffold passes eval.mjs's OWN deterministic rubric from birth +// (K2/K3/K5/K6/refCountConsistency + K10 sibling-non-overlap) — so this file +// re-runs the very eval checkers against the scaffold to prove it independently. +// +// Two layers, like every other B3 op: +// (A) pure core — planCreateSkill(name, ctx): mutates nothing, reads no disk, +// returns { entry, scaffold, guardrail }. Refuses (ok=false) on a failing +// criterion, a sibling overlap >= threshold, or a name collision. +// (B) impure shell — runCreatePlan(...) gated dry-run (0 writes under skills/) +// + applyApprovedAction(...) create branch (writes the scaffold on --apply). +// +// Every disk scenario runs against a tmpdir fixture — NEVER the real skills/. + +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { mkdtempSync, rmSync, mkdirSync, writeFileSync, readFileSync, existsSync, readdirSync, statSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { + planCreateSkill, + runCreatePlan, + applyApprovedAction, + TEST_DOMAIN_SPEC, + CREATE_OPERATION, +} from '../../scripts/kb-eval/lib/skill-ops.mjs'; +import { + loadDecisions, + saveDecisions, + recordAction, + actionKey, + isActionDecided, +} from '../../scripts/kb-update/lib/decisions-io.mjs'; +import { applyCategoryAdditions, applyCategoryReassignments } from '../../scripts/kb-update/lib/taxonomy.mjs'; +import { + splitFrontmatter, + extractDescription, + checkK2, + checkK3, + checkK5K6, + checkRefConsistency, +} from '../../scripts/kb-eval/eval.mjs'; + +// =========================================================================== +// (A) Pure core — planCreateSkill +// =========================================================================== + +const NAME = 'create-self-test'; + +test('planCreateSkill — guardrail OK for a clean test-domain scaffold; entry is a PENDING create_skill', () => { + const { entry, guardrail, scaffold } = planCreateSkill(NAME, { ...TEST_DOMAIN_SPEC, decided_at: '2026-06-20' }); + assert.equal(guardrail.ok, true, 'a born-compliant scaffold passes the constructive guardrail'); + assert.deepEqual(guardrail.failed, [], 'no failing criteria'); + assert.equal(guardrail.nameAvailable, true); + assert.equal(entry.operation_type, CREATE_OPERATION); + assert.equal(entry.status, 'pending', 'planner only proposes — never approved'); + assert.deepEqual(entry.targets, { name: NAME }); + assert.equal(entry.decided_at, '2026-06-20'); + // scaffold carries the SKILL.md + ref files + taxonomy additions + assert.equal(scaffold.skillMd.path, `skills/${NAME}/SKILL.md`); + assert.equal(scaffold.files.length, 3, 'TEST_DOMAIN_SPEC = 2 core + 1 patterns'); + assert.deepEqual( + scaffold.files.map((f) => f.path).sort(), + [ + `skills/${NAME}/references/core/mechanism.md`, + `skills/${NAME}/references/core/overview.md`, + `skills/${NAME}/references/patterns/routing.md`, + ], + ); + assert.deepEqual(scaffold.taxonomyAdditions, [ + { category: 'core', skill: NAME }, + { category: 'patterns', skill: NAME }, + ]); +}); + +test('planCreateSkill — THE gate-criterion: the scaffold passes eval.mjs deterministic K1–K10 from birth', () => { + const { scaffold } = planCreateSkill(NAME, { ...TEST_DOMAIN_SPEC }); + // Independently re-derive the verdict via eval.mjs's OWN checkers (not the + // planner's guardrail) against the generated SKILL.md. + const { frontmatter, body } = splitFrontmatter(scaffold.skillMd.content); + const description = extractDescription(frontmatter); + const totalRefFiles = scaffold.files.length; + // actual per-folder counts derived from the scaffold files (what eval reads off disk) + const perFolder = {}; + for (const f of scaffold.files) { + const cat = f.path.split('/references/')[1].split('/')[0]; + perFolder[cat] = (perFolder[cat] || 0) + 1; + } + const k2 = checkK2(description); + const k3 = checkK3(body); + const { K5, K6 } = checkK5K6(body, totalRefFiles); + const rc = checkRefConsistency(body, perFolder); + + assert.equal(k2.pass, true, 'K2 description format'); + assert.equal(k3.pass, true, 'K3 body <= 500 lines'); + assert.equal(K5.pass, true, `K5 progressive disclosure (ratio ${K5.namedRatio})`); + assert.equal(K6.pass, true, 'K6 routing table has named start files'); + assert.equal(rc.consistent, true, `refCountConsistency: ${JSON.stringify(rc.mismatches)}`); +}); + +test('planCreateSkill — guardrail FAILS when the description fails K2 (no use-when form, < 3 quoted phrases)', () => { + const { guardrail } = planCreateSkill(NAME, { + ...TEST_DOMAIN_SPEC, + description: 'A short blurb about a thing with no required format.', + }); + assert.equal(guardrail.ok, false, 'a sub-bar description must not be born compliant'); + assert.ok(guardrail.failed.includes('K2'), 'K2 surfaced as the failing criterion'); + assert.equal(guardrail.k2.pass, false); +}); + +test('planCreateSkill — guardrail FAILS on a name collision (skill already exists)', () => { + const { guardrail } = planCreateSkill(NAME, { + ...TEST_DOMAIN_SPEC, + existingSkills: ['ms-ai-engineering', NAME, 'ms-ai-security'], + }); + assert.equal(guardrail.ok, false, 'cannot create over an existing skill'); + assert.equal(guardrail.nameAvailable, false); + assert.ok(guardrail.failed.includes('name')); +}); + +test('planCreateSkill — guardrail FAILS when the new description overlaps a sibling at/above K10 threshold', () => { + // A sibling that shares a distinctive token with the new description, plus a + // tiny threshold so ANY overlap trips K10 — proves the sibling-non-overlap gate. + const ctx = { + ...TEST_DOMAIN_SPEC, + description: 'Use this skill for the widget gadget domain. Triggers on: "widget a", "widget b", "widget c".', + existingDescriptions: { 'sibling-skill': 'A sibling that also owns the widget surface entirely.' }, + promptSet: {}, + threshold: 0.0001, + }; + const { guardrail } = planCreateSkill(NAME, ctx); + assert.equal(guardrail.ok, false, 'an over-overlapping description must be refused'); + assert.equal(guardrail.k10.pass, false); + assert.ok(guardrail.k10.maxCombined > 0, 'shares a distinctive token with the sibling'); + assert.ok(guardrail.failed.includes('K10')); +}); + +test('planCreateSkill — a lone skill (no siblings) trivially passes K10', () => { + const { guardrail } = planCreateSkill(NAME, { ...TEST_DOMAIN_SPEC, existingDescriptions: {}, promptSet: {} }); + assert.equal(guardrail.k10.pass, true); + assert.equal(guardrail.k10.maxCombined, 0); + assert.equal(guardrail.k10.worstSibling, null); +}); + +test('planCreateSkill — refCountConsistency holds: the routing table cites the scaffold counts exactly', () => { + const { guardrail, scaffold } = planCreateSkill(NAME, { ...TEST_DOMAIN_SPEC }); + assert.equal(guardrail.refConsistency.consistent, true); + // the body must cite `references/core/` | 2 and `references/patterns/` | 1 + assert.match(scaffold.skillMd.content, /`references\/core\/`\s*\|\s*2/); + assert.match(scaffold.skillMd.content, /`references\/patterns\/`\s*\|\s*1/); +}); + +// =========================================================================== +// (B) Impure shell — runCreatePlan (read-only existing skills + gated write) +// =========================================================================== + +function withSkillsFixture(fn, { seedExisting = [] } = {}) { + const root = mkdtempSync(join(tmpdir(), 'skillops-create-')); + const skillsDir = join(root, 'skills'); + const dataDir = join(root, 'data'); + mkdirSync(skillsDir, { recursive: true }); + mkdirSync(dataDir, { recursive: true }); + // Seed any pre-existing skill dirs (for collision tests). + for (const s of seedExisting) { + mkdirSync(join(skillsDir, s), { recursive: true }); + writeFileSync(join(skillsDir, s, 'SKILL.md'), `---\nname: ${s}\ndescription: existing\n---\n# ${s}\n`); + } + // Minimal taxonomy so applyCategoryAdditions has something to load. + writeFileSync(join(dataDir, 'domain-taxonomy.json'), JSON.stringify({ category_skill: {} }, null, 2) + '\n'); + try { + return fn({ root, skillsDir, dataDir }); + } finally { + rmSync(root, { recursive: true, force: true }); + } +} + +function snapshot(dir) { + const out = {}; + const walk = (d) => { + for (const e of readdirSync(d, { withFileTypes: true })) { + const p = join(d, e.name); + if (e.isDirectory()) walk(p); + else { const s = statSync(p); out[p] = `${s.size}:${s.mtimeMs}`; } + } + }; + walk(dir); + return out; +} + +test('runCreatePlan --write — records a PENDING entry and touches NOTHING under skills/', () => { + withSkillsFixture(({ skillsDir, dataDir }) => { + const before = snapshot(skillsDir); + const { entry, guardrail } = runCreatePlan(NAME, { skillsDir, dataDir, write: true, decided_at: '2026-06-20', ...TEST_DOMAIN_SPEC }); + + assert.equal(entry.status, 'pending'); + assert.equal(guardrail.ok, true); + + const led = loadDecisions(dataDir); + assert.equal(isActionDecided(led, actionKey(entry)), true); + assert.equal(led.actions[actionKey(entry)].status, 'pending'); + assert.equal(actionKey(entry), `create_skill:${NAME}`); + + // THE CONSTRUCTIVE INVARIANT: skills/ is byte-for-byte unchanged by a dry-run. + assert.deepEqual(snapshot(skillsDir), before, 'dry-run must not write under skills/'); + }, { seedExisting: ['ms-ai-engineering'] }); +}); + +test('runCreatePlan without --write — pure preview, no ledger file created', () => { + withSkillsFixture(({ skillsDir, dataDir }) => { + const { entry } = runCreatePlan(NAME, { skillsDir, dataDir, write: false, ...TEST_DOMAIN_SPEC }); + assert.equal(entry.status, 'pending'); + assert.equal(existsSync(join(dataDir, 'decisions.json')), false, 'no write without --write'); + }); +}); + +test('runCreatePlan — refuses a name that already exists on disk (read-only collision check)', () => { + withSkillsFixture(({ skillsDir, dataDir }) => { + const { guardrail } = runCreatePlan('ms-ai-security', { skillsDir, dataDir, write: false, ...TEST_DOMAIN_SPEC }); + assert.equal(guardrail.ok, false); + assert.equal(guardrail.nameAvailable, false); + }, { seedExisting: ['ms-ai-security'] }); +}); + +// =========================================================================== +// (C) Impure executor — applyApprovedAction create (writes the scaffold) +// =========================================================================== + +// Seed an operator-approved create entry exactly as the gate would after a +// reviewed `plan-skill-op create … --write` dry-run. +function approveCreate(name, spec, { skillsDir, dataDir }) { + const { entry } = runCreatePlan(name, { skillsDir, dataDir, write: false, decided_at: '2026-06-20', ...spec }); + const approved = { ...entry, status: 'approved' }; + saveDecisions(recordAction(loadDecisions(dataDir), approved), dataDir); + return approved; +} + +test('applyApprovedAction create — writes the scaffold, persists taxonomy, flips the ledger; the created skill passes eval K1–K10', () => { + withSkillsFixture(({ root, skillsDir, dataDir }) => { + const approved = approveCreate(NAME, TEST_DOMAIN_SPEC, { skillsDir, dataDir }); + const res = applyApprovedAction(approved, { + pluginRoot: root, skillsDir, dataDir, taxonomyDataDir: dataDir, + createSpec: { description: TEST_DOMAIN_SPEC.description, categories: TEST_DOMAIN_SPEC.categories }, + apply: true, decided_at: '2026-06-21', + }); + + assert.equal(res.applied, true); + assert.equal(res.revalidate.ok, true); + + // SKILL.md + all 3 reference files exist on disk. + assert.ok(existsSync(join(skillsDir, NAME, 'SKILL.md')), 'SKILL.md written'); + assert.ok(existsSync(join(skillsDir, NAME, 'references/core/overview.md')), 'core/overview.md written'); + assert.ok(existsSync(join(skillsDir, NAME, 'references/core/mechanism.md')), 'core/mechanism.md written'); + assert.ok(existsSync(join(skillsDir, NAME, 'references/patterns/routing.md')), 'patterns/routing.md written'); + + // Taxonomy ownership persisted additively. + const tax = JSON.parse(readFileSync(join(dataDir, 'domain-taxonomy.json'), 'utf8')); + assert.equal(tax.category_skill.core, NAME); + assert.equal(tax.category_skill.patterns, NAME); + + // Ledger flipped approved -> applied. + assert.equal(loadDecisions(dataDir).actions[actionKey(approved)].status, 'applied'); + + // INDEPENDENT proof: the on-disk created skill passes the deterministic rubric. + const content = readFileSync(join(skillsDir, NAME, 'SKILL.md'), 'utf8'); + const { frontmatter, body } = splitFrontmatter(content); + const description = extractDescription(frontmatter); + const perFolder = { core: 2, patterns: 1 }; + assert.equal(checkK2(description).pass, true); + assert.equal(checkK3(body).pass, true); + const { K5, K6 } = checkK5K6(body, 3); + assert.equal(K5.pass, true); + assert.equal(K6.pass, true); + assert.equal(checkRefConsistency(body, perFolder).consistent, true); + }); +}); + +test('applyApprovedAction create — preview (apply:false) writes nothing and leaves the ledger at approved', () => { + withSkillsFixture(({ root, skillsDir, dataDir }) => { + const approved = approveCreate(NAME, TEST_DOMAIN_SPEC, { skillsDir, dataDir }); + const res = applyApprovedAction(approved, { + pluginRoot: root, skillsDir, dataDir, taxonomyDataDir: dataDir, + createSpec: { description: TEST_DOMAIN_SPEC.description, categories: TEST_DOMAIN_SPEC.categories }, + apply: false, + }); + assert.equal(res.applied, false); + assert.equal(res.preview, true); + assert.equal(res.revalidate.ok, true, 'name still free -> revalidation would pass'); + assert.equal(existsSync(join(skillsDir, NAME)), false, 'preview creates nothing'); + assert.equal(loadDecisions(dataDir).actions[actionKey(approved)].status, 'approved'); + }); +}); + +test('applyApprovedAction create — DRIFT (name taken after approval) aborts with ZERO mutation', () => { + withSkillsFixture(({ root, skillsDir, dataDir }) => { + const approved = approveCreate(NAME, TEST_DOMAIN_SPEC, { skillsDir, dataDir }); + // Disk drifts: a skill with the target name now exists -> fresh guardrail + // nameAvailable=false -> guardrail differs from the approved snapshot. + mkdirSync(join(skillsDir, NAME), { recursive: true }); + writeFileSync(join(skillsDir, NAME, 'SKILL.md'), '---\nname: squatter\ndescription: x\n---\n# squatter\n'); + + const res = applyApprovedAction(approved, { + pluginRoot: root, skillsDir, dataDir, taxonomyDataDir: dataDir, + createSpec: { description: TEST_DOMAIN_SPEC.description, categories: TEST_DOMAIN_SPEC.categories }, + apply: true, + }); + assert.equal(res.applied, false, 'a name collision after approval must abort'); + assert.equal(res.revalidate.ok, false); + // The pre-existing (squatter) SKILL.md is untouched; no references/ created. + assert.equal(existsSync(join(skillsDir, NAME, 'references')), false, 'no scaffold written on abort'); + assert.equal(loadDecisions(dataDir).actions[actionKey(approved)].status, 'approved', 'ledger left at approved'); + }); +}); + +test('applyApprovedAction create — refuses a non-approved (pending) entry with zero mutation', () => { + withSkillsFixture(({ root, skillsDir, dataDir }) => { + const { entry } = runCreatePlan(NAME, { skillsDir, dataDir, write: false, ...TEST_DOMAIN_SPEC }); + saveDecisions(recordAction(loadDecisions(dataDir), entry), dataDir); // pending, not approved + + const res = applyApprovedAction(entry, { + pluginRoot: root, skillsDir, dataDir, taxonomyDataDir: dataDir, + createSpec: { description: TEST_DOMAIN_SPEC.description, categories: TEST_DOMAIN_SPEC.categories }, + apply: true, + }); + assert.equal(res.applied, false); + assert.match(res.revalidate.reason, /approved/i); + assert.equal(existsSync(join(skillsDir, NAME)), false, 'no mutation'); + }); +}); + +// =========================================================================== +// (D) Taxonomy helper — applyCategoryAdditions (additive, never clobbers) +// =========================================================================== + +test('applyCategoryAdditions — adds new category ownership, no-ops on an already-owned category', () => { + const tax = { category_skill: { existing: 'owner-skill' } }; + const next = applyCategoryAdditions(tax, [ + { category: 'brand-new', skill: 'new-skill' }, + { category: 'existing', skill: 'new-skill' }, // must NOT clobber owner-skill + ]); + assert.equal(next.category_skill['brand-new'], 'new-skill', 'new category added'); + assert.equal(next.category_skill.existing, 'owner-skill', 'existing owner never clobbered'); + // pure: original untouched + assert.equal(tax.category_skill['brand-new'], undefined, 'input not mutated'); +}); + +test('applyCategoryAdditions — disjoint from applyCategoryReassignments (additions vs repoint stay separate ops)', () => { + const tax = { category_skill: { a: 's1' } }; + const added = applyCategoryAdditions(tax, [{ category: 'b', skill: 's2' }]); + const repointed = applyCategoryReassignments(added, [{ category: 'a', from: 's1', to: 's2' }]); + assert.deepEqual(repointed.category_skill, { a: 's2', b: 's2' }); +});