diff --git a/.gitignore b/.gitignore index c2c2dd1..de400bf 100644 --- a/.gitignore +++ b/.gitignore @@ -27,6 +27,9 @@ org/ scripts/kb-update/data/* !scripts/kb-update/data/domain-taxonomy.json !scripts/kb-update/data/decisions.json +# Generated skill-lifecycle detection report (Spor B / B1) — regenerated on demand, +# like the kb-update reports above. The detector script + curated inputs are tracked. +scripts/kb-eval/data/skill-lifecycle-report.json .kb-backup/ .rollback-in-progress diff --git a/scripts/kb-eval/detect-skill-lifecycle.mjs b/scripts/kb-eval/detect-skill-lifecycle.mjs new file mode 100644 index 0000000..cb049c6 --- /dev/null +++ b/scripts/kb-eval/detect-skill-lifecycle.mjs @@ -0,0 +1,212 @@ +#!/usr/bin/env node +// detect-skill-lifecycle.mjs — Spor B / lag-1-analog at SKILL granularity. +// +// PRODUCES ONLY REPORTS — never writes to skills/. Mirrors the lag-1 invariant: +// detection surfaces candidates; any skill-lifecycle op (merge/sanitize/retire/ +// create) goes through decisions.json + operator-gate (later phases B3). +// +// Sesjon 13 (B1, first detector): OVERLAP. Two deterministic signals, combined: +// (1) boundary-tension graph — operator-curated k1-trigger-prompts.json: +// each out_of_domain entry is a sibling prompt tagged belongs_to=, +// i.e. a hand-labelled "confusable neighbour". Symmetric counts = how much +// two skills sit on each other's trigger boundary. +// (2) df-weighted lexical trigger-surface overlap — shared content tokens +// between two descriptions, each weighted 1/df so domain-common vocabulary +// ("azure") contributes little and distinctive shared tokens contribute more. +// +// The eng<->infra pair (Azure-deployment boundary) is surfaced as focusPair — +// the operator-designated first target for B1. +// +// Usage: +// node scripts/kb-eval/detect-skill-lifecycle.mjs # human summary +// node scripts/kb-eval/detect-skill-lifecycle.mjs --json # machine output +// node scripts/kb-eval/detect-skill-lifecycle.mjs --write # persist report JSON +// +// Zero dependencies. Reuses eval.mjs extractors + kb-update atomic-write. + +import { readFileSync, readdirSync, existsSync, mkdirSync } from 'node:fs'; +import { join, dirname } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { splitFrontmatter, extractDescription } from './eval.mjs'; +import { atomicWriteJson } from '../kb-update/lib/atomic-write.mjs'; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const PLUGIN_ROOT = join(__dirname, '..', '..'); +const SKILLS_DIR = join(PLUGIN_ROOT, 'skills'); +const DATA_DIR = join(__dirname, 'data'); +const PROMPTS_FILE = join(DATA_DIR, 'k1-trigger-prompts.json'); +const OUT_FILE = join(DATA_DIR, 'skill-lifecycle-report.json'); + +// Operator-designated B1 focus boundary. +const FOCUS_PAIR = ['ms-ai-engineering', 'ms-ai-infrastructure']; + +// Function words (no + en) of length >= 3. Tokens < 3 chars are dropped anyway, +// so this list only needs the longer connectives. Domain nouns are NOT here — +// df-weighting handles common domain vocabulary instead. +const STOPWORDS = new Set([ + 'the', 'and', 'for', 'with', 'between', 'before', 'not', 'are', 'that', 'this', + 'into', 'over', 'per', 'use', 'used', 'when', 'which', 'how', 'via', 'from', + 'eller', 'som', 'til', 'med', 'mot', 'ved', 'for', 'har', 'kan', 'ikke', 'der', + 'det', 'den', 'ein', 'eit', 'sin', + // description-format boilerplate (every skill ends with "Triggers on:") + 'triggers', 'trigger', +]); + +/** Lowercase, split on non-alphanumeric, drop stopwords + tokens < 3 chars. */ +export function tokenize(text) { + return (text.toLowerCase().match(/[a-z0-9æøå]+/g) || []) + .filter((w) => w.length >= 3 && !STOPWORDS.has(w)); +} + +/** Trigger surface of a description: quoted phrases + content-token set. */ +export function extractTriggerSurface(description) { + const phrases = (description.match(/"([^"]+)"/g) || []).map((p) => p.slice(1, -1)); + return { phrases, tokens: new Set(tokenize(description)) }; +} + +/** token -> number of skill-surfaces that contain it. */ +export function buildDocumentFrequency(surfaces) { + const df = new Map(); + for (const s of surfaces) { + for (const t of s.tokens) df.set(t, (df.get(t) || 0) + 1); + } + return df; +} + +/** Order-independent pair key. */ +export function pairKey(a, b) { + return [a, b].sort().join('|'); +} + +/** Lexical overlap between two surfaces, df-weighted. */ +export function lexicalOverlap(surfaceA, surfaceB, df) { + const shared = []; + for (const t of surfaceA.tokens) if (surfaceB.tokens.has(t)) shared.push(t); + shared.sort(); + const union = new Set([...surfaceA.tokens, ...surfaceB.tokens]).size; + const jaccard = union > 0 ? shared.length / union : 0; + let weightedScore = 0; + for (const t of shared) weightedScore += 1 / (df.get(t) || 1); + return { + shared, + jaccard: Number(jaccard.toFixed(4)), + weightedScore: Number(weightedScore.toFixed(4)), + }; +} + +/** + * Symmetric boundary-tension matrix from the curated prompt set. + * Counts out_of_domain entries whose belongs_to is one of the real skills + * (controls / out-of-stack entries are ignored). Keyed by pairKey. + */ +export function boundaryTensionMatrix(promptSet) { + const skills = Object.keys(promptSet).filter((k) => k !== '_meta'); + const skillSet = new Set(skills); + const m = {}; + for (const s of skills) { + for (const e of promptSet[s].out_of_domain || []) { + const b = e && e.belongs_to; + if (!skillSet.has(b) || b === s) continue; + const k = pairKey(s, b); + m[k] = (m[k] || 0) + 1; + } + } + return m; +} + +/** + * Pure core: given { skill -> description } and the curated prompt set, compute + * the overlap report section. combined = boundaryTension + weightedScore + * (operator-grounded primary signal + distinctive-lexical corroboration). + */ +export function computeOverlapFromInputs(descriptionsBySkill, promptSet) { + const skills = Object.keys(descriptionsBySkill).sort(); + const surfaces = {}; + for (const s of skills) surfaces[s] = extractTriggerSurface(descriptionsBySkill[s]); + const df = buildDocumentFrequency(Object.values(surfaces)); + const tension = boundaryTensionMatrix(promptSet); + + const pairs = []; + for (let i = 0; i < skills.length; i++) { + for (let j = i + 1; j < skills.length; j++) { + const a = skills[i]; + const b = skills[j]; + const key = pairKey(a, b); + const lexical = lexicalOverlap(surfaces[a], surfaces[b], df); + const boundaryTension = tension[key] || 0; + const combined = Number((boundaryTension + lexical.weightedScore).toFixed(4)); + pairs.push({ pair: [a, b], key, boundaryTension, lexical, combined }); + } + } + // sort by combined desc, then key asc for stable ties + pairs.sort((x, y) => y.combined - x.combined || x.key.localeCompare(y.key)); + + const focusKey = pairKey(...FOCUS_PAIR); + const focusPair = pairs.find((p) => p.key === focusKey) || null; + + return { + method: + 'deterministic: (1) operator-curated boundary-tension (k1-trigger-prompts belongs_to), ' + + '(2) df-weighted lexical trigger-surface overlap. combined = boundaryTension + weightedScore.', + focusPairReason: + 'Azure-deployment boundary engineering(build) <-> infrastructure(operate) — operator-designated B1 target.', + pairs, + focusPair, + }; +} + +/** Read the five SKILL.md descriptions from disk. */ +function loadDescriptions() { + const out = {}; + for (const e of readdirSync(SKILLS_DIR, { withFileTypes: true })) { + if (!e.isDirectory()) continue; + const md = join(SKILLS_DIR, e.name, 'SKILL.md'); + if (!existsSync(md)) continue; + out[e.name] = extractDescription(splitFrontmatter(readFileSync(md, 'utf8')).frontmatter); + } + return out; +} + +function buildReport() { + const promptSet = JSON.parse(readFileSync(PROMPTS_FILE, 'utf8')); + const descriptions = loadDescriptions(); + return { + rubric: 'skill-lifecycle', + phase: 'B1', + note: 'Detection only — never writes to skills/. Candidates feed decisions.json + operator-gate (B3).', + overlap: computeOverlapFromInputs(descriptions, promptSet), + }; +} + +function main() { + const args = process.argv.slice(2); + const jsonOut = args.includes('--json'); + const doWrite = args.includes('--write'); + + const report = buildReport(); + + if (doWrite) { + mkdirSync(DATA_DIR, { recursive: true }); + atomicWriteJson(OUT_FILE, report); + } + if (jsonOut) { + process.stdout.write(JSON.stringify(report) + '\n'); + return; + } + + const o = report.overlap; + console.log(`\nSkill-livssyklus — B1 overlap-detektor (${o.pairs.length} par)\n`); + console.log('Par (sortert på combined = grensetension + df-vektet leksikalsk):\n'); + for (const p of o.pairs) { + const lex = p.lexical.shared.length ? p.lexical.shared.join(', ') : '—'; + const focus = p.key === pairKey(...FOCUS_PAIR) ? ' ◀ FOCUS (Azure-deployment)' : ''; + console.log(` ${p.combined.toFixed(2).padStart(6)} ${p.pair.join(' / ')}${focus}`); + console.log(` tension=${p.boundaryTension} lex(w=${p.lexical.weightedScore}, jac=${p.lexical.jaccard}) delt: ${lex}`); + } + console.log(`\nFocus-par: ${o.focusPair ? o.focusPair.pair.join(' <-> ') : '(ingen)'} — ${o.focusPairReason}`); + console.log('\n(Rapport skrives med --write til data/skill-lifecycle-report.json; aldri til skills/.)\n'); +} + +if (process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1]) { + main(); +} diff --git a/tests/kb-eval/test-skill-lifecycle-detect.test.mjs b/tests/kb-eval/test-skill-lifecycle-detect.test.mjs new file mode 100644 index 0000000..19b196a --- /dev/null +++ b/tests/kb-eval/test-skill-lifecycle-detect.test.mjs @@ -0,0 +1,194 @@ +// test-skill-lifecycle-detect.test.mjs — Spor B / B1 overlap-detektor. +// TDD: written before scripts/kb-eval/detect-skill-lifecycle.mjs exists. +// +// The overlap detector is DETERMINISTIC and combines two signals: +// (1) operator-curated boundary-tension graph (k1-trigger-prompts.json belongs_to) +// (2) df-weighted lexical trigger-surface overlap (down-weights domain-common tokens) +// It must NEVER write to skills/ — it only produces a report. + +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +import { + tokenize, + extractTriggerSurface, + buildDocumentFrequency, + lexicalOverlap, + pairKey, + boundaryTensionMatrix, + computeOverlapFromInputs, +} from '../../scripts/kb-eval/detect-skill-lifecycle.mjs'; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const PROMPTS_PATH = join(__dirname, '..', '..', 'scripts', 'kb-eval', 'data', 'k1-trigger-prompts.json'); +const promptSet = JSON.parse(readFileSync(PROMPTS_PATH, 'utf8')); + +// --------------------------------------------------------------------------- +// tokenize +// --------------------------------------------------------------------------- +test('tokenize: lowercases, strips punctuation, drops short + stopwords', () => { + const t = tokenize('Azure AI Services for the RAG-pipeline, and BCDR. Triggers on: x.'); + assert.ok(t.includes('azure'), 'keeps azure'); + assert.ok(t.includes('services'), 'keeps services'); + assert.ok(t.includes('rag'), 'splits hyphen -> rag'); + assert.ok(t.includes('pipeline'), 'splits hyphen -> pipeline'); + assert.ok(t.includes('bcdr'), 'keeps bcdr'); + assert.ok(!t.includes('ai'), 'drops len<3 token ai'); + assert.ok(!t.includes('for'), 'drops stopword for'); + assert.ok(!t.includes('the'), 'drops stopword the'); + assert.ok(!t.includes('and'), 'drops stopword and'); + assert.ok(!t.includes('triggers'), 'drops description-format word triggers'); + // all lowercase + assert.ok(t.every((w) => w === w.toLowerCase())); +}); + +// --------------------------------------------------------------------------- +// extractTriggerSurface +// --------------------------------------------------------------------------- +test('extractTriggerSurface: pulls quoted phrases + content tokens', () => { + const desc = + 'Deep guidance for building AI. Triggers on: "RAG architecture on Azure", "Azure AI Search".'; + const s = extractTriggerSurface(desc); + assert.ok(Array.isArray(s.phrases)); + assert.equal(s.phrases.length, 2, 'two quoted phrases'); + assert.ok(s.phrases.includes('RAG architecture on Azure')); + assert.ok(s.tokens instanceof Set); + assert.ok(s.tokens.has('azure')); + assert.ok(s.tokens.has('architecture')); +}); + +// --------------------------------------------------------------------------- +// buildDocumentFrequency +// --------------------------------------------------------------------------- +test('buildDocumentFrequency: counts skills containing each token', () => { + const surfaces = [ + { tokens: new Set(['azure', 'rag']) }, + { tokens: new Set(['azure', 'bcdr']) }, + { tokens: new Set(['azure', 'dpia']) }, + ]; + const df = buildDocumentFrequency(surfaces); + assert.equal(df.get('azure'), 3); + assert.equal(df.get('rag'), 1); + assert.equal(df.get('bcdr'), 1); +}); + +// --------------------------------------------------------------------------- +// lexicalOverlap — df-weighting down-weights common tokens +// --------------------------------------------------------------------------- +test('lexicalOverlap: shared tokens, jaccard, df-weighted score', () => { + const sA = { tokens: new Set(['azure', 'deployment', 'rag']) }; + const sB = { tokens: new Set(['azure', 'deployment', 'bcdr']) }; + const df = new Map([ + ['azure', 5], + ['deployment', 2], + ['rag', 1], + ['bcdr', 1], + ]); + const o = lexicalOverlap(sA, sB, df); + assert.deepEqual(o.shared, ['azure', 'deployment'], 'shared sorted'); + assert.equal(o.jaccard, 0.5, '2 shared / 4 union'); + // weighted = 1/5 (azure) + 1/2 (deployment) = 0.7 + assert.ok(Math.abs(o.weightedScore - 0.7) < 1e-9, `weightedScore=${o.weightedScore}`); +}); + +test('lexicalOverlap: a high-df token contributes less than a distinctive one', () => { + const df = new Map([['common', 5], ['rare', 2]]); + const a = { tokens: new Set(['common', 'rare']) }; + const onlyCommon = lexicalOverlap(a, { tokens: new Set(['common']) }, df); + const onlyRare = lexicalOverlap(a, { tokens: new Set(['rare']) }, df); + assert.ok(onlyRare.weightedScore > onlyCommon.weightedScore, 'rare token weighs more'); +}); + +// --------------------------------------------------------------------------- +// pairKey — order-independent, stable +// --------------------------------------------------------------------------- +test('pairKey: order-independent', () => { + assert.equal(pairKey('b', 'a'), pairKey('a', 'b')); + assert.equal(pairKey('a', 'b'), 'a|b'); +}); + +// --------------------------------------------------------------------------- +// boundaryTensionMatrix — differential check vs independent reducer +// --------------------------------------------------------------------------- +test('boundaryTensionMatrix: symmetric, matches independent count, ignores non-skill belongs_to', () => { + const skills = Object.keys(promptSet).filter((k) => k !== '_meta'); + const m = boundaryTensionMatrix(promptSet); + + // independent reimplementation of symmetric tension + const skillSet = new Set(skills); + const expected = {}; + for (const s of skills) { + for (const e of promptSet[s].out_of_domain || []) { + const b = e.belongs_to; + if (!skillSet.has(b)) continue; // controls / out-of-stack are not skills + const k = pairKey(s, b); + expected[k] = (expected[k] || 0) + 1; + } + } + for (const [k, v] of Object.entries(expected)) { + assert.equal(m[k], v, `tension ${k}`); + } + // anchored known value tied to current curated set (S11) + assert.equal(m[pairKey('ms-ai-engineering', 'ms-ai-infrastructure')], 6, 'eng<->infra Azure-deployment boundary = 6'); +}); + +// --------------------------------------------------------------------------- +// computeOverlapFromInputs — full report section +// --------------------------------------------------------------------------- +function realDescriptions() { + // Minimal stand-ins are not enough; use the real five descriptions via the + // same extractor eval.mjs uses, so the test exercises real data. + const root = join(__dirname, '..', '..'); + const names = [ + 'ms-ai-advisor', + 'ms-ai-engineering', + 'ms-ai-governance', + 'ms-ai-infrastructure', + 'ms-ai-security', + ]; + // lazy import to avoid top-level coupling + return import('../../scripts/kb-eval/eval.mjs').then((m) => { + const out = {}; + for (const n of names) { + const c = readFileSync(join(root, 'skills', n, 'SKILL.md'), 'utf8'); + out[n] = m.extractDescription(m.splitFrontmatter(c).frontmatter); + } + return out; + }); +} + +test('computeOverlapFromInputs: 10 pairs, eng<->infra focusPair, sorted, deterministic', async () => { + const descs = await realDescriptions(); + const r1 = computeOverlapFromInputs(descs, promptSet); + const r2 = computeOverlapFromInputs(descs, promptSet); + + // 5 skills -> C(5,2) = 10 unordered pairs + assert.equal(r1.pairs.length, 10); + + // each pair carries both signals + for (const p of r1.pairs) { + assert.ok(Array.isArray(p.pair) && p.pair.length === 2); + assert.equal(typeof p.boundaryTension, 'number'); + assert.ok(p.lexical && Array.isArray(p.lexical.shared)); + assert.equal(typeof p.combined, 'number'); + } + + // sorted by combined descending + for (let i = 1; i < r1.pairs.length; i++) { + assert.ok(r1.pairs[i - 1].combined >= r1.pairs[i].combined, 'pairs sorted desc by combined'); + } + + // focusPair = eng<->infra (operator-designated Azure-deployment boundary) + assert.ok(r1.focusPair, 'focusPair present'); + assert.deepEqual( + [...r1.focusPair.pair].sort(), + ['ms-ai-engineering', 'ms-ai-infrastructure'], + ); + assert.equal(r1.focusPair.boundaryTension, 6); + + // determinism: identical output across runs + assert.deepEqual(r1, r2); +});