// test-skill-lifecycle-detect.test.mjs — Spor B / B1 overlap-detektor. // TDD: written before scripts/kb-eval/detect-skill-lifecycle.mjs exists. // // The overlap detector is DETERMINISTIC and combines two signals: // (1) operator-curated boundary-tension graph (k1-trigger-prompts.json belongs_to) // (2) df-weighted lexical trigger-surface overlap (down-weights domain-common tokens) // It must NEVER write to skills/ — it only produces a report. import { test } from 'node:test'; import assert from 'node:assert/strict'; import { readFileSync } from 'node:fs'; import { dirname, join } from 'node:path'; import { fileURLToPath } from 'node:url'; import { tokenize, extractTriggerSurface, buildDocumentFrequency, lexicalOverlap, pairKey, boundaryTensionMatrix, computeOverlapFromInputs, } from '../../scripts/kb-eval/detect-skill-lifecycle.mjs'; const __dirname = dirname(fileURLToPath(import.meta.url)); const PROMPTS_PATH = join(__dirname, '..', '..', 'scripts', 'kb-eval', 'data', 'k1-trigger-prompts.json'); const promptSet = JSON.parse(readFileSync(PROMPTS_PATH, 'utf8')); // --------------------------------------------------------------------------- // tokenize // --------------------------------------------------------------------------- test('tokenize: lowercases, strips punctuation, drops short + stopwords', () => { const t = tokenize('Azure AI Services for the RAG-pipeline, and BCDR. Triggers on: x.'); assert.ok(t.includes('azure'), 'keeps azure'); assert.ok(t.includes('services'), 'keeps services'); assert.ok(t.includes('rag'), 'splits hyphen -> rag'); assert.ok(t.includes('pipeline'), 'splits hyphen -> pipeline'); assert.ok(t.includes('bcdr'), 'keeps bcdr'); assert.ok(!t.includes('ai'), 'drops len<3 token ai'); assert.ok(!t.includes('for'), 'drops stopword for'); assert.ok(!t.includes('the'), 'drops stopword the'); assert.ok(!t.includes('and'), 'drops stopword and'); assert.ok(!t.includes('triggers'), 'drops description-format word triggers'); // all lowercase assert.ok(t.every((w) => w === w.toLowerCase())); }); // --------------------------------------------------------------------------- // extractTriggerSurface // --------------------------------------------------------------------------- test('extractTriggerSurface: pulls quoted phrases + content tokens', () => { const desc = 'Deep guidance for building AI. Triggers on: "RAG architecture on Azure", "Azure AI Search".'; const s = extractTriggerSurface(desc); assert.ok(Array.isArray(s.phrases)); assert.equal(s.phrases.length, 2, 'two quoted phrases'); assert.ok(s.phrases.includes('RAG architecture on Azure')); assert.ok(s.tokens instanceof Set); assert.ok(s.tokens.has('azure')); assert.ok(s.tokens.has('architecture')); }); // --------------------------------------------------------------------------- // buildDocumentFrequency // --------------------------------------------------------------------------- test('buildDocumentFrequency: counts skills containing each token', () => { const surfaces = [ { tokens: new Set(['azure', 'rag']) }, { tokens: new Set(['azure', 'bcdr']) }, { tokens: new Set(['azure', 'dpia']) }, ]; const df = buildDocumentFrequency(surfaces); assert.equal(df.get('azure'), 3); assert.equal(df.get('rag'), 1); assert.equal(df.get('bcdr'), 1); }); // --------------------------------------------------------------------------- // lexicalOverlap — df-weighting down-weights common tokens // --------------------------------------------------------------------------- test('lexicalOverlap: shared tokens, jaccard, df-weighted score', () => { const sA = { tokens: new Set(['azure', 'deployment', 'rag']) }; const sB = { tokens: new Set(['azure', 'deployment', 'bcdr']) }; const df = new Map([ ['azure', 5], ['deployment', 2], ['rag', 1], ['bcdr', 1], ]); const o = lexicalOverlap(sA, sB, df); assert.deepEqual(o.shared, ['azure', 'deployment'], 'shared sorted'); assert.equal(o.jaccard, 0.5, '2 shared / 4 union'); // weighted = 1/5 (azure) + 1/2 (deployment) = 0.7 assert.ok(Math.abs(o.weightedScore - 0.7) < 1e-9, `weightedScore=${o.weightedScore}`); }); test('lexicalOverlap: a high-df token contributes less than a distinctive one', () => { const df = new Map([['common', 5], ['rare', 2]]); const a = { tokens: new Set(['common', 'rare']) }; const onlyCommon = lexicalOverlap(a, { tokens: new Set(['common']) }, df); const onlyRare = lexicalOverlap(a, { tokens: new Set(['rare']) }, df); assert.ok(onlyRare.weightedScore > onlyCommon.weightedScore, 'rare token weighs more'); }); // --------------------------------------------------------------------------- // pairKey — order-independent, stable // --------------------------------------------------------------------------- test('pairKey: order-independent', () => { assert.equal(pairKey('b', 'a'), pairKey('a', 'b')); assert.equal(pairKey('a', 'b'), 'a|b'); }); // --------------------------------------------------------------------------- // boundaryTensionMatrix — differential check vs independent reducer // --------------------------------------------------------------------------- test('boundaryTensionMatrix: symmetric, matches independent count, ignores non-skill belongs_to', () => { const skills = Object.keys(promptSet).filter((k) => k !== '_meta'); const m = boundaryTensionMatrix(promptSet); // independent reimplementation of symmetric tension const skillSet = new Set(skills); const expected = {}; for (const s of skills) { for (const e of promptSet[s].out_of_domain || []) { const b = e.belongs_to; if (!skillSet.has(b)) continue; // controls / out-of-stack are not skills const k = pairKey(s, b); expected[k] = (expected[k] || 0) + 1; } } for (const [k, v] of Object.entries(expected)) { assert.equal(m[k], v, `tension ${k}`); } // anchored known value tied to current curated set (S11) assert.equal(m[pairKey('ms-ai-engineering', 'ms-ai-infrastructure')], 6, 'eng<->infra Azure-deployment boundary = 6'); }); // --------------------------------------------------------------------------- // computeOverlapFromInputs — full report section // --------------------------------------------------------------------------- function realDescriptions() { // Minimal stand-ins are not enough; use the real five descriptions via the // same extractor eval.mjs uses, so the test exercises real data. const root = join(__dirname, '..', '..'); const names = [ 'ms-ai-advisor', 'ms-ai-engineering', 'ms-ai-governance', 'ms-ai-infrastructure', 'ms-ai-security', ]; // lazy import to avoid top-level coupling return import('../../scripts/kb-eval/eval.mjs').then((m) => { const out = {}; for (const n of names) { const c = readFileSync(join(root, 'skills', n, 'SKILL.md'), 'utf8'); out[n] = m.extractDescription(m.splitFrontmatter(c).frontmatter); } return out; }); } test('computeOverlapFromInputs: 10 pairs, eng<->infra focusPair, sorted, deterministic', async () => { const descs = await realDescriptions(); const r1 = computeOverlapFromInputs(descs, promptSet); const r2 = computeOverlapFromInputs(descs, promptSet); // 5 skills -> C(5,2) = 10 unordered pairs assert.equal(r1.pairs.length, 10); // each pair carries both signals for (const p of r1.pairs) { assert.ok(Array.isArray(p.pair) && p.pair.length === 2); assert.equal(typeof p.boundaryTension, 'number'); assert.ok(p.lexical && Array.isArray(p.lexical.shared)); assert.equal(typeof p.combined, 'number'); } // sorted by combined descending for (let i = 1; i < r1.pairs.length; i++) { assert.ok(r1.pairs[i - 1].combined >= r1.pairs[i].combined, 'pairs sorted desc by combined'); } // focusPair = eng<->infra (operator-designated Azure-deployment boundary) assert.ok(r1.focusPair, 'focusPair present'); assert.deepEqual( [...r1.focusPair.pair].sort(), ['ms-ai-engineering', 'ms-ai-infrastructure'], ); assert.equal(r1.focusPair.boundaryTension, 6); // determinism: identical output across runs assert.deepEqual(r1, r2); });