// tests/kb-eval/test-k10-sibling-overlap.test.mjs // Unit tests for K10 — sibling-scope-non-overlap (Sesjon 15 / B2). // Pure core: perSkillSiblingOverlap (lib/sibling-overlap.mjs) + the eval.mjs // integration attachSiblingOverlap. Deterministic — no LLM judge. import { test } from 'node:test'; import assert from 'node:assert/strict'; import { readFileSync } from 'node:fs'; import { dirname, join } from 'node:path'; import { fileURLToPath } from 'node:url'; import { perSkillSiblingOverlap, computeOverlapFromInputs, K10_OVERLAP_THRESHOLD, } from '../../scripts/kb-eval/lib/sibling-overlap.mjs'; import { attachSiblingOverlap, splitFrontmatter, extractDescription } from '../../scripts/kb-eval/eval.mjs'; const __dirname = dirname(fileURLToPath(import.meta.url)); const PROMPTS_PATH = join(__dirname, '..', '..', 'scripts', 'kb-eval', 'data', 'k1-trigger-prompts.json'); const promptSet = JSON.parse(readFileSync(PROMPTS_PATH, 'utf8')); // --------------------------------------------------------------------------- // perSkillSiblingOverlap — pure mapping pairs -> per-skill worst sibling // --------------------------------------------------------------------------- test('perSkillSiblingOverlap — each skill takes its worst (max combined) pair', () => { const pairs = [ { pair: ['a', 'b'], combined: 7.5 }, { pair: ['a', 'c'], combined: 3.0 }, { pair: ['b', 'c'], combined: 6.0 }, ]; const r = perSkillSiblingOverlap(pairs, { threshold: 7.0 }); assert.equal(r.a.maxCombined, 7.5); assert.equal(r.a.worstSibling, 'b'); assert.equal(r.b.maxCombined, 7.5); assert.equal(r.b.worstSibling, 'a'); assert.equal(r.c.maxCombined, 6.0); assert.equal(r.c.worstSibling, 'b'); }); test('perSkillSiblingOverlap — pass iff maxCombined strictly below threshold', () => { const pairs = [ { pair: ['a', 'b'], combined: 7.5 }, { pair: ['a', 'c'], combined: 3.0 }, { pair: ['b', 'c'], combined: 6.0 }, ]; const r = perSkillSiblingOverlap(pairs, { threshold: 7.0 }); assert.equal(r.a.pass, false); // 7.5 >= 7.0 assert.equal(r.b.pass, false); // 7.5 >= 7.0 assert.equal(r.c.pass, true); // 6.0 < 7.0 }); test('perSkillSiblingOverlap — combined exactly at threshold FAILS (strict <)', () => { const r = perSkillSiblingOverlap([{ pair: ['x', 'y'], combined: 7.0 }], { threshold: 7.0 }); assert.equal(r.x.pass, false); assert.equal(r.y.pass, false); }); test('perSkillSiblingOverlap — threshold is configurable; records it on each verdict', () => { const pairs = [{ pair: ['a', 'b'], combined: 7.5 }]; const lax = perSkillSiblingOverlap(pairs, { threshold: 8.0 }); assert.equal(lax.a.pass, true); // 7.5 < 8.0 assert.equal(lax.a.threshold, 8.0); }); test('perSkillSiblingOverlap — default threshold is the rubric K10 constant (7.0)', () => { assert.equal(K10_OVERLAP_THRESHOLD, 7.0); const r = perSkillSiblingOverlap([{ pair: ['a', 'b'], combined: 7.5 }]); assert.equal(r.a.threshold, 7.0); assert.equal(r.a.pass, false); }); test('perSkillSiblingOverlap — empty pairs yields empty verdict map', () => { assert.deepEqual(perSkillSiblingOverlap([]), {}); }); // --------------------------------------------------------------------------- // Real-data path — eng + infra FAIL at 7.0; the other three PASS // --------------------------------------------------------------------------- test('K10 on the real five — ALL pass (eng↔infra boundary sharpened S39, corpus invariant)', () => { // Build descriptions straight from the curated belongs_to graph + lexical // surfaces via the same core the detector uses, so this asserts on live tokens. // // S39 (steg A): the eng↔infra pair used to fail K10 at combined=7.4167 (the // Azure-deployment build↔operate boundary). The infra description was sharpened // — "multi-region"→"cross-region", "edge AI architecture"→"edge AI deployment" — // removing the shared distinctive tokens `multi` (0.5) and `architecture` (0.333), // dropping the pair under threshold. This test now pins the corpus invariant: // every skill passes K10 (a precondition for the ≥90 % score on all five). const skills = ['ms-ai-advisor', 'ms-ai-engineering', 'ms-ai-governance', 'ms-ai-infrastructure', 'ms-ai-security']; const descriptionsBySkill = {}; for (const s of skills) { const md = join(__dirname, '..', '..', 'skills', s, 'SKILL.md'); descriptionsBySkill[s] = extractDescription(splitFrontmatter(readFileSync(md, 'utf8')).frontmatter); } const overlap = computeOverlapFromInputs(descriptionsBySkill, promptSet); const k10 = perSkillSiblingOverlap(overlap.pairs, { threshold: 7.0 }); for (const s of skills) { assert.equal(k10[s].pass, true, `${s} must pass K10 (maxCombined ${k10[s].maxCombined} < 7.0)`); assert.ok(k10[s].maxCombined < 7.0, `${s} maxCombined ${k10[s].maxCombined} must be < 7.0`); } // The formerly-failing pair is now safely under threshold (was 7.4167). const engInfra = overlap.pairs.find( (p) => p.key === 'ms-ai-engineering|ms-ai-infrastructure', ); assert.ok(engInfra.combined < 7.0, `eng↔infra combined ${engInfra.combined} must be < 7.0`); }); // --------------------------------------------------------------------------- // attachSiblingOverlap — eval.mjs integration (injects K10 into deterministic) // --------------------------------------------------------------------------- test('attachSiblingOverlap — injects K10_siblingScopeOverlap into each skill deterministic block', () => { const skills = [ { name: 'a', deterministic: {}, judgeInputs: { description: 'Build RAG pipelines. Triggers on: "rag", "pipeline".' } }, { name: 'b', deterministic: {}, judgeInputs: { description: 'Operate clusters. Triggers on: "operate", "cluster".' } }, ]; const out = attachSiblingOverlap(skills, promptSet, { threshold: 7.0 }); assert.ok(out[0].deterministic.K10_siblingScopeOverlap, 'K10 present on skill a'); assert.ok(out[1].deterministic.K10_siblingScopeOverlap, 'K10 present on skill b'); assert.equal(typeof out[0].deterministic.K10_siblingScopeOverlap.maxCombined, 'number'); assert.equal(typeof out[0].deterministic.K10_siblingScopeOverlap.pass, 'boolean'); assert.equal(out[0].deterministic.K10_siblingScopeOverlap.threshold, 7.0); }); test('attachSiblingOverlap — uses K10 default threshold when none given', () => { const skills = [ { name: 'a', deterministic: {}, judgeInputs: { description: 'x "p" "q" "r".' } }, { name: 'b', deterministic: {}, judgeInputs: { description: 'y "s" "t" "u".' } }, ]; const out = attachSiblingOverlap(skills, promptSet); assert.equal(out[0].deterministic.K10_siblingScopeOverlap.threshold, K10_OVERLAP_THRESHOLD); });