ms-ai-architect/tests/kb-eval/test-k10-sibling-overlap.test.mjs

// tests/kb-eval/test-k10-sibling-overlap.test.mjs
// Unit tests for K10 — sibling-scope-non-overlap (Sesjon 15 / B2).
// Pure core: perSkillSiblingOverlap (lib/sibling-overlap.mjs) + the eval.mjs
// integration attachSiblingOverlap. Deterministic — no LLM judge.

import { test } from 'node:test';
import assert from 'node:assert/strict';
import { readFileSync } from 'node:fs';
import { dirname, join } from 'node:path';
import { fileURLToPath } from 'node:url';

import {
  perSkillSiblingOverlap,
  computeOverlapFromInputs,
  K10_OVERLAP_THRESHOLD,
} from '../../scripts/kb-eval/lib/sibling-overlap.mjs';
import { attachSiblingOverlap, splitFrontmatter, extractDescription } from '../../scripts/kb-eval/eval.mjs';

const __dirname = dirname(fileURLToPath(import.meta.url));
const PROMPTS_PATH = join(__dirname, '..', '..', 'scripts', 'kb-eval', 'data', 'k1-trigger-prompts.json');
const promptSet = JSON.parse(readFileSync(PROMPTS_PATH, 'utf8'));

// ---------------------------------------------------------------------------
// perSkillSiblingOverlap — pure mapping pairs -> per-skill worst sibling
// ---------------------------------------------------------------------------

test('perSkillSiblingOverlap — each skill takes its worst (max combined) pair', () => {
  const pairs = [
    { pair: ['a', 'b'], combined: 7.5 },
    { pair: ['a', 'c'], combined: 3.0 },
    { pair: ['b', 'c'], combined: 6.0 },
  ];
  const r = perSkillSiblingOverlap(pairs, { threshold: 7.0 });
  assert.equal(r.a.maxCombined, 7.5);
  assert.equal(r.a.worstSibling, 'b');
  assert.equal(r.b.maxCombined, 7.5);
  assert.equal(r.b.worstSibling, 'a');
  assert.equal(r.c.maxCombined, 6.0);
  assert.equal(r.c.worstSibling, 'b');
});

test('perSkillSiblingOverlap — pass iff maxCombined strictly below threshold', () => {
  const pairs = [
    { pair: ['a', 'b'], combined: 7.5 },
    { pair: ['a', 'c'], combined: 3.0 },
    { pair: ['b', 'c'], combined: 6.0 },
  ];
  const r = perSkillSiblingOverlap(pairs, { threshold: 7.0 });
  assert.equal(r.a.pass, false); // 7.5 >= 7.0
  assert.equal(r.b.pass, false); // 7.5 >= 7.0
  assert.equal(r.c.pass, true); //  6.0 <  7.0
});

test('perSkillSiblingOverlap — combined exactly at threshold FAILS (strict <)', () => {
  const r = perSkillSiblingOverlap([{ pair: ['x', 'y'], combined: 7.0 }], { threshold: 7.0 });
  assert.equal(r.x.pass, false);
  assert.equal(r.y.pass, false);
});

test('perSkillSiblingOverlap — threshold is configurable; records it on each verdict', () => {
  const pairs = [{ pair: ['a', 'b'], combined: 7.5 }];
  const lax = perSkillSiblingOverlap(pairs, { threshold: 8.0 });
  assert.equal(lax.a.pass, true); // 7.5 < 8.0
  assert.equal(lax.a.threshold, 8.0);
});

test('perSkillSiblingOverlap — default threshold is the rubric K10 constant (7.0)', () => {
  assert.equal(K10_OVERLAP_THRESHOLD, 7.0);
  const r = perSkillSiblingOverlap([{ pair: ['a', 'b'], combined: 7.5 }]);
  assert.equal(r.a.threshold, 7.0);
  assert.equal(r.a.pass, false);
});

test('perSkillSiblingOverlap — empty pairs yields empty verdict map', () => {
  assert.deepEqual(perSkillSiblingOverlap([]), {});
});

// ---------------------------------------------------------------------------
// Real-data path — eng + infra FAIL at 7.0; the other three PASS
// ---------------------------------------------------------------------------

test('K10 on the real five — ALL pass (eng↔infra boundary sharpened S39, corpus invariant)', () => {
  // Build descriptions straight from the curated belongs_to graph + lexical
  // surfaces via the same core the detector uses, so this asserts on live tokens.
  //
  // S39 (steg A): the eng↔infra pair used to fail K10 at combined=7.4167 (the
  // Azure-deployment build↔operate boundary). The infra description was sharpened
  // — "multi-region"→"cross-region", "edge AI architecture"→"edge AI deployment" —
  // removing the shared distinctive tokens `multi` (0.5) and `architecture` (0.333),
  // dropping the pair under threshold. This test now pins the corpus invariant:
  // every skill passes K10 (a precondition for the ≥90 % score on all five).
  const skills = ['ms-ai-advisor', 'ms-ai-engineering', 'ms-ai-governance', 'ms-ai-infrastructure', 'ms-ai-security'];
  const descriptionsBySkill = {};
  for (const s of skills) {
    const md = join(__dirname, '..', '..', 'skills', s, 'SKILL.md');
    descriptionsBySkill[s] = extractDescription(splitFrontmatter(readFileSync(md, 'utf8')).frontmatter);
  }
  const overlap = computeOverlapFromInputs(descriptionsBySkill, promptSet);
  const k10 = perSkillSiblingOverlap(overlap.pairs, { threshold: 7.0 });
  for (const s of skills) {
    assert.equal(k10[s].pass, true, `${s} must pass K10 (maxCombined ${k10[s].maxCombined} < 7.0)`);
    assert.ok(k10[s].maxCombined < 7.0, `${s} maxCombined ${k10[s].maxCombined} must be < 7.0`);
  }
  // The formerly-failing pair is now safely under threshold (was 7.4167).
  const engInfra = overlap.pairs.find(
    (p) => p.key === 'ms-ai-engineering|ms-ai-infrastructure',
  );
  assert.ok(engInfra.combined < 7.0, `eng↔infra combined ${engInfra.combined} must be < 7.0`);
});

// ---------------------------------------------------------------------------
// attachSiblingOverlap — eval.mjs integration (injects K10 into deterministic)
// ---------------------------------------------------------------------------

test('attachSiblingOverlap — injects K10_siblingScopeOverlap into each skill deterministic block', () => {
  const skills = [
    { name: 'a', deterministic: {}, judgeInputs: { description: 'Build RAG pipelines. Triggers on: "rag", "pipeline".' } },
    { name: 'b', deterministic: {}, judgeInputs: { description: 'Operate clusters. Triggers on: "operate", "cluster".' } },
  ];
  const out = attachSiblingOverlap(skills, promptSet, { threshold: 7.0 });
  assert.ok(out[0].deterministic.K10_siblingScopeOverlap, 'K10 present on skill a');
  assert.ok(out[1].deterministic.K10_siblingScopeOverlap, 'K10 present on skill b');
  assert.equal(typeof out[0].deterministic.K10_siblingScopeOverlap.maxCombined, 'number');
  assert.equal(typeof out[0].deterministic.K10_siblingScopeOverlap.pass, 'boolean');
  assert.equal(out[0].deterministic.K10_siblingScopeOverlap.threshold, 7.0);
});

test('attachSiblingOverlap — uses K10 default threshold when none given', () => {
  const skills = [
    { name: 'a', deterministic: {}, judgeInputs: { description: 'x "p" "q" "r".' } },
    { name: 'b', deterministic: {}, judgeInputs: { description: 'y "s" "t" "u".' } },
  ];
  const out = attachSiblingOverlap(skills, promptSet);
  assert.equal(out[0].deterministic.K10_siblingScopeOverlap.threshold, K10_OVERLAP_THRESHOLD);
});