ms-ai-architect/tests/kb-eval/test-skill-lifecycle-detect.test.mjs

// test-skill-lifecycle-detect.test.mjs — Spor B / B1 overlap-detektor.
// TDD: written before scripts/kb-eval/detect-skill-lifecycle.mjs exists.
//
// The overlap detector is DETERMINISTIC and combines two signals:
//   (1) operator-curated boundary-tension graph (k1-trigger-prompts.json belongs_to)
//   (2) df-weighted lexical trigger-surface overlap (down-weights domain-common tokens)
// It must NEVER write to skills/ — it only produces a report.

import { test } from 'node:test';
import assert from 'node:assert/strict';
import { readFileSync } from 'node:fs';
import { dirname, join } from 'node:path';
import { fileURLToPath } from 'node:url';

import {
  tokenize,
  extractTriggerSurface,
  buildDocumentFrequency,
  lexicalOverlap,
  pairKey,
  boundaryTensionMatrix,
  computeOverlapFromInputs,
} from '../../scripts/kb-eval/detect-skill-lifecycle.mjs';

const __dirname = dirname(fileURLToPath(import.meta.url));
const PROMPTS_PATH = join(__dirname, '..', '..', 'scripts', 'kb-eval', 'data', 'k1-trigger-prompts.json');
const promptSet = JSON.parse(readFileSync(PROMPTS_PATH, 'utf8'));

// ---------------------------------------------------------------------------
// tokenize
// ---------------------------------------------------------------------------
test('tokenize: lowercases, strips punctuation, drops short + stopwords', () => {
  const t = tokenize('Azure AI Services for the RAG-pipeline, and BCDR. Triggers on: x.');
  assert.ok(t.includes('azure'), 'keeps azure');
  assert.ok(t.includes('services'), 'keeps services');
  assert.ok(t.includes('rag'), 'splits hyphen -> rag');
  assert.ok(t.includes('pipeline'), 'splits hyphen -> pipeline');
  assert.ok(t.includes('bcdr'), 'keeps bcdr');
  assert.ok(!t.includes('ai'), 'drops len<3 token ai');
  assert.ok(!t.includes('for'), 'drops stopword for');
  assert.ok(!t.includes('the'), 'drops stopword the');
  assert.ok(!t.includes('and'), 'drops stopword and');
  assert.ok(!t.includes('triggers'), 'drops description-format word triggers');
  // all lowercase
  assert.ok(t.every((w) => w === w.toLowerCase()));
});

// ---------------------------------------------------------------------------
// extractTriggerSurface
// ---------------------------------------------------------------------------
test('extractTriggerSurface: pulls quoted phrases + content tokens', () => {
  const desc =
    'Deep guidance for building AI. Triggers on: "RAG architecture on Azure", "Azure AI Search".';
  const s = extractTriggerSurface(desc);
  assert.ok(Array.isArray(s.phrases));
  assert.equal(s.phrases.length, 2, 'two quoted phrases');
  assert.ok(s.phrases.includes('RAG architecture on Azure'));
  assert.ok(s.tokens instanceof Set);
  assert.ok(s.tokens.has('azure'));
  assert.ok(s.tokens.has('architecture'));
});

// ---------------------------------------------------------------------------
// buildDocumentFrequency
// ---------------------------------------------------------------------------
test('buildDocumentFrequency: counts skills containing each token', () => {
  const surfaces = [
    { tokens: new Set(['azure', 'rag']) },
    { tokens: new Set(['azure', 'bcdr']) },
    { tokens: new Set(['azure', 'dpia']) },
  ];
  const df = buildDocumentFrequency(surfaces);
  assert.equal(df.get('azure'), 3);
  assert.equal(df.get('rag'), 1);
  assert.equal(df.get('bcdr'), 1);
});

// ---------------------------------------------------------------------------
// lexicalOverlap — df-weighting down-weights common tokens
// ---------------------------------------------------------------------------
test('lexicalOverlap: shared tokens, jaccard, df-weighted score', () => {
  const sA = { tokens: new Set(['azure', 'deployment', 'rag']) };
  const sB = { tokens: new Set(['azure', 'deployment', 'bcdr']) };
  const df = new Map([
    ['azure', 5],
    ['deployment', 2],
    ['rag', 1],
    ['bcdr', 1],
  ]);
  const o = lexicalOverlap(sA, sB, df);
  assert.deepEqual(o.shared, ['azure', 'deployment'], 'shared sorted');
  assert.equal(o.jaccard, 0.5, '2 shared / 4 union');
  // weighted = 1/5 (azure) + 1/2 (deployment) = 0.7
  assert.ok(Math.abs(o.weightedScore - 0.7) < 1e-9, `weightedScore=${o.weightedScore}`);
});

test('lexicalOverlap: a high-df token contributes less than a distinctive one', () => {
  const df = new Map([['common', 5], ['rare', 2]]);
  const a = { tokens: new Set(['common', 'rare']) };
  const onlyCommon = lexicalOverlap(a, { tokens: new Set(['common']) }, df);
  const onlyRare = lexicalOverlap(a, { tokens: new Set(['rare']) }, df);
  assert.ok(onlyRare.weightedScore > onlyCommon.weightedScore, 'rare token weighs more');
});

// ---------------------------------------------------------------------------
// pairKey — order-independent, stable
// ---------------------------------------------------------------------------
test('pairKey: order-independent', () => {
  assert.equal(pairKey('b', 'a'), pairKey('a', 'b'));
  assert.equal(pairKey('a', 'b'), 'a|b');
});

// ---------------------------------------------------------------------------
// boundaryTensionMatrix — differential check vs independent reducer
// ---------------------------------------------------------------------------
test('boundaryTensionMatrix: symmetric, matches independent count, ignores non-skill belongs_to', () => {
  const skills = Object.keys(promptSet).filter((k) => k !== '_meta');
  const m = boundaryTensionMatrix(promptSet);

  // independent reimplementation of symmetric tension
  const skillSet = new Set(skills);
  const expected = {};
  for (const s of skills) {
    for (const e of promptSet[s].out_of_domain || []) {
      const b = e.belongs_to;
      if (!skillSet.has(b)) continue; // controls / out-of-stack are not skills
      const k = pairKey(s, b);
      expected[k] = (expected[k] || 0) + 1;
    }
  }
  for (const [k, v] of Object.entries(expected)) {
    assert.equal(m[k], v, `tension ${k}`);
  }
  // anchored known value tied to current curated set (S11)
  assert.equal(m[pairKey('ms-ai-engineering', 'ms-ai-infrastructure')], 6, 'eng<->infra Azure-deployment boundary = 6');
});

// ---------------------------------------------------------------------------
// computeOverlapFromInputs — full report section
// ---------------------------------------------------------------------------
function realDescriptions() {
  // Minimal stand-ins are not enough; use the real five descriptions via the
  // same extractor eval.mjs uses, so the test exercises real data.
  const root = join(__dirname, '..', '..');
  const names = [
    'ms-ai-advisor',
    'ms-ai-engineering',
    'ms-ai-governance',
    'ms-ai-infrastructure',
    'ms-ai-security',
  ];
  // lazy import to avoid top-level coupling
  return import('../../scripts/kb-eval/eval.mjs').then((m) => {
    const out = {};
    for (const n of names) {
      const c = readFileSync(join(root, 'skills', n, 'SKILL.md'), 'utf8');
      out[n] = m.extractDescription(m.splitFrontmatter(c).frontmatter);
    }
    return out;
  });
}

test('computeOverlapFromInputs: 10 pairs, eng<->infra focusPair, sorted, deterministic', async () => {
  const descs = await realDescriptions();
  const r1 = computeOverlapFromInputs(descs, promptSet);
  const r2 = computeOverlapFromInputs(descs, promptSet);

  // 5 skills -> C(5,2) = 10 unordered pairs
  assert.equal(r1.pairs.length, 10);

  // each pair carries both signals
  for (const p of r1.pairs) {
    assert.ok(Array.isArray(p.pair) && p.pair.length === 2);
    assert.equal(typeof p.boundaryTension, 'number');
    assert.ok(p.lexical && Array.isArray(p.lexical.shared));
    assert.equal(typeof p.combined, 'number');
  }

  // sorted by combined descending
  for (let i = 1; i < r1.pairs.length; i++) {
    assert.ok(r1.pairs[i - 1].combined >= r1.pairs[i].combined, 'pairs sorted desc by combined');
  }

  // focusPair = eng<->infra (operator-designated Azure-deployment boundary)
  assert.ok(r1.focusPair, 'focusPair present');
  assert.deepEqual(
    [...r1.focusPair.pair].sort(),
    ['ms-ai-engineering', 'ms-ai-infrastructure'],
  );
  assert.equal(r1.focusPair.boundaryTension, 6);

  // determinism: identical output across runs
  assert.deepEqual(r1, r2);
});