ms-ai-architect/tests/kb-eval/test-skill-lifecycle-detect.test.mjs

// test-skill-lifecycle-detect.test.mjs — Spor B / B1 detectors.
// TDD: written before scripts/kb-eval/detect-skill-lifecycle.mjs exists.
//
// Detector 1 (S13, OVERLAP) is DETERMINISTIC and combines two signals:
//   (1) operator-curated boundary-tension graph (k1-trigger-prompts.json belongs_to)
//   (2) df-weighted lexical trigger-surface overlap (down-weights domain-common tokens)
// Detectors 2+3 (S14): coverage/gap (taxonomy vs disk, in-domain only) and
//   bloat/stale (K3-margin per skill + dateless reference-file fraction).
// All detectors must NEVER write to skills/ — they only produce a report.

import { test } from 'node:test';
import assert from 'node:assert/strict';
import { readFileSync } from 'node:fs';
import { dirname, join } from 'node:path';
import { fileURLToPath } from 'node:url';

import {
  tokenize,
  extractTriggerSurface,
  buildDocumentFrequency,
  lexicalOverlap,
  pairKey,
  boundaryTensionMatrix,
  computeOverlapFromInputs,
  // S14 — coverage/gap + bloat/stale
  computeCoverageFromInputs,
  computeBloatFromInputs,
  extractLastUpdated,
  loadDiskCounts,
  loadBloatInputs,
} from '../../scripts/kb-eval/detect-skill-lifecycle.mjs';

const __dirname = dirname(fileURLToPath(import.meta.url));
const PROMPTS_PATH = join(__dirname, '..', '..', 'scripts', 'kb-eval', 'data', 'k1-trigger-prompts.json');
const promptSet = JSON.parse(readFileSync(PROMPTS_PATH, 'utf8'));
const TAX_PATH = join(__dirname, '..', '..', 'scripts', 'kb-update', 'data', 'domain-taxonomy.json');
const taxonomy = JSON.parse(readFileSync(TAX_PATH, 'utf8'));

// ---------------------------------------------------------------------------
// tokenize
// ---------------------------------------------------------------------------
test('tokenize: lowercases, strips punctuation, drops short + stopwords', () => {
  const t = tokenize('Azure AI Services for the RAG-pipeline, and BCDR. Triggers on: x.');
  assert.ok(t.includes('azure'), 'keeps azure');
  assert.ok(t.includes('services'), 'keeps services');
  assert.ok(t.includes('rag'), 'splits hyphen -> rag');
  assert.ok(t.includes('pipeline'), 'splits hyphen -> pipeline');
  assert.ok(t.includes('bcdr'), 'keeps bcdr');
  assert.ok(!t.includes('ai'), 'drops len<3 token ai');
  assert.ok(!t.includes('for'), 'drops stopword for');
  assert.ok(!t.includes('the'), 'drops stopword the');
  assert.ok(!t.includes('and'), 'drops stopword and');
  assert.ok(!t.includes('triggers'), 'drops description-format word triggers');
  // all lowercase
  assert.ok(t.every((w) => w === w.toLowerCase()));
});

// ---------------------------------------------------------------------------
// extractTriggerSurface
// ---------------------------------------------------------------------------
test('extractTriggerSurface: pulls quoted phrases + content tokens', () => {
  const desc =
    'Deep guidance for building AI. Triggers on: "RAG architecture on Azure", "Azure AI Search".';
  const s = extractTriggerSurface(desc);
  assert.ok(Array.isArray(s.phrases));
  assert.equal(s.phrases.length, 2, 'two quoted phrases');
  assert.ok(s.phrases.includes('RAG architecture on Azure'));
  assert.ok(s.tokens instanceof Set);
  assert.ok(s.tokens.has('azure'));
  assert.ok(s.tokens.has('architecture'));
});

// ---------------------------------------------------------------------------
// buildDocumentFrequency
// ---------------------------------------------------------------------------
test('buildDocumentFrequency: counts skills containing each token', () => {
  const surfaces = [
    { tokens: new Set(['azure', 'rag']) },
    { tokens: new Set(['azure', 'bcdr']) },
    { tokens: new Set(['azure', 'dpia']) },
  ];
  const df = buildDocumentFrequency(surfaces);
  assert.equal(df.get('azure'), 3);
  assert.equal(df.get('rag'), 1);
  assert.equal(df.get('bcdr'), 1);
});

// ---------------------------------------------------------------------------
// lexicalOverlap — df-weighting down-weights common tokens
// ---------------------------------------------------------------------------
test('lexicalOverlap: shared tokens, jaccard, df-weighted score', () => {
  const sA = { tokens: new Set(['azure', 'deployment', 'rag']) };
  const sB = { tokens: new Set(['azure', 'deployment', 'bcdr']) };
  const df = new Map([
    ['azure', 5],
    ['deployment', 2],
    ['rag', 1],
    ['bcdr', 1],
  ]);
  const o = lexicalOverlap(sA, sB, df);
  assert.deepEqual(o.shared, ['azure', 'deployment'], 'shared sorted');
  assert.equal(o.jaccard, 0.5, '2 shared / 4 union');
  // weighted = 1/5 (azure) + 1/2 (deployment) = 0.7
  assert.ok(Math.abs(o.weightedScore - 0.7) < 1e-9, `weightedScore=${o.weightedScore}`);
});

test('lexicalOverlap: a high-df token contributes less than a distinctive one', () => {
  const df = new Map([['common', 5], ['rare', 2]]);
  const a = { tokens: new Set(['common', 'rare']) };
  const onlyCommon = lexicalOverlap(a, { tokens: new Set(['common']) }, df);
  const onlyRare = lexicalOverlap(a, { tokens: new Set(['rare']) }, df);
  assert.ok(onlyRare.weightedScore > onlyCommon.weightedScore, 'rare token weighs more');
});

// ---------------------------------------------------------------------------
// pairKey — order-independent, stable
// ---------------------------------------------------------------------------
test('pairKey: order-independent', () => {
  assert.equal(pairKey('b', 'a'), pairKey('a', 'b'));
  assert.equal(pairKey('a', 'b'), 'a|b');
});

// ---------------------------------------------------------------------------
// boundaryTensionMatrix — differential check vs independent reducer
// ---------------------------------------------------------------------------
test('boundaryTensionMatrix: symmetric, matches independent count, ignores non-skill belongs_to', () => {
  const skills = Object.keys(promptSet).filter((k) => k !== '_meta');
  const m = boundaryTensionMatrix(promptSet);

  // independent reimplementation of symmetric tension
  const skillSet = new Set(skills);
  const expected = {};
  for (const s of skills) {
    for (const e of promptSet[s].out_of_domain || []) {
      const b = e.belongs_to;
      if (!skillSet.has(b)) continue; // controls / out-of-stack are not skills
      const k = pairKey(s, b);
      expected[k] = (expected[k] || 0) + 1;
    }
  }
  for (const [k, v] of Object.entries(expected)) {
    assert.equal(m[k], v, `tension ${k}`);
  }
  // anchored known value tied to current curated set (S11)
  assert.equal(m[pairKey('ms-ai-engineering', 'ms-ai-infrastructure')], 6, 'eng<->infra Azure-deployment boundary = 6');
});

// ---------------------------------------------------------------------------
// computeOverlapFromInputs — full report section
// ---------------------------------------------------------------------------
function realDescriptions() {
  // Minimal stand-ins are not enough; use the real five descriptions via the
  // same extractor eval.mjs uses, so the test exercises real data.
  const root = join(__dirname, '..', '..');
  const names = [
    'ms-ai-advisor',
    'ms-ai-engineering',
    'ms-ai-governance',
    'ms-ai-infrastructure',
    'ms-ai-security',
  ];
  // lazy import to avoid top-level coupling
  return import('../../scripts/kb-eval/eval.mjs').then((m) => {
    const out = {};
    for (const n of names) {
      const c = readFileSync(join(root, 'skills', n, 'SKILL.md'), 'utf8');
      out[n] = m.extractDescription(m.splitFrontmatter(c).frontmatter);
    }
    return out;
  });
}

test('computeOverlapFromInputs: 10 pairs, eng<->infra focusPair, sorted, deterministic', async () => {
  const descs = await realDescriptions();
  const r1 = computeOverlapFromInputs(descs, promptSet);
  const r2 = computeOverlapFromInputs(descs, promptSet);

  // 5 skills -> C(5,2) = 10 unordered pairs
  assert.equal(r1.pairs.length, 10);

  // each pair carries both signals
  for (const p of r1.pairs) {
    assert.ok(Array.isArray(p.pair) && p.pair.length === 2);
    assert.equal(typeof p.boundaryTension, 'number');
    assert.ok(p.lexical && Array.isArray(p.lexical.shared));
    assert.equal(typeof p.combined, 'number');
  }

  // sorted by combined descending
  for (let i = 1; i < r1.pairs.length; i++) {
    assert.ok(r1.pairs[i - 1].combined >= r1.pairs[i].combined, 'pairs sorted desc by combined');
  }

  // focusPair = eng<->infra (operator-designated Azure-deployment boundary)
  assert.ok(r1.focusPair, 'focusPair present');
  assert.deepEqual(
    [...r1.focusPair.pair].sort(),
    ['ms-ai-engineering', 'ms-ai-infrastructure'],
  );
  assert.equal(r1.focusPair.boundaryTension, 6);

  // determinism: identical output across runs
  assert.deepEqual(r1, r2);
});

// ===========================================================================
// S14 — Detector 2: coverage/gap (in-domain) — taxonomy vs physical disk
// ===========================================================================
test('computeCoverageFromInputs: gap / thin / ok classification', () => {
  const categorySkill = {
    'rag-architecture': 'eng',
    development: 'advisor',
    'security-scoring': 'sec',
  };
  const diskCounts = {
    eng: { 'rag-architecture': 28 },
    advisor: { development: 1 },
    sec: {}, // security-scoring declared but no folder on disk
  };
  const r = computeCoverageFromInputs(categorySkill, diskCounts, { thinThreshold: 10 });
  const byCat = Object.fromEntries(r.categories.map((c) => [c.category, c]));
  assert.equal(byCat['rag-architecture'].status, 'ok');
  assert.equal(byCat['rag-architecture'].covered, true);
  assert.equal(byCat.development.status, 'thin');
  assert.equal(byCat.development.covered, true);
  assert.equal(byCat['security-scoring'].status, 'gap');
  assert.equal(byCat['security-scoring'].covered, false);
  assert.equal(byCat['security-scoring'].onDiskCount, 0);
  assert.deepEqual(r.gaps.map((g) => g.category), ['security-scoring']);
  assert.deepEqual(r.thin.map((t) => t.category), ['development']);
});

test('computeCoverageFromInputs: detects orphan + misowned disk folders', () => {
  const categorySkill = { 'rag-architecture': 'eng' };
  const diskCounts = {
    eng: { 'rag-architecture': 28, 'mystery-folder': 3 }, // mystery -> orphan (undeclared)
    advisor: { 'rag-architecture': 4 }, // declared owner is eng -> misowned
  };
  const r = computeCoverageFromInputs(categorySkill, diskCounts, { thinThreshold: 10 });
  assert.deepEqual(r.orphans, [{ skill: 'eng', category: 'mystery-folder', onDiskCount: 3 }]);
  assert.deepEqual(r.misowned, [
    { skill: 'advisor', category: 'rag-architecture', declaredOwner: 'eng', onDiskCount: 4 },
  ]);
});

test('computeCoverageFromInputs: deterministic, sorted, summary counts', () => {
  const categorySkill = { b: 'x', a: 'x', c: 'y' };
  const diskCounts = { x: { a: 20, b: 0 }, y: { c: 2 } };
  const r1 = computeCoverageFromInputs(categorySkill, diskCounts, { thinThreshold: 10 });
  const r2 = computeCoverageFromInputs(categorySkill, diskCounts, { thinThreshold: 10 });
  assert.deepEqual(r1, r2);
  assert.deepEqual(r1.categories.map((c) => c.category), ['a', 'b', 'c']); // sorted
  assert.equal(r1.summary.declaredCategories, 3);
  assert.equal(r1.summary.gaps, 1); // b: 0 files
  assert.equal(r1.summary.thin, 1); // c: 2 files
  assert.equal(r1.summary.covered, 2); // a, c
});

test('coverage on real taxonomy + disk: declared-count matches, security-scoring gap, development thin', () => {
  const disk = loadDiskCounts();
  const r = computeCoverageFromInputs(taxonomy.category_skill, disk);
  assert.equal(r.summary.declaredCategories, Object.keys(taxonomy.category_skill).length);
  // security-scoring is declared (ms-ai-security) but has no folder on disk -> gap
  assert.ok(r.gaps.some((g) => g.category === 'security-scoring'), 'security-scoring gap surfaced');
  // development has a single reference file -> thin
  assert.ok(r.thin.some((t) => t.category === 'development'), 'development thin surfaced');
});

// ===========================================================================
// S14 — Detector 3: bloat/stale — K3-margin + dateless reference fraction
// ===========================================================================
test('extractLastUpdated: parses header variants, normalizes YYYY-MM, null when absent', () => {
  assert.equal(extractLastUpdated('# Title\n\n**Last updated:** 2026-03-14\n'), '2026-03-14');
  assert.equal(extractLastUpdated('**Sist oppdatert:** 2025-11\n'), '2025-11-01'); // YYYY-MM -> -01
  assert.equal(extractLastUpdated('**Sist verifisert:** 2026-01-09\n'), '2026-01-09');
  assert.equal(extractLastUpdated('**Dato:** 2024-12-31\n'), '2024-12-31');
  assert.equal(extractLastUpdated('# No header here\nbody text'), null);
});

test('computeBloatFromInputs: margins, ratios, candidate flags', () => {
  const inputs = {
    big: { bodyLines: 470, refTotal: 10, refDateless: 0, datelessFiles: [] },
    stale: { bodyLines: 100, refTotal: 20, refDateless: 4, datelessFiles: ['a.md', 'b.md', 'c.md', 'd.md'] },
    healthy: { bodyLines: 200, refTotal: 50, refDateless: 0, datelessFiles: [] },
  };
  const r = computeBloatFromInputs(inputs, { bloatMarginMin: 50, datelessWarnRatio: 0.05 });
  const by = Object.fromEntries(r.skills.map((s) => [s.name, s]));
  assert.equal(by.big.k3Margin, 30); // 500 - 470
  assert.equal(by.big.bloatCandidate, true); // margin 30 < 50
  assert.equal(by.big.staleCandidate, false);
  assert.equal(by.stale.datelessRatio, 0.2); // 4 / 20
  assert.equal(by.stale.staleCandidate, true); // 0.2 > 0.05
  assert.equal(by.stale.bloatCandidate, false); // margin 400
  assert.deepEqual(by.stale.datelessFiles, ['a.md', 'b.md', 'c.md', 'd.md']);
  assert.equal(by.healthy.bloatCandidate, false);
  assert.equal(by.healthy.staleCandidate, false);
  assert.deepEqual(r.summary.bloatCandidates, ['big']);
  assert.deepEqual(r.summary.staleCandidates, ['stale']);
});

test('computeBloatFromInputs: deterministic, sorted by name, no div-by-zero', () => {
  const inputs = {
    z: { bodyLines: 10, refTotal: 0, refDateless: 0, datelessFiles: [] },
    a: { bodyLines: 10, refTotal: 0, refDateless: 0, datelessFiles: [] },
  };
  const r1 = computeBloatFromInputs(inputs);
  const r2 = computeBloatFromInputs(inputs);
  assert.deepEqual(r1, r2);
  assert.deepEqual(r1.skills.map((s) => s.name), ['a', 'z']); // sorted
  assert.equal(r1.skills[0].datelessRatio, 0); // refTotal 0 -> ratio 0, not NaN
});

test('bloat on real disk: 5 skills, ratios in [0,1], deterministic, dateless count matches file list', () => {
  const inputs = loadBloatInputs();
  const r1 = computeBloatFromInputs(inputs);
  const r2 = computeBloatFromInputs(inputs);
  assert.deepEqual(r1, r2);
  assert.deepEqual(
    r1.skills.map((s) => s.name).sort(),
    ['ms-ai-advisor', 'ms-ai-engineering', 'ms-ai-governance', 'ms-ai-infrastructure', 'ms-ai-security'],
  );
  for (const s of r1.skills) {
    assert.ok(s.bodyLines > 0, `${s.name} has body lines`);
    assert.ok(s.datelessRatio >= 0 && s.datelessRatio <= 1, `${s.name} ratio in range`);
    assert.equal(s.refDateless, s.datelessFiles.length, `${s.name} dateless count matches list`);
  }
});