ms-ai-architect/scripts/kb-eval/detect-skill-lifecycle.mjs

#!/usr/bin/env node
// detect-skill-lifecycle.mjs — Spor B / lag-1-analog at SKILL granularity.
//
// PRODUCES ONLY REPORTS — never writes to skills/. Mirrors the lag-1 invariant:
// detection surfaces candidates; any skill-lifecycle op (merge/sanitize/retire/
// create) goes through decisions.json + operator-gate (later phases B3).
//
// Sesjon 13 (B1, first detector): OVERLAP. Two deterministic signals, combined:
//   (1) boundary-tension graph — operator-curated k1-trigger-prompts.json:
//       each out_of_domain entry is a sibling prompt tagged belongs_to=<skill>,
//       i.e. a hand-labelled "confusable neighbour". Symmetric counts = how much
//       two skills sit on each other's trigger boundary.
//   (2) df-weighted lexical trigger-surface overlap — shared content tokens
//       between two descriptions, each weighted 1/df so domain-common vocabulary
//       ("azure") contributes little and distinctive shared tokens contribute more.
//
// The eng<->infra pair (Azure-deployment boundary) is surfaced as focusPair —
// the operator-designated first target for B1.
//
// Sesjon 14 adds two more detectors to the same report (all deterministic):
//   [2] COVERAGE/GAP — taxonomy.category_skill (declared ownership) vs physical
//       disk folder counts. Surfaces gap (owned, 0 files), thin (< threshold),
//       orphan (undeclared folder), misowned (wrong owner). In-domain only —
//       no new domains proposed.
//   [3] BLOAT/STALE — per-skill K3-margin (eval.checkK3 bodyLines vs 500) plus
//       the dateless reference fraction (refs lacking a "Last updated:" header =
//       unverifiable currency). Flags split-/sanitize-candidates.
//
// Usage:
//   node scripts/kb-eval/detect-skill-lifecycle.mjs          # human summary
//   node scripts/kb-eval/detect-skill-lifecycle.mjs --json   # machine output
//   node scripts/kb-eval/detect-skill-lifecycle.mjs --write  # persist report JSON
//
// Zero dependencies. Reuses eval.mjs extractors + kb-update atomic-write.

import { readFileSync, readdirSync, existsSync, mkdirSync } from 'node:fs';
import { join, dirname, relative } from 'node:path';
import { fileURLToPath } from 'node:url';
import { splitFrontmatter, extractDescription, checkK3 } from './eval.mjs';
import { loadTaxonomy } from '../kb-update/lib/taxonomy.mjs';
import { atomicWriteJson } from '../kb-update/lib/atomic-write.mjs';

const __dirname = dirname(fileURLToPath(import.meta.url));
const PLUGIN_ROOT = join(__dirname, '..', '..');
const SKILLS_DIR = join(PLUGIN_ROOT, 'skills');
const DATA_DIR = join(__dirname, 'data');
const TAX_DATA_DIR = join(__dirname, '..', 'kb-update', 'data');
const PROMPTS_FILE = join(DATA_DIR, 'k1-trigger-prompts.json');
const OUT_FILE = join(DATA_DIR, 'skill-lifecycle-report.json');

// Operator-designated B1 focus boundary.
const FOCUS_PAIR = ['ms-ai-engineering', 'ms-ai-infrastructure'];

// --- S14 detector thresholds (named, documented) -------------------------
// K3 hard body-length limit (mirrors eval.mjs K3_MAX_BODY_LINES — single source
// of bodyLines is eval.checkK3; this constant only computes the margin).
const K3_MAX_BODY_LINES = 500;
// A declared category with fewer than this many reference files is "thin"
// relative to the ~19-file median across the 20 on-disk categories.
const THIN_REF_COUNT = 10;
// Flag a SKILL.md as a split-candidate when its body is within this many lines
// of the K3 limit (i.e. margin < 50 -> body > 450).
const BLOAT_MARGIN_MIN = 50;
// Flag a skill as a sanitize-candidate when more than this fraction of its
// reference files lack a verifiable "Last updated:" header (unverifiable currency).
const DATELESS_WARN_RATIO = 0.05;

// Mirrors report-changes.mjs LAST_UPDATED_PATTERNS (kept local: report-changes
// runs main() at import time, so it cannot be imported safely). Currency-header
// forms accepted across the KB.
const LAST_UPDATED_PATTERNS = [
  /\*\*Last updated:\*\*\s*([\d-]+)/i,
  /\*\*Sist (?:oppdatert|verifisert):\*\*\s*([\d-]+)/i,
  /\*\*Dato:\*\*\s*([\d-]+)/i,
];

// Function words (no + en) of length >= 3. Tokens < 3 chars are dropped anyway,
// so this list only needs the longer connectives. Domain nouns are NOT here —
// df-weighting handles common domain vocabulary instead.
const STOPWORDS = new Set([
  'the', 'and', 'for', 'with', 'between', 'before', 'not', 'are', 'that', 'this',
  'into', 'over', 'per', 'use', 'used', 'when', 'which', 'how', 'via', 'from',
  'eller', 'som', 'til', 'med', 'mot', 'ved', 'for', 'har', 'kan', 'ikke', 'der',
  'det', 'den', 'ein', 'eit', 'sin',
  // description-format boilerplate (every skill ends with "Triggers on:")
  'triggers', 'trigger',
]);

/** Lowercase, split on non-alphanumeric, drop stopwords + tokens < 3 chars. */
export function tokenize(text) {
  return (text.toLowerCase().match(/[a-z0-9æøå]+/g) || [])
    .filter((w) => w.length >= 3 && !STOPWORDS.has(w));
}

/** Trigger surface of a description: quoted phrases + content-token set. */
export function extractTriggerSurface(description) {
  const phrases = (description.match(/"([^"]+)"/g) || []).map((p) => p.slice(1, -1));
  return { phrases, tokens: new Set(tokenize(description)) };
}

/** token -> number of skill-surfaces that contain it. */
export function buildDocumentFrequency(surfaces) {
  const df = new Map();
  for (const s of surfaces) {
    for (const t of s.tokens) df.set(t, (df.get(t) || 0) + 1);
  }
  return df;
}

/** Order-independent pair key. */
export function pairKey(a, b) {
  return [a, b].sort().join('|');
}

/** Lexical overlap between two surfaces, df-weighted. */
export function lexicalOverlap(surfaceA, surfaceB, df) {
  const shared = [];
  for (const t of surfaceA.tokens) if (surfaceB.tokens.has(t)) shared.push(t);
  shared.sort();
  const union = new Set([...surfaceA.tokens, ...surfaceB.tokens]).size;
  const jaccard = union > 0 ? shared.length / union : 0;
  let weightedScore = 0;
  for (const t of shared) weightedScore += 1 / (df.get(t) || 1);
  return {
    shared,
    jaccard: Number(jaccard.toFixed(4)),
    weightedScore: Number(weightedScore.toFixed(4)),
  };
}

/**
 * Symmetric boundary-tension matrix from the curated prompt set.
 * Counts out_of_domain entries whose belongs_to is one of the real skills
 * (controls / out-of-stack entries are ignored). Keyed by pairKey.
 */
export function boundaryTensionMatrix(promptSet) {
  const skills = Object.keys(promptSet).filter((k) => k !== '_meta');
  const skillSet = new Set(skills);
  const m = {};
  for (const s of skills) {
    for (const e of promptSet[s].out_of_domain || []) {
      const b = e && e.belongs_to;
      if (!skillSet.has(b) || b === s) continue;
      const k = pairKey(s, b);
      m[k] = (m[k] || 0) + 1;
    }
  }
  return m;
}

/**
 * Pure core: given { skill -> description } and the curated prompt set, compute
 * the overlap report section. combined = boundaryTension + weightedScore
 * (operator-grounded primary signal + distinctive-lexical corroboration).
 */
export function computeOverlapFromInputs(descriptionsBySkill, promptSet) {
  const skills = Object.keys(descriptionsBySkill).sort();
  const surfaces = {};
  for (const s of skills) surfaces[s] = extractTriggerSurface(descriptionsBySkill[s]);
  const df = buildDocumentFrequency(Object.values(surfaces));
  const tension = boundaryTensionMatrix(promptSet);

  const pairs = [];
  for (let i = 0; i < skills.length; i++) {
    for (let j = i + 1; j < skills.length; j++) {
      const a = skills[i];
      const b = skills[j];
      const key = pairKey(a, b);
      const lexical = lexicalOverlap(surfaces[a], surfaces[b], df);
      const boundaryTension = tension[key] || 0;
      const combined = Number((boundaryTension + lexical.weightedScore).toFixed(4));
      pairs.push({ pair: [a, b], key, boundaryTension, lexical, combined });
    }
  }
  // sort by combined desc, then key asc for stable ties
  pairs.sort((x, y) => y.combined - x.combined || x.key.localeCompare(y.key));

  const focusKey = pairKey(...FOCUS_PAIR);
  const focusPair = pairs.find((p) => p.key === focusKey) || null;

  return {
    method:
      'deterministic: (1) operator-curated boundary-tension (k1-trigger-prompts belongs_to), ' +
      '(2) df-weighted lexical trigger-surface overlap. combined = boundaryTension + weightedScore.',
    focusPairReason:
      'Azure-deployment boundary engineering(build) <-> infrastructure(operate) — operator-designated B1 target.',
    pairs,
    focusPair,
  };
}

// ===========================================================================
// S14 — Detector 2: coverage/gap (in-domain) — taxonomy vs physical disk
// ===========================================================================

/**
 * Pure core: in-domain coverage. Compares the declared ownership map
 * (taxonomy.category_skill) against physical disk folder counts. NO new domains
 * are proposed — this only surfaces gaps WITHIN the already-covered domain.
 *   gap      = category declared but 0 files on disk under its owner
 *   thin     = declared + covered, but < thinThreshold files
 *   orphan   = a disk folder whose category is not declared anywhere
 *   misowned = a disk folder sitting under a skill other than its declared owner
 * @param {Record<string,string>} categorySkill  category -> owning skill
 * @param {Record<string,Record<string,number>>} diskCounts  skill -> {category: count}
 */
export function computeCoverageFromInputs(categorySkill, diskCounts, { thinThreshold = THIN_REF_COUNT } = {}) {
  const categories = [];
  for (const category of Object.keys(categorySkill).sort()) {
    const owningSkill = categorySkill[category];
    const onDiskCount = (diskCounts[owningSkill] && diskCounts[owningSkill][category]) || 0;
    const covered = onDiskCount > 0;
    const thin = covered && onDiskCount < thinThreshold;
    const status = !covered ? 'gap' : thin ? 'thin' : 'ok';
    categories.push({ category, owningSkill, onDiskCount, covered, thin, status });
  }

  const orphans = [];
  const misowned = [];
  for (const skill of Object.keys(diskCounts).sort()) {
    for (const category of Object.keys(diskCounts[skill]).sort()) {
      const declaredOwner = categorySkill[category];
      if (declaredOwner === undefined) {
        orphans.push({ skill, category, onDiskCount: diskCounts[skill][category] });
      } else if (declaredOwner !== skill) {
        misowned.push({ skill, category, declaredOwner, onDiskCount: diskCounts[skill][category] });
      }
    }
  }

  const gaps = categories.filter((c) => c.status === 'gap');
  const thinCats = categories.filter((c) => c.status === 'thin');
  return {
    method:
      'in-domain coverage: taxonomy.category_skill (declared ownership) vs physical-disk folder counts. ' +
      'gap = owned but 0 files; thin = owned but < thinThreshold files; ' +
      'orphan = disk folder with no declared owner; misowned = folder under a skill other than its declared owner. ' +
      'No new domains proposed — gaps are within the already-covered domain only.',
    thinThreshold,
    categories,
    gaps,
    thin: thinCats,
    orphans,
    misowned,
    summary: {
      declaredCategories: categories.length,
      covered: categories.filter((c) => c.covered).length,
      gaps: gaps.length,
      thin: thinCats.length,
      orphans: orphans.length,
      misowned: misowned.length,
    },
  };
}

// ===========================================================================
// S14 — Detector 3: bloat/stale — K3-margin + dateless reference fraction
// ===========================================================================

/**
 * Parse a "Last updated:" currency header from leading text. Mirrors
 * report-changes.parseLastUpdated: YYYY-MM normalizes to YYYY-MM-01; returns
 * null when no recognized header is present (unverifiable currency).
 */
export function extractLastUpdated(text) {
  const head = text.slice(0, 500);
  for (const re of LAST_UPDATED_PATTERNS) {
    const m = head.match(re);
    if (m) {
      const raw = m[1].trim();
      return raw.length === 7 ? raw + '-01' : raw;
    }
  }
  return null;
}

/**
 * Pure core: per-skill bloat (K3-margin) + stale (dateless reference fraction).
 *   bloatCandidate = body within bloatMarginMin lines of the K3 limit -> split
 *   staleCandidate = > datelessWarnRatio of refs lack a "Last updated:" header
 * Stale here means "unverifiable currency" (disk-only, deterministic). Genuine
 * poll-derived obsolescence is a separate, registry-dependent signal that lives
 * in change-report.json — intentionally out of this deterministic detector.
 * @param {Record<string,{bodyLines:number,refTotal:number,refDateless:number,datelessFiles?:string[]}>} skillInputs
 */
export function computeBloatFromInputs(skillInputs, { bloatMarginMin = BLOAT_MARGIN_MIN, datelessWarnRatio = DATELESS_WARN_RATIO } = {}) {
  const skills = [];
  for (const name of Object.keys(skillInputs).sort()) {
    const { bodyLines, refTotal, refDateless, datelessFiles = [] } = skillInputs[name];
    const k3Margin = K3_MAX_BODY_LINES - bodyLines;
    const bloatCandidate = k3Margin < bloatMarginMin;
    const datelessRatio = refTotal > 0 ? Number((refDateless / refTotal).toFixed(4)) : 0;
    const staleCandidate = datelessRatio > datelessWarnRatio;
    skills.push({ name, bodyLines, k3Margin, bloatCandidate, refTotal, refDateless, datelessRatio, staleCandidate, datelessFiles });
  }
  return {
    method:
      'bloat = eval.checkK3 bodyLines vs K3 limit (' + K3_MAX_BODY_LINES + '); margin < bloatMarginMin -> split-candidate. ' +
      'stale = fraction of reference files lacking a verifiable "Last updated:" header (unverifiable currency, disk-only). ' +
      'Poll-derived obsolescence (change-report.json) is a complementary registry-dependent signal, not computed here.',
    thresholds: { k3MaxBodyLines: K3_MAX_BODY_LINES, bloatMarginMin, datelessWarnRatio },
    skills,
    summary: {
      bloatCandidates: skills.filter((s) => s.bloatCandidate).map((s) => s.name),
      staleCandidates: skills.filter((s) => s.staleCandidate).map((s) => s.name),
    },
  };
}

/** Recursively count .md files under a directory. */
function countMarkdown(dir) {
  let n = 0;
  if (!existsSync(dir)) return n;
  for (const e of readdirSync(dir, { withFileTypes: true })) {
    const p = join(dir, e.name);
    if (e.isDirectory()) n += countMarkdown(p);
    else if (e.isFile() && e.name.endsWith('.md')) n++;
  }
  return n;
}

/** Recursively list .md file paths under a directory. */
function listMarkdown(dir) {
  const out = [];
  if (!existsSync(dir)) return out;
  for (const e of readdirSync(dir, { withFileTypes: true })) {
    const p = join(dir, e.name);
    if (e.isDirectory()) out.push(...listMarkdown(p));
    else if (e.isFile() && e.name.endsWith('.md')) out.push(p);
  }
  return out;
}

/** Physical disk truth: { skill -> { category -> mdFileCount } } (read-only). */
export function loadDiskCounts() {
  const out = {};
  for (const e of readdirSync(SKILLS_DIR, { withFileTypes: true })) {
    if (!e.isDirectory()) continue;
    const refDir = join(SKILLS_DIR, e.name, 'references');
    if (!existsSync(refDir)) continue;
    const cats = {};
    for (const c of readdirSync(refDir, { withFileTypes: true })) {
      if (c.isDirectory()) cats[c.name] = countMarkdown(join(refDir, c.name));
    }
    out[e.name] = cats;
  }
  return out;
}

/** Per-skill bloat/stale inputs read from disk (read-only). */
export function loadBloatInputs() {
  const out = {};
  for (const e of readdirSync(SKILLS_DIR, { withFileTypes: true })) {
    if (!e.isDirectory()) continue;
    const skillMd = join(SKILLS_DIR, e.name, 'SKILL.md');
    if (!existsSync(skillMd)) continue;
    const { body } = splitFrontmatter(readFileSync(skillMd, 'utf8'));
    const bodyLines = checkK3(body).bodyLines;
    const refFiles = listMarkdown(join(SKILLS_DIR, e.name, 'references'));
    const datelessFiles = [];
    for (const f of refFiles) {
      if (extractLastUpdated(readFileSync(f, 'utf8')) === null) datelessFiles.push(relative(PLUGIN_ROOT, f));
    }
    out[e.name] = { bodyLines, refTotal: refFiles.length, refDateless: datelessFiles.length, datelessFiles };
  }
  return out;
}

/** Read the five SKILL.md descriptions from disk. */
function loadDescriptions() {
  const out = {};
  for (const e of readdirSync(SKILLS_DIR, { withFileTypes: true })) {
    if (!e.isDirectory()) continue;
    const md = join(SKILLS_DIR, e.name, 'SKILL.md');
    if (!existsSync(md)) continue;
    out[e.name] = extractDescription(splitFrontmatter(readFileSync(md, 'utf8')).frontmatter);
  }
  return out;
}

function buildReport() {
  const promptSet = JSON.parse(readFileSync(PROMPTS_FILE, 'utf8'));
  const descriptions = loadDescriptions();
  const taxonomy = loadTaxonomy(TAX_DATA_DIR);
  return {
    rubric: 'skill-lifecycle',
    phase: 'B1',
    note: 'Detection only — never writes to skills/. Candidates feed decisions.json + operator-gate (B3).',
    overlap: computeOverlapFromInputs(descriptions, promptSet),
    coverage: computeCoverageFromInputs(taxonomy.category_skill, loadDiskCounts()),
    bloat: computeBloatFromInputs(loadBloatInputs()),
  };
}

function main() {
  const args = process.argv.slice(2);
  const jsonOut = args.includes('--json');
  const doWrite = args.includes('--write');

  const report = buildReport();

  if (doWrite) {
    mkdirSync(DATA_DIR, { recursive: true });
    atomicWriteJson(OUT_FILE, report);
  }
  if (jsonOut) {
    process.stdout.write(JSON.stringify(report) + '\n');
    return;
  }

  const o = report.overlap;
  console.log(`\nSkill-livssyklus — B1 deteksjon (overlap · coverage · bloat)\n`);
  console.log(`[1] Overlap — ${o.pairs.length} par (sortert på combined = grensetension + df-vektet leksikalsk):\n`);
  for (const p of o.pairs) {
    const lex = p.lexical.shared.length ? p.lexical.shared.join(', ') : '—';
    const focus = p.key === pairKey(...FOCUS_PAIR) ? '  ◀ FOCUS (Azure-deployment)' : '';
    console.log(`  ${p.combined.toFixed(2).padStart(6)}  ${p.pair.join(' / ')}${focus}`);
    console.log(`          tension=${p.boundaryTension}  lex(w=${p.lexical.weightedScore}, jac=${p.lexical.jaccard})  delt: ${lex}`);
  }
  console.log(`\n  Focus-par: ${o.focusPair ? o.focusPair.pair.join(' <-> ') : '(ingen)'} — ${o.focusPairReason}`);

  const c = report.coverage;
  console.log(`\n[2] Coverage/gap — ${c.summary.declaredCategories} deklarerte kategorier (terskel tynn < ${c.thinThreshold}):`);
  console.log(`    dekket=${c.summary.covered}  gap=${c.summary.gaps}  tynn=${c.summary.thin}  orphan=${c.summary.orphans}  misowned=${c.summary.misowned}`);
  if (c.gaps.length) {
    console.log('    GAP (eid, 0 filer på disk):');
    for (const g of c.gaps) console.log(`      • ${g.category} → ${g.owningSkill}`);
  }
  if (c.thin.length) {
    console.log('    TYNN (eid, men få filer):');
    for (const t of c.thin) console.log(`      • ${t.category} → ${t.owningSkill} (${t.onDiskCount})`);
  }
  for (const x of c.orphans) console.log(`      ⚠ orphan: ${x.skill}/${x.category} (${x.onDiskCount}) — ikke deklarert`);
  for (const x of c.misowned) console.log(`      ⚠ misowned: ${x.skill}/${x.category} — deklarert eier=${x.declaredOwner}`);

  const b = report.bloat;
  console.log(`\n[3] Bloat/stale — per skill (K3-grense ${b.thresholds.k3MaxBodyLines}; split-kandidat margin < ${b.thresholds.bloatMarginMin}; saner-kandidat dateless > ${b.thresholds.datelessWarnRatio}):`);
  for (const s of b.skills) {
    const flags = [s.bloatCandidate ? 'SPLIT' : '', s.staleCandidate ? 'SANER' : ''].filter(Boolean).join('+') || 'ok';
    console.log(`    ${s.name.padEnd(22)} body=${String(s.bodyLines).padStart(3)} (margin ${String(s.k3Margin).padStart(3)})  dateless=${s.refDateless}/${s.refTotal} (${s.datelessRatio})  [${flags}]`);
  }
  if (b.summary.bloatCandidates.length) console.log(`    Split-kandidater: ${b.summary.bloatCandidates.join(', ')}`);
  if (b.summary.staleCandidates.length) console.log(`    Saner-kandidater: ${b.summary.staleCandidates.join(', ')}`);

  console.log('\n(Rapport skrives med --write til data/skill-lifecycle-report.json; aldri til skills/.)\n');
}

if (process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1]) {
  main();
}