ms-ai-architect/scripts/kb-eval/eval.mjs

#!/usr/bin/env node
// eval.mjs — Skill quality baseline (rubrikk K1–K9) for ms-ai-architect.
//
// READ-ONLY by default. Computes the deterministic criteria (K2/K3/K5/K6 +
// reference-count consistency) for every skill and extracts the inputs an LLM
// judge needs for the semantic criteria (K1/K4/K7/K8/K9). The baseline file is
// written ONLY with --write (operator gate), per the redesign spec.
//
// Usage:
//   node scripts/kb-eval/eval.mjs            # human-readable summary
//   node scripts/kb-eval/eval.mjs --json     # machine output (pure JSON to stdout)
//   node scripts/kb-eval/eval.mjs --write    # also persist data/eval-baseline.json
//
// Zero dependencies. Reuses kb-update/lib/atomic-write.mjs for the gated write.

import { readdirSync, readFileSync, existsSync, mkdirSync } from 'node:fs';
import { join, dirname, relative } from 'node:path';
import { fileURLToPath } from 'node:url';
import { atomicWriteJson } from '../kb-update/lib/atomic-write.mjs';
import {
  computeOverlapFromInputs,
  perSkillSiblingOverlap,
  K10_OVERLAP_THRESHOLD,
} from './lib/sibling-overlap.mjs';

const __dirname = dirname(fileURLToPath(import.meta.url));
const PLUGIN_ROOT = join(__dirname, '..', '..');
const SKILLS_DIR = join(PLUGIN_ROOT, 'skills');
const OUT_DIR = join(__dirname, 'data');
const OUT_FILE = join(OUT_DIR, 'eval-baseline.json');
const PROMPTS_FILE = join(OUT_DIR, 'k1-trigger-prompts.json');

// Thresholds — see spec "Rubrikk K1–K10".
const K3_MAX_BODY_LINES = 500;
const K5_MIN_NAMED_RATIO = 0.2;

/** Recursively list .md files under dir. */
function listMarkdown(dir) {
  const out = [];
  if (!existsSync(dir)) return out;
  for (const e of readdirSync(dir, { withFileTypes: true })) {
    const p = join(dir, e.name);
    if (e.isDirectory()) out.push(...listMarkdown(p));
    else if (e.isFile() && e.name.endsWith('.md')) out.push(p);
  }
  return out;
}

/** Count .md files per immediate subfolder of references/. */
function refCountsPerFolder(refDir) {
  const counts = {};
  if (!existsSync(refDir)) return counts;
  for (const e of readdirSync(refDir, { withFileTypes: true })) {
    if (e.isDirectory()) counts[e.name] = listMarkdown(join(refDir, e.name)).length;
  }
  return counts;
}

/** Split YAML frontmatter from body. */
export function splitFrontmatter(content) {
  if (!content.startsWith('---')) return { frontmatter: '', body: content };
  const close = content.indexOf('\n---', 3);
  if (close === -1) return { frontmatter: '', body: content };
  const bodyStart = content.indexOf('\n', close + 1);
  return {
    frontmatter: content.slice(0, bodyStart === -1 ? content.length : bodyStart + 1),
    body: bodyStart === -1 ? '' : content.slice(bodyStart + 1),
  };
}

/** Extract the (possibly folded) description value from frontmatter. */
export function extractDescription(frontmatter) {
  const lines = frontmatter.split('\n');
  const out = [];
  let collecting = false;
  for (const line of lines) {
    if (/^description:/.test(line)) {
      collecting = true;
      out.push(line.replace(/^description:\s*>?-?\s*/, ''));
      continue;
    }
    if (collecting) {
      if (/^[A-Za-z0-9_-]+:\s/.test(line) || line.trim() === '---' || line.trim() === '') {
        if (line.trim() === '') continue; // folded blocks keep going across blanks
        break;
      }
      out.push(line.trim());
    }
  }
  return out.join(' ').replace(/\s+/g, ' ').trim();
}

/** K2 — description format: third-person/use-when OR >=3 quoted user phrases. */
export function checkK2(description) {
  const quoted = (description.match(/"[^"]+"/g) || []).length;
  const useWhenForm = /should be used when|use this skill|triggers on/i.test(description);
  const imperativeStart = /^(use|bruk|run|kjør)\b/i.test(description);
  const pass = useWhenForm || quoted >= 3;
  return { quotedPhrases: quoted, useWhenForm, imperativeStart, pass };
}

/** K3 — SKILL.md body length. */
export function checkK3(body) {
  const lines = body.replace(/\n+$/, '').split('\n').length;
  return { bodyLines: lines, pass: lines <= K3_MAX_BODY_LINES };
}

/** K5/K6 — progressive disclosure: named .md links vs folder-only references. */
export function checkK5K6(body, totalRefFiles) {
  const named = [...new Set((body.match(/references\/[A-Za-z0-9_\-/]+\.md/g) || []))];
  // folder references like `references/foo/` not immediately followed by a filename
  const folderRefs = [...new Set((body.match(/references\/[A-Za-z0-9_-]+\/(?![A-Za-z0-9_-]+\.md)/g) || []))];
  const namedRatio = totalRefFiles > 0 ? named.length / totalRefFiles : 0;
  return {
    K5: {
      namedFileLinks: named.length,
      folderRefs: folderRefs.length,
      totalRefFiles,
      namedRatio: Number(namedRatio.toFixed(4)),
      pass: namedRatio >= K5_MIN_NAMED_RATIO,
      sampleNamed: named.slice(0, 5),
    },
    K6: {
      // routing table = at least one named start file the model can Read directly
      namedStartFiles: named.length,
      pass: named.length >= 1,
    },
  };
}

/** Reference-count consistency: cited per-folder counts vs actual on disk. */
export function checkRefConsistency(body, actualPerFolder) {
  const mismatches = [];
  const seen = {};
  // prose form: references/<folder>/ (N files)   and   `references/<folder>/` (N files)
  const proseRe = /references\/([a-z0-9-]+)\/`?\s*\((\d+)\s*files?\)/gi;
  // table form: `references/<folder>/` | N
  const tableRe = /`references\/([a-z0-9-]+)\/`\s*\|\s*(\d+)/gi;
  for (const re of [proseRe, tableRe]) {
    let m;
    while ((m = re.exec(body)) !== null) {
      const folder = m[1];
      const cited = Number(m[2]);
      seen[folder] = seen[folder] || new Set();
      seen[folder].add(cited);
      const actual = actualPerFolder[folder];
      if (actual !== undefined && cited !== actual) {
        mismatches.push({ folder, cited, actual });
      }
    }
  }
  // also flag folders cited with two different numbers
  for (const [folder, set] of Object.entries(seen)) {
    if (set.size > 1) mismatches.push({ folder, citedMultiple: [...set], actual: actualPerFolder[folder] ?? null });
  }
  return { consistent: mismatches.length === 0, mismatches };
}

/** Light deterministic pre-scan for K9 (time-sensitive tokens in body). */
function scanK9Hints(body) {
  const tokens = body.match(/\b(GA|preview|deprecat\w*|retir\w*|20\d\d|v\d+(?:\.\d+)?)\b/gi) || [];
  return { timeSensitiveTokenHits: tokens.length, sample: [...new Set(tokens)].slice(0, 12) };
}

export function evalSkill(skillName) {
  const skillDir = join(SKILLS_DIR, skillName);
  const skillMd = join(skillDir, 'SKILL.md');
  if (!existsSync(skillMd)) return null;
  const content = readFileSync(skillMd, 'utf8');
  const { frontmatter, body } = splitFrontmatter(content);
  const description = extractDescription(frontmatter);
  const refDir = join(skillDir, 'references');
  const refFiles = listMarkdown(refDir);
  const perFolder = refCountsPerFolder(refDir);

  const K2 = checkK2(description);
  const K3 = checkK3(body);
  const { K5, K6 } = checkK5K6(body, refFiles.length);
  const refConsistency = checkRefConsistency(body, perFolder);
  const K9hints = scanK9Hints(body);

  return {
    name: skillName,
    skillMd: relative(PLUGIN_ROOT, skillMd),
    refFilesActual: refFiles.length,
    refCountsPerFolder: perFolder,
    deterministic: {
      K2_descriptionFormat: K2,
      K3_bodyLength: K3,
      K5_progressiveDisclosure: K5,
      K6_routingTable: K6,
      refCountConsistency: refConsistency,
      K9_timeSensitiveHints: K9hints,
    },
    // Pointers for the LLM-judge pass (K1/K4/K7/K8/K9). The judge reads the files
    // directly; we only carry the description + sampled ref files here.
    judgeInputs: {
      description,
      bodyLines: K3.bodyLines,
      refFileSample: refFiles.slice(0, 8).map((f) => relative(PLUGIN_ROOT, f)),
    },
    // Filled in by the operator-gated LLM-judge merge step. Null = not yet judged.
    judge: null,
  };
}

/**
 * K10 — sibling-scope-non-overlap. Deterministic, computed ACROSS all skills
 * (unlike K2/K3/K5/K6 which are per-skill), so it is attached after the
 * per-skill pass. Reuses the B1 overlap core: combined = boundaryTension +
 * df-weighted lexical overlap; each skill's K10 verdict is its worst sibling
 * pair (perSkillSiblingOverlap). Mutates + returns the skills array.
 */
export function attachSiblingOverlap(skills, promptSet, { threshold = K10_OVERLAP_THRESHOLD } = {}) {
  const descriptionsBySkill = {};
  for (const s of skills) descriptionsBySkill[s.name] = s.judgeInputs.description;
  const overlap = computeOverlapFromInputs(descriptionsBySkill, promptSet);
  const k10 = perSkillSiblingOverlap(overlap.pairs, { threshold });
  for (const s of skills) {
    s.deterministic.K10_siblingScopeOverlap =
      k10[s.name] || { maxCombined: 0, worstSibling: null, pass: true, threshold };
  }
  return skills;
}

function main() {
  const args = process.argv.slice(2);
  const jsonOut = args.includes('--json');
  const doWrite = args.includes('--write');

  const skillNames = readdirSync(SKILLS_DIR, { withFileTypes: true })
    .filter((e) => e.isDirectory() && existsSync(join(SKILLS_DIR, e.name, 'SKILL.md')))
    .map((e) => e.name)
    .sort();

  const skills = skillNames.map(evalSkill).filter(Boolean);

  // K10 — sibling-scope-non-overlap (deterministic, cross-skill). Attached after
  // the per-skill pass since it needs every description + the curated prompt set.
  if (existsSync(PROMPTS_FILE)) {
    const promptSet = JSON.parse(readFileSync(PROMPTS_FILE, 'utf8'));
    attachSiblingOverlap(skills, promptSet);
  }

  // Merge operator-gated LLM-judge results (K1/K4/K7/K8/K9) if present.
  const judgeFile = join(OUT_DIR, 'judge-results.json');
  if (existsSync(judgeFile)) {
    const jr = JSON.parse(readFileSync(judgeFile, 'utf8'));
    for (const s of skills) if (jr[s.name]) s.judge = jr[s.name];
  }

  const report = {
    rubric: 'K1-K10',
    note: 'Deterministic: K2,K3,K5,K6,refCountConsistency,K10(siblingScopeOverlap). LLM-judge (operator-gated): K1,K4,K7,K8,K9.',
    skills,
  };

  if (doWrite) {
    mkdirSync(OUT_DIR, { recursive: true });
    atomicWriteJson(OUT_FILE, report);
  }

  if (jsonOut) {
    process.stdout.write(JSON.stringify(report) + '\n');
    return;
  }

  // Human-readable summary
  console.log(`\nKB skill eval — ${skills.length} skills (deterministic pass)\n`);
  for (const s of skills) {
    const d = s.deterministic;
    const flag = (b) => (b ? 'PASS' : 'FAIL');
    console.log(`■ ${s.name}  (${s.refFilesActual} ref-filer)`);
    console.log(`   K2 description : ${flag(d.K2_descriptionFormat.pass)} (quoted=${d.K2_descriptionFormat.quotedPhrases}, useWhen=${d.K2_descriptionFormat.useWhenForm})`);
    console.log(`   K3 body ≤500   : ${flag(d.K3_bodyLength.pass)} (${d.K3_bodyLength.bodyLines} linjer)`);
    console.log(`   K5 prog.disc.  : ${flag(d.K5_progressiveDisclosure.pass)} (navngitte=${d.K5_progressiveDisclosure.namedFileLinks}, mapper=${d.K5_progressiveDisclosure.folderRefs}, ratio=${d.K5_progressiveDisclosure.namedRatio})`);
    console.log(`   K6 routing-tab : ${flag(d.K6_routingTable.pass)} (navngitte startfiler=${d.K6_routingTable.namedStartFiles})`);
    const rc = d.refCountConsistency;
    console.log(`   ref-tall       : ${flag(rc.consistent)}${rc.consistent ? '' : ' — ' + JSON.stringify(rc.mismatches)}`);
    const k10 = d.K10_siblingScopeOverlap;
    if (k10) {
      console.log(`   K10 søsken-ovl : ${flag(k10.pass)} (maks ${k10.maxCombined} mot ${k10.worstSibling}, terskel ${k10.threshold})`);
    }
    console.log(`   K9 hints       : ${d.K9_timeSensitiveHints.timeSensitiveTokenHits} tid-sensitive tokens i body`);
    console.log('');
  }
  console.log(`(LLM-judge K1/K4/K7/K8/K9 kjøres som operatør-gated subagent-steg; baseline skrives med --write.)\n`);
}

// Run only when invoked directly (not when imported by tests).
if (process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1]) {
  main();
}