ms-ai-architect/scripts/kb-eval/eval.mjs

#!/usr/bin/env node
// eval.mjs — Skill quality baseline (rubrikk K1–K9) for ms-ai-architect.
//
// READ-ONLY by default. Computes the deterministic criteria (K2/K3/K5/K6 +
// reference-count consistency) for every skill and extracts the inputs an LLM
// judge needs for the semantic criteria (K1/K4/K7/K9). The baseline file is
// written ONLY with --write (operator gate), per the redesign spec.
//
// Usage:
//   node scripts/kb-eval/eval.mjs            # human-readable summary
//   node scripts/kb-eval/eval.mjs --json     # machine output (pure JSON to stdout)
//   node scripts/kb-eval/eval.mjs --write    # also persist data/eval-baseline.json
//
// Zero dependencies. Reuses kb-update/lib/atomic-write.mjs for the gated write.

import { readdirSync, readFileSync, existsSync, mkdirSync } from 'node:fs';
import { join, dirname, relative } from 'node:path';
import { fileURLToPath } from 'node:url';
import { atomicWriteJson } from '../kb-update/lib/atomic-write.mjs';
import { parseTypeHeader, parseSourceHeader } from '../kb-update/lib/kb-headers.mjs';
import {
  computeOverlapFromInputs,
  perSkillSiblingOverlap,
  K10_OVERLAP_THRESHOLD,
} from './lib/sibling-overlap.mjs';

const __dirname = dirname(fileURLToPath(import.meta.url));
const PLUGIN_ROOT = join(__dirname, '..', '..');
const SKILLS_DIR = join(PLUGIN_ROOT, 'skills');
const OUT_DIR = join(__dirname, 'data');
const OUT_FILE = join(OUT_DIR, 'eval-baseline.json');
const PROMPTS_FILE = join(OUT_DIR, 'k1-trigger-prompts.json');

// Thresholds — see spec "Rubrikk K1–K10".
const K3_MAX_BODY_LINES = 500;
const K5_MIN_NAMED_RATIO = 0.2;
const CT5_SOURCED_RATIO_TARGET = 0.8; // pass at/above this share of sourced reference files (mirrors old K8)

/** Recursively list .md files under dir. */
function listMarkdown(dir) {
  const out = [];
  if (!existsSync(dir)) return out;
  for (const e of readdirSync(dir, { withFileTypes: true })) {
    const p = join(dir, e.name);
    if (e.isDirectory()) out.push(...listMarkdown(p));
    else if (e.isFile() && e.name.endsWith('.md')) out.push(p);
  }
  return out;
}

/** Count .md files per immediate subfolder of references/. */
function refCountsPerFolder(refDir) {
  const counts = {};
  if (!existsSync(refDir)) return counts;
  for (const e of readdirSync(refDir, { withFileTypes: true })) {
    if (e.isDirectory()) counts[e.name] = listMarkdown(join(refDir, e.name)).length;
  }
  return counts;
}

/** Split YAML frontmatter from body. */
export function splitFrontmatter(content) {
  if (!content.startsWith('---')) return { frontmatter: '', body: content };
  const close = content.indexOf('\n---', 3);
  if (close === -1) return { frontmatter: '', body: content };
  const bodyStart = content.indexOf('\n', close + 1);
  return {
    frontmatter: content.slice(0, bodyStart === -1 ? content.length : bodyStart + 1),
    body: bodyStart === -1 ? '' : content.slice(bodyStart + 1),
  };
}

/** Extract the `name:` scalar from frontmatter (single-line value). */
export function extractName(frontmatter) {
  for (const line of frontmatter.split('\n')) {
    const m = /^name:\s*(.+?)\s*$/.exec(line);
    if (m) return m[1].replace(/^["']|["']$/g, '').trim();
  }
  return '';
}

/** Extract the (possibly folded) description value from frontmatter. */
export function extractDescription(frontmatter) {
  const lines = frontmatter.split('\n');
  const out = [];
  let collecting = false;
  for (const line of lines) {
    if (/^description:/.test(line)) {
      collecting = true;
      out.push(line.replace(/^description:\s*>?-?\s*/, ''));
      continue;
    }
    if (collecting) {
      if (/^[A-Za-z0-9_-]+:\s/.test(line) || line.trim() === '---' || line.trim() === '') {
        if (line.trim() === '') continue; // folded blocks keep going across blanks
        break;
      }
      out.push(line.trim());
    }
  }
  return out.join(' ').replace(/\s+/g, ' ').trim();
}

/** K2 — description format: third-person/use-when OR >=3 quoted user phrases. */
export function checkK2(description) {
  const quoted = (description.match(/"[^"]+"/g) || []).length;
  const useWhenForm = /should be used when|use this skill|triggers on/i.test(description);
  const imperativeStart = /^(use|bruk|run|kjør)\b/i.test(description);
  const pass = useWhenForm || quoted >= 3;
  return { quotedPhrases: quoted, useWhenForm, imperativeStart, pass };
}

/** K3 — SKILL.md body length. */
export function checkK3(body) {
  const lines = body.replace(/\n+$/, '').split('\n').length;
  return { bodyLines: lines, pass: lines <= K3_MAX_BODY_LINES };
}

/** K5/K6 — progressive disclosure: named .md links vs folder-only references. */
export function checkK5K6(body, totalRefFiles) {
  const named = [...new Set((body.match(/references\/[A-Za-z0-9_\-/]+\.md/g) || []))];
  // folder references like `references/foo/` not immediately followed by a filename
  const folderRefs = [...new Set((body.match(/references\/[A-Za-z0-9_-]+\/(?![A-Za-z0-9_-]+\.md)/g) || []))];
  const namedRatio = totalRefFiles > 0 ? named.length / totalRefFiles : 0;
  return {
    K5: {
      namedFileLinks: named.length,
      folderRefs: folderRefs.length,
      totalRefFiles,
      namedRatio: Number(namedRatio.toFixed(4)),
      pass: namedRatio >= K5_MIN_NAMED_RATIO,
      sampleNamed: named.slice(0, 5),
    },
    K6: {
      // routing table = at least one named start file the model can Read directly
      namedStartFiles: named.length,
      pass: named.length >= 1,
    },
  };
}

/** Reference-count consistency: cited per-folder counts vs actual on disk. */
export function checkRefConsistency(body, actualPerFolder) {
  const mismatches = [];
  const seen = {};
  // prose form: references/<folder>/ (N files)   and   `references/<folder>/` (N files)
  const proseRe = /references\/([a-z0-9-]+)\/`?\s*\((\d+)\s*files?\)/gi;
  // table form: `references/<folder>/` | N
  const tableRe = /`references\/([a-z0-9-]+)\/`\s*\|\s*(\d+)/gi;
  for (const re of [proseRe, tableRe]) {
    let m;
    while ((m = re.exec(body)) !== null) {
      const folder = m[1];
      const cited = Number(m[2]);
      seen[folder] = seen[folder] || new Set();
      seen[folder].add(cited);
      const actual = actualPerFolder[folder];
      if (actual !== undefined && cited !== actual) {
        mismatches.push({ folder, cited, actual });
      }
    }
  }
  // also flag folders cited with two different numbers
  for (const [folder, set] of Object.entries(seen)) {
    if (set.size > 1) mismatches.push({ folder, citedMultiple: [...set], actual: actualPerFolder[folder] ?? null });
  }
  return { consistent: mismatches.length === 0, mismatches };
}

/** Light deterministic pre-scan for K9 (time-sensitive tokens in body). */
function scanK9Hints(body) {
  const tokens = body.match(/\b(GA|preview|deprecat\w*|retir\w*|20\d\d|v\d+(?:\.\d+)?)\b/gi) || [];
  return { timeSensitiveTokenHits: tokens.length, sample: [...new Set(tokens)].slice(0, 12) };
}

// --- N1-N5 — deterministic checks from Anthropic "Skill authoring best
// practices" that the K1-K10 rubric did not yet cover. See
// docs/skill-quality-scoring-plan.md §1 for exact rules + sources.

const N1_MAX_NAME_LEN = 64; // best-practices §Skill structure
const N1_RESERVED = ['claude', 'anthropic'];
const N2_MAX_DESC_LEN = 1024; // best-practices §Writing effective descriptions
const N4_LARGE_FILE_LINES = 100; // best-practices §Structure longer reference files

/** N1 — name validity: ≤64 chars, [a-z0-9-], no reserved words. Gerund is
 *  Anthropic-recommended but NOT required, so it is reported, never gated. */
export function checkN1(name) {
  const n = (name || '').trim();
  const withinLength = n.length > 0 && n.length <= N1_MAX_NAME_LEN;
  const validChars = /^[a-z0-9-]+$/.test(n);
  const lower = n.toLowerCase();
  const noReserved = !N1_RESERVED.some((w) => lower.includes(w));
  const gerund = /[a-z]ing(?:-|$)/.test(lower); // informational only
  return { name: n, length: n.length, withinLength, validChars, noReserved, gerund, pass: withinLength && validChars && noReserved };
}

/** N2 — description present and within the 1024-char hard limit. */
export function checkN2(description) {
  const d = description || '';
  const nonEmpty = d.trim().length > 0;
  const withinLimit = d.length <= N2_MAX_DESC_LEN;
  return { length: d.length, nonEmpty, withinLimit, pass: nonEmpty && withinLimit };
}

/** Markdown links to a local .md file (nesting candidates). Skips external
 *  URLs and pure in-page anchors (#section). */
function localMdLinks(content) {
  const out = [];
  const re = /\]\(([^)\s]+?\.md)(?:#[^)]*)?\)/g;
  let m;
  while ((m = re.exec(content)) !== null) {
    const target = m[1];
    if (/^[a-z][a-z0-9+.-]*:\/\//i.test(target)) continue; // external URL
    out.push(target);
  }
  return out;
}

/** N3 — references stay one level deep: a reference file must not link to
 *  another reference (.md) file. Input: [{ path, content }]. */
export function checkN3(refFiles) {
  const nested = [];
  for (const f of refFiles) {
    const links = localMdLinks(f.content);
    if (links.length > 0) nested.push({ file: f.path, links: [...new Set(links)].slice(0, 8) });
  }
  return { checked: refFiles.length, nested, pass: nested.length === 0 };
}

/** Does a reference file carry a table of contents? Either an explicit TOC
 *  heading or a cluster (≥3) of in-page anchor links near the top. */
function hasToc(content) {
  if (/^#{1,3}\s*(table of contents|contents|innhold(?:sfortegnelse)?|toc)\b/im.test(content)) return true;
  const head = content.split('\n').slice(0, 40).join('\n');
  return (head.match(/\]\(#/g) || []).length >= 3;
}

/** N4 — reference files longer than 100 lines should have a table of contents.
 *  Sub-score = fraction of large files that have one (vacuous pass if none). */
export function checkN4(refFiles) {
  const large = refFiles.filter((f) => f.content.replace(/\n+$/, '').split('\n').length > N4_LARGE_FILE_LINES);
  const withToc = large.filter((f) => hasToc(f.content));
  const ratio = large.length > 0 ? Number((withToc.length / large.length).toFixed(4)) : 1;
  return {
    largeFiles: large.length,
    withToc: withToc.length,
    ratio,
    missing: large.filter((f) => !hasToc(f.content)).map((f) => f.path).slice(0, 12),
    pass: large.length === 0 || withToc.length === large.length,
  };
}

/** N5 — forward-slash paths only (no Windows-style backslash paths) in body. */
export function checkN5(body) {
  const patterns = [
    /[A-Za-z]:\\/g, // drive-letter path: C:\
    /[\w-]+\\[\w-]+\.md/g, // backslash path ending in a .md file
    /\\(references|skills|scripts|docs|assets)\b/gi, // backslash before a known dir
  ];
  const hits = new Set();
  for (const re of patterns) for (const m of body.match(re) || []) hits.add(m);
  const windowsPaths = [...hits].slice(0, 12);
  return { windowsPaths, pass: windowsPaths.length === 0 };
}

/**
 * CT5 — sourcedness (deterministic, whole-corpus). REPLACES the LLM-sampled K8
 * (same signal; per ref-kb-direction-note §45 the two must not run in parallel
 * or they double-count and two dashboards diverge). Scope is the Port 1
 * contract: only files declaring `type: reference` are in scope; CT5 is the
 * fraction of those that cite a **Source:**. template/methodology/regulatory are
 * exempt (excluded from the denominator). On an unmigrated corpus no file
 * declares a type, so referenceFiles is 0 and the score layer drops CT5
 * (available:false) — it never tanks the score before migration adopts the
 * contract.
 * @param {Array<{path: string, content: string}>} refFiles
 * @returns {{referenceFiles: number, sourced: number, ratio: number, pass: boolean, unsourced: string[]}}
 */
export function checkCT5(refFiles) {
  let referenceFiles = 0;
  let sourced = 0;
  const unsourced = [];
  for (const f of refFiles) {
    if (parseTypeHeader(f.content) !== 'reference') continue; // out of scope
    referenceFiles++;
    if (parseSourceHeader(f.content)) sourced++;
    else unsourced.push(f.path);
  }
  const ratio = referenceFiles ? sourced / referenceFiles : 0;
  const pass = referenceFiles === 0 ? true : ratio >= CT5_SOURCED_RATIO_TARGET;
  return { referenceFiles, sourced, ratio, pass, unsourced: unsourced.slice(0, 12) };
}

export function evalSkill(skillName) {
  const skillDir = join(SKILLS_DIR, skillName);
  const skillMd = join(skillDir, 'SKILL.md');
  if (!existsSync(skillMd)) return null;
  const content = readFileSync(skillMd, 'utf8');
  const { frontmatter, body } = splitFrontmatter(content);
  const name = extractName(frontmatter);
  const description = extractDescription(frontmatter);
  const refDir = join(skillDir, 'references');
  const refFiles = listMarkdown(refDir);
  const refFileContents = refFiles.map((p) => ({ path: relative(PLUGIN_ROOT, p), content: readFileSync(p, 'utf8') }));
  const perFolder = refCountsPerFolder(refDir);

  const K2 = checkK2(description);
  const K3 = checkK3(body);
  const { K5, K6 } = checkK5K6(body, refFiles.length);
  const refConsistency = checkRefConsistency(body, perFolder);
  const K9hints = scanK9Hints(body);
  const N1 = checkN1(name);
  const N2 = checkN2(description);
  const N3 = checkN3(refFileContents);
  const N4 = checkN4(refFileContents);
  const N5 = checkN5(body);
  const CT5 = checkCT5(refFileContents);

  return {
    name: skillName,
    skillMd: relative(PLUGIN_ROOT, skillMd),
    refFilesActual: refFiles.length,
    refCountsPerFolder: perFolder,
    deterministic: {
      K2_descriptionFormat: K2,
      K3_bodyLength: K3,
      K5_progressiveDisclosure: K5,
      K6_routingTable: K6,
      refCountConsistency: refConsistency,
      K9_timeSensitiveHints: K9hints,
      N1_nameValidity: N1,
      N2_descriptionLength: N2,
      N3_refNestingDepth: N3,
      N4_refToc: N4,
      N5_forwardSlashPaths: N5,
      CT5_sourcedness: CT5,
    },
    // Pointers for the LLM-judge pass (K1/K4/K7/K9). The judge reads the files
    // directly; we only carry the description + sampled ref files here. (K8
    // sourcedness is now the deterministic CT5 above — no longer LLM-sampled.)
    judgeInputs: {
      description,
      bodyLines: K3.bodyLines,
      refFileSample: refFiles.slice(0, 8).map((f) => relative(PLUGIN_ROOT, f)),
    },
    // Filled in by the operator-gated LLM-judge merge step. Null = not yet judged.
    judge: null,
  };
}

/**
 * K10 — sibling-scope-non-overlap. Deterministic, computed ACROSS all skills
 * (unlike K2/K3/K5/K6 which are per-skill), so it is attached after the
 * per-skill pass. Reuses the B1 overlap core: combined = boundaryTension +
 * df-weighted lexical overlap; each skill's K10 verdict is its worst sibling
 * pair (perSkillSiblingOverlap). Mutates + returns the skills array.
 */
export function attachSiblingOverlap(skills, promptSet, { threshold = K10_OVERLAP_THRESHOLD } = {}) {
  const descriptionsBySkill = {};
  for (const s of skills) descriptionsBySkill[s.name] = s.judgeInputs.description;
  const overlap = computeOverlapFromInputs(descriptionsBySkill, promptSet);
  const k10 = perSkillSiblingOverlap(overlap.pairs, { threshold });
  for (const s of skills) {
    s.deterministic.K10_siblingScopeOverlap =
      k10[s.name] || { maxCombined: 0, worstSibling: null, pass: true, threshold };
  }
  return skills;
}

/**
 * Assemble the full corpus report: per-skill deterministic eval (incl. N1-N5) +
 * cross-skill K10 sibling-overlap + merged operator-gated judge cache. This is
 * the object the score-skill CLI consumes via scoreReport(). Reads disk.
 */
export function buildReport() {
  const skillNames = readdirSync(SKILLS_DIR, { withFileTypes: true })
    .filter((e) => e.isDirectory() && existsSync(join(SKILLS_DIR, e.name, 'SKILL.md')))
    .map((e) => e.name)
    .sort();

  const skills = skillNames.map(evalSkill).filter(Boolean);

  // K10 — sibling-scope-non-overlap (deterministic, cross-skill). Attached after
  // the per-skill pass since it needs every description + the curated prompt set.
  if (existsSync(PROMPTS_FILE)) {
    const promptSet = JSON.parse(readFileSync(PROMPTS_FILE, 'utf8'));
    attachSiblingOverlap(skills, promptSet);
  }

  // Merge operator-gated LLM-judge results (K1/K4/K7/K9) if present.
  const judgeFile = join(OUT_DIR, 'judge-results.json');
  if (existsSync(judgeFile)) {
    const jr = JSON.parse(readFileSync(judgeFile, 'utf8'));
    for (const s of skills) if (jr[s.name]) s.judge = jr[s.name];
  }

  return {
    rubric: 'K1-K10',
    note: 'Deterministic: K2,K3,K5,K6,refCountConsistency,K10(siblingScopeOverlap),N1-N5,CT5(sourcedness). LLM-judge (operator-gated): K1,K4,K7,K9.',
    skills,
  };
}

function main() {
  const args = process.argv.slice(2);
  const jsonOut = args.includes('--json');
  const doWrite = args.includes('--write');

  const report = buildReport();
  const skills = report.skills;

  if (doWrite) {
    mkdirSync(OUT_DIR, { recursive: true });
    atomicWriteJson(OUT_FILE, report);
  }

  if (jsonOut) {
    process.stdout.write(JSON.stringify(report) + '\n');
    return;
  }

  // Human-readable summary
  console.log(`\nKB skill eval — ${skills.length} skills (deterministic pass)\n`);
  for (const s of skills) {
    const d = s.deterministic;
    const flag = (b) => (b ? 'PASS' : 'FAIL');
    console.log(`■ ${s.name}  (${s.refFilesActual} ref-filer)`);
    console.log(`   K2 description : ${flag(d.K2_descriptionFormat.pass)} (quoted=${d.K2_descriptionFormat.quotedPhrases}, useWhen=${d.K2_descriptionFormat.useWhenForm})`);
    console.log(`   K3 body ≤500   : ${flag(d.K3_bodyLength.pass)} (${d.K3_bodyLength.bodyLines} linjer)`);
    console.log(`   K5 prog.disc.  : ${flag(d.K5_progressiveDisclosure.pass)} (navngitte=${d.K5_progressiveDisclosure.namedFileLinks}, mapper=${d.K5_progressiveDisclosure.folderRefs}, ratio=${d.K5_progressiveDisclosure.namedRatio})`);
    console.log(`   K6 routing-tab : ${flag(d.K6_routingTable.pass)} (navngitte startfiler=${d.K6_routingTable.namedStartFiles})`);
    const rc = d.refCountConsistency;
    console.log(`   ref-tall       : ${flag(rc.consistent)}${rc.consistent ? '' : ' — ' + JSON.stringify(rc.mismatches)}`);
    const k10 = d.K10_siblingScopeOverlap;
    if (k10) {
      console.log(`   K10 søsken-ovl : ${flag(k10.pass)} (maks ${k10.maxCombined} mot ${k10.worstSibling}, terskel ${k10.threshold})`);
    }
    console.log(`   K9 hints       : ${d.K9_timeSensitiveHints.timeSensitiveTokenHits} tid-sensitive tokens i body`);
    console.log('');
  }
  console.log(`(LLM-judge K1/K4/K7/K9 kjøres som operatør-gated subagent-steg; baseline skrives med --write.)\n`);
}

// Run only when invoked directly (not when imported by tests).
if (process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1]) {
  main();
}