ms-ai-architect/scripts/kb-eval/lib/skill-score.mjs

// skill-score.mjs — PURE skill-quality scoring (Spor D).
//
// Aggregates the K1-K10 rubric (eval.mjs) into a single 0-100 quality score per
// skill, using weighted partial-credit with a HARD FLOOR on the load-bearing
// criteria (K1 trigger-precision, K10 sibling-overlap): if either fails, the
// score is capped below the target — a skill that does not trigger correctly or
// overlaps a sibling cannot "pass", regardless of formatting. Grounded in
// Anthropic's skill-authoring best practices (description/triggering is
// "critical for skill selection"). See docs/skill-quality-scoring-plan.md §2.
//
// PURE: consumes the object produced by eval.mjs (evalSkill + attachSiblingOverlap
// + merged judge results). No disk access -> fully testable with fixtures.
// Degrades gracefully when unjudged: judge criteria are excluded from both the
// numerator and the denominator, and the score is flagged `provisional` because
// the K1 floor cannot be enforced without a judge verdict.

export const TARGET = 90; // every skill should sit at or above this
export const FLOOR_CAP = 89; // capped value when a load-bearing criterion fails
const K5_NAMED_RATIO_TARGET = 0.2; // full credit at/above this (eval.mjs K5 threshold)

const clamp01 = (n) => Math.max(0, Math.min(1, n));

/**
 * Criterion registry. Each entry maps the raw eval object to a 0-1 sub-score.
 * `read(evalObj)` returns { available, sub, pass, detail }; `available:false`
 * means the data is absent (e.g. judge criteria on an unjudged skill) and the
 * criterion is dropped from the weighted mean entirely.
 *   weight  — proportional to how load-bearing Anthropic treats the criterion
 *   floor   — failing it caps the score at FLOOR_CAP
 *   source  — 'det' (always available) | 'judge' (cached, operator-gated)
 *   fix     — concrete remediation surfaced in the improvement report
 */
export const CRITERIA = [
  {
    key: 'K1', label: 'trigger-presisjon', weight: 3, floor: true, source: 'judge',
    fix: 'Optimaliser description-feltet (kjør trigger-eval: in-domain ≥0.90, out-domain FP ≤0.10).',
    read: (e) => {
      const j = e.judge?.K1_triggerPrecision;
      if (!j) return { available: false };
      return { available: true, sub: clamp01(num(j.precision)), pass: !!j.pass, detail: `presisjon ${num(j.precision)}` };
    },
  },
  {
    key: 'K10', label: 'søsken-ikke-overlapp', weight: 3, floor: true, source: 'det',
    fix: 'Reduser overlapp mot søster-skill (plan-skill-op merge eller sanitize for å skjerpe scope-grensen).',
    read: (e) => {
      const k = e.deterministic?.K10_siblingScopeOverlap;
      if (!k) return { available: false };
      const thr = num(k.threshold) || 7.0;
      const sub = k.pass ? 1 : clamp01(1 - (num(k.maxCombined) - thr) / thr);
      return { available: true, sub, pass: !!k.pass, detail: `maks ${num(k.maxCombined)} mot ${k.worstSibling} (terskel ${thr})` };
    },
  },
  {
    key: 'K3', label: 'body ≤500 linjer', weight: 2, floor: false, source: 'det',
    fix: 'Trim SKILL.md til ≤500 linjer; flytt detalj til references/ (Anthropic token-budget).',
    read: (e) => {
      const k = e.deterministic?.K3_bodyLength;
      if (!k) return { available: false };
      const lines = num(k.bodyLines);
      const sub = lines <= 500 ? 1 : clamp01(1 - (lines - 500) / 500);
      return { available: true, sub, pass: !!k.pass, detail: `${lines} linjer` };
    },
  },
  {
    key: 'K4', label: 'ingen duplisering', weight: 2, floor: false, source: 'judge',
    fix: 'Fjern detalj fra SKILL.md som dupliserer ref-filer (info bor ett sted).',
    read: (e) => {
      const j = e.judge?.K4_noDuplication;
      if (!j) return { available: false };
      return { available: true, sub: clamp01(num(j.score) / 5), pass: !!j.pass, detail: `score ${num(j.score)}/5` };
    },
  },
  {
    key: 'K9', label: 'ingen volatil tid-info', weight: 2, floor: false, source: 'judge',
    fix: 'Flytt volatile påstander (GA/preview/versjoner/priser) til ref-filer eller «old patterns».',
    read: (e) => {
      const j = e.judge?.K9_noTimeSensitive;
      if (!j) return { available: false };
      return { available: true, sub: j.pass ? 1 : 0, pass: !!j.pass, detail: `${(j.findings || []).length} funn` };
    },
  },
  {
    key: 'K2', label: 'description-format', weight: 1, floor: false, source: 'det',
    fix: 'Skriv description i 3.-person «… should be used when …» eller med ≥3 siterte trigger-fraser.',
    read: (e) => readBinary(e.deterministic?.K2_descriptionFormat),
  },
  {
    key: 'K5', label: 'progressiv disclosure', weight: 1, floor: false, source: 'det',
    fix: 'Lenk ref-filer ved navn fra SKILL.md (ikke bare mappe-referanser).',
    read: (e) => {
      const k = e.deterministic?.K5_progressiveDisclosure;
      if (!k) return { available: false };
      return { available: true, sub: clamp01(num(k.namedRatio) / K5_NAMED_RATIO_TARGET), pass: !!k.pass, detail: `navngitt-ratio ${num(k.namedRatio)}` };
    },
  },
  {
    key: 'K6', label: 'routing-pekere', weight: 1, floor: false, source: 'det',
    fix: 'Legg til minst én navngitt startfil SKILL.md peker til.',
    read: (e) => readBinary(e.deterministic?.K6_routingTable),
  },
  {
    key: 'K7', label: 'imperativ stil', weight: 1, floor: false, source: 'judge',
    fix: 'Skriv instruksjoner i imperativ/infinitiv (verb-først), ikke 2.-person.',
    read: (e) => {
      const j = e.judge?.K7_imperativeStyle;
      if (!j) return { available: false };
      return { available: true, sub: clamp01(num(j.ratio)), pass: !!j.pass, detail: `ratio ${num(j.ratio)}` };
    },
  },
  {
    // CT5 — sourcedness (deterministic, whole-corpus). REPLACES the LLM-sampled
    // K8 (same signal; must not run in parallel — ref-kb-direction-note §45).
    // Dormant (available:false) until Port 1 migration declares reference types,
    // so it never tanks the score on an unmigrated corpus.
    key: 'CT5', label: 'sourcedness refs', weight: 1, floor: false, source: 'det',
    fix: 'Gi ref-filene **Source:**-URL (Port 1-kontrakt) — ≥0.80 andel av type:reference-filer.',
    read: (e) => {
      const k = e.deterministic?.CT5_sourcedness;
      if (!k || !k.referenceFiles) return { available: false };
      return { available: true, sub: clamp01(num(k.ratio)), pass: !!k.pass, detail: `${num(k.sourced)}/${num(k.referenceFiles)} sourced` };
    },
  },
  {
    key: 'refCount', label: 'ref-tall-konsistens', weight: 1, floor: false, source: 'det',
    fix: 'Rett opp ref-tall sitert i SKILL.md som ikke matcher disk.',
    read: (e) => {
      const k = e.deterministic?.refCountConsistency;
      if (!k) return { available: false };
      return { available: true, sub: k.consistent ? 1 : 0, pass: !!k.consistent, detail: k.consistent ? 'OK' : `${(k.mismatches || []).length} avvik` };
    },
  },
  {
    key: 'N1', label: 'name-validitet', weight: 1, floor: false, source: 'det',
    fix: 'Rett skill-name: ≤64 tegn, kun a-z0-9-, ingen «claude»/«anthropic».',
    read: (e) => readBinary(e.deterministic?.N1_nameValidity),
  },
  {
    key: 'N2', label: 'description ≤1024', weight: 1, floor: false, source: 'det',
    fix: 'Hold description ikke-tom og ≤1024 tegn.',
    read: (e) => readBinary(e.deterministic?.N2_descriptionLength),
  },
  {
    key: 'N3', label: 'refs én nivå dypt', weight: 1, floor: false, source: 'det',
    fix: 'Fjern lenker fra ref-filer til andre ref-filer (unngå nøstede referanser).',
    read: (e) => {
      const k = e.deterministic?.N3_refNestingDepth;
      if (!k) return { available: false };
      return { available: true, sub: k.pass ? 1 : 0, pass: !!k.pass, detail: k.pass ? 'OK' : `${(k.nested || []).length} nøstede` };
    },
  },
  {
    key: 'N4', label: 'TOC i store ref-filer', weight: 1, floor: false, source: 'det',
    fix: 'Legg innholdsfortegnelse i ref-filer >100 linjer (eksplisitt TOC-heading eller anker-cluster).',
    read: (e) => {
      const k = e.deterministic?.N4_refToc;
      if (!k) return { available: false };
      return { available: true, sub: clamp01(num(k.ratio)), pass: !!k.pass, detail: `${num(k.withToc)}/${num(k.largeFiles)} store filer med TOC` };
    },
  },
  {
    key: 'N5', label: 'forward-slash-stier', weight: 1, floor: false, source: 'det',
    fix: 'Erstatt Windows-stier (\\) med forward-slash i SKILL.md.',
    read: (e) => {
      const k = e.deterministic?.N5_forwardSlashPaths;
      if (!k) return { available: false };
      return { available: true, sub: k.pass ? 1 : 0, pass: !!k.pass, detail: k.pass ? 'OK' : `${(k.windowsPaths || []).length} Windows-stier` };
    },
  },
];

function num(v) {
  const n = Number(v);
  return Number.isFinite(n) ? n : 0;
}

/** Binary deterministic criterion: { pass } -> sub 1/0. */
function readBinary(k) {
  if (!k) return { available: false };
  return { available: true, sub: k.pass ? 1 : 0, pass: !!k.pass, detail: k.pass ? 'OK' : 'mangler' };
}

/**
 * Score one skill's eval object into a 0-100 quality score.
 * @param {object|null} evalObj  shape from eval.mjs evalSkill()+judge merge
 * @param {{target?: number, floorCap?: number}} [opts]
 * @returns {object|null} null on a missing/garbage object
 */
export function scoreSkill(evalObj, opts = {}) {
  if (!evalObj || typeof evalObj !== 'object' || !evalObj.deterministic) return null;
  const target = opts.target ?? TARGET;
  const floorCap = opts.floorCap ?? FLOOR_CAP;

  const criteria = [];
  let weightedSum = 0;
  let weightAvail = 0;
  let judged = false;
  let floorFailed = false;
  let provisional = false;

  for (const c of CRITERIA) {
    const r = c.read(evalObj);
    if (!r.available) {
      // A floor criterion we cannot evaluate (e.g. K1 without a judge) makes the
      // result provisional — the floor can't be enforced.
      if (c.floor) provisional = true;
      criteria.push({ key: c.key, label: c.label, weight: c.weight, floor: c.floor, source: c.source, available: false });
      continue;
    }
    if (c.source === 'judge') judged = true;
    const sub = clamp01(r.sub);
    weightedSum += c.weight * sub;
    weightAvail += c.weight;
    if (c.floor && !r.pass) floorFailed = true;
    criteria.push({
      key: c.key, label: c.label, weight: c.weight, floor: c.floor, source: c.source,
      available: true, sub, pass: !!r.pass, detail: r.detail || '',
      pointsLost: c.weight * (1 - sub), fix: c.fix,
    });
  }

  const rawScore = weightAvail > 0 ? Math.round((weightedSum / weightAvail) * 100) : 0;
  const score = floorFailed ? Math.min(rawScore, floorCap) : rawScore;

  const improvements = criteria
    .filter((c) => c.available && c.sub < 1)
    .sort((a, b) => b.pointsLost - a.pointsLost)
    .map((c) => ({ key: c.key, label: c.label, detail: c.detail, fix: c.fix, pointsLost: c.pointsLost, floor: c.floor }));

  return {
    name: evalObj.name,
    score,
    rawScore,
    floored: score < rawScore,
    judged,
    provisional,
    meetsTarget: score >= target,
    criteria,
    improvements,
  };
}

/**
 * Score every skill in an eval report ({ skills: [...] }) and summarize how many
 * fall below the target.
 * @param {{skills: object[]}} report
 * @param {{target?: number, floorCap?: number}} [opts]
 * @returns {{target: number, scored: object[], below: object[]}}
 */
export function scoreReport(report, opts = {}) {
  const target = opts.target ?? TARGET;
  const scored = (report?.skills || []).map((s) => scoreSkill(s, opts)).filter(Boolean);
  const below = scored.filter((s) => !s.meetsTarget);
  return { target, scored, below };
}

/**
 * Re-score ONE affected skill out of a freshly built corpus report and return a
 * lifecycle-gate verdict (Spor D, STEG B). Used by apply-skill-op.mjs after a
 * create/merge/sanitize mutation: the whole corpus is needed for K10 sibling
 * overlap, but only the affected skill is judged. The deterministic floor (K10)
 * is enforced immediately via `blocked`; an unjudged skill is `provisional`
 * (the K1 floor cannot be enforced -> the caller nudges to re-run the judge).
 * @param {{skills: object[]}} report  buildReport() output (fresh disk)
 * @param {string} skillName           the mutated skill to re-score
 * @param {{target?: number, floorCap?: number}} [opts]
 * @returns {object} { name, found, score?, meetsTarget?, blocked?, provisional?, judged?, floored?, improvements? }
 */
export function gateSkill(report, skillName, opts = {}) {
  const target = opts.target ?? TARGET;
  const { scored } = scoreReport(report, { ...opts, target });
  const s = scored.find((x) => x.name === skillName);
  if (!s) return { name: skillName, found: false };
  return {
    name: skillName,
    found: true,
    score: s.score,
    meetsTarget: s.meetsTarget,
    blocked: !s.meetsTarget,
    provisional: s.provisional,
    judged: s.judged,
    floored: s.floored,
    improvements: s.improvements,
  };
}

/**
 * Project a scoreReport() result into the compact, PURE cache shape persisted as
 * data/skill-score-report.json and consumed by the STEG C SessionStart surfacing
 * (which must read the cache, never score live). Drops the verbose criteria[] and
 * keeps only what the one-liner needs; `below` is a list of names. Deterministic
 * given an injected `generatedAt` (no Date.now in the pure layer).
 * @param {{target: number, scored: object[]}} result  scoreReport() output
 * @param {{generatedAt?: string|null}} [opts]
 * @returns {{generatedAt: string|null, target: number, scored: object[], below: string[]}}
 */
export function buildScoreCache(result, opts = {}) {
  const target = result?.target ?? TARGET;
  const scored = (result?.scored || []).map((s) => ({
    name: s.name,
    score: s.score,
    meetsTarget: s.meetsTarget,
    provisional: s.provisional,
    floored: s.floored,
    judged: s.judged,
    topFix: s.improvements?.[0]
      ? { key: s.improvements[0].key, label: s.improvements[0].label, fix: s.improvements[0].fix }
      : null,
  }));
  return {
    generatedAt: opts.generatedAt ?? null,
    target,
    scored,
    below: scored.filter((s) => !s.meetsTarget).map((s) => s.name),
  };
}

/**
 * Render a human-readable quality report (pure). Each skill shows its score and
 * status; below-target skills list their sorted improvements (biggest weighted
 * loss first). Returns a multi-line string.
 * @param {{target: number, scored: object[]}} result  scoreReport() output
 */
export function formatScoreReport(result) {
  const target = result?.target ?? TARGET;
  const scored = result?.scored || [];
  const below = scored.filter((s) => !s.meetsTarget);
  const lines = [];
  lines.push(`Skill-kvalitet — ${scored.length} skills, mål ≥${target} %  (${below.length} under mål)`);
  lines.push('');
  for (const s of scored) {
    const tags = [];
    if (s.floored) tags.push('floored');
    if (s.provisional) tags.push('provisional');
    if (!s.judged) tags.push('ujudget');
    const status = s.meetsTarget ? 'OK' : 'UNDER MÅL';
    const tagStr = tags.length ? ` [${tags.join(', ')}]` : '';
    lines.push(`■ ${s.name}  ${s.score}/100  ${status}${tagStr}`);
    if (!s.meetsTarget) {
      for (const imp of s.improvements) {
        const floor = imp.floor ? ' ⚑gulv' : '';
        lines.push(`   ↳ ${imp.key} ${imp.label}${floor} (${imp.detail}) [-${imp.pointsLost.toFixed(2)}] → ${imp.fix}`);
      }
    }
    lines.push('');
  }
  return lines.join('\n');
}