ms-ai-architect/scripts/kb-eval/lib/skill-score.mjs

// skill-score.mjs — PURE skill-quality scoring (Spor D).
//
// Aggregates the K1-K10 rubric (eval.mjs) into a single 0-100 quality score per
// skill, using weighted partial-credit with a HARD FLOOR on the load-bearing
// criteria (K1 trigger-precision, K10 sibling-overlap): if either fails, the
// score is capped below the target — a skill that does not trigger correctly or
// overlaps a sibling cannot "pass", regardless of formatting. Grounded in
// Anthropic's skill-authoring best practices (description/triggering is
// "critical for skill selection"). See docs/skill-quality-scoring-plan.md §2.
//
// PURE: consumes the object produced by eval.mjs (evalSkill + attachSiblingOverlap
// + merged judge results). No disk access -> fully testable with fixtures.
// Degrades gracefully when unjudged: judge criteria are excluded from both the
// numerator and the denominator, and the score is flagged `provisional` because
// the K1 floor cannot be enforced without a judge verdict.

export const TARGET = 90; // every skill should sit at or above this
export const FLOOR_CAP = 89; // capped value when a load-bearing criterion fails
const K5_NAMED_RATIO_TARGET = 0.2; // full credit at/above this (eval.mjs K5 threshold)

const clamp01 = (n) => Math.max(0, Math.min(1, n));

/**
 * Criterion registry. Each entry maps the raw eval object to a 0-1 sub-score.
 * `read(evalObj)` returns { available, sub, pass, detail }; `available:false`
 * means the data is absent (e.g. judge criteria on an unjudged skill) and the
 * criterion is dropped from the weighted mean entirely.
 *   weight  — proportional to how load-bearing Anthropic treats the criterion
 *   floor   — failing it caps the score at FLOOR_CAP
 *   source  — 'det' (always available) | 'judge' (cached, operator-gated)
 *   fix     — concrete remediation surfaced in the improvement report
 */
export const CRITERIA = [
  {
    key: 'K1', label: 'trigger-presisjon', weight: 3, floor: true, source: 'judge',
    fix: 'Optimaliser description-feltet (kjør trigger-eval: in-domain ≥0.90, out-domain FP ≤0.10).',
    read: (e) => {
      const j = e.judge?.K1_triggerPrecision;
      if (!j) return { available: false };
      return { available: true, sub: clamp01(num(j.precision)), pass: !!j.pass, detail: `presisjon ${num(j.precision)}` };
    },
  },
  {
    key: 'K10', label: 'søsken-ikke-overlapp', weight: 3, floor: true, source: 'det',
    fix: 'Reduser overlapp mot søster-skill (plan-skill-op merge eller sanitize for å skjerpe scope-grensen).',
    read: (e) => {
      const k = e.deterministic?.K10_siblingScopeOverlap;
      if (!k) return { available: false };
      const thr = num(k.threshold) || 7.0;
      const sub = k.pass ? 1 : clamp01(1 - (num(k.maxCombined) - thr) / thr);
      return { available: true, sub, pass: !!k.pass, detail: `maks ${num(k.maxCombined)} mot ${k.worstSibling} (terskel ${thr})` };
    },
  },
  {
    key: 'K3', label: 'body ≤500 linjer', weight: 2, floor: false, source: 'det',
    fix: 'Trim SKILL.md til ≤500 linjer; flytt detalj til references/ (Anthropic token-budget).',
    read: (e) => {
      const k = e.deterministic?.K3_bodyLength;
      if (!k) return { available: false };
      const lines = num(k.bodyLines);
      const sub = lines <= 500 ? 1 : clamp01(1 - (lines - 500) / 500);
      return { available: true, sub, pass: !!k.pass, detail: `${lines} linjer` };
    },
  },
  {
    key: 'K4', label: 'ingen duplisering', weight: 2, floor: false, source: 'judge',
    fix: 'Fjern detalj fra SKILL.md som dupliserer ref-filer (info bor ett sted).',
    read: (e) => {
      const j = e.judge?.K4_noDuplication;
      if (!j) return { available: false };
      return { available: true, sub: clamp01(num(j.score) / 5), pass: !!j.pass, detail: `score ${num(j.score)}/5` };
    },
  },
  {
    key: 'K9', label: 'ingen volatil tid-info', weight: 2, floor: false, source: 'judge',
    fix: 'Flytt volatile påstander (GA/preview/versjoner/priser) til ref-filer eller «old patterns».',
    read: (e) => {
      const j = e.judge?.K9_noTimeSensitive;
      if (!j) return { available: false };
      return { available: true, sub: j.pass ? 1 : 0, pass: !!j.pass, detail: `${(j.findings || []).length} funn` };
    },
  },
  {
    key: 'K2', label: 'description-format', weight: 1, floor: false, source: 'det',
    fix: 'Skriv description i 3.-person «… should be used when …» eller med ≥3 siterte trigger-fraser.',
    read: (e) => readBinary(e.deterministic?.K2_descriptionFormat),
  },
  {
    key: 'K5', label: 'progressiv disclosure', weight: 1, floor: false, source: 'det',
    fix: 'Lenk ref-filer ved navn fra SKILL.md (ikke bare mappe-referanser).',
    read: (e) => {
      const k = e.deterministic?.K5_progressiveDisclosure;
      if (!k) return { available: false };
      return { available: true, sub: clamp01(num(k.namedRatio) / K5_NAMED_RATIO_TARGET), pass: !!k.pass, detail: `navngitt-ratio ${num(k.namedRatio)}` };
    },
  },
  {
    key: 'K6', label: 'routing-pekere', weight: 1, floor: false, source: 'det',
    fix: 'Legg til minst én navngitt startfil SKILL.md peker til.',
    read: (e) => readBinary(e.deterministic?.K6_routingTable),
  },
  {
    key: 'K7', label: 'imperativ stil', weight: 1, floor: false, source: 'judge',
    fix: 'Skriv instruksjoner i imperativ/infinitiv (verb-først), ikke 2.-person.',
    read: (e) => {
      const j = e.judge?.K7_imperativeStyle;
      if (!j) return { available: false };
      return { available: true, sub: clamp01(num(j.ratio)), pass: !!j.pass, detail: `ratio ${num(j.ratio)}` };
    },
  },
  {
    key: 'K8', label: 'kildehenvisning refs', weight: 1, floor: false, source: 'judge',
    fix: 'Legg dated/Verified-header eller kilde-URL i ref-filene (≥0.80 andel).',
    read: (e) => {
      const j = e.judge?.K8_sourceCitation;
      if (!j) return { available: false };
      return { available: true, sub: clamp01(num(j.ratio)), pass: !!j.pass, detail: `ratio ${num(j.ratio)}` };
    },
  },
  {
    key: 'refCount', label: 'ref-tall-konsistens', weight: 1, floor: false, source: 'det',
    fix: 'Rett opp ref-tall sitert i SKILL.md som ikke matcher disk.',
    read: (e) => {
      const k = e.deterministic?.refCountConsistency;
      if (!k) return { available: false };
      return { available: true, sub: k.consistent ? 1 : 0, pass: !!k.consistent, detail: k.consistent ? 'OK' : `${(k.mismatches || []).length} avvik` };
    },
  },
];

function num(v) {
  const n = Number(v);
  return Number.isFinite(n) ? n : 0;
}

/** Binary deterministic criterion: { pass } -> sub 1/0. */
function readBinary(k) {
  if (!k) return { available: false };
  return { available: true, sub: k.pass ? 1 : 0, pass: !!k.pass, detail: k.pass ? 'OK' : 'mangler' };
}

/**
 * Score one skill's eval object into a 0-100 quality score.
 * @param {object|null} evalObj  shape from eval.mjs evalSkill()+judge merge
 * @param {{target?: number, floorCap?: number}} [opts]
 * @returns {object|null} null on a missing/garbage object
 */
export function scoreSkill(evalObj, opts = {}) {
  if (!evalObj || typeof evalObj !== 'object' || !evalObj.deterministic) return null;
  const target = opts.target ?? TARGET;
  const floorCap = opts.floorCap ?? FLOOR_CAP;

  const criteria = [];
  let weightedSum = 0;
  let weightAvail = 0;
  let judged = false;
  let floorFailed = false;
  let provisional = false;

  for (const c of CRITERIA) {
    const r = c.read(evalObj);
    if (!r.available) {
      // A floor criterion we cannot evaluate (e.g. K1 without a judge) makes the
      // result provisional — the floor can't be enforced.
      if (c.floor) provisional = true;
      criteria.push({ key: c.key, label: c.label, weight: c.weight, floor: c.floor, source: c.source, available: false });
      continue;
    }
    if (c.source === 'judge') judged = true;
    const sub = clamp01(r.sub);
    weightedSum += c.weight * sub;
    weightAvail += c.weight;
    if (c.floor && !r.pass) floorFailed = true;
    criteria.push({
      key: c.key, label: c.label, weight: c.weight, floor: c.floor, source: c.source,
      available: true, sub, pass: !!r.pass, detail: r.detail || '',
      pointsLost: c.weight * (1 - sub), fix: c.fix,
    });
  }

  const rawScore = weightAvail > 0 ? Math.round((weightedSum / weightAvail) * 100) : 0;
  const score = floorFailed ? Math.min(rawScore, floorCap) : rawScore;

  const improvements = criteria
    .filter((c) => c.available && c.sub < 1)
    .sort((a, b) => b.pointsLost - a.pointsLost)
    .map((c) => ({ key: c.key, label: c.label, detail: c.detail, fix: c.fix, pointsLost: c.pointsLost, floor: c.floor }));

  return {
    name: evalObj.name,
    score,
    rawScore,
    floored: score < rawScore,
    judged,
    provisional,
    meetsTarget: score >= target,
    criteria,
    improvements,
  };
}

/**
 * Score every skill in an eval report ({ skills: [...] }) and summarize how many
 * fall below the target.
 * @param {{skills: object[]}} report
 * @param {{target?: number, floorCap?: number}} [opts]
 * @returns {{target: number, scored: object[], below: object[]}}
 */
export function scoreReport(report, opts = {}) {
  const target = opts.target ?? TARGET;
  const scored = (report?.skills || []).map((s) => scoreSkill(s, opts)).filter(Boolean);
  const below = scored.filter((s) => !s.meetsTarget);
  return { target, scored, below };
}