// skill-score.mjs — PURE skill-quality scoring (Spor D). // // Aggregates the K1-K10 rubric (eval.mjs) into a single 0-100 quality score per // skill, using weighted partial-credit with a HARD FLOOR on the load-bearing // criteria (K1 trigger-precision, K10 sibling-overlap): if either fails, the // score is capped below the target — a skill that does not trigger correctly or // overlaps a sibling cannot "pass", regardless of formatting. Grounded in // Anthropic's skill-authoring best practices (description/triggering is // "critical for skill selection"). See docs/skill-quality-scoring-plan.md §2. // // PURE: consumes the object produced by eval.mjs (evalSkill + attachSiblingOverlap // + merged judge results). No disk access -> fully testable with fixtures. // Degrades gracefully when unjudged: judge criteria are excluded from both the // numerator and the denominator, and the score is flagged `provisional` because // the K1 floor cannot be enforced without a judge verdict. export const TARGET = 90; // every skill should sit at or above this export const FLOOR_CAP = 89; // capped value when a load-bearing criterion fails const K5_NAMED_RATIO_TARGET = 0.2; // full credit at/above this (eval.mjs K5 threshold) const clamp01 = (n) => Math.max(0, Math.min(1, n)); /** * Criterion registry. Each entry maps the raw eval object to a 0-1 sub-score. * `read(evalObj)` returns { available, sub, pass, detail }; `available:false` * means the data is absent (e.g. judge criteria on an unjudged skill) and the * criterion is dropped from the weighted mean entirely. * weight — proportional to how load-bearing Anthropic treats the criterion * floor — failing it caps the score at FLOOR_CAP * source — 'det' (always available) | 'judge' (cached, operator-gated) * fix — concrete remediation surfaced in the improvement report */ export const CRITERIA = [ { key: 'K1', label: 'trigger-presisjon', weight: 3, floor: true, source: 'judge', fix: 'Optimaliser description-feltet (kjør trigger-eval: in-domain ≥0.90, out-domain FP ≤0.10).', read: (e) => { const j = e.judge?.K1_triggerPrecision; if (!j) return { available: false }; return { available: true, sub: clamp01(num(j.precision)), pass: !!j.pass, detail: `presisjon ${num(j.precision)}` }; }, }, { key: 'K10', label: 'søsken-ikke-overlapp', weight: 3, floor: true, source: 'det', fix: 'Reduser overlapp mot søster-skill (plan-skill-op merge eller sanitize for å skjerpe scope-grensen).', read: (e) => { const k = e.deterministic?.K10_siblingScopeOverlap; if (!k) return { available: false }; const thr = num(k.threshold) || 7.0; const sub = k.pass ? 1 : clamp01(1 - (num(k.maxCombined) - thr) / thr); return { available: true, sub, pass: !!k.pass, detail: `maks ${num(k.maxCombined)} mot ${k.worstSibling} (terskel ${thr})` }; }, }, { key: 'K3', label: 'body ≤500 linjer', weight: 2, floor: false, source: 'det', fix: 'Trim SKILL.md til ≤500 linjer; flytt detalj til references/ (Anthropic token-budget).', read: (e) => { const k = e.deterministic?.K3_bodyLength; if (!k) return { available: false }; const lines = num(k.bodyLines); const sub = lines <= 500 ? 1 : clamp01(1 - (lines - 500) / 500); return { available: true, sub, pass: !!k.pass, detail: `${lines} linjer` }; }, }, { key: 'K4', label: 'ingen duplisering', weight: 2, floor: false, source: 'judge', fix: 'Fjern detalj fra SKILL.md som dupliserer ref-filer (info bor ett sted).', read: (e) => { const j = e.judge?.K4_noDuplication; if (!j) return { available: false }; return { available: true, sub: clamp01(num(j.score) / 5), pass: !!j.pass, detail: `score ${num(j.score)}/5` }; }, }, { key: 'K9', label: 'ingen volatil tid-info', weight: 2, floor: false, source: 'judge', fix: 'Flytt volatile påstander (GA/preview/versjoner/priser) til ref-filer eller «old patterns».', read: (e) => { const j = e.judge?.K9_noTimeSensitive; if (!j) return { available: false }; return { available: true, sub: j.pass ? 1 : 0, pass: !!j.pass, detail: `${(j.findings || []).length} funn` }; }, }, { key: 'K2', label: 'description-format', weight: 1, floor: false, source: 'det', fix: 'Skriv description i 3.-person «… should be used when …» eller med ≥3 siterte trigger-fraser.', read: (e) => readBinary(e.deterministic?.K2_descriptionFormat), }, { key: 'K5', label: 'progressiv disclosure', weight: 1, floor: false, source: 'det', fix: 'Lenk ref-filer ved navn fra SKILL.md (ikke bare mappe-referanser).', read: (e) => { const k = e.deterministic?.K5_progressiveDisclosure; if (!k) return { available: false }; return { available: true, sub: clamp01(num(k.namedRatio) / K5_NAMED_RATIO_TARGET), pass: !!k.pass, detail: `navngitt-ratio ${num(k.namedRatio)}` }; }, }, { key: 'K6', label: 'routing-pekere', weight: 1, floor: false, source: 'det', fix: 'Legg til minst én navngitt startfil SKILL.md peker til.', read: (e) => readBinary(e.deterministic?.K6_routingTable), }, { key: 'K7', label: 'imperativ stil', weight: 1, floor: false, source: 'judge', fix: 'Skriv instruksjoner i imperativ/infinitiv (verb-først), ikke 2.-person.', read: (e) => { const j = e.judge?.K7_imperativeStyle; if (!j) return { available: false }; return { available: true, sub: clamp01(num(j.ratio)), pass: !!j.pass, detail: `ratio ${num(j.ratio)}` }; }, }, { key: 'K8', label: 'kildehenvisning refs', weight: 1, floor: false, source: 'judge', fix: 'Legg dated/Verified-header eller kilde-URL i ref-filene (≥0.80 andel).', read: (e) => { const j = e.judge?.K8_sourceCitation; if (!j) return { available: false }; return { available: true, sub: clamp01(num(j.ratio)), pass: !!j.pass, detail: `ratio ${num(j.ratio)}` }; }, }, { key: 'refCount', label: 'ref-tall-konsistens', weight: 1, floor: false, source: 'det', fix: 'Rett opp ref-tall sitert i SKILL.md som ikke matcher disk.', read: (e) => { const k = e.deterministic?.refCountConsistency; if (!k) return { available: false }; return { available: true, sub: k.consistent ? 1 : 0, pass: !!k.consistent, detail: k.consistent ? 'OK' : `${(k.mismatches || []).length} avvik` }; }, }, ]; function num(v) { const n = Number(v); return Number.isFinite(n) ? n : 0; } /** Binary deterministic criterion: { pass } -> sub 1/0. */ function readBinary(k) { if (!k) return { available: false }; return { available: true, sub: k.pass ? 1 : 0, pass: !!k.pass, detail: k.pass ? 'OK' : 'mangler' }; } /** * Score one skill's eval object into a 0-100 quality score. * @param {object|null} evalObj shape from eval.mjs evalSkill()+judge merge * @param {{target?: number, floorCap?: number}} [opts] * @returns {object|null} null on a missing/garbage object */ export function scoreSkill(evalObj, opts = {}) { if (!evalObj || typeof evalObj !== 'object' || !evalObj.deterministic) return null; const target = opts.target ?? TARGET; const floorCap = opts.floorCap ?? FLOOR_CAP; const criteria = []; let weightedSum = 0; let weightAvail = 0; let judged = false; let floorFailed = false; let provisional = false; for (const c of CRITERIA) { const r = c.read(evalObj); if (!r.available) { // A floor criterion we cannot evaluate (e.g. K1 without a judge) makes the // result provisional — the floor can't be enforced. if (c.floor) provisional = true; criteria.push({ key: c.key, label: c.label, weight: c.weight, floor: c.floor, source: c.source, available: false }); continue; } if (c.source === 'judge') judged = true; const sub = clamp01(r.sub); weightedSum += c.weight * sub; weightAvail += c.weight; if (c.floor && !r.pass) floorFailed = true; criteria.push({ key: c.key, label: c.label, weight: c.weight, floor: c.floor, source: c.source, available: true, sub, pass: !!r.pass, detail: r.detail || '', pointsLost: c.weight * (1 - sub), fix: c.fix, }); } const rawScore = weightAvail > 0 ? Math.round((weightedSum / weightAvail) * 100) : 0; const score = floorFailed ? Math.min(rawScore, floorCap) : rawScore; const improvements = criteria .filter((c) => c.available && c.sub < 1) .sort((a, b) => b.pointsLost - a.pointsLost) .map((c) => ({ key: c.key, label: c.label, detail: c.detail, fix: c.fix, pointsLost: c.pointsLost, floor: c.floor })); return { name: evalObj.name, score, rawScore, floored: score < rawScore, judged, provisional, meetsTarget: score >= target, criteria, improvements, }; } /** * Score every skill in an eval report ({ skills: [...] }) and summarize how many * fall below the target. * @param {{skills: object[]}} report * @param {{target?: number, floorCap?: number}} [opts] * @returns {{target: number, scored: object[], below: object[]}} */ export function scoreReport(report, opts = {}) { const target = opts.target ?? TARGET; const scored = (report?.skills || []).map((s) => scoreSkill(s, opts)).filter(Boolean); const below = scored.filter((s) => !s.meetsTarget); return { target, scored, below }; }