// skill-score.mjs — PURE skill-quality scoring (Spor D). // // Aggregates the K1-K10 rubric (eval.mjs) into a single 0-100 quality score per // skill, using weighted partial-credit with a HARD FLOOR on the load-bearing // criteria (K1 trigger-precision, K10 sibling-overlap): if either fails, the // score is capped below the target — a skill that does not trigger correctly or // overlaps a sibling cannot "pass", regardless of formatting. Grounded in // Anthropic's skill-authoring best practices (description/triggering is // "critical for skill selection"). See docs/skill-quality-scoring-plan.md §2. // // PURE: consumes the object produced by eval.mjs (evalSkill + attachSiblingOverlap // + merged judge results). No disk access -> fully testable with fixtures. // Degrades gracefully when unjudged: judge criteria are excluded from both the // numerator and the denominator, and the score is flagged `provisional` because // the K1 floor cannot be enforced without a judge verdict. export const TARGET = 90; // every skill should sit at or above this export const FLOOR_CAP = 89; // capped value when a load-bearing criterion fails const K5_NAMED_RATIO_TARGET = 0.2; // full credit at/above this (eval.mjs K5 threshold) const clamp01 = (n) => Math.max(0, Math.min(1, n)); /** * Criterion registry. Each entry maps the raw eval object to a 0-1 sub-score. * `read(evalObj)` returns { available, sub, pass, detail }; `available:false` * means the data is absent (e.g. judge criteria on an unjudged skill) and the * criterion is dropped from the weighted mean entirely. * weight — proportional to how load-bearing Anthropic treats the criterion * floor — failing it caps the score at FLOOR_CAP * source — 'det' (always available) | 'judge' (cached, operator-gated) * fix — concrete remediation surfaced in the improvement report */ export const CRITERIA = [ { key: 'K1', label: 'trigger-presisjon', weight: 3, floor: true, source: 'judge', fix: 'Optimaliser description-feltet (kjør trigger-eval: in-domain ≥0.90, out-domain FP ≤0.10).', read: (e) => { const j = e.judge?.K1_triggerPrecision; if (!j) return { available: false }; return { available: true, sub: clamp01(num(j.precision)), pass: !!j.pass, detail: `presisjon ${num(j.precision)}` }; }, }, { key: 'K10', label: 'søsken-ikke-overlapp', weight: 3, floor: true, source: 'det', fix: 'Reduser overlapp mot søster-skill (plan-skill-op merge eller sanitize for å skjerpe scope-grensen).', read: (e) => { const k = e.deterministic?.K10_siblingScopeOverlap; if (!k) return { available: false }; const thr = num(k.threshold) || 7.0; const sub = k.pass ? 1 : clamp01(1 - (num(k.maxCombined) - thr) / thr); return { available: true, sub, pass: !!k.pass, detail: `maks ${num(k.maxCombined)} mot ${k.worstSibling} (terskel ${thr})` }; }, }, { key: 'K3', label: 'body ≤500 linjer', weight: 2, floor: false, source: 'det', fix: 'Trim SKILL.md til ≤500 linjer; flytt detalj til references/ (Anthropic token-budget).', read: (e) => { const k = e.deterministic?.K3_bodyLength; if (!k) return { available: false }; const lines = num(k.bodyLines); const sub = lines <= 500 ? 1 : clamp01(1 - (lines - 500) / 500); return { available: true, sub, pass: !!k.pass, detail: `${lines} linjer` }; }, }, { key: 'K4', label: 'ingen duplisering', weight: 2, floor: false, source: 'judge', fix: 'Fjern detalj fra SKILL.md som dupliserer ref-filer (info bor ett sted).', read: (e) => { const j = e.judge?.K4_noDuplication; if (!j) return { available: false }; return { available: true, sub: clamp01(num(j.score) / 5), pass: !!j.pass, detail: `score ${num(j.score)}/5` }; }, }, { key: 'K9', label: 'ingen volatil tid-info', weight: 2, floor: false, source: 'judge', fix: 'Flytt volatile påstander (GA/preview/versjoner/priser) til ref-filer eller «old patterns».', read: (e) => { const j = e.judge?.K9_noTimeSensitive; if (!j) return { available: false }; return { available: true, sub: j.pass ? 1 : 0, pass: !!j.pass, detail: `${(j.findings || []).length} funn` }; }, }, { key: 'K2', label: 'description-format', weight: 1, floor: false, source: 'det', fix: 'Skriv description i 3.-person «… should be used when …» eller med ≥3 siterte trigger-fraser.', read: (e) => readBinary(e.deterministic?.K2_descriptionFormat), }, { key: 'K5', label: 'progressiv disclosure', weight: 1, floor: false, source: 'det', fix: 'Lenk ref-filer ved navn fra SKILL.md (ikke bare mappe-referanser).', read: (e) => { const k = e.deterministic?.K5_progressiveDisclosure; if (!k) return { available: false }; return { available: true, sub: clamp01(num(k.namedRatio) / K5_NAMED_RATIO_TARGET), pass: !!k.pass, detail: `navngitt-ratio ${num(k.namedRatio)}` }; }, }, { key: 'K6', label: 'routing-pekere', weight: 1, floor: false, source: 'det', fix: 'Legg til minst én navngitt startfil SKILL.md peker til.', read: (e) => readBinary(e.deterministic?.K6_routingTable), }, { key: 'K7', label: 'imperativ stil', weight: 1, floor: false, source: 'judge', fix: 'Skriv instruksjoner i imperativ/infinitiv (verb-først), ikke 2.-person.', read: (e) => { const j = e.judge?.K7_imperativeStyle; if (!j) return { available: false }; return { available: true, sub: clamp01(num(j.ratio)), pass: !!j.pass, detail: `ratio ${num(j.ratio)}` }; }, }, { // CT5 — sourcedness (deterministic, whole-corpus). REPLACES the LLM-sampled // K8 (same signal; must not run in parallel — ref-kb-direction-note §45). // Dormant (available:false) until Port 1 migration declares reference types, // so it never tanks the score on an unmigrated corpus. key: 'CT5', label: 'sourcedness refs', weight: 1, floor: false, source: 'det', fix: 'Gi ref-filene **Source:**-URL (Port 1-kontrakt) — ≥0.80 andel av type:reference-filer.', read: (e) => { const k = e.deterministic?.CT5_sourcedness; if (!k || !k.referenceFiles) return { available: false }; return { available: true, sub: clamp01(num(k.ratio)), pass: !!k.pass, detail: `${num(k.sourced)}/${num(k.referenceFiles)} sourced` }; }, }, { key: 'refCount', label: 'ref-tall-konsistens', weight: 1, floor: false, source: 'det', fix: 'Rett opp ref-tall sitert i SKILL.md som ikke matcher disk.', read: (e) => { const k = e.deterministic?.refCountConsistency; if (!k) return { available: false }; return { available: true, sub: k.consistent ? 1 : 0, pass: !!k.consistent, detail: k.consistent ? 'OK' : `${(k.mismatches || []).length} avvik` }; }, }, { key: 'N1', label: 'name-validitet', weight: 1, floor: false, source: 'det', fix: 'Rett skill-name: ≤64 tegn, kun a-z0-9-, ingen «claude»/«anthropic».', read: (e) => readBinary(e.deterministic?.N1_nameValidity), }, { key: 'N2', label: 'description ≤1024', weight: 1, floor: false, source: 'det', fix: 'Hold description ikke-tom og ≤1024 tegn.', read: (e) => readBinary(e.deterministic?.N2_descriptionLength), }, { key: 'N3', label: 'refs én nivå dypt', weight: 1, floor: false, source: 'det', fix: 'Fjern lenker fra ref-filer til andre ref-filer (unngå nøstede referanser).', read: (e) => { const k = e.deterministic?.N3_refNestingDepth; if (!k) return { available: false }; return { available: true, sub: k.pass ? 1 : 0, pass: !!k.pass, detail: k.pass ? 'OK' : `${(k.nested || []).length} nøstede` }; }, }, { key: 'N4', label: 'TOC i store ref-filer', weight: 1, floor: false, source: 'det', fix: 'Legg innholdsfortegnelse i ref-filer >100 linjer (eksplisitt TOC-heading eller anker-cluster).', read: (e) => { const k = e.deterministic?.N4_refToc; if (!k) return { available: false }; return { available: true, sub: clamp01(num(k.ratio)), pass: !!k.pass, detail: `${num(k.withToc)}/${num(k.largeFiles)} store filer med TOC` }; }, }, { key: 'N5', label: 'forward-slash-stier', weight: 1, floor: false, source: 'det', fix: 'Erstatt Windows-stier (\\) med forward-slash i SKILL.md.', read: (e) => { const k = e.deterministic?.N5_forwardSlashPaths; if (!k) return { available: false }; return { available: true, sub: k.pass ? 1 : 0, pass: !!k.pass, detail: k.pass ? 'OK' : `${(k.windowsPaths || []).length} Windows-stier` }; }, }, ]; function num(v) { const n = Number(v); return Number.isFinite(n) ? n : 0; } /** Binary deterministic criterion: { pass } -> sub 1/0. */ function readBinary(k) { if (!k) return { available: false }; return { available: true, sub: k.pass ? 1 : 0, pass: !!k.pass, detail: k.pass ? 'OK' : 'mangler' }; } /** * Score one skill's eval object into a 0-100 quality score. * @param {object|null} evalObj shape from eval.mjs evalSkill()+judge merge * @param {{target?: number, floorCap?: number}} [opts] * @returns {object|null} null on a missing/garbage object */ export function scoreSkill(evalObj, opts = {}) { if (!evalObj || typeof evalObj !== 'object' || !evalObj.deterministic) return null; const target = opts.target ?? TARGET; const floorCap = opts.floorCap ?? FLOOR_CAP; const criteria = []; let weightedSum = 0; let weightAvail = 0; let judged = false; let floorFailed = false; let provisional = false; for (const c of CRITERIA) { const r = c.read(evalObj); if (!r.available) { // A floor criterion we cannot evaluate (e.g. K1 without a judge) makes the // result provisional — the floor can't be enforced. if (c.floor) provisional = true; criteria.push({ key: c.key, label: c.label, weight: c.weight, floor: c.floor, source: c.source, available: false }); continue; } if (c.source === 'judge') judged = true; const sub = clamp01(r.sub); weightedSum += c.weight * sub; weightAvail += c.weight; if (c.floor && !r.pass) floorFailed = true; criteria.push({ key: c.key, label: c.label, weight: c.weight, floor: c.floor, source: c.source, available: true, sub, pass: !!r.pass, detail: r.detail || '', pointsLost: c.weight * (1 - sub), fix: c.fix, }); } const rawScore = weightAvail > 0 ? Math.round((weightedSum / weightAvail) * 100) : 0; const score = floorFailed ? Math.min(rawScore, floorCap) : rawScore; const improvements = criteria .filter((c) => c.available && c.sub < 1) .sort((a, b) => b.pointsLost - a.pointsLost) .map((c) => ({ key: c.key, label: c.label, detail: c.detail, fix: c.fix, pointsLost: c.pointsLost, floor: c.floor })); return { name: evalObj.name, score, rawScore, floored: score < rawScore, judged, provisional, meetsTarget: score >= target, criteria, improvements, }; } /** * Score every skill in an eval report ({ skills: [...] }) and summarize how many * fall below the target. * @param {{skills: object[]}} report * @param {{target?: number, floorCap?: number}} [opts] * @returns {{target: number, scored: object[], below: object[]}} */ export function scoreReport(report, opts = {}) { const target = opts.target ?? TARGET; const scored = (report?.skills || []).map((s) => scoreSkill(s, opts)).filter(Boolean); const below = scored.filter((s) => !s.meetsTarget); return { target, scored, below }; } /** * Re-score ONE affected skill out of a freshly built corpus report and return a * lifecycle-gate verdict (Spor D, STEG B). Used by apply-skill-op.mjs after a * create/merge/sanitize mutation: the whole corpus is needed for K10 sibling * overlap, but only the affected skill is judged. The deterministic floor (K10) * is enforced immediately via `blocked`; an unjudged skill is `provisional` * (the K1 floor cannot be enforced -> the caller nudges to re-run the judge). * @param {{skills: object[]}} report buildReport() output (fresh disk) * @param {string} skillName the mutated skill to re-score * @param {{target?: number, floorCap?: number}} [opts] * @returns {object} { name, found, score?, meetsTarget?, blocked?, provisional?, judged?, floored?, improvements? } */ export function gateSkill(report, skillName, opts = {}) { const target = opts.target ?? TARGET; const { scored } = scoreReport(report, { ...opts, target }); const s = scored.find((x) => x.name === skillName); if (!s) return { name: skillName, found: false }; return { name: skillName, found: true, score: s.score, meetsTarget: s.meetsTarget, blocked: !s.meetsTarget, provisional: s.provisional, judged: s.judged, floored: s.floored, improvements: s.improvements, }; } /** * Project a scoreReport() result into the compact, PURE cache shape persisted as * data/skill-score-report.json and consumed by the STEG C SessionStart surfacing * (which must read the cache, never score live). Drops the verbose criteria[] and * keeps only what the one-liner needs; `below` is a list of names. Deterministic * given an injected `generatedAt` (no Date.now in the pure layer). * @param {{target: number, scored: object[]}} result scoreReport() output * @param {{generatedAt?: string|null}} [opts] * @returns {{generatedAt: string|null, target: number, scored: object[], below: string[]}} */ export function buildScoreCache(result, opts = {}) { const target = result?.target ?? TARGET; const scored = (result?.scored || []).map((s) => ({ name: s.name, score: s.score, meetsTarget: s.meetsTarget, provisional: s.provisional, floored: s.floored, judged: s.judged, topFix: s.improvements?.[0] ? { key: s.improvements[0].key, label: s.improvements[0].label, fix: s.improvements[0].fix } : null, })); return { generatedAt: opts.generatedAt ?? null, target, scored, below: scored.filter((s) => !s.meetsTarget).map((s) => s.name), }; } /** * Render a human-readable quality report (pure). Each skill shows its score and * status; below-target skills list their sorted improvements (biggest weighted * loss first). Returns a multi-line string. * @param {{target: number, scored: object[]}} result scoreReport() output */ export function formatScoreReport(result) { const target = result?.target ?? TARGET; const scored = result?.scored || []; const below = scored.filter((s) => !s.meetsTarget); const lines = []; lines.push(`Skill-kvalitet — ${scored.length} skills, mål ≥${target} % (${below.length} under mål)`); lines.push(''); for (const s of scored) { const tags = []; if (s.floored) tags.push('floored'); if (s.provisional) tags.push('provisional'); if (!s.judged) tags.push('ujudget'); const status = s.meetsTarget ? 'OK' : 'UNDER MÅL'; const tagStr = tags.length ? ` [${tags.join(', ')}]` : ''; lines.push(`■ ${s.name} ${s.score}/100 ${status}${tagStr}`); if (!s.meetsTarget) { for (const imp of s.improvements) { const floor = imp.floor ? ' ⚑gulv' : ''; lines.push(` ↳ ${imp.key} ${imp.label}${floor} (${imp.detail}) [-${imp.pointsLost.toFixed(2)}] → ${imp.fix}`); } } lines.push(''); } return lines.join('\n'); }