feat(ms-ai-architect): Spor D steg 2 — score-skill CLI (--json/--gate/--skill) (TDD)

Byggekloss for både SessionStart-surfacing og lifecycle-gate (D3).

- eval.mjs: ekstraher buildReport() (eksportert, testbar) ut av main() —
  samler hele korpus-rapporten (deterministisk + K10 + merget judge-cache).
- lib/skill-score.mjs: formatScoreReport() (ren) — sortert forbedringsrapport
  med score, status, floored/provisional/ujudget-tags og ⚑gulv-markering.
- score-skill.mjs (NY CLI): buildReport → scoreReport → render/json/gate.
  --skill scorer hele korpuset (K10 trenger alle søsken) men rapporterer/gater
  kun den valgte → inkrementell enkelt-skill-scoring for D3 lifecycle-gate.
- docs/development.md: ny «Skill-kvalitetsscore (Spor D)»-seksjon.

Live baseline: advisor 91, governance/security 96 (OK); engineering +
infrastructure 89 (floored av K10-gulv) UNDER mål. Gate exit=1.

Tester: kb-eval 134→142 (+8). validate 239, kb-update 316 uendret grønt.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Kjell Tore Guttormsen 2026-06-23 17:16:44 +02:00
commit c1a09062d4
5 changed files with 249 additions and 7 deletions

View file

@ -334,11 +334,12 @@ export function attachSiblingOverlap(skills, promptSet, { threshold = K10_OVERLA
return skills;
}
function main() {
const args = process.argv.slice(2);
const jsonOut = args.includes('--json');
const doWrite = args.includes('--write');
/**
* Assemble the full corpus report: per-skill deterministic eval (incl. N1-N5) +
* cross-skill K10 sibling-overlap + merged operator-gated judge cache. This is
* the object the score-skill CLI consumes via scoreReport(). Reads disk.
*/
export function buildReport() {
const skillNames = readdirSync(SKILLS_DIR, { withFileTypes: true })
.filter((e) => e.isDirectory() && existsSync(join(SKILLS_DIR, e.name, 'SKILL.md')))
.map((e) => e.name)
@ -360,11 +361,20 @@ function main() {
for (const s of skills) if (jr[s.name]) s.judge = jr[s.name];
}
const report = {
return {
rubric: 'K1-K10',
note: 'Deterministic: K2,K3,K5,K6,refCountConsistency,K10(siblingScopeOverlap). LLM-judge (operator-gated): K1,K4,K7,K8,K9.',
note: 'Deterministic: K2,K3,K5,K6,refCountConsistency,K10(siblingScopeOverlap),N1-N5. LLM-judge (operator-gated): K1,K4,K7,K8,K9.',
skills,
};
}
function main() {
const args = process.argv.slice(2);
const jsonOut = args.includes('--json');
const doWrite = args.includes('--write');
const report = buildReport();
const skills = report.skills;
if (doWrite) {
mkdirSync(OUT_DIR, { recursive: true });

View file

@ -249,3 +249,35 @@ export function scoreReport(report, opts = {}) {
const below = scored.filter((s) => !s.meetsTarget);
return { target, scored, below };
}
/**
* Render a human-readable quality report (pure). Each skill shows its score and
* status; below-target skills list their sorted improvements (biggest weighted
* loss first). Returns a multi-line string.
* @param {{target: number, scored: object[]}} result scoreReport() output
*/
export function formatScoreReport(result) {
const target = result?.target ?? TARGET;
const scored = result?.scored || [];
const below = scored.filter((s) => !s.meetsTarget);
const lines = [];
lines.push(`Skill-kvalitet — ${scored.length} skills, mål ≥${target} % (${below.length} under mål)`);
lines.push('');
for (const s of scored) {
const tags = [];
if (s.floored) tags.push('floored');
if (s.provisional) tags.push('provisional');
if (!s.judged) tags.push('ujudget');
const status = s.meetsTarget ? 'OK' : 'UNDER MÅL';
const tagStr = tags.length ? ` [${tags.join(', ')}]` : '';
lines.push(`${s.name} ${s.score}/100 ${status}${tagStr}`);
if (!s.meetsTarget) {
for (const imp of s.improvements) {
const floor = imp.floor ? ' ⚑gulv' : '';
lines.push(`${imp.key} ${imp.label}${floor} (${imp.detail}) [-${imp.pointsLost.toFixed(2)}] → ${imp.fix}`);
}
}
lines.push('');
}
return lines.join('\n');
}

View file

@ -0,0 +1,72 @@
#!/usr/bin/env node
// score-skill.mjs — Skill-quality score CLI (Spor D, step 2).
//
// Runs the eval pipeline (buildReport: deterministic K2/K3/K5/K6/refCount/N1-N5
// + cross-skill K10 + merged operator-gated judge cache), scores every skill via
// the PURE lib (scoreSkill/scoreReport), and reports a 0-100 quality score with
// a sorted improvement list. The corpus invariant is: every skill stays >=90 %.
//
// Usage:
// node scripts/kb-eval/score-skill.mjs # human summary
// node scripts/kb-eval/score-skill.mjs --json # machine output (scoreReport)
// node scripts/kb-eval/score-skill.mjs --gate 90 # non-zero exit if any skill < 90
// node scripts/kb-eval/score-skill.mjs --skill <name> # scope output to one skill
// # (still scores the full corpus
// # so K10 sibling-overlap is correct)
//
// K10 requires every sibling description, so even single-skill (incremental)
// scoring evaluates the whole corpus and then filters the reported set — this is
// the building block for the lifecycle-gate in apply-skill-op.mjs.
//
// Zero dependencies. Pure scoring lives in lib/skill-score.mjs.
import { fileURLToPath } from 'node:url';
import { buildReport } from './eval.mjs';
import { scoreReport, formatScoreReport, TARGET } from './lib/skill-score.mjs';
function argValue(args, flag) {
const i = args.indexOf(flag);
return i !== -1 && i + 1 < args.length ? args[i + 1] : null;
}
function main() {
const args = process.argv.slice(2);
const jsonOut = args.includes('--json');
const gateRaw = argValue(args, '--gate');
const gate = gateRaw !== null ? Number(gateRaw) : null;
const skillFilter = argValue(args, '--skill');
const target = Number.isFinite(gate) ? gate : TARGET;
const full = scoreReport(buildReport(), { target });
// Scope the reported/gated set to one skill if requested (scoring already ran
// over the whole corpus, so K10 is computed correctly).
let scored = full.scored;
if (skillFilter) {
scored = scored.filter((s) => s.name === skillFilter);
if (scored.length === 0) {
process.stderr.write(`score-skill: ukjent skill '${skillFilter}'. Tilgjengelig: ${full.scored.map((s) => s.name).join(', ')}\n`);
process.exitCode = 2;
return;
}
}
const below = scored.filter((s) => !s.meetsTarget);
const result = { target, scored, below };
if (jsonOut) {
process.stdout.write(JSON.stringify(result) + '\n');
} else {
process.stdout.write(formatScoreReport(result) + '\n');
if (gate !== null) {
process.stdout.write(below.length === 0
? `✓ Gate ≥${target}: alle ${scored.length} skills består.\n`
: `✗ Gate ≥${target}: ${below.length} skill(s) under mål — ${below.map((s) => s.name).join(', ')}.\n`);
}
}
if (gate !== null && below.length > 0) process.exitCode = 1;
}
if (process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1]) {
main();
}