feat(ms-ai-architect): Spor D Steg B — skill-quality lifecycle-gate + score-cache (TDD)

B1: gateSkill(report, name) — pure post-mutation verdict (blocked/provisional/
improvements), wired into apply-skill-op.mjs to re-score the affected skill from
fresh disk after create/merge/sanitize-apply. K10 floor enforced immediately;
unjudged → provisional + nudge to re-run the judge pass. retire → skipped.

B2: buildScoreCache(result) — compact, deterministic cache shape + score-skill.mjs
--write [path] → data/skill-score-report.json (always whole-corpus). Gitignored
(derived/regenerable; avoids churn in the public repo). Consumed by Steg C surfacing.

8 new tests (tests/kb-eval/test-skill-score-gate.test.mjs). kb-eval 150 pass,
validate 239 pass. Live score unchanged: advisor 91, eng/gov/infra/sec 96, 0 < 90.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Kjell Tore Guttormsen 2026-06-23 18:26:29 +02:00
commit 92cd93771b
5 changed files with 294 additions and 3 deletions

View file

@ -24,7 +24,8 @@
import { readFileSync, readdirSync, existsSync } from 'node:fs';
import { dirname, join } from 'node:path';
import { fileURLToPath } from 'node:url';
import { splitFrontmatter, extractDescription } from './eval.mjs';
import { splitFrontmatter, extractDescription, buildReport } from './eval.mjs';
import { gateSkill } from './lib/skill-score.mjs';
import { loadTaxonomy } from '../kb-update/lib/taxonomy.mjs';
import { loadDecisions, listActions, actionKey } from '../kb-update/lib/decisions-io.mjs';
import {
@ -50,6 +51,8 @@ const OP_BY_VERB = {
create: CREATE_OPERATION,
};
const QUALITY_TARGET = 90; // corpus invariant: every skill stays >=90 (Spor D)
/** Read the SKILL.md descriptions from disk (read-only) — create's K10 siblings. */
function loadDescriptions() {
const out = {};
@ -137,6 +140,54 @@ function printResult(res, verb, label, apply) {
console.log(`\nStatus: APPLIED. Verifiser med 'git status' og re-kjør validate/kb-integrity/kb-eval.\n`);
}
/**
* The mutated skill that should be re-scored after an apply. create/sanitize keep
* a scorable skill; merge leaves the absorber; retire removes the skill entirely
* (nothing to score -> null, gate skipped). Spor D STEG B lifecycle-gate.
*/
function affectedSkill(op, targets) {
if (op === CREATE_OPERATION) return targets.name;
if (op === MERGE_OPERATION) return targets.absorber;
if (op === SANITIZE_OPERATION) return targets.skill;
return null; // RETIRE: skill removed
}
/**
* Re-score the affected skill from fresh disk after a successful apply and surface
* a quality verdict. The deterministic floor (K10) is enforced immediately as a
* loud warning; an unjudged skill is flagged provisional with a nudge to re-run
* the judge pass (the K1 floor cannot be enforced from the cache alone).
*/
function printQualityGate(op, targets) {
const affected = affectedSkill(op, targets);
if (!affected) {
console.log(`\nKvalitetsgate: ingen gjenværende skill å re-score (retire). Hopper over.`);
return;
}
const v = gateSkill(buildReport(), affected, { target: QUALITY_TARGET });
if (!v.found) {
console.log(`\nKvalitetsgate: fant ikke '${affected}' på disk for re-scoring (hoppet over).`);
return;
}
const tags = [];
if (v.floored) tags.push('floored');
if (v.provisional) tags.push('provisional');
const tagStr = tags.length ? ` [${tags.join(', ')}]` : '';
if (v.blocked) {
console.log(`\n⚠ KVALITETSGATE: '${affected}' scorer ${v.score}/100 — UNDER MÅL ${QUALITY_TARGET} %${tagStr}.`);
for (const imp of (v.improvements || []).slice(0, 3)) {
const floor = imp.floor ? ' ⚑gulv' : '';
console.log(`${imp.key} ${imp.label}${floor} (${imp.detail}) [-${imp.pointsLost.toFixed(2)}] → ${imp.fix}`);
}
console.log(` Rett før commit, eller dokumenter avviket eksplisitt.`);
} else {
console.log(`\nKvalitetsgate: '${affected}' scorer ${v.score}/100 — OK (≥${QUALITY_TARGET} %)${tagStr}.`);
}
if (v.provisional) {
console.log(` judge-cache mangler/utdatert for '${affected}' → K1-gulv ikke håndhevet. Kjør judge-passet (scripts/kb-eval/judge-prompt.md) og re-merge data/judge-results.json, deretter re-score.`);
}
}
function main() {
const args = process.argv.slice(2);
const verb = args[0];
@ -209,6 +260,11 @@ function main() {
promptSet: isCreate ? loadPromptSet() : undefined,
});
printResult(res, verb, label, apply);
// STEG B: re-score the affected skill from fresh disk after a real mutation.
// Preview never mutates -> nothing to re-score (create's projected skill isn't
// on disk), so the gate runs only on applied mutations.
if (res.applied) printQualityGate(op, targets);
}
if (process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1]) {