feat(ms-ai-architect): Spor D Steg B — skill-quality lifecycle-gate + score-cache (TDD)

B1: gateSkill(report, name) — pure post-mutation verdict (blocked/provisional/
improvements), wired into apply-skill-op.mjs to re-score the affected skill from
fresh disk after create/merge/sanitize-apply. K10 floor enforced immediately;
unjudged → provisional + nudge to re-run the judge pass. retire → skipped.

B2: buildScoreCache(result) — compact, deterministic cache shape + score-skill.mjs
--write [path] → data/skill-score-report.json (always whole-corpus). Gitignored
(derived/regenerable; avoids churn in the public repo). Consumed by Steg C surfacing.

8 new tests (tests/kb-eval/test-skill-score-gate.test.mjs). kb-eval 150 pass,
validate 239 pass. Live score unchanged: advisor 91, eng/gov/infra/sec 96, 0 < 90.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Kjell Tore Guttormsen 2026-06-23 18:26:29 +02:00
commit 92cd93771b
5 changed files with 294 additions and 3 deletions

View file

@ -24,7 +24,8 @@
import { readFileSync, readdirSync, existsSync } from 'node:fs';
import { dirname, join } from 'node:path';
import { fileURLToPath } from 'node:url';
import { splitFrontmatter, extractDescription } from './eval.mjs';
import { splitFrontmatter, extractDescription, buildReport } from './eval.mjs';
import { gateSkill } from './lib/skill-score.mjs';
import { loadTaxonomy } from '../kb-update/lib/taxonomy.mjs';
import { loadDecisions, listActions, actionKey } from '../kb-update/lib/decisions-io.mjs';
import {
@ -50,6 +51,8 @@ const OP_BY_VERB = {
create: CREATE_OPERATION,
};
const QUALITY_TARGET = 90; // corpus invariant: every skill stays >=90 (Spor D)
/** Read the SKILL.md descriptions from disk (read-only) — create's K10 siblings. */
function loadDescriptions() {
const out = {};
@ -137,6 +140,54 @@ function printResult(res, verb, label, apply) {
console.log(`\nStatus: APPLIED. Verifiser med 'git status' og re-kjør validate/kb-integrity/kb-eval.\n`);
}
/**
* The mutated skill that should be re-scored after an apply. create/sanitize keep
* a scorable skill; merge leaves the absorber; retire removes the skill entirely
* (nothing to score -> null, gate skipped). Spor D STEG B lifecycle-gate.
*/
function affectedSkill(op, targets) {
if (op === CREATE_OPERATION) return targets.name;
if (op === MERGE_OPERATION) return targets.absorber;
if (op === SANITIZE_OPERATION) return targets.skill;
return null; // RETIRE: skill removed
}
/**
* Re-score the affected skill from fresh disk after a successful apply and surface
* a quality verdict. The deterministic floor (K10) is enforced immediately as a
* loud warning; an unjudged skill is flagged provisional with a nudge to re-run
* the judge pass (the K1 floor cannot be enforced from the cache alone).
*/
function printQualityGate(op, targets) {
const affected = affectedSkill(op, targets);
if (!affected) {
console.log(`\nKvalitetsgate: ingen gjenværende skill å re-score (retire). Hopper over.`);
return;
}
const v = gateSkill(buildReport(), affected, { target: QUALITY_TARGET });
if (!v.found) {
console.log(`\nKvalitetsgate: fant ikke '${affected}' på disk for re-scoring (hoppet over).`);
return;
}
const tags = [];
if (v.floored) tags.push('floored');
if (v.provisional) tags.push('provisional');
const tagStr = tags.length ? ` [${tags.join(', ')}]` : '';
if (v.blocked) {
console.log(`\n⚠ KVALITETSGATE: '${affected}' scorer ${v.score}/100 — UNDER MÅL ${QUALITY_TARGET} %${tagStr}.`);
for (const imp of (v.improvements || []).slice(0, 3)) {
const floor = imp.floor ? ' ⚑gulv' : '';
console.log(`${imp.key} ${imp.label}${floor} (${imp.detail}) [-${imp.pointsLost.toFixed(2)}] → ${imp.fix}`);
}
console.log(` Rett før commit, eller dokumenter avviket eksplisitt.`);
} else {
console.log(`\nKvalitetsgate: '${affected}' scorer ${v.score}/100 — OK (≥${QUALITY_TARGET} %)${tagStr}.`);
}
if (v.provisional) {
console.log(` judge-cache mangler/utdatert for '${affected}' → K1-gulv ikke håndhevet. Kjør judge-passet (scripts/kb-eval/judge-prompt.md) og re-merge data/judge-results.json, deretter re-score.`);
}
}
function main() {
const args = process.argv.slice(2);
const verb = args[0];
@ -209,6 +260,11 @@ function main() {
promptSet: isCreate ? loadPromptSet() : undefined,
});
printResult(res, verb, label, apply);
// STEG B: re-score the affected skill from fresh disk after a real mutation.
// Preview never mutates -> nothing to re-score (create's projected skill isn't
// on disk), so the gate runs only on applied mutations.
if (res.applied) printQualityGate(op, targets);
}
if (process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1]) {

View file

@ -250,6 +250,67 @@ export function scoreReport(report, opts = {}) {
return { target, scored, below };
}
/**
* Re-score ONE affected skill out of a freshly built corpus report and return a
* lifecycle-gate verdict (Spor D, STEG B). Used by apply-skill-op.mjs after a
* create/merge/sanitize mutation: the whole corpus is needed for K10 sibling
* overlap, but only the affected skill is judged. The deterministic floor (K10)
* is enforced immediately via `blocked`; an unjudged skill is `provisional`
* (the K1 floor cannot be enforced -> the caller nudges to re-run the judge).
* @param {{skills: object[]}} report buildReport() output (fresh disk)
* @param {string} skillName the mutated skill to re-score
* @param {{target?: number, floorCap?: number}} [opts]
* @returns {object} { name, found, score?, meetsTarget?, blocked?, provisional?, judged?, floored?, improvements? }
*/
export function gateSkill(report, skillName, opts = {}) {
const target = opts.target ?? TARGET;
const { scored } = scoreReport(report, { ...opts, target });
const s = scored.find((x) => x.name === skillName);
if (!s) return { name: skillName, found: false };
return {
name: skillName,
found: true,
score: s.score,
meetsTarget: s.meetsTarget,
blocked: !s.meetsTarget,
provisional: s.provisional,
judged: s.judged,
floored: s.floored,
improvements: s.improvements,
};
}
/**
* Project a scoreReport() result into the compact, PURE cache shape persisted as
* data/skill-score-report.json and consumed by the STEG C SessionStart surfacing
* (which must read the cache, never score live). Drops the verbose criteria[] and
* keeps only what the one-liner needs; `below` is a list of names. Deterministic
* given an injected `generatedAt` (no Date.now in the pure layer).
* @param {{target: number, scored: object[]}} result scoreReport() output
* @param {{generatedAt?: string|null}} [opts]
* @returns {{generatedAt: string|null, target: number, scored: object[], below: string[]}}
*/
export function buildScoreCache(result, opts = {}) {
const target = result?.target ?? TARGET;
const scored = (result?.scored || []).map((s) => ({
name: s.name,
score: s.score,
meetsTarget: s.meetsTarget,
provisional: s.provisional,
floored: s.floored,
judged: s.judged,
topFix: s.improvements?.[0]
? { key: s.improvements[0].key, label: s.improvements[0].label, fix: s.improvements[0].fix }
: null,
}));
return {
generatedAt: opts.generatedAt ?? null,
target,
scored,
below: scored.filter((s) => !s.meetsTarget).map((s) => s.name),
};
}
/**
* Render a human-readable quality report (pure). Each skill shows its score and
* status; below-target skills list their sorted improvements (biggest weighted

View file

@ -13,16 +13,25 @@
// node scripts/kb-eval/score-skill.mjs --skill <name> # scope output to one skill
// # (still scores the full corpus
// # so K10 sibling-overlap is correct)
// node scripts/kb-eval/score-skill.mjs --write [path] # persist the whole-corpus cache
// # (default: data/skill-score-report.json;
// # consumed by the STEG C surfacing)
//
// K10 requires every sibling description, so even single-skill (incremental)
// scoring evaluates the whole corpus and then filters the reported set — this is
// the building block for the lifecycle-gate in apply-skill-op.mjs.
// the building block for the lifecycle-gate in apply-skill-op.mjs. The --write
// cache is ALWAYS whole-corpus (surfacing needs every skill), independent of --skill.
//
// Zero dependencies. Pure scoring lives in lib/skill-score.mjs.
import { writeFileSync } from 'node:fs';
import { dirname, join } from 'node:path';
import { fileURLToPath } from 'node:url';
import { buildReport } from './eval.mjs';
import { scoreReport, formatScoreReport, TARGET } from './lib/skill-score.mjs';
import { scoreReport, formatScoreReport, buildScoreCache, TARGET } from './lib/skill-score.mjs';
const __dirname = dirname(fileURLToPath(import.meta.url));
const DEFAULT_CACHE = join(__dirname, 'data', 'skill-score-report.json');
function argValue(args, flag) {
const i = args.indexOf(flag);
@ -36,9 +45,24 @@ function main() {
const gate = gateRaw !== null ? Number(gateRaw) : null;
const skillFilter = argValue(args, '--skill');
// --write [path]: bare flag uses the default cache path; an explicit non-flag
// argument overrides it (the tests write to a temp path to avoid clobbering).
const writeIdx = args.indexOf('--write');
const doWrite = writeIdx !== -1;
const writePath = doWrite && args[writeIdx + 1] && !args[writeIdx + 1].startsWith('--')
? args[writeIdx + 1]
: DEFAULT_CACHE;
const target = Number.isFinite(gate) ? gate : TARGET;
const full = scoreReport(buildReport(), { target });
// The cache is whole-corpus regardless of --skill (surfacing needs every skill).
if (doWrite) {
const cache = buildScoreCache(full, { generatedAt: new Date().toISOString().slice(0, 10) });
writeFileSync(writePath, JSON.stringify(cache, null, 2) + '\n');
process.stdout.write(`✓ Skrev skill-score-cache → ${writePath} (${cache.scored.length} skills, ${cache.below.length} under mål).\n`);
}
// Scope the reported/gated set to one skill if requested (scoring already ran
// over the whole corpus, so K10 is computed correctly).
let scored = full.scored;