feat(ms-ai-architect): Spor D Steg B — skill-quality lifecycle-gate + score-cache (TDD)
B1: gateSkill(report, name) — pure post-mutation verdict (blocked/provisional/ improvements), wired into apply-skill-op.mjs to re-score the affected skill from fresh disk after create/merge/sanitize-apply. K10 floor enforced immediately; unjudged → provisional + nudge to re-run the judge pass. retire → skipped. B2: buildScoreCache(result) — compact, deterministic cache shape + score-skill.mjs --write [path] → data/skill-score-report.json (always whole-corpus). Gitignored (derived/regenerable; avoids churn in the public repo). Consumed by Steg C surfacing. 8 new tests (tests/kb-eval/test-skill-score-gate.test.mjs). kb-eval 150 pass, validate 239 pass. Live score unchanged: advisor 91, eng/gov/infra/sec 96, 0 < 90. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
c578482933
commit
92cd93771b
5 changed files with 294 additions and 3 deletions
|
|
@ -24,7 +24,8 @@
|
|||
import { readFileSync, readdirSync, existsSync } from 'node:fs';
|
||||
import { dirname, join } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { splitFrontmatter, extractDescription } from './eval.mjs';
|
||||
import { splitFrontmatter, extractDescription, buildReport } from './eval.mjs';
|
||||
import { gateSkill } from './lib/skill-score.mjs';
|
||||
import { loadTaxonomy } from '../kb-update/lib/taxonomy.mjs';
|
||||
import { loadDecisions, listActions, actionKey } from '../kb-update/lib/decisions-io.mjs';
|
||||
import {
|
||||
|
|
@ -50,6 +51,8 @@ const OP_BY_VERB = {
|
|||
create: CREATE_OPERATION,
|
||||
};
|
||||
|
||||
const QUALITY_TARGET = 90; // corpus invariant: every skill stays >=90 (Spor D)
|
||||
|
||||
/** Read the SKILL.md descriptions from disk (read-only) — create's K10 siblings. */
|
||||
function loadDescriptions() {
|
||||
const out = {};
|
||||
|
|
@ -137,6 +140,54 @@ function printResult(res, verb, label, apply) {
|
|||
console.log(`\nStatus: APPLIED. Verifiser med 'git status' og re-kjør validate/kb-integrity/kb-eval.\n`);
|
||||
}
|
||||
|
||||
/**
|
||||
* The mutated skill that should be re-scored after an apply. create/sanitize keep
|
||||
* a scorable skill; merge leaves the absorber; retire removes the skill entirely
|
||||
* (nothing to score -> null, gate skipped). Spor D STEG B lifecycle-gate.
|
||||
*/
|
||||
function affectedSkill(op, targets) {
|
||||
if (op === CREATE_OPERATION) return targets.name;
|
||||
if (op === MERGE_OPERATION) return targets.absorber;
|
||||
if (op === SANITIZE_OPERATION) return targets.skill;
|
||||
return null; // RETIRE: skill removed
|
||||
}
|
||||
|
||||
/**
|
||||
* Re-score the affected skill from fresh disk after a successful apply and surface
|
||||
* a quality verdict. The deterministic floor (K10) is enforced immediately as a
|
||||
* loud warning; an unjudged skill is flagged provisional with a nudge to re-run
|
||||
* the judge pass (the K1 floor cannot be enforced from the cache alone).
|
||||
*/
|
||||
function printQualityGate(op, targets) {
|
||||
const affected = affectedSkill(op, targets);
|
||||
if (!affected) {
|
||||
console.log(`\nKvalitetsgate: ingen gjenværende skill å re-score (retire). Hopper over.`);
|
||||
return;
|
||||
}
|
||||
const v = gateSkill(buildReport(), affected, { target: QUALITY_TARGET });
|
||||
if (!v.found) {
|
||||
console.log(`\nKvalitetsgate: fant ikke '${affected}' på disk for re-scoring (hoppet over).`);
|
||||
return;
|
||||
}
|
||||
const tags = [];
|
||||
if (v.floored) tags.push('floored');
|
||||
if (v.provisional) tags.push('provisional');
|
||||
const tagStr = tags.length ? ` [${tags.join(', ')}]` : '';
|
||||
if (v.blocked) {
|
||||
console.log(`\n⚠ KVALITETSGATE: '${affected}' scorer ${v.score}/100 — UNDER MÅL ${QUALITY_TARGET} %${tagStr}.`);
|
||||
for (const imp of (v.improvements || []).slice(0, 3)) {
|
||||
const floor = imp.floor ? ' ⚑gulv' : '';
|
||||
console.log(` ↳ ${imp.key} ${imp.label}${floor} (${imp.detail}) [-${imp.pointsLost.toFixed(2)}] → ${imp.fix}`);
|
||||
}
|
||||
console.log(` Rett før commit, eller dokumenter avviket eksplisitt.`);
|
||||
} else {
|
||||
console.log(`\nKvalitetsgate: '${affected}' scorer ${v.score}/100 — OK (≥${QUALITY_TARGET} %)${tagStr}.`);
|
||||
}
|
||||
if (v.provisional) {
|
||||
console.log(` ℹ judge-cache mangler/utdatert for '${affected}' → K1-gulv ikke håndhevet. Kjør judge-passet (scripts/kb-eval/judge-prompt.md) og re-merge data/judge-results.json, deretter re-score.`);
|
||||
}
|
||||
}
|
||||
|
||||
function main() {
|
||||
const args = process.argv.slice(2);
|
||||
const verb = args[0];
|
||||
|
|
@ -209,6 +260,11 @@ function main() {
|
|||
promptSet: isCreate ? loadPromptSet() : undefined,
|
||||
});
|
||||
printResult(res, verb, label, apply);
|
||||
|
||||
// STEG B: re-score the affected skill from fresh disk after a real mutation.
|
||||
// Preview never mutates -> nothing to re-score (create's projected skill isn't
|
||||
// on disk), so the gate runs only on applied mutations.
|
||||
if (res.applied) printQualityGate(op, targets);
|
||||
}
|
||||
|
||||
if (process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1]) {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue