feat(ms-ai-architect): Spor D Steg B — skill-quality lifecycle-gate + score-cache (TDD)
B1: gateSkill(report, name) — pure post-mutation verdict (blocked/provisional/ improvements), wired into apply-skill-op.mjs to re-score the affected skill from fresh disk after create/merge/sanitize-apply. K10 floor enforced immediately; unjudged → provisional + nudge to re-run the judge pass. retire → skipped. B2: buildScoreCache(result) — compact, deterministic cache shape + score-skill.mjs --write [path] → data/skill-score-report.json (always whole-corpus). Gitignored (derived/regenerable; avoids churn in the public repo). Consumed by Steg C surfacing. 8 new tests (tests/kb-eval/test-skill-score-gate.test.mjs). kb-eval 150 pass, validate 239 pass. Live score unchanged: advisor 91, eng/gov/infra/sec 96, 0 < 90. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
c578482933
commit
92cd93771b
5 changed files with 294 additions and 3 deletions
|
|
@ -24,7 +24,8 @@
|
|||
import { readFileSync, readdirSync, existsSync } from 'node:fs';
|
||||
import { dirname, join } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { splitFrontmatter, extractDescription } from './eval.mjs';
|
||||
import { splitFrontmatter, extractDescription, buildReport } from './eval.mjs';
|
||||
import { gateSkill } from './lib/skill-score.mjs';
|
||||
import { loadTaxonomy } from '../kb-update/lib/taxonomy.mjs';
|
||||
import { loadDecisions, listActions, actionKey } from '../kb-update/lib/decisions-io.mjs';
|
||||
import {
|
||||
|
|
@ -50,6 +51,8 @@ const OP_BY_VERB = {
|
|||
create: CREATE_OPERATION,
|
||||
};
|
||||
|
||||
const QUALITY_TARGET = 90; // corpus invariant: every skill stays >=90 (Spor D)
|
||||
|
||||
/** Read the SKILL.md descriptions from disk (read-only) — create's K10 siblings. */
|
||||
function loadDescriptions() {
|
||||
const out = {};
|
||||
|
|
@ -137,6 +140,54 @@ function printResult(res, verb, label, apply) {
|
|||
console.log(`\nStatus: APPLIED. Verifiser med 'git status' og re-kjør validate/kb-integrity/kb-eval.\n`);
|
||||
}
|
||||
|
||||
/**
|
||||
* The mutated skill that should be re-scored after an apply. create/sanitize keep
|
||||
* a scorable skill; merge leaves the absorber; retire removes the skill entirely
|
||||
* (nothing to score -> null, gate skipped). Spor D STEG B lifecycle-gate.
|
||||
*/
|
||||
function affectedSkill(op, targets) {
|
||||
if (op === CREATE_OPERATION) return targets.name;
|
||||
if (op === MERGE_OPERATION) return targets.absorber;
|
||||
if (op === SANITIZE_OPERATION) return targets.skill;
|
||||
return null; // RETIRE: skill removed
|
||||
}
|
||||
|
||||
/**
|
||||
* Re-score the affected skill from fresh disk after a successful apply and surface
|
||||
* a quality verdict. The deterministic floor (K10) is enforced immediately as a
|
||||
* loud warning; an unjudged skill is flagged provisional with a nudge to re-run
|
||||
* the judge pass (the K1 floor cannot be enforced from the cache alone).
|
||||
*/
|
||||
function printQualityGate(op, targets) {
|
||||
const affected = affectedSkill(op, targets);
|
||||
if (!affected) {
|
||||
console.log(`\nKvalitetsgate: ingen gjenværende skill å re-score (retire). Hopper over.`);
|
||||
return;
|
||||
}
|
||||
const v = gateSkill(buildReport(), affected, { target: QUALITY_TARGET });
|
||||
if (!v.found) {
|
||||
console.log(`\nKvalitetsgate: fant ikke '${affected}' på disk for re-scoring (hoppet over).`);
|
||||
return;
|
||||
}
|
||||
const tags = [];
|
||||
if (v.floored) tags.push('floored');
|
||||
if (v.provisional) tags.push('provisional');
|
||||
const tagStr = tags.length ? ` [${tags.join(', ')}]` : '';
|
||||
if (v.blocked) {
|
||||
console.log(`\n⚠ KVALITETSGATE: '${affected}' scorer ${v.score}/100 — UNDER MÅL ${QUALITY_TARGET} %${tagStr}.`);
|
||||
for (const imp of (v.improvements || []).slice(0, 3)) {
|
||||
const floor = imp.floor ? ' ⚑gulv' : '';
|
||||
console.log(` ↳ ${imp.key} ${imp.label}${floor} (${imp.detail}) [-${imp.pointsLost.toFixed(2)}] → ${imp.fix}`);
|
||||
}
|
||||
console.log(` Rett før commit, eller dokumenter avviket eksplisitt.`);
|
||||
} else {
|
||||
console.log(`\nKvalitetsgate: '${affected}' scorer ${v.score}/100 — OK (≥${QUALITY_TARGET} %)${tagStr}.`);
|
||||
}
|
||||
if (v.provisional) {
|
||||
console.log(` ℹ judge-cache mangler/utdatert for '${affected}' → K1-gulv ikke håndhevet. Kjør judge-passet (scripts/kb-eval/judge-prompt.md) og re-merge data/judge-results.json, deretter re-score.`);
|
||||
}
|
||||
}
|
||||
|
||||
function main() {
|
||||
const args = process.argv.slice(2);
|
||||
const verb = args[0];
|
||||
|
|
@ -209,6 +260,11 @@ function main() {
|
|||
promptSet: isCreate ? loadPromptSet() : undefined,
|
||||
});
|
||||
printResult(res, verb, label, apply);
|
||||
|
||||
// STEG B: re-score the affected skill from fresh disk after a real mutation.
|
||||
// Preview never mutates -> nothing to re-score (create's projected skill isn't
|
||||
// on disk), so the gate runs only on applied mutations.
|
||||
if (res.applied) printQualityGate(op, targets);
|
||||
}
|
||||
|
||||
if (process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1]) {
|
||||
|
|
|
|||
|
|
@ -250,6 +250,67 @@ export function scoreReport(report, opts = {}) {
|
|||
return { target, scored, below };
|
||||
}
|
||||
|
||||
/**
|
||||
* Re-score ONE affected skill out of a freshly built corpus report and return a
|
||||
* lifecycle-gate verdict (Spor D, STEG B). Used by apply-skill-op.mjs after a
|
||||
* create/merge/sanitize mutation: the whole corpus is needed for K10 sibling
|
||||
* overlap, but only the affected skill is judged. The deterministic floor (K10)
|
||||
* is enforced immediately via `blocked`; an unjudged skill is `provisional`
|
||||
* (the K1 floor cannot be enforced -> the caller nudges to re-run the judge).
|
||||
* @param {{skills: object[]}} report buildReport() output (fresh disk)
|
||||
* @param {string} skillName the mutated skill to re-score
|
||||
* @param {{target?: number, floorCap?: number}} [opts]
|
||||
* @returns {object} { name, found, score?, meetsTarget?, blocked?, provisional?, judged?, floored?, improvements? }
|
||||
*/
|
||||
export function gateSkill(report, skillName, opts = {}) {
|
||||
const target = opts.target ?? TARGET;
|
||||
const { scored } = scoreReport(report, { ...opts, target });
|
||||
const s = scored.find((x) => x.name === skillName);
|
||||
if (!s) return { name: skillName, found: false };
|
||||
return {
|
||||
name: skillName,
|
||||
found: true,
|
||||
score: s.score,
|
||||
meetsTarget: s.meetsTarget,
|
||||
blocked: !s.meetsTarget,
|
||||
provisional: s.provisional,
|
||||
judged: s.judged,
|
||||
floored: s.floored,
|
||||
improvements: s.improvements,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Project a scoreReport() result into the compact, PURE cache shape persisted as
|
||||
* data/skill-score-report.json and consumed by the STEG C SessionStart surfacing
|
||||
* (which must read the cache, never score live). Drops the verbose criteria[] and
|
||||
* keeps only what the one-liner needs; `below` is a list of names. Deterministic
|
||||
* given an injected `generatedAt` (no Date.now in the pure layer).
|
||||
* @param {{target: number, scored: object[]}} result scoreReport() output
|
||||
* @param {{generatedAt?: string|null}} [opts]
|
||||
* @returns {{generatedAt: string|null, target: number, scored: object[], below: string[]}}
|
||||
*/
|
||||
export function buildScoreCache(result, opts = {}) {
|
||||
const target = result?.target ?? TARGET;
|
||||
const scored = (result?.scored || []).map((s) => ({
|
||||
name: s.name,
|
||||
score: s.score,
|
||||
meetsTarget: s.meetsTarget,
|
||||
provisional: s.provisional,
|
||||
floored: s.floored,
|
||||
judged: s.judged,
|
||||
topFix: s.improvements?.[0]
|
||||
? { key: s.improvements[0].key, label: s.improvements[0].label, fix: s.improvements[0].fix }
|
||||
: null,
|
||||
}));
|
||||
return {
|
||||
generatedAt: opts.generatedAt ?? null,
|
||||
target,
|
||||
scored,
|
||||
below: scored.filter((s) => !s.meetsTarget).map((s) => s.name),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Render a human-readable quality report (pure). Each skill shows its score and
|
||||
* status; below-target skills list their sorted improvements (biggest weighted
|
||||
|
|
|
|||
|
|
@ -13,16 +13,25 @@
|
|||
// node scripts/kb-eval/score-skill.mjs --skill <name> # scope output to one skill
|
||||
// # (still scores the full corpus
|
||||
// # so K10 sibling-overlap is correct)
|
||||
// node scripts/kb-eval/score-skill.mjs --write [path] # persist the whole-corpus cache
|
||||
// # (default: data/skill-score-report.json;
|
||||
// # consumed by the STEG C surfacing)
|
||||
//
|
||||
// K10 requires every sibling description, so even single-skill (incremental)
|
||||
// scoring evaluates the whole corpus and then filters the reported set — this is
|
||||
// the building block for the lifecycle-gate in apply-skill-op.mjs.
|
||||
// the building block for the lifecycle-gate in apply-skill-op.mjs. The --write
|
||||
// cache is ALWAYS whole-corpus (surfacing needs every skill), independent of --skill.
|
||||
//
|
||||
// Zero dependencies. Pure scoring lives in lib/skill-score.mjs.
|
||||
|
||||
import { writeFileSync } from 'node:fs';
|
||||
import { dirname, join } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { buildReport } from './eval.mjs';
|
||||
import { scoreReport, formatScoreReport, TARGET } from './lib/skill-score.mjs';
|
||||
import { scoreReport, formatScoreReport, buildScoreCache, TARGET } from './lib/skill-score.mjs';
|
||||
|
||||
const __dirname = dirname(fileURLToPath(import.meta.url));
|
||||
const DEFAULT_CACHE = join(__dirname, 'data', 'skill-score-report.json');
|
||||
|
||||
function argValue(args, flag) {
|
||||
const i = args.indexOf(flag);
|
||||
|
|
@ -36,9 +45,24 @@ function main() {
|
|||
const gate = gateRaw !== null ? Number(gateRaw) : null;
|
||||
const skillFilter = argValue(args, '--skill');
|
||||
|
||||
// --write [path]: bare flag uses the default cache path; an explicit non-flag
|
||||
// argument overrides it (the tests write to a temp path to avoid clobbering).
|
||||
const writeIdx = args.indexOf('--write');
|
||||
const doWrite = writeIdx !== -1;
|
||||
const writePath = doWrite && args[writeIdx + 1] && !args[writeIdx + 1].startsWith('--')
|
||||
? args[writeIdx + 1]
|
||||
: DEFAULT_CACHE;
|
||||
|
||||
const target = Number.isFinite(gate) ? gate : TARGET;
|
||||
const full = scoreReport(buildReport(), { target });
|
||||
|
||||
// The cache is whole-corpus regardless of --skill (surfacing needs every skill).
|
||||
if (doWrite) {
|
||||
const cache = buildScoreCache(full, { generatedAt: new Date().toISOString().slice(0, 10) });
|
||||
writeFileSync(writePath, JSON.stringify(cache, null, 2) + '\n');
|
||||
process.stdout.write(`✓ Skrev skill-score-cache → ${writePath} (${cache.scored.length} skills, ${cache.below.length} under mål).\n`);
|
||||
}
|
||||
|
||||
// Scope the reported/gated set to one skill if requested (scoring already ran
|
||||
// over the whole corpus, so K10 is computed correctly).
|
||||
let scored = full.scored;
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue