diff --git a/.gitignore b/.gitignore index 2599ccc..6b9174f 100644 --- a/.gitignore +++ b/.gitignore @@ -30,6 +30,10 @@ scripts/kb-update/data/* # Generated skill-lifecycle detection report (Spor B / B1) — regenerated on demand, # like the kb-update reports above. The detector script + curated inputs are tracked. scripts/kb-eval/data/skill-lifecycle-report.json +# Generated skill-quality score cache (Spor D / STEG B) — derived from the corpus, +# regenerated by `score-skill.mjs --write` and the apply-skill-op gate. Consumed by +# the STEG C SessionStart surfacing; not committed (avoids churn in the public repo). +scripts/kb-eval/data/skill-score-report.json .kb-backup/ .rollback-in-progress diff --git a/scripts/kb-eval/apply-skill-op.mjs b/scripts/kb-eval/apply-skill-op.mjs index 6cdd5ac..03b3505 100644 --- a/scripts/kb-eval/apply-skill-op.mjs +++ b/scripts/kb-eval/apply-skill-op.mjs @@ -24,7 +24,8 @@ import { readFileSync, readdirSync, existsSync } from 'node:fs'; import { dirname, join } from 'node:path'; import { fileURLToPath } from 'node:url'; -import { splitFrontmatter, extractDescription } from './eval.mjs'; +import { splitFrontmatter, extractDescription, buildReport } from './eval.mjs'; +import { gateSkill } from './lib/skill-score.mjs'; import { loadTaxonomy } from '../kb-update/lib/taxonomy.mjs'; import { loadDecisions, listActions, actionKey } from '../kb-update/lib/decisions-io.mjs'; import { @@ -50,6 +51,8 @@ const OP_BY_VERB = { create: CREATE_OPERATION, }; +const QUALITY_TARGET = 90; // corpus invariant: every skill stays >=90 (Spor D) + /** Read the SKILL.md descriptions from disk (read-only) — create's K10 siblings. */ function loadDescriptions() { const out = {}; @@ -137,6 +140,54 @@ function printResult(res, verb, label, apply) { console.log(`\nStatus: APPLIED. Verifiser med 'git status' og re-kjør validate/kb-integrity/kb-eval.\n`); } +/** + * The mutated skill that should be re-scored after an apply. create/sanitize keep + * a scorable skill; merge leaves the absorber; retire removes the skill entirely + * (nothing to score -> null, gate skipped). Spor D STEG B lifecycle-gate. + */ +function affectedSkill(op, targets) { + if (op === CREATE_OPERATION) return targets.name; + if (op === MERGE_OPERATION) return targets.absorber; + if (op === SANITIZE_OPERATION) return targets.skill; + return null; // RETIRE: skill removed +} + +/** + * Re-score the affected skill from fresh disk after a successful apply and surface + * a quality verdict. The deterministic floor (K10) is enforced immediately as a + * loud warning; an unjudged skill is flagged provisional with a nudge to re-run + * the judge pass (the K1 floor cannot be enforced from the cache alone). + */ +function printQualityGate(op, targets) { + const affected = affectedSkill(op, targets); + if (!affected) { + console.log(`\nKvalitetsgate: ingen gjenværende skill å re-score (retire). Hopper over.`); + return; + } + const v = gateSkill(buildReport(), affected, { target: QUALITY_TARGET }); + if (!v.found) { + console.log(`\nKvalitetsgate: fant ikke '${affected}' på disk for re-scoring (hoppet over).`); + return; + } + const tags = []; + if (v.floored) tags.push('floored'); + if (v.provisional) tags.push('provisional'); + const tagStr = tags.length ? ` [${tags.join(', ')}]` : ''; + if (v.blocked) { + console.log(`\n⚠ KVALITETSGATE: '${affected}' scorer ${v.score}/100 — UNDER MÅL ${QUALITY_TARGET} %${tagStr}.`); + for (const imp of (v.improvements || []).slice(0, 3)) { + const floor = imp.floor ? ' ⚑gulv' : ''; + console.log(` ↳ ${imp.key} ${imp.label}${floor} (${imp.detail}) [-${imp.pointsLost.toFixed(2)}] → ${imp.fix}`); + } + console.log(` Rett før commit, eller dokumenter avviket eksplisitt.`); + } else { + console.log(`\nKvalitetsgate: '${affected}' scorer ${v.score}/100 — OK (≥${QUALITY_TARGET} %)${tagStr}.`); + } + if (v.provisional) { + console.log(` ℹ judge-cache mangler/utdatert for '${affected}' → K1-gulv ikke håndhevet. Kjør judge-passet (scripts/kb-eval/judge-prompt.md) og re-merge data/judge-results.json, deretter re-score.`); + } +} + function main() { const args = process.argv.slice(2); const verb = args[0]; @@ -209,6 +260,11 @@ function main() { promptSet: isCreate ? loadPromptSet() : undefined, }); printResult(res, verb, label, apply); + + // STEG B: re-score the affected skill from fresh disk after a real mutation. + // Preview never mutates -> nothing to re-score (create's projected skill isn't + // on disk), so the gate runs only on applied mutations. + if (res.applied) printQualityGate(op, targets); } if (process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1]) { diff --git a/scripts/kb-eval/lib/skill-score.mjs b/scripts/kb-eval/lib/skill-score.mjs index ee7dbe5..8e84aec 100644 --- a/scripts/kb-eval/lib/skill-score.mjs +++ b/scripts/kb-eval/lib/skill-score.mjs @@ -250,6 +250,67 @@ export function scoreReport(report, opts = {}) { return { target, scored, below }; } +/** + * Re-score ONE affected skill out of a freshly built corpus report and return a + * lifecycle-gate verdict (Spor D, STEG B). Used by apply-skill-op.mjs after a + * create/merge/sanitize mutation: the whole corpus is needed for K10 sibling + * overlap, but only the affected skill is judged. The deterministic floor (K10) + * is enforced immediately via `blocked`; an unjudged skill is `provisional` + * (the K1 floor cannot be enforced -> the caller nudges to re-run the judge). + * @param {{skills: object[]}} report buildReport() output (fresh disk) + * @param {string} skillName the mutated skill to re-score + * @param {{target?: number, floorCap?: number}} [opts] + * @returns {object} { name, found, score?, meetsTarget?, blocked?, provisional?, judged?, floored?, improvements? } + */ +export function gateSkill(report, skillName, opts = {}) { + const target = opts.target ?? TARGET; + const { scored } = scoreReport(report, { ...opts, target }); + const s = scored.find((x) => x.name === skillName); + if (!s) return { name: skillName, found: false }; + return { + name: skillName, + found: true, + score: s.score, + meetsTarget: s.meetsTarget, + blocked: !s.meetsTarget, + provisional: s.provisional, + judged: s.judged, + floored: s.floored, + improvements: s.improvements, + }; +} + +/** + * Project a scoreReport() result into the compact, PURE cache shape persisted as + * data/skill-score-report.json and consumed by the STEG C SessionStart surfacing + * (which must read the cache, never score live). Drops the verbose criteria[] and + * keeps only what the one-liner needs; `below` is a list of names. Deterministic + * given an injected `generatedAt` (no Date.now in the pure layer). + * @param {{target: number, scored: object[]}} result scoreReport() output + * @param {{generatedAt?: string|null}} [opts] + * @returns {{generatedAt: string|null, target: number, scored: object[], below: string[]}} + */ +export function buildScoreCache(result, opts = {}) { + const target = result?.target ?? TARGET; + const scored = (result?.scored || []).map((s) => ({ + name: s.name, + score: s.score, + meetsTarget: s.meetsTarget, + provisional: s.provisional, + floored: s.floored, + judged: s.judged, + topFix: s.improvements?.[0] + ? { key: s.improvements[0].key, label: s.improvements[0].label, fix: s.improvements[0].fix } + : null, + })); + return { + generatedAt: opts.generatedAt ?? null, + target, + scored, + below: scored.filter((s) => !s.meetsTarget).map((s) => s.name), + }; +} + /** * Render a human-readable quality report (pure). Each skill shows its score and * status; below-target skills list their sorted improvements (biggest weighted diff --git a/scripts/kb-eval/score-skill.mjs b/scripts/kb-eval/score-skill.mjs index b3dd26e..769a634 100644 --- a/scripts/kb-eval/score-skill.mjs +++ b/scripts/kb-eval/score-skill.mjs @@ -13,16 +13,25 @@ // node scripts/kb-eval/score-skill.mjs --skill # scope output to one skill // # (still scores the full corpus // # so K10 sibling-overlap is correct) +// node scripts/kb-eval/score-skill.mjs --write [path] # persist the whole-corpus cache +// # (default: data/skill-score-report.json; +// # consumed by the STEG C surfacing) // // K10 requires every sibling description, so even single-skill (incremental) // scoring evaluates the whole corpus and then filters the reported set — this is -// the building block for the lifecycle-gate in apply-skill-op.mjs. +// the building block for the lifecycle-gate in apply-skill-op.mjs. The --write +// cache is ALWAYS whole-corpus (surfacing needs every skill), independent of --skill. // // Zero dependencies. Pure scoring lives in lib/skill-score.mjs. +import { writeFileSync } from 'node:fs'; +import { dirname, join } from 'node:path'; import { fileURLToPath } from 'node:url'; import { buildReport } from './eval.mjs'; -import { scoreReport, formatScoreReport, TARGET } from './lib/skill-score.mjs'; +import { scoreReport, formatScoreReport, buildScoreCache, TARGET } from './lib/skill-score.mjs'; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const DEFAULT_CACHE = join(__dirname, 'data', 'skill-score-report.json'); function argValue(args, flag) { const i = args.indexOf(flag); @@ -36,9 +45,24 @@ function main() { const gate = gateRaw !== null ? Number(gateRaw) : null; const skillFilter = argValue(args, '--skill'); + // --write [path]: bare flag uses the default cache path; an explicit non-flag + // argument overrides it (the tests write to a temp path to avoid clobbering). + const writeIdx = args.indexOf('--write'); + const doWrite = writeIdx !== -1; + const writePath = doWrite && args[writeIdx + 1] && !args[writeIdx + 1].startsWith('--') + ? args[writeIdx + 1] + : DEFAULT_CACHE; + const target = Number.isFinite(gate) ? gate : TARGET; const full = scoreReport(buildReport(), { target }); + // The cache is whole-corpus regardless of --skill (surfacing needs every skill). + if (doWrite) { + const cache = buildScoreCache(full, { generatedAt: new Date().toISOString().slice(0, 10) }); + writeFileSync(writePath, JSON.stringify(cache, null, 2) + '\n'); + process.stdout.write(`✓ Skrev skill-score-cache → ${writePath} (${cache.scored.length} skills, ${cache.below.length} under mål).\n`); + } + // Scope the reported/gated set to one skill if requested (scoring already ran // over the whole corpus, so K10 is computed correctly). let scored = full.scored; diff --git a/tests/kb-eval/test-skill-score-gate.test.mjs b/tests/kb-eval/test-skill-score-gate.test.mjs new file mode 100644 index 0000000..d8540b4 --- /dev/null +++ b/tests/kb-eval/test-skill-score-gate.test.mjs @@ -0,0 +1,146 @@ +// tests/kb-eval/test-skill-score-gate.test.mjs +// Tests for the Spor D, STEG B lifecycle-gate building blocks: +// - lib/skill-score.mjs gateSkill(report, name): the PURE post-mutation verdict +// used by apply-skill-op.mjs to re-score the affected skill incrementally. +// The deterministic floor (K10) is always enforced; an unjudged skill is +// flagged `provisional` (the K1 floor cannot be enforced -> nudge to re-judge). +// - lib/skill-score.mjs buildScoreCache(result): the compact, PURE cache shape +// persisted as data/skill-score-report.json and consumed by STEG C surfacing. +// - the score-skill.mjs CLI --write flag: persists the whole-corpus cache. + +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { execFileSync } from 'node:child_process'; +import { readFileSync, rmSync } from 'node:fs'; +import { fileURLToPath } from 'node:url'; +import { dirname, join } from 'node:path'; +import { tmpdir } from 'node:os'; +import { gateSkill, buildScoreCache, scoreReport } from '../../scripts/kb-eval/lib/skill-score.mjs'; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const CLI = join(__dirname, '..', '..', 'scripts', 'kb-eval', 'score-skill.mjs'); + +// A synthetic skill whose deterministic criteria all pass; K10 + judge are +// parametrizable so a single helper drives the meets/blocked/provisional cases. +function skill(name, { k10Pass = true, k10Combined = 1, worst = 'other', judge = null } = {}) { + return { + name, + deterministic: { + K2_descriptionFormat: { pass: true }, + K3_bodyLength: { bodyLines: 100, pass: true }, + K5_progressiveDisclosure: { namedRatio: 0.5, pass: true }, + K6_routingTable: { pass: true }, + refCountConsistency: { consistent: true }, + K10_siblingScopeOverlap: { maxCombined: k10Combined, pass: k10Pass, threshold: 7, worstSibling: worst }, + N1_nameValidity: { pass: true }, + N2_descriptionLength: { pass: true }, + N3_refNestingDepth: { pass: true }, + N4_refToc: { ratio: 1, pass: true, withToc: 1, largeFiles: 1 }, + N5_forwardSlashPaths: { pass: true }, + }, + judge, + }; +} + +const FULL_JUDGE = { + K1_triggerPrecision: { precision: 1, pass: true }, + K4_noDuplication: { score: 5, pass: true }, + K9_noTimeSensitive: { pass: true, findings: [] }, + K7_imperativeStyle: { ratio: 1, pass: true }, + K8_sourceCitation: { ratio: 1, pass: true }, +}; + +test('gateSkill — affected skill that meets the target is not blocked', () => { + const report = { skills: [skill('alpha', { judge: FULL_JUDGE }), skill('beta', { judge: FULL_JUDGE })] }; + const v = gateSkill(report, 'alpha', { target: 90 }); + assert.equal(v.found, true); + assert.equal(v.meetsTarget, true); + assert.equal(v.blocked, false); + assert.equal(v.provisional, false); +}); + +test('gateSkill — a failing K10 floor blocks the affected skill (deterministic, enforced immediately)', () => { + const report = { + skills: [skill('good', { judge: FULL_JUDGE }), skill('bad', { k10Pass: false, k10Combined: 9, worst: 'good', judge: FULL_JUDGE })], + }; + const v = gateSkill(report, 'bad', { target: 90 }); + assert.equal(v.found, true); + assert.equal(v.meetsTarget, false); + assert.equal(v.blocked, true); + assert.ok(v.score <= 89, `K10 floor must cap below 90, got ${v.score}`); + assert.ok(v.improvements.some((i) => i.key === 'K10'), 'K10 must surface as an improvement'); +}); + +test('gateSkill — an unjudged skill is provisional but the deterministic floor still blocks it', () => { + // judge:null -> K1 floor cannot be enforced (provisional), but K10 (deterministic) still blocks. + const report = { skills: [skill('good'), skill('bad', { k10Pass: false, k10Combined: 9, worst: 'good' })] }; + const blocked = gateSkill(report, 'bad', { target: 90 }); + assert.equal(blocked.provisional, true, 'missing judge -> provisional'); + assert.equal(blocked.blocked, true, 'deterministic K10 floor still blocks when provisional'); + + const ok = gateSkill(report, 'good', { target: 90 }); + assert.equal(ok.provisional, true, 'missing judge -> provisional even when passing'); + assert.equal(ok.meetsTarget, true, 'deterministic-only score still passes'); +}); + +test('gateSkill — an unknown affected skill returns found:false (no crash)', () => { + const report = { skills: [skill('alpha', { judge: FULL_JUDGE })] }; + const v = gateSkill(report, 'does-not-exist', { target: 90 }); + assert.equal(v.found, false); + assert.equal(v.name, 'does-not-exist'); +}); + +test('buildScoreCache — compact whole-corpus shape with below[] as names', () => { + const report = { + skills: [skill('good', { judge: FULL_JUDGE }), skill('bad', { k10Pass: false, k10Combined: 9, worst: 'good', judge: FULL_JUDGE })], + }; + const result = scoreReport(report, { target: 90 }); + const cache = buildScoreCache(result, { generatedAt: '2026-06-23' }); + assert.equal(cache.target, 90); + assert.equal(cache.generatedAt, '2026-06-23'); + assert.equal(cache.scored.length, 2); + // compact per-skill entry: the fields STEG C surfacing needs, not the verbose criteria[] + const good = cache.scored.find((s) => s.name === 'good'); + assert.equal(typeof good.score, 'number'); + assert.equal(typeof good.meetsTarget, 'boolean'); + assert.ok(!('criteria' in good), 'cache must stay compact (no criteria[])'); + // below[] is a list of NAMES (cheap for the SessionStart one-liner) + assert.deepEqual(cache.below, ['bad']); +}); + +test('buildScoreCache — deterministic given an injected generatedAt (no Date.now)', () => { + const report = { skills: [skill('alpha', { judge: FULL_JUDGE })] }; + const result = scoreReport(report, { target: 90 }); + const a = buildScoreCache(result, { generatedAt: '2026-01-01' }); + const b = buildScoreCache(result, { generatedAt: '2026-01-01' }); + assert.deepEqual(a, b); +}); + +test('CLI --write — persists the whole-corpus cache (all 5 skills) to disk', () => { + const out = join(tmpdir(), `skill-score-cache-${process.pid}.json`); + try { + execFileSync('node', [CLI, '--write', out], { encoding: 'utf8' }); + const cache = JSON.parse(readFileSync(out, 'utf8')); + assert.equal(cache.target, 90); + assert.equal(cache.scored.length, 5, 'cache must cover the whole corpus, not a filtered view'); + assert.ok(Array.isArray(cache.below), 'below[] present'); + for (const s of cache.scored) { + assert.equal(typeof s.name, 'string'); + assert.equal(typeof s.score, 'number'); + assert.equal(typeof s.meetsTarget, 'boolean'); + } + } finally { + rmSync(out, { force: true }); + } +}); + +test('CLI --write with --skill still writes the whole corpus (cache is corpus-wide)', () => { + const out = join(tmpdir(), `skill-score-cache-corpus-${process.pid}.json`); + try { + execFileSync('node', [CLI, '--skill', 'ms-ai-advisor', '--write', out], { encoding: 'utf8' }); + const cache = JSON.parse(readFileSync(out, 'utf8')); + assert.equal(cache.scored.length, 5, '--skill scopes stdout, not the persisted cache'); + } finally { + rmSync(out, { force: true }); + } +});