diff --git a/scripts/kb-eval/apply-skill-op.mjs b/scripts/kb-eval/apply-skill-op.mjs index 03b3505..0d0ba26 100644 --- a/scripts/kb-eval/apply-skill-op.mjs +++ b/scripts/kb-eval/apply-skill-op.mjs @@ -21,11 +21,11 @@ // node scripts/kb-eval/apply-skill-op.mjs retire [--apply] [--date YYYY-MM-DD] // node scripts/kb-eval/apply-skill-op.mjs merge [--apply] [--date YYYY-MM-DD] -import { readFileSync, readdirSync, existsSync } from 'node:fs'; +import { readFileSync, readdirSync, existsSync, writeFileSync } from 'node:fs'; import { dirname, join } from 'node:path'; import { fileURLToPath } from 'node:url'; import { splitFrontmatter, extractDescription, buildReport } from './eval.mjs'; -import { gateSkill } from './lib/skill-score.mjs'; +import { gateSkill, scoreReport, buildScoreCache } from './lib/skill-score.mjs'; import { loadTaxonomy } from '../kb-update/lib/taxonomy.mjs'; import { loadDecisions, listActions, actionKey } from '../kb-update/lib/decisions-io.mjs'; import { @@ -43,6 +43,10 @@ const SKILLS_DIR = join(PLUGIN_ROOT, 'skills'); const AGENTS_DIR = join(PLUGIN_ROOT, 'agents'); const LEDGER_DATA_DIR = join(__dirname, '..', 'kb-update', 'data'); const PROMPTS_FILE = join(__dirname, 'data', 'k1-trigger-prompts.json'); +// Same cache score-skill.mjs --write maintains and the STEG C SessionStart +// surfacing reads — refreshed here after every applied mutation so the surfacing +// never goes stale until the next manual re-score. +export const SCORE_CACHE_FILE = join(__dirname, 'data', 'skill-score-report.json'); const OP_BY_VERB = { sanitize: SANITIZE_OPERATION, @@ -153,18 +157,20 @@ function affectedSkill(op, targets) { } /** - * Re-score the affected skill from fresh disk after a successful apply and surface - * a quality verdict. The deterministic floor (K10) is enforced immediately as a - * loud warning; an unjudged skill is flagged provisional with a nudge to re-run - * the judge pass (the K1 floor cannot be enforced from the cache alone). + * Re-score the affected skill from a freshly-built corpus report after a + * successful apply and surface a quality verdict. The deterministic floor (K10) + * is enforced immediately as a loud warning; an unjudged skill is flagged + * provisional with a nudge to re-run the judge pass (the K1 floor cannot be + * enforced from the cache alone). The report is built once by the caller and + * shared with refreshScoreCache so the corpus is scored a single time. */ -function printQualityGate(op, targets) { +function printQualityGate(report, op, targets) { const affected = affectedSkill(op, targets); if (!affected) { console.log(`\nKvalitetsgate: ingen gjenværende skill å re-score (retire). Hopper over.`); return; } - const v = gateSkill(buildReport(), affected, { target: QUALITY_TARGET }); + const v = gateSkill(report, affected, { target: QUALITY_TARGET }); if (!v.found) { console.log(`\nKvalitetsgate: fant ikke '${affected}' på disk for re-scoring (hoppet over).`); return; @@ -188,6 +194,25 @@ function printQualityGate(op, targets) { } } +/** + * Regenerate the WHOLE-CORPUS score-cache from a freshly-built report after a + * successful mutation, so the STEG C SessionStart surfacing reflects the new disk + * immediately. Whole-corpus on purpose: retire drops a skill, create adds one, and + * merge/sanitize shift sibling K10 — the cache is independent of WHICH skill + * changed. Pure scoring (scoreReport/buildScoreCache); only the write touches disk. + * generatedAt is injectable so tests stay deterministic. + * @param {{skills: object[]}} report buildReport() output (fresh disk) + * @param {{cachePath?: string, generatedAt?: string}} [opts] + * @returns {object} the cache object that was written + */ +export function refreshScoreCache(report, opts = {}) { + const cachePath = opts.cachePath ?? SCORE_CACHE_FILE; + const generatedAt = opts.generatedAt ?? new Date().toISOString().slice(0, 10); + const cache = buildScoreCache(scoreReport(report, { target: QUALITY_TARGET }), { generatedAt }); + writeFileSync(cachePath, JSON.stringify(cache, null, 2) + '\n'); + return cache; +} + function main() { const args = process.argv.slice(2); const verb = args[0]; @@ -261,10 +286,17 @@ function main() { }); printResult(res, verb, label, apply); - // STEG B: re-score the affected skill from fresh disk after a real mutation. - // Preview never mutates -> nothing to re-score (create's projected skill isn't - // on disk), so the gate runs only on applied mutations. - if (res.applied) printQualityGate(op, targets); + // STEG B+1: after a real mutation, build the corpus report ONCE and reuse it for + // both the affected-skill gate (STEG B) and the whole-corpus score-cache refresh + // (STEG 1) — so the STEG C SessionStart surfacing reflects the new disk instead of + // going stale until the next manual `score-skill.mjs --write`. Preview never + // mutates -> nothing to re-score/refresh, so both run only on applied mutations. + if (res.applied) { + const report = buildReport(); + printQualityGate(report, op, targets); + const cache = refreshScoreCache(report); + console.log(`\n↻ Score-cache oppdatert → ${SCORE_CACHE_FILE} (${cache.scored.length} skills, ${cache.below.length} under mål).`); + } } if (process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1]) { diff --git a/tests/kb-eval/test-apply-score-cache.test.mjs b/tests/kb-eval/test-apply-score-cache.test.mjs new file mode 100644 index 0000000..c47f4c9 --- /dev/null +++ b/tests/kb-eval/test-apply-score-cache.test.mjs @@ -0,0 +1,103 @@ +// tests/kb-eval/test-apply-score-cache.test.mjs +// Spor D STEG 1 (S42): apply-skill-op.mjs refreshes the WHOLE-CORPUS score-cache +// after a successful mutation, so the SessionStart surfacing (STEG C) reflects the +// new disk immediately instead of going stale until the next manual +// `score-skill.mjs --write`. +// +// refreshScoreCache(report, {cachePath, generatedAt}) is the pure-scoring/impure-write +// seam: scoreReport + buildScoreCache (pure) then a single writeFileSync. The cache is +// WHOLE-CORPUS (independent of WHICH skill changed) because retire drops a skill and +// create/merge/sanitize shift sibling K10 — every applied mutation must regenerate it. + +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { readFileSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { refreshScoreCache, SCORE_CACHE_FILE } from '../../scripts/kb-eval/apply-skill-op.mjs'; +import { scoreReport, buildScoreCache } from '../../scripts/kb-eval/lib/skill-score.mjs'; + +// A synthetic skill whose deterministic + judge criteria all pass; K10 is +// parametrizable so one helper drives both the OK and the below-target cases. +function skill(name, { k10Pass = true, k10Combined = 1, worst = 'other' } = {}) { + return { + name, + deterministic: { + K2_descriptionFormat: { pass: true }, + K3_bodyLength: { bodyLines: 100, pass: true }, + K5_progressiveDisclosure: { namedRatio: 0.5, pass: true }, + K6_routingTable: { pass: true }, + refCountConsistency: { consistent: true }, + K10_siblingScopeOverlap: { maxCombined: k10Combined, pass: k10Pass, threshold: 7, worstSibling: worst }, + N1_nameValidity: { pass: true }, + N2_descriptionLength: { pass: true }, + N3_refNestingDepth: { pass: true }, + N4_refToc: { ratio: 1, pass: true, withToc: 1, largeFiles: 1 }, + N5_forwardSlashPaths: { pass: true }, + }, + judge: { + K1_triggerPrecision: { precision: 1, pass: true }, + K4_noDuplication: { score: 5, pass: true }, + K9_noTimeSensitive: { pass: true, findings: [] }, + K7_imperativeStyle: { ratio: 1, pass: true }, + K8_sourceCitation: { ratio: 1, pass: true }, + }, + }; +} + +test('SCORE_CACHE_FILE points at the surfacing cache (scripts/kb-eval/data/skill-score-report.json)', () => { + assert.ok( + SCORE_CACHE_FILE.endsWith(join('scripts', 'kb-eval', 'data', 'skill-score-report.json')), + `cache must be the same file STEG C surfacing reads; got: ${SCORE_CACHE_FILE}`, + ); +}); + +test('refreshScoreCache — persists exactly buildScoreCache(scoreReport(report)) and returns it', () => { + const out = join(tmpdir(), `apply-score-cache-${process.pid}.json`); + try { + const report = { skills: [skill('alpha'), skill('beta', { k10Pass: false, k10Combined: 9, worst: 'alpha' })] }; + const returned = refreshScoreCache(report, { cachePath: out, generatedAt: '2026-06-23' }); + + const onDisk = JSON.parse(readFileSync(out, 'utf8')); + const expected = buildScoreCache(scoreReport(report, { target: 90 }), { generatedAt: '2026-06-23' }); + assert.deepEqual(onDisk, expected, 'persisted cache must equal buildScoreCache(scoreReport(report))'); + assert.deepEqual(returned, expected, 'return value mirrors what was written'); + + // whole-corpus + compact + below[] as names + assert.equal(onDisk.scored.length, 2); + assert.ok(!('criteria' in onDisk.scored[0]), 'cache stays compact (no verbose criteria[])'); + assert.deepEqual(onDisk.below, ['beta'], 'the K10-failing skill surfaces as below-target'); + assert.equal(onDisk.generatedAt, '2026-06-23'); + } finally { + rmSync(out, { force: true }); + } +}); + +test('refreshScoreCache — OVERWRITES (does not merge): a retire shrinks the cache', () => { + const out = join(tmpdir(), `apply-score-cache-retire-${process.pid}.json`); + try { + // Corpus BEFORE: 3 skills. + refreshScoreCache({ skills: [skill('alpha'), skill('beta'), skill('gamma')] }, { cachePath: out, generatedAt: '2026-06-23' }); + assert.equal(JSON.parse(readFileSync(out, 'utf8')).scored.length, 3); + + // A retire drops 'gamma' -> refresh with the smaller corpus. The cache is + // overwritten whole-corpus, so the retired skill must not linger and re-surface. + refreshScoreCache({ skills: [skill('alpha'), skill('beta')] }, { cachePath: out, generatedAt: '2026-06-23' }); + const after = JSON.parse(readFileSync(out, 'utf8')); + assert.equal(after.scored.length, 2, 'cache overwritten, not merged'); + assert.ok(!after.scored.some((s) => s.name === 'gamma'), 'retired skill dropped from the surfacing cache'); + } finally { + rmSync(out, { force: true }); + } +}); + +test('refreshScoreCache — generatedAt defaults to an ISO date (today) when not injected', () => { + const out = join(tmpdir(), `apply-score-cache-date-${process.pid}.json`); + try { + refreshScoreCache({ skills: [skill('alpha')] }, { cachePath: out }); + const onDisk = JSON.parse(readFileSync(out, 'utf8')); + assert.match(onDisk.generatedAt, /^\d{4}-\d{2}-\d{2}$/, 'impure layer stamps today as YYYY-MM-DD'); + } finally { + rmSync(out, { force: true }); + } +});