feat(ms-ai-architect): Spor D steg 1 — apply regenererer hel-korpus score-cache post-mutasjon (TDD) [skip-docs]

apply-skill-op.mjs skrev aldri score-cachen (data/skill-score-report.json) selv
— kun score-skill.mjs --write gjorde det. Etter en create/merge/sanitize/retire-
apply var derfor STEG C SessionStart-surfacingen stale til operatøren manuelt
re-scoret.

Ny eksportert refreshScoreCache(report, {cachePath, generatedAt}): ren
scoreReport+buildScoreCache, én writeFileSync, injiserbar generatedAt (determini-
stisk i test). main() bygger nå buildReport() ÉN gang ved res.applied og deler
den mellom printQualityGate(report,…) (refaktorert til å ta report) og
refreshScoreCache(report). Hel-korpus med vilje: retire dropper, create legger
til, merge/sanitize flytter søsken-K10 → cachen er uavhengig av HVILKEN skill
endret seg.

TDD: 4 nye assert i tests/kb-eval/test-apply-score-cache.test.mjs (RED→GREEN).
kb-eval 154/154, kb-update 323/323, validate 239/0, kb-integrity 192/0. Real
cache er gitignored → testene skriver kun til tmpdir.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Kjell Tore Guttormsen 2026-06-23 19:51:04 +02:00
commit c413d335bc
2 changed files with 147 additions and 12 deletions

View file

@ -21,11 +21,11 @@
// node scripts/kb-eval/apply-skill-op.mjs retire <skill> [--apply] [--date YYYY-MM-DD]
// node scripts/kb-eval/apply-skill-op.mjs merge <absorber> <absorbed> [--apply] [--date YYYY-MM-DD]
import { readFileSync, readdirSync, existsSync } from 'node:fs';
import { readFileSync, readdirSync, existsSync, writeFileSync } from 'node:fs';
import { dirname, join } from 'node:path';
import { fileURLToPath } from 'node:url';
import { splitFrontmatter, extractDescription, buildReport } from './eval.mjs';
import { gateSkill } from './lib/skill-score.mjs';
import { gateSkill, scoreReport, buildScoreCache } from './lib/skill-score.mjs';
import { loadTaxonomy } from '../kb-update/lib/taxonomy.mjs';
import { loadDecisions, listActions, actionKey } from '../kb-update/lib/decisions-io.mjs';
import {
@ -43,6 +43,10 @@ const SKILLS_DIR = join(PLUGIN_ROOT, 'skills');
const AGENTS_DIR = join(PLUGIN_ROOT, 'agents');
const LEDGER_DATA_DIR = join(__dirname, '..', 'kb-update', 'data');
const PROMPTS_FILE = join(__dirname, 'data', 'k1-trigger-prompts.json');
// Same cache score-skill.mjs --write maintains and the STEG C SessionStart
// surfacing reads — refreshed here after every applied mutation so the surfacing
// never goes stale until the next manual re-score.
export const SCORE_CACHE_FILE = join(__dirname, 'data', 'skill-score-report.json');
const OP_BY_VERB = {
sanitize: SANITIZE_OPERATION,
@ -153,18 +157,20 @@ function affectedSkill(op, targets) {
}
/**
* Re-score the affected skill from fresh disk after a successful apply and surface
* a quality verdict. The deterministic floor (K10) is enforced immediately as a
* loud warning; an unjudged skill is flagged provisional with a nudge to re-run
* the judge pass (the K1 floor cannot be enforced from the cache alone).
* Re-score the affected skill from a freshly-built corpus report after a
* successful apply and surface a quality verdict. The deterministic floor (K10)
* is enforced immediately as a loud warning; an unjudged skill is flagged
* provisional with a nudge to re-run the judge pass (the K1 floor cannot be
* enforced from the cache alone). The report is built once by the caller and
* shared with refreshScoreCache so the corpus is scored a single time.
*/
function printQualityGate(op, targets) {
function printQualityGate(report, op, targets) {
const affected = affectedSkill(op, targets);
if (!affected) {
console.log(`\nKvalitetsgate: ingen gjenværende skill å re-score (retire). Hopper over.`);
return;
}
const v = gateSkill(buildReport(), affected, { target: QUALITY_TARGET });
const v = gateSkill(report, affected, { target: QUALITY_TARGET });
if (!v.found) {
console.log(`\nKvalitetsgate: fant ikke '${affected}' på disk for re-scoring (hoppet over).`);
return;
@ -188,6 +194,25 @@ function printQualityGate(op, targets) {
}
}
/**
* Regenerate the WHOLE-CORPUS score-cache from a freshly-built report after a
* successful mutation, so the STEG C SessionStart surfacing reflects the new disk
* immediately. Whole-corpus on purpose: retire drops a skill, create adds one, and
* merge/sanitize shift sibling K10 the cache is independent of WHICH skill
* changed. Pure scoring (scoreReport/buildScoreCache); only the write touches disk.
* generatedAt is injectable so tests stay deterministic.
* @param {{skills: object[]}} report buildReport() output (fresh disk)
* @param {{cachePath?: string, generatedAt?: string}} [opts]
* @returns {object} the cache object that was written
*/
export function refreshScoreCache(report, opts = {}) {
const cachePath = opts.cachePath ?? SCORE_CACHE_FILE;
const generatedAt = opts.generatedAt ?? new Date().toISOString().slice(0, 10);
const cache = buildScoreCache(scoreReport(report, { target: QUALITY_TARGET }), { generatedAt });
writeFileSync(cachePath, JSON.stringify(cache, null, 2) + '\n');
return cache;
}
function main() {
const args = process.argv.slice(2);
const verb = args[0];
@ -261,10 +286,17 @@ function main() {
});
printResult(res, verb, label, apply);
// STEG B: re-score the affected skill from fresh disk after a real mutation.
// Preview never mutates -> nothing to re-score (create's projected skill isn't
// on disk), so the gate runs only on applied mutations.
if (res.applied) printQualityGate(op, targets);
// STEG B+1: after a real mutation, build the corpus report ONCE and reuse it for
// both the affected-skill gate (STEG B) and the whole-corpus score-cache refresh
// (STEG 1) — so the STEG C SessionStart surfacing reflects the new disk instead of
// going stale until the next manual `score-skill.mjs --write`. Preview never
// mutates -> nothing to re-score/refresh, so both run only on applied mutations.
if (res.applied) {
const report = buildReport();
printQualityGate(report, op, targets);
const cache = refreshScoreCache(report);
console.log(`\n↻ Score-cache oppdatert → ${SCORE_CACHE_FILE} (${cache.scored.length} skills, ${cache.below.length} under mål).`);
}
}
if (process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1]) {

View file

@ -0,0 +1,103 @@
// tests/kb-eval/test-apply-score-cache.test.mjs
// Spor D STEG 1 (S42): apply-skill-op.mjs refreshes the WHOLE-CORPUS score-cache
// after a successful mutation, so the SessionStart surfacing (STEG C) reflects the
// new disk immediately instead of going stale until the next manual
// `score-skill.mjs --write`.
//
// refreshScoreCache(report, {cachePath, generatedAt}) is the pure-scoring/impure-write
// seam: scoreReport + buildScoreCache (pure) then a single writeFileSync. The cache is
// WHOLE-CORPUS (independent of WHICH skill changed) because retire drops a skill and
// create/merge/sanitize shift sibling K10 — every applied mutation must regenerate it.
import { test } from 'node:test';
import assert from 'node:assert/strict';
import { readFileSync, rmSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { refreshScoreCache, SCORE_CACHE_FILE } from '../../scripts/kb-eval/apply-skill-op.mjs';
import { scoreReport, buildScoreCache } from '../../scripts/kb-eval/lib/skill-score.mjs';
// A synthetic skill whose deterministic + judge criteria all pass; K10 is
// parametrizable so one helper drives both the OK and the below-target cases.
function skill(name, { k10Pass = true, k10Combined = 1, worst = 'other' } = {}) {
return {
name,
deterministic: {
K2_descriptionFormat: { pass: true },
K3_bodyLength: { bodyLines: 100, pass: true },
K5_progressiveDisclosure: { namedRatio: 0.5, pass: true },
K6_routingTable: { pass: true },
refCountConsistency: { consistent: true },
K10_siblingScopeOverlap: { maxCombined: k10Combined, pass: k10Pass, threshold: 7, worstSibling: worst },
N1_nameValidity: { pass: true },
N2_descriptionLength: { pass: true },
N3_refNestingDepth: { pass: true },
N4_refToc: { ratio: 1, pass: true, withToc: 1, largeFiles: 1 },
N5_forwardSlashPaths: { pass: true },
},
judge: {
K1_triggerPrecision: { precision: 1, pass: true },
K4_noDuplication: { score: 5, pass: true },
K9_noTimeSensitive: { pass: true, findings: [] },
K7_imperativeStyle: { ratio: 1, pass: true },
K8_sourceCitation: { ratio: 1, pass: true },
},
};
}
test('SCORE_CACHE_FILE points at the surfacing cache (scripts/kb-eval/data/skill-score-report.json)', () => {
assert.ok(
SCORE_CACHE_FILE.endsWith(join('scripts', 'kb-eval', 'data', 'skill-score-report.json')),
`cache must be the same file STEG C surfacing reads; got: ${SCORE_CACHE_FILE}`,
);
});
test('refreshScoreCache — persists exactly buildScoreCache(scoreReport(report)) and returns it', () => {
const out = join(tmpdir(), `apply-score-cache-${process.pid}.json`);
try {
const report = { skills: [skill('alpha'), skill('beta', { k10Pass: false, k10Combined: 9, worst: 'alpha' })] };
const returned = refreshScoreCache(report, { cachePath: out, generatedAt: '2026-06-23' });
const onDisk = JSON.parse(readFileSync(out, 'utf8'));
const expected = buildScoreCache(scoreReport(report, { target: 90 }), { generatedAt: '2026-06-23' });
assert.deepEqual(onDisk, expected, 'persisted cache must equal buildScoreCache(scoreReport(report))');
assert.deepEqual(returned, expected, 'return value mirrors what was written');
// whole-corpus + compact + below[] as names
assert.equal(onDisk.scored.length, 2);
assert.ok(!('criteria' in onDisk.scored[0]), 'cache stays compact (no verbose criteria[])');
assert.deepEqual(onDisk.below, ['beta'], 'the K10-failing skill surfaces as below-target');
assert.equal(onDisk.generatedAt, '2026-06-23');
} finally {
rmSync(out, { force: true });
}
});
test('refreshScoreCache — OVERWRITES (does not merge): a retire shrinks the cache', () => {
const out = join(tmpdir(), `apply-score-cache-retire-${process.pid}.json`);
try {
// Corpus BEFORE: 3 skills.
refreshScoreCache({ skills: [skill('alpha'), skill('beta'), skill('gamma')] }, { cachePath: out, generatedAt: '2026-06-23' });
assert.equal(JSON.parse(readFileSync(out, 'utf8')).scored.length, 3);
// A retire drops 'gamma' -> refresh with the smaller corpus. The cache is
// overwritten whole-corpus, so the retired skill must not linger and re-surface.
refreshScoreCache({ skills: [skill('alpha'), skill('beta')] }, { cachePath: out, generatedAt: '2026-06-23' });
const after = JSON.parse(readFileSync(out, 'utf8'));
assert.equal(after.scored.length, 2, 'cache overwritten, not merged');
assert.ok(!after.scored.some((s) => s.name === 'gamma'), 'retired skill dropped from the surfacing cache');
} finally {
rmSync(out, { force: true });
}
});
test('refreshScoreCache — generatedAt defaults to an ISO date (today) when not injected', () => {
const out = join(tmpdir(), `apply-score-cache-date-${process.pid}.json`);
try {
refreshScoreCache({ skills: [skill('alpha')] }, { cachePath: out });
const onDisk = JSON.parse(readFileSync(out, 'utf8'));
assert.match(onDisk.generatedAt, /^\d{4}-\d{2}-\d{2}$/, 'impure layer stamps today as YYYY-MM-DD');
} finally {
rmSync(out, { force: true });
}
});