open/ms-ai-architect
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity Actions

feat(ms-ai-architect): Spor D steg 1 — apply regenererer hel-korpus score-cache post-mutasjon (TDD) [skip-docs]

Browse source 
apply-skill-op.mjs skrev aldri score-cachen (data/skill-score-report.json) selv
— kun score-skill.mjs --write gjorde det. Etter en create/merge/sanitize/retire-
apply var derfor STEG C SessionStart-surfacingen stale til operatøren manuelt
re-scoret.

Ny eksportert refreshScoreCache(report, {cachePath, generatedAt}): ren
scoreReport+buildScoreCache, én writeFileSync, injiserbar generatedAt (determini-
stisk i test). main() bygger nå buildReport() ÉN gang ved res.applied og deler
den mellom printQualityGate(report,…) (refaktorert til å ta report) og
refreshScoreCache(report). Hel-korpus med vilje: retire dropper, create legger
til, merge/sanitize flytter søsken-K10 → cachen er uavhengig av HVILKEN skill
endret seg.

TDD: 4 nye assert i tests/kb-eval/test-apply-score-cache.test.mjs (RED→GREEN).
kb-eval 154/154, kb-update 323/323, validate 239/0, kb-integrity 192/0. Real
cache er gitignored → testene skriver kun til tmpdir.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Kjell Tore Guttormsen 2026-06-23 19:51:04 +02:00
commit c413d335bc
2 changed files with 147 additions and 12 deletions
Download patch file Download diff file Expand all files Collapse all files

56
scripts/kb-eval/apply-skill-op.mjs
View file

@ -21,11 +21,11 @@
// node scripts/kb-eval/apply-skill-op.mjs retire <skill> [--apply] [--date YYYY-MM-DD]
// node scripts/kb-eval/apply-skill-op.mjs merge <absorber> <absorbed> [--apply] [--date YYYY-MM-DD]
import { readFileSync, readdirSync, existsSync } from 'node:fs';
import { readFileSync, readdirSync, existsSync, writeFileSync } from 'node:fs';
import { dirname, join } from 'node:path';
import { fileURLToPath } from 'node:url';
import { splitFrontmatter, extractDescription, buildReport } from './eval.mjs';
import { gateSkill } from './lib/skill-score.mjs';
import { gateSkill, scoreReport, buildScoreCache } from './lib/skill-score.mjs';
import { loadTaxonomy } from '../kb-update/lib/taxonomy.mjs';
import { loadDecisions, listActions, actionKey } from '../kb-update/lib/decisions-io.mjs';
import {
@ -43,6 +43,10 @@ const SKILLS_DIR = join(PLUGIN_ROOT, 'skills');
const AGENTS_DIR = join(PLUGIN_ROOT, 'agents');
const LEDGER_DATA_DIR = join(__dirname, '..', 'kb-update', 'data');
const PROMPTS_FILE = join(__dirname, 'data', 'k1-trigger-prompts.json');
// Same cache score-skill.mjs --write maintains and the STEG C SessionStart
// surfacing reads — refreshed here after every applied mutation so the surfacing
// never goes stale until the next manual re-score.
export const SCORE_CACHE_FILE = join(__dirname, 'data', 'skill-score-report.json');
const OP_BY_VERB = {
sanitize: SANITIZE_OPERATION,
@ -153,18 +157,20 @@ function affectedSkill(op, targets) {
}
/**
* Re-score the affected skill from fresh disk after a successful apply and surface
* a quality verdict. The deterministic floor (K10) is enforced immediately as a
* loud warning; an unjudged skill is flagged provisional with a nudge to re-run
* the judge pass (the K1 floor cannot be enforced from the cache alone).
* Re-score the affected skill from a freshly-built corpus report after a
* successful apply and surface a quality verdict. The deterministic floor (K10)
* is enforced immediately as a loud warning; an unjudged skill is flagged
* provisional with a nudge to re-run the judge pass (the K1 floor cannot be
* enforced from the cache alone). The report is built once by the caller and
* shared with refreshScoreCache so the corpus is scored a single time.
*/
function printQualityGate(op, targets) {
function printQualityGate(report, op, targets) {
const affected = affectedSkill(op, targets);
if (!affected) {
console.log(`\nKvalitetsgate: ingen gjenværende skill å re-score (retire). Hopper over.`);
return;
}
const v = gateSkill(buildReport(), affected, { target: QUALITY_TARGET });
const v = gateSkill(report, affected, { target: QUALITY_TARGET });
if (!v.found) {
console.log(`\nKvalitetsgate: fant ikke '${affected}' på disk for re-scoring (hoppet over).`);
return;
@ -188,6 +194,25 @@ function printQualityGate(op, targets) {
}
}
/**
* Regenerate the WHOLE-CORPUS score-cache from a freshly-built report after a
* successful mutation, so the STEG C SessionStart surfacing reflects the new disk
* immediately. Whole-corpus on purpose: retire drops a skill, create adds one, and
* merge/sanitize shift sibling K10 — the cache is independent of WHICH skill
* changed. Pure scoring (scoreReport/buildScoreCache); only the write touches disk.
* generatedAt is injectable so tests stay deterministic.
* @param {{skills: object[]}} report buildReport() output (fresh disk)
* @param {{cachePath?: string, generatedAt?: string}} [opts]
* @returns {object} the cache object that was written
*/
export function refreshScoreCache(report, opts = {}) {
const cachePath = opts.cachePath ?? SCORE_CACHE_FILE;
const generatedAt = opts.generatedAt ?? new Date().toISOString().slice(0, 10);
const cache = buildScoreCache(scoreReport(report, { target: QUALITY_TARGET }), { generatedAt });
writeFileSync(cachePath, JSON.stringify(cache, null, 2) + '\n');
return cache;
}
function main() {
const args = process.argv.slice(2);
const verb = args[0];
@ -261,10 +286,17 @@ function main() {
});
printResult(res, verb, label, apply);
// STEG B: re-score the affected skill from fresh disk after a real mutation.
// Preview never mutates -> nothing to re-score (create's projected skill isn't
// on disk), so the gate runs only on applied mutations.
if (res.applied) printQualityGate(op, targets);
// STEG B+1: after a real mutation, build the corpus report ONCE and reuse it for
// both the affected-skill gate (STEG B) and the whole-corpus score-cache refresh
// (STEG 1) — so the STEG C SessionStart surfacing reflects the new disk instead of
// going stale until the next manual `score-skill.mjs --write`. Preview never
// mutates -> nothing to re-score/refresh, so both run only on applied mutations.
if (res.applied) {
const report = buildReport();
printQualityGate(report, op, targets);
const cache = refreshScoreCache(report);
console.log(`\n↻ Score-cache oppdatert → ${SCORE_CACHE_FILE} (${cache.scored.length} skills, ${cache.below.length} under mål).`);
}
}
if (process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1]) {