feat(ms-ai-architect): Spor D steg 1 — apply regenererer hel-korpus score-cache post-mutasjon (TDD) [skip-docs]
apply-skill-op.mjs skrev aldri score-cachen (data/skill-score-report.json) selv
— kun score-skill.mjs --write gjorde det. Etter en create/merge/sanitize/retire-
apply var derfor STEG C SessionStart-surfacingen stale til operatøren manuelt
re-scoret.
Ny eksportert refreshScoreCache(report, {cachePath, generatedAt}): ren
scoreReport+buildScoreCache, én writeFileSync, injiserbar generatedAt (determini-
stisk i test). main() bygger nå buildReport() ÉN gang ved res.applied og deler
den mellom printQualityGate(report,…) (refaktorert til å ta report) og
refreshScoreCache(report). Hel-korpus med vilje: retire dropper, create legger
til, merge/sanitize flytter søsken-K10 → cachen er uavhengig av HVILKEN skill
endret seg.
TDD: 4 nye assert i tests/kb-eval/test-apply-score-cache.test.mjs (RED→GREEN).
kb-eval 154/154, kb-update 323/323, validate 239/0, kb-integrity 192/0. Real
cache er gitignored → testene skriver kun til tmpdir.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
d6a60ca480
commit
c413d335bc
2 changed files with 147 additions and 12 deletions
|
|
@ -21,11 +21,11 @@
|
|||
// node scripts/kb-eval/apply-skill-op.mjs retire <skill> [--apply] [--date YYYY-MM-DD]
|
||||
// node scripts/kb-eval/apply-skill-op.mjs merge <absorber> <absorbed> [--apply] [--date YYYY-MM-DD]
|
||||
|
||||
import { readFileSync, readdirSync, existsSync } from 'node:fs';
|
||||
import { readFileSync, readdirSync, existsSync, writeFileSync } from 'node:fs';
|
||||
import { dirname, join } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { splitFrontmatter, extractDescription, buildReport } from './eval.mjs';
|
||||
import { gateSkill } from './lib/skill-score.mjs';
|
||||
import { gateSkill, scoreReport, buildScoreCache } from './lib/skill-score.mjs';
|
||||
import { loadTaxonomy } from '../kb-update/lib/taxonomy.mjs';
|
||||
import { loadDecisions, listActions, actionKey } from '../kb-update/lib/decisions-io.mjs';
|
||||
import {
|
||||
|
|
@ -43,6 +43,10 @@ const SKILLS_DIR = join(PLUGIN_ROOT, 'skills');
|
|||
const AGENTS_DIR = join(PLUGIN_ROOT, 'agents');
|
||||
const LEDGER_DATA_DIR = join(__dirname, '..', 'kb-update', 'data');
|
||||
const PROMPTS_FILE = join(__dirname, 'data', 'k1-trigger-prompts.json');
|
||||
// Same cache score-skill.mjs --write maintains and the STEG C SessionStart
|
||||
// surfacing reads — refreshed here after every applied mutation so the surfacing
|
||||
// never goes stale until the next manual re-score.
|
||||
export const SCORE_CACHE_FILE = join(__dirname, 'data', 'skill-score-report.json');
|
||||
|
||||
const OP_BY_VERB = {
|
||||
sanitize: SANITIZE_OPERATION,
|
||||
|
|
@ -153,18 +157,20 @@ function affectedSkill(op, targets) {
|
|||
}
|
||||
|
||||
/**
|
||||
* Re-score the affected skill from fresh disk after a successful apply and surface
|
||||
* a quality verdict. The deterministic floor (K10) is enforced immediately as a
|
||||
* loud warning; an unjudged skill is flagged provisional with a nudge to re-run
|
||||
* the judge pass (the K1 floor cannot be enforced from the cache alone).
|
||||
* Re-score the affected skill from a freshly-built corpus report after a
|
||||
* successful apply and surface a quality verdict. The deterministic floor (K10)
|
||||
* is enforced immediately as a loud warning; an unjudged skill is flagged
|
||||
* provisional with a nudge to re-run the judge pass (the K1 floor cannot be
|
||||
* enforced from the cache alone). The report is built once by the caller and
|
||||
* shared with refreshScoreCache so the corpus is scored a single time.
|
||||
*/
|
||||
function printQualityGate(op, targets) {
|
||||
function printQualityGate(report, op, targets) {
|
||||
const affected = affectedSkill(op, targets);
|
||||
if (!affected) {
|
||||
console.log(`\nKvalitetsgate: ingen gjenværende skill å re-score (retire). Hopper over.`);
|
||||
return;
|
||||
}
|
||||
const v = gateSkill(buildReport(), affected, { target: QUALITY_TARGET });
|
||||
const v = gateSkill(report, affected, { target: QUALITY_TARGET });
|
||||
if (!v.found) {
|
||||
console.log(`\nKvalitetsgate: fant ikke '${affected}' på disk for re-scoring (hoppet over).`);
|
||||
return;
|
||||
|
|
@ -188,6 +194,25 @@ function printQualityGate(op, targets) {
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Regenerate the WHOLE-CORPUS score-cache from a freshly-built report after a
|
||||
* successful mutation, so the STEG C SessionStart surfacing reflects the new disk
|
||||
* immediately. Whole-corpus on purpose: retire drops a skill, create adds one, and
|
||||
* merge/sanitize shift sibling K10 — the cache is independent of WHICH skill
|
||||
* changed. Pure scoring (scoreReport/buildScoreCache); only the write touches disk.
|
||||
* generatedAt is injectable so tests stay deterministic.
|
||||
* @param {{skills: object[]}} report buildReport() output (fresh disk)
|
||||
* @param {{cachePath?: string, generatedAt?: string}} [opts]
|
||||
* @returns {object} the cache object that was written
|
||||
*/
|
||||
export function refreshScoreCache(report, opts = {}) {
|
||||
const cachePath = opts.cachePath ?? SCORE_CACHE_FILE;
|
||||
const generatedAt = opts.generatedAt ?? new Date().toISOString().slice(0, 10);
|
||||
const cache = buildScoreCache(scoreReport(report, { target: QUALITY_TARGET }), { generatedAt });
|
||||
writeFileSync(cachePath, JSON.stringify(cache, null, 2) + '\n');
|
||||
return cache;
|
||||
}
|
||||
|
||||
function main() {
|
||||
const args = process.argv.slice(2);
|
||||
const verb = args[0];
|
||||
|
|
@ -261,10 +286,17 @@ function main() {
|
|||
});
|
||||
printResult(res, verb, label, apply);
|
||||
|
||||
// STEG B: re-score the affected skill from fresh disk after a real mutation.
|
||||
// Preview never mutates -> nothing to re-score (create's projected skill isn't
|
||||
// on disk), so the gate runs only on applied mutations.
|
||||
if (res.applied) printQualityGate(op, targets);
|
||||
// STEG B+1: after a real mutation, build the corpus report ONCE and reuse it for
|
||||
// both the affected-skill gate (STEG B) and the whole-corpus score-cache refresh
|
||||
// (STEG 1) — so the STEG C SessionStart surfacing reflects the new disk instead of
|
||||
// going stale until the next manual `score-skill.mjs --write`. Preview never
|
||||
// mutates -> nothing to re-score/refresh, so both run only on applied mutations.
|
||||
if (res.applied) {
|
||||
const report = buildReport();
|
||||
printQualityGate(report, op, targets);
|
||||
const cache = refreshScoreCache(report);
|
||||
console.log(`\n↻ Score-cache oppdatert → ${SCORE_CACHE_FILE} (${cache.scored.length} skills, ${cache.below.length} under mål).`);
|
||||
}
|
||||
}
|
||||
|
||||
if (process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1]) {
|
||||
|
|
|
|||
103
tests/kb-eval/test-apply-score-cache.test.mjs
Normal file
103
tests/kb-eval/test-apply-score-cache.test.mjs
Normal file
|
|
@ -0,0 +1,103 @@
|
|||
// tests/kb-eval/test-apply-score-cache.test.mjs
|
||||
// Spor D STEG 1 (S42): apply-skill-op.mjs refreshes the WHOLE-CORPUS score-cache
|
||||
// after a successful mutation, so the SessionStart surfacing (STEG C) reflects the
|
||||
// new disk immediately instead of going stale until the next manual
|
||||
// `score-skill.mjs --write`.
|
||||
//
|
||||
// refreshScoreCache(report, {cachePath, generatedAt}) is the pure-scoring/impure-write
|
||||
// seam: scoreReport + buildScoreCache (pure) then a single writeFileSync. The cache is
|
||||
// WHOLE-CORPUS (independent of WHICH skill changed) because retire drops a skill and
|
||||
// create/merge/sanitize shift sibling K10 — every applied mutation must regenerate it.
|
||||
|
||||
import { test } from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { readFileSync, rmSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { refreshScoreCache, SCORE_CACHE_FILE } from '../../scripts/kb-eval/apply-skill-op.mjs';
|
||||
import { scoreReport, buildScoreCache } from '../../scripts/kb-eval/lib/skill-score.mjs';
|
||||
|
||||
// A synthetic skill whose deterministic + judge criteria all pass; K10 is
|
||||
// parametrizable so one helper drives both the OK and the below-target cases.
|
||||
function skill(name, { k10Pass = true, k10Combined = 1, worst = 'other' } = {}) {
|
||||
return {
|
||||
name,
|
||||
deterministic: {
|
||||
K2_descriptionFormat: { pass: true },
|
||||
K3_bodyLength: { bodyLines: 100, pass: true },
|
||||
K5_progressiveDisclosure: { namedRatio: 0.5, pass: true },
|
||||
K6_routingTable: { pass: true },
|
||||
refCountConsistency: { consistent: true },
|
||||
K10_siblingScopeOverlap: { maxCombined: k10Combined, pass: k10Pass, threshold: 7, worstSibling: worst },
|
||||
N1_nameValidity: { pass: true },
|
||||
N2_descriptionLength: { pass: true },
|
||||
N3_refNestingDepth: { pass: true },
|
||||
N4_refToc: { ratio: 1, pass: true, withToc: 1, largeFiles: 1 },
|
||||
N5_forwardSlashPaths: { pass: true },
|
||||
},
|
||||
judge: {
|
||||
K1_triggerPrecision: { precision: 1, pass: true },
|
||||
K4_noDuplication: { score: 5, pass: true },
|
||||
K9_noTimeSensitive: { pass: true, findings: [] },
|
||||
K7_imperativeStyle: { ratio: 1, pass: true },
|
||||
K8_sourceCitation: { ratio: 1, pass: true },
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
test('SCORE_CACHE_FILE points at the surfacing cache (scripts/kb-eval/data/skill-score-report.json)', () => {
|
||||
assert.ok(
|
||||
SCORE_CACHE_FILE.endsWith(join('scripts', 'kb-eval', 'data', 'skill-score-report.json')),
|
||||
`cache must be the same file STEG C surfacing reads; got: ${SCORE_CACHE_FILE}`,
|
||||
);
|
||||
});
|
||||
|
||||
test('refreshScoreCache — persists exactly buildScoreCache(scoreReport(report)) and returns it', () => {
|
||||
const out = join(tmpdir(), `apply-score-cache-${process.pid}.json`);
|
||||
try {
|
||||
const report = { skills: [skill('alpha'), skill('beta', { k10Pass: false, k10Combined: 9, worst: 'alpha' })] };
|
||||
const returned = refreshScoreCache(report, { cachePath: out, generatedAt: '2026-06-23' });
|
||||
|
||||
const onDisk = JSON.parse(readFileSync(out, 'utf8'));
|
||||
const expected = buildScoreCache(scoreReport(report, { target: 90 }), { generatedAt: '2026-06-23' });
|
||||
assert.deepEqual(onDisk, expected, 'persisted cache must equal buildScoreCache(scoreReport(report))');
|
||||
assert.deepEqual(returned, expected, 'return value mirrors what was written');
|
||||
|
||||
// whole-corpus + compact + below[] as names
|
||||
assert.equal(onDisk.scored.length, 2);
|
||||
assert.ok(!('criteria' in onDisk.scored[0]), 'cache stays compact (no verbose criteria[])');
|
||||
assert.deepEqual(onDisk.below, ['beta'], 'the K10-failing skill surfaces as below-target');
|
||||
assert.equal(onDisk.generatedAt, '2026-06-23');
|
||||
} finally {
|
||||
rmSync(out, { force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('refreshScoreCache — OVERWRITES (does not merge): a retire shrinks the cache', () => {
|
||||
const out = join(tmpdir(), `apply-score-cache-retire-${process.pid}.json`);
|
||||
try {
|
||||
// Corpus BEFORE: 3 skills.
|
||||
refreshScoreCache({ skills: [skill('alpha'), skill('beta'), skill('gamma')] }, { cachePath: out, generatedAt: '2026-06-23' });
|
||||
assert.equal(JSON.parse(readFileSync(out, 'utf8')).scored.length, 3);
|
||||
|
||||
// A retire drops 'gamma' -> refresh with the smaller corpus. The cache is
|
||||
// overwritten whole-corpus, so the retired skill must not linger and re-surface.
|
||||
refreshScoreCache({ skills: [skill('alpha'), skill('beta')] }, { cachePath: out, generatedAt: '2026-06-23' });
|
||||
const after = JSON.parse(readFileSync(out, 'utf8'));
|
||||
assert.equal(after.scored.length, 2, 'cache overwritten, not merged');
|
||||
assert.ok(!after.scored.some((s) => s.name === 'gamma'), 'retired skill dropped from the surfacing cache');
|
||||
} finally {
|
||||
rmSync(out, { force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('refreshScoreCache — generatedAt defaults to an ISO date (today) when not injected', () => {
|
||||
const out = join(tmpdir(), `apply-score-cache-date-${process.pid}.json`);
|
||||
try {
|
||||
refreshScoreCache({ skills: [skill('alpha')] }, { cachePath: out });
|
||||
const onDisk = JSON.parse(readFileSync(out, 'utf8'));
|
||||
assert.match(onDisk.generatedAt, /^\d{4}-\d{2}-\d{2}$/, 'impure layer stamps today as YYYY-MM-DD');
|
||||
} finally {
|
||||
rmSync(out, { force: true });
|
||||
}
|
||||
});
|
||||
Loading…
Add table
Add a link
Reference in a new issue