feat(ms-ai-architect): Spor D Steg B — skill-quality lifecycle-gate + score-cache (TDD)
B1: gateSkill(report, name) — pure post-mutation verdict (blocked/provisional/ improvements), wired into apply-skill-op.mjs to re-score the affected skill from fresh disk after create/merge/sanitize-apply. K10 floor enforced immediately; unjudged → provisional + nudge to re-run the judge pass. retire → skipped. B2: buildScoreCache(result) — compact, deterministic cache shape + score-skill.mjs --write [path] → data/skill-score-report.json (always whole-corpus). Gitignored (derived/regenerable; avoids churn in the public repo). Consumed by Steg C surfacing. 8 new tests (tests/kb-eval/test-skill-score-gate.test.mjs). kb-eval 150 pass, validate 239 pass. Live score unchanged: advisor 91, eng/gov/infra/sec 96, 0 < 90. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
c578482933
commit
92cd93771b
5 changed files with 294 additions and 3 deletions
146
tests/kb-eval/test-skill-score-gate.test.mjs
Normal file
146
tests/kb-eval/test-skill-score-gate.test.mjs
Normal file
|
|
@ -0,0 +1,146 @@
|
|||
// tests/kb-eval/test-skill-score-gate.test.mjs
|
||||
// Tests for the Spor D, STEG B lifecycle-gate building blocks:
|
||||
// - lib/skill-score.mjs gateSkill(report, name): the PURE post-mutation verdict
|
||||
// used by apply-skill-op.mjs to re-score the affected skill incrementally.
|
||||
// The deterministic floor (K10) is always enforced; an unjudged skill is
|
||||
// flagged `provisional` (the K1 floor cannot be enforced -> nudge to re-judge).
|
||||
// - lib/skill-score.mjs buildScoreCache(result): the compact, PURE cache shape
|
||||
// persisted as data/skill-score-report.json and consumed by STEG C surfacing.
|
||||
// - the score-skill.mjs CLI --write flag: persists the whole-corpus cache.
|
||||
|
||||
import { test } from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { execFileSync } from 'node:child_process';
|
||||
import { readFileSync, rmSync } from 'node:fs';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { dirname, join } from 'node:path';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { gateSkill, buildScoreCache, scoreReport } from '../../scripts/kb-eval/lib/skill-score.mjs';
|
||||
|
||||
const __dirname = dirname(fileURLToPath(import.meta.url));
|
||||
const CLI = join(__dirname, '..', '..', 'scripts', 'kb-eval', 'score-skill.mjs');
|
||||
|
||||
// A synthetic skill whose deterministic criteria all pass; K10 + judge are
|
||||
// parametrizable so a single helper drives the meets/blocked/provisional cases.
|
||||
function skill(name, { k10Pass = true, k10Combined = 1, worst = 'other', judge = null } = {}) {
|
||||
return {
|
||||
name,
|
||||
deterministic: {
|
||||
K2_descriptionFormat: { pass: true },
|
||||
K3_bodyLength: { bodyLines: 100, pass: true },
|
||||
K5_progressiveDisclosure: { namedRatio: 0.5, pass: true },
|
||||
K6_routingTable: { pass: true },
|
||||
refCountConsistency: { consistent: true },
|
||||
K10_siblingScopeOverlap: { maxCombined: k10Combined, pass: k10Pass, threshold: 7, worstSibling: worst },
|
||||
N1_nameValidity: { pass: true },
|
||||
N2_descriptionLength: { pass: true },
|
||||
N3_refNestingDepth: { pass: true },
|
||||
N4_refToc: { ratio: 1, pass: true, withToc: 1, largeFiles: 1 },
|
||||
N5_forwardSlashPaths: { pass: true },
|
||||
},
|
||||
judge,
|
||||
};
|
||||
}
|
||||
|
||||
const FULL_JUDGE = {
|
||||
K1_triggerPrecision: { precision: 1, pass: true },
|
||||
K4_noDuplication: { score: 5, pass: true },
|
||||
K9_noTimeSensitive: { pass: true, findings: [] },
|
||||
K7_imperativeStyle: { ratio: 1, pass: true },
|
||||
K8_sourceCitation: { ratio: 1, pass: true },
|
||||
};
|
||||
|
||||
test('gateSkill — affected skill that meets the target is not blocked', () => {
|
||||
const report = { skills: [skill('alpha', { judge: FULL_JUDGE }), skill('beta', { judge: FULL_JUDGE })] };
|
||||
const v = gateSkill(report, 'alpha', { target: 90 });
|
||||
assert.equal(v.found, true);
|
||||
assert.equal(v.meetsTarget, true);
|
||||
assert.equal(v.blocked, false);
|
||||
assert.equal(v.provisional, false);
|
||||
});
|
||||
|
||||
test('gateSkill — a failing K10 floor blocks the affected skill (deterministic, enforced immediately)', () => {
|
||||
const report = {
|
||||
skills: [skill('good', { judge: FULL_JUDGE }), skill('bad', { k10Pass: false, k10Combined: 9, worst: 'good', judge: FULL_JUDGE })],
|
||||
};
|
||||
const v = gateSkill(report, 'bad', { target: 90 });
|
||||
assert.equal(v.found, true);
|
||||
assert.equal(v.meetsTarget, false);
|
||||
assert.equal(v.blocked, true);
|
||||
assert.ok(v.score <= 89, `K10 floor must cap below 90, got ${v.score}`);
|
||||
assert.ok(v.improvements.some((i) => i.key === 'K10'), 'K10 must surface as an improvement');
|
||||
});
|
||||
|
||||
test('gateSkill — an unjudged skill is provisional but the deterministic floor still blocks it', () => {
|
||||
// judge:null -> K1 floor cannot be enforced (provisional), but K10 (deterministic) still blocks.
|
||||
const report = { skills: [skill('good'), skill('bad', { k10Pass: false, k10Combined: 9, worst: 'good' })] };
|
||||
const blocked = gateSkill(report, 'bad', { target: 90 });
|
||||
assert.equal(blocked.provisional, true, 'missing judge -> provisional');
|
||||
assert.equal(blocked.blocked, true, 'deterministic K10 floor still blocks when provisional');
|
||||
|
||||
const ok = gateSkill(report, 'good', { target: 90 });
|
||||
assert.equal(ok.provisional, true, 'missing judge -> provisional even when passing');
|
||||
assert.equal(ok.meetsTarget, true, 'deterministic-only score still passes');
|
||||
});
|
||||
|
||||
test('gateSkill — an unknown affected skill returns found:false (no crash)', () => {
|
||||
const report = { skills: [skill('alpha', { judge: FULL_JUDGE })] };
|
||||
const v = gateSkill(report, 'does-not-exist', { target: 90 });
|
||||
assert.equal(v.found, false);
|
||||
assert.equal(v.name, 'does-not-exist');
|
||||
});
|
||||
|
||||
test('buildScoreCache — compact whole-corpus shape with below[] as names', () => {
|
||||
const report = {
|
||||
skills: [skill('good', { judge: FULL_JUDGE }), skill('bad', { k10Pass: false, k10Combined: 9, worst: 'good', judge: FULL_JUDGE })],
|
||||
};
|
||||
const result = scoreReport(report, { target: 90 });
|
||||
const cache = buildScoreCache(result, { generatedAt: '2026-06-23' });
|
||||
assert.equal(cache.target, 90);
|
||||
assert.equal(cache.generatedAt, '2026-06-23');
|
||||
assert.equal(cache.scored.length, 2);
|
||||
// compact per-skill entry: the fields STEG C surfacing needs, not the verbose criteria[]
|
||||
const good = cache.scored.find((s) => s.name === 'good');
|
||||
assert.equal(typeof good.score, 'number');
|
||||
assert.equal(typeof good.meetsTarget, 'boolean');
|
||||
assert.ok(!('criteria' in good), 'cache must stay compact (no criteria[])');
|
||||
// below[] is a list of NAMES (cheap for the SessionStart one-liner)
|
||||
assert.deepEqual(cache.below, ['bad']);
|
||||
});
|
||||
|
||||
test('buildScoreCache — deterministic given an injected generatedAt (no Date.now)', () => {
|
||||
const report = { skills: [skill('alpha', { judge: FULL_JUDGE })] };
|
||||
const result = scoreReport(report, { target: 90 });
|
||||
const a = buildScoreCache(result, { generatedAt: '2026-01-01' });
|
||||
const b = buildScoreCache(result, { generatedAt: '2026-01-01' });
|
||||
assert.deepEqual(a, b);
|
||||
});
|
||||
|
||||
test('CLI --write <path> — persists the whole-corpus cache (all 5 skills) to disk', () => {
|
||||
const out = join(tmpdir(), `skill-score-cache-${process.pid}.json`);
|
||||
try {
|
||||
execFileSync('node', [CLI, '--write', out], { encoding: 'utf8' });
|
||||
const cache = JSON.parse(readFileSync(out, 'utf8'));
|
||||
assert.equal(cache.target, 90);
|
||||
assert.equal(cache.scored.length, 5, 'cache must cover the whole corpus, not a filtered view');
|
||||
assert.ok(Array.isArray(cache.below), 'below[] present');
|
||||
for (const s of cache.scored) {
|
||||
assert.equal(typeof s.name, 'string');
|
||||
assert.equal(typeof s.score, 'number');
|
||||
assert.equal(typeof s.meetsTarget, 'boolean');
|
||||
}
|
||||
} finally {
|
||||
rmSync(out, { force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('CLI --write with --skill still writes the whole corpus (cache is corpus-wide)', () => {
|
||||
const out = join(tmpdir(), `skill-score-cache-corpus-${process.pid}.json`);
|
||||
try {
|
||||
execFileSync('node', [CLI, '--skill', 'ms-ai-advisor', '--write', out], { encoding: 'utf8' });
|
||||
const cache = JSON.parse(readFileSync(out, 'utf8'));
|
||||
assert.equal(cache.scored.length, 5, '--skill scopes stdout, not the persisted cache');
|
||||
} finally {
|
||||
rmSync(out, { force: true });
|
||||
}
|
||||
});
|
||||
Loading…
Add table
Add a link
Reference in a new issue