feat(ms-ai-architect): Spor D steg 2 — score-skill CLI (--json/--gate/--skill) (TDD)
Byggekloss for både SessionStart-surfacing og lifecycle-gate (D3). - eval.mjs: ekstraher buildReport() (eksportert, testbar) ut av main() — samler hele korpus-rapporten (deterministisk + K10 + merget judge-cache). - lib/skill-score.mjs: formatScoreReport() (ren) — sortert forbedringsrapport med score, status, floored/provisional/ujudget-tags og ⚑gulv-markering. - score-skill.mjs (NY CLI): buildReport → scoreReport → render/json/gate. --skill scorer hele korpuset (K10 trenger alle søsken) men rapporterer/gater kun den valgte → inkrementell enkelt-skill-scoring for D3 lifecycle-gate. - docs/development.md: ny «Skill-kvalitetsscore (Spor D)»-seksjon. Live baseline: advisor 91, governance/security 96 (OK); engineering + infrastructure 89 (floored av K10-gulv) UNDER mål. Gate exit=1. Tester: kb-eval 134→142 (+8). validate 239, kb-update 316 uendret grønt. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
e87289d929
commit
c1a09062d4
5 changed files with 249 additions and 7 deletions
113
tests/kb-eval/test-score-skill.test.mjs
Normal file
113
tests/kb-eval/test-score-skill.test.mjs
Normal file
|
|
@ -0,0 +1,113 @@
|
|||
// tests/kb-eval/test-score-skill.test.mjs
|
||||
// Tests for the score-skill CLI building blocks (Spor D, step 2):
|
||||
// - eval.mjs buildReport() assembles the full corpus report (deterministic +
|
||||
// K10 sibling-overlap + merged judge cache).
|
||||
// - lib/skill-score.mjs formatScoreReport() renders the sorted improvement
|
||||
// report (pure).
|
||||
// - the score-skill.mjs CLI: --json emits the scoreReport, --gate sets a
|
||||
// non-zero exit when any skill is below target, --skill scopes the output.
|
||||
|
||||
import { test } from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { execFileSync } from 'node:child_process';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { dirname, join } from 'node:path';
|
||||
import { buildReport } from '../../scripts/kb-eval/eval.mjs';
|
||||
import { scoreReport, formatScoreReport } from '../../scripts/kb-eval/lib/skill-score.mjs';
|
||||
|
||||
const __dirname = dirname(fileURLToPath(import.meta.url));
|
||||
const CLI = join(__dirname, '..', '..', 'scripts', 'kb-eval', 'score-skill.mjs');
|
||||
|
||||
test('buildReport — assembles the full corpus with N1-N5 on every skill', () => {
|
||||
const report = buildReport();
|
||||
assert.ok(Array.isArray(report.skills) && report.skills.length === 5, `got ${report.skills?.length} skills`);
|
||||
for (const s of report.skills) {
|
||||
const d = s.deterministic;
|
||||
assert.ok(d.N1_nameValidity && d.N4_refToc, `${s.name} missing N1/N4`);
|
||||
assert.ok(d.K10_siblingScopeOverlap, `${s.name} missing K10 (sibling overlap not attached)`);
|
||||
}
|
||||
});
|
||||
|
||||
test('scoreReport — over the real corpus returns a score per skill and a below-target set', () => {
|
||||
const result = scoreReport(buildReport());
|
||||
assert.equal(result.target, 90);
|
||||
assert.equal(result.scored.length, 5);
|
||||
for (const s of result.scored) {
|
||||
assert.ok(Number.isFinite(s.score), `${s.name} score not finite`);
|
||||
assert.equal(typeof s.meetsTarget, 'boolean');
|
||||
}
|
||||
// below = subset that does not meet the target
|
||||
assert.ok(result.below.every((s) => !s.meetsTarget));
|
||||
});
|
||||
|
||||
test('scoreReport — a synthetic below-target skill lands in below[]', () => {
|
||||
const synthetic = {
|
||||
skills: [
|
||||
// one perfect-ish, one with a failing K10 floor (capped below 90)
|
||||
{
|
||||
name: 'good', deterministic: {
|
||||
K2_descriptionFormat: { pass: true }, K3_bodyLength: { bodyLines: 100, pass: true },
|
||||
K5_progressiveDisclosure: { namedRatio: 0.5, pass: true }, K6_routingTable: { pass: true },
|
||||
refCountConsistency: { consistent: true }, K10_siblingScopeOverlap: { maxCombined: 1, pass: true, threshold: 7 },
|
||||
N1_nameValidity: { pass: true }, N2_descriptionLength: { pass: true },
|
||||
N3_refNestingDepth: { pass: true }, N4_refToc: { ratio: 1, pass: true, withToc: 1, largeFiles: 1 },
|
||||
N5_forwardSlashPaths: { pass: true },
|
||||
}, judge: null,
|
||||
},
|
||||
{
|
||||
name: 'bad', deterministic: {
|
||||
K2_descriptionFormat: { pass: true }, K3_bodyLength: { bodyLines: 100, pass: true },
|
||||
K5_progressiveDisclosure: { namedRatio: 0.5, pass: true }, K6_routingTable: { pass: true },
|
||||
refCountConsistency: { consistent: true }, K10_siblingScopeOverlap: { maxCombined: 9, pass: false, threshold: 7, worstSibling: 'good' },
|
||||
N1_nameValidity: { pass: true }, N2_descriptionLength: { pass: true },
|
||||
N3_refNestingDepth: { pass: true }, N4_refToc: { ratio: 1, pass: true, withToc: 1, largeFiles: 1 },
|
||||
N5_forwardSlashPaths: { pass: true },
|
||||
}, judge: null,
|
||||
},
|
||||
],
|
||||
};
|
||||
const result = scoreReport(synthetic);
|
||||
assert.equal(result.below.length, 1);
|
||||
assert.equal(result.below[0].name, 'bad');
|
||||
});
|
||||
|
||||
test('formatScoreReport — renders skill names, the target, and improvement fixes', () => {
|
||||
const result = scoreReport(buildReport());
|
||||
const text = formatScoreReport(result);
|
||||
assert.match(text, /ms-ai-advisor/);
|
||||
assert.match(text, /90/); // the target appears
|
||||
// at least one improvement fix line is rendered for a below-target skill
|
||||
if (result.below.length > 0) {
|
||||
assert.ok(/↳|→/.test(text), 'expected improvement bullets in the report');
|
||||
}
|
||||
});
|
||||
|
||||
test('CLI --json — emits a parseable scoreReport with five scored skills (exit 0)', () => {
|
||||
const out = execFileSync('node', [CLI, '--json'], { encoding: 'utf8' });
|
||||
const parsed = JSON.parse(out);
|
||||
assert.equal(parsed.scored.length, 5);
|
||||
assert.equal(parsed.target, 90);
|
||||
});
|
||||
|
||||
test('CLI --skill — scopes the output to a single skill', () => {
|
||||
const out = execFileSync('node', [CLI, '--skill', 'ms-ai-advisor', '--json'], { encoding: 'utf8' });
|
||||
const parsed = JSON.parse(out);
|
||||
assert.equal(parsed.scored.length, 1);
|
||||
assert.equal(parsed.scored[0].name, 'ms-ai-advisor');
|
||||
});
|
||||
|
||||
test('CLI --gate — non-zero exit path: an unreachable target fails the gate (corpus-independent)', () => {
|
||||
let code = 0;
|
||||
try {
|
||||
execFileSync('node', [CLI, '--gate', '101'], { encoding: 'utf8', stdio: 'pipe' });
|
||||
} catch (e) {
|
||||
code = e.status;
|
||||
}
|
||||
assert.equal(code, 1, 'no skill can score 101 -> gate must fail');
|
||||
});
|
||||
|
||||
test('CLI --gate — zero exit path: a target of 0 passes the gate (corpus-independent)', () => {
|
||||
// exits 0; execFileSync throws only on non-zero, so reaching here is the pass
|
||||
execFileSync('node', [CLI, '--gate', '0'], { encoding: 'utf8', stdio: 'pipe' });
|
||||
assert.ok(true);
|
||||
});
|
||||
Loading…
Add table
Add a link
Reference in a new issue