feat(ms-ai-architect): Sesjon 14 — B1 fullført (coverage/gap + bloat/stale) [skip-docs]
Utvider scripts/kb-eval/detect-skill-lifecycle.mjs med to deterministiske detektorer (B1 nå komplett, 3 seksjoner i skill-lifecycle-report.json). Skriver ALDRI til skills/ — kun rapport; kandidater mater decisions.json + gate (B3). Intern kb-eval-tooling: ingen brukervendt kommando/agent/skill/hook endret. Detektor 2 — coverage/gap (innen-domene): taksonomi category_skill (deklarert eierskap) vs fysisk disk-mappetelling. Klasser gap/thin/orphan/misowned. Empirisk: 21 deklarerte kat, 20 dekket, 1 gap (security-scoring — deklarert men ingen disk-mappe; innhold bor i ai-security-engineering = fantom-rutekat), 2 tynne (development=1, platforms=5). Detektor 3 — bloat/stale (per skill): K3-margin (eval.checkK3 body vs 500) + dateless ref-andel (mangler Last updated:-header = uverifiserbar ferskhet). Disk-only/deterministisk; poll-avledet staleness (change-report) bevisst utenfor. Empirisk: 0 split-kandidater, 1 saner (ms-ai-governance 4/78 dateless). TDD: 8 nye tester (coverage 4, stale-primitiv 1, bloat 3). kb-eval 23->31. Gate: validate 239, kb-update 122, kb-eval 31, kb-integrity 192/192 — grønn. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01REiKFhP4w6xGXXqWKpPCJJ
This commit is contained in:
parent
2665a3a2d8
commit
5ad4ed025c
2 changed files with 379 additions and 8 deletions
|
|
@ -1,10 +1,12 @@
|
|||
// test-skill-lifecycle-detect.test.mjs — Spor B / B1 overlap-detektor.
|
||||
// test-skill-lifecycle-detect.test.mjs — Spor B / B1 detectors.
|
||||
// TDD: written before scripts/kb-eval/detect-skill-lifecycle.mjs exists.
|
||||
//
|
||||
// The overlap detector is DETERMINISTIC and combines two signals:
|
||||
// Detector 1 (S13, OVERLAP) is DETERMINISTIC and combines two signals:
|
||||
// (1) operator-curated boundary-tension graph (k1-trigger-prompts.json belongs_to)
|
||||
// (2) df-weighted lexical trigger-surface overlap (down-weights domain-common tokens)
|
||||
// It must NEVER write to skills/ — it only produces a report.
|
||||
// Detectors 2+3 (S14): coverage/gap (taxonomy vs disk, in-domain only) and
|
||||
// bloat/stale (K3-margin per skill + dateless reference-file fraction).
|
||||
// All detectors must NEVER write to skills/ — they only produce a report.
|
||||
|
||||
import { test } from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
|
|
@ -20,11 +22,19 @@ import {
|
|||
pairKey,
|
||||
boundaryTensionMatrix,
|
||||
computeOverlapFromInputs,
|
||||
// S14 — coverage/gap + bloat/stale
|
||||
computeCoverageFromInputs,
|
||||
computeBloatFromInputs,
|
||||
extractLastUpdated,
|
||||
loadDiskCounts,
|
||||
loadBloatInputs,
|
||||
} from '../../scripts/kb-eval/detect-skill-lifecycle.mjs';
|
||||
|
||||
const __dirname = dirname(fileURLToPath(import.meta.url));
|
||||
const PROMPTS_PATH = join(__dirname, '..', '..', 'scripts', 'kb-eval', 'data', 'k1-trigger-prompts.json');
|
||||
const promptSet = JSON.parse(readFileSync(PROMPTS_PATH, 'utf8'));
|
||||
const TAX_PATH = join(__dirname, '..', '..', 'scripts', 'kb-update', 'data', 'domain-taxonomy.json');
|
||||
const taxonomy = JSON.parse(readFileSync(TAX_PATH, 'utf8'));
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// tokenize
|
||||
|
|
@ -192,3 +202,126 @@ test('computeOverlapFromInputs: 10 pairs, eng<->infra focusPair, sorted, determi
|
|||
// determinism: identical output across runs
|
||||
assert.deepEqual(r1, r2);
|
||||
});
|
||||
|
||||
// ===========================================================================
|
||||
// S14 — Detector 2: coverage/gap (in-domain) — taxonomy vs physical disk
|
||||
// ===========================================================================
|
||||
test('computeCoverageFromInputs: gap / thin / ok classification', () => {
|
||||
const categorySkill = {
|
||||
'rag-architecture': 'eng',
|
||||
development: 'advisor',
|
||||
'security-scoring': 'sec',
|
||||
};
|
||||
const diskCounts = {
|
||||
eng: { 'rag-architecture': 28 },
|
||||
advisor: { development: 1 },
|
||||
sec: {}, // security-scoring declared but no folder on disk
|
||||
};
|
||||
const r = computeCoverageFromInputs(categorySkill, diskCounts, { thinThreshold: 10 });
|
||||
const byCat = Object.fromEntries(r.categories.map((c) => [c.category, c]));
|
||||
assert.equal(byCat['rag-architecture'].status, 'ok');
|
||||
assert.equal(byCat['rag-architecture'].covered, true);
|
||||
assert.equal(byCat.development.status, 'thin');
|
||||
assert.equal(byCat.development.covered, true);
|
||||
assert.equal(byCat['security-scoring'].status, 'gap');
|
||||
assert.equal(byCat['security-scoring'].covered, false);
|
||||
assert.equal(byCat['security-scoring'].onDiskCount, 0);
|
||||
assert.deepEqual(r.gaps.map((g) => g.category), ['security-scoring']);
|
||||
assert.deepEqual(r.thin.map((t) => t.category), ['development']);
|
||||
});
|
||||
|
||||
test('computeCoverageFromInputs: detects orphan + misowned disk folders', () => {
|
||||
const categorySkill = { 'rag-architecture': 'eng' };
|
||||
const diskCounts = {
|
||||
eng: { 'rag-architecture': 28, 'mystery-folder': 3 }, // mystery -> orphan (undeclared)
|
||||
advisor: { 'rag-architecture': 4 }, // declared owner is eng -> misowned
|
||||
};
|
||||
const r = computeCoverageFromInputs(categorySkill, diskCounts, { thinThreshold: 10 });
|
||||
assert.deepEqual(r.orphans, [{ skill: 'eng', category: 'mystery-folder', onDiskCount: 3 }]);
|
||||
assert.deepEqual(r.misowned, [
|
||||
{ skill: 'advisor', category: 'rag-architecture', declaredOwner: 'eng', onDiskCount: 4 },
|
||||
]);
|
||||
});
|
||||
|
||||
test('computeCoverageFromInputs: deterministic, sorted, summary counts', () => {
|
||||
const categorySkill = { b: 'x', a: 'x', c: 'y' };
|
||||
const diskCounts = { x: { a: 20, b: 0 }, y: { c: 2 } };
|
||||
const r1 = computeCoverageFromInputs(categorySkill, diskCounts, { thinThreshold: 10 });
|
||||
const r2 = computeCoverageFromInputs(categorySkill, diskCounts, { thinThreshold: 10 });
|
||||
assert.deepEqual(r1, r2);
|
||||
assert.deepEqual(r1.categories.map((c) => c.category), ['a', 'b', 'c']); // sorted
|
||||
assert.equal(r1.summary.declaredCategories, 3);
|
||||
assert.equal(r1.summary.gaps, 1); // b: 0 files
|
||||
assert.equal(r1.summary.thin, 1); // c: 2 files
|
||||
assert.equal(r1.summary.covered, 2); // a, c
|
||||
});
|
||||
|
||||
test('coverage on real taxonomy + disk: declared-count matches, security-scoring gap, development thin', () => {
|
||||
const disk = loadDiskCounts();
|
||||
const r = computeCoverageFromInputs(taxonomy.category_skill, disk);
|
||||
assert.equal(r.summary.declaredCategories, Object.keys(taxonomy.category_skill).length);
|
||||
// security-scoring is declared (ms-ai-security) but has no folder on disk -> gap
|
||||
assert.ok(r.gaps.some((g) => g.category === 'security-scoring'), 'security-scoring gap surfaced');
|
||||
// development has a single reference file -> thin
|
||||
assert.ok(r.thin.some((t) => t.category === 'development'), 'development thin surfaced');
|
||||
});
|
||||
|
||||
// ===========================================================================
|
||||
// S14 — Detector 3: bloat/stale — K3-margin + dateless reference fraction
|
||||
// ===========================================================================
|
||||
test('extractLastUpdated: parses header variants, normalizes YYYY-MM, null when absent', () => {
|
||||
assert.equal(extractLastUpdated('# Title\n\n**Last updated:** 2026-03-14\n'), '2026-03-14');
|
||||
assert.equal(extractLastUpdated('**Sist oppdatert:** 2025-11\n'), '2025-11-01'); // YYYY-MM -> -01
|
||||
assert.equal(extractLastUpdated('**Sist verifisert:** 2026-01-09\n'), '2026-01-09');
|
||||
assert.equal(extractLastUpdated('**Dato:** 2024-12-31\n'), '2024-12-31');
|
||||
assert.equal(extractLastUpdated('# No header here\nbody text'), null);
|
||||
});
|
||||
|
||||
test('computeBloatFromInputs: margins, ratios, candidate flags', () => {
|
||||
const inputs = {
|
||||
big: { bodyLines: 470, refTotal: 10, refDateless: 0, datelessFiles: [] },
|
||||
stale: { bodyLines: 100, refTotal: 20, refDateless: 4, datelessFiles: ['a.md', 'b.md', 'c.md', 'd.md'] },
|
||||
healthy: { bodyLines: 200, refTotal: 50, refDateless: 0, datelessFiles: [] },
|
||||
};
|
||||
const r = computeBloatFromInputs(inputs, { bloatMarginMin: 50, datelessWarnRatio: 0.05 });
|
||||
const by = Object.fromEntries(r.skills.map((s) => [s.name, s]));
|
||||
assert.equal(by.big.k3Margin, 30); // 500 - 470
|
||||
assert.equal(by.big.bloatCandidate, true); // margin 30 < 50
|
||||
assert.equal(by.big.staleCandidate, false);
|
||||
assert.equal(by.stale.datelessRatio, 0.2); // 4 / 20
|
||||
assert.equal(by.stale.staleCandidate, true); // 0.2 > 0.05
|
||||
assert.equal(by.stale.bloatCandidate, false); // margin 400
|
||||
assert.deepEqual(by.stale.datelessFiles, ['a.md', 'b.md', 'c.md', 'd.md']);
|
||||
assert.equal(by.healthy.bloatCandidate, false);
|
||||
assert.equal(by.healthy.staleCandidate, false);
|
||||
assert.deepEqual(r.summary.bloatCandidates, ['big']);
|
||||
assert.deepEqual(r.summary.staleCandidates, ['stale']);
|
||||
});
|
||||
|
||||
test('computeBloatFromInputs: deterministic, sorted by name, no div-by-zero', () => {
|
||||
const inputs = {
|
||||
z: { bodyLines: 10, refTotal: 0, refDateless: 0, datelessFiles: [] },
|
||||
a: { bodyLines: 10, refTotal: 0, refDateless: 0, datelessFiles: [] },
|
||||
};
|
||||
const r1 = computeBloatFromInputs(inputs);
|
||||
const r2 = computeBloatFromInputs(inputs);
|
||||
assert.deepEqual(r1, r2);
|
||||
assert.deepEqual(r1.skills.map((s) => s.name), ['a', 'z']); // sorted
|
||||
assert.equal(r1.skills[0].datelessRatio, 0); // refTotal 0 -> ratio 0, not NaN
|
||||
});
|
||||
|
||||
test('bloat on real disk: 5 skills, ratios in [0,1], deterministic, dateless count matches file list', () => {
|
||||
const inputs = loadBloatInputs();
|
||||
const r1 = computeBloatFromInputs(inputs);
|
||||
const r2 = computeBloatFromInputs(inputs);
|
||||
assert.deepEqual(r1, r2);
|
||||
assert.deepEqual(
|
||||
r1.skills.map((s) => s.name).sort(),
|
||||
['ms-ai-advisor', 'ms-ai-engineering', 'ms-ai-governance', 'ms-ai-infrastructure', 'ms-ai-security'],
|
||||
);
|
||||
for (const s of r1.skills) {
|
||||
assert.ok(s.bodyLines > 0, `${s.name} has body lines`);
|
||||
assert.ok(s.datelessRatio >= 0 && s.datelessRatio <= 1, `${s.name} ratio in range`);
|
||||
assert.equal(s.refDateless, s.datelessFiles.length, `${s.name} dateless count matches list`);
|
||||
}
|
||||
});
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue