feat(ms-ai-architect): Sesjon 14 — B1 fullført (coverage/gap + bloat/stale) [skip-docs]

Utvider scripts/kb-eval/detect-skill-lifecycle.mjs med to deterministiske
detektorer (B1 nå komplett, 3 seksjoner i skill-lifecycle-report.json).
Skriver ALDRI til skills/ — kun rapport; kandidater mater decisions.json + gate (B3).
Intern kb-eval-tooling: ingen brukervendt kommando/agent/skill/hook endret.

Detektor 2 — coverage/gap (innen-domene): taksonomi category_skill (deklarert
eierskap) vs fysisk disk-mappetelling. Klasser gap/thin/orphan/misowned.
Empirisk: 21 deklarerte kat, 20 dekket, 1 gap (security-scoring — deklarert
men ingen disk-mappe; innhold bor i ai-security-engineering = fantom-rutekat),
2 tynne (development=1, platforms=5).

Detektor 3 — bloat/stale (per skill): K3-margin (eval.checkK3 body vs 500) +
dateless ref-andel (mangler Last updated:-header = uverifiserbar ferskhet).
Disk-only/deterministisk; poll-avledet staleness (change-report) bevisst utenfor.
Empirisk: 0 split-kandidater, 1 saner (ms-ai-governance 4/78 dateless).

TDD: 8 nye tester (coverage 4, stale-primitiv 1, bloat 3). kb-eval 23->31.
Gate: validate 239, kb-update 122, kb-eval 31, kb-integrity 192/192 — grønn.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01REiKFhP4w6xGXXqWKpPCJJ
This commit is contained in:
Kjell Tore Guttormsen 2026-06-20 11:34:57 +02:00
commit 5ad4ed025c
2 changed files with 379 additions and 8 deletions

View file

@ -1,10 +1,12 @@
// test-skill-lifecycle-detect.test.mjs — Spor B / B1 overlap-detektor.
// test-skill-lifecycle-detect.test.mjs — Spor B / B1 detectors.
// TDD: written before scripts/kb-eval/detect-skill-lifecycle.mjs exists.
//
// The overlap detector is DETERMINISTIC and combines two signals:
// Detector 1 (S13, OVERLAP) is DETERMINISTIC and combines two signals:
// (1) operator-curated boundary-tension graph (k1-trigger-prompts.json belongs_to)
// (2) df-weighted lexical trigger-surface overlap (down-weights domain-common tokens)
// It must NEVER write to skills/ — it only produces a report.
// Detectors 2+3 (S14): coverage/gap (taxonomy vs disk, in-domain only) and
// bloat/stale (K3-margin per skill + dateless reference-file fraction).
// All detectors must NEVER write to skills/ — they only produce a report.
import { test } from 'node:test';
import assert from 'node:assert/strict';
@ -20,11 +22,19 @@ import {
pairKey,
boundaryTensionMatrix,
computeOverlapFromInputs,
// S14 — coverage/gap + bloat/stale
computeCoverageFromInputs,
computeBloatFromInputs,
extractLastUpdated,
loadDiskCounts,
loadBloatInputs,
} from '../../scripts/kb-eval/detect-skill-lifecycle.mjs';
const __dirname = dirname(fileURLToPath(import.meta.url));
const PROMPTS_PATH = join(__dirname, '..', '..', 'scripts', 'kb-eval', 'data', 'k1-trigger-prompts.json');
const promptSet = JSON.parse(readFileSync(PROMPTS_PATH, 'utf8'));
const TAX_PATH = join(__dirname, '..', '..', 'scripts', 'kb-update', 'data', 'domain-taxonomy.json');
const taxonomy = JSON.parse(readFileSync(TAX_PATH, 'utf8'));
// ---------------------------------------------------------------------------
// tokenize
@ -192,3 +202,126 @@ test('computeOverlapFromInputs: 10 pairs, eng<->infra focusPair, sorted, determi
// determinism: identical output across runs
assert.deepEqual(r1, r2);
});
// ===========================================================================
// S14 — Detector 2: coverage/gap (in-domain) — taxonomy vs physical disk
// ===========================================================================
test('computeCoverageFromInputs: gap / thin / ok classification', () => {
const categorySkill = {
'rag-architecture': 'eng',
development: 'advisor',
'security-scoring': 'sec',
};
const diskCounts = {
eng: { 'rag-architecture': 28 },
advisor: { development: 1 },
sec: {}, // security-scoring declared but no folder on disk
};
const r = computeCoverageFromInputs(categorySkill, diskCounts, { thinThreshold: 10 });
const byCat = Object.fromEntries(r.categories.map((c) => [c.category, c]));
assert.equal(byCat['rag-architecture'].status, 'ok');
assert.equal(byCat['rag-architecture'].covered, true);
assert.equal(byCat.development.status, 'thin');
assert.equal(byCat.development.covered, true);
assert.equal(byCat['security-scoring'].status, 'gap');
assert.equal(byCat['security-scoring'].covered, false);
assert.equal(byCat['security-scoring'].onDiskCount, 0);
assert.deepEqual(r.gaps.map((g) => g.category), ['security-scoring']);
assert.deepEqual(r.thin.map((t) => t.category), ['development']);
});
test('computeCoverageFromInputs: detects orphan + misowned disk folders', () => {
const categorySkill = { 'rag-architecture': 'eng' };
const diskCounts = {
eng: { 'rag-architecture': 28, 'mystery-folder': 3 }, // mystery -> orphan (undeclared)
advisor: { 'rag-architecture': 4 }, // declared owner is eng -> misowned
};
const r = computeCoverageFromInputs(categorySkill, diskCounts, { thinThreshold: 10 });
assert.deepEqual(r.orphans, [{ skill: 'eng', category: 'mystery-folder', onDiskCount: 3 }]);
assert.deepEqual(r.misowned, [
{ skill: 'advisor', category: 'rag-architecture', declaredOwner: 'eng', onDiskCount: 4 },
]);
});
test('computeCoverageFromInputs: deterministic, sorted, summary counts', () => {
const categorySkill = { b: 'x', a: 'x', c: 'y' };
const diskCounts = { x: { a: 20, b: 0 }, y: { c: 2 } };
const r1 = computeCoverageFromInputs(categorySkill, diskCounts, { thinThreshold: 10 });
const r2 = computeCoverageFromInputs(categorySkill, diskCounts, { thinThreshold: 10 });
assert.deepEqual(r1, r2);
assert.deepEqual(r1.categories.map((c) => c.category), ['a', 'b', 'c']); // sorted
assert.equal(r1.summary.declaredCategories, 3);
assert.equal(r1.summary.gaps, 1); // b: 0 files
assert.equal(r1.summary.thin, 1); // c: 2 files
assert.equal(r1.summary.covered, 2); // a, c
});
test('coverage on real taxonomy + disk: declared-count matches, security-scoring gap, development thin', () => {
const disk = loadDiskCounts();
const r = computeCoverageFromInputs(taxonomy.category_skill, disk);
assert.equal(r.summary.declaredCategories, Object.keys(taxonomy.category_skill).length);
// security-scoring is declared (ms-ai-security) but has no folder on disk -> gap
assert.ok(r.gaps.some((g) => g.category === 'security-scoring'), 'security-scoring gap surfaced');
// development has a single reference file -> thin
assert.ok(r.thin.some((t) => t.category === 'development'), 'development thin surfaced');
});
// ===========================================================================
// S14 — Detector 3: bloat/stale — K3-margin + dateless reference fraction
// ===========================================================================
test('extractLastUpdated: parses header variants, normalizes YYYY-MM, null when absent', () => {
assert.equal(extractLastUpdated('# Title\n\n**Last updated:** 2026-03-14\n'), '2026-03-14');
assert.equal(extractLastUpdated('**Sist oppdatert:** 2025-11\n'), '2025-11-01'); // YYYY-MM -> -01
assert.equal(extractLastUpdated('**Sist verifisert:** 2026-01-09\n'), '2026-01-09');
assert.equal(extractLastUpdated('**Dato:** 2024-12-31\n'), '2024-12-31');
assert.equal(extractLastUpdated('# No header here\nbody text'), null);
});
test('computeBloatFromInputs: margins, ratios, candidate flags', () => {
const inputs = {
big: { bodyLines: 470, refTotal: 10, refDateless: 0, datelessFiles: [] },
stale: { bodyLines: 100, refTotal: 20, refDateless: 4, datelessFiles: ['a.md', 'b.md', 'c.md', 'd.md'] },
healthy: { bodyLines: 200, refTotal: 50, refDateless: 0, datelessFiles: [] },
};
const r = computeBloatFromInputs(inputs, { bloatMarginMin: 50, datelessWarnRatio: 0.05 });
const by = Object.fromEntries(r.skills.map((s) => [s.name, s]));
assert.equal(by.big.k3Margin, 30); // 500 - 470
assert.equal(by.big.bloatCandidate, true); // margin 30 < 50
assert.equal(by.big.staleCandidate, false);
assert.equal(by.stale.datelessRatio, 0.2); // 4 / 20
assert.equal(by.stale.staleCandidate, true); // 0.2 > 0.05
assert.equal(by.stale.bloatCandidate, false); // margin 400
assert.deepEqual(by.stale.datelessFiles, ['a.md', 'b.md', 'c.md', 'd.md']);
assert.equal(by.healthy.bloatCandidate, false);
assert.equal(by.healthy.staleCandidate, false);
assert.deepEqual(r.summary.bloatCandidates, ['big']);
assert.deepEqual(r.summary.staleCandidates, ['stale']);
});
test('computeBloatFromInputs: deterministic, sorted by name, no div-by-zero', () => {
const inputs = {
z: { bodyLines: 10, refTotal: 0, refDateless: 0, datelessFiles: [] },
a: { bodyLines: 10, refTotal: 0, refDateless: 0, datelessFiles: [] },
};
const r1 = computeBloatFromInputs(inputs);
const r2 = computeBloatFromInputs(inputs);
assert.deepEqual(r1, r2);
assert.deepEqual(r1.skills.map((s) => s.name), ['a', 'z']); // sorted
assert.equal(r1.skills[0].datelessRatio, 0); // refTotal 0 -> ratio 0, not NaN
});
test('bloat on real disk: 5 skills, ratios in [0,1], deterministic, dateless count matches file list', () => {
const inputs = loadBloatInputs();
const r1 = computeBloatFromInputs(inputs);
const r2 = computeBloatFromInputs(inputs);
assert.deepEqual(r1, r2);
assert.deepEqual(
r1.skills.map((s) => s.name).sort(),
['ms-ai-advisor', 'ms-ai-engineering', 'ms-ai-governance', 'ms-ai-infrastructure', 'ms-ai-security'],
);
for (const s of r1.skills) {
assert.ok(s.bodyLines > 0, `${s.name} has body lines`);
assert.ok(s.datelessRatio >= 0 && s.datelessRatio <= 1, `${s.name} ratio in range`);
assert.equal(s.refDateless, s.datelessFiles.length, `${s.name} dateless count matches list`);
}
});