feat(ms-ai-architect): Sesjon 14 — B1 fullført (coverage/gap + bloat/stale) [skip-docs]
Utvider scripts/kb-eval/detect-skill-lifecycle.mjs med to deterministiske detektorer (B1 nå komplett, 3 seksjoner i skill-lifecycle-report.json). Skriver ALDRI til skills/ — kun rapport; kandidater mater decisions.json + gate (B3). Intern kb-eval-tooling: ingen brukervendt kommando/agent/skill/hook endret. Detektor 2 — coverage/gap (innen-domene): taksonomi category_skill (deklarert eierskap) vs fysisk disk-mappetelling. Klasser gap/thin/orphan/misowned. Empirisk: 21 deklarerte kat, 20 dekket, 1 gap (security-scoring — deklarert men ingen disk-mappe; innhold bor i ai-security-engineering = fantom-rutekat), 2 tynne (development=1, platforms=5). Detektor 3 — bloat/stale (per skill): K3-margin (eval.checkK3 body vs 500) + dateless ref-andel (mangler Last updated:-header = uverifiserbar ferskhet). Disk-only/deterministisk; poll-avledet staleness (change-report) bevisst utenfor. Empirisk: 0 split-kandidater, 1 saner (ms-ai-governance 4/78 dateless). TDD: 8 nye tester (coverage 4, stale-primitiv 1, bloat 3). kb-eval 23->31. Gate: validate 239, kb-update 122, kb-eval 31, kb-integrity 192/192 — grønn. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01REiKFhP4w6xGXXqWKpPCJJ
This commit is contained in:
parent
2665a3a2d8
commit
5ad4ed025c
2 changed files with 379 additions and 8 deletions
|
|
@ -17,6 +17,15 @@
|
|||
// The eng<->infra pair (Azure-deployment boundary) is surfaced as focusPair —
|
||||
// the operator-designated first target for B1.
|
||||
//
|
||||
// Sesjon 14 adds two more detectors to the same report (all deterministic):
|
||||
// [2] COVERAGE/GAP — taxonomy.category_skill (declared ownership) vs physical
|
||||
// disk folder counts. Surfaces gap (owned, 0 files), thin (< threshold),
|
||||
// orphan (undeclared folder), misowned (wrong owner). In-domain only —
|
||||
// no new domains proposed.
|
||||
// [3] BLOAT/STALE — per-skill K3-margin (eval.checkK3 bodyLines vs 500) plus
|
||||
// the dateless reference fraction (refs lacking a "Last updated:" header =
|
||||
// unverifiable currency). Flags split-/sanitize-candidates.
|
||||
//
|
||||
// Usage:
|
||||
// node scripts/kb-eval/detect-skill-lifecycle.mjs # human summary
|
||||
// node scripts/kb-eval/detect-skill-lifecycle.mjs --json # machine output
|
||||
|
|
@ -25,21 +34,46 @@
|
|||
// Zero dependencies. Reuses eval.mjs extractors + kb-update atomic-write.
|
||||
|
||||
import { readFileSync, readdirSync, existsSync, mkdirSync } from 'node:fs';
|
||||
import { join, dirname } from 'node:path';
|
||||
import { join, dirname, relative } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { splitFrontmatter, extractDescription } from './eval.mjs';
|
||||
import { splitFrontmatter, extractDescription, checkK3 } from './eval.mjs';
|
||||
import { loadTaxonomy } from '../kb-update/lib/taxonomy.mjs';
|
||||
import { atomicWriteJson } from '../kb-update/lib/atomic-write.mjs';
|
||||
|
||||
const __dirname = dirname(fileURLToPath(import.meta.url));
|
||||
const PLUGIN_ROOT = join(__dirname, '..', '..');
|
||||
const SKILLS_DIR = join(PLUGIN_ROOT, 'skills');
|
||||
const DATA_DIR = join(__dirname, 'data');
|
||||
const TAX_DATA_DIR = join(__dirname, '..', 'kb-update', 'data');
|
||||
const PROMPTS_FILE = join(DATA_DIR, 'k1-trigger-prompts.json');
|
||||
const OUT_FILE = join(DATA_DIR, 'skill-lifecycle-report.json');
|
||||
|
||||
// Operator-designated B1 focus boundary.
|
||||
const FOCUS_PAIR = ['ms-ai-engineering', 'ms-ai-infrastructure'];
|
||||
|
||||
// --- S14 detector thresholds (named, documented) -------------------------
|
||||
// K3 hard body-length limit (mirrors eval.mjs K3_MAX_BODY_LINES — single source
|
||||
// of bodyLines is eval.checkK3; this constant only computes the margin).
|
||||
const K3_MAX_BODY_LINES = 500;
|
||||
// A declared category with fewer than this many reference files is "thin"
|
||||
// relative to the ~19-file median across the 20 on-disk categories.
|
||||
const THIN_REF_COUNT = 10;
|
||||
// Flag a SKILL.md as a split-candidate when its body is within this many lines
|
||||
// of the K3 limit (i.e. margin < 50 -> body > 450).
|
||||
const BLOAT_MARGIN_MIN = 50;
|
||||
// Flag a skill as a sanitize-candidate when more than this fraction of its
|
||||
// reference files lack a verifiable "Last updated:" header (unverifiable currency).
|
||||
const DATELESS_WARN_RATIO = 0.05;
|
||||
|
||||
// Mirrors report-changes.mjs LAST_UPDATED_PATTERNS (kept local: report-changes
|
||||
// runs main() at import time, so it cannot be imported safely). Currency-header
|
||||
// forms accepted across the KB.
|
||||
const LAST_UPDATED_PATTERNS = [
|
||||
/\*\*Last updated:\*\*\s*([\d-]+)/i,
|
||||
/\*\*Sist (?:oppdatert|verifisert):\*\*\s*([\d-]+)/i,
|
||||
/\*\*Dato:\*\*\s*([\d-]+)/i,
|
||||
];
|
||||
|
||||
// Function words (no + en) of length >= 3. Tokens < 3 chars are dropped anyway,
|
||||
// so this list only needs the longer connectives. Domain nouns are NOT here —
|
||||
// df-weighting handles common domain vocabulary instead.
|
||||
|
|
@ -155,6 +189,183 @@ export function computeOverlapFromInputs(descriptionsBySkill, promptSet) {
|
|||
};
|
||||
}
|
||||
|
||||
// ===========================================================================
|
||||
// S14 — Detector 2: coverage/gap (in-domain) — taxonomy vs physical disk
|
||||
// ===========================================================================
|
||||
|
||||
/**
|
||||
* Pure core: in-domain coverage. Compares the declared ownership map
|
||||
* (taxonomy.category_skill) against physical disk folder counts. NO new domains
|
||||
* are proposed — this only surfaces gaps WITHIN the already-covered domain.
|
||||
* gap = category declared but 0 files on disk under its owner
|
||||
* thin = declared + covered, but < thinThreshold files
|
||||
* orphan = a disk folder whose category is not declared anywhere
|
||||
* misowned = a disk folder sitting under a skill other than its declared owner
|
||||
* @param {Record<string,string>} categorySkill category -> owning skill
|
||||
* @param {Record<string,Record<string,number>>} diskCounts skill -> {category: count}
|
||||
*/
|
||||
export function computeCoverageFromInputs(categorySkill, diskCounts, { thinThreshold = THIN_REF_COUNT } = {}) {
|
||||
const categories = [];
|
||||
for (const category of Object.keys(categorySkill).sort()) {
|
||||
const owningSkill = categorySkill[category];
|
||||
const onDiskCount = (diskCounts[owningSkill] && diskCounts[owningSkill][category]) || 0;
|
||||
const covered = onDiskCount > 0;
|
||||
const thin = covered && onDiskCount < thinThreshold;
|
||||
const status = !covered ? 'gap' : thin ? 'thin' : 'ok';
|
||||
categories.push({ category, owningSkill, onDiskCount, covered, thin, status });
|
||||
}
|
||||
|
||||
const orphans = [];
|
||||
const misowned = [];
|
||||
for (const skill of Object.keys(diskCounts).sort()) {
|
||||
for (const category of Object.keys(diskCounts[skill]).sort()) {
|
||||
const declaredOwner = categorySkill[category];
|
||||
if (declaredOwner === undefined) {
|
||||
orphans.push({ skill, category, onDiskCount: diskCounts[skill][category] });
|
||||
} else if (declaredOwner !== skill) {
|
||||
misowned.push({ skill, category, declaredOwner, onDiskCount: diskCounts[skill][category] });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const gaps = categories.filter((c) => c.status === 'gap');
|
||||
const thinCats = categories.filter((c) => c.status === 'thin');
|
||||
return {
|
||||
method:
|
||||
'in-domain coverage: taxonomy.category_skill (declared ownership) vs physical-disk folder counts. ' +
|
||||
'gap = owned but 0 files; thin = owned but < thinThreshold files; ' +
|
||||
'orphan = disk folder with no declared owner; misowned = folder under a skill other than its declared owner. ' +
|
||||
'No new domains proposed — gaps are within the already-covered domain only.',
|
||||
thinThreshold,
|
||||
categories,
|
||||
gaps,
|
||||
thin: thinCats,
|
||||
orphans,
|
||||
misowned,
|
||||
summary: {
|
||||
declaredCategories: categories.length,
|
||||
covered: categories.filter((c) => c.covered).length,
|
||||
gaps: gaps.length,
|
||||
thin: thinCats.length,
|
||||
orphans: orphans.length,
|
||||
misowned: misowned.length,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
// ===========================================================================
|
||||
// S14 — Detector 3: bloat/stale — K3-margin + dateless reference fraction
|
||||
// ===========================================================================
|
||||
|
||||
/**
|
||||
* Parse a "Last updated:" currency header from leading text. Mirrors
|
||||
* report-changes.parseLastUpdated: YYYY-MM normalizes to YYYY-MM-01; returns
|
||||
* null when no recognized header is present (unverifiable currency).
|
||||
*/
|
||||
export function extractLastUpdated(text) {
|
||||
const head = text.slice(0, 500);
|
||||
for (const re of LAST_UPDATED_PATTERNS) {
|
||||
const m = head.match(re);
|
||||
if (m) {
|
||||
const raw = m[1].trim();
|
||||
return raw.length === 7 ? raw + '-01' : raw;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Pure core: per-skill bloat (K3-margin) + stale (dateless reference fraction).
|
||||
* bloatCandidate = body within bloatMarginMin lines of the K3 limit -> split
|
||||
* staleCandidate = > datelessWarnRatio of refs lack a "Last updated:" header
|
||||
* Stale here means "unverifiable currency" (disk-only, deterministic). Genuine
|
||||
* poll-derived obsolescence is a separate, registry-dependent signal that lives
|
||||
* in change-report.json — intentionally out of this deterministic detector.
|
||||
* @param {Record<string,{bodyLines:number,refTotal:number,refDateless:number,datelessFiles?:string[]}>} skillInputs
|
||||
*/
|
||||
export function computeBloatFromInputs(skillInputs, { bloatMarginMin = BLOAT_MARGIN_MIN, datelessWarnRatio = DATELESS_WARN_RATIO } = {}) {
|
||||
const skills = [];
|
||||
for (const name of Object.keys(skillInputs).sort()) {
|
||||
const { bodyLines, refTotal, refDateless, datelessFiles = [] } = skillInputs[name];
|
||||
const k3Margin = K3_MAX_BODY_LINES - bodyLines;
|
||||
const bloatCandidate = k3Margin < bloatMarginMin;
|
||||
const datelessRatio = refTotal > 0 ? Number((refDateless / refTotal).toFixed(4)) : 0;
|
||||
const staleCandidate = datelessRatio > datelessWarnRatio;
|
||||
skills.push({ name, bodyLines, k3Margin, bloatCandidate, refTotal, refDateless, datelessRatio, staleCandidate, datelessFiles });
|
||||
}
|
||||
return {
|
||||
method:
|
||||
'bloat = eval.checkK3 bodyLines vs K3 limit (' + K3_MAX_BODY_LINES + '); margin < bloatMarginMin -> split-candidate. ' +
|
||||
'stale = fraction of reference files lacking a verifiable "Last updated:" header (unverifiable currency, disk-only). ' +
|
||||
'Poll-derived obsolescence (change-report.json) is a complementary registry-dependent signal, not computed here.',
|
||||
thresholds: { k3MaxBodyLines: K3_MAX_BODY_LINES, bloatMarginMin, datelessWarnRatio },
|
||||
skills,
|
||||
summary: {
|
||||
bloatCandidates: skills.filter((s) => s.bloatCandidate).map((s) => s.name),
|
||||
staleCandidates: skills.filter((s) => s.staleCandidate).map((s) => s.name),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/** Recursively count .md files under a directory. */
|
||||
function countMarkdown(dir) {
|
||||
let n = 0;
|
||||
if (!existsSync(dir)) return n;
|
||||
for (const e of readdirSync(dir, { withFileTypes: true })) {
|
||||
const p = join(dir, e.name);
|
||||
if (e.isDirectory()) n += countMarkdown(p);
|
||||
else if (e.isFile() && e.name.endsWith('.md')) n++;
|
||||
}
|
||||
return n;
|
||||
}
|
||||
|
||||
/** Recursively list .md file paths under a directory. */
|
||||
function listMarkdown(dir) {
|
||||
const out = [];
|
||||
if (!existsSync(dir)) return out;
|
||||
for (const e of readdirSync(dir, { withFileTypes: true })) {
|
||||
const p = join(dir, e.name);
|
||||
if (e.isDirectory()) out.push(...listMarkdown(p));
|
||||
else if (e.isFile() && e.name.endsWith('.md')) out.push(p);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Physical disk truth: { skill -> { category -> mdFileCount } } (read-only). */
|
||||
export function loadDiskCounts() {
|
||||
const out = {};
|
||||
for (const e of readdirSync(SKILLS_DIR, { withFileTypes: true })) {
|
||||
if (!e.isDirectory()) continue;
|
||||
const refDir = join(SKILLS_DIR, e.name, 'references');
|
||||
if (!existsSync(refDir)) continue;
|
||||
const cats = {};
|
||||
for (const c of readdirSync(refDir, { withFileTypes: true })) {
|
||||
if (c.isDirectory()) cats[c.name] = countMarkdown(join(refDir, c.name));
|
||||
}
|
||||
out[e.name] = cats;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Per-skill bloat/stale inputs read from disk (read-only). */
|
||||
export function loadBloatInputs() {
|
||||
const out = {};
|
||||
for (const e of readdirSync(SKILLS_DIR, { withFileTypes: true })) {
|
||||
if (!e.isDirectory()) continue;
|
||||
const skillMd = join(SKILLS_DIR, e.name, 'SKILL.md');
|
||||
if (!existsSync(skillMd)) continue;
|
||||
const { body } = splitFrontmatter(readFileSync(skillMd, 'utf8'));
|
||||
const bodyLines = checkK3(body).bodyLines;
|
||||
const refFiles = listMarkdown(join(SKILLS_DIR, e.name, 'references'));
|
||||
const datelessFiles = [];
|
||||
for (const f of refFiles) {
|
||||
if (extractLastUpdated(readFileSync(f, 'utf8')) === null) datelessFiles.push(relative(PLUGIN_ROOT, f));
|
||||
}
|
||||
out[e.name] = { bodyLines, refTotal: refFiles.length, refDateless: datelessFiles.length, datelessFiles };
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Read the five SKILL.md descriptions from disk. */
|
||||
function loadDescriptions() {
|
||||
const out = {};
|
||||
|
|
@ -170,11 +381,14 @@ function loadDescriptions() {
|
|||
function buildReport() {
|
||||
const promptSet = JSON.parse(readFileSync(PROMPTS_FILE, 'utf8'));
|
||||
const descriptions = loadDescriptions();
|
||||
const taxonomy = loadTaxonomy(TAX_DATA_DIR);
|
||||
return {
|
||||
rubric: 'skill-lifecycle',
|
||||
phase: 'B1',
|
||||
note: 'Detection only — never writes to skills/. Candidates feed decisions.json + operator-gate (B3).',
|
||||
overlap: computeOverlapFromInputs(descriptions, promptSet),
|
||||
coverage: computeCoverageFromInputs(taxonomy.category_skill, loadDiskCounts()),
|
||||
bloat: computeBloatFromInputs(loadBloatInputs()),
|
||||
};
|
||||
}
|
||||
|
||||
|
|
@ -195,15 +409,39 @@ function main() {
|
|||
}
|
||||
|
||||
const o = report.overlap;
|
||||
console.log(`\nSkill-livssyklus — B1 overlap-detektor (${o.pairs.length} par)\n`);
|
||||
console.log('Par (sortert på combined = grensetension + df-vektet leksikalsk):\n');
|
||||
console.log(`\nSkill-livssyklus — B1 deteksjon (overlap · coverage · bloat)\n`);
|
||||
console.log(`[1] Overlap — ${o.pairs.length} par (sortert på combined = grensetension + df-vektet leksikalsk):\n`);
|
||||
for (const p of o.pairs) {
|
||||
const lex = p.lexical.shared.length ? p.lexical.shared.join(', ') : '—';
|
||||
const focus = p.key === pairKey(...FOCUS_PAIR) ? ' ◀ FOCUS (Azure-deployment)' : '';
|
||||
console.log(` ${p.combined.toFixed(2).padStart(6)} ${p.pair.join(' / ')}${focus}`);
|
||||
console.log(` tension=${p.boundaryTension} lex(w=${p.lexical.weightedScore}, jac=${p.lexical.jaccard}) delt: ${lex}`);
|
||||
}
|
||||
console.log(`\nFocus-par: ${o.focusPair ? o.focusPair.pair.join(' <-> ') : '(ingen)'} — ${o.focusPairReason}`);
|
||||
console.log(`\n Focus-par: ${o.focusPair ? o.focusPair.pair.join(' <-> ') : '(ingen)'} — ${o.focusPairReason}`);
|
||||
|
||||
const c = report.coverage;
|
||||
console.log(`\n[2] Coverage/gap — ${c.summary.declaredCategories} deklarerte kategorier (terskel tynn < ${c.thinThreshold}):`);
|
||||
console.log(` dekket=${c.summary.covered} gap=${c.summary.gaps} tynn=${c.summary.thin} orphan=${c.summary.orphans} misowned=${c.summary.misowned}`);
|
||||
if (c.gaps.length) {
|
||||
console.log(' GAP (eid, 0 filer på disk):');
|
||||
for (const g of c.gaps) console.log(` • ${g.category} → ${g.owningSkill}`);
|
||||
}
|
||||
if (c.thin.length) {
|
||||
console.log(' TYNN (eid, men få filer):');
|
||||
for (const t of c.thin) console.log(` • ${t.category} → ${t.owningSkill} (${t.onDiskCount})`);
|
||||
}
|
||||
for (const x of c.orphans) console.log(` ⚠ orphan: ${x.skill}/${x.category} (${x.onDiskCount}) — ikke deklarert`);
|
||||
for (const x of c.misowned) console.log(` ⚠ misowned: ${x.skill}/${x.category} — deklarert eier=${x.declaredOwner}`);
|
||||
|
||||
const b = report.bloat;
|
||||
console.log(`\n[3] Bloat/stale — per skill (K3-grense ${b.thresholds.k3MaxBodyLines}; split-kandidat margin < ${b.thresholds.bloatMarginMin}; saner-kandidat dateless > ${b.thresholds.datelessWarnRatio}):`);
|
||||
for (const s of b.skills) {
|
||||
const flags = [s.bloatCandidate ? 'SPLIT' : '', s.staleCandidate ? 'SANER' : ''].filter(Boolean).join('+') || 'ok';
|
||||
console.log(` ${s.name.padEnd(22)} body=${String(s.bodyLines).padStart(3)} (margin ${String(s.k3Margin).padStart(3)}) dateless=${s.refDateless}/${s.refTotal} (${s.datelessRatio}) [${flags}]`);
|
||||
}
|
||||
if (b.summary.bloatCandidates.length) console.log(` Split-kandidater: ${b.summary.bloatCandidates.join(', ')}`);
|
||||
if (b.summary.staleCandidates.length) console.log(` Saner-kandidater: ${b.summary.staleCandidates.join(', ')}`);
|
||||
|
||||
console.log('\n(Rapport skrives med --write til data/skill-lifecycle-report.json; aldri til skills/.)\n');
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue