feat(ms-ai-architect): Sesjon 13 — B1 overlap-detektor (skill-livssyklus)
Spor B fase B1, første detektor (lag-1-analog på SKILL-granularitet). PRODUSERER KUN RAPPORT — skriver aldri til skills/ (invariant bekreftet). Detektor (scripts/kb-eval/detect-skill-lifecycle.mjs) kombinerer to deterministiske overlapp-signaler per skill-par: - grensetension: operatør-kuratert k1-trigger-prompts.json belongs_to-graf (out_of_domain = håndmerkede confusable-naboer), symmetrisk telt - df-vektet leksikalsk trigger-surface-overlapp (1/df nedvekter domene- vanlige ord som «azure»; format-boilerplate «triggers» filtrert) combined = grensetension + weightedScore. Empirisk: eng↔infra topper (combined 7.42, tension 6, delt: architecture/ azure/data/multi) — operatørens Azure-deployment-grenseinstinkt bekreftet. Surfaces som focusPair (operatør-utpekt B1-mål). CLI: default human-summary · --json · --write (rapport gitignored som de andre deteksjonsrapportene; detektor + kuraterte inputs er tracked). TDD: 8 nye tester i tests/kb-eval/ (tokenize, surface, df, lexical, pairKey, tension differensial-sjekk, full compute m/determinisme). Gate møtt. Ingen regresjon: validate 239 · kb-update 122 · kb-eval 23 (15+8) · kb-integrity 192/192. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01REiKFhP4w6xGXXqWKpPCJJ
This commit is contained in:
parent
e8ceb7a8b4
commit
2665a3a2d8
3 changed files with 409 additions and 0 deletions
212
scripts/kb-eval/detect-skill-lifecycle.mjs
Normal file
212
scripts/kb-eval/detect-skill-lifecycle.mjs
Normal file
|
|
@ -0,0 +1,212 @@
|
|||
#!/usr/bin/env node
|
||||
// detect-skill-lifecycle.mjs — Spor B / lag-1-analog at SKILL granularity.
|
||||
//
|
||||
// PRODUCES ONLY REPORTS — never writes to skills/. Mirrors the lag-1 invariant:
|
||||
// detection surfaces candidates; any skill-lifecycle op (merge/sanitize/retire/
|
||||
// create) goes through decisions.json + operator-gate (later phases B3).
|
||||
//
|
||||
// Sesjon 13 (B1, first detector): OVERLAP. Two deterministic signals, combined:
|
||||
// (1) boundary-tension graph — operator-curated k1-trigger-prompts.json:
|
||||
// each out_of_domain entry is a sibling prompt tagged belongs_to=<skill>,
|
||||
// i.e. a hand-labelled "confusable neighbour". Symmetric counts = how much
|
||||
// two skills sit on each other's trigger boundary.
|
||||
// (2) df-weighted lexical trigger-surface overlap — shared content tokens
|
||||
// between two descriptions, each weighted 1/df so domain-common vocabulary
|
||||
// ("azure") contributes little and distinctive shared tokens contribute more.
|
||||
//
|
||||
// The eng<->infra pair (Azure-deployment boundary) is surfaced as focusPair —
|
||||
// the operator-designated first target for B1.
|
||||
//
|
||||
// Usage:
|
||||
// node scripts/kb-eval/detect-skill-lifecycle.mjs # human summary
|
||||
// node scripts/kb-eval/detect-skill-lifecycle.mjs --json # machine output
|
||||
// node scripts/kb-eval/detect-skill-lifecycle.mjs --write # persist report JSON
|
||||
//
|
||||
// Zero dependencies. Reuses eval.mjs extractors + kb-update atomic-write.
|
||||
|
||||
import { readFileSync, readdirSync, existsSync, mkdirSync } from 'node:fs';
|
||||
import { join, dirname } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { splitFrontmatter, extractDescription } from './eval.mjs';
|
||||
import { atomicWriteJson } from '../kb-update/lib/atomic-write.mjs';
|
||||
|
||||
const __dirname = dirname(fileURLToPath(import.meta.url));
|
||||
const PLUGIN_ROOT = join(__dirname, '..', '..');
|
||||
const SKILLS_DIR = join(PLUGIN_ROOT, 'skills');
|
||||
const DATA_DIR = join(__dirname, 'data');
|
||||
const PROMPTS_FILE = join(DATA_DIR, 'k1-trigger-prompts.json');
|
||||
const OUT_FILE = join(DATA_DIR, 'skill-lifecycle-report.json');
|
||||
|
||||
// Operator-designated B1 focus boundary.
|
||||
const FOCUS_PAIR = ['ms-ai-engineering', 'ms-ai-infrastructure'];
|
||||
|
||||
// Function words (no + en) of length >= 3. Tokens < 3 chars are dropped anyway,
|
||||
// so this list only needs the longer connectives. Domain nouns are NOT here —
|
||||
// df-weighting handles common domain vocabulary instead.
|
||||
const STOPWORDS = new Set([
|
||||
'the', 'and', 'for', 'with', 'between', 'before', 'not', 'are', 'that', 'this',
|
||||
'into', 'over', 'per', 'use', 'used', 'when', 'which', 'how', 'via', 'from',
|
||||
'eller', 'som', 'til', 'med', 'mot', 'ved', 'for', 'har', 'kan', 'ikke', 'der',
|
||||
'det', 'den', 'ein', 'eit', 'sin',
|
||||
// description-format boilerplate (every skill ends with "Triggers on:")
|
||||
'triggers', 'trigger',
|
||||
]);
|
||||
|
||||
/** Lowercase, split on non-alphanumeric, drop stopwords + tokens < 3 chars. */
|
||||
export function tokenize(text) {
|
||||
return (text.toLowerCase().match(/[a-z0-9æøå]+/g) || [])
|
||||
.filter((w) => w.length >= 3 && !STOPWORDS.has(w));
|
||||
}
|
||||
|
||||
/** Trigger surface of a description: quoted phrases + content-token set. */
|
||||
export function extractTriggerSurface(description) {
|
||||
const phrases = (description.match(/"([^"]+)"/g) || []).map((p) => p.slice(1, -1));
|
||||
return { phrases, tokens: new Set(tokenize(description)) };
|
||||
}
|
||||
|
||||
/** token -> number of skill-surfaces that contain it. */
|
||||
export function buildDocumentFrequency(surfaces) {
|
||||
const df = new Map();
|
||||
for (const s of surfaces) {
|
||||
for (const t of s.tokens) df.set(t, (df.get(t) || 0) + 1);
|
||||
}
|
||||
return df;
|
||||
}
|
||||
|
||||
/** Order-independent pair key. */
|
||||
export function pairKey(a, b) {
|
||||
return [a, b].sort().join('|');
|
||||
}
|
||||
|
||||
/** Lexical overlap between two surfaces, df-weighted. */
|
||||
export function lexicalOverlap(surfaceA, surfaceB, df) {
|
||||
const shared = [];
|
||||
for (const t of surfaceA.tokens) if (surfaceB.tokens.has(t)) shared.push(t);
|
||||
shared.sort();
|
||||
const union = new Set([...surfaceA.tokens, ...surfaceB.tokens]).size;
|
||||
const jaccard = union > 0 ? shared.length / union : 0;
|
||||
let weightedScore = 0;
|
||||
for (const t of shared) weightedScore += 1 / (df.get(t) || 1);
|
||||
return {
|
||||
shared,
|
||||
jaccard: Number(jaccard.toFixed(4)),
|
||||
weightedScore: Number(weightedScore.toFixed(4)),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Symmetric boundary-tension matrix from the curated prompt set.
|
||||
* Counts out_of_domain entries whose belongs_to is one of the real skills
|
||||
* (controls / out-of-stack entries are ignored). Keyed by pairKey.
|
||||
*/
|
||||
export function boundaryTensionMatrix(promptSet) {
|
||||
const skills = Object.keys(promptSet).filter((k) => k !== '_meta');
|
||||
const skillSet = new Set(skills);
|
||||
const m = {};
|
||||
for (const s of skills) {
|
||||
for (const e of promptSet[s].out_of_domain || []) {
|
||||
const b = e && e.belongs_to;
|
||||
if (!skillSet.has(b) || b === s) continue;
|
||||
const k = pairKey(s, b);
|
||||
m[k] = (m[k] || 0) + 1;
|
||||
}
|
||||
}
|
||||
return m;
|
||||
}
|
||||
|
||||
/**
|
||||
* Pure core: given { skill -> description } and the curated prompt set, compute
|
||||
* the overlap report section. combined = boundaryTension + weightedScore
|
||||
* (operator-grounded primary signal + distinctive-lexical corroboration).
|
||||
*/
|
||||
export function computeOverlapFromInputs(descriptionsBySkill, promptSet) {
|
||||
const skills = Object.keys(descriptionsBySkill).sort();
|
||||
const surfaces = {};
|
||||
for (const s of skills) surfaces[s] = extractTriggerSurface(descriptionsBySkill[s]);
|
||||
const df = buildDocumentFrequency(Object.values(surfaces));
|
||||
const tension = boundaryTensionMatrix(promptSet);
|
||||
|
||||
const pairs = [];
|
||||
for (let i = 0; i < skills.length; i++) {
|
||||
for (let j = i + 1; j < skills.length; j++) {
|
||||
const a = skills[i];
|
||||
const b = skills[j];
|
||||
const key = pairKey(a, b);
|
||||
const lexical = lexicalOverlap(surfaces[a], surfaces[b], df);
|
||||
const boundaryTension = tension[key] || 0;
|
||||
const combined = Number((boundaryTension + lexical.weightedScore).toFixed(4));
|
||||
pairs.push({ pair: [a, b], key, boundaryTension, lexical, combined });
|
||||
}
|
||||
}
|
||||
// sort by combined desc, then key asc for stable ties
|
||||
pairs.sort((x, y) => y.combined - x.combined || x.key.localeCompare(y.key));
|
||||
|
||||
const focusKey = pairKey(...FOCUS_PAIR);
|
||||
const focusPair = pairs.find((p) => p.key === focusKey) || null;
|
||||
|
||||
return {
|
||||
method:
|
||||
'deterministic: (1) operator-curated boundary-tension (k1-trigger-prompts belongs_to), ' +
|
||||
'(2) df-weighted lexical trigger-surface overlap. combined = boundaryTension + weightedScore.',
|
||||
focusPairReason:
|
||||
'Azure-deployment boundary engineering(build) <-> infrastructure(operate) — operator-designated B1 target.',
|
||||
pairs,
|
||||
focusPair,
|
||||
};
|
||||
}
|
||||
|
||||
/** Read the five SKILL.md descriptions from disk. */
|
||||
function loadDescriptions() {
|
||||
const out = {};
|
||||
for (const e of readdirSync(SKILLS_DIR, { withFileTypes: true })) {
|
||||
if (!e.isDirectory()) continue;
|
||||
const md = join(SKILLS_DIR, e.name, 'SKILL.md');
|
||||
if (!existsSync(md)) continue;
|
||||
out[e.name] = extractDescription(splitFrontmatter(readFileSync(md, 'utf8')).frontmatter);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function buildReport() {
|
||||
const promptSet = JSON.parse(readFileSync(PROMPTS_FILE, 'utf8'));
|
||||
const descriptions = loadDescriptions();
|
||||
return {
|
||||
rubric: 'skill-lifecycle',
|
||||
phase: 'B1',
|
||||
note: 'Detection only — never writes to skills/. Candidates feed decisions.json + operator-gate (B3).',
|
||||
overlap: computeOverlapFromInputs(descriptions, promptSet),
|
||||
};
|
||||
}
|
||||
|
||||
function main() {
|
||||
const args = process.argv.slice(2);
|
||||
const jsonOut = args.includes('--json');
|
||||
const doWrite = args.includes('--write');
|
||||
|
||||
const report = buildReport();
|
||||
|
||||
if (doWrite) {
|
||||
mkdirSync(DATA_DIR, { recursive: true });
|
||||
atomicWriteJson(OUT_FILE, report);
|
||||
}
|
||||
if (jsonOut) {
|
||||
process.stdout.write(JSON.stringify(report) + '\n');
|
||||
return;
|
||||
}
|
||||
|
||||
const o = report.overlap;
|
||||
console.log(`\nSkill-livssyklus — B1 overlap-detektor (${o.pairs.length} par)\n`);
|
||||
console.log('Par (sortert på combined = grensetension + df-vektet leksikalsk):\n');
|
||||
for (const p of o.pairs) {
|
||||
const lex = p.lexical.shared.length ? p.lexical.shared.join(', ') : '—';
|
||||
const focus = p.key === pairKey(...FOCUS_PAIR) ? ' ◀ FOCUS (Azure-deployment)' : '';
|
||||
console.log(` ${p.combined.toFixed(2).padStart(6)} ${p.pair.join(' / ')}${focus}`);
|
||||
console.log(` tension=${p.boundaryTension} lex(w=${p.lexical.weightedScore}, jac=${p.lexical.jaccard}) delt: ${lex}`);
|
||||
}
|
||||
console.log(`\nFocus-par: ${o.focusPair ? o.focusPair.pair.join(' <-> ') : '(ingen)'} — ${o.focusPairReason}`);
|
||||
console.log('\n(Rapport skrives med --write til data/skill-lifecycle-report.json; aldri til skills/.)\n');
|
||||
}
|
||||
|
||||
if (process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1]) {
|
||||
main();
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue