feat(ms-ai-architect): Sesjon 15 — B2 K10 søsken-scope-ikke-overlapp
- Refaktor: overlap-kjerne flyttet til scripts/kb-eval/lib/sibling-overlap.mjs (bryter sirkulær import eval.mjs<->detect); detect re-eksporterer → B1-tester urørt - K10 = søsken-scope-ikke-overlapp, deterministisk cross-skill: perSkillSiblingOverlap + attachSiblingOverlap i eval.mjs. combined = boundaryTension + df-vektet leksikalsk; per-skill verdikt = verste søskenpar; terskel 7.0 (naturlig gap 7.42→6.67) - Empirisk (alle 5): eng+infra FAIL (7.42 mot hverandre), advisor/gov/sec PASS → eng↔infra-signal mater B3 merge/saner (ikke blokkering) - Gated baseline-regen via --write (descriptions urørt → judge K1/K4/K7/K8/K9 merget uendret, ikke fabrikkert); rubric K1-K10 - TDD: +9 tester (tests/kb-eval/test-k10-sibling-overlap.test.mjs), kb-eval 31→40 - 0 skriving til skills/. Suiter: validate 239 · kb-update 122 · kb-integrity 192/192
This commit is contained in:
parent
5ad4ed025c
commit
ba597eb988
5 changed files with 400 additions and 123 deletions
|
|
@ -1,6 +1,6 @@
|
|||
{
|
||||
"rubric": "K1-K9",
|
||||
"note": "Deterministic: K2,K3,K5,K6,refCountConsistency. LLM-judge (operator-gated): K1,K4,K7,K8,K9.",
|
||||
"rubric": "K1-K10",
|
||||
"note": "Deterministic: K2,K3,K5,K6,refCountConsistency,K10(siblingScopeOverlap). LLM-judge (operator-gated): K1,K4,K7,K8,K9.",
|
||||
"skills": [
|
||||
{
|
||||
"name": "ms-ai-advisor",
|
||||
|
|
@ -55,6 +55,12 @@
|
|||
"2026",
|
||||
"v3.0"
|
||||
]
|
||||
},
|
||||
"K10_siblingScopeOverlap": {
|
||||
"maxCombined": 4.5833,
|
||||
"worstSibling": "ms-ai-engineering",
|
||||
"pass": true,
|
||||
"threshold": 7
|
||||
}
|
||||
},
|
||||
"judgeInputs": {
|
||||
|
|
@ -158,6 +164,12 @@
|
|||
"preview",
|
||||
"GA"
|
||||
]
|
||||
},
|
||||
"K10_siblingScopeOverlap": {
|
||||
"maxCombined": 7.4167,
|
||||
"worstSibling": "ms-ai-infrastructure",
|
||||
"pass": false,
|
||||
"threshold": 7
|
||||
}
|
||||
},
|
||||
"judgeInputs": {
|
||||
|
|
@ -253,6 +265,12 @@
|
|||
"sample": [
|
||||
"2024"
|
||||
]
|
||||
},
|
||||
"K10_siblingScopeOverlap": {
|
||||
"maxCombined": 6.6667,
|
||||
"worstSibling": "ms-ai-engineering",
|
||||
"pass": true,
|
||||
"threshold": 7
|
||||
}
|
||||
},
|
||||
"judgeInputs": {
|
||||
|
|
@ -348,6 +366,12 @@
|
|||
"preview",
|
||||
"GA"
|
||||
]
|
||||
},
|
||||
"K10_siblingScopeOverlap": {
|
||||
"maxCombined": 7.4167,
|
||||
"worstSibling": "ms-ai-engineering",
|
||||
"pass": false,
|
||||
"threshold": 7
|
||||
}
|
||||
},
|
||||
"judgeInputs": {
|
||||
|
|
@ -443,6 +467,12 @@
|
|||
"sample": [
|
||||
"2025"
|
||||
]
|
||||
},
|
||||
"K10_siblingScopeOverlap": {
|
||||
"maxCombined": 6.5,
|
||||
"worstSibling": "ms-ai-governance",
|
||||
"pass": true,
|
||||
"threshold": 7
|
||||
}
|
||||
},
|
||||
"judgeInputs": {
|
||||
|
|
|
|||
|
|
@ -39,6 +39,29 @@ import { fileURLToPath } from 'node:url';
|
|||
import { splitFrontmatter, extractDescription, checkK3 } from './eval.mjs';
|
||||
import { loadTaxonomy } from '../kb-update/lib/taxonomy.mjs';
|
||||
import { atomicWriteJson } from '../kb-update/lib/atomic-write.mjs';
|
||||
import {
|
||||
FOCUS_PAIR,
|
||||
tokenize,
|
||||
extractTriggerSurface,
|
||||
buildDocumentFrequency,
|
||||
pairKey,
|
||||
lexicalOverlap,
|
||||
boundaryTensionMatrix,
|
||||
computeOverlapFromInputs,
|
||||
} from './lib/sibling-overlap.mjs';
|
||||
|
||||
// Re-export the overlap core (moved to lib/sibling-overlap.mjs in S15/B2 to
|
||||
// avoid a circular import with eval.mjs) so existing B1 callers/tests keep
|
||||
// importing it from this module unchanged.
|
||||
export {
|
||||
tokenize,
|
||||
extractTriggerSurface,
|
||||
buildDocumentFrequency,
|
||||
pairKey,
|
||||
lexicalOverlap,
|
||||
boundaryTensionMatrix,
|
||||
computeOverlapFromInputs,
|
||||
};
|
||||
|
||||
const __dirname = dirname(fileURLToPath(import.meta.url));
|
||||
const PLUGIN_ROOT = join(__dirname, '..', '..');
|
||||
|
|
@ -48,9 +71,6 @@ const TAX_DATA_DIR = join(__dirname, '..', 'kb-update', 'data');
|
|||
const PROMPTS_FILE = join(DATA_DIR, 'k1-trigger-prompts.json');
|
||||
const OUT_FILE = join(DATA_DIR, 'skill-lifecycle-report.json');
|
||||
|
||||
// Operator-designated B1 focus boundary.
|
||||
const FOCUS_PAIR = ['ms-ai-engineering', 'ms-ai-infrastructure'];
|
||||
|
||||
// --- S14 detector thresholds (named, documented) -------------------------
|
||||
// K3 hard body-length limit (mirrors eval.mjs K3_MAX_BODY_LINES — single source
|
||||
// of bodyLines is eval.checkK3; this constant only computes the margin).
|
||||
|
|
@ -74,121 +94,6 @@ const LAST_UPDATED_PATTERNS = [
|
|||
/\*\*Dato:\*\*\s*([\d-]+)/i,
|
||||
];
|
||||
|
||||
// Function words (no + en) of length >= 3. Tokens < 3 chars are dropped anyway,
|
||||
// so this list only needs the longer connectives. Domain nouns are NOT here —
|
||||
// df-weighting handles common domain vocabulary instead.
|
||||
const STOPWORDS = new Set([
|
||||
'the', 'and', 'for', 'with', 'between', 'before', 'not', 'are', 'that', 'this',
|
||||
'into', 'over', 'per', 'use', 'used', 'when', 'which', 'how', 'via', 'from',
|
||||
'eller', 'som', 'til', 'med', 'mot', 'ved', 'for', 'har', 'kan', 'ikke', 'der',
|
||||
'det', 'den', 'ein', 'eit', 'sin',
|
||||
// description-format boilerplate (every skill ends with "Triggers on:")
|
||||
'triggers', 'trigger',
|
||||
]);
|
||||
|
||||
/** Lowercase, split on non-alphanumeric, drop stopwords + tokens < 3 chars. */
|
||||
export function tokenize(text) {
|
||||
return (text.toLowerCase().match(/[a-z0-9æøå]+/g) || [])
|
||||
.filter((w) => w.length >= 3 && !STOPWORDS.has(w));
|
||||
}
|
||||
|
||||
/** Trigger surface of a description: quoted phrases + content-token set. */
|
||||
export function extractTriggerSurface(description) {
|
||||
const phrases = (description.match(/"([^"]+)"/g) || []).map((p) => p.slice(1, -1));
|
||||
return { phrases, tokens: new Set(tokenize(description)) };
|
||||
}
|
||||
|
||||
/** token -> number of skill-surfaces that contain it. */
|
||||
export function buildDocumentFrequency(surfaces) {
|
||||
const df = new Map();
|
||||
for (const s of surfaces) {
|
||||
for (const t of s.tokens) df.set(t, (df.get(t) || 0) + 1);
|
||||
}
|
||||
return df;
|
||||
}
|
||||
|
||||
/** Order-independent pair key. */
|
||||
export function pairKey(a, b) {
|
||||
return [a, b].sort().join('|');
|
||||
}
|
||||
|
||||
/** Lexical overlap between two surfaces, df-weighted. */
|
||||
export function lexicalOverlap(surfaceA, surfaceB, df) {
|
||||
const shared = [];
|
||||
for (const t of surfaceA.tokens) if (surfaceB.tokens.has(t)) shared.push(t);
|
||||
shared.sort();
|
||||
const union = new Set([...surfaceA.tokens, ...surfaceB.tokens]).size;
|
||||
const jaccard = union > 0 ? shared.length / union : 0;
|
||||
let weightedScore = 0;
|
||||
for (const t of shared) weightedScore += 1 / (df.get(t) || 1);
|
||||
return {
|
||||
shared,
|
||||
jaccard: Number(jaccard.toFixed(4)),
|
||||
weightedScore: Number(weightedScore.toFixed(4)),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Symmetric boundary-tension matrix from the curated prompt set.
|
||||
* Counts out_of_domain entries whose belongs_to is one of the real skills
|
||||
* (controls / out-of-stack entries are ignored). Keyed by pairKey.
|
||||
*/
|
||||
export function boundaryTensionMatrix(promptSet) {
|
||||
const skills = Object.keys(promptSet).filter((k) => k !== '_meta');
|
||||
const skillSet = new Set(skills);
|
||||
const m = {};
|
||||
for (const s of skills) {
|
||||
for (const e of promptSet[s].out_of_domain || []) {
|
||||
const b = e && e.belongs_to;
|
||||
if (!skillSet.has(b) || b === s) continue;
|
||||
const k = pairKey(s, b);
|
||||
m[k] = (m[k] || 0) + 1;
|
||||
}
|
||||
}
|
||||
return m;
|
||||
}
|
||||
|
||||
/**
|
||||
* Pure core: given { skill -> description } and the curated prompt set, compute
|
||||
* the overlap report section. combined = boundaryTension + weightedScore
|
||||
* (operator-grounded primary signal + distinctive-lexical corroboration).
|
||||
*/
|
||||
export function computeOverlapFromInputs(descriptionsBySkill, promptSet) {
|
||||
const skills = Object.keys(descriptionsBySkill).sort();
|
||||
const surfaces = {};
|
||||
for (const s of skills) surfaces[s] = extractTriggerSurface(descriptionsBySkill[s]);
|
||||
const df = buildDocumentFrequency(Object.values(surfaces));
|
||||
const tension = boundaryTensionMatrix(promptSet);
|
||||
|
||||
const pairs = [];
|
||||
for (let i = 0; i < skills.length; i++) {
|
||||
for (let j = i + 1; j < skills.length; j++) {
|
||||
const a = skills[i];
|
||||
const b = skills[j];
|
||||
const key = pairKey(a, b);
|
||||
const lexical = lexicalOverlap(surfaces[a], surfaces[b], df);
|
||||
const boundaryTension = tension[key] || 0;
|
||||
const combined = Number((boundaryTension + lexical.weightedScore).toFixed(4));
|
||||
pairs.push({ pair: [a, b], key, boundaryTension, lexical, combined });
|
||||
}
|
||||
}
|
||||
// sort by combined desc, then key asc for stable ties
|
||||
pairs.sort((x, y) => y.combined - x.combined || x.key.localeCompare(y.key));
|
||||
|
||||
const focusKey = pairKey(...FOCUS_PAIR);
|
||||
const focusPair = pairs.find((p) => p.key === focusKey) || null;
|
||||
|
||||
return {
|
||||
method:
|
||||
'deterministic: (1) operator-curated boundary-tension (k1-trigger-prompts belongs_to), ' +
|
||||
'(2) df-weighted lexical trigger-surface overlap. combined = boundaryTension + weightedScore.',
|
||||
focusPairReason:
|
||||
'Azure-deployment boundary engineering(build) <-> infrastructure(operate) — operator-designated B1 target.',
|
||||
pairs,
|
||||
focusPair,
|
||||
};
|
||||
}
|
||||
|
||||
// ===========================================================================
|
||||
// S14 — Detector 2: coverage/gap (in-domain) — taxonomy vs physical disk
|
||||
// ===========================================================================
|
||||
|
|
|
|||
|
|
@ -17,14 +17,20 @@ import { readdirSync, readFileSync, existsSync, mkdirSync } from 'node:fs';
|
|||
import { join, dirname, relative } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { atomicWriteJson } from '../kb-update/lib/atomic-write.mjs';
|
||||
import {
|
||||
computeOverlapFromInputs,
|
||||
perSkillSiblingOverlap,
|
||||
K10_OVERLAP_THRESHOLD,
|
||||
} from './lib/sibling-overlap.mjs';
|
||||
|
||||
const __dirname = dirname(fileURLToPath(import.meta.url));
|
||||
const PLUGIN_ROOT = join(__dirname, '..', '..');
|
||||
const SKILLS_DIR = join(PLUGIN_ROOT, 'skills');
|
||||
const OUT_DIR = join(__dirname, 'data');
|
||||
const OUT_FILE = join(OUT_DIR, 'eval-baseline.json');
|
||||
const PROMPTS_FILE = join(OUT_DIR, 'k1-trigger-prompts.json');
|
||||
|
||||
// Thresholds — see spec "Rubrikk K1–K9".
|
||||
// Thresholds — see spec "Rubrikk K1–K10".
|
||||
const K3_MAX_BODY_LINES = 500;
|
||||
const K5_MIN_NAMED_RATIO = 0.2;
|
||||
|
||||
|
|
@ -198,6 +204,25 @@ export function evalSkill(skillName) {
|
|||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* K10 — sibling-scope-non-overlap. Deterministic, computed ACROSS all skills
|
||||
* (unlike K2/K3/K5/K6 which are per-skill), so it is attached after the
|
||||
* per-skill pass. Reuses the B1 overlap core: combined = boundaryTension +
|
||||
* df-weighted lexical overlap; each skill's K10 verdict is its worst sibling
|
||||
* pair (perSkillSiblingOverlap). Mutates + returns the skills array.
|
||||
*/
|
||||
export function attachSiblingOverlap(skills, promptSet, { threshold = K10_OVERLAP_THRESHOLD } = {}) {
|
||||
const descriptionsBySkill = {};
|
||||
for (const s of skills) descriptionsBySkill[s.name] = s.judgeInputs.description;
|
||||
const overlap = computeOverlapFromInputs(descriptionsBySkill, promptSet);
|
||||
const k10 = perSkillSiblingOverlap(overlap.pairs, { threshold });
|
||||
for (const s of skills) {
|
||||
s.deterministic.K10_siblingScopeOverlap =
|
||||
k10[s.name] || { maxCombined: 0, worstSibling: null, pass: true, threshold };
|
||||
}
|
||||
return skills;
|
||||
}
|
||||
|
||||
function main() {
|
||||
const args = process.argv.slice(2);
|
||||
const jsonOut = args.includes('--json');
|
||||
|
|
@ -210,6 +235,13 @@ function main() {
|
|||
|
||||
const skills = skillNames.map(evalSkill).filter(Boolean);
|
||||
|
||||
// K10 — sibling-scope-non-overlap (deterministic, cross-skill). Attached after
|
||||
// the per-skill pass since it needs every description + the curated prompt set.
|
||||
if (existsSync(PROMPTS_FILE)) {
|
||||
const promptSet = JSON.parse(readFileSync(PROMPTS_FILE, 'utf8'));
|
||||
attachSiblingOverlap(skills, promptSet);
|
||||
}
|
||||
|
||||
// Merge operator-gated LLM-judge results (K1/K4/K7/K8/K9) if present.
|
||||
const judgeFile = join(OUT_DIR, 'judge-results.json');
|
||||
if (existsSync(judgeFile)) {
|
||||
|
|
@ -218,8 +250,8 @@ function main() {
|
|||
}
|
||||
|
||||
const report = {
|
||||
rubric: 'K1-K9',
|
||||
note: 'Deterministic: K2,K3,K5,K6,refCountConsistency. LLM-judge (operator-gated): K1,K4,K7,K8,K9.',
|
||||
rubric: 'K1-K10',
|
||||
note: 'Deterministic: K2,K3,K5,K6,refCountConsistency,K10(siblingScopeOverlap). LLM-judge (operator-gated): K1,K4,K7,K8,K9.',
|
||||
skills,
|
||||
};
|
||||
|
||||
|
|
@ -245,6 +277,10 @@ function main() {
|
|||
console.log(` K6 routing-tab : ${flag(d.K6_routingTable.pass)} (navngitte startfiler=${d.K6_routingTable.namedStartFiles})`);
|
||||
const rc = d.refCountConsistency;
|
||||
console.log(` ref-tall : ${flag(rc.consistent)}${rc.consistent ? '' : ' — ' + JSON.stringify(rc.mismatches)}`);
|
||||
const k10 = d.K10_siblingScopeOverlap;
|
||||
if (k10) {
|
||||
console.log(` K10 søsken-ovl : ${flag(k10.pass)} (maks ${k10.maxCombined} mot ${k10.worstSibling}, terskel ${k10.threshold})`);
|
||||
}
|
||||
console.log(` K9 hints : ${d.K9_timeSensitiveHints.timeSensitiveTokenHits} tid-sensitive tokens i body`);
|
||||
console.log('');
|
||||
}
|
||||
|
|
|
|||
180
scripts/kb-eval/lib/sibling-overlap.mjs
Normal file
180
scripts/kb-eval/lib/sibling-overlap.mjs
Normal file
|
|
@ -0,0 +1,180 @@
|
|||
// sibling-overlap.mjs — shared deterministic overlap core for the skill rubric.
|
||||
//
|
||||
// Extracted from detect-skill-lifecycle.mjs (Sesjon 15 / B2) so BOTH the B1
|
||||
// detector and eval.mjs can use the same overlap mechanics without a circular
|
||||
// import (detect already imports from eval.mjs; eval.mjs now needs the overlap
|
||||
// core, so it lives here and both import it).
|
||||
//
|
||||
// Two deterministic signals, combined per skill pair:
|
||||
// (1) boundary-tension graph — operator-curated k1-trigger-prompts.json:
|
||||
// each out_of_domain entry is a sibling prompt tagged belongs_to=<skill>,
|
||||
// a hand-labelled "confusable neighbour". Symmetric counts = how much two
|
||||
// skills sit on each other's trigger boundary.
|
||||
// (2) df-weighted lexical trigger-surface overlap — shared content tokens
|
||||
// between two descriptions, each weighted 1/df so domain-common vocabulary
|
||||
// ("azure") contributes little and distinctive shared tokens contribute more.
|
||||
// combined = boundaryTension + weightedScore.
|
||||
//
|
||||
// K10 (B2): perSkillSiblingOverlap maps the pairwise combined scores to a
|
||||
// per-skill verdict — a skill's K10 is its worst sibling pair; it FAILS when
|
||||
// that exceeds the rubric threshold. This is the bar B3 merge/sanitize rests on.
|
||||
//
|
||||
// Zero dependencies — pure functions only (no fs/path). Inputs are passed in.
|
||||
|
||||
// Operator-designated B1 focus boundary (Azure-deployment: build <-> operate).
|
||||
export const FOCUS_PAIR = ['ms-ai-engineering', 'ms-ai-infrastructure'];
|
||||
|
||||
// K10 rubric threshold — a skill FAILS sibling-scope-non-overlap when its worst
|
||||
// sibling pair's combined score is at or above this. Set in the natural
|
||||
// distribution gap (7.42 -> 6.67) just below the operator-designated eng<->infra
|
||||
// boundary, so exactly that pair is surfaced as the B3 merge/sanitize signal.
|
||||
export const K10_OVERLAP_THRESHOLD = 7.0;
|
||||
|
||||
// Function words (no + en) of length >= 3. Tokens < 3 chars are dropped anyway,
|
||||
// so this list only needs the longer connectives. Domain nouns are NOT here —
|
||||
// df-weighting handles common domain vocabulary instead.
|
||||
const STOPWORDS = new Set([
|
||||
'the', 'and', 'for', 'with', 'between', 'before', 'not', 'are', 'that', 'this',
|
||||
'into', 'over', 'per', 'use', 'used', 'when', 'which', 'how', 'via', 'from',
|
||||
'eller', 'som', 'til', 'med', 'mot', 'ved', 'for', 'har', 'kan', 'ikke', 'der',
|
||||
'det', 'den', 'ein', 'eit', 'sin',
|
||||
// description-format boilerplate (every skill ends with "Triggers on:")
|
||||
'triggers', 'trigger',
|
||||
]);
|
||||
|
||||
/** Lowercase, split on non-alphanumeric, drop stopwords + tokens < 3 chars. */
|
||||
export function tokenize(text) {
|
||||
return (text.toLowerCase().match(/[a-z0-9æøå]+/g) || [])
|
||||
.filter((w) => w.length >= 3 && !STOPWORDS.has(w));
|
||||
}
|
||||
|
||||
/** Trigger surface of a description: quoted phrases + content-token set. */
|
||||
export function extractTriggerSurface(description) {
|
||||
const phrases = (description.match(/"([^"]+)"/g) || []).map((p) => p.slice(1, -1));
|
||||
return { phrases, tokens: new Set(tokenize(description)) };
|
||||
}
|
||||
|
||||
/** token -> number of skill-surfaces that contain it. */
|
||||
export function buildDocumentFrequency(surfaces) {
|
||||
const df = new Map();
|
||||
for (const s of surfaces) {
|
||||
for (const t of s.tokens) df.set(t, (df.get(t) || 0) + 1);
|
||||
}
|
||||
return df;
|
||||
}
|
||||
|
||||
/** Order-independent pair key. */
|
||||
export function pairKey(a, b) {
|
||||
return [a, b].sort().join('|');
|
||||
}
|
||||
|
||||
/** Lexical overlap between two surfaces, df-weighted. */
|
||||
export function lexicalOverlap(surfaceA, surfaceB, df) {
|
||||
const shared = [];
|
||||
for (const t of surfaceA.tokens) if (surfaceB.tokens.has(t)) shared.push(t);
|
||||
shared.sort();
|
||||
const union = new Set([...surfaceA.tokens, ...surfaceB.tokens]).size;
|
||||
const jaccard = union > 0 ? shared.length / union : 0;
|
||||
let weightedScore = 0;
|
||||
for (const t of shared) weightedScore += 1 / (df.get(t) || 1);
|
||||
return {
|
||||
shared,
|
||||
jaccard: Number(jaccard.toFixed(4)),
|
||||
weightedScore: Number(weightedScore.toFixed(4)),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Symmetric boundary-tension matrix from the curated prompt set.
|
||||
* Counts out_of_domain entries whose belongs_to is one of the real skills
|
||||
* (controls / out-of-stack entries are ignored). Keyed by pairKey.
|
||||
*/
|
||||
export function boundaryTensionMatrix(promptSet) {
|
||||
const skills = Object.keys(promptSet).filter((k) => k !== '_meta');
|
||||
const skillSet = new Set(skills);
|
||||
const m = {};
|
||||
for (const s of skills) {
|
||||
for (const e of promptSet[s].out_of_domain || []) {
|
||||
const b = e && e.belongs_to;
|
||||
if (!skillSet.has(b) || b === s) continue;
|
||||
const k = pairKey(s, b);
|
||||
m[k] = (m[k] || 0) + 1;
|
||||
}
|
||||
}
|
||||
return m;
|
||||
}
|
||||
|
||||
/**
|
||||
* Pure core: given { skill -> description } and the curated prompt set, compute
|
||||
* the overlap report section. combined = boundaryTension + weightedScore
|
||||
* (operator-grounded primary signal + distinctive-lexical corroboration).
|
||||
*/
|
||||
export function computeOverlapFromInputs(descriptionsBySkill, promptSet) {
|
||||
const skills = Object.keys(descriptionsBySkill).sort();
|
||||
const surfaces = {};
|
||||
for (const s of skills) surfaces[s] = extractTriggerSurface(descriptionsBySkill[s]);
|
||||
const df = buildDocumentFrequency(Object.values(surfaces));
|
||||
const tension = boundaryTensionMatrix(promptSet);
|
||||
|
||||
const pairs = [];
|
||||
for (let i = 0; i < skills.length; i++) {
|
||||
for (let j = i + 1; j < skills.length; j++) {
|
||||
const a = skills[i];
|
||||
const b = skills[j];
|
||||
const key = pairKey(a, b);
|
||||
const lexical = lexicalOverlap(surfaces[a], surfaces[b], df);
|
||||
const boundaryTension = tension[key] || 0;
|
||||
const combined = Number((boundaryTension + lexical.weightedScore).toFixed(4));
|
||||
pairs.push({ pair: [a, b], key, boundaryTension, lexical, combined });
|
||||
}
|
||||
}
|
||||
// sort by combined desc, then key asc for stable ties
|
||||
pairs.sort((x, y) => y.combined - x.combined || x.key.localeCompare(y.key));
|
||||
|
||||
const focusKey = pairKey(...FOCUS_PAIR);
|
||||
const focusPair = pairs.find((p) => p.key === focusKey) || null;
|
||||
|
||||
return {
|
||||
method:
|
||||
'deterministic: (1) operator-curated boundary-tension (k1-trigger-prompts belongs_to), ' +
|
||||
'(2) df-weighted lexical trigger-surface overlap. combined = boundaryTension + weightedScore.',
|
||||
focusPairReason:
|
||||
'Azure-deployment boundary engineering(build) <-> infrastructure(operate) — operator-designated B1 target.',
|
||||
pairs,
|
||||
focusPair,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* K10 — sibling-scope-non-overlap, per skill.
|
||||
*
|
||||
* Maps the pairwise `combined` scores (from computeOverlapFromInputs) to a
|
||||
* per-skill verdict: a skill's K10 is its WORST sibling pair (max combined among
|
||||
* pairs containing it). pass = maxCombined < threshold. A failing skill is a
|
||||
* B3 merge/sanitize candidate against its worst sibling — never a blocker here.
|
||||
*
|
||||
* @param {Array<{pair:string[], combined:number}>} pairs from computeOverlapFromInputs
|
||||
* @param {{threshold?:number}} [opts]
|
||||
* @returns {Record<string,{maxCombined:number, worstSibling:string|null, pass:boolean, threshold:number}>}
|
||||
*/
|
||||
export function perSkillSiblingOverlap(pairs, { threshold = K10_OVERLAP_THRESHOLD } = {}) {
|
||||
const out = {};
|
||||
const ensure = (s) => {
|
||||
if (!out[s]) out[s] = { maxCombined: 0, worstSibling: null, pass: true, threshold };
|
||||
};
|
||||
for (const p of pairs) {
|
||||
const [a, b] = p.pair;
|
||||
ensure(a);
|
||||
ensure(b);
|
||||
if (p.combined > out[a].maxCombined) {
|
||||
out[a].maxCombined = p.combined;
|
||||
out[a].worstSibling = b;
|
||||
}
|
||||
if (p.combined > out[b].maxCombined) {
|
||||
out[b].maxCombined = p.combined;
|
||||
out[b].worstSibling = a;
|
||||
}
|
||||
}
|
||||
for (const s of Object.keys(out)) out[s].pass = out[s].maxCombined < threshold;
|
||||
return out;
|
||||
}
|
||||
126
tests/kb-eval/test-k10-sibling-overlap.test.mjs
Normal file
126
tests/kb-eval/test-k10-sibling-overlap.test.mjs
Normal file
|
|
@ -0,0 +1,126 @@
|
|||
// tests/kb-eval/test-k10-sibling-overlap.test.mjs
|
||||
// Unit tests for K10 — sibling-scope-non-overlap (Sesjon 15 / B2).
|
||||
// Pure core: perSkillSiblingOverlap (lib/sibling-overlap.mjs) + the eval.mjs
|
||||
// integration attachSiblingOverlap. Deterministic — no LLM judge.
|
||||
|
||||
import { test } from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { readFileSync } from 'node:fs';
|
||||
import { dirname, join } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
|
||||
import {
|
||||
perSkillSiblingOverlap,
|
||||
computeOverlapFromInputs,
|
||||
K10_OVERLAP_THRESHOLD,
|
||||
} from '../../scripts/kb-eval/lib/sibling-overlap.mjs';
|
||||
import { attachSiblingOverlap, splitFrontmatter, extractDescription } from '../../scripts/kb-eval/eval.mjs';
|
||||
|
||||
const __dirname = dirname(fileURLToPath(import.meta.url));
|
||||
const PROMPTS_PATH = join(__dirname, '..', '..', 'scripts', 'kb-eval', 'data', 'k1-trigger-prompts.json');
|
||||
const promptSet = JSON.parse(readFileSync(PROMPTS_PATH, 'utf8'));
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// perSkillSiblingOverlap — pure mapping pairs -> per-skill worst sibling
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
test('perSkillSiblingOverlap — each skill takes its worst (max combined) pair', () => {
|
||||
const pairs = [
|
||||
{ pair: ['a', 'b'], combined: 7.5 },
|
||||
{ pair: ['a', 'c'], combined: 3.0 },
|
||||
{ pair: ['b', 'c'], combined: 6.0 },
|
||||
];
|
||||
const r = perSkillSiblingOverlap(pairs, { threshold: 7.0 });
|
||||
assert.equal(r.a.maxCombined, 7.5);
|
||||
assert.equal(r.a.worstSibling, 'b');
|
||||
assert.equal(r.b.maxCombined, 7.5);
|
||||
assert.equal(r.b.worstSibling, 'a');
|
||||
assert.equal(r.c.maxCombined, 6.0);
|
||||
assert.equal(r.c.worstSibling, 'b');
|
||||
});
|
||||
|
||||
test('perSkillSiblingOverlap — pass iff maxCombined strictly below threshold', () => {
|
||||
const pairs = [
|
||||
{ pair: ['a', 'b'], combined: 7.5 },
|
||||
{ pair: ['a', 'c'], combined: 3.0 },
|
||||
{ pair: ['b', 'c'], combined: 6.0 },
|
||||
];
|
||||
const r = perSkillSiblingOverlap(pairs, { threshold: 7.0 });
|
||||
assert.equal(r.a.pass, false); // 7.5 >= 7.0
|
||||
assert.equal(r.b.pass, false); // 7.5 >= 7.0
|
||||
assert.equal(r.c.pass, true); // 6.0 < 7.0
|
||||
});
|
||||
|
||||
test('perSkillSiblingOverlap — combined exactly at threshold FAILS (strict <)', () => {
|
||||
const r = perSkillSiblingOverlap([{ pair: ['x', 'y'], combined: 7.0 }], { threshold: 7.0 });
|
||||
assert.equal(r.x.pass, false);
|
||||
assert.equal(r.y.pass, false);
|
||||
});
|
||||
|
||||
test('perSkillSiblingOverlap — threshold is configurable; records it on each verdict', () => {
|
||||
const pairs = [{ pair: ['a', 'b'], combined: 7.5 }];
|
||||
const lax = perSkillSiblingOverlap(pairs, { threshold: 8.0 });
|
||||
assert.equal(lax.a.pass, true); // 7.5 < 8.0
|
||||
assert.equal(lax.a.threshold, 8.0);
|
||||
});
|
||||
|
||||
test('perSkillSiblingOverlap — default threshold is the rubric K10 constant (7.0)', () => {
|
||||
assert.equal(K10_OVERLAP_THRESHOLD, 7.0);
|
||||
const r = perSkillSiblingOverlap([{ pair: ['a', 'b'], combined: 7.5 }]);
|
||||
assert.equal(r.a.threshold, 7.0);
|
||||
assert.equal(r.a.pass, false);
|
||||
});
|
||||
|
||||
test('perSkillSiblingOverlap — empty pairs yields empty verdict map', () => {
|
||||
assert.deepEqual(perSkillSiblingOverlap([]), {});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Real-data path — eng + infra FAIL at 7.0; the other three PASS
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
test('K10 on the real five — eng+infra FAIL (Azure-deployment boundary), rest PASS', () => {
|
||||
// Build descriptions straight from the curated belongs_to graph + lexical
|
||||
// surfaces via the same core the detector uses, so this asserts on live tokens.
|
||||
const skills = ['ms-ai-advisor', 'ms-ai-engineering', 'ms-ai-governance', 'ms-ai-infrastructure', 'ms-ai-security'];
|
||||
const descriptionsBySkill = {};
|
||||
for (const s of skills) {
|
||||
const md = join(__dirname, '..', '..', 'skills', s, 'SKILL.md');
|
||||
descriptionsBySkill[s] = extractDescription(splitFrontmatter(readFileSync(md, 'utf8')).frontmatter);
|
||||
}
|
||||
const overlap = computeOverlapFromInputs(descriptionsBySkill, promptSet);
|
||||
const k10 = perSkillSiblingOverlap(overlap.pairs, { threshold: 7.0 });
|
||||
assert.equal(k10['ms-ai-engineering'].pass, false);
|
||||
assert.equal(k10['ms-ai-infrastructure'].pass, false);
|
||||
assert.equal(k10['ms-ai-engineering'].worstSibling, 'ms-ai-infrastructure');
|
||||
assert.equal(k10['ms-ai-infrastructure'].worstSibling, 'ms-ai-engineering');
|
||||
assert.equal(k10['ms-ai-advisor'].pass, true);
|
||||
assert.equal(k10['ms-ai-governance'].pass, true);
|
||||
assert.equal(k10['ms-ai-security'].pass, true);
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// attachSiblingOverlap — eval.mjs integration (injects K10 into deterministic)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
test('attachSiblingOverlap — injects K10_siblingScopeOverlap into each skill deterministic block', () => {
|
||||
const skills = [
|
||||
{ name: 'a', deterministic: {}, judgeInputs: { description: 'Build RAG pipelines. Triggers on: "rag", "pipeline".' } },
|
||||
{ name: 'b', deterministic: {}, judgeInputs: { description: 'Operate clusters. Triggers on: "operate", "cluster".' } },
|
||||
];
|
||||
const out = attachSiblingOverlap(skills, promptSet, { threshold: 7.0 });
|
||||
assert.ok(out[0].deterministic.K10_siblingScopeOverlap, 'K10 present on skill a');
|
||||
assert.ok(out[1].deterministic.K10_siblingScopeOverlap, 'K10 present on skill b');
|
||||
assert.equal(typeof out[0].deterministic.K10_siblingScopeOverlap.maxCombined, 'number');
|
||||
assert.equal(typeof out[0].deterministic.K10_siblingScopeOverlap.pass, 'boolean');
|
||||
assert.equal(out[0].deterministic.K10_siblingScopeOverlap.threshold, 7.0);
|
||||
});
|
||||
|
||||
test('attachSiblingOverlap — uses K10 default threshold when none given', () => {
|
||||
const skills = [
|
||||
{ name: 'a', deterministic: {}, judgeInputs: { description: 'x "p" "q" "r".' } },
|
||||
{ name: 'b', deterministic: {}, judgeInputs: { description: 'y "s" "t" "u".' } },
|
||||
];
|
||||
const out = attachSiblingOverlap(skills, promptSet);
|
||||
assert.equal(out[0].deterministic.K10_siblingScopeOverlap.threshold, K10_OVERLAP_THRESHOLD);
|
||||
});
|
||||
Loading…
Add table
Add a link
Reference in a new issue