feat(ms-ai-architect): Sesjon 15 — B2 K10 søsken-scope-ikke-overlapp

- Refaktor: overlap-kjerne flyttet til scripts/kb-eval/lib/sibling-overlap.mjs
  (bryter sirkulær import eval.mjs<->detect); detect re-eksporterer → B1-tester urørt
- K10 = søsken-scope-ikke-overlapp, deterministisk cross-skill: perSkillSiblingOverlap
  + attachSiblingOverlap i eval.mjs. combined = boundaryTension + df-vektet leksikalsk;
  per-skill verdikt = verste søskenpar; terskel 7.0 (naturlig gap 7.42→6.67)
- Empirisk (alle 5): eng+infra FAIL (7.42 mot hverandre), advisor/gov/sec PASS
  → eng↔infra-signal mater B3 merge/saner (ikke blokkering)
- Gated baseline-regen via --write (descriptions urørt → judge K1/K4/K7/K8/K9 merget
  uendret, ikke fabrikkert); rubric K1-K10
- TDD: +9 tester (tests/kb-eval/test-k10-sibling-overlap.test.mjs), kb-eval 31→40
- 0 skriving til skills/. Suiter: validate 239 · kb-update 122 · kb-integrity 192/192
This commit is contained in:
Kjell Tore Guttormsen 2026-06-20 11:59:59 +02:00
commit ba597eb988
5 changed files with 400 additions and 123 deletions

View file

@ -17,14 +17,20 @@ import { readdirSync, readFileSync, existsSync, mkdirSync } from 'node:fs';
import { join, dirname, relative } from 'node:path';
import { fileURLToPath } from 'node:url';
import { atomicWriteJson } from '../kb-update/lib/atomic-write.mjs';
import {
computeOverlapFromInputs,
perSkillSiblingOverlap,
K10_OVERLAP_THRESHOLD,
} from './lib/sibling-overlap.mjs';
const __dirname = dirname(fileURLToPath(import.meta.url));
const PLUGIN_ROOT = join(__dirname, '..', '..');
const SKILLS_DIR = join(PLUGIN_ROOT, 'skills');
const OUT_DIR = join(__dirname, 'data');
const OUT_FILE = join(OUT_DIR, 'eval-baseline.json');
const PROMPTS_FILE = join(OUT_DIR, 'k1-trigger-prompts.json');
// Thresholds — see spec "Rubrikk K1K9".
// Thresholds — see spec "Rubrikk K1K10".
const K3_MAX_BODY_LINES = 500;
const K5_MIN_NAMED_RATIO = 0.2;
@ -198,6 +204,25 @@ export function evalSkill(skillName) {
};
}
/**
* K10 sibling-scope-non-overlap. Deterministic, computed ACROSS all skills
* (unlike K2/K3/K5/K6 which are per-skill), so it is attached after the
* per-skill pass. Reuses the B1 overlap core: combined = boundaryTension +
* df-weighted lexical overlap; each skill's K10 verdict is its worst sibling
* pair (perSkillSiblingOverlap). Mutates + returns the skills array.
*/
export function attachSiblingOverlap(skills, promptSet, { threshold = K10_OVERLAP_THRESHOLD } = {}) {
const descriptionsBySkill = {};
for (const s of skills) descriptionsBySkill[s.name] = s.judgeInputs.description;
const overlap = computeOverlapFromInputs(descriptionsBySkill, promptSet);
const k10 = perSkillSiblingOverlap(overlap.pairs, { threshold });
for (const s of skills) {
s.deterministic.K10_siblingScopeOverlap =
k10[s.name] || { maxCombined: 0, worstSibling: null, pass: true, threshold };
}
return skills;
}
function main() {
const args = process.argv.slice(2);
const jsonOut = args.includes('--json');
@ -210,6 +235,13 @@ function main() {
const skills = skillNames.map(evalSkill).filter(Boolean);
// K10 — sibling-scope-non-overlap (deterministic, cross-skill). Attached after
// the per-skill pass since it needs every description + the curated prompt set.
if (existsSync(PROMPTS_FILE)) {
const promptSet = JSON.parse(readFileSync(PROMPTS_FILE, 'utf8'));
attachSiblingOverlap(skills, promptSet);
}
// Merge operator-gated LLM-judge results (K1/K4/K7/K8/K9) if present.
const judgeFile = join(OUT_DIR, 'judge-results.json');
if (existsSync(judgeFile)) {
@ -218,8 +250,8 @@ function main() {
}
const report = {
rubric: 'K1-K9',
note: 'Deterministic: K2,K3,K5,K6,refCountConsistency. LLM-judge (operator-gated): K1,K4,K7,K8,K9.',
rubric: 'K1-K10',
note: 'Deterministic: K2,K3,K5,K6,refCountConsistency,K10(siblingScopeOverlap). LLM-judge (operator-gated): K1,K4,K7,K8,K9.',
skills,
};
@ -245,6 +277,10 @@ function main() {
console.log(` K6 routing-tab : ${flag(d.K6_routingTable.pass)} (navngitte startfiler=${d.K6_routingTable.namedStartFiles})`);
const rc = d.refCountConsistency;
console.log(` ref-tall : ${flag(rc.consistent)}${rc.consistent ? '' : ' — ' + JSON.stringify(rc.mismatches)}`);
const k10 = d.K10_siblingScopeOverlap;
if (k10) {
console.log(` K10 søsken-ovl : ${flag(k10.pass)} (maks ${k10.maxCombined} mot ${k10.worstSibling}, terskel ${k10.threshold})`);
}
console.log(` K9 hints : ${d.K9_timeSensitiveHints.timeSensitiveTokenHits} tid-sensitive tokens i body`);
console.log('');
}