feat(ms-ai-architect): Sesjon 15 — B2 K10 søsken-scope-ikke-overlapp
- Refaktor: overlap-kjerne flyttet til scripts/kb-eval/lib/sibling-overlap.mjs (bryter sirkulær import eval.mjs<->detect); detect re-eksporterer → B1-tester urørt - K10 = søsken-scope-ikke-overlapp, deterministisk cross-skill: perSkillSiblingOverlap + attachSiblingOverlap i eval.mjs. combined = boundaryTension + df-vektet leksikalsk; per-skill verdikt = verste søskenpar; terskel 7.0 (naturlig gap 7.42→6.67) - Empirisk (alle 5): eng+infra FAIL (7.42 mot hverandre), advisor/gov/sec PASS → eng↔infra-signal mater B3 merge/saner (ikke blokkering) - Gated baseline-regen via --write (descriptions urørt → judge K1/K4/K7/K8/K9 merget uendret, ikke fabrikkert); rubric K1-K10 - TDD: +9 tester (tests/kb-eval/test-k10-sibling-overlap.test.mjs), kb-eval 31→40 - 0 skriving til skills/. Suiter: validate 239 · kb-update 122 · kb-integrity 192/192
This commit is contained in:
parent
5ad4ed025c
commit
ba597eb988
5 changed files with 400 additions and 123 deletions
|
|
@ -17,14 +17,20 @@ import { readdirSync, readFileSync, existsSync, mkdirSync } from 'node:fs';
|
|||
import { join, dirname, relative } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { atomicWriteJson } from '../kb-update/lib/atomic-write.mjs';
|
||||
import {
|
||||
computeOverlapFromInputs,
|
||||
perSkillSiblingOverlap,
|
||||
K10_OVERLAP_THRESHOLD,
|
||||
} from './lib/sibling-overlap.mjs';
|
||||
|
||||
const __dirname = dirname(fileURLToPath(import.meta.url));
|
||||
const PLUGIN_ROOT = join(__dirname, '..', '..');
|
||||
const SKILLS_DIR = join(PLUGIN_ROOT, 'skills');
|
||||
const OUT_DIR = join(__dirname, 'data');
|
||||
const OUT_FILE = join(OUT_DIR, 'eval-baseline.json');
|
||||
const PROMPTS_FILE = join(OUT_DIR, 'k1-trigger-prompts.json');
|
||||
|
||||
// Thresholds — see spec "Rubrikk K1–K9".
|
||||
// Thresholds — see spec "Rubrikk K1–K10".
|
||||
const K3_MAX_BODY_LINES = 500;
|
||||
const K5_MIN_NAMED_RATIO = 0.2;
|
||||
|
||||
|
|
@ -198,6 +204,25 @@ export function evalSkill(skillName) {
|
|||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* K10 — sibling-scope-non-overlap. Deterministic, computed ACROSS all skills
|
||||
* (unlike K2/K3/K5/K6 which are per-skill), so it is attached after the
|
||||
* per-skill pass. Reuses the B1 overlap core: combined = boundaryTension +
|
||||
* df-weighted lexical overlap; each skill's K10 verdict is its worst sibling
|
||||
* pair (perSkillSiblingOverlap). Mutates + returns the skills array.
|
||||
*/
|
||||
export function attachSiblingOverlap(skills, promptSet, { threshold = K10_OVERLAP_THRESHOLD } = {}) {
|
||||
const descriptionsBySkill = {};
|
||||
for (const s of skills) descriptionsBySkill[s.name] = s.judgeInputs.description;
|
||||
const overlap = computeOverlapFromInputs(descriptionsBySkill, promptSet);
|
||||
const k10 = perSkillSiblingOverlap(overlap.pairs, { threshold });
|
||||
for (const s of skills) {
|
||||
s.deterministic.K10_siblingScopeOverlap =
|
||||
k10[s.name] || { maxCombined: 0, worstSibling: null, pass: true, threshold };
|
||||
}
|
||||
return skills;
|
||||
}
|
||||
|
||||
function main() {
|
||||
const args = process.argv.slice(2);
|
||||
const jsonOut = args.includes('--json');
|
||||
|
|
@ -210,6 +235,13 @@ function main() {
|
|||
|
||||
const skills = skillNames.map(evalSkill).filter(Boolean);
|
||||
|
||||
// K10 — sibling-scope-non-overlap (deterministic, cross-skill). Attached after
|
||||
// the per-skill pass since it needs every description + the curated prompt set.
|
||||
if (existsSync(PROMPTS_FILE)) {
|
||||
const promptSet = JSON.parse(readFileSync(PROMPTS_FILE, 'utf8'));
|
||||
attachSiblingOverlap(skills, promptSet);
|
||||
}
|
||||
|
||||
// Merge operator-gated LLM-judge results (K1/K4/K7/K8/K9) if present.
|
||||
const judgeFile = join(OUT_DIR, 'judge-results.json');
|
||||
if (existsSync(judgeFile)) {
|
||||
|
|
@ -218,8 +250,8 @@ function main() {
|
|||
}
|
||||
|
||||
const report = {
|
||||
rubric: 'K1-K9',
|
||||
note: 'Deterministic: K2,K3,K5,K6,refCountConsistency. LLM-judge (operator-gated): K1,K4,K7,K8,K9.',
|
||||
rubric: 'K1-K10',
|
||||
note: 'Deterministic: K2,K3,K5,K6,refCountConsistency,K10(siblingScopeOverlap). LLM-judge (operator-gated): K1,K4,K7,K8,K9.',
|
||||
skills,
|
||||
};
|
||||
|
||||
|
|
@ -245,6 +277,10 @@ function main() {
|
|||
console.log(` K6 routing-tab : ${flag(d.K6_routingTable.pass)} (navngitte startfiler=${d.K6_routingTable.namedStartFiles})`);
|
||||
const rc = d.refCountConsistency;
|
||||
console.log(` ref-tall : ${flag(rc.consistent)}${rc.consistent ? '' : ' — ' + JSON.stringify(rc.mismatches)}`);
|
||||
const k10 = d.K10_siblingScopeOverlap;
|
||||
if (k10) {
|
||||
console.log(` K10 søsken-ovl : ${flag(k10.pass)} (maks ${k10.maxCombined} mot ${k10.worstSibling}, terskel ${k10.threshold})`);
|
||||
}
|
||||
console.log(` K9 hints : ${d.K9_timeSensitiveHints.timeSensitiveTokenHits} tid-sensitive tokens i body`);
|
||||
console.log('');
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue