feat(ms-ai-architect): Sesjon 1 - skill eval-baseline (rubrikk K1-K9) + ai-foundry probe
Read-only eval-verktoy scripts/kb-eval/eval.mjs (determ. K2/K3/K5/K6 + ref-tall) + operator-gated LLM-judge (judge-prompt.md -> data/judge-results.json) flettet til data/eval-baseline.json. 13 enhetstester (tests/kb-eval/), 42 kb-update-tester gronne. Baseline: K5 3/5 fail, K9 4/5 fail, K4 2/5 fail, ref-tall 2/5 fail; 2 konkrete bugs flagget (security 6x5-vekting-motstrid, governance broken ref-path). Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01REiKFhP4w6xGXXqWKpPCJJ
This commit is contained in:
parent
bae0977292
commit
215772df87
5 changed files with 936 additions and 0 deletions
257
scripts/kb-eval/eval.mjs
Normal file
257
scripts/kb-eval/eval.mjs
Normal file
|
|
@ -0,0 +1,257 @@
|
|||
#!/usr/bin/env node
|
||||
// eval.mjs — Skill quality baseline (rubrikk K1–K9) for ms-ai-architect.
|
||||
//
|
||||
// READ-ONLY by default. Computes the deterministic criteria (K2/K3/K5/K6 +
|
||||
// reference-count consistency) for every skill and extracts the inputs an LLM
|
||||
// judge needs for the semantic criteria (K1/K4/K7/K8/K9). The baseline file is
|
||||
// written ONLY with --write (operator gate), per the redesign spec.
|
||||
//
|
||||
// Usage:
|
||||
// node scripts/kb-eval/eval.mjs # human-readable summary
|
||||
// node scripts/kb-eval/eval.mjs --json # machine output (pure JSON to stdout)
|
||||
// node scripts/kb-eval/eval.mjs --write # also persist data/eval-baseline.json
|
||||
//
|
||||
// Zero dependencies. Reuses kb-update/lib/atomic-write.mjs for the gated write.
|
||||
|
||||
import { readdirSync, readFileSync, existsSync, mkdirSync } from 'node:fs';
|
||||
import { join, dirname, relative } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { atomicWriteJson } from '../kb-update/lib/atomic-write.mjs';
|
||||
|
||||
const __dirname = dirname(fileURLToPath(import.meta.url));
|
||||
const PLUGIN_ROOT = join(__dirname, '..', '..');
|
||||
const SKILLS_DIR = join(PLUGIN_ROOT, 'skills');
|
||||
const OUT_DIR = join(__dirname, 'data');
|
||||
const OUT_FILE = join(OUT_DIR, 'eval-baseline.json');
|
||||
|
||||
// Thresholds — see spec "Rubrikk K1–K9".
|
||||
const K3_MAX_BODY_LINES = 500;
|
||||
const K5_MIN_NAMED_RATIO = 0.2;
|
||||
|
||||
/** Recursively list .md files under dir. */
|
||||
function listMarkdown(dir) {
|
||||
const out = [];
|
||||
if (!existsSync(dir)) return out;
|
||||
for (const e of readdirSync(dir, { withFileTypes: true })) {
|
||||
const p = join(dir, e.name);
|
||||
if (e.isDirectory()) out.push(...listMarkdown(p));
|
||||
else if (e.isFile() && e.name.endsWith('.md')) out.push(p);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Count .md files per immediate subfolder of references/. */
|
||||
function refCountsPerFolder(refDir) {
|
||||
const counts = {};
|
||||
if (!existsSync(refDir)) return counts;
|
||||
for (const e of readdirSync(refDir, { withFileTypes: true })) {
|
||||
if (e.isDirectory()) counts[e.name] = listMarkdown(join(refDir, e.name)).length;
|
||||
}
|
||||
return counts;
|
||||
}
|
||||
|
||||
/** Split YAML frontmatter from body. */
|
||||
export function splitFrontmatter(content) {
|
||||
if (!content.startsWith('---')) return { frontmatter: '', body: content };
|
||||
const close = content.indexOf('\n---', 3);
|
||||
if (close === -1) return { frontmatter: '', body: content };
|
||||
const bodyStart = content.indexOf('\n', close + 1);
|
||||
return {
|
||||
frontmatter: content.slice(0, bodyStart === -1 ? content.length : bodyStart + 1),
|
||||
body: bodyStart === -1 ? '' : content.slice(bodyStart + 1),
|
||||
};
|
||||
}
|
||||
|
||||
/** Extract the (possibly folded) description value from frontmatter. */
|
||||
export function extractDescription(frontmatter) {
|
||||
const lines = frontmatter.split('\n');
|
||||
const out = [];
|
||||
let collecting = false;
|
||||
for (const line of lines) {
|
||||
if (/^description:/.test(line)) {
|
||||
collecting = true;
|
||||
out.push(line.replace(/^description:\s*>?-?\s*/, ''));
|
||||
continue;
|
||||
}
|
||||
if (collecting) {
|
||||
if (/^[A-Za-z0-9_-]+:\s/.test(line) || line.trim() === '---' || line.trim() === '') {
|
||||
if (line.trim() === '') continue; // folded blocks keep going across blanks
|
||||
break;
|
||||
}
|
||||
out.push(line.trim());
|
||||
}
|
||||
}
|
||||
return out.join(' ').replace(/\s+/g, ' ').trim();
|
||||
}
|
||||
|
||||
/** K2 — description format: third-person/use-when OR >=3 quoted user phrases. */
|
||||
export function checkK2(description) {
|
||||
const quoted = (description.match(/"[^"]+"/g) || []).length;
|
||||
const useWhenForm = /should be used when|use this skill|triggers on/i.test(description);
|
||||
const imperativeStart = /^(use|bruk|run|kjør)\b/i.test(description);
|
||||
const pass = useWhenForm || quoted >= 3;
|
||||
return { quotedPhrases: quoted, useWhenForm, imperativeStart, pass };
|
||||
}
|
||||
|
||||
/** K3 — SKILL.md body length. */
|
||||
export function checkK3(body) {
|
||||
const lines = body.replace(/\n+$/, '').split('\n').length;
|
||||
return { bodyLines: lines, pass: lines <= K3_MAX_BODY_LINES };
|
||||
}
|
||||
|
||||
/** K5/K6 — progressive disclosure: named .md links vs folder-only references. */
|
||||
export function checkK5K6(body, totalRefFiles) {
|
||||
const named = [...new Set((body.match(/references\/[A-Za-z0-9_\-/]+\.md/g) || []))];
|
||||
// folder references like `references/foo/` not immediately followed by a filename
|
||||
const folderRefs = [...new Set((body.match(/references\/[A-Za-z0-9_-]+\/(?![A-Za-z0-9_-]+\.md)/g) || []))];
|
||||
const namedRatio = totalRefFiles > 0 ? named.length / totalRefFiles : 0;
|
||||
return {
|
||||
K5: {
|
||||
namedFileLinks: named.length,
|
||||
folderRefs: folderRefs.length,
|
||||
totalRefFiles,
|
||||
namedRatio: Number(namedRatio.toFixed(4)),
|
||||
pass: namedRatio >= K5_MIN_NAMED_RATIO,
|
||||
sampleNamed: named.slice(0, 5),
|
||||
},
|
||||
K6: {
|
||||
// routing table = at least one named start file the model can Read directly
|
||||
namedStartFiles: named.length,
|
||||
pass: named.length >= 1,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/** Reference-count consistency: cited per-folder counts vs actual on disk. */
|
||||
export function checkRefConsistency(body, actualPerFolder) {
|
||||
const mismatches = [];
|
||||
const seen = {};
|
||||
// prose form: references/<folder>/ (N files) and `references/<folder>/` (N files)
|
||||
const proseRe = /references\/([a-z0-9-]+)\/`?\s*\((\d+)\s*files?\)/gi;
|
||||
// table form: `references/<folder>/` | N
|
||||
const tableRe = /`references\/([a-z0-9-]+)\/`\s*\|\s*(\d+)/gi;
|
||||
for (const re of [proseRe, tableRe]) {
|
||||
let m;
|
||||
while ((m = re.exec(body)) !== null) {
|
||||
const folder = m[1];
|
||||
const cited = Number(m[2]);
|
||||
seen[folder] = seen[folder] || new Set();
|
||||
seen[folder].add(cited);
|
||||
const actual = actualPerFolder[folder];
|
||||
if (actual !== undefined && cited !== actual) {
|
||||
mismatches.push({ folder, cited, actual });
|
||||
}
|
||||
}
|
||||
}
|
||||
// also flag folders cited with two different numbers
|
||||
for (const [folder, set] of Object.entries(seen)) {
|
||||
if (set.size > 1) mismatches.push({ folder, citedMultiple: [...set], actual: actualPerFolder[folder] ?? null });
|
||||
}
|
||||
return { consistent: mismatches.length === 0, mismatches };
|
||||
}
|
||||
|
||||
/** Light deterministic pre-scan for K9 (time-sensitive tokens in body). */
|
||||
function scanK9Hints(body) {
|
||||
const tokens = body.match(/\b(GA|preview|deprecat\w*|retir\w*|20\d\d|v\d+(?:\.\d+)?)\b/gi) || [];
|
||||
return { timeSensitiveTokenHits: tokens.length, sample: [...new Set(tokens)].slice(0, 12) };
|
||||
}
|
||||
|
||||
function evalSkill(skillName) {
|
||||
const skillDir = join(SKILLS_DIR, skillName);
|
||||
const skillMd = join(skillDir, 'SKILL.md');
|
||||
if (!existsSync(skillMd)) return null;
|
||||
const content = readFileSync(skillMd, 'utf8');
|
||||
const { frontmatter, body } = splitFrontmatter(content);
|
||||
const description = extractDescription(frontmatter);
|
||||
const refDir = join(skillDir, 'references');
|
||||
const refFiles = listMarkdown(refDir);
|
||||
const perFolder = refCountsPerFolder(refDir);
|
||||
|
||||
const K2 = checkK2(description);
|
||||
const K3 = checkK3(body);
|
||||
const { K5, K6 } = checkK5K6(body, refFiles.length);
|
||||
const refConsistency = checkRefConsistency(body, perFolder);
|
||||
const K9hints = scanK9Hints(body);
|
||||
|
||||
return {
|
||||
name: skillName,
|
||||
skillMd: relative(PLUGIN_ROOT, skillMd),
|
||||
refFilesActual: refFiles.length,
|
||||
refCountsPerFolder: perFolder,
|
||||
deterministic: {
|
||||
K2_descriptionFormat: K2,
|
||||
K3_bodyLength: K3,
|
||||
K5_progressiveDisclosure: K5,
|
||||
K6_routingTable: K6,
|
||||
refCountConsistency: refConsistency,
|
||||
K9_timeSensitiveHints: K9hints,
|
||||
},
|
||||
// Pointers for the LLM-judge pass (K1/K4/K7/K8/K9). The judge reads the files
|
||||
// directly; we only carry the description + sampled ref files here.
|
||||
judgeInputs: {
|
||||
description,
|
||||
bodyLines: K3.bodyLines,
|
||||
refFileSample: refFiles.slice(0, 8).map((f) => relative(PLUGIN_ROOT, f)),
|
||||
},
|
||||
// Filled in by the operator-gated LLM-judge merge step. Null = not yet judged.
|
||||
judge: null,
|
||||
};
|
||||
}
|
||||
|
||||
function main() {
|
||||
const args = process.argv.slice(2);
|
||||
const jsonOut = args.includes('--json');
|
||||
const doWrite = args.includes('--write');
|
||||
|
||||
const skillNames = readdirSync(SKILLS_DIR, { withFileTypes: true })
|
||||
.filter((e) => e.isDirectory() && existsSync(join(SKILLS_DIR, e.name, 'SKILL.md')))
|
||||
.map((e) => e.name)
|
||||
.sort();
|
||||
|
||||
const skills = skillNames.map(evalSkill).filter(Boolean);
|
||||
|
||||
// Merge operator-gated LLM-judge results (K1/K4/K7/K8/K9) if present.
|
||||
const judgeFile = join(OUT_DIR, 'judge-results.json');
|
||||
if (existsSync(judgeFile)) {
|
||||
const jr = JSON.parse(readFileSync(judgeFile, 'utf8'));
|
||||
for (const s of skills) if (jr[s.name]) s.judge = jr[s.name];
|
||||
}
|
||||
|
||||
const report = {
|
||||
rubric: 'K1-K9',
|
||||
note: 'Deterministic: K2,K3,K5,K6,refCountConsistency. LLM-judge (operator-gated): K1,K4,K7,K8,K9.',
|
||||
skills,
|
||||
};
|
||||
|
||||
if (doWrite) {
|
||||
mkdirSync(OUT_DIR, { recursive: true });
|
||||
atomicWriteJson(OUT_FILE, report);
|
||||
}
|
||||
|
||||
if (jsonOut) {
|
||||
process.stdout.write(JSON.stringify(report) + '\n');
|
||||
return;
|
||||
}
|
||||
|
||||
// Human-readable summary
|
||||
console.log(`\nKB skill eval — ${skills.length} skills (deterministic pass)\n`);
|
||||
for (const s of skills) {
|
||||
const d = s.deterministic;
|
||||
const flag = (b) => (b ? 'PASS' : 'FAIL');
|
||||
console.log(`■ ${s.name} (${s.refFilesActual} ref-filer)`);
|
||||
console.log(` K2 description : ${flag(d.K2_descriptionFormat.pass)} (quoted=${d.K2_descriptionFormat.quotedPhrases}, useWhen=${d.K2_descriptionFormat.useWhenForm})`);
|
||||
console.log(` K3 body ≤500 : ${flag(d.K3_bodyLength.pass)} (${d.K3_bodyLength.bodyLines} linjer)`);
|
||||
console.log(` K5 prog.disc. : ${flag(d.K5_progressiveDisclosure.pass)} (navngitte=${d.K5_progressiveDisclosure.namedFileLinks}, mapper=${d.K5_progressiveDisclosure.folderRefs}, ratio=${d.K5_progressiveDisclosure.namedRatio})`);
|
||||
console.log(` K6 routing-tab : ${flag(d.K6_routingTable.pass)} (navngitte startfiler=${d.K6_routingTable.namedStartFiles})`);
|
||||
const rc = d.refCountConsistency;
|
||||
console.log(` ref-tall : ${flag(rc.consistent)}${rc.consistent ? '' : ' — ' + JSON.stringify(rc.mismatches)}`);
|
||||
console.log(` K9 hints : ${d.K9_timeSensitiveHints.timeSensitiveTokenHits} tid-sensitive tokens i body`);
|
||||
console.log('');
|
||||
}
|
||||
console.log(`(LLM-judge K1/K4/K7/K8/K9 kjøres som operatør-gated subagent-steg; baseline skrives med --write.)\n`);
|
||||
}
|
||||
|
||||
// Run only when invoked directly (not when imported by tests).
|
||||
if (process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1]) {
|
||||
main();
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue