diff --git a/scripts/kb-eval/eval.mjs b/scripts/kb-eval/eval.mjs index fac76ae..9fe2796 100644 --- a/scripts/kb-eval/eval.mjs +++ b/scripts/kb-eval/eval.mjs @@ -3,7 +3,7 @@ // // READ-ONLY by default. Computes the deterministic criteria (K2/K3/K5/K6 + // reference-count consistency) for every skill and extracts the inputs an LLM -// judge needs for the semantic criteria (K1/K4/K7/K8/K9). The baseline file is +// judge needs for the semantic criteria (K1/K4/K7/K9). The baseline file is // written ONLY with --write (operator gate), per the redesign spec. // // Usage: @@ -17,6 +17,7 @@ import { readdirSync, readFileSync, existsSync, mkdirSync } from 'node:fs'; import { join, dirname, relative } from 'node:path'; import { fileURLToPath } from 'node:url'; import { atomicWriteJson } from '../kb-update/lib/atomic-write.mjs'; +import { parseTypeHeader, parseSourceHeader } from '../kb-update/lib/kb-headers.mjs'; import { computeOverlapFromInputs, perSkillSiblingOverlap, @@ -33,6 +34,7 @@ const PROMPTS_FILE = join(OUT_DIR, 'k1-trigger-prompts.json'); // Thresholds — see spec "Rubrikk K1–K10". const K3_MAX_BODY_LINES = 500; const K5_MIN_NAMED_RATIO = 0.2; +const CT5_SOURCED_RATIO_TARGET = 0.8; // pass at/above this share of sourced reference files (mirrors old K8) /** Recursively list .md files under dir. */ function listMarkdown(dir) { @@ -261,6 +263,34 @@ export function checkN5(body) { return { windowsPaths, pass: windowsPaths.length === 0 }; } +/** + * CT5 — sourcedness (deterministic, whole-corpus). REPLACES the LLM-sampled K8 + * (same signal; per ref-kb-direction-note §45 the two must not run in parallel + * or they double-count and two dashboards diverge). Scope is the Port 1 + * contract: only files declaring `type: reference` are in scope; CT5 is the + * fraction of those that cite a **Source:**. template/methodology/regulatory are + * exempt (excluded from the denominator). On an unmigrated corpus no file + * declares a type, so referenceFiles is 0 and the score layer drops CT5 + * (available:false) — it never tanks the score before migration adopts the + * contract. + * @param {Array<{path: string, content: string}>} refFiles + * @returns {{referenceFiles: number, sourced: number, ratio: number, pass: boolean, unsourced: string[]}} + */ +export function checkCT5(refFiles) { + let referenceFiles = 0; + let sourced = 0; + const unsourced = []; + for (const f of refFiles) { + if (parseTypeHeader(f.content) !== 'reference') continue; // out of scope + referenceFiles++; + if (parseSourceHeader(f.content)) sourced++; + else unsourced.push(f.path); + } + const ratio = referenceFiles ? sourced / referenceFiles : 0; + const pass = referenceFiles === 0 ? true : ratio >= CT5_SOURCED_RATIO_TARGET; + return { referenceFiles, sourced, ratio, pass, unsourced: unsourced.slice(0, 12) }; +} + export function evalSkill(skillName) { const skillDir = join(SKILLS_DIR, skillName); const skillMd = join(skillDir, 'SKILL.md'); @@ -284,6 +314,7 @@ export function evalSkill(skillName) { const N3 = checkN3(refFileContents); const N4 = checkN4(refFileContents); const N5 = checkN5(body); + const CT5 = checkCT5(refFileContents); return { name: skillName, @@ -302,9 +333,11 @@ export function evalSkill(skillName) { N3_refNestingDepth: N3, N4_refToc: N4, N5_forwardSlashPaths: N5, + CT5_sourcedness: CT5, }, - // Pointers for the LLM-judge pass (K1/K4/K7/K8/K9). The judge reads the files - // directly; we only carry the description + sampled ref files here. + // Pointers for the LLM-judge pass (K1/K4/K7/K9). The judge reads the files + // directly; we only carry the description + sampled ref files here. (K8 + // sourcedness is now the deterministic CT5 above — no longer LLM-sampled.) judgeInputs: { description, bodyLines: K3.bodyLines, @@ -354,7 +387,7 @@ export function buildReport() { attachSiblingOverlap(skills, promptSet); } - // Merge operator-gated LLM-judge results (K1/K4/K7/K8/K9) if present. + // Merge operator-gated LLM-judge results (K1/K4/K7/K9) if present. const judgeFile = join(OUT_DIR, 'judge-results.json'); if (existsSync(judgeFile)) { const jr = JSON.parse(readFileSync(judgeFile, 'utf8')); @@ -363,7 +396,7 @@ export function buildReport() { return { rubric: 'K1-K10', - note: 'Deterministic: K2,K3,K5,K6,refCountConsistency,K10(siblingScopeOverlap),N1-N5. LLM-judge (operator-gated): K1,K4,K7,K8,K9.', + note: 'Deterministic: K2,K3,K5,K6,refCountConsistency,K10(siblingScopeOverlap),N1-N5,CT5(sourcedness). LLM-judge (operator-gated): K1,K4,K7,K9.', skills, }; } @@ -405,7 +438,7 @@ function main() { console.log(` K9 hints : ${d.K9_timeSensitiveHints.timeSensitiveTokenHits} tid-sensitive tokens i body`); console.log(''); } - console.log(`(LLM-judge K1/K4/K7/K8/K9 kjøres som operatør-gated subagent-steg; baseline skrives med --write.)\n`); + console.log(`(LLM-judge K1/K4/K7/K9 kjøres som operatør-gated subagent-steg; baseline skrives med --write.)\n`); } // Run only when invoked directly (not when imported by tests). diff --git a/scripts/kb-eval/judge-prompt.md b/scripts/kb-eval/judge-prompt.md index 47a782e..43aee0e 100644 --- a/scripts/kb-eval/judge-prompt.md +++ b/scripts/kb-eval/judge-prompt.md @@ -1,19 +1,19 @@ -# LLM-judge-rubrikk (K1/K4/K7/K8/K9) — skill-kvalitet, ms-ai-architect +# LLM-judge-rubrikk (K1/K4/K7/K9) — skill-kvalitet, ms-ai-architect Pinnet rubrikk for de semantiske eval-kriteriene som ikke kan måles deterministisk. Kjøres som operatør-gated subagent-steg (Opus, én dommer per skill). De deterministiske -kriteriene (K2/K3/K5/K6 + ref-tall) måles av `eval.mjs`; denne dekker resten. +kriteriene (K2/K3/K5/K6 + ref-tall + CT5 sourcedness) måles av `eval.mjs`; denne dekker +resten. (Sourcedness var tidligere LLM-samplet K8; nå deterministisk CT5 — se eval.mjs.) Placeholders: `` = skill-navn, `` = plugin-rot, `` = sti til SKILL.md. --- Du er en skill-kvalitets-dommer (LLM-as-judge) for ms-ai-architect. Vurder skillen -`` mot K1/K4/K7/K8/K9. Plugin-rot: ``. SKILL.md: ``. Referansefiler: +`` mot K1/K4/K7/K9. Plugin-rot: ``. SKILL.md: ``. Referansefiler: `/skills//references/`. -Les SKILL.md i sin helhet. For K8: sample 5 tilfeldige referansefiler og les headeren deres. -Vær streng og adversariell — ikke ros, ikke pynt på tall. +Les SKILL.md i sin helhet. Vær streng og adversariell — ikke ros, ikke pynt på tall. - **K1 trigger-presisjon:** last det operatør-kuraterte settet for `` fra `/scripts/kb-eval/data/k1-trigger-prompts.json` (nøkkel ``: `in_domain` [10] + @@ -28,8 +28,6 @@ Vær streng og adversariell — ikke ros, ikke pynt på tall. i SKILL.md body som dupliserer ref-filer? Gi konkret eksempel. pass = score ≥ 4. - **K7 imperativ/instruksjons-stil:** sample 10 instruksjonssetninger fra body; andel i imperativ/infinitiv. pass = ratio ≥ 0.80. -- **K8 kildehenvisning i ref-filer:** for 5 samplede ref-filer, har header `Last updated` / - `Verified` / kilde-URL? Rapporter andel. pass = ratio ≥ 0.80. - **K9 ingen VOLATIL tid-sensitiv info i SKILL.md body:** finnes **volatile** påstander DIREKTE i body (ikke i ref-filer) — GA/preview-release-status, modellversjoner, SLA-/ytelses-/pris-tall, regions- tilgjengelighet? List funn. pass = ingen volatile funn. **Utenfor scope (ikke funn):** stabile @@ -40,5 +38,5 @@ Vær streng og adversariell — ikke ros, ikke pynt på tall. RETURNER KUN dette JSON-objektet (ingen annen tekst, ingen markdown-fence): ``` -{"skill":"","K1_triggerPrecision":{"provisional":false,"inDomainHitRate":0.0,"outDomainFalsePositiveRate":0.0,"precision":0.0,"pass":false,"misclassified":[],"notes":""},"K4_noDuplication":{"score":0,"pass":false,"evidence":""},"K7_imperativeStyle":{"ratio":0.0,"pass":false,"notes":""},"K8_sourceCitation":{"ratio":0.0,"pass":false,"sampledFiles":[],"notes":""},"K9_noTimeSensitive":{"pass":false,"findings":[]}} +{"skill":"","K1_triggerPrecision":{"provisional":false,"inDomainHitRate":0.0,"outDomainFalsePositiveRate":0.0,"precision":0.0,"pass":false,"misclassified":[],"notes":""},"K4_noDuplication":{"score":0,"pass":false,"evidence":""},"K7_imperativeStyle":{"ratio":0.0,"pass":false,"notes":""},"K9_noTimeSensitive":{"pass":false,"findings":[]}} ``` diff --git a/scripts/kb-eval/lib/skill-score.mjs b/scripts/kb-eval/lib/skill-score.mjs index 8e84aec..bfcadee 100644 --- a/scripts/kb-eval/lib/skill-score.mjs +++ b/scripts/kb-eval/lib/skill-score.mjs @@ -109,12 +109,16 @@ export const CRITERIA = [ }, }, { - key: 'K8', label: 'kildehenvisning refs', weight: 1, floor: false, source: 'judge', - fix: 'Legg dated/Verified-header eller kilde-URL i ref-filene (≥0.80 andel).', + // CT5 — sourcedness (deterministic, whole-corpus). REPLACES the LLM-sampled + // K8 (same signal; must not run in parallel — ref-kb-direction-note §45). + // Dormant (available:false) until Port 1 migration declares reference types, + // so it never tanks the score on an unmigrated corpus. + key: 'CT5', label: 'sourcedness refs', weight: 1, floor: false, source: 'det', + fix: 'Gi ref-filene **Source:**-URL (Port 1-kontrakt) — ≥0.80 andel av type:reference-filer.', read: (e) => { - const j = e.judge?.K8_sourceCitation; - if (!j) return { available: false }; - return { available: true, sub: clamp01(num(j.ratio)), pass: !!j.pass, detail: `ratio ${num(j.ratio)}` }; + const k = e.deterministic?.CT5_sourcedness; + if (!k || !k.referenceFiles) return { available: false }; + return { available: true, sub: clamp01(num(k.ratio)), pass: !!k.pass, detail: `${num(k.sourced)}/${num(k.referenceFiles)} sourced` }; }, }, { diff --git a/tests/kb-eval/test-apply-score-cache.test.mjs b/tests/kb-eval/test-apply-score-cache.test.mjs index c47f4c9..3f06fb3 100644 --- a/tests/kb-eval/test-apply-score-cache.test.mjs +++ b/tests/kb-eval/test-apply-score-cache.test.mjs @@ -40,7 +40,6 @@ function skill(name, { k10Pass = true, k10Combined = 1, worst = 'other' } = {}) K4_noDuplication: { score: 5, pass: true }, K9_noTimeSensitive: { pass: true, findings: [] }, K7_imperativeStyle: { ratio: 1, pass: true }, - K8_sourceCitation: { ratio: 1, pass: true }, }, }; } diff --git a/tests/kb-eval/test-eval.test.mjs b/tests/kb-eval/test-eval.test.mjs index 8fd5921..44471db 100644 --- a/tests/kb-eval/test-eval.test.mjs +++ b/tests/kb-eval/test-eval.test.mjs @@ -16,6 +16,7 @@ import { checkN3, checkN4, checkN5, + checkCT5, } from '../../scripts/kb-eval/eval.mjs'; test('splitFrontmatter — separates frontmatter from body', () => { @@ -229,3 +230,67 @@ test('checkN5 — Windows backslash path fails', () => { test('checkN5 — drive-letter path fails', () => { assert.equal(checkN5('Save to C:\\Users\\x\\file.md').pass, false); }); + +// --- CT5 — sourcedness (deterministic, whole-corpus). REPLACES the LLM-sampled +// K8 (same signal, must not live in parallel — ref-kb-direction-note §45). Scope +// is the Port 1 contract: only files declared `type: reference` are in scope; +// CT5 = fraction of those that cite a **Source:**. template/methodology/ +// regulatory are exempt (excluded from the denominator). When no file declares +// `type: reference` (unmigrated legacy corpus) the denominator is 0 — CT5 is then +// dormant (the score layer drops it; it does not tank the score). + +const refFile = (over) => ({ path: 'references/x.md', content: '', ...over }); +const srcLine = '**Source:** https://learn.microsoft.com/azure/x\n'; + +test('checkCT5 — all declared reference files cite a source → ratio 1, pass', () => { + const refs = [ + refFile({ path: 'references/a.md', content: '# A\n**Type:** reference\n' + srcLine }), + refFile({ path: 'references/b.md', content: '# B\n**Type:** reference\n' + srcLine }), + ]; + const r = checkCT5(refs); + assert.equal(r.referenceFiles, 2); + assert.equal(r.sourced, 2); + assert.equal(r.ratio, 1); + assert.equal(r.pass, true); +}); + +test('checkCT5 — a reference file missing **Source:** lowers the ratio', () => { + const refs = [ + refFile({ path: 'references/a.md', content: '# A\n**Type:** reference\n' + srcLine }), + refFile({ path: 'references/b.md', content: '# B\n**Type:** reference\n' }), // no source + ]; + const r = checkCT5(refs); + assert.equal(r.referenceFiles, 2); + assert.equal(r.sourced, 1); + assert.equal(r.ratio, 0.5); + assert.equal(r.pass, false); +}); + +test('checkCT5 — template/methodology/regulatory are excluded from the denominator', () => { + const refs = [ + refFile({ path: 'references/r.md', content: '# R\n**Type:** reference\n' + srcLine }), + refFile({ path: 'references/t.md', content: '# T\n**Type:** template\n' }), + refFile({ path: 'references/m.md', content: '# M\n**Type:** methodology\n' }), + refFile({ path: 'references/g.md', content: '# G\n**Type:** regulatory\n' }), + ]; + const r = checkCT5(refs); + assert.equal(r.referenceFiles, 1, 'only the one reference file is in scope'); + assert.equal(r.sourced, 1); + assert.equal(r.ratio, 1); +}); + +test('checkCT5 — no file declares type:reference (unmigrated legacy) → referenceFiles 0', () => { + const refs = [ + refFile({ path: 'references/a.md', content: '# A\n**Last updated:** 2026-04\n' }), + refFile({ path: 'references/b.md', content: '# B\nplain prose, no headers\n' }), + ]; + const r = checkCT5(refs); + assert.equal(r.referenceFiles, 0, 'denominator empty until Port 1 migration declares types'); +}); + +test('checkCT5 — 0.80 is the pass threshold (mirrors the old K8 ratio target)', () => { + const mk = (n, withSrc) => Array.from({ length: n }, (_, i) => + refFile({ path: `references/f${i}.md`, content: '**Type:** reference\n' + (i < withSrc ? srcLine : '') })); + assert.equal(checkCT5(mk(5, 4)).pass, true, '4/5 = 0.80 passes'); + assert.equal(checkCT5(mk(5, 3)).pass, false, '3/5 = 0.60 fails'); +}); diff --git a/tests/kb-eval/test-skill-score-gate.test.mjs b/tests/kb-eval/test-skill-score-gate.test.mjs index d8540b4..5a7b857 100644 --- a/tests/kb-eval/test-skill-score-gate.test.mjs +++ b/tests/kb-eval/test-skill-score-gate.test.mjs @@ -47,7 +47,6 @@ const FULL_JUDGE = { K4_noDuplication: { score: 5, pass: true }, K9_noTimeSensitive: { pass: true, findings: [] }, K7_imperativeStyle: { ratio: 1, pass: true }, - K8_sourceCitation: { ratio: 1, pass: true }, }; test('gateSkill — affected skill that meets the target is not blocked', () => { diff --git a/tests/kb-eval/test-skill-score.test.mjs b/tests/kb-eval/test-skill-score.test.mjs index 368fd23..9799e07 100644 --- a/tests/kb-eval/test-skill-score.test.mjs +++ b/tests/kb-eval/test-skill-score.test.mjs @@ -7,7 +7,7 @@ import { test } from 'node:test'; import assert from 'node:assert/strict'; -import { scoreSkill, TARGET, FLOOR_CAP } from '../../scripts/kb-eval/lib/skill-score.mjs'; +import { scoreSkill, TARGET, FLOOR_CAP, CRITERIA } from '../../scripts/kb-eval/lib/skill-score.mjs'; // --- fixture builder: a fully-passing eval object (all sub-scores = 1) ------- function perfectEval(overrides = {}) { @@ -26,13 +26,13 @@ function perfectEval(overrides = {}) { N3_refNestingDepth: { pass: true, nested: [] }, N4_refToc: { largeFiles: 5, withToc: 5, ratio: 1, pass: true }, N5_forwardSlashPaths: { pass: true, windowsPaths: [] }, + CT5_sourcedness: { referenceFiles: 5, sourced: 5, ratio: 1, pass: true }, }, judgeInputs: { description: 'desc', bodyLines: 120 }, judge: { K1_triggerPrecision: { precision: 1, pass: true }, K4_noDuplication: { score: 5, pass: true }, K7_imperativeStyle: { ratio: 1, pass: true }, - K8_sourceCitation: { ratio: 1, pass: true }, K9_noTimeSensitive: { pass: true, findings: [] }, }, }; @@ -59,6 +59,38 @@ test('scoreSkill — TARGET is 90 and FLOOR_CAP is below it', () => { assert.ok(FLOOR_CAP < TARGET); }); +// CT5 (sourcedness) replaced the LLM-sampled K8. On an unmigrated corpus +// (referenceFiles: 0) it is dormant — dropped from the weighted mean, so it +// never tanks the score before Port 1 migration. Once reference files exist it +// contributes deterministically. + +test('scoreSkill — CT5 dormant (no declared reference files) is dropped, score stays 100', () => { + const r = scoreSkill(perfectEval({ + deterministic: { CT5_sourcedness: { referenceFiles: 0, sourced: 0, ratio: 0, pass: true } }, + })); + assert.equal(r.score, 100, 'CT5 available:false => excluded from the mean, no score impact'); + const ct5 = r.criteria.find((c) => c.key === 'CT5'); + assert.equal(ct5.available, false); +}); + +test('scoreSkill — CT5 contributes deterministically once reference files exist', () => { + const r = scoreSkill(perfectEval({ + deterministic: { CT5_sourcedness: { referenceFiles: 4, sourced: 2, ratio: 0.5, pass: false } }, + })); + assert.ok(r.score < 100, `imperfect sourcedness should cost points, got ${r.score}`); + const ct5 = r.criteria.find((c) => c.key === 'CT5'); + assert.equal(ct5.available, true); + assert.equal(ct5.sub, 0.5); + assert.ok(r.improvements.some((i) => i.key === 'CT5'), 'CT5 surfaces as an improvement'); +}); + +test('scoreSkill — CT5 is deterministic (source det), not a judge criterion', () => { + const ct5 = CRITERIA.find((c) => c.key === 'CT5'); + assert.ok(ct5, 'CT5 exists in the rubric'); + assert.equal(ct5.source, 'det'); + assert.equal(CRITERIA.find((c) => c.key === 'K8'), undefined, 'K8 no longer in the rubric'); +}); + test('scoreSkill — failing K10 (deterministic floor) caps the score below target despite a perfect rest', () => { const r = scoreSkill(perfectEval({ deterministic: { K10_siblingScopeOverlap: { maxCombined: 7.4, worstSibling: 'y', pass: false, threshold: 7.0 } },