feat(ms-ai-architect): Spor 3 Port 3 (CT5) — deterministisk sourcedness erstatter LLM-samplet K8 [skip-docs]
§4 Port 3 verifiseringskrav: «CT5 (sourcedness) ERSTATTER K8s rolle». K8 var LLM-judge (sample 5 ref-filer, ikke-deterministisk); CT5 er samme signal gjort deterministisk + hel-korpus fra Port 1-kontrakten. Per ref-kb-direction-note §45 MÅ de ikke leve parallelt (dobbelttelling + divergerende dashbord) — så K8 er fjernet, ikke duplisert. eval.mjs: checkCT5(refFiles) — blant filer som deklarerer type:reference, andel med **Source:** (template/methodology/regulatory ekskludert fra nevneren). Wiret som deterministic.CT5_sourcedness. parseTypeHeader/parseSourceHeader importert fra kb-update/lib/kb-headers (tillatt retning). skill-score.mjs: K8-kriteriet (source:judge) → CT5 (source:det, vekt 1). available:false når referenceFiles=0 → droppet fra vektet snitt → tanker IKKE scoren på umigrert korpus (alarm-tretthet unngått, direction-note §45). judge-prompt.md: K8 fjernet fra rubrikk/instruksjon/JSON (judgen gjør nå K1/K4/K7/K9). Ekte kjøring: alle 5 skills CT5 dormant (referenceFiles:0, umigrert), scorer uendret (91-96, 0 under mål). CT5 aktiveres deterministisk når Spor 1 migrerer. TDD (Iron Law): 5 checkCT5 + 3 CT5-integrasjon; død K8-fixture-data ryddet. Suite 620/620. Plugin-validering 239/0/0.
This commit is contained in:
parent
08e7e72c62
commit
75ee9ec062
7 changed files with 153 additions and 23 deletions
|
|
@ -3,7 +3,7 @@
|
|||
//
|
||||
// READ-ONLY by default. Computes the deterministic criteria (K2/K3/K5/K6 +
|
||||
// reference-count consistency) for every skill and extracts the inputs an LLM
|
||||
// judge needs for the semantic criteria (K1/K4/K7/K8/K9). The baseline file is
|
||||
// judge needs for the semantic criteria (K1/K4/K7/K9). The baseline file is
|
||||
// written ONLY with --write (operator gate), per the redesign spec.
|
||||
//
|
||||
// Usage:
|
||||
|
|
@ -17,6 +17,7 @@ import { readdirSync, readFileSync, existsSync, mkdirSync } from 'node:fs';
|
|||
import { join, dirname, relative } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { atomicWriteJson } from '../kb-update/lib/atomic-write.mjs';
|
||||
import { parseTypeHeader, parseSourceHeader } from '../kb-update/lib/kb-headers.mjs';
|
||||
import {
|
||||
computeOverlapFromInputs,
|
||||
perSkillSiblingOverlap,
|
||||
|
|
@ -33,6 +34,7 @@ const PROMPTS_FILE = join(OUT_DIR, 'k1-trigger-prompts.json');
|
|||
// Thresholds — see spec "Rubrikk K1–K10".
|
||||
const K3_MAX_BODY_LINES = 500;
|
||||
const K5_MIN_NAMED_RATIO = 0.2;
|
||||
const CT5_SOURCED_RATIO_TARGET = 0.8; // pass at/above this share of sourced reference files (mirrors old K8)
|
||||
|
||||
/** Recursively list .md files under dir. */
|
||||
function listMarkdown(dir) {
|
||||
|
|
@ -261,6 +263,34 @@ export function checkN5(body) {
|
|||
return { windowsPaths, pass: windowsPaths.length === 0 };
|
||||
}
|
||||
|
||||
/**
|
||||
* CT5 — sourcedness (deterministic, whole-corpus). REPLACES the LLM-sampled K8
|
||||
* (same signal; per ref-kb-direction-note §45 the two must not run in parallel
|
||||
* or they double-count and two dashboards diverge). Scope is the Port 1
|
||||
* contract: only files declaring `type: reference` are in scope; CT5 is the
|
||||
* fraction of those that cite a **Source:**. template/methodology/regulatory are
|
||||
* exempt (excluded from the denominator). On an unmigrated corpus no file
|
||||
* declares a type, so referenceFiles is 0 and the score layer drops CT5
|
||||
* (available:false) — it never tanks the score before migration adopts the
|
||||
* contract.
|
||||
* @param {Array<{path: string, content: string}>} refFiles
|
||||
* @returns {{referenceFiles: number, sourced: number, ratio: number, pass: boolean, unsourced: string[]}}
|
||||
*/
|
||||
export function checkCT5(refFiles) {
|
||||
let referenceFiles = 0;
|
||||
let sourced = 0;
|
||||
const unsourced = [];
|
||||
for (const f of refFiles) {
|
||||
if (parseTypeHeader(f.content) !== 'reference') continue; // out of scope
|
||||
referenceFiles++;
|
||||
if (parseSourceHeader(f.content)) sourced++;
|
||||
else unsourced.push(f.path);
|
||||
}
|
||||
const ratio = referenceFiles ? sourced / referenceFiles : 0;
|
||||
const pass = referenceFiles === 0 ? true : ratio >= CT5_SOURCED_RATIO_TARGET;
|
||||
return { referenceFiles, sourced, ratio, pass, unsourced: unsourced.slice(0, 12) };
|
||||
}
|
||||
|
||||
export function evalSkill(skillName) {
|
||||
const skillDir = join(SKILLS_DIR, skillName);
|
||||
const skillMd = join(skillDir, 'SKILL.md');
|
||||
|
|
@ -284,6 +314,7 @@ export function evalSkill(skillName) {
|
|||
const N3 = checkN3(refFileContents);
|
||||
const N4 = checkN4(refFileContents);
|
||||
const N5 = checkN5(body);
|
||||
const CT5 = checkCT5(refFileContents);
|
||||
|
||||
return {
|
||||
name: skillName,
|
||||
|
|
@ -302,9 +333,11 @@ export function evalSkill(skillName) {
|
|||
N3_refNestingDepth: N3,
|
||||
N4_refToc: N4,
|
||||
N5_forwardSlashPaths: N5,
|
||||
CT5_sourcedness: CT5,
|
||||
},
|
||||
// Pointers for the LLM-judge pass (K1/K4/K7/K8/K9). The judge reads the files
|
||||
// directly; we only carry the description + sampled ref files here.
|
||||
// Pointers for the LLM-judge pass (K1/K4/K7/K9). The judge reads the files
|
||||
// directly; we only carry the description + sampled ref files here. (K8
|
||||
// sourcedness is now the deterministic CT5 above — no longer LLM-sampled.)
|
||||
judgeInputs: {
|
||||
description,
|
||||
bodyLines: K3.bodyLines,
|
||||
|
|
@ -354,7 +387,7 @@ export function buildReport() {
|
|||
attachSiblingOverlap(skills, promptSet);
|
||||
}
|
||||
|
||||
// Merge operator-gated LLM-judge results (K1/K4/K7/K8/K9) if present.
|
||||
// Merge operator-gated LLM-judge results (K1/K4/K7/K9) if present.
|
||||
const judgeFile = join(OUT_DIR, 'judge-results.json');
|
||||
if (existsSync(judgeFile)) {
|
||||
const jr = JSON.parse(readFileSync(judgeFile, 'utf8'));
|
||||
|
|
@ -363,7 +396,7 @@ export function buildReport() {
|
|||
|
||||
return {
|
||||
rubric: 'K1-K10',
|
||||
note: 'Deterministic: K2,K3,K5,K6,refCountConsistency,K10(siblingScopeOverlap),N1-N5. LLM-judge (operator-gated): K1,K4,K7,K8,K9.',
|
||||
note: 'Deterministic: K2,K3,K5,K6,refCountConsistency,K10(siblingScopeOverlap),N1-N5,CT5(sourcedness). LLM-judge (operator-gated): K1,K4,K7,K9.',
|
||||
skills,
|
||||
};
|
||||
}
|
||||
|
|
@ -405,7 +438,7 @@ function main() {
|
|||
console.log(` K9 hints : ${d.K9_timeSensitiveHints.timeSensitiveTokenHits} tid-sensitive tokens i body`);
|
||||
console.log('');
|
||||
}
|
||||
console.log(`(LLM-judge K1/K4/K7/K8/K9 kjøres som operatør-gated subagent-steg; baseline skrives med --write.)\n`);
|
||||
console.log(`(LLM-judge K1/K4/K7/K9 kjøres som operatør-gated subagent-steg; baseline skrives med --write.)\n`);
|
||||
}
|
||||
|
||||
// Run only when invoked directly (not when imported by tests).
|
||||
|
|
|
|||
|
|
@ -1,19 +1,19 @@
|
|||
# LLM-judge-rubrikk (K1/K4/K7/K8/K9) — skill-kvalitet, ms-ai-architect
|
||||
# LLM-judge-rubrikk (K1/K4/K7/K9) — skill-kvalitet, ms-ai-architect
|
||||
|
||||
Pinnet rubrikk for de semantiske eval-kriteriene som ikke kan måles deterministisk.
|
||||
Kjøres som operatør-gated subagent-steg (Opus, én dommer per skill). De deterministiske
|
||||
kriteriene (K2/K3/K5/K6 + ref-tall) måles av `eval.mjs`; denne dekker resten.
|
||||
kriteriene (K2/K3/K5/K6 + ref-tall + CT5 sourcedness) måles av `eval.mjs`; denne dekker
|
||||
resten. (Sourcedness var tidligere LLM-samplet K8; nå deterministisk CT5 — se eval.mjs.)
|
||||
|
||||
Placeholders: `<NAME>` = skill-navn, `<ROOT>` = plugin-rot, `<PATH>` = sti til SKILL.md.
|
||||
|
||||
---
|
||||
|
||||
Du er en skill-kvalitets-dommer (LLM-as-judge) for ms-ai-architect. Vurder skillen
|
||||
`<NAME>` mot K1/K4/K7/K8/K9. Plugin-rot: `<ROOT>`. SKILL.md: `<PATH>`. Referansefiler:
|
||||
`<NAME>` mot K1/K4/K7/K9. Plugin-rot: `<ROOT>`. SKILL.md: `<PATH>`. Referansefiler:
|
||||
`<ROOT>/skills/<NAME>/references/`.
|
||||
|
||||
Les SKILL.md i sin helhet. For K8: sample 5 tilfeldige referansefiler og les headeren deres.
|
||||
Vær streng og adversariell — ikke ros, ikke pynt på tall.
|
||||
Les SKILL.md i sin helhet. Vær streng og adversariell — ikke ros, ikke pynt på tall.
|
||||
|
||||
- **K1 trigger-presisjon:** last det operatør-kuraterte settet for `<NAME>` fra
|
||||
`<ROOT>/scripts/kb-eval/data/k1-trigger-prompts.json` (nøkkel `<NAME>`: `in_domain` [10] +
|
||||
|
|
@ -28,8 +28,6 @@ Vær streng og adversariell — ikke ros, ikke pynt på tall.
|
|||
i SKILL.md body som dupliserer ref-filer? Gi konkret eksempel. pass = score ≥ 4.
|
||||
- **K7 imperativ/instruksjons-stil:** sample 10 instruksjonssetninger fra body; andel i
|
||||
imperativ/infinitiv. pass = ratio ≥ 0.80.
|
||||
- **K8 kildehenvisning i ref-filer:** for 5 samplede ref-filer, har header `Last updated` /
|
||||
`Verified` / kilde-URL? Rapporter andel. pass = ratio ≥ 0.80.
|
||||
- **K9 ingen VOLATIL tid-sensitiv info i SKILL.md body:** finnes **volatile** påstander DIREKTE i body
|
||||
(ikke i ref-filer) — GA/preview-release-status, modellversjoner, SLA-/ytelses-/pris-tall, regions-
|
||||
tilgjengelighet? List funn. pass = ingen volatile funn. **Utenfor scope (ikke funn):** stabile
|
||||
|
|
@ -40,5 +38,5 @@ Vær streng og adversariell — ikke ros, ikke pynt på tall.
|
|||
RETURNER KUN dette JSON-objektet (ingen annen tekst, ingen markdown-fence):
|
||||
|
||||
```
|
||||
{"skill":"<NAME>","K1_triggerPrecision":{"provisional":false,"inDomainHitRate":0.0,"outDomainFalsePositiveRate":0.0,"precision":0.0,"pass":false,"misclassified":[],"notes":""},"K4_noDuplication":{"score":0,"pass":false,"evidence":""},"K7_imperativeStyle":{"ratio":0.0,"pass":false,"notes":""},"K8_sourceCitation":{"ratio":0.0,"pass":false,"sampledFiles":[],"notes":""},"K9_noTimeSensitive":{"pass":false,"findings":[]}}
|
||||
{"skill":"<NAME>","K1_triggerPrecision":{"provisional":false,"inDomainHitRate":0.0,"outDomainFalsePositiveRate":0.0,"precision":0.0,"pass":false,"misclassified":[],"notes":""},"K4_noDuplication":{"score":0,"pass":false,"evidence":""},"K7_imperativeStyle":{"ratio":0.0,"pass":false,"notes":""},"K9_noTimeSensitive":{"pass":false,"findings":[]}}
|
||||
```
|
||||
|
|
|
|||
|
|
@ -109,12 +109,16 @@ export const CRITERIA = [
|
|||
},
|
||||
},
|
||||
{
|
||||
key: 'K8', label: 'kildehenvisning refs', weight: 1, floor: false, source: 'judge',
|
||||
fix: 'Legg dated/Verified-header eller kilde-URL i ref-filene (≥0.80 andel).',
|
||||
// CT5 — sourcedness (deterministic, whole-corpus). REPLACES the LLM-sampled
|
||||
// K8 (same signal; must not run in parallel — ref-kb-direction-note §45).
|
||||
// Dormant (available:false) until Port 1 migration declares reference types,
|
||||
// so it never tanks the score on an unmigrated corpus.
|
||||
key: 'CT5', label: 'sourcedness refs', weight: 1, floor: false, source: 'det',
|
||||
fix: 'Gi ref-filene **Source:**-URL (Port 1-kontrakt) — ≥0.80 andel av type:reference-filer.',
|
||||
read: (e) => {
|
||||
const j = e.judge?.K8_sourceCitation;
|
||||
if (!j) return { available: false };
|
||||
return { available: true, sub: clamp01(num(j.ratio)), pass: !!j.pass, detail: `ratio ${num(j.ratio)}` };
|
||||
const k = e.deterministic?.CT5_sourcedness;
|
||||
if (!k || !k.referenceFiles) return { available: false };
|
||||
return { available: true, sub: clamp01(num(k.ratio)), pass: !!k.pass, detail: `${num(k.sourced)}/${num(k.referenceFiles)} sourced` };
|
||||
},
|
||||
},
|
||||
{
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue