feat(ms-ai-architect): Spor 3 Port 3 (CT5) — deterministisk sourcedness erstatter LLM-samplet K8 [skip-docs]
§4 Port 3 verifiseringskrav: «CT5 (sourcedness) ERSTATTER K8s rolle». K8 var LLM-judge (sample 5 ref-filer, ikke-deterministisk); CT5 er samme signal gjort deterministisk + hel-korpus fra Port 1-kontrakten. Per ref-kb-direction-note §45 MÅ de ikke leve parallelt (dobbelttelling + divergerende dashbord) — så K8 er fjernet, ikke duplisert. eval.mjs: checkCT5(refFiles) — blant filer som deklarerer type:reference, andel med **Source:** (template/methodology/regulatory ekskludert fra nevneren). Wiret som deterministic.CT5_sourcedness. parseTypeHeader/parseSourceHeader importert fra kb-update/lib/kb-headers (tillatt retning). skill-score.mjs: K8-kriteriet (source:judge) → CT5 (source:det, vekt 1). available:false når referenceFiles=0 → droppet fra vektet snitt → tanker IKKE scoren på umigrert korpus (alarm-tretthet unngått, direction-note §45). judge-prompt.md: K8 fjernet fra rubrikk/instruksjon/JSON (judgen gjør nå K1/K4/K7/K9). Ekte kjøring: alle 5 skills CT5 dormant (referenceFiles:0, umigrert), scorer uendret (91-96, 0 under mål). CT5 aktiveres deterministisk når Spor 1 migrerer. TDD (Iron Law): 5 checkCT5 + 3 CT5-integrasjon; død K8-fixture-data ryddet. Suite 620/620. Plugin-validering 239/0/0.
This commit is contained in:
parent
08e7e72c62
commit
75ee9ec062
7 changed files with 153 additions and 23 deletions
|
|
@ -16,6 +16,7 @@ import {
|
|||
checkN3,
|
||||
checkN4,
|
||||
checkN5,
|
||||
checkCT5,
|
||||
} from '../../scripts/kb-eval/eval.mjs';
|
||||
|
||||
test('splitFrontmatter — separates frontmatter from body', () => {
|
||||
|
|
@ -229,3 +230,67 @@ test('checkN5 — Windows backslash path fails', () => {
|
|||
test('checkN5 — drive-letter path fails', () => {
|
||||
assert.equal(checkN5('Save to C:\\Users\\x\\file.md').pass, false);
|
||||
});
|
||||
|
||||
// --- CT5 — sourcedness (deterministic, whole-corpus). REPLACES the LLM-sampled
|
||||
// K8 (same signal, must not live in parallel — ref-kb-direction-note §45). Scope
|
||||
// is the Port 1 contract: only files declared `type: reference` are in scope;
|
||||
// CT5 = fraction of those that cite a **Source:**. template/methodology/
|
||||
// regulatory are exempt (excluded from the denominator). When no file declares
|
||||
// `type: reference` (unmigrated legacy corpus) the denominator is 0 — CT5 is then
|
||||
// dormant (the score layer drops it; it does not tank the score).
|
||||
|
||||
const refFile = (over) => ({ path: 'references/x.md', content: '', ...over });
|
||||
const srcLine = '**Source:** https://learn.microsoft.com/azure/x\n';
|
||||
|
||||
test('checkCT5 — all declared reference files cite a source → ratio 1, pass', () => {
|
||||
const refs = [
|
||||
refFile({ path: 'references/a.md', content: '# A\n**Type:** reference\n' + srcLine }),
|
||||
refFile({ path: 'references/b.md', content: '# B\n**Type:** reference\n' + srcLine }),
|
||||
];
|
||||
const r = checkCT5(refs);
|
||||
assert.equal(r.referenceFiles, 2);
|
||||
assert.equal(r.sourced, 2);
|
||||
assert.equal(r.ratio, 1);
|
||||
assert.equal(r.pass, true);
|
||||
});
|
||||
|
||||
test('checkCT5 — a reference file missing **Source:** lowers the ratio', () => {
|
||||
const refs = [
|
||||
refFile({ path: 'references/a.md', content: '# A\n**Type:** reference\n' + srcLine }),
|
||||
refFile({ path: 'references/b.md', content: '# B\n**Type:** reference\n' }), // no source
|
||||
];
|
||||
const r = checkCT5(refs);
|
||||
assert.equal(r.referenceFiles, 2);
|
||||
assert.equal(r.sourced, 1);
|
||||
assert.equal(r.ratio, 0.5);
|
||||
assert.equal(r.pass, false);
|
||||
});
|
||||
|
||||
test('checkCT5 — template/methodology/regulatory are excluded from the denominator', () => {
|
||||
const refs = [
|
||||
refFile({ path: 'references/r.md', content: '# R\n**Type:** reference\n' + srcLine }),
|
||||
refFile({ path: 'references/t.md', content: '# T\n**Type:** template\n' }),
|
||||
refFile({ path: 'references/m.md', content: '# M\n**Type:** methodology\n' }),
|
||||
refFile({ path: 'references/g.md', content: '# G\n**Type:** regulatory\n' }),
|
||||
];
|
||||
const r = checkCT5(refs);
|
||||
assert.equal(r.referenceFiles, 1, 'only the one reference file is in scope');
|
||||
assert.equal(r.sourced, 1);
|
||||
assert.equal(r.ratio, 1);
|
||||
});
|
||||
|
||||
test('checkCT5 — no file declares type:reference (unmigrated legacy) → referenceFiles 0', () => {
|
||||
const refs = [
|
||||
refFile({ path: 'references/a.md', content: '# A\n**Last updated:** 2026-04\n' }),
|
||||
refFile({ path: 'references/b.md', content: '# B\nplain prose, no headers\n' }),
|
||||
];
|
||||
const r = checkCT5(refs);
|
||||
assert.equal(r.referenceFiles, 0, 'denominator empty until Port 1 migration declares types');
|
||||
});
|
||||
|
||||
test('checkCT5 — 0.80 is the pass threshold (mirrors the old K8 ratio target)', () => {
|
||||
const mk = (n, withSrc) => Array.from({ length: n }, (_, i) =>
|
||||
refFile({ path: `references/f${i}.md`, content: '**Type:** reference\n' + (i < withSrc ? srcLine : '') }));
|
||||
assert.equal(checkCT5(mk(5, 4)).pass, true, '4/5 = 0.80 passes');
|
||||
assert.equal(checkCT5(mk(5, 3)).pass, false, '3/5 = 0.60 fails');
|
||||
});
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue