feat(ms-ai-architect): Spor 3 Port 3 (CT5) — deterministisk sourcedness erstatter LLM-samplet K8 [skip-docs]
§4 Port 3 verifiseringskrav: «CT5 (sourcedness) ERSTATTER K8s rolle». K8 var LLM-judge (sample 5 ref-filer, ikke-deterministisk); CT5 er samme signal gjort deterministisk + hel-korpus fra Port 1-kontrakten. Per ref-kb-direction-note §45 MÅ de ikke leve parallelt (dobbelttelling + divergerende dashbord) — så K8 er fjernet, ikke duplisert. eval.mjs: checkCT5(refFiles) — blant filer som deklarerer type:reference, andel med **Source:** (template/methodology/regulatory ekskludert fra nevneren). Wiret som deterministic.CT5_sourcedness. parseTypeHeader/parseSourceHeader importert fra kb-update/lib/kb-headers (tillatt retning). skill-score.mjs: K8-kriteriet (source:judge) → CT5 (source:det, vekt 1). available:false når referenceFiles=0 → droppet fra vektet snitt → tanker IKKE scoren på umigrert korpus (alarm-tretthet unngått, direction-note §45). judge-prompt.md: K8 fjernet fra rubrikk/instruksjon/JSON (judgen gjør nå K1/K4/K7/K9). Ekte kjøring: alle 5 skills CT5 dormant (referenceFiles:0, umigrert), scorer uendret (91-96, 0 under mål). CT5 aktiveres deterministisk når Spor 1 migrerer. TDD (Iron Law): 5 checkCT5 + 3 CT5-integrasjon; død K8-fixture-data ryddet. Suite 620/620. Plugin-validering 239/0/0.
This commit is contained in:
parent
08e7e72c62
commit
75ee9ec062
7 changed files with 153 additions and 23 deletions
|
|
@ -7,7 +7,7 @@
|
|||
|
||||
import { test } from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { scoreSkill, TARGET, FLOOR_CAP } from '../../scripts/kb-eval/lib/skill-score.mjs';
|
||||
import { scoreSkill, TARGET, FLOOR_CAP, CRITERIA } from '../../scripts/kb-eval/lib/skill-score.mjs';
|
||||
|
||||
// --- fixture builder: a fully-passing eval object (all sub-scores = 1) -------
|
||||
function perfectEval(overrides = {}) {
|
||||
|
|
@ -26,13 +26,13 @@ function perfectEval(overrides = {}) {
|
|||
N3_refNestingDepth: { pass: true, nested: [] },
|
||||
N4_refToc: { largeFiles: 5, withToc: 5, ratio: 1, pass: true },
|
||||
N5_forwardSlashPaths: { pass: true, windowsPaths: [] },
|
||||
CT5_sourcedness: { referenceFiles: 5, sourced: 5, ratio: 1, pass: true },
|
||||
},
|
||||
judgeInputs: { description: 'desc', bodyLines: 120 },
|
||||
judge: {
|
||||
K1_triggerPrecision: { precision: 1, pass: true },
|
||||
K4_noDuplication: { score: 5, pass: true },
|
||||
K7_imperativeStyle: { ratio: 1, pass: true },
|
||||
K8_sourceCitation: { ratio: 1, pass: true },
|
||||
K9_noTimeSensitive: { pass: true, findings: [] },
|
||||
},
|
||||
};
|
||||
|
|
@ -59,6 +59,38 @@ test('scoreSkill — TARGET is 90 and FLOOR_CAP is below it', () => {
|
|||
assert.ok(FLOOR_CAP < TARGET);
|
||||
});
|
||||
|
||||
// CT5 (sourcedness) replaced the LLM-sampled K8. On an unmigrated corpus
|
||||
// (referenceFiles: 0) it is dormant — dropped from the weighted mean, so it
|
||||
// never tanks the score before Port 1 migration. Once reference files exist it
|
||||
// contributes deterministically.
|
||||
|
||||
test('scoreSkill — CT5 dormant (no declared reference files) is dropped, score stays 100', () => {
|
||||
const r = scoreSkill(perfectEval({
|
||||
deterministic: { CT5_sourcedness: { referenceFiles: 0, sourced: 0, ratio: 0, pass: true } },
|
||||
}));
|
||||
assert.equal(r.score, 100, 'CT5 available:false => excluded from the mean, no score impact');
|
||||
const ct5 = r.criteria.find((c) => c.key === 'CT5');
|
||||
assert.equal(ct5.available, false);
|
||||
});
|
||||
|
||||
test('scoreSkill — CT5 contributes deterministically once reference files exist', () => {
|
||||
const r = scoreSkill(perfectEval({
|
||||
deterministic: { CT5_sourcedness: { referenceFiles: 4, sourced: 2, ratio: 0.5, pass: false } },
|
||||
}));
|
||||
assert.ok(r.score < 100, `imperfect sourcedness should cost points, got ${r.score}`);
|
||||
const ct5 = r.criteria.find((c) => c.key === 'CT5');
|
||||
assert.equal(ct5.available, true);
|
||||
assert.equal(ct5.sub, 0.5);
|
||||
assert.ok(r.improvements.some((i) => i.key === 'CT5'), 'CT5 surfaces as an improvement');
|
||||
});
|
||||
|
||||
test('scoreSkill — CT5 is deterministic (source det), not a judge criterion', () => {
|
||||
const ct5 = CRITERIA.find((c) => c.key === 'CT5');
|
||||
assert.ok(ct5, 'CT5 exists in the rubric');
|
||||
assert.equal(ct5.source, 'det');
|
||||
assert.equal(CRITERIA.find((c) => c.key === 'K8'), undefined, 'K8 no longer in the rubric');
|
||||
});
|
||||
|
||||
test('scoreSkill — failing K10 (deterministic floor) caps the score below target despite a perfect rest', () => {
|
||||
const r = scoreSkill(perfectEval({
|
||||
deterministic: { K10_siblingScopeOverlap: { maxCombined: 7.4, worstSibling: 'y', pass: false, threshold: 7.0 } },
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue