feat(ms-ai-architect): Fase 0 steg 1 — deterministisk sample-frame (45 volatil + 10 kontroll) + TDD [skip-docs]
Internt Fase 0-måleverktøy (ingen brukervendt flate: ingen ny kommando/agent/skill). - build-sample-frame.mjs: volatilitets-scorer (K9-grense for stabile identifikatorer) + sti-boost (cost-optimization/platforms) + stratifisering (floor + masse-vektet largest-remainder). Deterministisk, ingen Math.random. - 14 unit-tester (suite 509 -> 523, alle grønne) - fase0-sample-frame.json: 55 filer fra 306 sourced-populasjon; oversampler pris/SKU/TPM/region/versjon/preview-tette filer; kontroll-stratum fra responsible-ai/npsg/architecture (stabile påstander)
This commit is contained in:
parent
91556a38b5
commit
f7fba5e4e7
3 changed files with 1742 additions and 0 deletions
148
tests/kb-eval/test-build-sample-frame.test.mjs
Normal file
148
tests/kb-eval/test-build-sample-frame.test.mjs
Normal file
|
|
@ -0,0 +1,148 @@
|
|||
// tests/kb-eval/test-build-sample-frame.test.mjs
|
||||
// Unit tests for the deterministic sample-frame builder (Fase 0, steg 1).
|
||||
//
|
||||
// The builder selects a volatility-weighted, stratified set of reference files
|
||||
// for manual correctness verification against live MS Learn. The selection MUST
|
||||
// be deterministic (same input -> same output; no Math.random). The volatility
|
||||
// scorer is a RANKING heuristic for sample selection, not the correctness
|
||||
// classifier — so its only hard contract is the K9 boundary (stable identifiers
|
||||
// score zero) and reproducibility.
|
||||
|
||||
import { test } from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import {
|
||||
skillOf,
|
||||
topFolder,
|
||||
scoreVolatility,
|
||||
pathBoost,
|
||||
fileScore,
|
||||
allocateQuota,
|
||||
stratify,
|
||||
} from '../../scripts/kb-eval/build-sample-frame.mjs';
|
||||
|
||||
// --- skillOf -----------------------------------------------------------------
|
||||
test('skillOf — extracts skill name from a references path', () => {
|
||||
assert.equal(
|
||||
skillOf('skills/ms-ai-advisor/references/platforms/m365-copilot.md'),
|
||||
'ms-ai-advisor',
|
||||
);
|
||||
});
|
||||
test('skillOf — non-skill path returns empty string', () => {
|
||||
assert.equal(skillOf('docs/ref-kb-workflow-plan-2026-06.md'), '');
|
||||
});
|
||||
|
||||
// --- topFolder ---------------------------------------------------------------
|
||||
test('topFolder — immediate folder under references/', () => {
|
||||
assert.equal(topFolder('skills/x/references/cost-optimization/a/b.md'), 'cost-optimization');
|
||||
assert.equal(topFolder('skills/x/references/platforms/p.md'), 'platforms');
|
||||
});
|
||||
test('topFolder — file directly under references/ has no top folder', () => {
|
||||
assert.equal(topFolder('skills/x/references/top.md'), '');
|
||||
});
|
||||
|
||||
// --- scoreVolatility ---------------------------------------------------------
|
||||
test('scoreVolatility — volatile claims accumulate signal hits and a positive score', () => {
|
||||
const t =
|
||||
'GPT-5 koster 30 NOK, 4750 TPM per PTU, public preview i Norway East, GlobalStandard SKU.';
|
||||
const r = scoreVolatility(t);
|
||||
assert.equal(r.signals.tpmPtu, 2); // TPM + PTU
|
||||
assert.ok(r.signals.version >= 1); // GPT-5
|
||||
assert.ok(r.signals.region >= 1); // Norway East
|
||||
assert.ok(r.signals.previewGa >= 1); // public preview
|
||||
assert.ok(r.signals.sku >= 1); // GlobalStandard / SKU
|
||||
assert.ok(r.signals.price >= 1); // NOK
|
||||
assert.ok(r.score > 0);
|
||||
});
|
||||
test('scoreVolatility — stable identifiers score zero (K9 boundary)', () => {
|
||||
const t = 'Forordning (EU) 2024/1689, sak C-311/18, MADR v3.0, OWASP LLM Top 10 2025.';
|
||||
const r = scoreVolatility(t);
|
||||
assert.equal(r.score, 0);
|
||||
assert.deepEqual(r.signals, {
|
||||
tpmPtu: 0,
|
||||
price: 0,
|
||||
sku: 0,
|
||||
region: 0,
|
||||
version: 0,
|
||||
previewGa: 0,
|
||||
});
|
||||
});
|
||||
|
||||
// --- pathBoost ---------------------------------------------------------------
|
||||
test('pathBoost — cost-optimization and platforms folders are boosted', () => {
|
||||
assert.ok(pathBoost('skills/x/references/cost-optimization/a.md') > 0);
|
||||
assert.ok(pathBoost('skills/x/references/platforms/a.md') > 0);
|
||||
});
|
||||
test('pathBoost — neutral folders get no boost', () => {
|
||||
assert.equal(pathBoost('skills/x/references/methodology/a.md'), 0);
|
||||
});
|
||||
|
||||
// --- fileScore ---------------------------------------------------------------
|
||||
test('fileScore — combines content volatility and path boost', () => {
|
||||
const relpath = 'skills/x/references/platforms/p.md';
|
||||
const text = 'GPT-5 in public preview.';
|
||||
assert.equal(fileScore({ relpath, text }), scoreVolatility(text).score + pathBoost(relpath));
|
||||
});
|
||||
|
||||
// --- allocateQuota -----------------------------------------------------------
|
||||
test('allocateQuota — floor per skill, remainder by mass, sums to total', () => {
|
||||
const q = allocateQuota({ a: 90, b: 10, c: 0, d: 0, e: 0 }, 45, 5);
|
||||
const sum = Object.values(q).reduce((s, n) => s + n, 0);
|
||||
assert.equal(sum, 45);
|
||||
assert.ok(q.a > q.b); // mass-weighted oversample
|
||||
assert.ok(Object.values(q).every((n) => n >= 5)); // floor honored
|
||||
});
|
||||
test('allocateQuota — deterministic for equal mass (tiebreak by skill name)', () => {
|
||||
const q1 = allocateQuota({ a: 1, b: 1, c: 1 }, 10, 3);
|
||||
const q2 = allocateQuota({ a: 1, b: 1, c: 1 }, 10, 3);
|
||||
assert.deepEqual(q1, q2);
|
||||
assert.equal(Object.values(q1).reduce((s, n) => s + n, 0), 10);
|
||||
});
|
||||
|
||||
// --- stratify ----------------------------------------------------------------
|
||||
const SCORED = [
|
||||
{ file: 'skills/s1/references/platforms/a.md', skill: 's1', topFolder: 'platforms', score: 20 },
|
||||
{ file: 'skills/s1/references/platforms/b.md', skill: 's1', topFolder: 'platforms', score: 12 },
|
||||
{
|
||||
file: 'skills/s2/references/cost-optimization/c.md',
|
||||
skill: 's2',
|
||||
topFolder: 'cost-optimization',
|
||||
score: 18,
|
||||
},
|
||||
{
|
||||
file: 'skills/s2/references/methodology/d.md',
|
||||
skill: 's2',
|
||||
topFolder: 'methodology',
|
||||
score: 0,
|
||||
},
|
||||
{
|
||||
file: 'skills/s1/references/regulatory/e.md',
|
||||
skill: 's1',
|
||||
topFolder: 'regulatory',
|
||||
score: 1,
|
||||
},
|
||||
];
|
||||
const CFG = {
|
||||
volatileTarget: 3,
|
||||
floorPerSkill: 1,
|
||||
controlFolders: ['methodology', 'regulatory'],
|
||||
controlMaxScore: 2,
|
||||
controlTarget: 2,
|
||||
};
|
||||
test('stratify — control stratum drawn from low-score methodology/regulatory files', () => {
|
||||
const { control } = stratify(SCORED, CFG);
|
||||
const files = control.map((c) => c.file).sort();
|
||||
assert.deepEqual(files, [
|
||||
'skills/s1/references/regulatory/e.md',
|
||||
'skills/s2/references/methodology/d.md',
|
||||
]);
|
||||
assert.ok(control.every((c) => c.stratum === 'control'));
|
||||
});
|
||||
test('stratify — volatile stratum excludes control files and prefers high score', () => {
|
||||
const { volatile } = stratify(SCORED, CFG);
|
||||
assert.ok(volatile.every((v) => v.stratum === 'volatile'));
|
||||
assert.ok(!volatile.some((v) => v.topFolder === 'methodology'));
|
||||
assert.ok(volatile.some((v) => v.file === 'skills/s1/references/platforms/a.md'));
|
||||
});
|
||||
test('stratify — deterministic (same input yields identical selection)', () => {
|
||||
assert.deepEqual(stratify(SCORED, CFG), stratify(SCORED, CFG));
|
||||
});
|
||||
Loading…
Add table
Add a link
Reference in a new issue