ms-ai-architect/tests/kb-eval/test-build-sample-frame.test.mjs

// tests/kb-eval/test-build-sample-frame.test.mjs
// Unit tests for the deterministic sample-frame builder (Fase 0, steg 1).
//
// The builder selects a volatility-weighted, stratified set of reference files
// for manual correctness verification against live MS Learn. The selection MUST
// be deterministic (same input -> same output; no Math.random). The volatility
// scorer is a RANKING heuristic for sample selection, not the correctness
// classifier — so its only hard contract is the K9 boundary (stable identifiers
// score zero) and reproducibility.

import { test } from 'node:test';
import assert from 'node:assert/strict';
import {
  skillOf,
  topFolder,
  scoreVolatility,
  pathBoost,
  fileScore,
  allocateQuota,
  stratify,
} from '../../scripts/kb-eval/build-sample-frame.mjs';

// --- skillOf -----------------------------------------------------------------
test('skillOf — extracts skill name from a references path', () => {
  assert.equal(
    skillOf('skills/ms-ai-advisor/references/platforms/m365-copilot.md'),
    'ms-ai-advisor',
  );
});
test('skillOf — non-skill path returns empty string', () => {
  assert.equal(skillOf('docs/ref-kb-workflow-plan-2026-06.md'), '');
});

// --- topFolder ---------------------------------------------------------------
test('topFolder — immediate folder under references/', () => {
  assert.equal(topFolder('skills/x/references/cost-optimization/a/b.md'), 'cost-optimization');
  assert.equal(topFolder('skills/x/references/platforms/p.md'), 'platforms');
});
test('topFolder — file directly under references/ has no top folder', () => {
  assert.equal(topFolder('skills/x/references/top.md'), '');
});

// --- scoreVolatility ---------------------------------------------------------
test('scoreVolatility — volatile claims accumulate signal hits and a positive score', () => {
  const t =
    'GPT-5 koster 30 NOK, 4750 TPM per PTU, public preview i Norway East, GlobalStandard SKU.';
  const r = scoreVolatility(t);
  assert.equal(r.signals.tpmPtu, 2); // TPM + PTU
  assert.ok(r.signals.version >= 1); // GPT-5
  assert.ok(r.signals.region >= 1); // Norway East
  assert.ok(r.signals.previewGa >= 1); // public preview
  assert.ok(r.signals.sku >= 1); // GlobalStandard / SKU
  assert.ok(r.signals.price >= 1); // NOK
  assert.ok(r.score > 0);
});
test('scoreVolatility — stable identifiers score zero (K9 boundary)', () => {
  const t = 'Forordning (EU) 2024/1689, sak C-311/18, MADR v3.0, OWASP LLM Top 10 2025.';
  const r = scoreVolatility(t);
  assert.equal(r.score, 0);
  assert.deepEqual(r.signals, {
    tpmPtu: 0,
    price: 0,
    sku: 0,
    region: 0,
    version: 0,
    previewGa: 0,
  });
});

// --- pathBoost ---------------------------------------------------------------
test('pathBoost — cost-optimization and platforms folders are boosted', () => {
  assert.ok(pathBoost('skills/x/references/cost-optimization/a.md') > 0);
  assert.ok(pathBoost('skills/x/references/platforms/a.md') > 0);
});
test('pathBoost — neutral folders get no boost', () => {
  assert.equal(pathBoost('skills/x/references/methodology/a.md'), 0);
});

// --- fileScore ---------------------------------------------------------------
test('fileScore — combines content volatility and path boost', () => {
  const relpath = 'skills/x/references/platforms/p.md';
  const text = 'GPT-5 in public preview.';
  assert.equal(fileScore({ relpath, text }), scoreVolatility(text).score + pathBoost(relpath));
});

// --- allocateQuota -----------------------------------------------------------
test('allocateQuota — floor per skill, remainder by mass, sums to total', () => {
  const q = allocateQuota({ a: 90, b: 10, c: 0, d: 0, e: 0 }, 45, 5);
  const sum = Object.values(q).reduce((s, n) => s + n, 0);
  assert.equal(sum, 45);
  assert.ok(q.a > q.b); // mass-weighted oversample
  assert.ok(Object.values(q).every((n) => n >= 5)); // floor honored
});
test('allocateQuota — deterministic for equal mass (tiebreak by skill name)', () => {
  const q1 = allocateQuota({ a: 1, b: 1, c: 1 }, 10, 3);
  const q2 = allocateQuota({ a: 1, b: 1, c: 1 }, 10, 3);
  assert.deepEqual(q1, q2);
  assert.equal(Object.values(q1).reduce((s, n) => s + n, 0), 10);
});

// --- stratify ----------------------------------------------------------------
const SCORED = [
  { file: 'skills/s1/references/platforms/a.md', skill: 's1', topFolder: 'platforms', score: 20 },
  { file: 'skills/s1/references/platforms/b.md', skill: 's1', topFolder: 'platforms', score: 12 },
  {
    file: 'skills/s2/references/cost-optimization/c.md',
    skill: 's2',
    topFolder: 'cost-optimization',
    score: 18,
  },
  {
    file: 'skills/s2/references/methodology/d.md',
    skill: 's2',
    topFolder: 'methodology',
    score: 0,
  },
  {
    file: 'skills/s1/references/regulatory/e.md',
    skill: 's1',
    topFolder: 'regulatory',
    score: 1,
  },
];
const CFG = {
  volatileTarget: 3,
  floorPerSkill: 1,
  controlFolders: ['methodology', 'regulatory'],
  controlMaxScore: 2,
  controlTarget: 2,
};
test('stratify — control stratum drawn from low-score methodology/regulatory files', () => {
  const { control } = stratify(SCORED, CFG);
  const files = control.map((c) => c.file).sort();
  assert.deepEqual(files, [
    'skills/s1/references/regulatory/e.md',
    'skills/s2/references/methodology/d.md',
  ]);
  assert.ok(control.every((c) => c.stratum === 'control'));
});
test('stratify — volatile stratum excludes control files and prefers high score', () => {
  const { volatile } = stratify(SCORED, CFG);
  assert.ok(volatile.every((v) => v.stratum === 'volatile'));
  assert.ok(!volatile.some((v) => v.topFolder === 'methodology'));
  assert.ok(volatile.some((v) => v.file === 'skills/s1/references/platforms/a.md'));
});
test('stratify — deterministic (same input yields identical selection)', () => {
  assert.deepEqual(stratify(SCORED, CFG), stratify(SCORED, CFG));
});