feat(ms-ai-architect): Fase 0 steg 1 — deterministisk sample-frame (45 volatil + 10 kontroll) + TDD [skip-docs]

Internt Fase 0-måleverktøy (ingen brukervendt flate: ingen ny kommando/agent/skill).

- build-sample-frame.mjs: volatilitets-scorer (K9-grense for stabile identifikatorer)
  + sti-boost (cost-optimization/platforms) + stratifisering (floor + masse-vektet
  largest-remainder). Deterministisk, ingen Math.random.
- 14 unit-tester (suite 509 -> 523, alle grønne)
- fase0-sample-frame.json: 55 filer fra 306 sourced-populasjon; oversampler
  pris/SKU/TPM/region/versjon/preview-tette filer; kontroll-stratum fra
  responsible-ai/npsg/architecture (stabile påstander)

scripts/kb-eval/build-sample-frame.mjs

@@ -0,0 +1,265 @@
+#!/usr/bin/env node
+// build-sample-frame.mjs — Fase 0, steg 1: deterministic volatility-weighted,
+// stratified sample frame for ref-file correctness verification.
+//
+// Selects ~45 volatile-dense reference files (oversampling cost/platform/SKU/
+// price/TPM/region/version/preview-dense files across all 5 skills) plus a
+// control stratum of low-volatility methodology/regulatory files. Selection is
+// DETERMINISTIC — same input yields the same frame (no Math.random).
+//
+// The volatility scorer is a RANKING heuristic for sample SELECTION, not the
+// correctness classifier (that is the subagent step against live MS Learn).
+// Its signal set + stable-identifier boundary follow the K9 rule in
+// scripts/kb-eval/judge-prompt.md: regulation years, case numbers, standard
+// version names (OWASP ... 2025, MADR v3.0) and filenames are NOT volatile.
+//
+// Verifiable population = the 306 ref files that cite >=1 MS Learn URL, derived
+// by inverting scripts/kb-update/data/url-registry.json (urls{}.reference_files[]).
+//
+// Usage:
+//   node scripts/kb-eval/build-sample-frame.mjs           # human-readable summary
+//   node scripts/kb-eval/build-sample-frame.mjs --json    # frame JSON to stdout
+//   node scripts/kb-eval/build-sample-frame.mjs --write   # persist data/fase0-sample-frame.json
+//
+// Zero dependencies. Reuses kb-update/lib/atomic-write.mjs for the gated write.
+
+import { readFileSync, existsSync } from 'node:fs';
+import { join, dirname } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { atomicWriteJson } from '../kb-update/lib/atomic-write.mjs';
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const PLUGIN_ROOT = join(__dirname, '..', '..');
+const REGISTRY = join(PLUGIN_ROOT, 'scripts/kb-update/data/url-registry.json');
+const OUT_FILE = join(__dirname, 'data', 'fase0-sample-frame.json');
+
+// --- volatility signal patterns (ranking heuristic; K9 boundary) -------------
+// Stable identifiers deliberately do NOT match: regulation years (2024/1689),
+// case numbers (C-311/18), standard version names (MADR v3.0, OWASP LLM Top 10
+// 2025) and filenames — see judge-prompt.md K9.
+const SIGNAL_PATTERNS = {
+  tpmPtu: /\b(TPM|PTU|tokens?[ -]per[ -]minute|provisioned throughput)\b/gi,
+  price:
+    /(\bNOK\b|\bUSD\b|\bEUR\b|\$\s?\d|\bkr\b|per\s?1[ ,.]?0{3,}\s?tokens|per\s?1\s?[MK]\b|\/month|\/m[åa]ned)/gi,
+  sku: /\b(SKU|GlobalStandard|DataZoneStandard|DataZone|Pay-?as-?you-?go|PayGo|provisioned deployment|deployment type)\b/gi,
+  region:
+    /\b(East US|West US|West Europe|North Europe|Sweden Central|Norway East|norwayeast|swedencentral|region(?:al)? availability|available in (?:the )?following regions)\b/gi,
+  version:
+    /\b(GPT-[0-9](?:\.[0-9])?|GPT-4o|o1|o3|o4-mini|Claude\s?[0-9]|Gemini\s?[0-9]|text-embedding-[0-9]|api-version=\d{4}-\d{2}-\d{2})\b/gi,
+  previewGa: /(public preview|private preview|in preview|\(preview\)|generally available|now available)/gi,
+};
+const SIGNAL_WEIGHTS = { tpmPtu: 3, price: 2, sku: 2, region: 2, version: 3, previewGa: 2 };
+const PATH_BOOST = { 'cost-optimization': 5, platforms: 5 };
+
+/** Skill name from a `skills/<skill>/references/...` path, else ''. */
+export function skillOf(relpath) {
+  const m = /^skills\/([^/]+)\/references\//.exec(relpath);
+  return m ? m[1] : '';
+}
+
+/** Immediate folder under `references/`, else '' (file sits directly in references/). */
+export function topFolder(relpath) {
+  const m = /^skills\/[^/]+\/references\/([^/]+)\//.exec(relpath);
+  return m ? m[1] : '';
+}
+
+/** Count volatility-signal hits in text; weighted sum = ranking score. */
+export function scoreVolatility(text) {
+  const signals = {};
+  let score = 0;
+  for (const [key, pat] of Object.entries(SIGNAL_PATTERNS)) {
+    const matches = text.match(pat);
+    const n = matches ? matches.length : 0;
+    signals[key] = n;
+    score += n * SIGNAL_WEIGHTS[key];
+  }
+  return { score, signals };
+}
+
+/** Path-based boost for volatility-dense folders (cost-optimization, platforms). */
+export function pathBoost(relpath) {
+  return PATH_BOOST[topFolder(relpath)] || 0;
+}
+
+/** Total ranking score for a file = content volatility + path boost. */
+export function fileScore({ relpath, text }) {
+  return scoreVolatility(text).score + pathBoost(relpath);
+}
+
+/**
+ * Apportion `total` selections across skills: a floor per skill (balance) plus
+ * the remainder distributed by volatility mass (oversample dense skills), using
+ * largest-remainder apportionment. Deterministic — ties broken by skill name.
+ */
+export function allocateQuota(massBySkill, total, floorPerSkill) {
+  const skills = Object.keys(massBySkill).sort();
+  const quotas = {};
+  for (const s of skills) quotas[s] = floorPerSkill;
+  let remaining = total - floorPerSkill * skills.length;
+  if (remaining <= 0) return quotas;
+  const totalMass = skills.reduce((sum, s) => sum + massBySkill[s], 0);
+  if (totalMass <= 0) {
+    // no mass signal — distribute round-robin deterministically by skill name
+    let i = 0;
+    while (remaining > 0) {
+      quotas[skills[i % skills.length]] += 1;
+      remaining--;
+      i++;
+    }
+    return quotas;
+  }
+  const shares = skills.map((s) => {
+    const exact = (massBySkill[s] / totalMass) * remaining;
+    const base = Math.floor(exact);
+    return { s, base, frac: exact - base };
+  });
+  for (const sh of shares) {
+    quotas[sh.s] += sh.base;
+    remaining -= sh.base;
+  }
+  shares.sort((a, b) => b.frac - a.frac || (a.s < b.s ? -1 : 1));
+  for (let i = 0; i < shares.length && remaining > 0; i++) {
+    quotas[shares[i].s] += 1;
+    remaining--;
+  }
+  return quotas;
+}
+
+const byScoreThenPath = (a, b) =>
+  b.score - a.score || (a.file < b.file ? -1 : a.file > b.file ? 1 : 0);
+
+/**
+ * Split scored files into a volatile stratum (~volatileTarget, balanced across
+ * skills, oversampling dense ones) and a control stratum (low-volatility files
+ * from controlFolders). Deterministic.
+ */
+export function stratify(scored, cfg) {
+  const controlSet = new Set();
+  const control = scored
+    .filter((f) => cfg.controlFolders.includes(f.topFolder) && f.score <= cfg.controlMaxScore)
+    .sort((a, b) => a.score - b.score || (a.file < b.file ? -1 : a.file > b.file ? 1 : 0))
+    .slice(0, cfg.controlTarget)
+    .map((f) => {
+      controlSet.add(f.file);
+      return { ...f, stratum: 'control' };
+    });
+
+  const pool = scored.filter((f) => !controlSet.has(f.file) && f.skill);
+  const bySkill = {};
+  const massBySkill = {};
+  for (const f of pool) {
+    (bySkill[f.skill] ||= []).push(f);
+    massBySkill[f.skill] = (massBySkill[f.skill] || 0) + f.score;
+  }
+  const quotas = allocateQuota(massBySkill, cfg.volatileTarget, cfg.floorPerSkill);
+  const volatile = [];
+  for (const skill of Object.keys(bySkill).sort()) {
+    for (const f of bySkill[skill].sort(byScoreThenPath).slice(0, quotas[skill] || 0)) {
+      volatile.push({ ...f, stratum: 'volatile' });
+    }
+  }
+  volatile.sort(byScoreThenPath);
+  return { volatile, control };
+}
+
+// --- CLI / frame assembly ----------------------------------------------------
+const CFG = {
+  volatileTarget: 45,
+  floorPerSkill: 5,
+  // Control = stable-claim folders (the real analogues of the contract's
+  // "methodology/regulatory"): responsible-AI principles, Norwegian public-sector
+  // governance/law, and architecture methodology. Filtered to low volatility score
+  // so the control stratum measures the stable-claim sanity rate, not volatile churn.
+  controlFolders: ['responsible-ai', 'norwegian-public-sector-governance', 'architecture'],
+  controlMaxScore: 4,
+  controlTarget: 10,
+};
+
+/** Invert url-registry → Map(reference_file -> sorted unique cited URLs). */
+function invertRegistry() {
+  const reg = JSON.parse(readFileSync(REGISTRY, 'utf8'));
+  const inv = new Map();
+  for (const [url, meta] of Object.entries(reg.urls || {})) {
+    for (const rf of meta.reference_files || []) {
+      if (!inv.has(rf)) inv.set(rf, new Set());
+      inv.get(rf).add(url);
+    }
+  }
+  return inv;
+}
+
+function scoreSourcedFiles() {
+  const inv = invertRegistry();
+  const scored = [];
+  for (const [relpath, urlSet] of inv) {
+    const abs = join(PLUGIN_ROOT, relpath);
+    if (!existsSync(abs)) continue; // sourced file since deleted — skip
+    const text = readFileSync(abs, 'utf8');
+    const { score, signals } = scoreVolatility(text);
+    scored.push({
+      file: relpath,
+      skill: skillOf(relpath),
+      topFolder: topFolder(relpath),
+      score: score + pathBoost(relpath),
+      signals,
+      citedUrls: [...urlSet].sort(),
+    });
+  }
+  return scored;
+}
+
+function buildFrame() {
+  const scored = scoreSourcedFiles();
+  const { volatile, control } = stratify(scored, CFG);
+  const stamp = new Date().toISOString();
+  return {
+    _meta: {
+      created: stamp,
+      method:
+        'volatility-weighted stratified selection; deterministic (no random). ' +
+        'Scorer is a ranking heuristic for sample selection, not the correctness classifier.',
+      population: scored.length,
+      volatile_count: volatile.length,
+      control_count: control.length,
+      config: CFG,
+      signal_weights: SIGNAL_WEIGHTS,
+    },
+    volatile,
+    control,
+  };
+}
+
+function summarize(frame) {
+  const perSkill = {};
+  for (const f of [...frame.volatile, ...frame.control]) {
+    const k = `${f.skill}/${f.stratum}`;
+    perSkill[k] = (perSkill[k] || 0) + 1;
+  }
+  console.log(`Sourced population: ${frame._meta.population} files`);
+  console.log(`Volatile stratum:   ${frame.volatile.length}`);
+  console.log(`Control stratum:    ${frame.control.length}`);
+  console.log('\nPer skill / stratum:');
+  for (const k of Object.keys(perSkill).sort()) console.log(`  ${k.padEnd(34)} ${perSkill[k]}`);
+  console.log('\nTop 10 volatile (score · file):');
+  for (const f of frame.volatile.slice(0, 10)) {
+    console.log(`  ${String(f.score).padStart(4)}  ${f.file}`);
+  }
+}
+
+function main() {
+  const args = process.argv.slice(2);
+  const frame = buildFrame();
+  if (args.includes('--json')) {
+    process.stdout.write(JSON.stringify(frame, null, 2) + '\n');
+    return;
+  }
+  summarize(frame);
+  if (args.includes('--write')) {
+    atomicWriteJson(OUT_FILE, frame);
+    console.log(`\nWrote ${OUT_FILE}`);
+  } else {
+    console.log('\n(dry run — pass --write to persist data/fase0-sample-frame.json)');
+  }
+}
+
+if (import.meta.url === `file://${process.argv[1]}`) main();

tests/kb-eval/test-build-sample-frame.test.mjs

@@ -0,0 +1,148 @@
+// tests/kb-eval/test-build-sample-frame.test.mjs
+// Unit tests for the deterministic sample-frame builder (Fase 0, steg 1).
+//
+// The builder selects a volatility-weighted, stratified set of reference files
+// for manual correctness verification against live MS Learn. The selection MUST
+// be deterministic (same input -> same output; no Math.random). The volatility
+// scorer is a RANKING heuristic for sample selection, not the correctness
+// classifier — so its only hard contract is the K9 boundary (stable identifiers
+// score zero) and reproducibility.
+
+import { test } from 'node:test';
+import assert from 'node:assert/strict';
+import {
+  skillOf,
+  topFolder,
+  scoreVolatility,
+  pathBoost,
+  fileScore,
+  allocateQuota,
+  stratify,
+} from '../../scripts/kb-eval/build-sample-frame.mjs';
+
+// --- skillOf -----------------------------------------------------------------
+test('skillOf — extracts skill name from a references path', () => {
+  assert.equal(
+    skillOf('skills/ms-ai-advisor/references/platforms/m365-copilot.md'),
+    'ms-ai-advisor',
+  );
+});
+test('skillOf — non-skill path returns empty string', () => {
+  assert.equal(skillOf('docs/ref-kb-workflow-plan-2026-06.md'), '');
+});
+
+// --- topFolder ---------------------------------------------------------------
+test('topFolder — immediate folder under references/', () => {
+  assert.equal(topFolder('skills/x/references/cost-optimization/a/b.md'), 'cost-optimization');
+  assert.equal(topFolder('skills/x/references/platforms/p.md'), 'platforms');
+});
+test('topFolder — file directly under references/ has no top folder', () => {
+  assert.equal(topFolder('skills/x/references/top.md'), '');
+});
+
+// --- scoreVolatility ---------------------------------------------------------
+test('scoreVolatility — volatile claims accumulate signal hits and a positive score', () => {
+  const t =
+    'GPT-5 koster 30 NOK, 4750 TPM per PTU, public preview i Norway East, GlobalStandard SKU.';
+  const r = scoreVolatility(t);
+  assert.equal(r.signals.tpmPtu, 2); // TPM + PTU
+  assert.ok(r.signals.version >= 1); // GPT-5
+  assert.ok(r.signals.region >= 1); // Norway East
+  assert.ok(r.signals.previewGa >= 1); // public preview
+  assert.ok(r.signals.sku >= 1); // GlobalStandard / SKU
+  assert.ok(r.signals.price >= 1); // NOK
+  assert.ok(r.score > 0);
+});
+test('scoreVolatility — stable identifiers score zero (K9 boundary)', () => {
+  const t = 'Forordning (EU) 2024/1689, sak C-311/18, MADR v3.0, OWASP LLM Top 10 2025.';
+  const r = scoreVolatility(t);
+  assert.equal(r.score, 0);
+  assert.deepEqual(r.signals, {
+    tpmPtu: 0,
+    price: 0,
+    sku: 0,
+    region: 0,
+    version: 0,
+    previewGa: 0,
+  });
+});
+
+// --- pathBoost ---------------------------------------------------------------
+test('pathBoost — cost-optimization and platforms folders are boosted', () => {
+  assert.ok(pathBoost('skills/x/references/cost-optimization/a.md') > 0);
+  assert.ok(pathBoost('skills/x/references/platforms/a.md') > 0);
+});
+test('pathBoost — neutral folders get no boost', () => {
+  assert.equal(pathBoost('skills/x/references/methodology/a.md'), 0);
+});
+
+// --- fileScore ---------------------------------------------------------------
+test('fileScore — combines content volatility and path boost', () => {
+  const relpath = 'skills/x/references/platforms/p.md';
+  const text = 'GPT-5 in public preview.';
+  assert.equal(fileScore({ relpath, text }), scoreVolatility(text).score + pathBoost(relpath));
+});
+
+// --- allocateQuota -----------------------------------------------------------
+test('allocateQuota — floor per skill, remainder by mass, sums to total', () => {
+  const q = allocateQuota({ a: 90, b: 10, c: 0, d: 0, e: 0 }, 45, 5);
+  const sum = Object.values(q).reduce((s, n) => s + n, 0);
+  assert.equal(sum, 45);
+  assert.ok(q.a > q.b); // mass-weighted oversample
+  assert.ok(Object.values(q).every((n) => n >= 5)); // floor honored
+});
+test('allocateQuota — deterministic for equal mass (tiebreak by skill name)', () => {
+  const q1 = allocateQuota({ a: 1, b: 1, c: 1 }, 10, 3);
+  const q2 = allocateQuota({ a: 1, b: 1, c: 1 }, 10, 3);
+  assert.deepEqual(q1, q2);
+  assert.equal(Object.values(q1).reduce((s, n) => s + n, 0), 10);
+});
+
+// --- stratify ----------------------------------------------------------------
+const SCORED = [
+  { file: 'skills/s1/references/platforms/a.md', skill: 's1', topFolder: 'platforms', score: 20 },
+  { file: 'skills/s1/references/platforms/b.md', skill: 's1', topFolder: 'platforms', score: 12 },
+  {
+    file: 'skills/s2/references/cost-optimization/c.md',
+    skill: 's2',
+    topFolder: 'cost-optimization',
+    score: 18,
+  },
+  {
+    file: 'skills/s2/references/methodology/d.md',
+    skill: 's2',
+    topFolder: 'methodology',
+    score: 0,
+  },
+  {
+    file: 'skills/s1/references/regulatory/e.md',
+    skill: 's1',
+    topFolder: 'regulatory',
+    score: 1,
+  },
+];
+const CFG = {
+  volatileTarget: 3,
+  floorPerSkill: 1,
+  controlFolders: ['methodology', 'regulatory'],
+  controlMaxScore: 2,
+  controlTarget: 2,
+};
+test('stratify — control stratum drawn from low-score methodology/regulatory files', () => {
+  const { control } = stratify(SCORED, CFG);
+  const files = control.map((c) => c.file).sort();
+  assert.deepEqual(files, [
+    'skills/s1/references/regulatory/e.md',
+    'skills/s2/references/methodology/d.md',
+  ]);
+  assert.ok(control.every((c) => c.stratum === 'control'));
+});
+test('stratify — volatile stratum excludes control files and prefers high score', () => {
+  const { volatile } = stratify(SCORED, CFG);
+  assert.ok(volatile.every((v) => v.stratum === 'volatile'));
+  assert.ok(!volatile.some((v) => v.topFolder === 'methodology'));
+  assert.ok(volatile.some((v) => v.file === 'skills/s1/references/platforms/a.md'));
+});
+test('stratify — deterministic (same input yields identical selection)', () => {
+  assert.deepEqual(stratify(SCORED, CFG), stratify(SCORED, CFG));
+});