ms-ai-architect/scripts/kb-eval/build-sample-frame.mjs

#!/usr/bin/env node
// build-sample-frame.mjs — Fase 0, steg 1: deterministic volatility-weighted,
// stratified sample frame for ref-file correctness verification.
//
// Selects ~45 volatile-dense reference files (oversampling cost/platform/SKU/
// price/TPM/region/version/preview-dense files across all 5 skills) plus a
// control stratum of low-volatility methodology/regulatory files. Selection is
// DETERMINISTIC — same input yields the same frame (no Math.random).
//
// The volatility scorer is a RANKING heuristic for sample SELECTION, not the
// correctness classifier (that is the subagent step against live MS Learn).
// Its signal set + stable-identifier boundary follow the K9 rule in
// scripts/kb-eval/judge-prompt.md: regulation years, case numbers, standard
// version names (OWASP ... 2025, MADR v3.0) and filenames are NOT volatile.
//
// Verifiable population = the 306 ref files that cite >=1 MS Learn URL, derived
// by inverting scripts/kb-update/data/url-registry.json (urls{}.reference_files[]).
//
// Usage:
//   node scripts/kb-eval/build-sample-frame.mjs           # human-readable summary
//   node scripts/kb-eval/build-sample-frame.mjs --json    # frame JSON to stdout
//   node scripts/kb-eval/build-sample-frame.mjs --write   # persist data/fase0-sample-frame.json
//
// Zero dependencies. Reuses kb-update/lib/atomic-write.mjs for the gated write.

import { readFileSync, existsSync } from 'node:fs';
import { join, dirname } from 'node:path';
import { fileURLToPath } from 'node:url';
import { atomicWriteJson } from '../kb-update/lib/atomic-write.mjs';

const __dirname = dirname(fileURLToPath(import.meta.url));
const PLUGIN_ROOT = join(__dirname, '..', '..');
const REGISTRY = join(PLUGIN_ROOT, 'scripts/kb-update/data/url-registry.json');
const OUT_FILE = join(__dirname, 'data', 'fase0-sample-frame.json');

// --- volatility signal patterns (ranking heuristic; K9 boundary) -------------
// Stable identifiers deliberately do NOT match: regulation years (2024/1689),
// case numbers (C-311/18), standard version names (MADR v3.0, OWASP LLM Top 10
// 2025) and filenames — see judge-prompt.md K9.
const SIGNAL_PATTERNS = {
  tpmPtu: /\b(TPM|PTU|tokens?[ -]per[ -]minute|provisioned throughput)\b/gi,
  price:
    /(\bNOK\b|\bUSD\b|\bEUR\b|\$\s?\d|\bkr\b|per\s?1[ ,.]?0{3,}\s?tokens|per\s?1\s?[MK]\b|\/month|\/m[åa]ned)/gi,
  sku: /\b(SKU|GlobalStandard|DataZoneStandard|DataZone|Pay-?as-?you-?go|PayGo|provisioned deployment|deployment type)\b/gi,
  region:
    /\b(East US|West US|West Europe|North Europe|Sweden Central|Norway East|norwayeast|swedencentral|region(?:al)? availability|available in (?:the )?following regions)\b/gi,
  version:
    /\b(GPT-[0-9](?:\.[0-9])?|GPT-4o|o1|o3|o4-mini|Claude\s?[0-9]|Gemini\s?[0-9]|text-embedding-[0-9]|api-version=\d{4}-\d{2}-\d{2})\b/gi,
  previewGa: /(public preview|private preview|in preview|\(preview\)|generally available|now available)/gi,
};
const SIGNAL_WEIGHTS = { tpmPtu: 3, price: 2, sku: 2, region: 2, version: 3, previewGa: 2 };
const PATH_BOOST = { 'cost-optimization': 5, platforms: 5 };

/** Skill name from a `skills/<skill>/references/...` path, else ''. */
export function skillOf(relpath) {
  const m = /^skills\/([^/]+)\/references\//.exec(relpath);
  return m ? m[1] : '';
}

/** Immediate folder under `references/`, else '' (file sits directly in references/). */
export function topFolder(relpath) {
  const m = /^skills\/[^/]+\/references\/([^/]+)\//.exec(relpath);
  return m ? m[1] : '';
}

/** Count volatility-signal hits in text; weighted sum = ranking score. */
export function scoreVolatility(text) {
  const signals = {};
  let score = 0;
  for (const [key, pat] of Object.entries(SIGNAL_PATTERNS)) {
    const matches = text.match(pat);
    const n = matches ? matches.length : 0;
    signals[key] = n;
    score += n * SIGNAL_WEIGHTS[key];
  }
  return { score, signals };
}

/** Path-based boost for volatility-dense folders (cost-optimization, platforms). */
export function pathBoost(relpath) {
  return PATH_BOOST[topFolder(relpath)] || 0;
}

/** Total ranking score for a file = content volatility + path boost. */
export function fileScore({ relpath, text }) {
  return scoreVolatility(text).score + pathBoost(relpath);
}

/**
 * Apportion `total` selections across skills: a floor per skill (balance) plus
 * the remainder distributed by volatility mass (oversample dense skills), using
 * largest-remainder apportionment. Deterministic — ties broken by skill name.
 */
export function allocateQuota(massBySkill, total, floorPerSkill) {
  const skills = Object.keys(massBySkill).sort();
  const quotas = {};
  for (const s of skills) quotas[s] = floorPerSkill;
  let remaining = total - floorPerSkill * skills.length;
  if (remaining <= 0) return quotas;
  const totalMass = skills.reduce((sum, s) => sum + massBySkill[s], 0);
  if (totalMass <= 0) {
    // no mass signal — distribute round-robin deterministically by skill name
    let i = 0;
    while (remaining > 0) {
      quotas[skills[i % skills.length]] += 1;
      remaining--;
      i++;
    }
    return quotas;
  }
  const shares = skills.map((s) => {
    const exact = (massBySkill[s] / totalMass) * remaining;
    const base = Math.floor(exact);
    return { s, base, frac: exact - base };
  });
  for (const sh of shares) {
    quotas[sh.s] += sh.base;
    remaining -= sh.base;
  }
  shares.sort((a, b) => b.frac - a.frac || (a.s < b.s ? -1 : 1));
  for (let i = 0; i < shares.length && remaining > 0; i++) {
    quotas[shares[i].s] += 1;
    remaining--;
  }
  return quotas;
}

const byScoreThenPath = (a, b) =>
  b.score - a.score || (a.file < b.file ? -1 : a.file > b.file ? 1 : 0);

/**
 * Split scored files into a volatile stratum (~volatileTarget, balanced across
 * skills, oversampling dense ones) and a control stratum (low-volatility files
 * from controlFolders). Deterministic.
 */
export function stratify(scored, cfg) {
  const controlSet = new Set();
  const control = scored
    .filter((f) => cfg.controlFolders.includes(f.topFolder) && f.score <= cfg.controlMaxScore)
    .sort((a, b) => a.score - b.score || (a.file < b.file ? -1 : a.file > b.file ? 1 : 0))
    .slice(0, cfg.controlTarget)
    .map((f) => {
      controlSet.add(f.file);
      return { ...f, stratum: 'control' };
    });

  const pool = scored.filter((f) => !controlSet.has(f.file) && f.skill);
  const bySkill = {};
  const massBySkill = {};
  for (const f of pool) {
    (bySkill[f.skill] ||= []).push(f);
    massBySkill[f.skill] = (massBySkill[f.skill] || 0) + f.score;
  }
  const quotas = allocateQuota(massBySkill, cfg.volatileTarget, cfg.floorPerSkill);
  const volatile = [];
  for (const skill of Object.keys(bySkill).sort()) {
    for (const f of bySkill[skill].sort(byScoreThenPath).slice(0, quotas[skill] || 0)) {
      volatile.push({ ...f, stratum: 'volatile' });
    }
  }
  volatile.sort(byScoreThenPath);
  return { volatile, control };
}

// --- CLI / frame assembly ----------------------------------------------------
const CFG = {
  volatileTarget: 45,
  floorPerSkill: 5,
  // Control = stable-claim folders (the real analogues of the contract's
  // "methodology/regulatory"): responsible-AI principles, Norwegian public-sector
  // governance/law, and architecture methodology. Filtered to low volatility score
  // so the control stratum measures the stable-claim sanity rate, not volatile churn.
  controlFolders: ['responsible-ai', 'norwegian-public-sector-governance', 'architecture'],
  controlMaxScore: 4,
  controlTarget: 10,
};

/** Invert url-registry → Map(reference_file -> sorted unique cited URLs). */
function invertRegistry() {
  const reg = JSON.parse(readFileSync(REGISTRY, 'utf8'));
  const inv = new Map();
  for (const [url, meta] of Object.entries(reg.urls || {})) {
    for (const rf of meta.reference_files || []) {
      if (!inv.has(rf)) inv.set(rf, new Set());
      inv.get(rf).add(url);
    }
  }
  return inv;
}

function scoreSourcedFiles() {
  const inv = invertRegistry();
  const scored = [];
  for (const [relpath, urlSet] of inv) {
    const abs = join(PLUGIN_ROOT, relpath);
    if (!existsSync(abs)) continue; // sourced file since deleted — skip
    const text = readFileSync(abs, 'utf8');
    const { score, signals } = scoreVolatility(text);
    scored.push({
      file: relpath,
      skill: skillOf(relpath),
      topFolder: topFolder(relpath),
      score: score + pathBoost(relpath),
      signals,
      citedUrls: [...urlSet].sort(),
    });
  }
  return scored;
}

function buildFrame() {
  const scored = scoreSourcedFiles();
  const { volatile, control } = stratify(scored, CFG);
  const stamp = new Date().toISOString();
  return {
    _meta: {
      created: stamp,
      method:
        'volatility-weighted stratified selection; deterministic (no random). ' +
        'Scorer is a ranking heuristic for sample selection, not the correctness classifier.',
      population: scored.length,
      volatile_count: volatile.length,
      control_count: control.length,
      config: CFG,
      signal_weights: SIGNAL_WEIGHTS,
    },
    volatile,
    control,
  };
}

function summarize(frame) {
  const perSkill = {};
  for (const f of [...frame.volatile, ...frame.control]) {
    const k = `${f.skill}/${f.stratum}`;
    perSkill[k] = (perSkill[k] || 0) + 1;
  }
  console.log(`Sourced population: ${frame._meta.population} files`);
  console.log(`Volatile stratum:   ${frame.volatile.length}`);
  console.log(`Control stratum:    ${frame.control.length}`);
  console.log('\nPer skill / stratum:');
  for (const k of Object.keys(perSkill).sort()) console.log(`  ${k.padEnd(34)} ${perSkill[k]}`);
  console.log('\nTop 10 volatile (score · file):');
  for (const f of frame.volatile.slice(0, 10)) {
    console.log(`  ${String(f.score).padStart(4)}  ${f.file}`);
  }
}

function main() {
  const args = process.argv.slice(2);
  const frame = buildFrame();
  if (args.includes('--json')) {
    process.stdout.write(JSON.stringify(frame, null, 2) + '\n');
    return;
  }
  summarize(frame);
  if (args.includes('--write')) {
    atomicWriteJson(OUT_FILE, frame);
    console.log(`\nWrote ${OUT_FILE}`);
  } else {
    console.log('\n(dry run — pass --write to persist data/fase0-sample-frame.json)');
  }
}

if (import.meta.url === `file://${process.argv[1]}`) main();