feat(ms-ai-architect): Fase 0 steg 2 — verifiserings-fan-out (373 påstander) + gull-sett m/ deterministisk lastmod-join + TDD [skip-docs]

15 batcher (Opus 4.8 xhigh, read-only) verifiserte 55 sample-frame-filer mot live
MS Learn (strikt v2 bevis-krav: correct/outdated/wrong krever hentet learn-sitat,
ellers unsourced). 373 påstander: 259 correct, 29 outdated, 11 wrong, 74 unsourced.
40 reelle feil. lastmod_changed joinet i kode (registry sitemap_lastmod vs filens
dato): 0 filer ville staleness-loopen flagget -> alle 40 feil er judge-unike (rå
funn, krever tolkning i base-rate-rapporten). lib/base-rate.mjs (Wilson, lastmod,
aggregering) TDD 15 tester. Suite 538/538. Steg 3 (compute-base-rate + gate) gjenstår.

scripts/kb-eval/build-gold-set.mjs

@@ -0,0 +1,105 @@
+#!/usr/bin/env node
+// build-gold-set.mjs — Fase 0, steg 2/3 glue: assemble the gold correctness set.
+//
+// Consolidates the per-batch verification returns (scripts/kb-eval/data/
+// fase0-returns/batch-*.json — produced by Opus 4.8 subagents that checked each
+// volatile claim against live MS Learn) into a single, reusable gold set, and
+// joins the DETERMINISTIC lastmod_changed signal in code (never an LLM call):
+// for each file, did any cited source change on the MS Learn sitemap after the
+// file's own "last updated" date? That is exactly what the existing KB staleness
+// loop sees, so it tells us which genuine errors a correctness judge would catch
+// that the staleness loop would miss.
+//
+// The non-trivial logic (date parse, lastmod comparison) lives in tested
+// lib/base-rate.mjs; this CLI is thin wiring over it.
+//
+// Usage:  node scripts/kb-eval/build-gold-set.mjs [--write]
+//   (default: print summary; --write: persist gold-correctness-set.json)
+
+import fs from 'node:fs';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { parseLastUpdated, fileLastmodChanged } from './lib/base-rate.mjs';
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+const ROOT = path.resolve(__dirname, '..', '..');
+const DATA = path.join(__dirname, 'data');
+const RETURNS = path.join(DATA, 'fase0-returns');
+
+const frame = JSON.parse(fs.readFileSync(path.join(DATA, 'fase0-sample-frame.json'), 'utf8'));
+const registry = JSON.parse(
+  fs.readFileSync(path.join(ROOT, 'scripts/kb-update/data/url-registry.json'), 'utf8'),
+);
+
+// file -> citedUrls (from the deterministic sample frame)
+const citedByFile = {};
+for (const e of [...frame.volatile, ...frame.control]) citedByFile[e.file] = e.citedUrls;
+
+// file -> { date, lastmod_changed } (read each file once)
+const fileMeta = {};
+function metaFor(file) {
+  if (fileMeta[file]) return fileMeta[file];
+  let date = null;
+  try {
+    date = parseLastUpdated(fs.readFileSync(path.join(ROOT, file), 'utf8'));
+  } catch {
+    date = null;
+  }
+  const changed = fileLastmodChanged(citedByFile[file] || [], registry, date);
+  return (fileMeta[file] = { date, lastmod_changed: changed });
+}
+
+// relpath after skills/<skill>/references/ for a compact, stable claim id
+function relOf(file, skill) {
+  const prefix = `skills/${skill}/references/`;
+  return file.startsWith(prefix) ? file.slice(prefix.length) : file;
+}
+
+const claims = [];
+const batchFiles = fs.readdirSync(RETURNS).filter((f) => /^batch-\d+\.json$/.test(f)).sort();
+for (const bf of batchFiles) {
+  const batch = JSON.parse(fs.readFileSync(path.join(RETURNS, bf), 'utf8'));
+  for (const c of batch.claims) {
+    const meta = metaFor(c.file);
+    claims.push({
+      id: `${c.skill}/${relOf(c.file, c.skill)}#${c.n}`,
+      file: c.file,
+      skill: c.skill,
+      stratum: c.stratum,
+      claim: c.claim,
+      claim_type: c.claim_type,
+      verdict: c.verdict,
+      evidence_url: c.evidence_url ?? null,
+      lastmod_changed: meta.lastmod_changed,
+      file_last_updated: meta.date,
+      notes: c.notes ?? '',
+    });
+  }
+}
+
+const goldSet = {
+  _meta: {
+    created: '2026-06-26',
+    method:
+      'Per-file volatile-claim verification against live MS Learn by Opus 4.8 subagents (strict v2 evidence rule: correct/outdated/wrong require a fetched learn.microsoft.com quote; otherwise unsourced). lastmod_changed joined deterministically in code from url-registry sitemap_lastmod vs the file last-updated date.',
+    sample_frame: 'fase0-sample-frame.json',
+    raw_returns: 'fase0-returns/batch-*.json',
+    note_evidence_quote:
+      'Per-claim evidence_quote omitted here to bound size; full verbatim quotes live in the session transcript. notes + evidence_url capture each verdict basis.',
+    claim_count: claims.length,
+  },
+  claims,
+};
+
+if (process.argv.includes('--write')) {
+  const out = path.join(DATA, 'gold-correctness-set.json');
+  fs.writeFileSync(out, JSON.stringify(goldSet, null, 2) + '\n');
+  console.log(`wrote ${out} (${claims.length} claims)`);
+} else {
+  const filesWithChange = Object.entries(fileMeta).filter(([, m]) => m.lastmod_changed === true).length;
+  const filesNoChange = Object.entries(fileMeta).filter(([, m]) => m.lastmod_changed === false).length;
+  const filesUnknown = Object.entries(fileMeta).filter(([, m]) => m.lastmod_changed === null).length;
+  console.log(`claims: ${claims.length} | files: ${Object.keys(fileMeta).length}`);
+  console.log(`file lastmod_changed: true=${filesWithChange} false=${filesNoChange} unknown(null date)=${filesUnknown}`);
+  console.log('(dry run — pass --write to persist gold-correctness-set.json)');
+}

scripts/kb-eval/lib/base-rate.mjs

@@ -0,0 +1,125 @@
+// scripts/kb-eval/lib/base-rate.mjs — Fase 0 base-rate aggregation (deterministic).
+//
+// Turns the per-claim subagent verdicts (gold-correctness-set) into a defensible
+// base-rate report. Pure functions only — no I/O, no Date.now/Math.random.
+//
+// Verdict vocabulary (set by the per-file verification subagents):
+//   correct   — a fetched learn.microsoft.com page states the claimed value
+//   outdated  — fetched source shows a different, superseded value (time drift)
+//   wrong     — fetched source contradicts the claim; it was never correct
+//   unsourced — no fetchable MS Learn page states the value (cannot verify)
+//
+// "Genuine errors" = outdated + wrong. unsourced is NOT an error (it is the
+// unverifiable mass: prices on JS-rendered Azure pages a fetch-based judge also
+// cannot reach). The verifiable error rate therefore EXCLUDES unsourced from its
+// denominator. The gate-critical figure is errorsJudgeUnique: genuine errors
+// whose cited source lastmod did NOT change since the file date — the only class
+// a correctness judge catches that the existing staleness loop would miss.
+
+const ERROR_VERDICTS = new Set(['outdated', 'wrong']);
+const VERDICTS = ['correct', 'outdated', 'wrong', 'unsourced'];
+
+// Parse the "**Last updated:** YYYY-MM[-DD]" header. Month-only normalises to the
+// first of the month (the conservative choice for the gate: it makes more sources
+// count as "changed since the file", which understates — never overstates — the
+// judge's unique value). Returns an ISO date string or null.
+export function parseLastUpdated(text) {
+  if (!text) return null;
+  const m = text.match(/\*\*Last updated:\*\*\s*(\d{4})-(\d{2})(?:-(\d{2}))?/i);
+  if (!m) return null;
+  const [, y, mo, d] = m;
+  return `${y}-${mo}-${d || '01'}`;
+}
+
+// Wilson score interval for a binomial proportion. Returns {p, low, high}.
+// Empty sample -> all zero (never NaN). Default z = 1.96 (95%).
+export function wilson(successes, n, z = 1.96) {
+  if (!n || n <= 0) return { p: 0, low: 0, high: 0 };
+  const phat = successes / n;
+  const z2 = z * z;
+  const denom = 1 + z2 / n;
+  const center = (phat + z2 / (2 * n)) / denom;
+  const margin =
+    (z / denom) * Math.sqrt((phat * (1 - phat)) / n + z2 / (4 * n * n));
+  return {
+    p: phat,
+    low: Math.max(0, center - margin),
+    high: Math.min(1, center + margin),
+  };
+}
+
+// Did any of the file's cited sources change on the MS Learn sitemap after the
+// file's own "last updated" date? This is exactly the signal the existing KB
+// staleness loop sees (it polls sitemap_lastmod per cited URL). Returns:
+//   true  — at least one cited source has sitemap_lastmod > fileDate
+//   false — no cited source is newer (staleness loop would NOT flag the file)
+//   null  — fileDate unknown (cannot determine)
+export function fileLastmodChanged(citedUrls, registry, fileDate) {
+  if (!fileDate) return null;
+  const urls = (registry && registry.urls) || {};
+  for (const u of citedUrls || []) {
+    const entry = urls[u];
+    if (entry && entry.sitemap_lastmod && entry.sitemap_lastmod > fileDate) {
+      return true;
+    }
+  }
+  return false;
+}
+
+function emptyBucket() {
+  return {
+    total: 0,
+    byVerdict: { correct: 0, outdated: 0, wrong: 0, unsourced: 0 },
+    errors: 0,
+    verifiable: 0,
+    unsourced: 0,
+    errorsStalenessCatchable: 0,
+    errorsJudgeUnique: 0,
+  };
+}
+
+function tally(bucket, claim) {
+  bucket.total += 1;
+  const v = claim.verdict;
+  if (v in bucket.byVerdict) bucket.byVerdict[v] += 1;
+  if (v === 'unsourced') {
+    bucket.unsourced += 1;
+  } else {
+    bucket.verifiable += 1;
+  }
+  if (ERROR_VERDICTS.has(v)) {
+    bucket.errors += 1;
+    // lastmod_changed === true  -> staleness loop already flags this file
+    // anything else (false/null) -> only a correctness judge would catch it
+    if (claim.lastmod_changed === true) bucket.errorsStalenessCatchable += 1;
+    else bucket.errorsJudgeUnique += 1;
+  }
+}
+
+function finalize(bucket) {
+  bucket.errorRate = bucket.verifiable ? bucket.errors / bucket.verifiable : 0;
+  bucket.errorRateWilson = wilson(bucket.errors, bucket.verifiable);
+  bucket.unsourcedRate = bucket.total ? bucket.unsourced / bucket.total : 0;
+  return bucket;
+}
+
+// Aggregate a flat list of claims into overall + per-skill + per-stratum +
+// per-claim_type base-rate buckets. Each claim needs: verdict, skill, stratum,
+// claim_type, lastmod_changed.
+export function computeBaseRate(claims) {
+  const overall = emptyBucket();
+  const bySkill = {};
+  const byStratum = {};
+  const byClaimType = {};
+  for (const c of claims) {
+    tally(overall, c);
+    (bySkill[c.skill] ||= emptyBucket()), tally(bySkill[c.skill], c);
+    (byStratum[c.stratum] ||= emptyBucket()), tally(byStratum[c.stratum], c);
+    (byClaimType[c.claim_type] ||= emptyBucket()), tally(byClaimType[c.claim_type], c);
+  }
+  finalize(overall);
+  for (const b of Object.values(bySkill)) finalize(b);
+  for (const b of Object.values(byStratum)) finalize(b);
+  for (const b of Object.values(byClaimType)) finalize(b);
+  return { overall, bySkill, byStratum, byClaimType, _verdicts: VERDICTS };
+}

tests/kb-eval/test-base-rate.test.mjs

@@ -0,0 +1,133 @@
+// tests/kb-eval/test-base-rate.test.mjs
+// Unit tests for the Fase 0 base-rate aggregation lib.
+//
+// These functions turn the per-claim subagent verdicts (gold-correctness-set)
+// into a defensible base-rate report: a verifiable error rate with a Wilson 95%
+// score interval, broken down per skill / stratum / claim_type, plus the gate-
+// critical "errors a correctness judge would catch over the existing staleness
+// loop" (= genuine errors whose cited source lastmod did NOT change since the
+// file's date). All computation is deterministic.
+
+import { test } from 'node:test';
+import assert from 'node:assert/strict';
+import {
+  parseLastUpdated,
+  wilson,
+  fileLastmodChanged,
+  computeBaseRate,
+} from '../../scripts/kb-eval/lib/base-rate.mjs';
+
+// --- parseLastUpdated --------------------------------------------------------
+test('parseLastUpdated — full YYYY-MM-DD header', () => {
+  assert.equal(
+    parseLastUpdated('# Title\n\n**Last updated:** 2026-06-24 (research via MCP)\n'),
+    '2026-06-24',
+  );
+});
+test('parseLastUpdated — month-only header normalises to first of month', () => {
+  assert.equal(
+    parseLastUpdated('**Last updated:** 2026-06 | Priser verifisert juni 2026\n'),
+    '2026-06-01',
+  );
+});
+test('parseLastUpdated — no header returns null', () => {
+  assert.equal(parseLastUpdated('# Title\n\nNo date here.\n'), null);
+});
+
+// --- wilson ------------------------------------------------------------------
+test('wilson — empty sample is all-zero, never NaN', () => {
+  const w = wilson(0, 0);
+  assert.equal(w.p, 0);
+  assert.equal(w.low, 0);
+  assert.equal(w.high, 0);
+});
+test('wilson — all successes: point estimate 1, upper bound 1, lower < 1', () => {
+  const w = wilson(10, 10);
+  assert.equal(w.p, 1);
+  assert.ok(Math.abs(w.high - 1) < 1e-9);
+  assert.ok(w.low < 1 && w.low > 0.6);
+});
+test('wilson — 50/100 matches known 95% interval [0.404, 0.596]', () => {
+  const w = wilson(50, 100);
+  assert.equal(w.p, 0.5);
+  assert.ok(Math.abs(w.low - 0.4038) < 0.01, `low=${w.low}`);
+  assert.ok(Math.abs(w.high - 0.5962) < 0.01, `high=${w.high}`);
+});
+
+// --- fileLastmodChanged ------------------------------------------------------
+const REG = {
+  urls: {
+    'https://l/a': { sitemap_lastmod: '2026-05-01' },
+    'https://l/b': { sitemap_lastmod: '2024-01-01' },
+  },
+};
+test('fileLastmodChanged — a cited source changed after the file date -> true', () => {
+  assert.equal(fileLastmodChanged(['https://l/a', 'https://l/b'], REG, '2026-01-01'), true);
+});
+test('fileLastmodChanged — no cited source newer than file date -> false', () => {
+  assert.equal(fileLastmodChanged(['https://l/b'], REG, '2026-01-01'), false);
+});
+test('fileLastmodChanged — unknown file date -> null', () => {
+  assert.equal(fileLastmodChanged(['https://l/a'], REG, null), null);
+});
+test('fileLastmodChanged — cited url absent from registry is ignored', () => {
+  assert.equal(fileLastmodChanged(['https://l/missing'], REG, '2026-01-01'), false);
+});
+
+// --- computeBaseRate ---------------------------------------------------------
+// Synthetic gold set: 10 claims across two skills and two strata.
+const CLAIMS = [
+  // skill X, volatile
+  { skill: 'x', stratum: 'volatile', claim_type: 'price', verdict: 'correct', lastmod_changed: false },
+  { skill: 'x', stratum: 'volatile', claim_type: 'version', verdict: 'outdated', lastmod_changed: true },  // staleness-catchable
+  { skill: 'x', stratum: 'volatile', claim_type: 'sku', verdict: 'wrong', lastmod_changed: false },        // judge-unique
+  { skill: 'x', stratum: 'volatile', claim_type: 'price', verdict: 'unsourced', lastmod_changed: false },
+  { skill: 'x', stratum: 'volatile', claim_type: 'status', verdict: 'outdated', lastmod_changed: false },  // judge-unique
+  // skill y, volatile
+  { skill: 'y', stratum: 'volatile', claim_type: 'tpm', verdict: 'correct', lastmod_changed: true },
+  { skill: 'y', stratum: 'volatile', claim_type: 'price', verdict: 'unsourced', lastmod_changed: null },
+  // skill y, control
+  { skill: 'y', stratum: 'control', claim_type: 'taxonomy', verdict: 'correct', lastmod_changed: false },
+  { skill: 'y', stratum: 'control', claim_type: 'taxonomy', verdict: 'correct', lastmod_changed: false },
+  { skill: 'y', stratum: 'control', claim_type: 'status', verdict: 'wrong', lastmod_changed: true },        // staleness-catchable
+];
+
+test('computeBaseRate — overall verdict tally and totals', () => {
+  const r = computeBaseRate(CLAIMS);
+  assert.equal(r.overall.total, 10);
+  assert.equal(r.overall.byVerdict.correct, 4);
+  assert.equal(r.overall.byVerdict.outdated, 2);
+  assert.equal(r.overall.byVerdict.wrong, 2);
+  assert.equal(r.overall.byVerdict.unsourced, 2);
+});
+test('computeBaseRate — verifiable error rate excludes unsourced from denominator', () => {
+  const r = computeBaseRate(CLAIMS);
+  // errors = outdated+wrong = 4; verifiable = total - unsourced = 8
+  assert.equal(r.overall.errors, 4);
+  assert.equal(r.overall.verifiable, 8);
+  assert.ok(Math.abs(r.overall.errorRate - 0.5) < 1e-9);
+  assert.equal(r.overall.unsourced, 2);
+  assert.ok(Math.abs(r.overall.unsourcedRate - 0.2) < 1e-9);
+  // Wilson band present and brackets the point estimate
+  assert.ok(r.overall.errorRateWilson.low < 0.5 && r.overall.errorRateWilson.high > 0.5);
+});
+test('computeBaseRate — judge-unique errors = errors with lastmod_changed !== true', () => {
+  const r = computeBaseRate(CLAIMS);
+  // errors: outdated(true)=staleness, wrong(false)=judge, outdated(false)=judge, wrong(true)=staleness
+  assert.equal(r.overall.errorsStalenessCatchable, 2); // lastmod_changed === true
+  assert.equal(r.overall.errorsJudgeUnique, 2);        // lastmod_changed !== true
+});
+test('computeBaseRate — per-stratum split (volatile vs control)', () => {
+  const r = computeBaseRate(CLAIMS);
+  assert.equal(r.byStratum.volatile.total, 7);
+  assert.equal(r.byStratum.volatile.errors, 3);
+  assert.equal(r.byStratum.control.total, 3);
+  assert.equal(r.byStratum.control.errors, 1);
+});
+test('computeBaseRate — per-skill breakdown present', () => {
+  const r = computeBaseRate(CLAIMS);
+  assert.equal(r.bySkill.x.total, 5);
+  assert.equal(r.bySkill.x.errors, 3);
+  assert.equal(r.bySkill.y.total, 5);
+  assert.equal(r.bySkill.y.errors, 1);
+});