ms-ai-architect/scripts/kb-eval/lib/base-rate.mjs

// scripts/kb-eval/lib/base-rate.mjs — Fase 0 base-rate aggregation (deterministic).
//
// Turns the per-claim subagent verdicts (gold-correctness-set) into a defensible
// base-rate report. Pure functions only — no I/O, no Date.now/Math.random.
//
// Verdict vocabulary (set by the per-file verification subagents):
//   correct   — a fetched learn.microsoft.com page states the claimed value
//   outdated  — fetched source shows a different, superseded value (time drift)
//   wrong     — fetched source contradicts the claim; it was never correct
//   unsourced — no fetchable MS Learn page states the value (cannot verify)
//
// "Genuine errors" = outdated + wrong. unsourced is NOT an error (it is the
// unverifiable mass: prices on JS-rendered Azure pages a fetch-based judge also
// cannot reach). The verifiable error rate therefore EXCLUDES unsourced from its
// denominator. The gate-critical figure is errorsJudgeUnique: genuine errors
// whose cited source lastmod did NOT change since the file date — the only class
// a correctness judge catches that the existing staleness loop would miss.

const ERROR_VERDICTS = new Set(['outdated', 'wrong']);
const VERDICTS = ['correct', 'outdated', 'wrong', 'unsourced'];

// Parse the "**Last updated:** YYYY-MM[-DD]" header. Month-only normalises to the
// first of the month (the conservative choice for the gate: it makes more sources
// count as "changed since the file", which understates — never overstates — the
// judge's unique value). Returns an ISO date string or null.
export function parseLastUpdated(text) {
  if (!text) return null;
  const m = text.match(/\*\*Last updated:\*\*\s*(\d{4})-(\d{2})(?:-(\d{2}))?/i);
  if (!m) return null;
  const [, y, mo, d] = m;
  return `${y}-${mo}-${d || '01'}`;
}

// Wilson score interval for a binomial proportion. Returns {p, low, high}.
// Empty sample -> all zero (never NaN). Default z = 1.96 (95%).
export function wilson(successes, n, z = 1.96) {
  if (!n || n <= 0) return { p: 0, low: 0, high: 0 };
  const phat = successes / n;
  const z2 = z * z;
  const denom = 1 + z2 / n;
  const center = (phat + z2 / (2 * n)) / denom;
  const margin =
    (z / denom) * Math.sqrt((phat * (1 - phat)) / n + z2 / (4 * n * n));
  return {
    p: phat,
    low: Math.max(0, center - margin),
    high: Math.min(1, center + margin),
  };
}

// Did any of the file's cited sources change on the MS Learn sitemap after the
// file's own "last updated" date? This is exactly the signal the existing KB
// staleness loop sees (it polls sitemap_lastmod per cited URL). Returns:
//   true  — at least one cited source has sitemap_lastmod > fileDate
//   false — no cited source is newer (staleness loop would NOT flag the file)
//   null  — fileDate unknown (cannot determine)
export function fileLastmodChanged(citedUrls, registry, fileDate) {
  if (!fileDate) return null;
  const urls = (registry && registry.urls) || {};
  for (const u of citedUrls || []) {
    const entry = urls[u];
    if (entry && entry.sitemap_lastmod && entry.sitemap_lastmod > fileDate) {
      return true;
    }
  }
  return false;
}

function emptyBucket() {
  return {
    total: 0,
    byVerdict: { correct: 0, outdated: 0, wrong: 0, unsourced: 0 },
    errors: 0,
    verifiable: 0,
    unsourced: 0,
    errorsStalenessCatchable: 0,
    errorsJudgeUnique: 0,
  };
}

function tally(bucket, claim) {
  bucket.total += 1;
  const v = claim.verdict;
  if (v in bucket.byVerdict) bucket.byVerdict[v] += 1;
  if (v === 'unsourced') {
    bucket.unsourced += 1;
  } else {
    bucket.verifiable += 1;
  }
  if (ERROR_VERDICTS.has(v)) {
    bucket.errors += 1;
    // lastmod_changed === true  -> staleness loop already flags this file
    // anything else (false/null) -> only a correctness judge would catch it
    if (claim.lastmod_changed === true) bucket.errorsStalenessCatchable += 1;
    else bucket.errorsJudgeUnique += 1;
  }
}

function finalize(bucket) {
  bucket.errorRate = bucket.verifiable ? bucket.errors / bucket.verifiable : 0;
  bucket.errorRateWilson = wilson(bucket.errors, bucket.verifiable);
  bucket.unsourcedRate = bucket.total ? bucket.unsourced / bucket.total : 0;
  return bucket;
}

// Aggregate a flat list of claims into overall + per-skill + per-stratum +
// per-claim_type base-rate buckets. Each claim needs: verdict, skill, stratum,
// claim_type, lastmod_changed.
export function computeBaseRate(claims) {
  const overall = emptyBucket();
  const bySkill = {};
  const byStratum = {};
  const byClaimType = {};
  for (const c of claims) {
    tally(overall, c);
    (bySkill[c.skill] ||= emptyBucket()), tally(bySkill[c.skill], c);
    (byStratum[c.stratum] ||= emptyBucket()), tally(byStratum[c.stratum], c);
    (byClaimType[c.claim_type] ||= emptyBucket()), tally(byClaimType[c.claim_type], c);
  }
  finalize(overall);
  for (const b of Object.values(bySkill)) finalize(b);
  for (const b of Object.values(byStratum)) finalize(b);
  for (const b of Object.values(byClaimType)) finalize(b);
  return { overall, bySkill, byStratum, byClaimType, _verdicts: VERDICTS };
}