ms-ai-architect/scripts/kb-eval/lib/judge-bakeoff.mjs

// scripts/kb-eval/lib/judge-bakeoff.mjs — S1 judge bake-off aggregation (deterministic).
//
// De-risks the Fase 3 correctness judge BEFORE production scaffolding (S3) is built:
// grade a per-claim groundedness judge against the frozen gold set and decide whether
// it beats the cheap staleness baseline (recall 0/40) by enough to justify ~2700
// non-batchable microsoft_docs_fetch calls per full pass. Pure functions only — no
// I/O, no Date.now/Math.random.
//
// Eval population P = volatile stratum + fetchable claim_type (price excluded). This
// is the only population that matters: it is where the real errors concentrate, and
// scoping to it structurally avoids the "inverted leverage" trap (a judge that wins
// only by auto-scoring cheap stable claims it was never at risk on).
//
// Detection confusion matrix vs the gold verdicts:
//   gold positive  = verdict ∈ {outdated, wrong}   (a real error the judge should catch)
//   gold negative  = verdict === 'correct'
//   excluded       = verdict === 'unsourced'        (no ground-truth value to grade against)
//   predicted positive (flag) = judge_verdict === 'not_grounded'
// judge_verdict vocabulary: 'grounded' | 'not_grounded' | 'source_silent'.

import { wilson } from './base-rate.mjs';

export const FETCHABLE_TYPES = new Set([
  'taxonomy', 'status', 'version', 'tpm', 'sku', 'region',
]);
const ERROR_VERDICTS = new Set(['outdated', 'wrong']);

// P = the claims the bake-off is measured on. A claim is in P iff it is volatile and
// of a fetchable claim_type (price is excluded — 74% unsourced, JS-rendered Azure
// pages defeat microsoft_docs_fetch, so a fetch-based judge cannot reach it either).
export function evalPopulation(claims) {
  return claims.filter(
    (c) => c.stratum === 'volatile' && FETCHABLE_TYPES.has(c.claim_type),
  );
}

// The BLIND manifest handed to the judge subagents: every claim in P, stripped to
// the fields the judge is allowed to see. The gold verdict, notes, lastmod_changed
// and stratum are deliberately withheld so the eval is blind (no label leakage).
// All of P is included (unsourced too) — the judge must not know which claims the
// human could not source; reproducing that boundary as source_silent is itself a
// measured signal.
const BLIND_FIELDS = ['id', 'file', 'skill', 'claim', 'claim_type', 'evidence_url'];
export function blindClaims(claims) {
  return evalPopulation(claims).map((c) => {
    const out = {};
    for (const k of BLIND_FIELDS) out[k] = c[k];
    return out;
  });
}

export function stalenessFlag(claim) {
  return claim.lastmod_changed === true;
}
export function judgeFlag(claim) {
  return claim.judge_verdict === 'not_grounded';
}
export function hybridFlag(claim) {
  return stalenessFlag(claim) || judgeFlag(claim);
}

// Grade one detection arm over a list of *verifiable* claims (gold correct/outdated/
// wrong only — unsourced must be filtered out before calling). flagFn(claim) -> bool
// is the arm's predicted-error rule. Returns the confusion matrix plus precision /
// recall / F1. precision is null when nothing is flagged (0/0, not NaN); recall is
// null when there are no positives at all.
export function gradeArm(verifiableClaims, flagFn) {
  let tp = 0, fp = 0, fn = 0, tn = 0;
  for (const c of verifiableClaims) {
    const isError = ERROR_VERDICTS.has(c.verdict);
    const flagged = flagFn(c);
    if (isError && flagged) tp += 1;
    else if (!isError && flagged) fp += 1;
    else if (isError && !flagged) fn += 1;
    else tn += 1;
  }
  const positives = tp + fn;
  const negatives = fp + tn;
  const flagged = tp + fp;
  const precision = flagged ? tp / flagged : null;
  const recall = positives ? tp / positives : null;
  const f1 =
    precision != null && recall != null && precision + recall > 0
      ? (2 * precision * recall) / (precision + recall)
      : null;
  // Wilson 95% bands surface sampling noise — the positive sample is small (≈38),
  // so a point estimate near the threshold should not be read as crisp. Bands are
  // reported context; the gate itself uses the point estimate.
  const recallWilson = positives ? wilson(tp, positives) : null;
  const precisionWilson = flagged ? wilson(tp, flagged) : null;
  return {
    tp, fp, fn, tn, positives, negatives, flagged,
    precision, recall, f1, recallWilson, precisionWilson,
  };
}

// The PRE-REGISTERED gate. thresholds = { minRecall, minPrecision }. Pass requires
// the judge to clear both thresholds AND strictly beat the staleness baseline's
// recall (the directional point of S1: staleness recall is 0/40, so any positive
// recall beats it — but the threshold guards against a judge too weak to justify
// the fetch cost). A null judge precision (judge flagged nothing) cannot clear a
// positive minPrecision.
export function gateDecision(judgeArm, stalenessArm, thresholds) {
  const { minRecall, minPrecision } = thresholds;
  const recall = judgeArm.recall ?? 0;
  const precision = judgeArm.precision; // may be null
  const staleRecall = stalenessArm.recall ?? 0;

  const recallOk = recall >= minRecall;
  const precisionOk = precision != null && precision >= minPrecision;
  const beatsStaleness = recall > staleRecall;
  const pass = recallOk && precisionOk && beatsStaleness;

  const reasons = [];
  if (!recallOk) reasons.push(`recall ${fmt(recall)} < minRecall ${minRecall}`);
  if (!precisionOk) {
    reasons.push(
      precision == null
        ? 'judge flagged nothing (precision undefined)'
        : `precision ${fmt(precision)} < minPrecision ${minPrecision}`,
    );
  }
  if (!beatsStaleness) {
    reasons.push(`recall ${fmt(recall)} does not beat staleness ${fmt(staleRecall)}`);
  }
  if (pass) reasons.push('all criteria met');
  return { pass, recallOk, precisionOk, beatsStaleness, thresholds, reasons };
}

function fmt(x) {
  return x == null ? 'n/a' : x.toFixed(3);
}

// source_silent is the judge's "I fetched but the page does not state this value"
// verdict. It is diagnostic, not a flag. Two populations matter:
//   - on verifiable claims (the human DID find a value): every source_silent is a
//     judge verification miss. Split by whether the gold value was correct vs an error
//     (a source_silent on a real error is a missed catch via "can't verify").
//   - on unsourced-in-P claims (the human ALSO found no value): source_silent here is
//     agreement — a good sign the judge reproduces the unverifiable boundary.
function sourceSilentDiagnostics(P) {
  let onVerifiableNegative = 0;
  let onVerifiableError = 0;
  let agreesWithUnsourced = 0;
  let disagreesWithUnsourced = 0;
  for (const c of P) {
    const silent = c.judge_verdict === 'source_silent';
    if (c.verdict === 'unsourced') {
      if (silent) agreesWithUnsourced += 1;
      else disagreesWithUnsourced += 1;
    } else if (silent) {
      if (ERROR_VERDICTS.has(c.verdict)) onVerifiableError += 1;
      else onVerifiableNegative += 1;
    }
  }
  return {
    onVerifiableNegative,
    onVerifiableError,
    agreesWithUnsourced,
    disagreesWithUnsourced,
  };
}

// Per claim_type judge confusion matrix over the verifiable subset (so the report can
// show where the judge earns its keep — sku/version/tpm carry the highest error rates).
function judgeByClaimType(verifiable) {
  const byType = {};
  for (const c of verifiable) {
    (byType[c.claim_type] ||= []).push(c);
  }
  const out = {};
  for (const [t, arr] of Object.entries(byType)) {
    out[t] = gradeArm(arr, judgeFlag);
  }
  return out;
}

// Top-level: join already done by the caller (each claim carries judge_verdict).
// Filters to P, splits verifiable vs unsourced, grades all three arms over the same
// verifiable set, computes diagnostics + the gate, returns one report object.
export function computeBakeoff(joinedClaims, thresholds) {
  const P = evalPopulation(joinedClaims);
  const verifiable = P.filter((c) => c.verdict !== 'unsourced');
  const unsourcedInP = P.length - verifiable.length;

  const arms = {
    staleness: gradeArm(verifiable, stalenessFlag),
    judge: gradeArm(verifiable, judgeFlag),
    hybrid: gradeArm(verifiable, hybridFlag),
  };
  const gate = gateDecision(arms.judge, arms.staleness, thresholds);

  return {
    population: {
      total: P.length,
      verifiable: verifiable.length,
      positives: verifiable.filter((c) => ERROR_VERDICTS.has(c.verdict)).length,
      negatives: verifiable.filter((c) => c.verdict === 'correct').length,
      unsourcedInP,
    },
    arms,
    sourceSilent: sourceSilentDiagnostics(P),
    byClaimType: judgeByClaimType(verifiable),
    gate,
  };
}