ms-ai-architect/scripts/kb-eval/compute-base-rate.mjs

#!/usr/bin/env node
// compute-base-rate.mjs — Fase 0, steg 3 glue: turn the gold correctness set into
// a defensible base-rate report (the directional input for the Fase 3 gate).
//
// All non-trivial math (verifiable error rate, Wilson bands, the staleness-
// catchable vs judge-unique split) lives in tested lib/base-rate.mjs. This CLI is
// thin wiring: read gold-correctness-set.json -> computeBaseRate() -> write a
// machine-readable .json and a human-readable .md. Same shape as build-gold-set.mjs.
//
// Usage:  node scripts/kb-eval/compute-base-rate.mjs [--write]
//   (default: print the overall + per-stratum summary; --write persists
//    data/base-rate-report.json and data/base-rate-report.md)

import fs from 'node:fs';
import path from 'node:path';
import { fileURLToPath } from 'node:url';
import { computeBaseRate } from './lib/base-rate.mjs';

const __dirname = path.dirname(fileURLToPath(import.meta.url));
const DATA = path.join(__dirname, 'data');

const gold = JSON.parse(fs.readFileSync(path.join(DATA, 'gold-correctness-set.json'), 'utf8'));
const claims = gold.claims;
const report = computeBaseRate(claims);

const pct = (x) => `${(x * 100).toFixed(1)}%`;
const band = (w) => `[${pct(w.low)}, ${pct(w.high)}]`;

// One row of the per-dimension tables. b = a finalized bucket from the lib.
function row(key, b) {
  const v = b.byVerdict;
  return `| ${key} | ${b.total} | ${v.correct} | ${v.outdated} | ${v.wrong} | ${v.unsourced} | ${b.errors}/${b.verifiable} = ${pct(b.errorRate)} | ${band(b.errorRateWilson)} | ${b.errorsJudgeUnique} |`;
}
const HEAD =
  '| key | n | correct | outdated | wrong | unsourced | err-rate (verifiable) | Wilson 95% | judge-unique |\n' +
  '|---|---|---|---|---|---|---|---|---|';

// Sort dimension entries by descending total so the heaviest buckets read first.
function table(title, byKey) {
  const rows = Object.entries(byKey)
    .sort((a, b) => b[1].total - a[1].total)
    .map(([k, b]) => row(k, b));
  return `### ${title}\n\n${HEAD}\n${rows.join('\n')}\n`;
}

const o = report.overall;
const md = `# Base-rate-rapport — Fase 0 (KB korrekthet)

_Generert deterministisk av \`compute-base-rate.mjs\` over \`gold-correctness-set.json\`. Tall fra testet \`lib/base-rate.mjs\` (15 tester). Ikke rediger for hånd — regenerer._

**Gull-sett:** ${claims.length} påstander · metode: ${gold._meta?.method ? 'se gold-correctness-set.json `_meta.method`' : 'n/a'}

## Verdict-vokabular

- **correct** — en hentet learn.microsoft.com-side oppgir den påståtte verdien
- **outdated** — hentet kilde viser en annen, erstattet verdi (tidsdrift)
- **wrong** — hentet kilde motsier påstanden; den var aldri korrekt
- **unsourced** — ingen hentbar MS Learn-side oppgir verdien (kan ikke verifiseres)

«Reelle feil» = outdated + wrong. **unsourced er IKKE en feil** — det er den uverifiserbare massen (priser på JS-rendrede Azure-sider som en fetch-basert judge heller ikke når). Den verifiserbare feilraten ekskluderer derfor unsourced fra nevneren.

## Overall

| metrikk | verdi |
|---|---|
| Påstander totalt | ${o.total} |
| correct / outdated / wrong / unsourced | ${o.byVerdict.correct} / ${o.byVerdict.outdated} / ${o.byVerdict.wrong} / ${o.byVerdict.unsourced} |
| Reelle feil (outdated+wrong) | ${o.errors} |
| Verifiserbare påstander (nevner) | ${o.verifiable} |
| **Verifiserbar feilrate** | **${o.errors}/${o.verifiable} = ${pct(o.errorRate)}** |
| Wilson 95 % | ${band(o.errorRateWilson)} |
| Unsourced-andel | ${o.unsourced}/${o.total} = ${pct(o.unsourcedRate)} |
| Feil staleness-loopen fanger (lastmod_changed=true) | ${o.errorsStalenessCatchable} |
| **Feil kun en korrekthets-judge fanger (judge-unique)** | **${o.errorsJudgeUnique}** |

> **Gate-kritisk:** \`errorsJudgeUnique\` = reelle feil hvis siterte kilde-lastmod IKKE endret seg etter fildato — den eneste klassen en korrekthets-judge fanger som den eksisterende staleness-loopen bommer på. Staleness-recall på de reelle feilene = ${o.errors ? `${o.errorsStalenessCatchable}/${o.errors} = ${pct(o.errorsStalenessCatchable / o.errors)}` : 'n/a'}.

## Per stratum

${table('Stratum', report.byStratum)}
## Per skill

${table('Skill', report.bySkill)}
## Per claim_type

${table('Claim type', report.byClaimType)}
`;

if (process.argv.includes('--write')) {
  const jsonOut = path.join(DATA, 'base-rate-report.json');
  const mdOut = path.join(DATA, 'base-rate-report.md');
  fs.writeFileSync(jsonOut, JSON.stringify({ _meta: { source: 'gold-correctness-set.json', claim_count: claims.length }, ...report }, null, 2) + '\n');
  fs.writeFileSync(mdOut, md);
  console.log(`wrote ${jsonOut}`);
  console.log(`wrote ${mdOut}`);
} else {
  console.log(`claims: ${o.total} | correct=${o.byVerdict.correct} outdated=${o.byVerdict.outdated} wrong=${o.byVerdict.wrong} unsourced=${o.byVerdict.unsourced}`);
  console.log(`verifiable error rate: ${o.errors}/${o.verifiable} = ${pct(o.errorRate)}  Wilson ${band(o.errorRateWilson)}`);
  console.log(`unsourced: ${o.unsourced}/${o.total} = ${pct(o.unsourcedRate)}`);
  console.log(`staleness-catchable=${o.errorsStalenessCatchable}  judge-unique=${o.errorsJudgeUnique}`);
  console.log('(dry run — pass --write to persist base-rate-report.json + .md)');
}