ms-ai-architect/tests/kb-eval/test-base-rate.test.mjs

// tests/kb-eval/test-base-rate.test.mjs
// Unit tests for the Fase 0 base-rate aggregation lib.
//
// These functions turn the per-claim subagent verdicts (gold-correctness-set)
// into a defensible base-rate report: a verifiable error rate with a Wilson 95%
// score interval, broken down per skill / stratum / claim_type, plus the gate-
// critical "errors a correctness judge would catch over the existing staleness
// loop" (= genuine errors whose cited source lastmod did NOT change since the
// file's date). All computation is deterministic.

import { test } from 'node:test';
import assert from 'node:assert/strict';
import {
  parseLastUpdated,
  wilson,
  fileLastmodChanged,
  computeBaseRate,
} from '../../scripts/kb-eval/lib/base-rate.mjs';

// --- parseLastUpdated --------------------------------------------------------
test('parseLastUpdated — full YYYY-MM-DD header', () => {
  assert.equal(
    parseLastUpdated('# Title\n\n**Last updated:** 2026-06-24 (research via MCP)\n'),
    '2026-06-24',
  );
});
test('parseLastUpdated — month-only header normalises to first of month', () => {
  assert.equal(
    parseLastUpdated('**Last updated:** 2026-06 | Priser verifisert juni 2026\n'),
    '2026-06-01',
  );
});
test('parseLastUpdated — no header returns null', () => {
  assert.equal(parseLastUpdated('# Title\n\nNo date here.\n'), null);
});

// --- wilson ------------------------------------------------------------------
test('wilson — empty sample is all-zero, never NaN', () => {
  const w = wilson(0, 0);
  assert.equal(w.p, 0);
  assert.equal(w.low, 0);
  assert.equal(w.high, 0);
});
test('wilson — all successes: point estimate 1, upper bound 1, lower < 1', () => {
  const w = wilson(10, 10);
  assert.equal(w.p, 1);
  assert.ok(Math.abs(w.high - 1) < 1e-9);
  assert.ok(w.low < 1 && w.low > 0.6);
});
test('wilson — 50/100 matches known 95% interval [0.404, 0.596]', () => {
  const w = wilson(50, 100);
  assert.equal(w.p, 0.5);
  assert.ok(Math.abs(w.low - 0.4038) < 0.01, `low=${w.low}`);
  assert.ok(Math.abs(w.high - 0.5962) < 0.01, `high=${w.high}`);
});

// --- fileLastmodChanged ------------------------------------------------------
const REG = {
  urls: {
    'https://l/a': { sitemap_lastmod: '2026-05-01' },
    'https://l/b': { sitemap_lastmod: '2024-01-01' },
  },
};
test('fileLastmodChanged — a cited source changed after the file date -> true', () => {
  assert.equal(fileLastmodChanged(['https://l/a', 'https://l/b'], REG, '2026-01-01'), true);
});
test('fileLastmodChanged — no cited source newer than file date -> false', () => {
  assert.equal(fileLastmodChanged(['https://l/b'], REG, '2026-01-01'), false);
});
test('fileLastmodChanged — unknown file date -> null', () => {
  assert.equal(fileLastmodChanged(['https://l/a'], REG, null), null);
});
test('fileLastmodChanged — cited url absent from registry is ignored', () => {
  assert.equal(fileLastmodChanged(['https://l/missing'], REG, '2026-01-01'), false);
});

// --- computeBaseRate ---------------------------------------------------------
// Synthetic gold set: 10 claims across two skills and two strata.
const CLAIMS = [
  // skill X, volatile
  { skill: 'x', stratum: 'volatile', claim_type: 'price', verdict: 'correct', lastmod_changed: false },
  { skill: 'x', stratum: 'volatile', claim_type: 'version', verdict: 'outdated', lastmod_changed: true },  // staleness-catchable
  { skill: 'x', stratum: 'volatile', claim_type: 'sku', verdict: 'wrong', lastmod_changed: false },        // judge-unique
  { skill: 'x', stratum: 'volatile', claim_type: 'price', verdict: 'unsourced', lastmod_changed: false },
  { skill: 'x', stratum: 'volatile', claim_type: 'status', verdict: 'outdated', lastmod_changed: false },  // judge-unique
  // skill y, volatile
  { skill: 'y', stratum: 'volatile', claim_type: 'tpm', verdict: 'correct', lastmod_changed: true },
  { skill: 'y', stratum: 'volatile', claim_type: 'price', verdict: 'unsourced', lastmod_changed: null },
  // skill y, control
  { skill: 'y', stratum: 'control', claim_type: 'taxonomy', verdict: 'correct', lastmod_changed: false },
  { skill: 'y', stratum: 'control', claim_type: 'taxonomy', verdict: 'correct', lastmod_changed: false },
  { skill: 'y', stratum: 'control', claim_type: 'status', verdict: 'wrong', lastmod_changed: true },        // staleness-catchable
];

test('computeBaseRate — overall verdict tally and totals', () => {
  const r = computeBaseRate(CLAIMS);
  assert.equal(r.overall.total, 10);
  assert.equal(r.overall.byVerdict.correct, 4);
  assert.equal(r.overall.byVerdict.outdated, 2);
  assert.equal(r.overall.byVerdict.wrong, 2);
  assert.equal(r.overall.byVerdict.unsourced, 2);
});
test('computeBaseRate — verifiable error rate excludes unsourced from denominator', () => {
  const r = computeBaseRate(CLAIMS);
  // errors = outdated+wrong = 4; verifiable = total - unsourced = 8
  assert.equal(r.overall.errors, 4);
  assert.equal(r.overall.verifiable, 8);
  assert.ok(Math.abs(r.overall.errorRate - 0.5) < 1e-9);
  assert.equal(r.overall.unsourced, 2);
  assert.ok(Math.abs(r.overall.unsourcedRate - 0.2) < 1e-9);
  // Wilson band present and brackets the point estimate
  assert.ok(r.overall.errorRateWilson.low < 0.5 && r.overall.errorRateWilson.high > 0.5);
});
test('computeBaseRate — judge-unique errors = errors with lastmod_changed !== true', () => {
  const r = computeBaseRate(CLAIMS);
  // errors: outdated(true)=staleness, wrong(false)=judge, outdated(false)=judge, wrong(true)=staleness
  assert.equal(r.overall.errorsStalenessCatchable, 2); // lastmod_changed === true
  assert.equal(r.overall.errorsJudgeUnique, 2);        // lastmod_changed !== true
});
test('computeBaseRate — per-stratum split (volatile vs control)', () => {
  const r = computeBaseRate(CLAIMS);
  assert.equal(r.byStratum.volatile.total, 7);
  assert.equal(r.byStratum.volatile.errors, 3);
  assert.equal(r.byStratum.control.total, 3);
  assert.equal(r.byStratum.control.errors, 1);
});
test('computeBaseRate — per-skill breakdown present', () => {
  const r = computeBaseRate(CLAIMS);
  assert.equal(r.bySkill.x.total, 5);
  assert.equal(r.bySkill.x.errors, 3);
  assert.equal(r.bySkill.y.total, 5);
  assert.equal(r.bySkill.y.errors, 1);
});