voyage/scripts/bakeoff-fidelity.mjs

// scripts/bakeoff-fidelity.mjs
// NW2 bake-off — fidelity analysis across ≥3 runs/arm (S10 part B).
//
// Consumes the per-run structured arm outputs ({verdict, findings:[...]}) and
// computes the T2 §5 fidelity picture:
//
//   - CROSS-ARM (PRIMARY): every (Ai, Bj) pair via fidelityDiffStructured
//     (lib/review/fidelity-diff.mjs) → median jaccard, verdict-match rate,
//     equivalence rate, severity/rule_key mismatch tallies.
//   - WITHIN-ARM (context): each arm's own run-to-run variance (Ai vs Aj),
//     so cross-arm divergence can be read against each arm's intrinsic noise.
//   - Distributions: per-arm verdicts + finding counts.
//
// Pure analysis over JSON the live runs produced — no LLM, deterministic.
//
// Usage:
//   node scripts/bakeoff-fidelity.mjs --armA a1.json a2.json a3.json \
//                                      --armB b1.json b2.json b3.json [--json]
//   Each *.json is a structured arm result: {"verdict": "...", "findings": [...]}.

import { readFileSync } from 'node:fs';
import { fidelityDiffStructured } from '../lib/review/fidelity-diff.mjs';
import { isMain } from '../lib/util/is-main.mjs';

function median(nums) {
  if (nums.length === 0) return null;
  const s = [...nums].sort((a, b) => a - b);
  const mid = Math.floor(s.length / 2);
  return s.length % 2 ? s[mid] : (s[mid - 1] + s[mid]) / 2;
}

function round(n, d = 4) {
  return n === null ? null : Number(n.toFixed(d));
}

function pairwise(arms, label) {
  const out = [];
  for (let i = 0; i < arms.length; i++) {
    for (let j = i + 1; j < arms.length; j++) {
      const d = fidelityDiffStructured(arms[i].result, arms[j].result);
      out.push({ pair: `${label}${i + 1}×${label}${j + 1}`, jaccard: round(d.jaccard), verdictMatch: d.verdictMatch, equivalent: d.equivalent });
    }
  }
  return out;
}

/**
 * @param {Array<{name, result:{verdict,findings}}>} armA
 * @param {Array<{name, result:{verdict,findings}}>} armB
 */
export function analyze(armA, armB) {
  // Cross-arm — the substrate comparison.
  const cross = [];
  for (let i = 0; i < armA.length; i++) {
    for (let j = 0; j < armB.length; j++) {
      const d = fidelityDiffStructured(armA[i].result, armB[j].result);
      cross.push({
        pair: `A${i + 1}×B${j + 1}`,
        verdictA: d.verdictA, verdictB: d.verdictB, verdictMatch: d.verdictMatch,
        jaccard: round(d.jaccard), countA: d.countA, countB: d.countB,
        severityMismatches: d.severityMismatches.length,
        ruleKeyMismatches: d.ruleKeyMismatches.length,
        equivalent: d.equivalent,
      });
    }
  }
  const crossJ = cross.map((c) => c.jaccard);
  const summary = {
    runs: { armA: armA.length, armB: armB.length },
    cross_arm: {
      median_jaccard: round(median(crossJ)),
      min_jaccard: round(Math.min(...crossJ)),
      max_jaccard: round(Math.max(...crossJ)),
      verdict_match_rate: round(cross.filter((c) => c.verdictMatch).length / cross.length, 3),
      equivalent_rate: round(cross.filter((c) => c.equivalent).length / cross.length, 3),
      severity_mismatch_pairs: cross.filter((c) => c.severityMismatches > 0).length,
      rulekey_mismatch_pairs: cross.filter((c) => c.ruleKeyMismatches > 0).length,
    },
    within_arm_A: pairwise(armA, 'A'),
    within_arm_B: pairwise(armB, 'B'),
    distributions: {
      armA_verdicts: armA.map((a) => a.result.verdict),
      armB_verdicts: armB.map((b) => b.result.verdict),
      armA_counts: armA.map((a) => (a.result.findings || []).length),
      armB_counts: armB.map((b) => (b.result.findings || []).length),
    },
  };
  return { summary, cross };
}

// ---- CLI shim ----------------------------------------------------------------

if (isMain(import.meta.url)) {
  const argv = process.argv.slice(2);
  const asJson = argv.includes('--json');
  function collect(flag) {
    const i = argv.indexOf(flag);
    if (i === -1) return [];
    const files = [];
    for (let k = i + 1; k < argv.length && !argv[k].startsWith('--'); k++) files.push(argv[k]);
    return files.map((f) => ({ name: f, result: JSON.parse(readFileSync(f, 'utf-8')) }));
  }
  const armA = collect('--armA');
  const armB = collect('--armB');
  if (armA.length === 0 || armB.length === 0) {
    process.stderr.write('Usage: bakeoff-fidelity.mjs --armA a*.json --armB b*.json [--json]\n');
    process.exit(2);
  }
  const { summary, cross } = analyze(armA, armB);
  if (asJson) {
    process.stdout.write(JSON.stringify({ summary, cross }, null, 2) + '\n');
  } else {
    const x = summary.cross_arm;
    process.stdout.write('NW2 bake-off fidelity (cross-arm = PRIMARY)\n');
    process.stdout.write(`  runs: A=${summary.runs.armA} B=${summary.runs.armB}\n`);
    process.stdout.write(`  median jaccard: ${x.median_jaccard} (min ${x.min_jaccard}, max ${x.max_jaccard})\n`);
    process.stdout.write(`  verdict-match rate: ${x.verdict_match_rate}\n`);
    process.stdout.write(`  equivalent rate: ${x.equivalent_rate}\n`);
    process.stdout.write(`  severity-mismatch pairs: ${x.severity_mismatch_pairs}; rule_key-mismatch pairs: ${x.rulekey_mismatch_pairs}\n`);
    process.stdout.write(`  armA verdicts: ${summary.distributions.armA_verdicts.join(',')} | counts ${summary.distributions.armA_counts.join(',')}\n`);
    process.stdout.write(`  armB verdicts: ${summary.distributions.armB_verdicts.join(',')} | counts ${summary.distributions.armB_counts.join(',')}\n`);
  }
  process.exit(0);
}