feat(voyage): S10 part B — NW2 full bake-off (rich fixture) → verdict POSITIVE

Run the full T2 §5 prose-vs-Workflow /trekreview bake-off (operator GO,
choice "a"): 3 runs/arm on a rich-finding JWT-auth fixture, resolving the
smoke's 0-finding limitation.

Deliverables:
- tests/fixtures/bakeoff-rich/ — JWT-auth brief + diff with 5 seeded blatant,
  brief-traceable issues (varied severity/rule_key, one dual-flaggable).
- scripts/bakeoff-armA-merge.mjs — Arm A (prose) validate (NW1) + triplet-dedup,
  matching Arm B's dedup exactly.
- scripts/bakeoff-fidelity.mjs — cross-arm + within-arm + granularity-ladder
  fidelity analysis over the structured arm outputs.
- docs/T2-bakeoff-results.md §Full run — the T2 §5 verdict.

Result (3 runs/arm, both arms ran the coordinator):
- Verdict fidelity EQUIVALENT — all 6 runs BLOCK, cross-arm verdict-match 1.0.
- Finding-set: substrate is fidelity-neutral. Cross-arm jaccard 0.41 (triplet)
  → 0.71 (file,rule_key) → 1.0 (file); cross-arm ≈ within-arm at every
  granularity. Issue coverage 5/5 in 6/6 runs. Low triplet jaccard is
  line-citation noise shared by both arms, not a substrate effect.
- Token +4.4% (Arm B vs A; <=+15%). Classifier interference 0 at 9-agent
  concurrency. JSON-robustness: Arm B schema-forced; Arm A 6/6 valid via NW1.
- VERDICT POSITIVE → S11 proceeds with opt-in --workflow flag.

Caveat (per plan posture): strict triplet-jaccard>=0.7 flag is 0/9, a
metric-calibration artifact (both arms sub-0.7 against themselves), not a
regression. Residual: F4 auto/bypass explicit-mode check (mode not settable
in-session).

Suite green (662/660 pass, 2 skip); plugin validate clean (modulo the
pre-existing root-CLAUDE.md warning). No production code changed.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01LqBYc8Ltrk7LipyJmGxXiB

scripts/bakeoff-fidelity.mjs

@@ -0,0 +1,123 @@
+// scripts/bakeoff-fidelity.mjs
+// NW2 bake-off — fidelity analysis across ≥3 runs/arm (S10 part B).
+//
+// Consumes the per-run structured arm outputs ({verdict, findings:[...]}) and
+// computes the T2 §5 fidelity picture:
+//
+//   - CROSS-ARM (PRIMARY): every (Ai, Bj) pair via fidelityDiffStructured
+//     (lib/review/fidelity-diff.mjs) → median jaccard, verdict-match rate,
+//     equivalence rate, severity/rule_key mismatch tallies.
+//   - WITHIN-ARM (context): each arm's own run-to-run variance (Ai vs Aj),
+//     so cross-arm divergence can be read against each arm's intrinsic noise.
+//   - Distributions: per-arm verdicts + finding counts.
+//
+// Pure analysis over JSON the live runs produced — no LLM, deterministic.
+//
+// Usage:
+//   node scripts/bakeoff-fidelity.mjs --armA a1.json a2.json a3.json \
+//                                      --armB b1.json b2.json b3.json [--json]
+//   Each *.json is a structured arm result: {"verdict": "...", "findings": [...]}.
+
+import { readFileSync } from 'node:fs';
+import { fidelityDiffStructured } from '../lib/review/fidelity-diff.mjs';
+
+function median(nums) {
+  if (nums.length === 0) return null;
+  const s = [...nums].sort((a, b) => a - b);
+  const mid = Math.floor(s.length / 2);
+  return s.length % 2 ? s[mid] : (s[mid - 1] + s[mid]) / 2;
+}
+
+function round(n, d = 4) {
+  return n === null ? null : Number(n.toFixed(d));
+}
+
+function pairwise(arms, label) {
+  const out = [];
+  for (let i = 0; i < arms.length; i++) {
+    for (let j = i + 1; j < arms.length; j++) {
+      const d = fidelityDiffStructured(arms[i].result, arms[j].result);
+      out.push({ pair: `${label}${i + 1}×${label}${j + 1}`, jaccard: round(d.jaccard), verdictMatch: d.verdictMatch, equivalent: d.equivalent });
+    }
+  }
+  return out;
+}
+
+/**
+ * @param {Array<{name, result:{verdict,findings}}>} armA
+ * @param {Array<{name, result:{verdict,findings}}>} armB
+ */
+export function analyze(armA, armB) {
+  // Cross-arm — the substrate comparison.
+  const cross = [];
+  for (let i = 0; i < armA.length; i++) {
+    for (let j = 0; j < armB.length; j++) {
+      const d = fidelityDiffStructured(armA[i].result, armB[j].result);
+      cross.push({
+        pair: `A${i + 1}×B${j + 1}`,
+        verdictA: d.verdictA, verdictB: d.verdictB, verdictMatch: d.verdictMatch,
+        jaccard: round(d.jaccard), countA: d.countA, countB: d.countB,
+        severityMismatches: d.severityMismatches.length,
+        ruleKeyMismatches: d.ruleKeyMismatches.length,
+        equivalent: d.equivalent,
+      });
+    }
+  }
+  const crossJ = cross.map((c) => c.jaccard);
+  const summary = {
+    runs: { armA: armA.length, armB: armB.length },
+    cross_arm: {
+      median_jaccard: round(median(crossJ)),
+      min_jaccard: round(Math.min(...crossJ)),
+      max_jaccard: round(Math.max(...crossJ)),
+      verdict_match_rate: round(cross.filter((c) => c.verdictMatch).length / cross.length, 3),
+      equivalent_rate: round(cross.filter((c) => c.equivalent).length / cross.length, 3),
+      severity_mismatch_pairs: cross.filter((c) => c.severityMismatches > 0).length,
+      rulekey_mismatch_pairs: cross.filter((c) => c.ruleKeyMismatches > 0).length,
+    },
+    within_arm_A: pairwise(armA, 'A'),
+    within_arm_B: pairwise(armB, 'B'),
+    distributions: {
+      armA_verdicts: armA.map((a) => a.result.verdict),
+      armB_verdicts: armB.map((b) => b.result.verdict),
+      armA_counts: armA.map((a) => (a.result.findings || []).length),
+      armB_counts: armB.map((b) => (b.result.findings || []).length),
+    },
+  };
+  return { summary, cross };
+}
+
+// ---- CLI shim ----------------------------------------------------------------
+
+if (import.meta.url === `file://${process.argv[1]}`) {
+  const argv = process.argv.slice(2);
+  const asJson = argv.includes('--json');
+  function collect(flag) {
+    const i = argv.indexOf(flag);
+    if (i === -1) return [];
+    const files = [];
+    for (let k = i + 1; k < argv.length && !argv[k].startsWith('--'); k++) files.push(argv[k]);
+    return files.map((f) => ({ name: f, result: JSON.parse(readFileSync(f, 'utf-8')) }));
+  }
+  const armA = collect('--armA');
+  const armB = collect('--armB');
+  if (armA.length === 0 || armB.length === 0) {
+    process.stderr.write('Usage: bakeoff-fidelity.mjs --armA a*.json --armB b*.json [--json]\n');
+    process.exit(2);
+  }
+  const { summary, cross } = analyze(armA, armB);
+  if (asJson) {
+    process.stdout.write(JSON.stringify({ summary, cross }, null, 2) + '\n');
+  } else {
+    const x = summary.cross_arm;
+    process.stdout.write('NW2 bake-off fidelity (cross-arm = PRIMARY)\n');
+    process.stdout.write(`  runs: A=${summary.runs.armA} B=${summary.runs.armB}\n`);
+    process.stdout.write(`  median jaccard: ${x.median_jaccard} (min ${x.min_jaccard}, max ${x.max_jaccard})\n`);
+    process.stdout.write(`  verdict-match rate: ${x.verdict_match_rate}\n`);
+    process.stdout.write(`  equivalent rate: ${x.equivalent_rate}\n`);
+    process.stdout.write(`  severity-mismatch pairs: ${x.severity_mismatch_pairs}; rule_key-mismatch pairs: ${x.rulekey_mismatch_pairs}\n`);
+    process.stdout.write(`  armA verdicts: ${summary.distributions.armA_verdicts.join(',')} | counts ${summary.distributions.armA_counts.join(',')}\n`);
+    process.stdout.write(`  armB verdicts: ${summary.distributions.armB_verdicts.join(',')} | counts ${summary.distributions.armB_counts.join(',')}\n`);
+  }
+  process.exit(0);
+}