voyage/lib/review/fidelity-diff.mjs

// lib/review/fidelity-diff.mjs
// Fidelity comparison of two review.md artifacts — the PRIMARY metric of the
// NW2 (S10) prose-vs-Workflow bake-off (T2 §5).
//
// A substrate swap (prose Arm A → Workflow Arm B) passes the gate only if it
// produces a fidelity-equivalent review.md: SAME verdict and an equivalent
// finding set (IDs / severities / rule_keys). This module computes that diff
// from two rendered review.md texts, reusing the determinism-pipeline
// primitives (jaccard over finding-IDs + frontmatter parse + the NW1 trailing-
// block extractor).
//
// Pure JS, zero deps beyond existing lib modules — unit-testable without any
// live LLM run.

import { parseDocument } from '../util/frontmatter.mjs';
import { jaccardSimilarity } from '../parsers/jaccard.mjs';
import { computeFindingId } from '../parsers/finding-id.mjs';
import { extractFindingsBlock } from './findings-schema.mjs';
import { isMain } from '../util/is-main.mjs';

export const DEFAULT_JACCARD_TOLERANCE = 0.7;

/**
 * Parse a rendered review.md into its comparable shape.
 * - verdict + finding-ID list come from frontmatter (the validated contract).
 * - per-finding severity / rule_key / file / line come from the trailing JSON
 *   block. Both the `rule_key` (real reviewer output) and `rule` (fixtures)
 *   keys are accepted.
 * @param {string} text
 * @returns {{ verdict: string|null, findingIds: string[],
 *             details: Array<{id, severity, rule_key, file, line}> }}
 */
export function parseReviewArtifact(text) {
  const doc = parseDocument(text);
  const fm = (doc.valid && doc.parsed && doc.parsed.frontmatter) || {};
  const verdict = typeof fm.verdict === 'string' ? fm.verdict : null;
  const findingIds = Array.isArray(fm.findings) ? fm.findings.filter((x) => typeof x === 'string') : [];

  let details = [];
  const block = extractFindingsBlock(text);
  if (block !== null) {
    try {
      const parsed = JSON.parse(block);
      const arr = Array.isArray(parsed) ? parsed : (Array.isArray(parsed.findings) ? parsed.findings : []);
      details = arr.map((f) => ({
        id: f.id ?? null,
        severity: f.severity ?? null,
        rule_key: f.rule_key ?? f.rule ?? null,
        file: f.file ?? null,
        line: f.line ?? null,
      }));
    } catch {
      details = [];
    }
  }

  return { verdict, findingIds, details };
}

function detailMap(details) {
  const m = new Map();
  for (const d of details) {
    if (d.id) m.set(d.id, d);
  }
  return m;
}

/**
 * Core comparison over two normalized artifacts: {verdict, findingIds, details}.
 * Shared by fidelityDiff (review.md text) and fidelityDiffStructured (arm output).
 */
function compareArtifacts(a, b, opts = {}) {
  const tol = typeof opts.jaccardTolerance === 'number' ? opts.jaccardTolerance : DEFAULT_JACCARD_TOLERANCE;
  const verdictMatch = a.verdict === b.verdict;
  const jaccard = jaccardSimilarity(a.findingIds, b.findingIds);

  // Cross-check severity + rule_key on findings present in BOTH arms.
  const mapA = detailMap(a.details);
  const mapB = detailMap(b.details);
  const severityMismatches = [];
  const ruleKeyMismatches = [];
  for (const [id, da] of mapA) {
    const db = mapB.get(id);
    if (!db) continue;
    if (da.severity !== db.severity) severityMismatches.push({ id, a: da.severity, b: db.severity });
    if (da.rule_key !== db.rule_key) ruleKeyMismatches.push({ id, a: da.rule_key, b: db.rule_key });
  }

  const equivalent =
    verdictMatch &&
    jaccard >= tol &&
    severityMismatches.length === 0 &&
    ruleKeyMismatches.length === 0;

  return {
    verdictA: a.verdict,
    verdictB: b.verdict,
    verdictMatch,
    jaccard,
    countA: a.findingIds.length,
    countB: b.findingIds.length,
    severityMismatches,
    ruleKeyMismatches,
    equivalent,
  };
}

/**
 * Compute the fidelity diff between two rendered review.md artifacts.
 * @param {string} textA — baseline (Arm A — prose)
 * @param {string} textB — candidate (Arm B — Workflow)
 * @param {{ jaccardTolerance?: number }} [opts]
 */
export function fidelityDiff(textA, textB, opts = {}) {
  return compareArtifacts(parseReviewArtifact(textA), parseReviewArtifact(textB), opts);
}

/**
 * Normalize a structured arm output ({verdict, findings:[{severity,rule_key,
 * file,line}]}) into the comparable shape, recomputing canonical finding-IDs
 * from the (file, line, rule_key) triplet. Findings missing file/rule_key are
 * dropped from the ID set (they cannot dedupe), but counted is by valid IDs.
 */
export function normalizeArmOutput(arm) {
  const verdict = arm && typeof arm.verdict === 'string' ? arm.verdict : null;
  const findings = (arm && Array.isArray(arm.findings)) ? arm.findings : [];
  const findingIds = [];
  const details = [];
  for (const f of findings) {
    const file = f.file;
    const rule_key = f.rule_key ?? f.rule ?? null;
    const line = f.line;
    let id = null;
    if (typeof file === 'string' && file.length > 0 && rule_key && line !== null && line !== undefined) {
      try { id = computeFindingId(file, line, rule_key); } catch { id = null; }
    }
    if (id) findingIds.push(id);
    details.push({ id, severity: f.severity ?? null, rule_key, file: file ?? null, line: line ?? null });
  }
  return { verdict, findingIds, details };
}

/**
 * Fidelity diff between two structured arm outputs (the bake-off comparison —
 * avoids rendering review.md for each run).
 * @param {{verdict, findings}} armA
 * @param {{verdict, findings}} armB
 * @param {{ jaccardTolerance?: number }} [opts]
 */
export function fidelityDiffStructured(armA, armB, opts = {}) {
  return compareArtifacts(normalizeArmOutput(armA), normalizeArmOutput(armB), opts);
}

// ---- CLI shim ----------------------------------------------------------------

if (isMain(import.meta.url)) {
  const { readFileSync } = await import('node:fs');
  const args = process.argv.slice(2);
  const files = args.filter((x) => !x.startsWith('--'));
  if (files.length !== 2) {
    process.stderr.write('Usage: fidelity-diff.mjs [--json] <review-A.md> <review-B.md>\n');
    process.exit(2);
  }
  const d = fidelityDiff(readFileSync(files[0], 'utf-8'), readFileSync(files[1], 'utf-8'));
  if (args.includes('--json')) {
    process.stdout.write(JSON.stringify(d, null, 2) + '\n');
  } else {
    process.stdout.write(`fidelity-diff: ${d.equivalent ? 'EQUIVALENT' : 'DIVERGENT'}\n`);
    process.stdout.write(`  verdict: ${d.verdictA} vs ${d.verdictB} (match=${d.verdictMatch})\n`);
    process.stdout.write(`  jaccard: ${d.jaccard.toFixed(4)} (findings ${d.countA} vs ${d.countB})\n`);
    process.stdout.write(`  severity mismatches: ${d.severityMismatches.length}; rule_key mismatches: ${d.ruleKeyMismatches.length}\n`);
  }
  process.exit(d.equivalent ? 0 : 1);
}