// lib/review/fidelity-diff.mjs // Fidelity comparison of two review.md artifacts — the PRIMARY metric of the // NW2 (S10) prose-vs-Workflow bake-off (T2 §5). // // A substrate swap (prose Arm A → Workflow Arm B) passes the gate only if it // produces a fidelity-equivalent review.md: SAME verdict and an equivalent // finding set (IDs / severities / rule_keys). This module computes that diff // from two rendered review.md texts, reusing the determinism-pipeline // primitives (jaccard over finding-IDs + frontmatter parse + the NW1 trailing- // block extractor). // // Pure JS, zero deps beyond existing lib modules — unit-testable without any // live LLM run. import { parseDocument } from '../util/frontmatter.mjs'; import { jaccardSimilarity } from '../parsers/jaccard.mjs'; import { computeFindingId } from '../parsers/finding-id.mjs'; import { extractFindingsBlock } from './findings-schema.mjs'; export const DEFAULT_JACCARD_TOLERANCE = 0.7; /** * Parse a rendered review.md into its comparable shape. * - verdict + finding-ID list come from frontmatter (the validated contract). * - per-finding severity / rule_key / file / line come from the trailing JSON * block. Both the `rule_key` (real reviewer output) and `rule` (fixtures) * keys are accepted. * @param {string} text * @returns {{ verdict: string|null, findingIds: string[], * details: Array<{id, severity, rule_key, file, line}> }} */ export function parseReviewArtifact(text) { const doc = parseDocument(text); const fm = (doc.valid && doc.parsed && doc.parsed.frontmatter) || {}; const verdict = typeof fm.verdict === 'string' ? fm.verdict : null; const findingIds = Array.isArray(fm.findings) ? fm.findings.filter((x) => typeof x === 'string') : []; let details = []; const block = extractFindingsBlock(text); if (block !== null) { try { const parsed = JSON.parse(block); const arr = Array.isArray(parsed) ? parsed : (Array.isArray(parsed.findings) ? parsed.findings : []); details = arr.map((f) => ({ id: f.id ?? null, severity: f.severity ?? null, rule_key: f.rule_key ?? f.rule ?? null, file: f.file ?? null, line: f.line ?? null, })); } catch { details = []; } } return { verdict, findingIds, details }; } function detailMap(details) { const m = new Map(); for (const d of details) { if (d.id) m.set(d.id, d); } return m; } /** * Core comparison over two normalized artifacts: {verdict, findingIds, details}. * Shared by fidelityDiff (review.md text) and fidelityDiffStructured (arm output). */ function compareArtifacts(a, b, opts = {}) { const tol = typeof opts.jaccardTolerance === 'number' ? opts.jaccardTolerance : DEFAULT_JACCARD_TOLERANCE; const verdictMatch = a.verdict === b.verdict; const jaccard = jaccardSimilarity(a.findingIds, b.findingIds); // Cross-check severity + rule_key on findings present in BOTH arms. const mapA = detailMap(a.details); const mapB = detailMap(b.details); const severityMismatches = []; const ruleKeyMismatches = []; for (const [id, da] of mapA) { const db = mapB.get(id); if (!db) continue; if (da.severity !== db.severity) severityMismatches.push({ id, a: da.severity, b: db.severity }); if (da.rule_key !== db.rule_key) ruleKeyMismatches.push({ id, a: da.rule_key, b: db.rule_key }); } const equivalent = verdictMatch && jaccard >= tol && severityMismatches.length === 0 && ruleKeyMismatches.length === 0; return { verdictA: a.verdict, verdictB: b.verdict, verdictMatch, jaccard, countA: a.findingIds.length, countB: b.findingIds.length, severityMismatches, ruleKeyMismatches, equivalent, }; } /** * Compute the fidelity diff between two rendered review.md artifacts. * @param {string} textA — baseline (Arm A — prose) * @param {string} textB — candidate (Arm B — Workflow) * @param {{ jaccardTolerance?: number }} [opts] */ export function fidelityDiff(textA, textB, opts = {}) { return compareArtifacts(parseReviewArtifact(textA), parseReviewArtifact(textB), opts); } /** * Normalize a structured arm output ({verdict, findings:[{severity,rule_key, * file,line}]}) into the comparable shape, recomputing canonical finding-IDs * from the (file, line, rule_key) triplet. Findings missing file/rule_key are * dropped from the ID set (they cannot dedupe), but counted is by valid IDs. */ export function normalizeArmOutput(arm) { const verdict = arm && typeof arm.verdict === 'string' ? arm.verdict : null; const findings = (arm && Array.isArray(arm.findings)) ? arm.findings : []; const findingIds = []; const details = []; for (const f of findings) { const file = f.file; const rule_key = f.rule_key ?? f.rule ?? null; const line = f.line; let id = null; if (typeof file === 'string' && file.length > 0 && rule_key && line !== null && line !== undefined) { try { id = computeFindingId(file, line, rule_key); } catch { id = null; } } if (id) findingIds.push(id); details.push({ id, severity: f.severity ?? null, rule_key, file: file ?? null, line: line ?? null }); } return { verdict, findingIds, details }; } /** * Fidelity diff between two structured arm outputs (the bake-off comparison — * avoids rendering review.md for each run). * @param {{verdict, findings}} armA * @param {{verdict, findings}} armB * @param {{ jaccardTolerance?: number }} [opts] */ export function fidelityDiffStructured(armA, armB, opts = {}) { return compareArtifacts(normalizeArmOutput(armA), normalizeArmOutput(armB), opts); } // ---- CLI shim ---------------------------------------------------------------- if (import.meta.url === `file://${process.argv[1]}`) { const { readFileSync } = await import('node:fs'); const args = process.argv.slice(2); const files = args.filter((x) => !x.startsWith('--')); if (files.length !== 2) { process.stderr.write('Usage: fidelity-diff.mjs [--json] \n'); process.exit(2); } const d = fidelityDiff(readFileSync(files[0], 'utf-8'), readFileSync(files[1], 'utf-8')); if (args.includes('--json')) { process.stdout.write(JSON.stringify(d, null, 2) + '\n'); } else { process.stdout.write(`fidelity-diff: ${d.equivalent ? 'EQUIVALENT' : 'DIVERGENT'}\n`); process.stdout.write(` verdict: ${d.verdictA} vs ${d.verdictB} (match=${d.verdictMatch})\n`); process.stdout.write(` jaccard: ${d.jaccard.toFixed(4)} (findings ${d.countA} vs ${d.countB})\n`); process.stdout.write(` severity mismatches: ${d.severityMismatches.length}; rule_key mismatches: ${d.ruleKeyMismatches.length}\n`); } process.exit(d.equivalent ? 0 : 1); }