voyage/tests/lib/fidelity-diff.test.mjs

import { test } from 'node:test';
import { strict as assert } from 'node:assert';
import { readFileSync } from 'node:fs';
import { join, dirname } from 'node:path';
import { fileURLToPath } from 'node:url';
import {
  parseReviewArtifact,
  fidelityDiff,
  fidelityDiffStructured,
  normalizeArmOutput,
} from '../../lib/review/fidelity-diff.mjs';
import { computeFindingId } from '../../lib/parsers/finding-id.mjs';

const HERE = dirname(fileURLToPath(import.meta.url));
const ROOT = join(HERE, '..', '..');

function fixture(name) {
  return readFileSync(join(ROOT, 'tests/fixtures/trekreview', name), 'utf-8');
}

const RUN_A = fixture('review-run-A.md');
const RUN_B = fixture('review-run-B.md');

// ---- parseReviewArtifact ----------------------------------------------------

test('parseReviewArtifact — extracts verdict + finding IDs from frontmatter', () => {
  const a = parseReviewArtifact(RUN_A);
  assert.equal(a.verdict, 'WARN');
  assert.ok(Array.isArray(a.findingIds));
  assert.equal(a.findingIds.length, 5);
  for (const id of a.findingIds) assert.match(id, /^[0-9a-f]{40}$/);
});

test('parseReviewArtifact — extracts per-finding details from trailing JSON', () => {
  const a = parseReviewArtifact(RUN_A);
  assert.ok(Array.isArray(a.details));
  assert.equal(a.details.length, 5);
  const blocker = a.details.find((d) => d.severity === 'BLOCKER');
  assert.ok(blocker);
  // fixtures use the key "rule"; real reviewer output uses "rule_key" — both accepted
  assert.equal(blocker.rule_key, 'UNIMPLEMENTED_CRITERION');
  assert.equal(blocker.file, 'lib/handlers/login.mjs');
  assert.equal(blocker.line, 23);
});

test('parseReviewArtifact — tolerates a missing trailing JSON block (details = [])', () => {
  const noJson = '---\ntype: trekreview\nverdict: ALLOW\nfindings: []\n---\n\n# Review\n\nNo findings.\n';
  const r = parseReviewArtifact(noJson);
  assert.equal(r.verdict, 'ALLOW');
  assert.deepEqual(r.findingIds, []);
  assert.deepEqual(r.details, []);
});

// ---- fidelityDiff -----------------------------------------------------------

test('fidelityDiff — identical artifact is fully equivalent (jaccard 1.0)', () => {
  const d = fidelityDiff(RUN_A, RUN_A);
  assert.equal(d.verdictMatch, true);
  assert.equal(d.jaccard, 1);
  assert.equal(d.equivalent, true);
  assert.equal(d.severityMismatches.length, 0);
  assert.equal(d.ruleKeyMismatches.length, 0);
});

test('fidelityDiff — A vs B (A ⊂ B): same verdict, jaccard 5/6, equivalent at default tolerance', () => {
  const d = fidelityDiff(RUN_A, RUN_B);
  assert.equal(d.verdictMatch, true);
  assert.ok(Math.abs(d.jaccard - 5 / 6) < 1e-9, `jaccard=${d.jaccard}`);
  assert.equal(d.equivalent, true); // default jaccardTolerance 0.7
});

test('fidelityDiff — divergent verdict makes it non-equivalent even at jaccard 1.0', () => {
  const a = parseReviewArtifact(RUN_A);
  const flipped = RUN_A.replace('verdict: WARN', 'verdict: BLOCK');
  const d = fidelityDiff(RUN_A, flipped);
  assert.equal(d.verdictMatch, false);
  assert.equal(d.jaccard, 1);
  assert.equal(d.equivalent, false);
  void a;
});

test('fidelityDiff — tolerance gate: jaccard below tolerance is non-equivalent', () => {
  const dLoose = fidelityDiff(RUN_A, RUN_B, { jaccardTolerance: 0.7 });
  const dStrict = fidelityDiff(RUN_A, RUN_B, { jaccardTolerance: 0.95 });
  assert.equal(dLoose.equivalent, true);
  assert.equal(dStrict.equivalent, false);
});

test('fidelityDiff — flags severity mismatch on a shared finding ID', () => {
  // Flip the BLOCKER severity in B's trailing JSON for the shared login.mjs finding.
  const sharedId = '763d174e6c519fafbadcba5d1706708479e36e61';
  const tampered = RUN_B.replace(
    `"id": "${sharedId}", "severity": "BLOCKER"`,
    `"id": "${sharedId}", "severity": "MINOR"`,
  );
  const d = fidelityDiff(RUN_A, tampered);
  assert.ok(d.severityMismatches.some((m) => m.id === sharedId),
    `expected a severity mismatch for ${sharedId}; got ${JSON.stringify(d.severityMismatches)}`);
  assert.equal(d.equivalent, false);
});

test('fidelityDiff — exposes finding counts for both arms', () => {
  const d = fidelityDiff(RUN_A, RUN_B);
  assert.equal(d.countA, 5);
  assert.equal(d.countB, 6);
});

// ---- structured arm-output path (the bake-off comparison) -------------------

const ARM = (verdict, findings) => ({ verdict, findings });
const F = (severity, rule_key, file, line) => ({ severity, rule_key, file, line });

test('normalizeArmOutput — recomputes canonical IDs from the (file,line,rule_key) triplet', () => {
  const arm = ARM('WARN', [F('BLOCKER', 'SECURITY_INJECTION', 'lib/exec.mjs', 23)]);
  const n = normalizeArmOutput(arm);
  assert.equal(n.verdict, 'WARN');
  assert.equal(n.findingIds.length, 1);
  assert.equal(n.findingIds[0], computeFindingId('lib/exec.mjs', 23, 'SECURITY_INJECTION'));
});

test('normalizeArmOutput — drops findings missing file/rule_key from the ID set', () => {
  const arm = ARM('WARN', [F('MAJOR', 'MISSING_TEST', '', 0), { severity: 'MINOR', line: 5 }]);
  const n = normalizeArmOutput(arm);
  assert.equal(n.findingIds.length, 0);
  assert.equal(n.details.length, 2); // still recorded, just not ID-keyed
});

test('fidelityDiffStructured — identical arms are equivalent (jaccard 1.0)', () => {
  const findings = [
    F('BLOCKER', 'UNIMPLEMENTED_CRITERION', 'a.mjs', 1),
    F('MAJOR', 'MISSING_TEST', 'b.mjs', 0),
  ];
  const d = fidelityDiffStructured(ARM('BLOCK', findings), ARM('BLOCK', findings));
  assert.equal(d.equivalent, true);
  assert.equal(d.jaccard, 1);
  assert.equal(d.verdictMatch, true);
});

test('fidelityDiffStructured — divergent verdict fails even with identical findings', () => {
  const findings = [F('BLOCKER', 'UNIMPLEMENTED_CRITERION', 'a.mjs', 1)];
  const d = fidelityDiffStructured(ARM('BLOCK', findings), ARM('WARN', findings));
  assert.equal(d.verdictMatch, false);
  assert.equal(d.equivalent, false);
});

test('fidelityDiffStructured — extra finding in Arm B lowers jaccard', () => {
  const a = ARM('WARN', [F('MAJOR', 'MISSING_TEST', 'a.mjs', 10)]);
  const b = ARM('WARN', [
    F('MAJOR', 'MISSING_TEST', 'a.mjs', 10),
    F('MINOR', 'PLACEHOLDER_IN_CODE', 'b.mjs', 14),
  ]);
  const d = fidelityDiffStructured(a, b);
  assert.ok(Math.abs(d.jaccard - 1 / 2) < 1e-9, `jaccard=${d.jaccard}`);
  assert.equal(d.countA, 1);
  assert.equal(d.countB, 2);
});

test('fidelityDiffStructured — same triplet, different severity → severity mismatch', () => {
  const a = ARM('BLOCK', [F('BLOCKER', 'UNIMPLEMENTED_CRITERION', 'a.mjs', 1)]);
  const b = ARM('BLOCK', [F('MAJOR', 'UNIMPLEMENTED_CRITERION', 'a.mjs', 1)]);
  const d = fidelityDiffStructured(a, b);
  assert.equal(d.jaccard, 1); // same triplet → same ID
  assert.equal(d.severityMismatches.length, 1);
  assert.equal(d.equivalent, false);
});