import { test } from 'node:test'; import { strict as assert } from 'node:assert'; import { readFileSync } from 'node:fs'; import { join, dirname } from 'node:path'; import { fileURLToPath } from 'node:url'; import { parseReviewArtifact, fidelityDiff, fidelityDiffStructured, normalizeArmOutput, } from '../../lib/review/fidelity-diff.mjs'; import { computeFindingId } from '../../lib/parsers/finding-id.mjs'; const HERE = dirname(fileURLToPath(import.meta.url)); const ROOT = join(HERE, '..', '..'); function fixture(name) { return readFileSync(join(ROOT, 'tests/fixtures/trekreview', name), 'utf-8'); } const RUN_A = fixture('review-run-A.md'); const RUN_B = fixture('review-run-B.md'); // ---- parseReviewArtifact ---------------------------------------------------- test('parseReviewArtifact — extracts verdict + finding IDs from frontmatter', () => { const a = parseReviewArtifact(RUN_A); assert.equal(a.verdict, 'WARN'); assert.ok(Array.isArray(a.findingIds)); assert.equal(a.findingIds.length, 5); for (const id of a.findingIds) assert.match(id, /^[0-9a-f]{40}$/); }); test('parseReviewArtifact — extracts per-finding details from trailing JSON', () => { const a = parseReviewArtifact(RUN_A); assert.ok(Array.isArray(a.details)); assert.equal(a.details.length, 5); const blocker = a.details.find((d) => d.severity === 'BLOCKER'); assert.ok(blocker); // fixtures use the key "rule"; real reviewer output uses "rule_key" — both accepted assert.equal(blocker.rule_key, 'UNIMPLEMENTED_CRITERION'); assert.equal(blocker.file, 'lib/handlers/login.mjs'); assert.equal(blocker.line, 23); }); test('parseReviewArtifact — tolerates a missing trailing JSON block (details = [])', () => { const noJson = '---\ntype: trekreview\nverdict: ALLOW\nfindings: []\n---\n\n# Review\n\nNo findings.\n'; const r = parseReviewArtifact(noJson); assert.equal(r.verdict, 'ALLOW'); assert.deepEqual(r.findingIds, []); assert.deepEqual(r.details, []); }); // ---- fidelityDiff ----------------------------------------------------------- test('fidelityDiff — identical artifact is fully equivalent (jaccard 1.0)', () => { const d = fidelityDiff(RUN_A, RUN_A); assert.equal(d.verdictMatch, true); assert.equal(d.jaccard, 1); assert.equal(d.equivalent, true); assert.equal(d.severityMismatches.length, 0); assert.equal(d.ruleKeyMismatches.length, 0); }); test('fidelityDiff — A vs B (A ⊂ B): same verdict, jaccard 5/6, equivalent at default tolerance', () => { const d = fidelityDiff(RUN_A, RUN_B); assert.equal(d.verdictMatch, true); assert.ok(Math.abs(d.jaccard - 5 / 6) < 1e-9, `jaccard=${d.jaccard}`); assert.equal(d.equivalent, true); // default jaccardTolerance 0.7 }); test('fidelityDiff — divergent verdict makes it non-equivalent even at jaccard 1.0', () => { const a = parseReviewArtifact(RUN_A); const flipped = RUN_A.replace('verdict: WARN', 'verdict: BLOCK'); const d = fidelityDiff(RUN_A, flipped); assert.equal(d.verdictMatch, false); assert.equal(d.jaccard, 1); assert.equal(d.equivalent, false); void a; }); test('fidelityDiff — tolerance gate: jaccard below tolerance is non-equivalent', () => { const dLoose = fidelityDiff(RUN_A, RUN_B, { jaccardTolerance: 0.7 }); const dStrict = fidelityDiff(RUN_A, RUN_B, { jaccardTolerance: 0.95 }); assert.equal(dLoose.equivalent, true); assert.equal(dStrict.equivalent, false); }); test('fidelityDiff — flags severity mismatch on a shared finding ID', () => { // Flip the BLOCKER severity in B's trailing JSON for the shared login.mjs finding. const sharedId = '763d174e6c519fafbadcba5d1706708479e36e61'; const tampered = RUN_B.replace( `"id": "${sharedId}", "severity": "BLOCKER"`, `"id": "${sharedId}", "severity": "MINOR"`, ); const d = fidelityDiff(RUN_A, tampered); assert.ok(d.severityMismatches.some((m) => m.id === sharedId), `expected a severity mismatch for ${sharedId}; got ${JSON.stringify(d.severityMismatches)}`); assert.equal(d.equivalent, false); }); test('fidelityDiff — exposes finding counts for both arms', () => { const d = fidelityDiff(RUN_A, RUN_B); assert.equal(d.countA, 5); assert.equal(d.countB, 6); }); // ---- structured arm-output path (the bake-off comparison) ------------------- const ARM = (verdict, findings) => ({ verdict, findings }); const F = (severity, rule_key, file, line) => ({ severity, rule_key, file, line }); test('normalizeArmOutput — recomputes canonical IDs from the (file,line,rule_key) triplet', () => { const arm = ARM('WARN', [F('BLOCKER', 'SECURITY_INJECTION', 'lib/exec.mjs', 23)]); const n = normalizeArmOutput(arm); assert.equal(n.verdict, 'WARN'); assert.equal(n.findingIds.length, 1); assert.equal(n.findingIds[0], computeFindingId('lib/exec.mjs', 23, 'SECURITY_INJECTION')); }); test('normalizeArmOutput — drops findings missing file/rule_key from the ID set', () => { const arm = ARM('WARN', [F('MAJOR', 'MISSING_TEST', '', 0), { severity: 'MINOR', line: 5 }]); const n = normalizeArmOutput(arm); assert.equal(n.findingIds.length, 0); assert.equal(n.details.length, 2); // still recorded, just not ID-keyed }); test('fidelityDiffStructured — identical arms are equivalent (jaccard 1.0)', () => { const findings = [ F('BLOCKER', 'UNIMPLEMENTED_CRITERION', 'a.mjs', 1), F('MAJOR', 'MISSING_TEST', 'b.mjs', 0), ]; const d = fidelityDiffStructured(ARM('BLOCK', findings), ARM('BLOCK', findings)); assert.equal(d.equivalent, true); assert.equal(d.jaccard, 1); assert.equal(d.verdictMatch, true); }); test('fidelityDiffStructured — divergent verdict fails even with identical findings', () => { const findings = [F('BLOCKER', 'UNIMPLEMENTED_CRITERION', 'a.mjs', 1)]; const d = fidelityDiffStructured(ARM('BLOCK', findings), ARM('WARN', findings)); assert.equal(d.verdictMatch, false); assert.equal(d.equivalent, false); }); test('fidelityDiffStructured — extra finding in Arm B lowers jaccard', () => { const a = ARM('WARN', [F('MAJOR', 'MISSING_TEST', 'a.mjs', 10)]); const b = ARM('WARN', [ F('MAJOR', 'MISSING_TEST', 'a.mjs', 10), F('MINOR', 'PLACEHOLDER_IN_CODE', 'b.mjs', 14), ]); const d = fidelityDiffStructured(a, b); assert.ok(Math.abs(d.jaccard - 1 / 2) < 1e-9, `jaccard=${d.jaccard}`); assert.equal(d.countA, 1); assert.equal(d.countB, 2); }); test('fidelityDiffStructured — same triplet, different severity → severity mismatch', () => { const a = ARM('BLOCK', [F('BLOCKER', 'UNIMPLEMENTED_CRITERION', 'a.mjs', 1)]); const b = ARM('BLOCK', [F('MAJOR', 'UNIMPLEMENTED_CRITERION', 'a.mjs', 1)]); const d = fidelityDiffStructured(a, b); assert.equal(d.jaccard, 1); // same triplet → same ID assert.equal(d.severityMismatches.length, 1); assert.equal(d.equivalent, false); });