voyage/tests/lib/gold-scorer.test.mjs

// tests/lib/gold-scorer.test.mjs
// SKAL-1·4b — behavior test for the gold scorer.
//
// scoreFindings compares a recorded agent-run's findings against a golden
// corpus record at (file, rule_key) granularity (line + severity deliberately
// ignored). This pins the precision/recall/f1 math AND the discriminating
// paths (false negatives + false positives), not merely the all-match case —
// a scorer that always returned 1.0 would pass an all-match-only test.

import { test } from 'node:test';
import { strict as assert } from 'node:assert';
import { scoreFindings, scoreVerdict } from '../../lib/review/gold-scorer.mjs';

// A minimal gold set: two distinct (file, rule_key) pairs.
const GOLD = [
  { file: 'a.mjs', line: 10, rule_key: 'SECURITY_INJECTION', severity: 'BLOCKER' },
  { file: 'b.mjs', line: 0, rule_key: 'MISSING_TEST', severity: 'MAJOR' },
];

test('scoreFindings — perfect match scores precision/recall/f1 = 1', () => {
  // Same pairs, different line/severity (must be ignored at (file,rule_key) granularity).
  const run = [
    { file: 'a.mjs', line: 99, rule_key: 'SECURITY_INJECTION', severity: 'MINOR' },
    { file: 'b.mjs', line: 7, rule_key: 'MISSING_TEST', severity: 'BLOCKER' },
  ];
  const s = scoreFindings(run, GOLD);
  assert.equal(s.tp, 2);
  assert.equal(s.fp, 0);
  assert.equal(s.fn, 0);
  assert.equal(s.precision, 1);
  assert.equal(s.recall, 1);
  assert.equal(s.f1, 1);
});

test('scoreFindings — a missed gold pair is a false negative (recall < 1)', () => {
  const run = [{ file: 'a.mjs', line: 10, rule_key: 'SECURITY_INJECTION', severity: 'BLOCKER' }];
  const s = scoreFindings(run, GOLD);
  assert.equal(s.tp, 1);
  assert.equal(s.fp, 0);
  assert.equal(s.fn, 1);
  assert.equal(s.precision, 1);
  assert.equal(s.recall, 0.5);
  assert.deepEqual(s.missed, ['b.mjs MISSING_TEST']);
});

test('scoreFindings — a spurious pair is a false positive (precision < 1)', () => {
  const run = [
    { file: 'a.mjs', line: 10, rule_key: 'SECURITY_INJECTION', severity: 'BLOCKER' },
    { file: 'b.mjs', line: 0, rule_key: 'MISSING_TEST', severity: 'MAJOR' },
    { file: 'c.mjs', line: 3, rule_key: 'PLACEHOLDER_IN_CODE', severity: 'MAJOR' },
  ];
  const s = scoreFindings(run, GOLD);
  assert.equal(s.tp, 2);
  assert.equal(s.fp, 1);
  assert.equal(s.fn, 0);
  assert.equal(s.recall, 1);
  assert.equal(s.precision, 2 / 3);
  assert.deepEqual(s.spurious, ['c.mjs PLACEHOLDER_IN_CODE']);
});

test('scoreFindings — mixed FN + FP', () => {
  const run = [
    { file: 'a.mjs', line: 10, rule_key: 'SECURITY_INJECTION', severity: 'BLOCKER' }, // match
    { file: 'c.mjs', line: 3, rule_key: 'PLACEHOLDER_IN_CODE', severity: 'MAJOR' }, // spurious
  ];
  const s = scoreFindings(run, GOLD);
  assert.equal(s.tp, 1);
  assert.equal(s.fp, 1);
  assert.equal(s.fn, 1);
  assert.equal(s.precision, 0.5);
  assert.equal(s.recall, 0.5);
  assert.equal(s.f1, 0.5);
});

test('scoreFindings — duplicate (file,rule_key) pairs collapse (set semantics)', () => {
  const run = [
    { file: 'a.mjs', line: 10, rule_key: 'SECURITY_INJECTION', severity: 'BLOCKER' },
    { file: 'a.mjs', line: 22, rule_key: 'SECURITY_INJECTION', severity: 'BLOCKER' }, // same pair
    { file: 'b.mjs', line: 0, rule_key: 'MISSING_TEST', severity: 'MAJOR' },
  ];
  const s = scoreFindings(run, GOLD);
  assert.equal(s.tp, 2);
  assert.equal(s.fp, 0);
  assert.equal(s.fn, 0);
});

test('scoreFindings — empty run: recall 0 (nothing found), precision vacuously 1, f1 0', () => {
  const s = scoreFindings([], GOLD);
  assert.equal(s.tp, 0);
  assert.equal(s.fp, 0);
  assert.equal(s.fn, 2);
  assert.equal(s.recall, 0);
  assert.equal(s.precision, 1);
  assert.equal(s.f1, 0);
});

test('scoreFindings — empty gold: precision 0 (all spurious), recall vacuously 1, f1 0', () => {
  const run = [{ file: 'a.mjs', line: 10, rule_key: 'SECURITY_INJECTION', severity: 'BLOCKER' }];
  const s = scoreFindings(run, []);
  assert.equal(s.tp, 0);
  assert.equal(s.fp, 1);
  assert.equal(s.fn, 0);
  assert.equal(s.precision, 0);
  assert.equal(s.recall, 1);
  assert.equal(s.f1, 0);
});

test('scoreFindings — both empty: precision/recall/f1 = 1 (matched nothing perfectly)', () => {
  const s = scoreFindings([], []);
  assert.equal(s.precision, 1);
  assert.equal(s.recall, 1);
  assert.equal(s.f1, 1);
});

test('scoreFindings — tolerates null/undefined finding arrays', () => {
  const s = scoreFindings(null, undefined);
  assert.equal(s.tp, 0);
  assert.equal(s.fp, 0);
  assert.equal(s.fn, 0);
});

test('scoreVerdict — exact verdict match is true, mismatch is false', () => {
  assert.equal(scoreVerdict('BLOCK', 'BLOCK'), true);
  assert.equal(scoreVerdict('WARN', 'BLOCK'), false);
  assert.equal(scoreVerdict('ALLOW', 'ALLOW'), true);
});