voyage/lib/review/gold-scorer.mjs

// lib/review/gold-scorer.mjs
// SKAL-1·4b — offline gold-scored output eval scorer.
//
// Scores a recorded agent-run's findings against a golden corpus record
// (voyage-eval-gold/1, see docs/eval-corpus/README.md) at (file, rule_key)
// granularity. line and severity are deliberately NOT part of the match key:
// the eval asks "did the run surface this issue at all", and (file, rule_key)
// is the identity pair a delivered diff + the catalogue uniquely determine.
//
// Pairs with coordinator-contract.mjs (4a): that module turns committed
// reviewer payloads into the run findings scored here. This module is the
// offline counterpart to the live LLM-in-the-loop eval (4c) — pure: no I/O,
// no LLM, no network, no time, no randomness.
//
// Vacuous-set conventions (documented so the degenerate numbers are honest):
//   - precision when nothing is predicted (tp+fp === 0)  -> 1 (no false positives)
//   - recall when there is nothing to find (tp+fn === 0) -> 1 (found all zero)
//   - f1 collapses to 0 whenever precision or recall is 0, so it stays the
//     honest single-number summary for the degenerate run/gold cases.

// NUL separator: file paths and rule_keys never contain it, so the join is
// an injective (file, rule_key) -> string key.
const SEP = '';

function pairKey(f) {
  return `${f.file}${SEP}${f.rule_key}`;
}

// Human-readable form of a pair key (for matched/missed/spurious reporting).
function pairLabel(key) {
  return key.replace(SEP, ' ');
}

// Set of unique (file, rule_key) pair keys from a findings list. Findings
// missing either field are skipped (they cannot identify a pair).
function pairSet(findings) {
  const s = new Set();
  for (const f of findings ?? []) {
    if (f && typeof f.file === 'string' && typeof f.rule_key === 'string') {
      s.add(pairKey(f));
    }
  }
  return s;
}

/**
 * Score run findings against gold findings at (file, rule_key) granularity.
 * @param {object[]|null|undefined} runFindings findings produced by the recorded run
 * @param {object[]|null|undefined} goldFindings the golden corpus findings
 * @returns {{
 *   tp: number, fp: number, fn: number,
 *   precision: number, recall: number, f1: number,
 *   matched: string[], missed: string[], spurious: string[]
 * }} matched/missed/spurious are "<file> <rule_key>" labels.
 */
export function scoreFindings(runFindings, goldFindings) {
  const runPairs = pairSet(runFindings);
  const goldPairs = pairSet(goldFindings);

  const matched = [];
  const spurious = [];
  for (const p of runPairs) {
    (goldPairs.has(p) ? matched : spurious).push(pairLabel(p));
  }
  const missed = [];
  for (const p of goldPairs) {
    if (!runPairs.has(p)) missed.push(pairLabel(p));
  }

  const tp = matched.length;
  const fp = spurious.length;
  const fn = missed.length;

  const precision = tp + fp === 0 ? 1 : tp / (tp + fp);
  const recall = tp + fn === 0 ? 1 : tp / (tp + fn);
  const f1 = precision + recall === 0 ? 0 : (2 * precision * recall) / (precision + recall);

  return { tp, fp, fn, precision, recall, f1, matched, missed, spurious };
}

/**
 * Exact match of a run's coordinator verdict against the gold expected_verdict.
 * @param {string} runVerdict
 * @param {string} goldVerdict
 * @returns {boolean}
 */
export function scoreVerdict(runVerdict, goldVerdict) {
  return runVerdict === goldVerdict;
}