// lib/review/gold-scorer.mjs // SKAL-1·4b — offline gold-scored output eval scorer. // // Scores a recorded agent-run's findings against a golden corpus record // (voyage-eval-gold/1, see docs/eval-corpus/README.md) at (file, rule_key) // granularity. line and severity are deliberately NOT part of the match key: // the eval asks "did the run surface this issue at all", and (file, rule_key) // is the identity pair a delivered diff + the catalogue uniquely determine. // // Pairs with coordinator-contract.mjs (4a): that module turns committed // reviewer payloads into the run findings scored here. This module is the // offline counterpart to the live LLM-in-the-loop eval (4c) — pure: no I/O, // no LLM, no network, no time, no randomness. // // Vacuous-set conventions (documented so the degenerate numbers are honest): // - precision when nothing is predicted (tp+fp === 0) -> 1 (no false positives) // - recall when there is nothing to find (tp+fn === 0) -> 1 (found all zero) // - f1 collapses to 0 whenever precision or recall is 0, so it stays the // honest single-number summary for the degenerate run/gold cases. // NUL separator: file paths and rule_keys never contain it, so the join is // an injective (file, rule_key) -> string key. const SEP = ''; function pairKey(f) { return `${f.file}${SEP}${f.rule_key}`; } // Human-readable form of a pair key (for matched/missed/spurious reporting). function pairLabel(key) { return key.replace(SEP, ' '); } // Set of unique (file, rule_key) pair keys from a findings list. Findings // missing either field are skipped (they cannot identify a pair). function pairSet(findings) { const s = new Set(); for (const f of findings ?? []) { if (f && typeof f.file === 'string' && typeof f.rule_key === 'string') { s.add(pairKey(f)); } } return s; } /** * Score run findings against gold findings at (file, rule_key) granularity. * @param {object[]|null|undefined} runFindings findings produced by the recorded run * @param {object[]|null|undefined} goldFindings the golden corpus findings * @returns {{ * tp: number, fp: number, fn: number, * precision: number, recall: number, f1: number, * matched: string[], missed: string[], spurious: string[] * }} matched/missed/spurious are " " labels. */ export function scoreFindings(runFindings, goldFindings) { const runPairs = pairSet(runFindings); const goldPairs = pairSet(goldFindings); const matched = []; const spurious = []; for (const p of runPairs) { (goldPairs.has(p) ? matched : spurious).push(pairLabel(p)); } const missed = []; for (const p of goldPairs) { if (!runPairs.has(p)) missed.push(pairLabel(p)); } const tp = matched.length; const fp = spurious.length; const fn = missed.length; const precision = tp + fp === 0 ? 1 : tp / (tp + fp); const recall = tp + fn === 0 ? 1 : tp / (tp + fn); const f1 = precision + recall === 0 ? 0 : (2 * precision * recall) / (precision + recall); return { tp, fp, fn, precision, recall, f1, matched, missed, spurious }; } /** * Exact match of a run's coordinator verdict against the gold expected_verdict. * @param {string} runVerdict * @param {string} goldVerdict * @returns {boolean} */ export function scoreVerdict(runVerdict, goldVerdict) { return runVerdict === goldVerdict; }