Kjell Tore Guttormsen
f1c2c9eb4a
lib/review/gold-scorer.mjs held its (file, rule_key) separator as a literal NUL byte
inside a string literal. Git's binary heuristic therefore classed the whole file as
binary. Every diff of the scorer read "Binary files differ", `--numstat` printed "- -",
and plain grep matched nothing in it. The separator is now written as the escape
sequence '\x00', which is the same one-character string at runtime.
Proof that the change is behaviour-neutral:
- A snapshot script recorded every number the scorer produces, before and after:
- the committed run-perfect fixture through the coordinator contract;
- empty, half, spurious (keys containing spaces and colons), duplicated and empty-gold
runs;
- all 9 verdict pairs.
`diff` of the two outputs is empty (141 lines each).
- tests/lib/gold-scorer.test.mjs + tests/lib/gold-eval.test.mjs: 13/13 before and after.
- `git diff --no-index --numstat /dev/null <file>`: the new file reads "89 0" (text); the
old blob, as a control, reads "- -" (binary).
End-state gate D-06: open -> closed (defects 4 -> 3 of 7). Suite 1059 (1057/0/2), run
on a clean export of the index.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
89 lines
3.3 KiB
JavaScript
89 lines
3.3 KiB
JavaScript
// lib/review/gold-scorer.mjs
|
|
// SKAL-1·4b — offline gold-scored output eval scorer.
|
|
//
|
|
// Scores a recorded agent-run's findings against a golden corpus record
|
|
// (voyage-eval-gold/1, see docs/eval-corpus/README.md) at (file, rule_key)
|
|
// granularity. line and severity are deliberately NOT part of the match key:
|
|
// the eval asks "did the run surface this issue at all", and (file, rule_key)
|
|
// is the identity pair a delivered diff + the catalogue uniquely determine.
|
|
//
|
|
// Pairs with coordinator-contract.mjs (4a): that module turns committed
|
|
// reviewer payloads into the run findings scored here. This module is the
|
|
// offline counterpart to the live LLM-in-the-loop eval (4c) — pure: no I/O,
|
|
// no LLM, no network, no time, no randomness.
|
|
//
|
|
// Vacuous-set conventions (documented so the degenerate numbers are honest):
|
|
// - precision when nothing is predicted (tp+fp === 0) -> 1 (no false positives)
|
|
// - recall when there is nothing to find (tp+fn === 0) -> 1 (found all zero)
|
|
// - f1 collapses to 0 whenever precision or recall is 0, so it stays the
|
|
// honest single-number summary for the degenerate run/gold cases.
|
|
|
|
// NUL separator: file paths and rule_keys never contain it, so the join is
|
|
// an injective (file, rule_key) -> string key.
|
|
const SEP = '\x00';
|
|
|
|
function pairKey(f) {
|
|
return `${f.file}${SEP}${f.rule_key}`;
|
|
}
|
|
|
|
// Human-readable form of a pair key (for matched/missed/spurious reporting).
|
|
function pairLabel(key) {
|
|
return key.replace(SEP, ' ');
|
|
}
|
|
|
|
// Set of unique (file, rule_key) pair keys from a findings list. Findings
|
|
// missing either field are skipped (they cannot identify a pair).
|
|
function pairSet(findings) {
|
|
const s = new Set();
|
|
for (const f of findings ?? []) {
|
|
if (f && typeof f.file === 'string' && typeof f.rule_key === 'string') {
|
|
s.add(pairKey(f));
|
|
}
|
|
}
|
|
return s;
|
|
}
|
|
|
|
/**
|
|
* Score run findings against gold findings at (file, rule_key) granularity.
|
|
* @param {object[]|null|undefined} runFindings findings produced by the recorded run
|
|
* @param {object[]|null|undefined} goldFindings the golden corpus findings
|
|
* @returns {{
|
|
* tp: number, fp: number, fn: number,
|
|
* precision: number, recall: number, f1: number,
|
|
* matched: string[], missed: string[], spurious: string[]
|
|
* }} matched/missed/spurious are "<file> <rule_key>" labels.
|
|
*/
|
|
export function scoreFindings(runFindings, goldFindings) {
|
|
const runPairs = pairSet(runFindings);
|
|
const goldPairs = pairSet(goldFindings);
|
|
|
|
const matched = [];
|
|
const spurious = [];
|
|
for (const p of runPairs) {
|
|
(goldPairs.has(p) ? matched : spurious).push(pairLabel(p));
|
|
}
|
|
const missed = [];
|
|
for (const p of goldPairs) {
|
|
if (!runPairs.has(p)) missed.push(pairLabel(p));
|
|
}
|
|
|
|
const tp = matched.length;
|
|
const fp = spurious.length;
|
|
const fn = missed.length;
|
|
|
|
const precision = tp + fp === 0 ? 1 : tp / (tp + fp);
|
|
const recall = tp + fn === 0 ? 1 : tp / (tp + fn);
|
|
const f1 = precision + recall === 0 ? 0 : (2 * precision * recall) / (precision + recall);
|
|
|
|
return { tp, fp, fn, precision, recall, f1, matched, missed, spurious };
|
|
}
|
|
|
|
/**
|
|
* Exact match of a run's coordinator verdict against the gold expected_verdict.
|
|
* @param {string} runVerdict
|
|
* @param {string} goldVerdict
|
|
* @returns {boolean}
|
|
*/
|
|
export function scoreVerdict(runVerdict, goldVerdict) {
|
|
return runVerdict === goldVerdict;
|
|
}
|