From e374a7c0ff67c698cdb914e7cf70b3e3242393d6 Mon Sep 17 00:00:00 2001
From: Kjell Tore Guttormsen <hello@fromaitochitta.com>
Date: Fri, 26 Jun 2026 11:51:46 +0200
Subject: [PATCH] feat(eval): add gold.json golden corpus + loader/validator
 test

---
 tests/fixtures/bakeoff-rich/gold.json | 44 +++++++++++++++++
 tests/lib/gold-corpus.test.mjs        | 68 +++++++++++++++++++++++++++
 2 files changed, 112 insertions(+)
 create mode 100644 tests/fixtures/bakeoff-rich/gold.json
 create mode 100644 tests/lib/gold-corpus.test.mjs

diff --git a/tests/fixtures/bakeoff-rich/gold.json b/tests/fixtures/bakeoff-rich/gold.json
new file mode 100644
index 0000000..5e3e711
--- /dev/null
+++ b/tests/fixtures/bakeoff-rich/gold.json
@@ -0,0 +1,44 @@
+{
+  "schema": "voyage-eval-gold/1",
+  "source": "tests/fixtures/bakeoff-rich/README.md",
+  "description": "Golden corpus for SKAL-1·4a — the 5 brief-traceable seeded findings of the bakeoff-rich JWT-auth fixture, machine-readable. expected_verdict is the review-coordinator Pass-4 outcome (BLOCKER >= 1 -> BLOCK).",
+  "expected_verdict": "BLOCK",
+  "findings": [
+    {
+      "file": "lib/handlers/login.mjs",
+      "line": 17,
+      "rule_key": "UNIMPLEMENTED_CRITERION",
+      "severity": "BLOCKER",
+      "owner_reviewer": "conformance"
+    },
+    {
+      "file": "lib/auth/jwt.mjs",
+      "line": 19,
+      "rule_key": "SECURITY_INJECTION",
+      "severity": "BLOCKER",
+      "owner_reviewer": "correctness",
+      "dual_flaggable": "NON_GOAL_VIOLATED"
+    },
+    {
+      "file": "lib/auth/refresh.mjs",
+      "line": 0,
+      "rule_key": "MISSING_TEST",
+      "severity": "MAJOR",
+      "owner_reviewer": "correctness"
+    },
+    {
+      "file": "lib/handlers/login.mjs",
+      "line": 13,
+      "rule_key": "PLAN_EXECUTE_DRIFT",
+      "severity": "MAJOR",
+      "owner_reviewer": "conformance"
+    },
+    {
+      "file": "lib/auth/refresh.mjs",
+      "line": 10,
+      "rule_key": "MISSING_ERROR_HANDLING",
+      "severity": "MINOR",
+      "owner_reviewer": "correctness"
+    }
+  ]
+}
diff --git a/tests/lib/gold-corpus.test.mjs b/tests/lib/gold-corpus.test.mjs
new file mode 100644
index 0000000..3ec9df6
--- /dev/null
+++ b/tests/lib/gold-corpus.test.mjs
@@ -0,0 +1,68 @@
+// tests/lib/gold-corpus.test.mjs
+// SKAL-1·4a — golden corpus loader/validator.
+//
+// gold.json is the machine-readable form of the 5 seeded findings in
+// tests/fixtures/bakeoff-rich/README.md. This test pins its shape and ties
+// every rule_key / severity back to the canonical catalogue so the corpus
+// can never drift to an invented rule_key or severity tier.
+
+import { test } from 'node:test';
+import { strict as assert } from 'node:assert';
+import { readFileSync } from 'node:fs';
+import { join, dirname } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { RULE_KEYS, SEVERITY_VALUES } from '../../lib/review/rule-catalogue.mjs';
+import { FINDING_REQUIRED_FIELDS } from '../../lib/review/findings-schema.mjs';
+
+const HERE = dirname(fileURLToPath(import.meta.url));
+const ROOT = join(HERE, '..', '..');
+const GOLD = join(ROOT, 'tests/fixtures/bakeoff-rich/gold.json');
+
+const gold = JSON.parse(readFileSync(GOLD, 'utf-8'));
+
+test('gold.json — parses and has exactly 5 findings', () => {
+  assert.ok(Array.isArray(gold.findings), 'findings must be an array');
+  assert.equal(gold.findings.length, 5);
+});
+
+test('gold.json — corpus-level expected_verdict is BLOCK', () => {
+  assert.equal(gold.expected_verdict, 'BLOCK');
+});
+
+test('gold.json — every finding carries the 5 required fields', () => {
+  // FINDING_REQUIRED_FIELDS (severity, rule_key, file, line) + the eval addition owner_reviewer.
+  const required = [...FINDING_REQUIRED_FIELDS, 'owner_reviewer'];
+  for (const [i, f] of gold.findings.entries()) {
+    for (const field of required) {
+      assert.ok(field in f, `findings[${i}] missing required field "${field}"`);
+    }
+    assert.ok(typeof f.file === 'string' && f.file.length > 0, `findings[${i}].file empty`);
+    assert.ok(Number.isInteger(f.line) && f.line >= 0, `findings[${i}].line must be an int >= 0`);
+  }
+});
+
+test('gold.json — every rule_key is a member of the canonical RULE_CATALOGUE', () => {
+  for (const [i, f] of gold.findings.entries()) {
+    assert.ok(RULE_KEYS.has(f.rule_key), `findings[${i}].rule_key "${f.rule_key}" not in catalogue`);
+  }
+});
+
+test('gold.json — every severity is a valid catalogue tier', () => {
+  for (const [i, f] of gold.findings.entries()) {
+    assert.ok(SEVERITY_VALUES.includes(f.severity), `findings[${i}].severity "${f.severity}" invalid`);
+  }
+});
+
+test('gold.json — owner_reviewer is one of the two reviewer roles', () => {
+  for (const [i, f] of gold.findings.entries()) {
+    assert.ok(
+      f.owner_reviewer === 'conformance' || f.owner_reviewer === 'correctness',
+      `findings[${i}].owner_reviewer "${f.owner_reviewer}" must be conformance|correctness`,
+    );
+  }
+});
+
+test('gold.json — contains at least one BLOCKER (consistent with expected_verdict BLOCK)', () => {
+  const hasBlocker = gold.findings.some((f) => f.severity === 'BLOCKER');
+  assert.ok(hasBlocker, 'expected_verdict is BLOCK but no BLOCKER finding present');
+});