feat(eval): add gold.json golden corpus + loader/validator test
This commit is contained in:
parent
84fa055e9f
commit
e374a7c0ff
2 changed files with 112 additions and 0 deletions
68
tests/lib/gold-corpus.test.mjs
Normal file
68
tests/lib/gold-corpus.test.mjs
Normal file
|
|
@ -0,0 +1,68 @@
|
|||
// tests/lib/gold-corpus.test.mjs
|
||||
// SKAL-1·4a — golden corpus loader/validator.
|
||||
//
|
||||
// gold.json is the machine-readable form of the 5 seeded findings in
|
||||
// tests/fixtures/bakeoff-rich/README.md. This test pins its shape and ties
|
||||
// every rule_key / severity back to the canonical catalogue so the corpus
|
||||
// can never drift to an invented rule_key or severity tier.
|
||||
|
||||
import { test } from 'node:test';
|
||||
import { strict as assert } from 'node:assert';
|
||||
import { readFileSync } from 'node:fs';
|
||||
import { join, dirname } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { RULE_KEYS, SEVERITY_VALUES } from '../../lib/review/rule-catalogue.mjs';
|
||||
import { FINDING_REQUIRED_FIELDS } from '../../lib/review/findings-schema.mjs';
|
||||
|
||||
const HERE = dirname(fileURLToPath(import.meta.url));
|
||||
const ROOT = join(HERE, '..', '..');
|
||||
const GOLD = join(ROOT, 'tests/fixtures/bakeoff-rich/gold.json');
|
||||
|
||||
const gold = JSON.parse(readFileSync(GOLD, 'utf-8'));
|
||||
|
||||
test('gold.json — parses and has exactly 5 findings', () => {
|
||||
assert.ok(Array.isArray(gold.findings), 'findings must be an array');
|
||||
assert.equal(gold.findings.length, 5);
|
||||
});
|
||||
|
||||
test('gold.json — corpus-level expected_verdict is BLOCK', () => {
|
||||
assert.equal(gold.expected_verdict, 'BLOCK');
|
||||
});
|
||||
|
||||
test('gold.json — every finding carries the 5 required fields', () => {
|
||||
// FINDING_REQUIRED_FIELDS (severity, rule_key, file, line) + the eval addition owner_reviewer.
|
||||
const required = [...FINDING_REQUIRED_FIELDS, 'owner_reviewer'];
|
||||
for (const [i, f] of gold.findings.entries()) {
|
||||
for (const field of required) {
|
||||
assert.ok(field in f, `findings[${i}] missing required field "${field}"`);
|
||||
}
|
||||
assert.ok(typeof f.file === 'string' && f.file.length > 0, `findings[${i}].file empty`);
|
||||
assert.ok(Number.isInteger(f.line) && f.line >= 0, `findings[${i}].line must be an int >= 0`);
|
||||
}
|
||||
});
|
||||
|
||||
test('gold.json — every rule_key is a member of the canonical RULE_CATALOGUE', () => {
|
||||
for (const [i, f] of gold.findings.entries()) {
|
||||
assert.ok(RULE_KEYS.has(f.rule_key), `findings[${i}].rule_key "${f.rule_key}" not in catalogue`);
|
||||
}
|
||||
});
|
||||
|
||||
test('gold.json — every severity is a valid catalogue tier', () => {
|
||||
for (const [i, f] of gold.findings.entries()) {
|
||||
assert.ok(SEVERITY_VALUES.includes(f.severity), `findings[${i}].severity "${f.severity}" invalid`);
|
||||
}
|
||||
});
|
||||
|
||||
test('gold.json — owner_reviewer is one of the two reviewer roles', () => {
|
||||
for (const [i, f] of gold.findings.entries()) {
|
||||
assert.ok(
|
||||
f.owner_reviewer === 'conformance' || f.owner_reviewer === 'correctness',
|
||||
`findings[${i}].owner_reviewer "${f.owner_reviewer}" must be conformance|correctness`,
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
test('gold.json — contains at least one BLOCKER (consistent with expected_verdict BLOCK)', () => {
|
||||
const hasBlocker = gold.findings.some((f) => f.severity === 'BLOCKER');
|
||||
assert.ok(hasBlocker, 'expected_verdict is BLOCK but no BLOCKER finding present');
|
||||
});
|
||||
Loading…
Add table
Add a link
Reference in a new issue