open/voyage
1
0
Fork 0
Code Issues Releases 1 Activity

feat(eval): add gold.json golden corpus + loader/validator test

Browse source
This commit is contained in:
Kjell Tore Guttormsen 2026-06-26 11:51:46 +02:00
commit e374a7c0ff
2 changed files with 112 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

44
tests/fixtures/bakeoff-rich/gold.json vendored Normal file
View file

@ -0,0 +1,44 @@
{
"schema": "voyage-eval-gold/1",
"source": "tests/fixtures/bakeoff-rich/README.md",
"description": "Golden corpus for SKAL-1·4a — the 5 brief-traceable seeded findings of the bakeoff-rich JWT-auth fixture, machine-readable. expected_verdict is the review-coordinator Pass-4 outcome (BLOCKER >= 1 -> BLOCK).",
"expected_verdict": "BLOCK",
"findings": [
{
"file": "lib/handlers/login.mjs",
"line": 17,
"rule_key": "UNIMPLEMENTED_CRITERION",
"severity": "BLOCKER",
"owner_reviewer": "conformance"
},
{
"file": "lib/auth/jwt.mjs",
"line": 19,
"rule_key": "SECURITY_INJECTION",
"severity": "BLOCKER",
"owner_reviewer": "correctness",
"dual_flaggable": "NON_GOAL_VIOLATED"
},
{
"file": "lib/auth/refresh.mjs",
"line": 0,
"rule_key": "MISSING_TEST",
"severity": "MAJOR",
"owner_reviewer": "correctness"
},
{
"file": "lib/handlers/login.mjs",
"line": 13,
"rule_key": "PLAN_EXECUTE_DRIFT",
"severity": "MAJOR",
"owner_reviewer": "conformance"
},
{
"file": "lib/auth/refresh.mjs",
"line": 10,
"rule_key": "MISSING_ERROR_HANDLING",
"severity": "MINOR",
"owner_reviewer": "correctness"
}
]
}