open/llm-ingestion-okf
1
0
Fork 0
Code Issues Releases 4 Activity

feat(retrieval-gate): row 5 reads the threshold as a number and compares it with the hold-out

Browse source 
Two checks replace `bool(threshold)`: `_as_share` parses it as a share in
[0, 1] -- so `report-only; any number is acceptable for v1` is a NO, and so
is `80` -- and `_hold_out_verdict` RUNS the registered set against the
registered bundle and prints `answered of asked = share against threshold`.

A registration naming an absent set, an unreadable bundle or an empty set
is a NO with its reason, never an exception and never a silent pass. The
row is 11 checks; it stays RED today because no registration exists.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Kjell Tore Guttormsen 2026-09-20 08:11:03 +02:00
commit 735fb237e4
Signed by: ktg
SSH key fingerprint: SHA256:JakMjO6FTBBzN0Bhfj9saOoEjaFxlSdYuZQQpM/lF9Q
2 changed files with 117 additions and 68 deletions
Download patch file Download diff file Expand all files Collapse all files

98
tests/test_retrieval_gate.py
View file

@ -349,49 +349,33 @@ def test_row_five_is_red_while_no_hold_out_is_registered(tmp_path: Path) -> None
assert (row.k, row.m, row.status) == (0, 1, gate.RED)
def test_row_five_is_green_for_a_registration_that_carries_all_seven(tmp_path: Path) -> None:
held = tmp_path / "held.json"
held.write_text('{"questions": []}', encoding="utf-8")
registration = tmp_path / "registration.json"
registration.write_text(
json.dumps(
{
"set": str(held),
"sha256": gate.sha256_of(held),
"threshold": "hit@payload at unit granularity >= 0.8",
"threshold_written_at": "2026-09-19T10:00:00Z",
"written_by": "the session that wrote the eval, never the one that "
"changes the ranking",
"readings": [{"at": "2026-09-20T09:00:00Z", "value": "not read yet"}],
}
),
encoding="utf-8",
def test_row_five_is_green_for_a_registration_that_carries_all_eleven(
tmp_path: Path,
) -> None:
registration = _registration(
tmp_path,
set_path=FIXTURES / "set-positive.json",
bundle=_bundles(tmp_path)["positive"],
threshold=0.8,
)
row = gate.row_five(registration, provenance=lambda _: _carried_by_git())
assert (row.k, row.m, row.status) == (10, 10, gate.GREEN)
assert (row.k, row.m, row.status) == (11, 11, gate.GREEN)
def test_row_five_falls_on_a_number_read_before_its_threshold_was_written(
tmp_path: Path,
) -> None:
held = tmp_path / "held.json"
held.write_text('{"questions": []}', encoding="utf-8")
registration = tmp_path / "registration.json"
registration.write_text(
json.dumps(
{
"set": str(held),
"sha256": gate.sha256_of(held),
"threshold": "hit@payload at unit granularity >= 0.8",
"threshold_written_at": "2026-09-19T10:00:00Z",
"written_by": "somebody",
"readings": [{"at": "2026-09-18T09:00:00Z", "value": "0.62"}],
}
),
encoding="utf-8",
registration = _registration(
tmp_path,
set_path=FIXTURES / "set-positive.json",
bundle=_bundles(tmp_path)["positive"],
threshold=0.8,
)
spec = json.loads(registration.read_text(encoding="utf-8"))
spec["readings"] = [{"at": "2026-09-18T09:00:00Z", "value": "0.62"}]
registration.write_text(json.dumps(spec), encoding="utf-8")
row = gate.row_five(registration, provenance=lambda _: _carried_by_git())
assert (row.k, row.m, row.status) == (9, 10, gate.RED)
assert (row.k, row.m, row.status) == (10, 11, gate.RED)
assert any("no reading predates the threshold: NO" in detail for detail in row.details)
@ -401,21 +385,11 @@ def test_row_five_falls_when_the_registration_rides_in_on_the_ranking_change(
"""The three git checks, each driven red on its own: a file nobody
committed, a threshold committed together with the ranking change, and a
registration no ranking change has come after."""
held = tmp_path / "held.json"
held.write_text('{"questions": []}', encoding="utf-8")
registration = tmp_path / "registration.json"
registration.write_text(
json.dumps(
{
"set": str(held),
"sha256": gate.sha256_of(held),
"threshold": "hit@payload at unit granularity >= 0.8",
"threshold_written_at": "2026-09-19T10:00:00Z",
"written_by": "somebody",
"readings": [],
}
),
encoding="utf-8",
registration = _registration(
tmp_path,
set_path=FIXTURES / "set-positive.json",
bundle=_bundles(tmp_path)["positive"],
threshold=0.8,
)
arms = {
"uncommitted": (_carried_by_git(tracked=False), "committed"),
@ -957,22 +931,15 @@ def test_j1_a_registration_this_session_wrote_is_not_a_hold_out(tmp_path: Path)
`7 of 7 GREEN`. Every check was an assertion the registration made about
itself -- `written_by` is `bool()` of a string the file sets, and the
readings are read from a list the same file owns."""
held = tmp_path / "held-out.json"
held.write_text('{"questions": []}', encoding="utf-8")
registration = tmp_path / "registration.json"
registration.write_text(
json.dumps(
{
"set": str(held),
"sha256": gate.sha256_of(held),
"threshold": "hit@8 must not fall below 6 of 8",
"threshold_written_at": "2026-01-01",
"written_by": "the same session, lying",
"readings": [],
}
),
encoding="utf-8",
registration = _registration(
tmp_path,
set_path=FIXTURES / "set-positive.json",
bundle=_bundles(tmp_path)["positive"],
threshold=0.8,
)
spec = json.loads(registration.read_text(encoding="utf-8"))
spec["written_by"] = "the same session, lying"
registration.write_text(json.dumps(spec), encoding="utf-8")
row = gate.row_five(registration)
assert row.status != gate.GREEN, row
assert row.fails
@ -1107,8 +1074,7 @@ def test_row_eight_names_the_bundle_identity_of_every_set_it_measured(
"""
bundles = {"positive": _bundles(tmp_path)["positive"]}
real = [
(_hitting_set(name, quote="innen 1. november"), bundles)
for name in gate.REQUIRED_REAL_SETS
(_hitting_set(name, quote="innen 1. november"), bundles) for name in gate.REQUIRED_REAL_SETS
]
row = gate.row_eight(real)
# The identity is read here, independently of the gate, from the same