open/llm-ingestion-okf
1
0
Fork 0
Code Issues Releases 4 Activity

feat(retrieval-gate): row 5 reads the threshold as a number and compares it with the hold-out

Browse source 
Two checks replace `bool(threshold)`: `_as_share` parses it as a share in
[0, 1] -- so `report-only; any number is acceptable for v1` is a NO, and so
is `80` -- and `_hold_out_verdict` RUNS the registered set against the
registered bundle and prints `answered of asked = share against threshold`.

A registration naming an absent set, an unreadable bundle or an empty set
is a NO with its reason, never an exception and never a silent pass. The
row is 11 checks; it stays RED today because no registration exists.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Kjell Tore Guttormsen 2026-09-20 08:11:03 +02:00
commit 735fb237e4
Signed by: ktg
SSH key fingerprint: SHA256:JakMjO6FTBBzN0Bhfj9saOoEjaFxlSdYuZQQpM/lF9Q
2 changed files with 117 additions and 68 deletions
Download patch file Download diff file Expand all files Collapse all files

98
tests/test_retrieval_gate.py
View file

@ -349,49 +349,33 @@ def test_row_five_is_red_while_no_hold_out_is_registered(tmp_path: Path) -> None
assert (row.k, row.m, row.status) == (0, 1, gate.RED)
def test_row_five_is_green_for_a_registration_that_carries_all_seven(tmp_path: Path) -> None:
held = tmp_path / "held.json"
held.write_text('{"questions": []}', encoding="utf-8")
registration = tmp_path / "registration.json"
registration.write_text(
json.dumps(
{
"set": str(held),
"sha256": gate.sha256_of(held),
"threshold": "hit@payload at unit granularity >= 0.8",
"threshold_written_at": "2026-09-19T10:00:00Z",
"written_by": "the session that wrote the eval, never the one that "
"changes the ranking",
"readings": [{"at": "2026-09-20T09:00:00Z", "value": "not read yet"}],
}
),
encoding="utf-8",
def test_row_five_is_green_for_a_registration_that_carries_all_eleven(
tmp_path: Path,
) -> None:
registration = _registration(
tmp_path,
set_path=FIXTURES / "set-positive.json",
bundle=_bundles(tmp_path)["positive"],
threshold=0.8,
)
row = gate.row_five(registration, provenance=lambda _: _carried_by_git())
assert (row.k, row.m, row.status) == (10, 10, gate.GREEN)
assert (row.k, row.m, row.status) == (11, 11, gate.GREEN)
def test_row_five_falls_on_a_number_read_before_its_threshold_was_written(
tmp_path: Path,
) -> None:
held = tmp_path / "held.json"
held.write_text('{"questions": []}', encoding="utf-8")
registration = tmp_path / "registration.json"
registration.write_text(
json.dumps(
{
"set": str(held),
"sha256": gate.sha256_of(held),
"threshold": "hit@payload at unit granularity >= 0.8",
"threshold_written_at": "2026-09-19T10:00:00Z",
"written_by": "somebody",
"readings": [{"at": "2026-09-18T09:00:00Z", "value": "0.62"}],
}
),
encoding="utf-8",
registration = _registration(
tmp_path,
set_path=FIXTURES / "set-positive.json",
bundle=_bundles(tmp_path)["positive"],
threshold=0.8,
)
spec = json.loads(registration.read_text(encoding="utf-8"))
spec["readings"] = [{"at": "2026-09-18T09:00:00Z", "value": "0.62"}]
registration.write_text(json.dumps(spec), encoding="utf-8")
row = gate.row_five(registration, provenance=lambda _: _carried_by_git())
assert (row.k, row.m, row.status) == (9, 10, gate.RED)
assert (row.k, row.m, row.status) == (10, 11, gate.RED)
assert any("no reading predates the threshold: NO" in detail for detail in row.details)
@ -401,21 +385,11 @@ def test_row_five_falls_when_the_registration_rides_in_on_the_ranking_change(
"""The three git checks, each driven red on its own: a file nobody
committed, a threshold committed together with the ranking change, and a
registration no ranking change has come after."""
held = tmp_path / "held.json"
held.write_text('{"questions": []}', encoding="utf-8")
registration = tmp_path / "registration.json"
registration.write_text(
json.dumps(
{
"set": str(held),
"sha256": gate.sha256_of(held),
"threshold": "hit@payload at unit granularity >= 0.8",
"threshold_written_at": "2026-09-19T10:00:00Z",
"written_by": "somebody",
"readings": [],
}
),
encoding="utf-8",
registration = _registration(
tmp_path,
set_path=FIXTURES / "set-positive.json",
bundle=_bundles(tmp_path)["positive"],
threshold=0.8,
)
arms = {
"uncommitted": (_carried_by_git(tracked=False), "committed"),
@ -957,22 +931,15 @@ def test_j1_a_registration_this_session_wrote_is_not_a_hold_out(tmp_path: Path)
`7 of 7 GREEN`. Every check was an assertion the registration made about
itself -- `written_by` is `bool()` of a string the file sets, and the
readings are read from a list the same file owns."""
held = tmp_path / "held-out.json"
held.write_text('{"questions": []}', encoding="utf-8")
registration = tmp_path / "registration.json"
registration.write_text(
json.dumps(
{
"set": str(held),
"sha256": gate.sha256_of(held),
"threshold": "hit@8 must not fall below 6 of 8",
"threshold_written_at": "2026-01-01",
"written_by": "the same session, lying",
"readings": [],
}
),
encoding="utf-8",
registration = _registration(
tmp_path,
set_path=FIXTURES / "set-positive.json",
bundle=_bundles(tmp_path)["positive"],
threshold=0.8,
)
spec = json.loads(registration.read_text(encoding="utf-8"))
spec["written_by"] = "the same session, lying"
registration.write_text(json.dumps(spec), encoding="utf-8")
row = gate.row_five(registration)
assert row.status != gate.GREEN, row
assert row.fails
@ -1107,8 +1074,7 @@ def test_row_eight_names_the_bundle_identity_of_every_set_it_measured(
"""
bundles = {"positive": _bundles(tmp_path)["positive"]}
real = [
(_hitting_set(name, quote="innen 1. november"), bundles)
for name in gate.REQUIRED_REAL_SETS
(_hitting_set(name, quote="innen 1. november"), bundles) for name in gate.REQUIRED_REAL_SETS
]
row = gate.row_eight(real)
# The identity is read here, independently of the gate, from the same

87
tools/okf_retrieval_gate.py
View file

@ -1285,6 +1285,61 @@ def _display(path: Path) -> str:
return str(path)
def _as_share(value: object) -> float | None:
"""A threshold as a number in [0, 1], or None.
A SHARE and not any float: every metric this gate carries is `k of N`, so
a threshold of `80` is either 80 % written wrongly or a bar no run can
clear, and both are refusals rather than guesses.
"""
try:
number = float(str(value).strip())
except (TypeError, ValueError):
return None
if not 0.0 <= number <= 1.0:
return None
return number
def _hold_out_verdict(
spec: Mapping[str, object],
set_path: Path,
pinned: str,
threshold: float | None,
) -> tuple[bool, str]:
"""Run the registered hold-out set against the registered bundle and put
its share beside the threshold.
Every refusal is a NO with its reason and never an exception: a
registration naming an absent set is a finding about the registration.
"""
if threshold is None:
return False, "no numeric threshold to compare with"
bundle = str(spec.get("bundle", ""))
if not bundle:
return False, "the registration names no bundle to measure against"
if not set_path.is_file():
return False, f"the set is absent at {_display(set_path)}"
try:
question_set = load_set(set_path, pinned)
bundles = _bundle_map(bundle)
if list(bundles) == [""] and question_set.bundle:
bundles = {question_set.bundle: bundles[""]}
units = [
unit
for question in question_set.questions
for unit in measure_units(bundles[question.bundle or question_set.bundle], question)
]
except Exception as error: # a registration that cannot be run is a NO
return False, f"the hold-out did not run: {type(error).__name__}: {error}"
asked = len({unit.question_id for unit in units})
if not asked:
return False, "the hold-out set carries no question; a share over 0 is not a number"
answered = len({unit.question_id for unit in units if unit.hit})
share = answered / asked
return share >= threshold, f"{answered} of {asked} = {share:.4f} against {threshold:.4f}"
def row_five(
registration: Path = HOLDOUT_REGISTRATION,
*,
@ -1296,6 +1351,14 @@ def row_five(
Report-only without a written threshold is not a protection, so the
absence of a threshold is red rather than absent.
AND A THRESHOLD IS A NUMBER THAT IS COMPARED WITH ONE. Until 2026-09-20
the check was `bool(threshold)`, so the threshold `report-only; any number
is acceptable for v1` read as `a threshold is written: yes`. Two checks
now: it parses as a share in [0, 1], and the registered set is RUN against
the registered bundle so its own `answered of asked` stands beside it. A
registration naming an absent set, an unreadable bundle or an empty set is
a NO with its reason -- never an exception, and never a silent pass.
AND NEITHER IS A PROTECTION THE FILE WRITES ABOUT ITSELF. Until
2026-09-19 every check here read a field the registration owned, and two
files PM wrote in the moment came back `7 of 7 GREEN`. Three checks now
@ -1322,7 +1385,12 @@ def row_five(
[
" a set written by a session other than the one that changes the "
"ranking, frozen with sha256 before the first capability line",
" its threshold written, with a date, before its number is read",
" its threshold written as a NUMBER in [0, 1], with a date, "
"before its number is read -- `report-only` passed the old check "
"and could fell nothing",
" and a bundle named, so the set can be RUN and its share put "
"beside the threshold. A number never compared with a measurement "
"is a note",
" and COMMITTED before the ranking moves: git must show the "
f"registration in a commit of its own, with a later commit to "
f"{RANKING_PATH}. That is the half a session cannot write about itself",
@ -1340,7 +1408,14 @@ def row_five(
readings = spec.get("readings", [])
checks.append(("a set is named", bool(str(spec.get("set", ""))), str(set_path)))
checks.append(("a sha256 is pinned", len(pinned) == 64, pinned[:12]))
checks.append(("a threshold is written", bool(threshold), threshold))
number = _as_share(threshold)
checks.append(
(
"the threshold is a number",
number is not None,
threshold if number is None else f"{number:.4f}",
)
)
checks.append(("the threshold carries a date", bool(written_at), written_at))
checks.append(
(
@ -1368,6 +1443,14 @@ def row_five(
f"{len(early)} reading(s) before {written_at}" if early else "0 early readings",
)
)
# AND IT IS COMPARED WITH SOMETHING. A number written down and never put
# beside a measurement is a note, not a gate: `bool(threshold)` was the
# whole check until 2026-09-20, and `report-only; any number is acceptable
# for v1` passed it. The hold-out is run HERE, against the bundle the
# registration names, and the row says what it measured.
cleared, note = _hold_out_verdict(spec, set_path, pinned, number)
checks.append(("the measured hold-out clears the threshold", cleared, note))
history = provenance(registration)
checks.append(
(