open/llm-ingestion-okf
1
0
Fork 0
Code Issues Releases 4 Activity

test(retrieval-gate): row 5 must name no machine and measure the bundle it registered — red

Browse source 
The hold-out registration is committed to a public repository, so its paths
must be written under the home directory (`~/...`) and expanded when read; an
absolute path is a NO with its reason. And the bundle it names is the
registered bundle only while its tree measures the registered `bundle_ref`.

Red on ef76a24: an absolute path is followed, `~/` is never expanded, no
line compares the bundle's ref, an absent path is not named. Existing row-5
tests now write `~/` registrations under a `home` fixture and count twelve
checks.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Kjell Tore Guttormsen 2026-09-23 10:22:47 +02:00
commit 9787b7fc93
Signed by: ktg
SSH key fingerprint: SHA256:JakMjO6FTBBzN0Bhfj9saOoEjaFxlSdYuZQQpM/lF9Q
1 changed files with 179 additions and 20 deletions
Download patch file Download diff file Expand all files Collapse all files

199
tests/test_retrieval_gate.py
View file

@ -394,8 +394,8 @@ def test_row_five_is_red_while_no_hold_out_is_registered(tmp_path: Path) -> None
assert (row.k, row.m, row.status) == (0, 1, gate.RED)
def test_row_five_is_green_for_a_registration_that_carries_all_eleven(
tmp_path: Path,
def test_row_five_is_green_for_a_registration_that_carries_all_twelve(
tmp_path: Path, home: Path
) -> None:
registration = _registration(
tmp_path,
@ -404,11 +404,11 @@ def test_row_five_is_green_for_a_registration_that_carries_all_eleven(
threshold=0.8,
)
row = gate.row_five(registration, provenance=lambda _: _carried_by_git())
assert (row.k, row.m, row.status) == (11, 11, gate.GREEN)
assert (row.k, row.m, row.status) == (12, 12, gate.GREEN)
def test_row_five_falls_on_a_number_read_before_its_threshold_was_written(
tmp_path: Path,
tmp_path: Path, home: Path
) -> None:
registration = _registration(
tmp_path,
@ -420,12 +420,12 @@ def test_row_five_falls_on_a_number_read_before_its_threshold_was_written(
spec["readings"] = [{"at": "2026-09-18T09:00:00Z", "value": "0.62"}]
registration.write_text(json.dumps(spec), encoding="utf-8")
row = gate.row_five(registration, provenance=lambda _: _carried_by_git())
assert (row.k, row.m, row.status) == (10, 11, gate.RED)
assert (row.k, row.m, row.status) == (11, 12, gate.RED)
assert any("no reading predates the threshold: NO" in detail for detail in row.details)
def test_row_five_falls_when_the_registration_rides_in_on_the_ranking_change(
tmp_path: Path,
tmp_path: Path, home: Path
) -> None:
"""The three git checks, each driven red on its own: a file nobody
committed, a threshold committed together with the ranking change, and a
@ -964,7 +964,7 @@ def test_a_forced_fixture_that_stops_missing_counts_against_row_two(
assert any("Z9" in detail and "premise" in detail for detail in row.details)
def test_j1_a_registration_this_session_wrote_is_not_a_hold_out(tmp_path: Path) -> None:
def test_j1_a_registration_this_session_wrote_is_not_a_hold_out(tmp_path: Path, home: Path) -> None:
"""PM's J1: two files written by the session under test came back
`7 of 7 GREEN`. Every check was an assertion the registration made about
itself -- `written_by` is `bool()` of a string the file sets, and the
@ -1294,6 +1294,18 @@ def test_a_wiki_set_of_one_question_is_refused_on_the_command_line(
# --- step 0: the threshold is a number, and it is compared with something ----
@pytest.fixture
def home(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path:
"""Row 5 reads a registration's paths as `~/...`, expanded against the
home directory; every test writing one makes its `tmp_path` that home."""
monkeypatch.setenv("HOME", str(tmp_path))
return tmp_path
def _under_home(path: Path, home: Path) -> str:
return "~/" + path.relative_to(home).as_posix()
def _registration(
tmp_path: Path,
*,
@ -1301,14 +1313,26 @@ def _registration(
bundle: Path,
threshold: object,
name: str = "registration.json",
bundle_ref: str | None = None,
) -> Path:
"""A registration in the form the committed one must have: its paths
under the home directory, written `~/...`, and the bundle's tree ref
pinned. The home is `tmp_path` (the `home` fixture); a set living outside
it is copied in byte for byte, so the sha256 pinned is the set's own."""
assert Path.home() == tmp_path, "a row-5 test must request the `home` fixture"
if not set_path.is_relative_to(tmp_path):
copied = tmp_path / "sets" / set_path.name
copied.parent.mkdir(exist_ok=True)
copied.write_bytes(set_path.read_bytes())
set_path = copied
registration = tmp_path / name
registration.write_text(
json.dumps(
{
"set": str(set_path),
"set": _under_home(set_path, tmp_path),
"sha256": gate.sha256_of(set_path),
"bundle": str(bundle),
"bundle": _under_home(bundle, tmp_path),
"bundle_ref": consume.bundle_ref(bundle) if bundle_ref is None else bundle_ref,
"metric": "questions answered over questions asked",
"threshold": threshold,
"threshold_written_at": "2026-09-19T10:00:00Z",
@ -1321,7 +1345,7 @@ def _registration(
return registration
def test_a_threshold_that_is_not_a_number_is_refused(tmp_path: Path) -> None:
def test_a_threshold_that_is_not_a_number_is_refused(tmp_path: Path, home: Path) -> None:
"""`bool(threshold)` was the whole check, so `report-only; any number is
acceptable for v1` read as `a threshold is written: yes`. A threshold that
cannot be compared with a number cannot fell anything."""
@ -1337,7 +1361,7 @@ def test_a_threshold_that_is_not_a_number_is_refused(tmp_path: Path) -> None:
assert row.fails
def test_the_threshold_is_compared_with_the_measured_hold_out(tmp_path: Path) -> None:
def test_the_threshold_is_compared_with_the_measured_hold_out(tmp_path: Path, home: Path) -> None:
"""Both directions, from the same code path: a set the bundle answers
clears a threshold under it, and a set it does not answer falls under one
over it. The measured share is counted here as well, off the set's own
@ -1354,7 +1378,7 @@ def test_the_threshold_is_compared_with_the_measured_hold_out(tmp_path: Path) ->
provenance=lambda _: _carried_by_git(),
)
assert any("clears the threshold: yes" in detail for detail in clears.details)
assert (clears.k, clears.m, clears.status) == (11, 11, gate.GREEN)
assert (clears.k, clears.m, clears.status) == (12, 12, gate.GREEN)
# The independent count: set-miss carries one question, forced to miss.
missing = json.loads((FIXTURES / "set-miss.json").read_text(encoding="utf-8"))
@ -1370,7 +1394,7 @@ def test_the_threshold_is_compared_with_the_measured_hold_out(tmp_path: Path) ->
provenance=lambda _: _carried_by_git(),
)
assert any("clears the threshold: NO" in detail for detail in falls.details)
assert (falls.k, falls.m, falls.status) == (10, 11, gate.RED)
assert (falls.k, falls.m, falls.status) == (11, 12, gate.RED)
# --- step 1: row 4, the marking a consumer can act on -------------------------
@ -1458,7 +1482,7 @@ def test_a_set_of_the_right_size_and_the_wrong_bytes_is_still_refused(
assert "sha256" in str(refusal.value)
def test_a_threshold_outside_nought_to_one_is_not_a_share(tmp_path: Path) -> None:
def test_a_threshold_outside_nought_to_one_is_not_a_share(tmp_path: Path, home: Path) -> None:
"""`80` is either 80 % written wrongly or a bar no run can clear, and
guessing which is not this row's job."""
bundles = _bundles(tmp_path)
@ -1488,7 +1512,7 @@ def test_a_threshold_outside_nought_to_one_is_not_a_share(tmp_path: Path) -> Non
assert any("the threshold is a number: yes" in detail for detail in inside.details)
def test_a_hold_out_set_with_no_question_clears_no_threshold(tmp_path: Path) -> None:
def test_a_hold_out_set_with_no_question_clears_no_threshold(tmp_path: Path, home: Path) -> None:
"""A share over a denominator of nought is not a number, and an empty set
was the shape every row-5 test used before 2026-09-20 -- so `>=` over it
would have made the comparison vacuous the moment it was added."""
@ -1562,7 +1586,7 @@ def _fase_set(path: Path, *, schema: str = "fase-sporsmaal/1") -> Path:
return path
def test_row_five_measures_a_hold_out_set_in_the_wiki_schema(tmp_path: Path) -> None:
def test_row_five_measures_a_hold_out_set_in_the_wiki_schema(tmp_path: Path, home: Path) -> None:
"""Both directions through the same set and the same bundle: 4 of 5
clears 0.8 and falls under 0.9. The share is counted off the file first,
so the row is not the only thing that knows it."""
@ -1579,7 +1603,7 @@ def test_row_five_measures_a_hold_out_set_in_the_wiki_schema(tmp_path: Path) ->
"the measured hold-out clears the threshold: yes (4 of 5 = 0.8000 against 0.8000)" in detail
for detail in clears.details
), clears.details
assert (clears.k, clears.m, clears.status) == (11, 11, gate.GREEN)
assert (clears.k, clears.m, clears.status) == (12, 12, gate.GREEN)
falls = gate.row_five(
_registration(tmp_path, set_path=held_out, bundle=bundle, threshold=0.9, name="f.json"),
@ -1589,10 +1613,10 @@ def test_row_five_measures_a_hold_out_set_in_the_wiki_schema(tmp_path: Path) ->
"the measured hold-out clears the threshold: NO (4 of 5 = 0.8000 against 0.9000)" in detail
for detail in falls.details
), falls.details
assert (falls.k, falls.m, falls.status) == (10, 11, gate.RED)
assert (falls.k, falls.m, falls.status) == (11, 12, gate.RED)
def test_a_hold_out_set_in_an_unknown_schema_is_a_named_no(tmp_path: Path) -> None:
def test_a_hold_out_set_in_an_unknown_schema_is_a_named_no(tmp_path: Path, home: Path) -> None:
"""Fail closed: a set read in a shape it was not written in measures
nothing, so an unknown schema is a NO that NAMES the schema -- never a
guess at a reader and never an exception."""
@ -1612,7 +1636,7 @@ def test_a_hold_out_set_in_an_unknown_schema_is_a_named_no(tmp_path: Path) -> No
def test_row_five_prints_the_share_and_never_which_hold_out_question_missed(
tmp_path: Path,
tmp_path: Path, home: Path
) -> None:
"""A hold-out set whose misses are printed is a tuning set by the next
session. The known-positive in the same run: the row DID measure, one
@ -1637,3 +1661,138 @@ def test_row_five_prints_the_share_and_never_which_hold_out_question_missed(
assert "fyrlykta" not in text
for hit in _HOLD_OUT_HIT_IDS:
assert hit not in text
# --- row 5 names no machine, and measures the bundle it registered ------------
#
# The registration is committed to a PUBLIC repository, so the paths it names
# are written relative to the home directory and expanded when read: an
# absolute path names one machine and is a NO, never a path the row follows.
# And the bundle it names is only the registered bundle while its tree still
# measures the registered `bundle_ref`: a bundle path is a place, and a place
# can be filled with other bytes.
def _row_five_line(row: gate.Row, name: str) -> str:
lines = [detail for detail in row.details if detail.strip().startswith(f"{name}:")]
assert len(lines) == 1, (name, row.details)
return lines[0]
def test_an_absolute_path_in_the_registration_is_a_named_no(tmp_path: Path, home: Path) -> None:
"""Both paths, each on its own: the set and the bundle written as the
absolute path of the very file the `~/` form names. The row does not
follow either, and says why."""
held_out = _fase_set(tmp_path / "held-out.json")
bundle = _bundles(tmp_path)["positive"]
registration = _registration(tmp_path, set_path=held_out, bundle=bundle, threshold=0.8)
written = json.loads(registration.read_text(encoding="utf-8"))
for key, absolute, line in (
("set", str(held_out), "a set is named"),
("bundle", str(bundle), "the bundle is the registered tree"),
):
spec = dict(written, **{key: absolute})
registration.write_text(json.dumps(spec), encoding="utf-8")
row = gate.row_five(registration, provenance=lambda _: _carried_by_git())
verdict = _row_five_line(row, line)
assert ": NO (" in verdict, verdict
assert "absolute" in verdict, verdict
assert "~/" in verdict, verdict
assert row.status == gate.RED, key
# A path the row does not follow is a set it does not measure.
assert "4 of 5" not in gate.render([row]), key
def test_a_home_relative_path_is_expanded_and_read(tmp_path: Path, home: Path) -> None:
"""The known-positive for the rule above: the registration's own text
carries `~/` and no path of this machine, and the row expands it, reads
the set and measures the bundle -- 4 of 5, counted off the file."""
held_out = _fase_set(tmp_path / "held-out.json")
registration = _registration(
tmp_path, set_path=held_out, bundle=_bundles(tmp_path)["positive"], threshold=0.8
)
text = registration.read_text(encoding="utf-8")
spec = json.loads(text)
assert spec["set"].startswith("~/") and spec["bundle"].startswith("~/"), spec
assert str(tmp_path) not in text
row = gate.row_five(registration, provenance=lambda _: _carried_by_git())
assert ": yes (" in _row_five_line(row, "the pinned bytes are the bytes on disk")
assert ": yes (" in _row_five_line(row, "the bundle is the registered tree")
assert "(4 of 5 = 0.8000 against 0.8000)" in _row_five_line(
row, "the measured hold-out clears the threshold"
)
assert (row.k, row.m, row.status) == (12, 12, gate.GREEN)
def test_a_bundle_whose_tree_is_not_the_registered_ref_is_a_no(tmp_path: Path, home: Path) -> None:
"""Two ways the path stays and the bytes go: a registration pinning
another bundle's ref, and the registered bundle's concepts edited after
the ref was written. Neither is measured -- a share read off other bytes
is not the registered hold-out."""
held_out = _fase_set(tmp_path / "held-out.json")
bundles = _bundles(tmp_path)
foreign = consume.bundle_ref(bundles["miss"])
assert foreign != consume.bundle_ref(bundles["positive"]), "the control must differ"
pinned_elsewhere = gate.row_five(
_registration(
tmp_path,
set_path=held_out,
bundle=bundles["positive"],
threshold=0.8,
bundle_ref=foreign,
name="foreign.json",
),
provenance=lambda _: _carried_by_git(),
)
verdict = _row_five_line(pinned_elsewhere, "the bundle is the registered tree")
assert ": NO (" in verdict and foreign in verdict, verdict
assert pinned_elsewhere.status == gate.RED
assert "4 of 5" not in gate.render([pinned_elsewhere])
registration = _registration(
tmp_path, set_path=held_out, bundle=bundles["positive"], threshold=0.8, name="edited.json"
)
concepts = [p for p in bundles["positive"].rglob("*.md") if p.name != "index.md"]
assert concepts, "the control needs a concept to edit"
for concept in concepts:
concept.write_bytes(concept.read_bytes() + b"\nOther bytes at the same path.\n")
edited = gate.row_five(registration, provenance=lambda _: _carried_by_git())
assert ": NO (" in _row_five_line(edited, "the bundle is the registered tree")
assert edited.status == gate.RED
def test_a_registered_set_or_bundle_absent_on_this_machine_is_a_named_no(
tmp_path: Path, home: Path
) -> None:
"""A registration read on a machine that does not carry what it names:
a NO with the registered `~/` path, never an exception, never green."""
held_out = _fase_set(tmp_path / "held-out.json")
bundle = _bundles(tmp_path)["positive"]
written = json.loads(
_registration(tmp_path, set_path=held_out, bundle=bundle, threshold=0.8).read_text(
encoding="utf-8"
)
)
registration = tmp_path / "absent.json"
for key, value, line in (
("set", "~/nowhere/held-out.json", "the pinned bytes are the bytes on disk"),
("bundle", "~/nowhere/bundle", "the bundle is the registered tree"),
):
registration.write_text(json.dumps(dict(written, **{key: value})), encoding="utf-8")
row = gate.row_five(registration, provenance=lambda _: _carried_by_git())
verdict = _row_five_line(row, line)
assert ": NO (" in verdict and "absent" in verdict and value in verdict, verdict
assert row.status == gate.RED, key
assert ": NO (" in _row_five_line(row, "the measured hold-out clears the threshold")
def test_the_committed_registration_names_no_machine_path() -> None:
"""The file this repository publishes: its paths are `~/...`, and no
value in it names a directory of the machine it was written on."""
text = gate.HOLDOUT_REGISTRATION.read_text(encoding="utf-8")
spec = json.loads(text)
assert spec["set"].startswith("~/"), spec["set"]
assert spec["bundle"].startswith("~/"), spec["bundle"]
assert spec["bundle_ref"].startswith("sha256-tree:"), spec["bundle_ref"]
for marker in ("/Users/", "/home/", "/private/", "/var/folders/"):
assert marker not in text, marker