test(retrieval-gate): the pin needs the bytes, the threshold needs a range, and an empty hold-out clears nothing

Three guards for the code the two step-0 commits added, each written
against a mutation that would otherwise survive: a wiki-shaped file
carrying the pinned 20 questions and 29 fasit entries but invented content
(so the counts alone are a one-line forgery), a threshold of 80 or 1.5
(a share or a typo, and guessing is not this row's job), and a hold-out set
of no questions -- the shape every row-5 test used until today, which would
have made the new comparison vacuous the moment it was added.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Kjell Tore Guttormsen 2026-09-20 08:45:27 +02:00
commit c5b0dc6d9a
Signed by: ktg
SSH key fingerprint: SHA256:JakMjO6FTBBzN0Bhfj9saOoEjaFxlSdYuZQQpM/lF9Q

View file

@ -1342,3 +1342,96 @@ def test_the_marking_stays_off_every_question_the_synthetic_sets_do_answer(
assert not gate.marked(payload), f"{question_set.set_id}/{question.id}"
answered += 1
assert answered == 10, answered
def test_a_set_of_the_right_size_and_the_wrong_bytes_is_still_refused(
tmp_path: Path,
) -> None:
"""The size pin alone would be a one-line forgery: twenty invented
questions in the wiki shape carry the pinned count and none of the pinned
content."""
path = tmp_path / "wiki-sized.json"
path.write_text(
json.dumps(
{
"questions": [
{
"id": f"W{index}",
"question": "Naar kontrolleres vinterberedskapen?",
"fasit": [
{"doc": "haandbok", "quote": "innen 1. november"},
*([{"doc": "haandbok", "quote": ""}] if index < 9 else []),
],
}
for index in range(20)
]
}
),
encoding="utf-8",
)
# The counts this file carries ARE the pinned ones, measured here.
question_set = gate.read_real_set("wiki", path, gate.sha256_of(path))
pinned = gate.REAL_SET_PINS["wiki-20"]
assert (len(question_set.questions), question_set.units) == (
pinned.questions,
pinned.fasit_entries,
)
with pytest.raises(gate.GateUsage) as refusal:
gate._real_sets(
[("wiki", str(path), gate.sha256_of(path), str(_bundles(tmp_path)["positive"]))]
)
assert "sha256" in str(refusal.value)
def test_a_threshold_outside_nought_to_one_is_not_a_share(tmp_path: Path) -> None:
"""`80` is either 80 % written wrongly or a bar no run can clear, and
guessing which is not this row's job."""
bundles = _bundles(tmp_path)
for value in (80, -0.1, 1.5):
row = gate.row_five(
_registration(
tmp_path,
set_path=FIXTURES / "set-positive.json",
bundle=bundles["positive"],
threshold=value,
name=f"t{value}.json",
),
provenance=lambda _: _carried_by_git(),
)
assert any("the threshold is a number: NO" in detail for detail in row.details), value
# The control: a share inside the range is accepted as one.
inside = gate.row_five(
_registration(
tmp_path,
set_path=FIXTURES / "set-positive.json",
bundle=bundles["positive"],
threshold=1.0,
name="inside.json",
),
provenance=lambda _: _carried_by_git(),
)
assert any("the threshold is a number: yes" in detail for detail in inside.details)
def test_a_hold_out_set_with_no_question_clears_no_threshold(tmp_path: Path) -> None:
"""A share over a denominator of nought is not a number, and an empty set
was the shape every row-5 test used before 2026-09-20 -- so `>=` over it
would have made the comparison vacuous the moment it was added."""
empty = tmp_path / "empty-set.json"
empty.write_text(
json.dumps({"set_id": "empty", "bundle": "positive", "questions": []}),
encoding="utf-8",
)
row = gate.row_five(
_registration(
tmp_path,
set_path=empty,
bundle=_bundles(tmp_path)["positive"],
threshold=0.0,
),
provenance=lambda _: _carried_by_git(),
)
assert any(
"clears the threshold: NO (the hold-out set carries no question" in detail
for detail in row.details
), row.details