test(retrieval-gate): the pin needs the bytes, the threshold needs a range, and an empty hold-out clears nothing
Three guards for the code the two step-0 commits added, each written against a mutation that would otherwise survive: a wiki-shaped file carrying the pinned 20 questions and 29 fasit entries but invented content (so the counts alone are a one-line forgery), a threshold of 80 or 1.5 (a share or a typo, and guessing is not this row's job), and a hold-out set of no questions -- the shape every row-5 test used until today, which would have made the new comparison vacuous the moment it was added. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
parent
05cb19087a
commit
c5b0dc6d9a
1 changed files with 93 additions and 0 deletions
|
|
@ -1342,3 +1342,96 @@ def test_the_marking_stays_off_every_question_the_synthetic_sets_do_answer(
|
|||
assert not gate.marked(payload), f"{question_set.set_id}/{question.id}"
|
||||
answered += 1
|
||||
assert answered == 10, answered
|
||||
|
||||
|
||||
def test_a_set_of_the_right_size_and_the_wrong_bytes_is_still_refused(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
"""The size pin alone would be a one-line forgery: twenty invented
|
||||
questions in the wiki shape carry the pinned count and none of the pinned
|
||||
content."""
|
||||
path = tmp_path / "wiki-sized.json"
|
||||
path.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"questions": [
|
||||
{
|
||||
"id": f"W{index}",
|
||||
"question": "Naar kontrolleres vinterberedskapen?",
|
||||
"fasit": [
|
||||
{"doc": "haandbok", "quote": "innen 1. november"},
|
||||
*([{"doc": "haandbok", "quote": ""}] if index < 9 else []),
|
||||
],
|
||||
}
|
||||
for index in range(20)
|
||||
]
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
# The counts this file carries ARE the pinned ones, measured here.
|
||||
question_set = gate.read_real_set("wiki", path, gate.sha256_of(path))
|
||||
pinned = gate.REAL_SET_PINS["wiki-20"]
|
||||
assert (len(question_set.questions), question_set.units) == (
|
||||
pinned.questions,
|
||||
pinned.fasit_entries,
|
||||
)
|
||||
with pytest.raises(gate.GateUsage) as refusal:
|
||||
gate._real_sets(
|
||||
[("wiki", str(path), gate.sha256_of(path), str(_bundles(tmp_path)["positive"]))]
|
||||
)
|
||||
assert "sha256" in str(refusal.value)
|
||||
|
||||
|
||||
def test_a_threshold_outside_nought_to_one_is_not_a_share(tmp_path: Path) -> None:
|
||||
"""`80` is either 80 % written wrongly or a bar no run can clear, and
|
||||
guessing which is not this row's job."""
|
||||
bundles = _bundles(tmp_path)
|
||||
for value in (80, -0.1, 1.5):
|
||||
row = gate.row_five(
|
||||
_registration(
|
||||
tmp_path,
|
||||
set_path=FIXTURES / "set-positive.json",
|
||||
bundle=bundles["positive"],
|
||||
threshold=value,
|
||||
name=f"t{value}.json",
|
||||
),
|
||||
provenance=lambda _: _carried_by_git(),
|
||||
)
|
||||
assert any("the threshold is a number: NO" in detail for detail in row.details), value
|
||||
# The control: a share inside the range is accepted as one.
|
||||
inside = gate.row_five(
|
||||
_registration(
|
||||
tmp_path,
|
||||
set_path=FIXTURES / "set-positive.json",
|
||||
bundle=bundles["positive"],
|
||||
threshold=1.0,
|
||||
name="inside.json",
|
||||
),
|
||||
provenance=lambda _: _carried_by_git(),
|
||||
)
|
||||
assert any("the threshold is a number: yes" in detail for detail in inside.details)
|
||||
|
||||
|
||||
def test_a_hold_out_set_with_no_question_clears_no_threshold(tmp_path: Path) -> None:
|
||||
"""A share over a denominator of nought is not a number, and an empty set
|
||||
was the shape every row-5 test used before 2026-09-20 -- so `>=` over it
|
||||
would have made the comparison vacuous the moment it was added."""
|
||||
empty = tmp_path / "empty-set.json"
|
||||
empty.write_text(
|
||||
json.dumps({"set_id": "empty", "bundle": "positive", "questions": []}),
|
||||
encoding="utf-8",
|
||||
)
|
||||
row = gate.row_five(
|
||||
_registration(
|
||||
tmp_path,
|
||||
set_path=empty,
|
||||
bundle=_bundles(tmp_path)["positive"],
|
||||
threshold=0.0,
|
||||
),
|
||||
provenance=lambda _: _carried_by_git(),
|
||||
)
|
||||
assert any(
|
||||
"clears the threshold: NO (the hold-out set carries no question" in detail
|
||||
for detail in row.details
|
||||
), row.details
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue