test(retrieval-gate): the pin needs the bytes, the threshold needs a range, and an empty hold-out clears nothing
Three guards for the code the two step-0 commits added, each written against a mutation that would otherwise survive: a wiki-shaped file carrying the pinned 20 questions and 29 fasit entries but invented content (so the counts alone are a one-line forgery), a threshold of 80 or 1.5 (a share or a typo, and guessing is not this row's job), and a hold-out set of no questions -- the shape every row-5 test used until today, which would have made the new comparison vacuous the moment it was added. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
parent
05cb19087a
commit
c5b0dc6d9a
1 changed files with 93 additions and 0 deletions
|
|
@ -1342,3 +1342,96 @@ def test_the_marking_stays_off_every_question_the_synthetic_sets_do_answer(
|
||||||
assert not gate.marked(payload), f"{question_set.set_id}/{question.id}"
|
assert not gate.marked(payload), f"{question_set.set_id}/{question.id}"
|
||||||
answered += 1
|
answered += 1
|
||||||
assert answered == 10, answered
|
assert answered == 10, answered
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_set_of_the_right_size_and_the_wrong_bytes_is_still_refused(
|
||||||
|
tmp_path: Path,
|
||||||
|
) -> None:
|
||||||
|
"""The size pin alone would be a one-line forgery: twenty invented
|
||||||
|
questions in the wiki shape carry the pinned count and none of the pinned
|
||||||
|
content."""
|
||||||
|
path = tmp_path / "wiki-sized.json"
|
||||||
|
path.write_text(
|
||||||
|
json.dumps(
|
||||||
|
{
|
||||||
|
"questions": [
|
||||||
|
{
|
||||||
|
"id": f"W{index}",
|
||||||
|
"question": "Naar kontrolleres vinterberedskapen?",
|
||||||
|
"fasit": [
|
||||||
|
{"doc": "haandbok", "quote": "innen 1. november"},
|
||||||
|
*([{"doc": "haandbok", "quote": ""}] if index < 9 else []),
|
||||||
|
],
|
||||||
|
}
|
||||||
|
for index in range(20)
|
||||||
|
]
|
||||||
|
}
|
||||||
|
),
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
# The counts this file carries ARE the pinned ones, measured here.
|
||||||
|
question_set = gate.read_real_set("wiki", path, gate.sha256_of(path))
|
||||||
|
pinned = gate.REAL_SET_PINS["wiki-20"]
|
||||||
|
assert (len(question_set.questions), question_set.units) == (
|
||||||
|
pinned.questions,
|
||||||
|
pinned.fasit_entries,
|
||||||
|
)
|
||||||
|
with pytest.raises(gate.GateUsage) as refusal:
|
||||||
|
gate._real_sets(
|
||||||
|
[("wiki", str(path), gate.sha256_of(path), str(_bundles(tmp_path)["positive"]))]
|
||||||
|
)
|
||||||
|
assert "sha256" in str(refusal.value)
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_threshold_outside_nought_to_one_is_not_a_share(tmp_path: Path) -> None:
|
||||||
|
"""`80` is either 80 % written wrongly or a bar no run can clear, and
|
||||||
|
guessing which is not this row's job."""
|
||||||
|
bundles = _bundles(tmp_path)
|
||||||
|
for value in (80, -0.1, 1.5):
|
||||||
|
row = gate.row_five(
|
||||||
|
_registration(
|
||||||
|
tmp_path,
|
||||||
|
set_path=FIXTURES / "set-positive.json",
|
||||||
|
bundle=bundles["positive"],
|
||||||
|
threshold=value,
|
||||||
|
name=f"t{value}.json",
|
||||||
|
),
|
||||||
|
provenance=lambda _: _carried_by_git(),
|
||||||
|
)
|
||||||
|
assert any("the threshold is a number: NO" in detail for detail in row.details), value
|
||||||
|
# The control: a share inside the range is accepted as one.
|
||||||
|
inside = gate.row_five(
|
||||||
|
_registration(
|
||||||
|
tmp_path,
|
||||||
|
set_path=FIXTURES / "set-positive.json",
|
||||||
|
bundle=bundles["positive"],
|
||||||
|
threshold=1.0,
|
||||||
|
name="inside.json",
|
||||||
|
),
|
||||||
|
provenance=lambda _: _carried_by_git(),
|
||||||
|
)
|
||||||
|
assert any("the threshold is a number: yes" in detail for detail in inside.details)
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_hold_out_set_with_no_question_clears_no_threshold(tmp_path: Path) -> None:
|
||||||
|
"""A share over a denominator of nought is not a number, and an empty set
|
||||||
|
was the shape every row-5 test used before 2026-09-20 -- so `>=` over it
|
||||||
|
would have made the comparison vacuous the moment it was added."""
|
||||||
|
empty = tmp_path / "empty-set.json"
|
||||||
|
empty.write_text(
|
||||||
|
json.dumps({"set_id": "empty", "bundle": "positive", "questions": []}),
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
row = gate.row_five(
|
||||||
|
_registration(
|
||||||
|
tmp_path,
|
||||||
|
set_path=empty,
|
||||||
|
bundle=_bundles(tmp_path)["positive"],
|
||||||
|
threshold=0.0,
|
||||||
|
),
|
||||||
|
provenance=lambda _: _carried_by_git(),
|
||||||
|
)
|
||||||
|
assert any(
|
||||||
|
"clears the threshold: NO (the hold-out set carries no question" in detail
|
||||||
|
for detail in row.details
|
||||||
|
), row.details
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue