test(consume, retrieval-gate): the payload must say what of the question it covered, and row 4 must see it

Red first, on behaviour: three asserts about what the payload carries and
what row 4 scores, none on an import or an attribute.

Measured 2026-09-19: N3, N4 and N5 come back with 8, 8 and 1 excerpts and
nothing that says they are weak, so `marked = nothing delivered` reads
three uncovered questions as answered ones. The second gate test is the
known-negative that a marking firing on everything would fail: the 10
questions the three synthetic sets DO answer must stay unmarked.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Kjell Tore Guttormsen 2026-09-20 08:25:38 +02:00
commit 90394c383d
Signed by: ktg
SSH key fingerprint: SHA256:JakMjO6FTBBzN0Bhfj9saOoEjaFxlSdYuZQQpM/lF9Q
2 changed files with 118 additions and 0 deletions

View file

@ -1287,3 +1287,49 @@ def test_the_threshold_is_compared_with_the_measured_hold_out(tmp_path: Path) ->
)
assert any("clears the threshold: NO" in detail for detail in falls.details)
assert (falls.k, falls.m, falls.status) == (10, 11, gate.RED)
# --- step 1: row 4, the marking a consumer can act on -------------------------
def test_row_four_marks_every_uncovered_control_and_leaves_the_covered_one(
tmp_path: Path,
) -> None:
"""Six controls, five uncovered and one covered, and the row must get all
six right.
Three of the five were measured red on 2026-09-19: N3, N4 and N5 came back
with 8, 8 and 1 excerpts and nothing in the payload that said they were
weak, so `marked = nothing delivered` could not see them.
"""
case = _case(tmp_path, "set-controls.json")
# The denominator, counted from the pinned file rather than from the run.
declared = json.loads((FIXTURES / "set-controls.json").read_text(encoding="utf-8"))
assert len(declared["controls"]) == 6
assert sum(1 for control in declared["controls"] if control.get("covered")) == 1
row = gate.row_four([case])
assert (row.k, row.m, row.status) == (6, 6, gate.GREEN)
assert not row.details
def test_the_marking_stays_off_every_question_the_synthetic_sets_do_answer(
tmp_path: Path,
) -> None:
"""The known-negative, and it is wider than row 4's own denominator.
A marking that fires on the questions the bundles DO answer would take row
4 to 6 of 6 and be worthless. Every question of the three sets whose fasit
row 1 finds is measured here, and none of them may come back marked.
"""
bundles = _bundles(tmp_path)
answered = 0
for name in ("set-positive.json", "set-signals.json", "set-quota.json"):
question_set = gate.load_set(FIXTURES / name, gate.SYNTHETIC_SETS[name])
for question in question_set.questions:
bundle = bundles[question.bundle or question_set.bundle]
payload = consume.build_payload(
bundle, question=question.question, k=question.k, limit=question.limit
)
assert not gate.marked(payload), f"{question_set.set_id}/{question.id}"
answered += 1
assert answered == 10, answered