test(retrieval-gate): red -- one real set of three is not a measurement of three

PM's checkpoint on 2c8296b ran eight cheating attacks at this gate and four
went through: a row came back GREEN without one label becoming true or one
concept ranking better. This is the first of them, written as a test that must
refuse it.

J2: `row_eight` iterates over whatever `--real` handed it and counts. One set
of three came back `6 of 6 GREEN`, and the realistic route is `--real r761`
alone -- the set PM's own registration puts at 7 of 7, with the two that miss
left out. The docstring already said "never green by leaving a set out"; the
code did not, and THIS REPOSITORY'S OWN TEST asserted the opposite at
`tests/test_retrieval_gate.py:486`, `(1, 1, GREEN)` for a single wiki set.
That assertion is corrected here rather than worked around: a test that pins
the defect is the defect.

J2b: the headline `k of N` is `quoted_hits + concept_hits` over
`quoted_units + concept_units`, written on the line directly above the detail
that says the two granularities are NOT summed into one number.

3 red on an assertion about behaviour (GREEN where NOT RUN is required, and a
headline that is a sum), 46 passed. The green direction -- all three named
sets present -- is in the same commit and passes already, so the fix cannot
close the rows by making the row unreachable.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Kjell Tore Guttormsen 2026-09-19 20:49:42 +02:00
commit 3d496de2c6
Signed by: ktg
SSH key fingerprint: SHA256:JakMjO6FTBBzN0Bhfj9saOoEjaFxlSdYuZQQpM/lF9Q

View file

@ -483,7 +483,9 @@ def test_the_wiki_adapter_reads_its_own_shape_and_hits_by_source_file(tmp_path:
assert question_set.questions[0].fasit[0].by == "source_file"
assert question_set.questions[0].fasit[0].value == "haandbok.md"
row = gate.row_eight([(question_set, {"wiki": _bundles(tmp_path)["positive"]})])
assert (row.k, row.m, row.status) == (1, 1, gate.GREEN)
# J2: one set of three is NOT a measurement of the three -- this asserted
# GREEN until 2026-09-19, which is the breakthrough PM measured.
assert row.status == gate.NOT_RUN
assert any("citation granularity" in detail for detail in row.details)
@ -736,3 +738,79 @@ def _mapping_is_sorted(mapping: Mapping[str, object]) -> bool:
def test_every_class_is_documented_in_the_output() -> None:
assert [letter for letter, _ in gate.CLASSES] == ["a", "b", "c", "d", "e"]
assert all(description for _, description in gate.CLASSES)
# --- the eight attacks PM ran against this gate 2026-09-19 ---------------------
#
# Four of them went through: a row went GREEN without one label becoming true or
# one concept ranking better. Each is reproduced here as a test that must REFUSE
# it, and the four that were already refused stay as regression guards, so the
# table is 8 of 8 rather than 4 of 4.
def _hitting_set(set_id: str, *, quote: str) -> gate.QuestionSet:
"""A set the positive bundle answers, named by concept: the shortest way to
drive row 8 green without an adapter between the set and the row."""
return gate.QuestionSet(
set_id=set_id,
bundle="positive",
path=Path(f"/nowhere/{set_id}.json"),
sha256="0" * 64,
questions=(
gate.Question(
id=f"{set_id}-1",
question="Naar kontrolleres vinterberedskapen paa hytta?",
fasit=(
gate.Fasit(
by="concept", value="haandbok/vinterberedskap", quote=quote
),
),
),
),
controls=(),
)
def test_j2_row_eight_is_not_run_when_a_required_set_is_left_out(tmp_path: Path) -> None:
"""PM's J2: one set of three came back `6 of 6 GREEN`."""
bundles = {"positive": _bundles(tmp_path)["positive"]}
row = gate.row_eight([(_hitting_set("wiki-20", quote="innen 1. november"), bundles)])
assert row.status == gate.NOT_RUN
assert row.fails
assert "r761-sk2" in row.reason and "vegnormal-32" in row.reason
# The numbers it DID measure are still carried: a missing set must not cost
# the reader the set that ran.
assert any("wiki-20: " in detail for detail in row.details)
def test_row_eight_is_green_only_with_all_three_named_sets(tmp_path: Path) -> None:
bundles = {"positive": _bundles(tmp_path)["positive"]}
real = [
(_hitting_set(name, quote="innen 1. november"), bundles)
for name in ("wiki-20", "r761-sk2", "vegnormal-32")
]
row = gate.row_eight(real)
assert row.status == gate.GREEN
assert (row.k, row.m) == (3, 3)
def test_j2b_row_eight_never_sums_the_two_granularities_into_its_headline(
tmp_path: Path,
) -> None:
"""PM's J2b: the headline was `quoted + concept` on the line above a detail
saying the two are not summed."""
bundles = {"positive": _bundles(tmp_path)["positive"]}
real = [
(_hitting_set("wiki-20", quote="innen 1. november"), bundles),
(_hitting_set("r761-sk2", quote=""), bundles),
(_hitting_set("vegnormal-32", quote=""), bundles),
]
row = gate.row_eight(real)
# Three questions, one per set: the headline is at QUESTION granularity and
# is never the sum of one citation unit and two concept units.
assert (row.k, row.m) == (3, 3)
assert "question" in row.reason
assert any(
"1 of 1 at citation granularity, 2 of 2 at concept granularity" in detail
for detail in row.details
)