test(retrieval-gate): red -- one real set of three is not a measurement of three
PM's checkpoint on 2c8296b ran eight cheating attacks at this gate and four
went through: a row came back GREEN without one label becoming true or one
concept ranking better. This is the first of them, written as a test that must
refuse it.
J2: `row_eight` iterates over whatever `--real` handed it and counts. One set
of three came back `6 of 6 GREEN`, and the realistic route is `--real r761`
alone -- the set PM's own registration puts at 7 of 7, with the two that miss
left out. The docstring already said "never green by leaving a set out"; the
code did not, and THIS REPOSITORY'S OWN TEST asserted the opposite at
`tests/test_retrieval_gate.py:486`, `(1, 1, GREEN)` for a single wiki set.
That assertion is corrected here rather than worked around: a test that pins
the defect is the defect.
J2b: the headline `k of N` is `quoted_hits + concept_hits` over
`quoted_units + concept_units`, written on the line directly above the detail
that says the two granularities are NOT summed into one number.
3 red on an assertion about behaviour (GREEN where NOT RUN is required, and a
headline that is a sum), 46 passed. The green direction -- all three named
sets present -- is in the same commit and passes already, so the fix cannot
close the rows by making the row unreachable.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
parent
44ad845e29
commit
3d496de2c6
1 changed files with 79 additions and 1 deletions
|
|
@ -483,7 +483,9 @@ def test_the_wiki_adapter_reads_its_own_shape_and_hits_by_source_file(tmp_path:
|
|||
assert question_set.questions[0].fasit[0].by == "source_file"
|
||||
assert question_set.questions[0].fasit[0].value == "haandbok.md"
|
||||
row = gate.row_eight([(question_set, {"wiki": _bundles(tmp_path)["positive"]})])
|
||||
assert (row.k, row.m, row.status) == (1, 1, gate.GREEN)
|
||||
# J2: one set of three is NOT a measurement of the three -- this asserted
|
||||
# GREEN until 2026-09-19, which is the breakthrough PM measured.
|
||||
assert row.status == gate.NOT_RUN
|
||||
assert any("citation granularity" in detail for detail in row.details)
|
||||
|
||||
|
||||
|
|
@ -736,3 +738,79 @@ def _mapping_is_sorted(mapping: Mapping[str, object]) -> bool:
|
|||
def test_every_class_is_documented_in_the_output() -> None:
|
||||
assert [letter for letter, _ in gate.CLASSES] == ["a", "b", "c", "d", "e"]
|
||||
assert all(description for _, description in gate.CLASSES)
|
||||
|
||||
|
||||
# --- the eight attacks PM ran against this gate 2026-09-19 ---------------------
|
||||
#
|
||||
# Four of them went through: a row went GREEN without one label becoming true or
|
||||
# one concept ranking better. Each is reproduced here as a test that must REFUSE
|
||||
# it, and the four that were already refused stay as regression guards, so the
|
||||
# table is 8 of 8 rather than 4 of 4.
|
||||
|
||||
|
||||
def _hitting_set(set_id: str, *, quote: str) -> gate.QuestionSet:
|
||||
"""A set the positive bundle answers, named by concept: the shortest way to
|
||||
drive row 8 green without an adapter between the set and the row."""
|
||||
return gate.QuestionSet(
|
||||
set_id=set_id,
|
||||
bundle="positive",
|
||||
path=Path(f"/nowhere/{set_id}.json"),
|
||||
sha256="0" * 64,
|
||||
questions=(
|
||||
gate.Question(
|
||||
id=f"{set_id}-1",
|
||||
question="Naar kontrolleres vinterberedskapen paa hytta?",
|
||||
fasit=(
|
||||
gate.Fasit(
|
||||
by="concept", value="haandbok/vinterberedskap", quote=quote
|
||||
),
|
||||
),
|
||||
),
|
||||
),
|
||||
controls=(),
|
||||
)
|
||||
|
||||
|
||||
def test_j2_row_eight_is_not_run_when_a_required_set_is_left_out(tmp_path: Path) -> None:
|
||||
"""PM's J2: one set of three came back `6 of 6 GREEN`."""
|
||||
bundles = {"positive": _bundles(tmp_path)["positive"]}
|
||||
row = gate.row_eight([(_hitting_set("wiki-20", quote="innen 1. november"), bundles)])
|
||||
assert row.status == gate.NOT_RUN
|
||||
assert row.fails
|
||||
assert "r761-sk2" in row.reason and "vegnormal-32" in row.reason
|
||||
# The numbers it DID measure are still carried: a missing set must not cost
|
||||
# the reader the set that ran.
|
||||
assert any("wiki-20: " in detail for detail in row.details)
|
||||
|
||||
|
||||
def test_row_eight_is_green_only_with_all_three_named_sets(tmp_path: Path) -> None:
|
||||
bundles = {"positive": _bundles(tmp_path)["positive"]}
|
||||
real = [
|
||||
(_hitting_set(name, quote="innen 1. november"), bundles)
|
||||
for name in ("wiki-20", "r761-sk2", "vegnormal-32")
|
||||
]
|
||||
row = gate.row_eight(real)
|
||||
assert row.status == gate.GREEN
|
||||
assert (row.k, row.m) == (3, 3)
|
||||
|
||||
|
||||
def test_j2b_row_eight_never_sums_the_two_granularities_into_its_headline(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
"""PM's J2b: the headline was `quoted + concept` on the line above a detail
|
||||
saying the two are not summed."""
|
||||
bundles = {"positive": _bundles(tmp_path)["positive"]}
|
||||
real = [
|
||||
(_hitting_set("wiki-20", quote="innen 1. november"), bundles),
|
||||
(_hitting_set("r761-sk2", quote=""), bundles),
|
||||
(_hitting_set("vegnormal-32", quote=""), bundles),
|
||||
]
|
||||
row = gate.row_eight(real)
|
||||
# Three questions, one per set: the headline is at QUESTION granularity and
|
||||
# is never the sum of one citation unit and two concept units.
|
||||
assert (row.k, row.m) == (3, 3)
|
||||
assert "question" in row.reason
|
||||
assert any(
|
||||
"1 of 1 at citation granularity, 2 of 2 at concept granularity" in detail
|
||||
for detail in row.details
|
||||
)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue