test(retrieval-gate): red -- rows 2 and 3 take their denominator from the run
PM's J10 and J8, the two attacks on the denominator, written as tests that must refuse them. J10 (`k = 32`): rows 1, 2, 3 and 6 all came back green at once and not one label had become true. Rows 2 and 3 count against the misses and the withheld, so a cut that delivers more broadly does not answer their question -- it shrinks their denominator to the cases that were already honest. The three fixtures that DECLARE class b are delivered under that cut, so their premise is broken, and a broken premise is not an absence. J8 (`--source-quota` off): every printed reason became true and row 3 read `6 of 6 GREEN` while row 1 fell to 8 of 9. That reading is not a lie -- with no quota there is no quota to name falsely -- but the row measured nothing, because the label it judges was never printed. 3 red on an assertion about behaviour (GREEN where RED or NOT RUN is required, and a broken premise counted as 0 of 0), 49 passed. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
parent
b4f03e44cb
commit
6105fc4193
1 changed files with 66 additions and 0 deletions
|
|
@ -810,3 +810,69 @@ def test_j2b_row_eight_never_sums_the_two_granularities_into_its_headline(
|
|||
"1 of 1 at citation granularity, 2 of 2 at concept granularity" in detail
|
||||
for detail in row.details
|
||||
)
|
||||
|
||||
|
||||
def _attack(tmp_path: Path, **cut: object) -> dict[int, gate.Row]:
|
||||
"""Rows 1-6 remeasured under a cut this gate did not ship."""
|
||||
cases, _ = gate.synthetic_cases(tmp_path / "bundles", FIXTURES)
|
||||
with gate._wrap_cut(**cut):
|
||||
remeasured = [gate.measure_case(case.question_set, case.bundles) for case in cases]
|
||||
return {row.number: row for row in gate.deterministic_rows(remeasured)}
|
||||
|
||||
|
||||
def test_j10_a_wider_cut_does_not_make_rows_two_and_three_green(tmp_path: Path) -> None:
|
||||
"""PM's J10: `k = 32` took rows 1, 2, 3 and 6 green at once, and not one
|
||||
label had become true -- the denominator of rows 2 and 3 IS the misses, so
|
||||
delivering more broadly shrinks it to the cases that were already honest."""
|
||||
rows = _attack(tmp_path, k=32)
|
||||
assert rows[2].status == gate.RED, rows[2]
|
||||
assert rows[3].status == gate.RED, rows[3]
|
||||
# The three fixtures that declare class b are delivered under this cut, so
|
||||
# their premise no longer holds -- and a broken premise counts against the
|
||||
# row rather than leaving it.
|
||||
assert any("premise" in detail for detail in rows[2].details), rows[2].details
|
||||
assert rows[2].m >= 7
|
||||
|
||||
|
||||
def test_j8_removing_the_quota_leaves_row_three_unable_to_say_anything(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
"""PM's J8: with `--source-quota` off every printed reason became true and
|
||||
row 3 read `6 of 6 GREEN`, while row 1 fell to 8 of 9. The label this row
|
||||
judges was not printed at all, so the row did not measure."""
|
||||
rows = _attack(tmp_path, source_quota=None)
|
||||
assert rows[3].status == gate.NOT_RUN, rows[3]
|
||||
assert "quota" in rows[3].reason
|
||||
|
||||
|
||||
def test_a_forced_fixture_that_stops_missing_counts_against_row_two(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
"""The fixed denominator, driven through a set file: a question that
|
||||
DECLARES the class it forces and then comes back a hit has not been
|
||||
classified -- its fixture's premise broke, and the row must carry it rather
|
||||
than lose it."""
|
||||
path, sha = _set_file(
|
||||
tmp_path / "set.json",
|
||||
set_id="row-two-premise",
|
||||
bundle="positive",
|
||||
questions=[
|
||||
{
|
||||
"id": "Z9",
|
||||
"question": "Naar kontrolleres vinterberedskapen paa hytta?",
|
||||
"expect_class": "b",
|
||||
"fasit": [
|
||||
{
|
||||
"by": "concept",
|
||||
"value": "haandbok/vinterberedskap",
|
||||
"quote": "innen 1. november",
|
||||
}
|
||||
],
|
||||
}
|
||||
],
|
||||
)
|
||||
case = gate.measure_case(gate.load_set(path, sha), _bundles(tmp_path))
|
||||
assert [unit.hit for unit in case.units] == [True], "the fixture must HIT for this test"
|
||||
row = gate.row_two([case])
|
||||
assert (row.k, row.m, row.status) == (0, 1, gate.RED)
|
||||
assert any("Z9" in detail and "premise" in detail for detail in row.details)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue