test(retrieval-gate): red -- rows 2 and 3 take their denominator from the run
PM's J10 and J8, the two attacks on the denominator, written as tests that must refuse them. J10 (`k = 32`): rows 1, 2, 3 and 6 all came back green at once and not one label had become true. Rows 2 and 3 count against the misses and the withheld, so a cut that delivers more broadly does not answer their question -- it shrinks their denominator to the cases that were already honest. The three fixtures that DECLARE class b are delivered under that cut, so their premise is broken, and a broken premise is not an absence. J8 (`--source-quota` off): every printed reason became true and row 3 read `6 of 6 GREEN` while row 1 fell to 8 of 9. That reading is not a lie -- with no quota there is no quota to name falsely -- but the row measured nothing, because the label it judges was never printed. 3 red on an assertion about behaviour (GREEN where RED or NOT RUN is required, and a broken premise counted as 0 of 0), 49 passed. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
parent
b4f03e44cb
commit
6105fc4193
1 changed files with 66 additions and 0 deletions
|
|
@ -810,3 +810,69 @@ def test_j2b_row_eight_never_sums_the_two_granularities_into_its_headline(
|
||||||
"1 of 1 at citation granularity, 2 of 2 at concept granularity" in detail
|
"1 of 1 at citation granularity, 2 of 2 at concept granularity" in detail
|
||||||
for detail in row.details
|
for detail in row.details
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _attack(tmp_path: Path, **cut: object) -> dict[int, gate.Row]:
|
||||||
|
"""Rows 1-6 remeasured under a cut this gate did not ship."""
|
||||||
|
cases, _ = gate.synthetic_cases(tmp_path / "bundles", FIXTURES)
|
||||||
|
with gate._wrap_cut(**cut):
|
||||||
|
remeasured = [gate.measure_case(case.question_set, case.bundles) for case in cases]
|
||||||
|
return {row.number: row for row in gate.deterministic_rows(remeasured)}
|
||||||
|
|
||||||
|
|
||||||
|
def test_j10_a_wider_cut_does_not_make_rows_two_and_three_green(tmp_path: Path) -> None:
|
||||||
|
"""PM's J10: `k = 32` took rows 1, 2, 3 and 6 green at once, and not one
|
||||||
|
label had become true -- the denominator of rows 2 and 3 IS the misses, so
|
||||||
|
delivering more broadly shrinks it to the cases that were already honest."""
|
||||||
|
rows = _attack(tmp_path, k=32)
|
||||||
|
assert rows[2].status == gate.RED, rows[2]
|
||||||
|
assert rows[3].status == gate.RED, rows[3]
|
||||||
|
# The three fixtures that declare class b are delivered under this cut, so
|
||||||
|
# their premise no longer holds -- and a broken premise counts against the
|
||||||
|
# row rather than leaving it.
|
||||||
|
assert any("premise" in detail for detail in rows[2].details), rows[2].details
|
||||||
|
assert rows[2].m >= 7
|
||||||
|
|
||||||
|
|
||||||
|
def test_j8_removing_the_quota_leaves_row_three_unable_to_say_anything(
|
||||||
|
tmp_path: Path,
|
||||||
|
) -> None:
|
||||||
|
"""PM's J8: with `--source-quota` off every printed reason became true and
|
||||||
|
row 3 read `6 of 6 GREEN`, while row 1 fell to 8 of 9. The label this row
|
||||||
|
judges was not printed at all, so the row did not measure."""
|
||||||
|
rows = _attack(tmp_path, source_quota=None)
|
||||||
|
assert rows[3].status == gate.NOT_RUN, rows[3]
|
||||||
|
assert "quota" in rows[3].reason
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_forced_fixture_that_stops_missing_counts_against_row_two(
|
||||||
|
tmp_path: Path,
|
||||||
|
) -> None:
|
||||||
|
"""The fixed denominator, driven through a set file: a question that
|
||||||
|
DECLARES the class it forces and then comes back a hit has not been
|
||||||
|
classified -- its fixture's premise broke, and the row must carry it rather
|
||||||
|
than lose it."""
|
||||||
|
path, sha = _set_file(
|
||||||
|
tmp_path / "set.json",
|
||||||
|
set_id="row-two-premise",
|
||||||
|
bundle="positive",
|
||||||
|
questions=[
|
||||||
|
{
|
||||||
|
"id": "Z9",
|
||||||
|
"question": "Naar kontrolleres vinterberedskapen paa hytta?",
|
||||||
|
"expect_class": "b",
|
||||||
|
"fasit": [
|
||||||
|
{
|
||||||
|
"by": "concept",
|
||||||
|
"value": "haandbok/vinterberedskap",
|
||||||
|
"quote": "innen 1. november",
|
||||||
|
}
|
||||||
|
],
|
||||||
|
}
|
||||||
|
],
|
||||||
|
)
|
||||||
|
case = gate.measure_case(gate.load_set(path, sha), _bundles(tmp_path))
|
||||||
|
assert [unit.hit for unit in case.units] == [True], "the fixture must HIT for this test"
|
||||||
|
row = gate.row_two([case])
|
||||||
|
assert (row.k, row.m, row.status) == (0, 1, gate.RED)
|
||||||
|
assert any("Z9" in detail and "premise" in detail for detail in row.details)
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue