test(retrieval-gate): red -- row 5's hold-out is an assertion the file makes about itself
PM's J1: two files written by the session under test came back `7 of 7 GREEN`. Every one of row 5's checks reads a field the registration owns. "Written by another session than the ranking change" is `bool()` of a string the file sets; "no reading predates the threshold" reads a `readings` list from the same file. The order's point -- report-only without a pre-written threshold is not a protection -- is not met by a protection that is itself a claim. 1 red on an assertion about behaviour (GREEN for a registration this test wrote seconds earlier), and it names what a session cannot write about itself: that git already carried the file before the ranking moved. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
parent
be899106c4
commit
e207935ba6
1 changed files with 30 additions and 0 deletions
|
|
@ -883,3 +883,33 @@ def test_a_forced_fixture_that_stops_missing_counts_against_row_two(
|
||||||
row = gate.row_two([case])
|
row = gate.row_two([case])
|
||||||
assert (row.k, row.m, row.status) == (0, 1, gate.RED)
|
assert (row.k, row.m, row.status) == (0, 1, gate.RED)
|
||||||
assert any("Z9" in detail and "premise" in detail for detail in row.details)
|
assert any("Z9" in detail and "premise" in detail for detail in row.details)
|
||||||
|
|
||||||
|
|
||||||
|
def test_j1_a_registration_this_session_wrote_is_not_a_hold_out(tmp_path: Path) -> None:
|
||||||
|
"""PM's J1: two files written by the session under test came back
|
||||||
|
`7 of 7 GREEN`. Every check was an assertion the registration made about
|
||||||
|
itself -- `written_by` is `bool()` of a string the file sets, and the
|
||||||
|
readings are read from a list the same file owns."""
|
||||||
|
held = tmp_path / "held-out.json"
|
||||||
|
held.write_text('{"questions": []}', encoding="utf-8")
|
||||||
|
registration = tmp_path / "registration.json"
|
||||||
|
registration.write_text(
|
||||||
|
json.dumps(
|
||||||
|
{
|
||||||
|
"set": str(held),
|
||||||
|
"sha256": gate.sha256_of(held),
|
||||||
|
"threshold": "hit@8 must not fall below 6 of 8",
|
||||||
|
"threshold_written_at": "2026-01-01",
|
||||||
|
"written_by": "the same session, lying",
|
||||||
|
"readings": [],
|
||||||
|
}
|
||||||
|
),
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
row = gate.row_five(registration)
|
||||||
|
assert row.status != gate.GREEN, row
|
||||||
|
assert row.fails
|
||||||
|
# What a session cannot write about itself: that git already carried the
|
||||||
|
# file before the ranking moved.
|
||||||
|
assert any("committed" in detail for detail in row.details), row.details
|
||||||
|
assert any("ranking" in detail for detail in row.details), row.details
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue