test(retrieval-gate): red -- row 5's hold-out is an assertion the file makes about itself
PM's J1: two files written by the session under test came back `7 of 7 GREEN`. Every one of row 5's checks reads a field the registration owns. "Written by another session than the ranking change" is `bool()` of a string the file sets; "no reading predates the threshold" reads a `readings` list from the same file. The order's point -- report-only without a pre-written threshold is not a protection -- is not met by a protection that is itself a claim. 1 red on an assertion about behaviour (GREEN for a registration this test wrote seconds earlier), and it names what a session cannot write about itself: that git already carried the file before the ranking moved. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
parent
be899106c4
commit
e207935ba6
1 changed files with 30 additions and 0 deletions
|
|
@ -883,3 +883,33 @@ def test_a_forced_fixture_that_stops_missing_counts_against_row_two(
|
|||
row = gate.row_two([case])
|
||||
assert (row.k, row.m, row.status) == (0, 1, gate.RED)
|
||||
assert any("Z9" in detail and "premise" in detail for detail in row.details)
|
||||
|
||||
|
||||
def test_j1_a_registration_this_session_wrote_is_not_a_hold_out(tmp_path: Path) -> None:
|
||||
"""PM's J1: two files written by the session under test came back
|
||||
`7 of 7 GREEN`. Every check was an assertion the registration made about
|
||||
itself -- `written_by` is `bool()` of a string the file sets, and the
|
||||
readings are read from a list the same file owns."""
|
||||
held = tmp_path / "held-out.json"
|
||||
held.write_text('{"questions": []}', encoding="utf-8")
|
||||
registration = tmp_path / "registration.json"
|
||||
registration.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"set": str(held),
|
||||
"sha256": gate.sha256_of(held),
|
||||
"threshold": "hit@8 must not fall below 6 of 8",
|
||||
"threshold_written_at": "2026-01-01",
|
||||
"written_by": "the same session, lying",
|
||||
"readings": [],
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
row = gate.row_five(registration)
|
||||
assert row.status != gate.GREEN, row
|
||||
assert row.fails
|
||||
# What a session cannot write about itself: that git already carried the
|
||||
# file before the ranking moved.
|
||||
assert any("committed" in detail for detail in row.details), row.details
|
||||
assert any("ranking" in detail for detail in row.details), row.details
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue