feat(consume): the payload says what of the question it reached, and row 4 reads it
`coverage` carries three lists: the terms the pre-pass read the question as, the terms no concept in the bundle answers, and the terms no delivered excerpt answers. Without it a reader holding eight excerpts cannot tell a bundle that ANSWERED its question from one that merely ranked something -- the two payloads have the same shape. FACTS, AND NO VERDICT, which is a measurement and not caution. Two readings were built and both falsified over 81 questions (16 synthetic, 65 across the three real sets, 2026-09-20): the share of question terms a delivered excerpt answers separates the synthetic controls at 0.33 against 0.50 and REVERSES on real data (covered questions down to 0.27, one genuinely uncovered question at 0.71); the share of a bundle tying the best lexical match is ~0.00 for every real question either way. Question style dominates the first, corpus size the second. The one bar this repository declares is the gate's: `UNANSWERED_BAR = 2/3` over `unanswered_in_bundle`, swept and collapsing at both ends -- at 0.50 eleven real covered questions are marked, at 0.70 the row falls to 5 of 6, at 2/3 the row is 6 of 6 and 0 of 65 real questions are marked. The margin is thin (0.6087 against 0.6667) and is published that way, together with what it does not catch: r761-sk2's own known-negative sits at 0.2857. Row 4: 3 of 6 RED -> 6 of 6 GREEN, with the 10 answered synthetic questions held unmarked as the known-negative. The contract's SS 8 gains point 7, the consumption skill is told to read the block, and the SS 7.4 known-positive moves with the document (14 721/375 -> 16 389/417). Suite 2292 passed. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
parent
90394c383d
commit
05cb19087a
8 changed files with 216 additions and 42 deletions
|
|
@ -2661,14 +2661,14 @@ def test_the_payload_names_the_question_terms_the_bundle_answers_to_none_of() ->
|
|||
the ones no delivered excerpt answers.
|
||||
"""
|
||||
payload = okf_consume.build_payload(
|
||||
FIXTURE, question="Hva koster et doegn paa hytta for gjester?"
|
||||
FIXTURE, question="Hva koster et doegn for gjester i kravet?"
|
||||
)
|
||||
assert "coverage" in payload, "the payload says nothing about what it covered"
|
||||
coverage = payload["coverage"]
|
||||
assert isinstance(coverage, dict)
|
||||
# The terms are the ranker's own, counted here rather than read back.
|
||||
expected = list(
|
||||
dict.fromkeys(okf_consume.normalise("Hva koster et doegn paa hytta for gjester?"))
|
||||
dict.fromkeys(okf_consume.normalise("Hva koster et doegn for gjester i kravet?"))
|
||||
)
|
||||
assert coverage["question_terms"] == expected
|
||||
assert expected, "a question with no terms would make every assertion below vacuous"
|
||||
|
|
|
|||
|
|
@ -297,9 +297,18 @@ def test_row_three_is_green_when_the_printed_reason_is_the_true_one(tmp_path: Pa
|
|||
# --- row 4 --------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_row_four_is_red_because_an_uncovered_question_still_gets_excerpts(
|
||||
tmp_path: Path,
|
||||
def test_row_four_goes_red_again_when_the_payload_stops_saying_what_it_missed(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
"""The RED direction, driven by an input: the payload's own words.
|
||||
|
||||
This row was 3 of 6 until 2026-09-20 because the payload said nothing
|
||||
about what of the question it reached, so N3, N4 and N5 -- 8, 8 and 1
|
||||
excerpts -- were indistinguishable from an answered question. A payload
|
||||
that reports every term as answered puts the row back there, with the same
|
||||
detail line.
|
||||
"""
|
||||
monkeypatch.setattr(consume, "unanswered_terms", lambda *args, **kwargs: [])
|
||||
row = gate.row_four([_case(tmp_path, "set-controls.json")])
|
||||
assert row.status == gate.RED
|
||||
assert row.m == 6
|
||||
|
|
@ -639,7 +648,7 @@ def test_the_gate_is_red_today_and_says_which_rows(tmp_path: Path) -> None:
|
|||
rows = gate.evaluate(tmp_path / "bundles")
|
||||
by_number = {row.number: row for row in rows}
|
||||
assert sorted(by_number) == [1, 2, 3, 4, 5, 6, 7, 8, 9]
|
||||
assert [row.number for row in rows if row.fails] == [4, 5, 7, 8, 9]
|
||||
assert [row.number for row in rows if row.fails] == [5, 7, 8, 9]
|
||||
# 10, not 9: `set-quota.json` adds row 3's known-positive, one question the
|
||||
# source quota genuinely decides, and it is a hit.
|
||||
assert (by_number[1].k, by_number[1].m) == (10, 10)
|
||||
|
|
@ -661,7 +670,7 @@ def test_the_command_exits_one_and_prints_every_row(
|
|||
printed = capsys.readouterr().out
|
||||
for number in range(1, 10):
|
||||
assert f"\n{number} " in f"\n{printed}"
|
||||
assert "GATE RED: rows 4, 5, 7, 8, 9" in printed
|
||||
assert "GATE RED: rows 5, 7, 8, 9" in printed
|
||||
|
||||
|
||||
def test_the_json_form_carries_the_same_rows(capsys: pytest.CaptureFixture[str]) -> None:
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue