test(consume): a drop the rank had already made must not be named as the quota's
Red first, on behaviour: the withheld set names c4 and c5 as `source_quota_exceeded` where the same cut without the quota withholds them `below_k`. The test measures the truth itself, off a second cut, and carries a known-positive so the assertion is not vacuous. Measured by PM on 25 real misses 2026-09-17: 13 were labelled by the quota and decided by the rank. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
parent
735fb237e4
commit
6b62ecea34
1 changed files with 37 additions and 0 deletions
|
|
@ -2609,3 +2609,40 @@ def test_a_source_key_is_a_prefix_and_never_a_substring(tmp_path: Path) -> None:
|
|||
)
|
||||
assert excerpt is not None
|
||||
assert "resource_owner" not in excerpt
|
||||
|
||||
|
||||
def test_a_drop_the_rank_had_already_made_is_not_named_as_the_quotas() -> None:
|
||||
"""The reason a withheld concept carries must be true OF THAT DROP.
|
||||
|
||||
The quota filters the WHOLE candidate list, not the top k, so every
|
||||
over-quota candidate came back `source_quota_exceeded` -- including the
|
||||
ones the rank had already put outside k, which the quota only reached
|
||||
because it ran first. Measured by PM on 25 real misses 2026-09-17: 13 of
|
||||
them were labelled by the quota and decided by the rank.
|
||||
|
||||
The truth is the SAME cut run without the quota, which is what the
|
||||
retrieval gate's row 3 compares the label against.
|
||||
"""
|
||||
concepts = tuple(
|
||||
_quota_concept(f"c{index}", source_file="o.md" if index < 6 else f"x{index}.md")
|
||||
for index in range(10)
|
||||
)
|
||||
ranked = tuple((concept, 1.0 / (index + 1), 3) for index, concept in enumerate(concepts))
|
||||
_, withheld, _ = okf_consume.cut(ranked, k=4, limit=okf_consume.DEFAULT_LIMIT, source_quota=2)
|
||||
rules = dict(withheld)
|
||||
# The truth, measured here rather than taken from the cut under test.
|
||||
off_delivered, _, _ = okf_consume.cut(
|
||||
ranked, k=4, limit=okf_consume.DEFAULT_LIMIT, source_quota=None
|
||||
)
|
||||
would_have_been_delivered = {str(entry["concept_id"]) for entry in off_delivered}
|
||||
assert would_have_been_delivered == {"c0", "c1", "c2", "c3"}
|
||||
|
||||
named_by_the_quota = {cid for cid, rule in withheld if rule == "source_quota_exceeded"}
|
||||
# KNOWN-POSITIVE: the quota really does decide something on this fixture,
|
||||
# so the assertion below is not vacuous over an empty set.
|
||||
assert named_by_the_quota, "the quota decided nothing here; the test measures nothing"
|
||||
assert named_by_the_quota == {"c2", "c3"}
|
||||
assert named_by_the_quota <= would_have_been_delivered
|
||||
# And the two the rank had already lost say so.
|
||||
assert rules["c4"] == "below_k"
|
||||
assert rules["c5"] == "below_k"
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue