feat(consume): cut by exact knapsack with a closing partition

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

tests/test_okf_consume.py

@@ -438,3 +438,114 @@ def test_the_price_concept_leads_on_the_price_question_in_the_fixture() -> None:
     scores = okf_consume.document_scores(FIXTURE, "Hvordan skal prisene fylles ut?")
     ranked = okf_consume.concept_scores(concepts, "Hvordan skal prisene fylles ut?", scores)
     assert ranked[0][0].concept_id == "krav/prissammenstilling"
+
+
+# --- Step 7: the cut ----------------------------------------------------------
+
+
+def _cut_fixture(
+    question: str = "Hvordan skal prisene fylles ut?", k: int = 8, limit: int | None = None
+) -> tuple[list[dict[str, object]], list[tuple[str, str]], int]:
+    concepts = _fixture_concepts()
+    scores = okf_consume.document_scores(FIXTURE, question)
+    ranked = okf_consume.concept_scores(concepts, question, scores)
+    delivered, withheld = okf_consume.cut(
+        ranked, k=k, limit=okf_consume.DEFAULT_LIMIT if limit is None else limit
+    )
+    return list(delivered), list(withheld), len(ranked)
+
+
+def test_the_fixture_has_both_a_verdict_concept_and_a_delivered_one() -> None:
+    # The control on every count below: neither zero may come from an empty
+    # fixture.
+    delivered, withheld, considered = _cut_fixture()
+    assert delivered, "nothing was delivered, so 'excluded' would measure nothing"
+    assert withheld, "nothing was withheld, so the rules would measure nothing"
+    assert considered == len(okf_consume.enumerate_concepts(FIXTURE))
+
+
+def test_a_verdict_concept_is_withheld_by_rule_and_reaches_no_excerpt() -> None:
+    delivered, withheld, _ = _cut_fixture()
+    rules = dict(withheld)
+    assert rules["dyp/nivaa/alminnelig-notat"] == "verdict_layer_excluded"
+    assert all(excerpt["concept_id"] != "dyp/nivaa/alminnelig-notat" for excerpt in delivered)
+
+
+def test_the_verdict_exclusion_is_a_type_check_and_never_a_path_filter() -> None:
+    # A `type: reference` file NAMED `verdict-lookalike` is delivered; a
+    # `type: verdict` file under an ordinary name at depth 3 is withheld. A path
+    # filter gets both backwards.
+    delivered, withheld, _ = _cut_fixture()
+    delivered_ids = {excerpt["concept_id"] for excerpt in delivered}
+    rules = dict(withheld)
+    assert "krav/verdict-lookalike" in delivered_ids
+    assert "dyp/nivaa/alminnelig-notat" not in delivered_ids
+    assert rules["dyp/nivaa/alminnelig-notat"] == "verdict_layer_excluded"
+
+
+def test_a_capital_l_log_type_does_not_crash_the_reader() -> None:
+    # `type: Log` really occurs in the K2 corpus. Case handling is a test here
+    # rather than an accident.
+    delivered, withheld, _ = _cut_fixture()
+    seen = {excerpt["concept_id"] for excerpt in delivered} | {cid for cid, _ in withheld}
+    assert "krav/loggnotat" in seen
+
+
+def test_a_concept_whose_verified_cannot_be_read_is_withheld_by_name() -> None:
+    # SS 6.2 requires a tier on every excerpt and SS 6.4 forbids reading absence
+    # as negation. Emitting `unverified` for an unreadable value asserts a fact
+    # nobody measured.
+    _, withheld, _ = _cut_fixture()
+    assert dict(withheld)["dyp/nivaa/blokkform-verifisert"] == "verified_unreadable"
+
+
+def test_delivered_and_withheld_partition_the_considered_set() -> None:
+    delivered, withheld, considered = _cut_fixture()
+    delivered_ids = {excerpt["concept_id"] for excerpt in delivered}
+    withheld_ids = {concept_id for concept_id, _ in withheld}
+    assert delivered_ids & withheld_ids == set()
+    assert len(delivered_ids) + len(withheld_ids) == considered
+    assert delivered_ids | withheld_ids == set(okf_consume.enumerate_concepts(FIXTURE))
+
+
+def test_every_withheld_entry_names_a_rule_from_the_closed_set() -> None:
+    _, withheld, _ = _cut_fixture()
+    assert {rule for _, rule in withheld} <= set(okf_consume.WITHHOLDING_RULES)
+
+
+def test_a_concept_larger_than_the_limit_is_excluded_by_name_before_the_dp() -> None:
+    # Named as a RULE rather than left as a packing artefact: "it did not fit"
+    # and "it could never fit" are different facts about the cut.
+    _, withheld, _ = _cut_fixture(limit=200)
+    rules = {rule for _, rule in withheld}
+    assert "over_budget_alone" in rules
+    assert "over_budget_after_knapsack" not in rules
+
+
+def test_concepts_ranked_beyond_k_are_withheld_as_below_k() -> None:
+    _, withheld, _ = _cut_fixture(k=1)
+    assert "below_k" in {rule for _, rule in withheld}
+
+
+def test_the_exact_knapsack_beats_greedy_by_density() -> None:
+    # Greedy takes the densest item first and is then unable to fit either of
+    # the two that together are worth more. Greedy-by-density has an unbounded
+    # approximation factor; an exact DP over at most `k` items is microseconds.
+    items = ((10.0, 6), (7.0, 5), (7.0, 5))
+    chosen = okf_consume.knapsack(items, capacity=10)
+    assert sorted(chosen) == [1, 2]
+    assert sum(items[index][0] for index in chosen) == 14.0
+
+
+def test_the_knapsack_is_deterministic_over_equal_value_subsets() -> None:
+    items = ((5.0, 5), (5.0, 5), (5.0, 5))
+    assert okf_consume.knapsack(items, capacity=10) == okf_consume.knapsack(items, capacity=10)
+
+
+def test_excerpts_come_back_in_rank_order_and_carry_that_rank() -> None:
+    # An id-sorted payload would turn "position in the payload" into a
+    # different number from "position in the ranking", and hit@k reads the
+    # second one.
+    delivered, _, _ = _cut_fixture()
+    assert [excerpt["rank"] for excerpt in delivered] == list(range(1, len(delivered) + 1))
+    assert delivered[0]["concept_id"] == "krav/prissammenstilling"