feat(consume): cut by exact knapsack with a closing partition

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

tests/test_okf_consume.py

@@ -438,3 +438,114 @@ def test_the_price_concept_leads_on_the_price_question_in_the_fixture() -> None:
     scores = okf_consume.document_scores(FIXTURE, "Hvordan skal prisene fylles ut?")
     ranked = okf_consume.concept_scores(concepts, "Hvordan skal prisene fylles ut?", scores)
     assert ranked[0][0].concept_id == "krav/prissammenstilling"
+
+
+# --- Step 7: the cut ----------------------------------------------------------
+
+
+def _cut_fixture(
+    question: str = "Hvordan skal prisene fylles ut?", k: int = 8, limit: int | None = None
+) -> tuple[list[dict[str, object]], list[tuple[str, str]], int]:
+    concepts = _fixture_concepts()
+    scores = okf_consume.document_scores(FIXTURE, question)
+    ranked = okf_consume.concept_scores(concepts, question, scores)
+    delivered, withheld = okf_consume.cut(
+        ranked, k=k, limit=okf_consume.DEFAULT_LIMIT if limit is None else limit
+    )
+    return list(delivered), list(withheld), len(ranked)
+
+
+def test_the_fixture_has_both_a_verdict_concept_and_a_delivered_one() -> None:
+    # The control on every count below: neither zero may come from an empty
+    # fixture.
+    delivered, withheld, considered = _cut_fixture()
+    assert delivered, "nothing was delivered, so 'excluded' would measure nothing"
+    assert withheld, "nothing was withheld, so the rules would measure nothing"
+    assert considered == len(okf_consume.enumerate_concepts(FIXTURE))
+
+
+def test_a_verdict_concept_is_withheld_by_rule_and_reaches_no_excerpt() -> None:
+    delivered, withheld, _ = _cut_fixture()
+    rules = dict(withheld)
+    assert rules["dyp/nivaa/alminnelig-notat"] == "verdict_layer_excluded"
+    assert all(excerpt["concept_id"] != "dyp/nivaa/alminnelig-notat" for excerpt in delivered)
+
+
+def test_the_verdict_exclusion_is_a_type_check_and_never_a_path_filter() -> None:
+    # A `type: reference` file NAMED `verdict-lookalike` is delivered; a
+    # `type: verdict` file under an ordinary name at depth 3 is withheld. A path
+    # filter gets both backwards.
+    delivered, withheld, _ = _cut_fixture()
+    delivered_ids = {excerpt["concept_id"] for excerpt in delivered}
+    rules = dict(withheld)
+    assert "krav/verdict-lookalike" in delivered_ids
+    assert "dyp/nivaa/alminnelig-notat" not in delivered_ids
+    assert rules["dyp/nivaa/alminnelig-notat"] == "verdict_layer_excluded"
+
+
+def test_a_capital_l_log_type_does_not_crash_the_reader() -> None:
+    # `type: Log` really occurs in the K2 corpus. Case handling is a test here
+    # rather than an accident.
+    delivered, withheld, _ = _cut_fixture()
+    seen = {excerpt["concept_id"] for excerpt in delivered} | {cid for cid, _ in withheld}
+    assert "krav/loggnotat" in seen
+
+
+def test_a_concept_whose_verified_cannot_be_read_is_withheld_by_name() -> None:
+    # SS 6.2 requires a tier on every excerpt and SS 6.4 forbids reading absence
+    # as negation. Emitting `unverified` for an unreadable value asserts a fact
+    # nobody measured.
+    _, withheld, _ = _cut_fixture()
+    assert dict(withheld)["dyp/nivaa/blokkform-verifisert"] == "verified_unreadable"
+
+
+def test_delivered_and_withheld_partition_the_considered_set() -> None:
+    delivered, withheld, considered = _cut_fixture()
+    delivered_ids = {excerpt["concept_id"] for excerpt in delivered}
+    withheld_ids = {concept_id for concept_id, _ in withheld}
+    assert delivered_ids & withheld_ids == set()
+    assert len(delivered_ids) + len(withheld_ids) == considered
+    assert delivered_ids | withheld_ids == set(okf_consume.enumerate_concepts(FIXTURE))
+
+
+def test_every_withheld_entry_names_a_rule_from_the_closed_set() -> None:
+    _, withheld, _ = _cut_fixture()
+    assert {rule for _, rule in withheld} <= set(okf_consume.WITHHOLDING_RULES)
+
+
+def test_a_concept_larger_than_the_limit_is_excluded_by_name_before_the_dp() -> None:
+    # Named as a RULE rather than left as a packing artefact: "it did not fit"
+    # and "it could never fit" are different facts about the cut.
+    _, withheld, _ = _cut_fixture(limit=200)
+    rules = {rule for _, rule in withheld}
+    assert "over_budget_alone" in rules
+    assert "over_budget_after_knapsack" not in rules
+
+
+def test_concepts_ranked_beyond_k_are_withheld_as_below_k() -> None:
+    _, withheld, _ = _cut_fixture(k=1)
+    assert "below_k" in {rule for _, rule in withheld}
+
+
+def test_the_exact_knapsack_beats_greedy_by_density() -> None:
+    # Greedy takes the densest item first and is then unable to fit either of
+    # the two that together are worth more. Greedy-by-density has an unbounded
+    # approximation factor; an exact DP over at most `k` items is microseconds.
+    items = ((10.0, 6), (7.0, 5), (7.0, 5))
+    chosen = okf_consume.knapsack(items, capacity=10)
+    assert sorted(chosen) == [1, 2]
+    assert sum(items[index][0] for index in chosen) == 14.0
+
+
+def test_the_knapsack_is_deterministic_over_equal_value_subsets() -> None:
+    items = ((5.0, 5), (5.0, 5), (5.0, 5))
+    assert okf_consume.knapsack(items, capacity=10) == okf_consume.knapsack(items, capacity=10)
+
+
+def test_excerpts_come_back_in_rank_order_and_carry_that_rank() -> None:
+    # An id-sorted payload would turn "position in the payload" into a
+    # different number from "position in the ranking", and hit@k reads the
+    # second one.
+    delivered, _, _ = _cut_fixture()
+    assert [excerpt["rank"] for excerpt in delivered] == list(range(1, len(delivered) + 1))
+    assert delivered[0]["concept_id"] == "krav/prissammenstilling"

tools/okf_consume.py

@@ -49,7 +49,11 @@ from llm_ingestion_okf.inbox import (  # noqa: E402
     ADJUDICATION_STATES,
 )
 from llm_ingestion_okf.materialize import parse_frontmatter  # noqa: E402
-from llm_ingestion_okf.profiles import SEGMENTED_OKF_V0_2, BundleProfile  # noqa: E402
+from llm_ingestion_okf.profiles import (  # noqa: E402
+    RESERVED_OKF_TYPE,
+    SEGMENTED_OKF_V0_2,
+    BundleProfile,
+)
 
 #: The profile whose index policy reads a faceted, per-directory index -- the
 #: shape both the in-repo golden bundle and the K2 corpus carry. Named as this
@@ -638,3 +642,149 @@ def concept_scores(
     by_id = {concept.concept_id: concept for concept in concepts}
     ranked_ids = sorted(fused, key=lambda key: (-fused[key], key))
     return [(by_id[concept_id], fused[concept_id]) for concept_id in ranked_ids]
+
+
+# --- The cut (SS 5, SS 7, SS 9.1) ---------------------------------------------
+
+#: Every rule this instrument may drop a concept under, spelled once. CLOSED:
+#: a drop with no rule is the silent cut SS 5.3 exists to forbid, and a rule
+#: invented at the drop site is a vocabulary no consumer can be held to.
+WITHHOLDING_RULES = (
+    "verdict_layer_excluded",
+    "verified_unreadable",
+    "over_budget_alone",
+    "below_k",
+    "over_budget_after_knapsack",
+)
+
+#: The knapsack's weight granularity, in bytes. Bucketing keeps the DP table
+#: small; bucketing UP the item and DOWN the capacity keeps the error one-sided,
+#: so the pack may under-deliver by a bucket and can never over-spend.
+WEIGHT_BUCKET = 500
+
+
+def excerpt_for(concept: Concept) -> dict[str, object] | None:
+    """One concept as a payload excerpt, or `None` when it cannot be tiered.
+
+    Carries the SS 8 members plus three the contract permits and this profile
+    needs:
+
+    - **`text`** -- the concept body. SS 8 names no content member, but SS 1
+      defines an excerpt as "one delivered unit of bundle CONTENT", and without
+      a body the budget gate would measure a two-kilobyte skeleton while the
+      skill went to the bundle itself, breaking SS 2.2.
+    - **`text_sha256`** -- the digest of the delivered bytes. `sha256` is the
+      digest of the WHOLE concept file (SS 3.2), so without this second digest a
+      consumer holds an identity that cannot verify what it was handed.
+    - **`bundle_id_inherited`** -- whether the first half of the SS 3.1 identity
+      tuple came from the concept or from the root index.
+
+    Trailing whitespace is stripped per line: a spreadsheet render is padded to
+    hundreds of trailing spaces per line, and unstripped, most of a budget goes
+    on padding.
+    """
+    tier = trust_tier(concept.frontmatter.get("verified"))
+    if tier is None:
+        return None
+    text = "\n".join(
+        line.rstrip() for line in unicodedata.normalize("NFC", concept.body).split("\n")
+    )
+    return {
+        "bundle_id": concept.bundle_id,
+        "concept_id": concept.concept_id,
+        "sha256": concept.sha256,
+        "adjudication": concept.adjudication,
+        "trust_tier": tier,
+        "bundle_id_inherited": concept.bundle_id_inherited,
+        "text_sha256": hashlib.sha256(text.encode("utf-8")).hexdigest(),
+        "text": text,
+    }
+
+
+def excerpt_weight(excerpt: Mapping[str, object]) -> int:
+    """What this excerpt costs by the gate's own instrument.
+
+    The ENCODED JSON object, never `stat().st_size`: measured, the two differ by
+    7.1 % over the K2 corpus, and a cut computed as fitting would then be
+    refused by a gate measuring different bytes.
+    """
+    return len(json.dumps(excerpt, ensure_ascii=False).encode("utf-8"))
+
+
+def knapsack(items: Sequence[tuple[float, int]], *, capacity: int) -> tuple[int, ...]:
+    """The exact 0/1 knapsack: indices of the highest-value subset that fits.
+
+    Exact rather than greedy-by-density, which has an unbounded approximation
+    factor -- and over at most `k` items the DP is microseconds, so the
+    approximation buys nothing. Deterministic: a subset replaces the incumbent
+    only on a STRICT improvement, so equal-value subsets resolve to the one
+    built from earlier items, and the caller sorts the pool by `concept_id`
+    first.
+    """
+    best: list[tuple[float, tuple[int, ...]]] = [(0.0, ()) for _ in range(capacity + 1)]
+    for index, (value, weight) in enumerate(items):
+        if weight > capacity:
+            continue
+        for room in range(capacity, weight - 1, -1):
+            candidate_value = best[room - weight][0] + value
+            if candidate_value > best[room][0]:
+                best[room] = (candidate_value, (*best[room - weight][1], index))
+    return max(best, key=lambda entry: entry[0])[1]
+
+
+def cut(
+    ranked: Sequence[tuple[Concept, float]], *, k: int, limit: int
+) -> tuple[tuple[dict[str, object], ...], tuple[tuple[str, str], ...]]:
+    """The ranked concepts split into delivered excerpts and named drops.
+
+    **The partition is the invariant, not a consequence.** Every considered
+    concept lands in exactly one of the two, so `considered == withheld +
+    delivered` closes by construction rather than by a count computed twice.
+
+    Exclusions run before the pack, each naming its rule, because "it did not
+    fit" and "it could never be delivered" are different facts about the cut.
+    """
+    withheld: list[tuple[str, str]] = []
+    candidates: list[tuple[Concept, float, dict[str, object], int]] = []
+    for concept, score in ranked:
+        # SS 9.1: a TYPE check at every level, never a path filter. Case-folded
+        # against the profile's own constant, because `type: Log` (capital L)
+        # really occurs in the corpus.
+        if concept.okf_type.casefold() == RESERVED_OKF_TYPE:
+            withheld.append((concept.concept_id, "verdict_layer_excluded"))
+            continue
+        excerpt = excerpt_for(concept)
+        if excerpt is None:
+            withheld.append((concept.concept_id, "verified_unreadable"))
+            continue
+        weight = excerpt_weight(excerpt)
+        if weight > limit:
+            withheld.append((concept.concept_id, "over_budget_alone"))
+            continue
+        candidates.append((concept, score, excerpt, weight))
+    for concept, _, _, _ in candidates[k:]:
+        withheld.append((concept.concept_id, "below_k"))
+    shortlist = candidates[:k]
+    # The DP POOL is sorted by `concept_id`, so which of two equal-value subsets
+    # wins is a property of the input set rather than of the order the ranker
+    # happened to emit. The OUTPUT is not: excerpts come back in fused-rank
+    # order, because the rank is what a hit@k measurement reads, and an id-sorted
+    # list would silently turn "position in the payload" into a different number
+    # from "position in the ranking".
+    pool = sorted(shortlist, key=lambda entry: entry[0].concept_id)
+    capacity = limit // WEIGHT_BUCKET
+    packed = {
+        id(pool[index][0])
+        for index in knapsack(
+            tuple((score, -(-weight // WEIGHT_BUCKET)) for _, score, _, weight in pool),
+            capacity=capacity,
+        )
+    }
+    delivered: list[dict[str, object]] = []
+    for concept, _, excerpt, _ in shortlist:
+        if id(concept) in packed:
+            delivered.append({**excerpt, "rank": len(delivered) + 1})
+        else:
+            withheld.append((concept.concept_id, "over_budget_after_knapsack"))
+    withheld.sort()
+    return tuple(delivered), tuple(withheld)