feat(consume): cut by exact knapsack with a closing partition

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

tools/okf_consume.py

@@ -49,7 +49,11 @@ from llm_ingestion_okf.inbox import (  # noqa: E402
     ADJUDICATION_STATES,
 )
 from llm_ingestion_okf.materialize import parse_frontmatter  # noqa: E402
-from llm_ingestion_okf.profiles import SEGMENTED_OKF_V0_2, BundleProfile  # noqa: E402
+from llm_ingestion_okf.profiles import (  # noqa: E402
+    RESERVED_OKF_TYPE,
+    SEGMENTED_OKF_V0_2,
+    BundleProfile,
+)
 
 #: The profile whose index policy reads a faceted, per-directory index -- the
 #: shape both the in-repo golden bundle and the K2 corpus carry. Named as this
@@ -638,3 +642,149 @@ def concept_scores(
     by_id = {concept.concept_id: concept for concept in concepts}
     ranked_ids = sorted(fused, key=lambda key: (-fused[key], key))
     return [(by_id[concept_id], fused[concept_id]) for concept_id in ranked_ids]
+
+
+# --- The cut (SS 5, SS 7, SS 9.1) ---------------------------------------------
+
+#: Every rule this instrument may drop a concept under, spelled once. CLOSED:
+#: a drop with no rule is the silent cut SS 5.3 exists to forbid, and a rule
+#: invented at the drop site is a vocabulary no consumer can be held to.
+WITHHOLDING_RULES = (
+    "verdict_layer_excluded",
+    "verified_unreadable",
+    "over_budget_alone",
+    "below_k",
+    "over_budget_after_knapsack",
+)
+
+#: The knapsack's weight granularity, in bytes. Bucketing keeps the DP table
+#: small; bucketing UP the item and DOWN the capacity keeps the error one-sided,
+#: so the pack may under-deliver by a bucket and can never over-spend.
+WEIGHT_BUCKET = 500
+
+
+def excerpt_for(concept: Concept) -> dict[str, object] | None:
+    """One concept as a payload excerpt, or `None` when it cannot be tiered.
+
+    Carries the SS 8 members plus three the contract permits and this profile
+    needs:
+
+    - **`text`** -- the concept body. SS 8 names no content member, but SS 1
+      defines an excerpt as "one delivered unit of bundle CONTENT", and without
+      a body the budget gate would measure a two-kilobyte skeleton while the
+      skill went to the bundle itself, breaking SS 2.2.
+    - **`text_sha256`** -- the digest of the delivered bytes. `sha256` is the
+      digest of the WHOLE concept file (SS 3.2), so without this second digest a
+      consumer holds an identity that cannot verify what it was handed.
+    - **`bundle_id_inherited`** -- whether the first half of the SS 3.1 identity
+      tuple came from the concept or from the root index.
+
+    Trailing whitespace is stripped per line: a spreadsheet render is padded to
+    hundreds of trailing spaces per line, and unstripped, most of a budget goes
+    on padding.
+    """
+    tier = trust_tier(concept.frontmatter.get("verified"))
+    if tier is None:
+        return None
+    text = "\n".join(
+        line.rstrip() for line in unicodedata.normalize("NFC", concept.body).split("\n")
+    )
+    return {
+        "bundle_id": concept.bundle_id,
+        "concept_id": concept.concept_id,
+        "sha256": concept.sha256,
+        "adjudication": concept.adjudication,
+        "trust_tier": tier,
+        "bundle_id_inherited": concept.bundle_id_inherited,
+        "text_sha256": hashlib.sha256(text.encode("utf-8")).hexdigest(),
+        "text": text,
+    }
+
+
+def excerpt_weight(excerpt: Mapping[str, object]) -> int:
+    """What this excerpt costs by the gate's own instrument.
+
+    The ENCODED JSON object, never `stat().st_size`: measured, the two differ by
+    7.1 % over the K2 corpus, and a cut computed as fitting would then be
+    refused by a gate measuring different bytes.
+    """
+    return len(json.dumps(excerpt, ensure_ascii=False).encode("utf-8"))
+
+
+def knapsack(items: Sequence[tuple[float, int]], *, capacity: int) -> tuple[int, ...]:
+    """The exact 0/1 knapsack: indices of the highest-value subset that fits.
+
+    Exact rather than greedy-by-density, which has an unbounded approximation
+    factor -- and over at most `k` items the DP is microseconds, so the
+    approximation buys nothing. Deterministic: a subset replaces the incumbent
+    only on a STRICT improvement, so equal-value subsets resolve to the one
+    built from earlier items, and the caller sorts the pool by `concept_id`
+    first.
+    """
+    best: list[tuple[float, tuple[int, ...]]] = [(0.0, ()) for _ in range(capacity + 1)]
+    for index, (value, weight) in enumerate(items):
+        if weight > capacity:
+            continue
+        for room in range(capacity, weight - 1, -1):
+            candidate_value = best[room - weight][0] + value
+            if candidate_value > best[room][0]:
+                best[room] = (candidate_value, (*best[room - weight][1], index))
+    return max(best, key=lambda entry: entry[0])[1]
+
+
+def cut(
+    ranked: Sequence[tuple[Concept, float]], *, k: int, limit: int
+) -> tuple[tuple[dict[str, object], ...], tuple[tuple[str, str], ...]]:
+    """The ranked concepts split into delivered excerpts and named drops.
+
+    **The partition is the invariant, not a consequence.** Every considered
+    concept lands in exactly one of the two, so `considered == withheld +
+    delivered` closes by construction rather than by a count computed twice.
+
+    Exclusions run before the pack, each naming its rule, because "it did not
+    fit" and "it could never be delivered" are different facts about the cut.
+    """
+    withheld: list[tuple[str, str]] = []
+    candidates: list[tuple[Concept, float, dict[str, object], int]] = []
+    for concept, score in ranked:
+        # SS 9.1: a TYPE check at every level, never a path filter. Case-folded
+        # against the profile's own constant, because `type: Log` (capital L)
+        # really occurs in the corpus.
+        if concept.okf_type.casefold() == RESERVED_OKF_TYPE:
+            withheld.append((concept.concept_id, "verdict_layer_excluded"))
+            continue
+        excerpt = excerpt_for(concept)
+        if excerpt is None:
+            withheld.append((concept.concept_id, "verified_unreadable"))
+            continue
+        weight = excerpt_weight(excerpt)
+        if weight > limit:
+            withheld.append((concept.concept_id, "over_budget_alone"))
+            continue
+        candidates.append((concept, score, excerpt, weight))
+    for concept, _, _, _ in candidates[k:]:
+        withheld.append((concept.concept_id, "below_k"))
+    shortlist = candidates[:k]
+    # The DP POOL is sorted by `concept_id`, so which of two equal-value subsets
+    # wins is a property of the input set rather than of the order the ranker
+    # happened to emit. The OUTPUT is not: excerpts come back in fused-rank
+    # order, because the rank is what a hit@k measurement reads, and an id-sorted
+    # list would silently turn "position in the payload" into a different number
+    # from "position in the ranking".
+    pool = sorted(shortlist, key=lambda entry: entry[0].concept_id)
+    capacity = limit // WEIGHT_BUCKET
+    packed = {
+        id(pool[index][0])
+        for index in knapsack(
+            tuple((score, -(-weight // WEIGHT_BUCKET)) for _, score, _, weight in pool),
+            capacity=capacity,
+        )
+    }
+    delivered: list[dict[str, object]] = []
+    for concept, _, excerpt, _ in shortlist:
+        if id(concept) in packed:
+            delivered.append({**excerpt, "rank": len(delivered) + 1})
+        else:
+            withheld.append((concept.concept_id, "over_budget_after_knapsack"))
+    withheld.sort()
+    return tuple(delivered), tuple(withheld)