feat(consume): assemble a payload that measures its own emitted bytes

Two corrections the plan did not carry, both found by running the instrument:

- A question matching nothing still returned the top eight by tie-break -- a
  confident guess wearing a denominator. The closed rule set gains
  no_lexical_match, so the known-negative returns a measured empty set with
  every considered concept named in withheld.
- The stage-one document prior summed overlap across a document's units, so it
  measured document SIZE. Measured on K2 for the price question: the
  competition document sums to 6.0 over 79 concepts (0.076 each), the price
  document to 2.0 over 1. The prior is now a density; the price-form gold moves
  from outside the top eight to rank 1. Found with that answer visible, which
  the measurement document states beside the number.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

tests/test_okf_consume.py

@@ -34,9 +34,11 @@ PROJECT_ROOT = Path(__file__).resolve().parents[1]
 sys.path.insert(0, str(PROJECT_ROOT / "tools"))
 
 import okf_consume  # noqa: E402
+import okf_contract_check  # noqa: E402
 
 from llm_ingestion_okf.materialize import parse_frontmatter  # noqa: E402
 
+TEMPLATE = PROJECT_ROOT / "skills" / "okf-consume-template" / "SKILL.md"
 GOLDEN = PROJECT_ROOT / "examples" / "ingest-golden-segmented-okf-v0-2" / "expected-bundle"
 
 
@@ -401,16 +403,16 @@ def test_concepts_tying_on_every_signal_come_back_in_concept_id_order() -> None:
     # A question matching nothing makes every signal identical, so the ONLY
     # thing left deciding the order is the declared tie-break.
     ranked = okf_consume.concept_scores(concepts, "zzzz qqqq", {})
-    ids = [concept.concept_id for concept, _ in ranked]
+    ids = [concept.concept_id for concept, _, _ in ranked]
     assert ids == sorted(ids)
 
 
 def test_reversing_the_input_order_does_not_change_the_output_order() -> None:
     concepts = _fixture_concepts()
-    forward = [c.concept_id for c, _ in okf_consume.concept_scores(concepts, "zzzz qqqq", {})]
+    forward = [c.concept_id for c, _, _ in okf_consume.concept_scores(concepts, "zzzz qqqq", {})]
     backward = [
         c.concept_id
-        for c, _ in okf_consume.concept_scores(list(reversed(concepts)), "zzzz qqqq", {})
+        for c, _, _ in okf_consume.concept_scores(list(reversed(concepts)), "zzzz qqqq", {})
     ]
     assert forward == backward
 
@@ -420,8 +422,8 @@ def test_a_concept_in_a_high_scoring_document_outranks_an_equally_lexical_one()
     question = "Hvordan skal prisene fylles ut?"
     lifted = okf_consume.concept_scores(concepts, question, {"krav": 10.0, "dyp": 0.0})
     dropped = okf_consume.concept_scores(concepts, question, {"krav": 0.0, "dyp": 10.0})
-    krav_first = [c.concept_id for c, _ in lifted].index("krav/prissammenstilling")
-    krav_later = [c.concept_id for c, _ in dropped].index("krav/prissammenstilling")
+    krav_first = [c.concept_id for c, _, _ in lifted].index("krav/prissammenstilling")
+    krav_later = [c.concept_id for c, _, _ in dropped].index("krav/prissammenstilling")
     assert krav_first < krav_later
 
 
@@ -430,7 +432,7 @@ def test_the_ranked_order_is_identical_across_two_calls() -> None:
     scores = okf_consume.document_scores(FIXTURE, "Hvordan skal prisene fylles ut?")
     first = okf_consume.concept_scores(concepts, "Hvordan skal prisene fylles ut?", scores)
     second = okf_consume.concept_scores(concepts, "Hvordan skal prisene fylles ut?", scores)
-    assert [c.concept_id for c, _ in first] == [c.concept_id for c, _ in second]
+    assert [c.concept_id for c, _, _ in first] == [c.concept_id for c, _, _ in second]
 
 
 def test_the_price_concept_leads_on_the_price_question_in_the_fixture() -> None:
@@ -472,15 +474,17 @@ def test_a_verdict_concept_is_withheld_by_rule_and_reaches_no_excerpt() -> None:
 
 
 def test_the_verdict_exclusion_is_a_type_check_and_never_a_path_filter() -> None:
-    # A `type: reference` file NAMED `verdict-lookalike` is delivered; a
-    # `type: verdict` file under an ordinary name at depth 3 is withheld. A path
-    # filter gets both backwards.
-    delivered, withheld, _ = _cut_fixture()
+    # One question reaching BOTH: a `type: reference` file NAMED
+    # `verdict-lookalike` is delivered, and a `type: verdict` file under an
+    # ordinary name at depth 3 is withheld. A path filter gets both backwards,
+    # and neither half of this is measured unless both concepts match.
+    delivered, withheld, _ = _cut_fixture(question="Hva sier notatet om stifilter og typesjekk?")
     delivered_ids = {excerpt["concept_id"] for excerpt in delivered}
     rules = dict(withheld)
     assert "krav/verdict-lookalike" in delivered_ids
     assert "dyp/nivaa/alminnelig-notat" not in delivered_ids
     assert rules["dyp/nivaa/alminnelig-notat"] == "verdict_layer_excluded"
+    assert rules["dyp/nivaa/alminnelig-notat"] != "no_lexical_match"
 
 
 def test_a_capital_l_log_type_does_not_crash_the_reader() -> None:
@@ -495,7 +499,9 @@ def test_a_concept_whose_verified_cannot_be_read_is_withheld_by_name() -> None:
     # SS 6.2 requires a tier on every excerpt and SS 6.4 forbids reading absence
     # as negation. Emitting `unverified` for an unreadable value asserts a fact
     # nobody measured.
-    _, withheld, _ = _cut_fixture()
+    # The question must REACH the concept: a relevance drop fires first, and a
+    # `no_lexical_match` here would prove nothing about tiering.
+    _, withheld, _ = _cut_fixture(question="Hva staar i blokkform?")
     assert dict(withheld)["dyp/nivaa/blokkform-verifisert"] == "verified_unreadable"
 
 
@@ -523,8 +529,13 @@ def test_a_concept_larger_than_the_limit_is_excluded_by_name_before_the_dp() ->
 
 
 def test_concepts_ranked_beyond_k_are_withheld_as_below_k() -> None:
-    _, withheld, _ = _cut_fixture(k=1)
-    assert "below_k" in {rule for _, rule in withheld}
+    # A question matching TWO concepts, so that capping at one leaves a real
+    # `below_k` drop rather than an empty set.
+    matching = "kontroll av prisene"
+    _, wide, _ = _cut_fixture(question=matching, k=8)
+    assert "below_k" not in {rule for _, rule in wide}
+    _, narrow, _ = _cut_fixture(question=matching, k=1)
+    assert "below_k" in {rule for _, rule in narrow}
 
 
 def test_the_exact_knapsack_beats_greedy_by_density() -> None:
@@ -549,3 +560,215 @@ def test_excerpts_come_back_in_rank_order_and_carry_that_rank() -> None:
     delivered, _, _ = _cut_fixture()
     assert [excerpt["rank"] for excerpt in delivered] == list(range(1, len(delivered) + 1))
     assert delivered[0]["concept_id"] == "krav/prissammenstilling"
+
+
+# --- Step 8: the payload ------------------------------------------------------
+
+
+def _payload(
+    root: Path = FIXTURE, question: str = "Hvordan skal prisene fylles ut?", **kwargs: object
+) -> dict[str, object]:
+    return okf_consume.build_payload(root, question=question, **kwargs)  # type: ignore[arg-type]
+
+
+def test_the_payload_passes_the_checker_against_the_template_with_zero_findings() -> None:
+    report = okf_contract_check.check(TEMPLATE.read_text(encoding="utf-8"), _payload())
+    assert report.findings == ()
+
+
+def test_the_payload_carries_every_section_eight_member() -> None:
+    payload = _payload()
+    assert payload["contract"] == "okf-consumption/1"
+    assert set(payload) >= {
+        "contract",
+        "bundle",
+        "budget",
+        "denominators",
+        "excerpts",
+        "withheld",
+    }
+    bundle = payload["bundle"]
+    assert isinstance(bundle, dict)
+    assert bundle["bundle_id"] == "consume-fixture"
+    assert str(bundle["ref"]).startswith("sha256-tree:")
+
+
+def test_spent_is_the_cost_of_the_delivered_set_and_not_of_the_whole_payload() -> None:
+    # SS 7.2 verbatim: "what the DELIVERED SET spent by that same instrument".
+    # Measured on K2 at k=8, the whole-payload reading puts a 628-entry
+    # `withheld` list (81 565 B) plus one gold excerpt (101 576 B) against a
+    # 120 000 B limit -- so a CORRECT implementation would exit 1 and fail its
+    # own SC1 and SC6.
+    payload = _payload()
+    budget = payload["budget"]
+    excerpts = payload["excerpts"]
+    assert isinstance(budget, dict) and isinstance(excerpts, list)
+    assert budget["spent"] == sum(okf_consume.excerpt_weight(e) for e in excerpts)
+
+
+def test_spent_moves_when_an_excerpt_moves_and_holds_when_withheld_grows() -> None:
+    # The property that distinguishes SS 7.2's reading from the whole-payload
+    # one, asserted rather than described.
+    matching = "kontroll av prisene"
+    wide = _payload(question=matching, k=8)
+    narrow = _payload(question=matching, k=1)
+    wide_budget, narrow_budget = wide["budget"], narrow["budget"]
+    wide_counts, narrow_counts = wide["denominators"], narrow["denominators"]
+    assert isinstance(wide_budget, dict) and isinstance(narrow_budget, dict)
+    assert isinstance(wide_counts, dict) and isinstance(narrow_counts, dict)
+    assert narrow_counts["withheld"] > wide_counts["withheld"]
+    assert narrow_budget["spent"] < wide_budget["spent"]
+
+
+def test_the_counts_and_the_lists_are_two_statements_of_one_fact() -> None:
+    payload = _payload()
+    counts, excerpts, withheld = payload["denominators"], payload["excerpts"], payload["withheld"]
+    assert isinstance(counts, dict) and isinstance(excerpts, list) and isinstance(withheld, list)
+    assert counts["delivered"] == len(excerpts)
+    assert counts["withheld"] == len(withheld)
+    assert counts["considered"] == counts["delivered"] + counts["withheld"]
+
+
+def test_every_excerpt_digest_recomputes_from_the_named_concepts_bytes() -> None:
+    payload = _payload()
+    excerpts = payload["excerpts"]
+    assert isinstance(excerpts, list) and excerpts
+    for excerpt in excerpts:
+        on_disk = FIXTURE / f"{excerpt['concept_id']}.md"
+        assert excerpt["sha256"] == hashlib.sha256(on_disk.read_bytes()).hexdigest()
+        assert (
+            excerpt["text_sha256"]
+            == hashlib.sha256(str(excerpt["text"]).encode("utf-8")).hexdigest()
+        )
+
+
+def test_every_concept_id_keeps_the_slash_import_slug_would_have_removed() -> None:
+    payload = _payload()
+    excerpts = payload["excerpts"]
+    assert isinstance(excerpts, list)
+    assert any("/" in str(excerpt["concept_id"]) for excerpt in excerpts)
+
+
+def test_the_budget_refuses_rather_than_narrowing_when_the_cut_cannot_fit() -> None:
+    # SS 7.3: exceeding the gate is a finding requiring a decision, never
+    # something to retry narrower.
+    with pytest.raises(okf_consume.ConsumeError) as raised:
+        _payload(limit=100)
+    assert raised.value.code == "budget_admits_nothing"
+
+
+def test_the_serialised_payload_is_lf_only_and_ends_in_exactly_one_newline() -> None:
+    text = okf_consume.serialise(_payload())
+    assert "\r" not in text
+    assert text.endswith("}\n")
+    assert not text.endswith("}\n\n")
+
+
+def test_the_serialised_payload_does_not_escape_norwegian_letters() -> None:
+    # `ensure_ascii=True` inflates this corpus by 7.1 %, which is more than the
+    # headroom the gate leaves.
+    text = okf_consume.serialise(_payload(question="Hvor ofte er den årlige kontrollen?"))
+    assert "\\u00e5" not in text
+
+
+def test_a_question_with_no_answer_returns_a_measured_empty_set_not_a_guess() -> None:
+    # The order's known-negative control, and the reason the `no_lexical_match`
+    # rule exists: a ranker that always returns its top eight scores well on
+    # every positive question and is useless. The emptiness must be POSITIVE --
+    # every considered concept named in `withheld` under a rule, so the identity
+    # still closes and the skill can say "measured, nothing cleared the bar"
+    # rather than "nothing was found".
+    payload = _payload(question="Hva er reglene for sveising av titan i vakuum?")
+    counts, excerpts, withheld = payload["denominators"], payload["excerpts"], payload["withheld"]
+    assert isinstance(counts, dict) and isinstance(excerpts, list) and isinstance(withheld, list)
+    assert excerpts == []
+    assert counts["delivered"] == 0
+    assert (
+        counts["withheld"] == counts["considered"] == len(okf_consume.enumerate_concepts(FIXTURE))
+    )
+    assert {entry["rule"] for entry in withheld} == {"no_lexical_match", "verdict_layer_excluded"}
+    # And the control: the SAME payload builder returns a non-empty set for a
+    # question this bundle does answer, so the zero is a measurement.
+    answered = _payload()
+    answered_excerpts = answered["excerpts"]
+    assert isinstance(answered_excerpts, list) and answered_excerpts
+
+
+def test_the_empty_payload_still_passes_the_checker() -> None:
+    payload = _payload(question="Hva er reglene for sveising av titan i vakuum?")
+    assert okf_contract_check.check(TEMPLATE.read_text(encoding="utf-8"), payload).findings == ()
+
+
+# --- Corpus-conditional arms --------------------------------------------------
+
+K2_BUNDLE = Path.home() / "corpora" / "okf-telling-20260829" / "K2-bundle-20260903"
+K2_CONCEPTS = 629
+K2_PROPOSED = 618
+K2_KEYLESS = 11
+
+requires_k2 = pytest.mark.skipif(
+    not K2_BUNDLE.is_dir(),
+    reason=(
+        f"the K2 corpus is not present at {K2_BUNDLE}. NOT MEASURED, not zero: "
+        f"this arm covers a denominator of {K2_CONCEPTS} concepts, of which "
+        f"{K2_PROPOSED} carry `adjudication: proposed` and {K2_KEYLESS} carry no "
+        "`adjudication` key at all. A skip here is an unmeasured denominator, "
+        "never a pass."
+    ),
+)
+
+
+@requires_k2
+def test_the_eleven_keyless_k2_concepts_come_back_unknown_over_a_stated_denominator() -> None:
+    # SS 6.1's third state, on real data rather than on a fixture. The 11 are
+    # asserted as ONE named set: measured, the concepts carrying no
+    # `adjudication` are EXACTLY those carrying no `bundle_id`, so three
+    # independent counts would share one blind spot.
+    root_bundle_id = parse_frontmatter(K2_BUNDLE / "index.md")["bundle_id"]
+    concepts = [
+        okf_consume.read_concept(
+            K2_BUNDLE / f"{concept_id}.md",
+            bundle_root=K2_BUNDLE,
+            root_bundle_id=root_bundle_id,
+        )
+        for concept_id in okf_consume.enumerate_concepts(K2_BUNDLE)
+    ]
+    assert len(concepts) == K2_CONCEPTS
+    unknown = {c.concept_id for c in concepts if c.adjudication == "unknown"}
+    inherited = {c.concept_id for c in concepts if c.bundle_id_inherited}
+    proposed = [c for c in concepts if c.adjudication == "proposed"]
+    assert len(proposed) == K2_PROPOSED
+    assert len(unknown) == K2_KEYLESS
+    assert unknown == inherited, "the two sets diverged; the fallback is no longer one fact"
+    assert all(c.bundle_id == root_bundle_id for c in concepts if c.bundle_id_inherited)
+    # `adjudicated` has denominator ZERO on this corpus. Stated, not implied.
+    assert [c for c in concepts if c.adjudication == "adjudicated"] == []
+
+
+@requires_k2
+def test_spent_is_the_delivered_set_where_the_whole_payload_reading_would_refuse() -> None:
+    # The regression guard, with figures RE-MEASURED here rather than carried
+    # from the plan: the plan predicted 101 576 B for this excerpt and 188 758 B
+    # for the payload, both taken before per-line trailing-whitespace stripping
+    # landed. What this build actually produces is recorded instead.
+    payload = okf_consume.build_payload(K2_BUNDLE, question="Hvordan skal prisene fylles ut?")
+    budget, excerpts = payload["budget"], payload["excerpts"]
+    assert isinstance(budget, dict) and isinstance(excerpts, list)
+    whole_payload = len(okf_consume.serialise(payload).encode("utf-8"))
+    assert whole_payload > int(budget["limit"]), (
+        "the guard measures nothing: the whole payload already fits, so the two "
+        "readings of SS 7.2 cannot be told apart on this case"
+    )
+    assert int(budget["spent"]) <= int(budget["limit"])
+
+
+@requires_k2
+def test_the_price_form_gold_is_delivered_for_the_price_question() -> None:
+    # SC6. The one question whose gold is confirmed by a signal from outside
+    # this repository: a live model reached that directory unprompted in three
+    # navigation steps on 2026-09-06.
+    payload = okf_consume.build_payload(K2_BUNDLE, question="Hvordan skal prisene fylles ut?")
+    excerpts = payload["excerpts"]
+    assert isinstance(excerpts, list)
+    ids = [str(excerpt["concept_id"]) for excerpt in excerpts]
+    assert "del-ii-bilag-7-prisskjema/prissammenstilling-sheet-1" in ids