feat(consume): a byte budget instrument that validates before it reports

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

tests/test_okf_consume.py

@@ -22,6 +22,7 @@ house pattern rather than new inventions:
 from __future__ import annotations
 
 import hashlib
+import json
 import os
 import sys
 from pathlib import Path
@@ -253,3 +254,58 @@ def test_the_block_form_case_is_real_in_the_fixture_and_not_only_in_the_unit_tes
     frontmatter = parse_frontmatter(FIXTURE / "dyp" / "nivaa" / "blokkform-verifisert.md")
     assert frontmatter["verified"] == ""
     assert okf_consume.trust_tier(frontmatter["verified"]) is None
+
+
+# --- Step 4: the budget instrument -------------------------------------------
+
+CONTRACT = PROJECT_ROOT / "docs" / "consumption-contract.md"
+
+
+def test_measure_counts_bytes_and_not_characters() -> None:
+    # The exact conflation the brief records itself making once: a chars/token
+    # ratio quoted where a bytes/token one was needed. `æøå` is three
+    # characters and six bytes, and the two only differ outside ASCII.
+    assert okf_consume.measure("æøå") == len('"æøå"'.encode())
+    assert okf_consume.measure("æøå") != len("æøå")
+
+
+def test_measure_counts_the_encoded_form_the_payload_actually_costs() -> None:
+    # `json.dumps` defaults to `ensure_ascii=True`, which inflates this corpus
+    # by 7.1 %. A gate measuring one form while the knapsack weighs the other
+    # disagrees by more than the headroom.
+    norwegian = "årlig kontroll av anlegget"
+    assert okf_consume.measure(norwegian) == len(
+        json.dumps(norwegian, ensure_ascii=False).encode("utf-8")
+    )
+    assert okf_consume.measure(norwegian) < len(
+        json.dumps(norwegian, ensure_ascii=True).encode("utf-8")
+    )
+
+
+def test_the_known_positive_is_reproduced_by_the_gates_own_instrument() -> None:
+    case, expected, measured = okf_consume.known_positive()
+    assert case
+    assert expected == measured, "SS 7.4: the instrument has not been shown to count"
+
+
+def test_the_known_positive_is_not_the_raw_byte_count_of_the_same_file() -> None:
+    # Validating one instrument while gating with another is the SS 7.4 failure
+    # the rule exists to prevent. The delta is derivable by a second, wholly
+    # independent route (`wc -c`) and moves the moment `measure` changes what it
+    # counts -- which is what keeps `expected == measured` from being vacuous.
+    _, expected, _ = okf_consume.known_positive()
+    raw = len(CONTRACT.read_bytes())
+    assert expected != raw
+    assert expected - raw == okf_consume.KNOWN_POSITIVE_ENCODING_DELTA
+
+
+def test_the_default_limit_admits_a_concept_the_size_of_the_price_form() -> None:
+    # Measured during planning: at the drafted 60 000 B default the SC6 gold
+    # concept (101 313 B encoded) falls to the "cannot fit alone" pre-exclusion,
+    # so SC1 and SC6 were mutually unsatisfiable on a CORRECT implementation.
+    assert okf_consume.DEFAULT_LIMIT >= 101_313
+
+
+def test_the_budget_unit_and_instrument_are_named_rather_than_implied() -> None:
+    assert "byte" in okf_consume.BUDGET_UNIT
+    assert "ensure_ascii=False" in okf_consume.BUDGET_INSTRUMENT

tools/okf_consume.py

@@ -32,6 +32,7 @@ wheel-installed command is a move rather than a rewrite.
 from __future__ import annotations
 
 import hashlib
+import json
 import sys
 from collections.abc import Mapping
 from dataclasses import dataclass
@@ -403,3 +404,71 @@ def _split_top_level(body: str, opener: str, closer: str) -> list[str]:
         current.append(character)
     parts.append("".join(current))
     return [part for part in parts if part.strip()]
+
+
+# --- The budget instrument (SS 7) --------------------------------------------
+
+#: SS 7.5 fixes no unit deliberately -- "a token is one encoder family's unit
+#: and fixing it would adopt one vendor's arithmetic as everyone's". This
+#: profile chooses utf-8 bytes of the EMITTED JSON, which the repository can
+#: count with no dependency at all. `tiktoken` would be runtime dependency
+#: number two behind a second optional extra plus a tokenizer-version fixture
+#: migration, bought to answer one comparison in its own unit.
+BUDGET_UNIT = "utf-8 bytes of emitted JSON"
+
+#: SS 7.1 requires the instrument to be NAMED, not merely used. The name is the
+#: function plus the one flag that changes its answer.
+BUDGET_INSTRUMENT = "okf_consume.measure (len of the ensure_ascii=False JSON encoding, utf-8)"
+
+#: Chosen, not derived, and the reason is a measurement rather than a taste:
+#: at 60 000 the largest realistic gold concept (101 313 B encoded) falls to the
+#: `over_budget_alone` pre-exclusion, so a CORRECT implementation would fail its
+#: own acceptance criteria. At 120 000 that concept fits with 18 424 B of
+#: headroom, and 3 of the K2 corpus's 629 concepts still cannot fit alone
+#: (4 at 60 000). A starting point to be moved by measurement.
+DEFAULT_LIMIT = 120_000
+
+#: The known-positive artefact (SS 7.4). A SHIPPED file rather than the bundle
+#: under test, because a per-bundle known-positive can only be one of two
+#: useless things: a constant that is wrong for every bundle but one, or the
+#: instrument's own output, which makes `expected == measured` true by
+#: construction and the rule decorative.
+#:
+#: The coupling is stated rather than hidden: if this document's bytes move, the
+#: literal below goes stale and the pre-pass refuses until it is updated. That
+#: is the intended direction -- a stale known-positive is a loud failure, and
+#: the document is normative and not edited from this repository.
+KNOWN_POSITIVE_CASE = "docs/consumption-contract.md, encoded as a JSON string"
+
+#: `measure()`'s own answer for that file. Vacuous ALONE -- which is why the
+#: delta below exists.
+KNOWN_POSITIVE_EXPECTED = 10_349
+
+#: The second, independent route. `wc -c` reports 10 060 raw bytes for the same
+#: file; the difference is this file's JSON quoting and escaping overhead. A
+#: reader can derive it without running `measure()` at all, and it moves the
+#: moment `measure()` changes what it counts -- which is what stops
+#: `expected == measured` from proving nothing.
+KNOWN_POSITIVE_ENCODING_DELTA = 289
+
+_KNOWN_POSITIVE_PATH = Path(__file__).resolve().parents[1] / "docs" / "consumption-contract.md"
+
+
+def measure(value: str) -> int:
+    """The cost of `value` in the unit the gate enforces.
+
+    The ENCODED JSON form, because that is what the payload actually costs. A
+    knapsack weighing `stat().st_size` while the gate measures this would let a
+    cut computed as fitting be refused by the gate -- measured, the two differ
+    by 7.1 % over the K2 corpus.
+    """
+    return len(json.dumps(value, ensure_ascii=False).encode("utf-8"))
+
+
+def known_positive() -> tuple[str, int, int]:
+    """The case, the figure expected of it, and the figure measured (SS 7.4).
+
+    Takes no bundle argument on purpose: see `KNOWN_POSITIVE_CASE`.
+    """
+    measured = measure(_KNOWN_POSITIVE_PATH.read_text(encoding="utf-8"))
+    return KNOWN_POSITIVE_CASE, KNOWN_POSITIVE_EXPECTED, measured