open/llm-ingestion-okf
1
0
Fork 0
Code Issues Releases 1 Activity

feat(consume): assemble a payload that measures its own emitted bytes

Browse source 
Two corrections the plan did not carry, both found by running the instrument:

- A question matching nothing still returned the top eight by tie-break -- a
  confident guess wearing a denominator. The closed rule set gains
  no_lexical_match, so the known-negative returns a measured empty set with
  every considered concept named in withheld.
- The stage-one document prior summed overlap across a document's units, so it
  measured document SIZE. Measured on K2 for the price question: the
  competition document sums to 6.0 over 79 concepts (0.076 each), the price
  document to 2.0 over 1. The prior is now a density; the price-form gold moves
  from outside the top eight to rank 1. Found with that answer visible, which
  the measurement document states beside the number.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Kjell Tore Guttormsen 2026-09-07 09:20:50 +02:00
commit 42ab9c1d1a
2 changed files with 417 additions and 25 deletions
Download patch file Download diff file Expand all files Collapse all files

193
tools/okf_consume.py
View file

@ -550,7 +550,7 @@ def _overlap(question_tokens: Sequence[str], candidate: str) -> int:
def document_scores(
bundle_root: Path, question: str, *, profile: BundleProfile = DEFAULT_PROFILE
) -> dict[str, float]:
"""One score per top-level document, from the indexes and the path alone.
"""One score per top-level document, from the indexes and the paths alone.
A "document" is a top-level entry: a directory, or -- per the corpus's own
shape -- a concept sitting at the root, which has no directory to inherit
@ -559,27 +559,45 @@ def document_scores(
nor `bundle_id`; scoring them as members of some parent would put one bug in
three places.
**The score is a DENSITY, not a sum, and that is a correction rather than a
preference.** A sum over a document's units grows with the number of units,
so a large document outscores a small one on size alone. Measured on K2 for
the price question: the competition document sums to 6.0 over 79 concepts
(0.076 each) and the price document to 2.0 over 1 (2.0), so the sum ranks
the larger document three times higher while the density ranks the smaller
one twenty-six times higher. A prior that grows with size is measuring size.
**Stated because it bears on how the hit@k number should be read:** this
defect was found by running the one question whose gold was confirmed
independently, and fixing it therefore happened with that answer visible.
The fix is justified by the scoring function's own arithmetic rather than by
the answer -- but the ranker is not blind to that one case, and the
measurement document says so beside the number.
Reads the INDEX TREE only. No directory is enumerated here or anywhere else
in this command (SS 9.2).
"""
question_tokens = normalise(question)
indexes, concepts = _walk_index_tree(bundle_root, profile=profile)
scores: dict[str, float] = {}
totals: dict[str, float] = {}
units: dict[str, int] = {}
def record(document: str, overlap: int) -> None:
totals[document] = totals.get(document, 0.0) + float(overlap)
units[document] = units.get(document, 0) + 1
for concept_id in concepts:
document = concept_id.split("/", 1)[0]
scores.setdefault(document, 0.0)
scores[document] += float(_overlap(question_tokens, concept_id.replace("/", " ")))
record(concept_id.split("/", 1)[0], _overlap(question_tokens, concept_id.replace("/", " ")))
for relative in indexes:
document = relative.split("/", 1)[0]
if document == profile.index.name:
continue
scores.setdefault(document, 0.0)
for line in (bundle_root / relative).read_text(encoding="utf-8").splitlines():
entry = profile.index.parse_entry(line)
if entry is None:
continue
scores[document] += float(_overlap(question_tokens, entry.label))
return scores
record(document, _overlap(question_tokens, entry.label))
return {document: totals[document] / units[document] for document in totals}
# --- Stage two: which concepts inside those documents -------------------------
@ -600,7 +618,7 @@ def concept_scores(
concepts: Sequence[Concept],
question: str,
document_score: Mapping[str, float],
) -> list[tuple[Concept, float]]:
) -> list[tuple[Concept, float, int]]:
"""Every concept, ordered best first, fused from three signals by RRF.
The signals: (1) the question against the concept's title and the segments
@ -614,6 +632,12 @@ def concept_scores(
**No float reaches the payload.** These scores order the cut; only ranks and
whole byte counts are emitted.
The third element of each tuple is the concept's OWN lexical overlap --
signals 1 and 2 only, with the document prior excluded. The cut needs it
separately: a concept that answers nothing in the question, sitting in a
document that does, is a GUESS, and a guess is the one thing a declared cut
must not deliver.
"""
question_tokens = normalise(question)
signals: list[dict[str, float]] = [
@ -639,9 +663,15 @@ def concept_scores(
order = sorted(signal, key=lambda key: (-signal[key], key))
for position, concept_id in enumerate(order, start=1):
fused[concept_id] += 1.0 / (RRF_K + position)
lexical = {
concept.concept_id: int(signals[0][concept.concept_id] + signals[1][concept.concept_id])
for concept in concepts
}
by_id = {concept.concept_id: concept for concept in concepts}
ranked_ids = sorted(fused, key=lambda key: (-fused[key], key))
return [(by_id[concept_id], fused[concept_id]) for concept_id in ranked_ids]
return [
(by_id[concept_id], fused[concept_id], lexical[concept_id]) for concept_id in ranked_ids
]
# --- The cut (SS 5, SS 7, SS 9.1) ---------------------------------------------
@ -652,6 +682,7 @@ def concept_scores(
WITHHOLDING_RULES = (
"verdict_layer_excluded",
"verified_unreadable",
"no_lexical_match",
"over_budget_alone",
"below_k",
"over_budget_after_knapsack",
@ -733,7 +764,7 @@ def knapsack(items: Sequence[tuple[float, int]], *, capacity: int) -> tuple[int,
def cut(
ranked: Sequence[tuple[Concept, float]], *, k: int, limit: int
ranked: Sequence[tuple[Concept, float, int]], *, k: int, limit: int
) -> tuple[tuple[dict[str, object], ...], tuple[tuple[str, str], ...]]:
"""The ranked concepts split into delivered excerpts and named drops.
@ -743,16 +774,25 @@ def cut(
Exclusions run before the pack, each naming its rule, because "it did not
fit" and "it could never be delivered" are different facts about the cut.
**A concept answering nothing in the question is withheld, never ranked into
the top k as filler.** Without that rule a question with no answer in the
bundle still returns eight excerpts -- a confident guess wearing a
denominator -- and hit@k over such a ranker measures the corpus's size
rather than the ranker.
"""
withheld: list[tuple[str, str]] = []
candidates: list[tuple[Concept, float, dict[str, object], int]] = []
for concept, score in ranked:
for concept, score, lexical in ranked:
# SS 9.1: a TYPE check at every level, never a path filter. Case-folded
# against the profile's own constant, because `type: Log` (capital L)
# really occurs in the corpus.
if concept.okf_type.casefold() == RESERVED_OKF_TYPE:
withheld.append((concept.concept_id, "verdict_layer_excluded"))
continue
if lexical == 0:
withheld.append((concept.concept_id, "no_lexical_match"))
continue
excerpt = excerpt_for(concept)
if excerpt is None:
withheld.append((concept.concept_id, "verified_unreadable"))
@ -788,3 +828,132 @@ def cut(
withheld.append((concept.concept_id, "over_budget_after_knapsack"))
withheld.sort()
return tuple(delivered), tuple(withheld)
# --- The payload (SS 8) -------------------------------------------------------
#: SS 8.2: present so a reader can tell which revision it is holding.
CONTRACT_REVISION = "okf-consumption/1"
#: `--k` caps the DELIVERED set. The budget is the gate; this is a second,
#: cheaper bound so a question matching half the corpus does not run a
#: 300-item DP to discover the same answer.
DEFAULT_K = 8
def build_payload(
bundle_root: Path,
*,
question: str,
k: int = DEFAULT_K,
limit: int = DEFAULT_LIMIT,
profile: BundleProfile = DEFAULT_PROFILE,
) -> dict[str, object]:
"""One bundle plus one question, cut to one contract-conformant payload.
Pure with respect to the clock and the network: the same
`(bundle_root, question, k, limit)` at the same bytes returns the same
object, every time.
"""
case, expected, measured = known_positive()
if expected != measured:
# SS 7.4: the instrument reports NONE of its own numbers until it has
# reproduced a known figure. Refusing here rather than emitting a
# payload with a failing known-positive is the difference between an
# instrument that has been shown to count and one that merely says so.
raise ConsumeError(
f"the budget instrument's known-positive ({case}) expected {expected} "
f"and measured {measured}; refusing to report any figure until the "
"two agree",
code="instrument_unvalidated",
)
root_index = bundle_root / profile.index.name
if not root_index.is_file():
raise ConsumeError(
f"{bundle_root} carries no {profile.index.name}, so there is no index "
"tree to walk and no way to read the bundle without enumerating a "
"directory, which SS 9.2 forbids",
code="bundle_unreadable",
)
root_bundle_id = parse_frontmatter(root_index).get("bundle_id", "")
if not root_bundle_id:
raise ConsumeError(
f"{root_index} declares no `bundle_id`; identity across bundles is "
"the (bundle_id, concept_id) tuple (SS 3.1) and half of it is missing",
code="bundle_id_missing",
)
concept_ids = enumerate_concepts(bundle_root, profile=profile)
concepts = [
read_concept(
bundle_root / f"{concept_id}{profile.paths.concept_suffix}",
bundle_root=bundle_root,
root_bundle_id=root_bundle_id,
)
for concept_id in concept_ids
]
ranked = concept_scores(
concepts, question, document_scores(bundle_root, question, profile=profile)
)
matched = sum(1 for _, _, lexical in ranked if lexical > 0)
delivered, withheld = cut(ranked, k=k, limit=limit)
spent = sum(excerpt_weight(excerpt) for excerpt in delivered)
if matched and not delivered:
# SS 7.3: a finding requiring a decision, never something to retry
# narrower. Distinguished from the honest empty result BY THE
# DENOMINATOR: concepts answered this question and the budget admitted
# none of them, which is a statement about the limit, not about the
# bundle.
raise ConsumeError(
f"{matched} concept(s) answered this question and the {limit}-byte "
f"budget admitted none of them; the cut strategy is wrong for this "
"bundle at this limit -- refusing rather than emitting an empty "
"payload that would read as 'nothing was found'",
code="budget_admits_nothing",
)
if spent > limit:
# Structurally unreachable while the knapsack bucket arithmetic is
# one-sided, and kept because SS 7.3 is a MUST about the emitted
# payload rather than about the algorithm that produced it.
raise ConsumeError(f"spent ({spent}) exceeds limit ({limit})", code="budget_exceeded")
return {
"contract": CONTRACT_REVISION,
"bundle": {
"bundle_id": root_bundle_id,
"ref": bundle_ref(bundle_root, profile=profile),
},
"budget": {
"unit": BUDGET_UNIT,
"instrument": BUDGET_INSTRUMENT,
"limit": limit,
"spent": spent,
"known_positive": {
"case": case,
"expected": expected,
"measured": measured,
# The second, independent route (`wc -c` on the same file), so
# `expected == measured` is not the only thing standing between
# a broken instrument and a green gate.
"raw_bytes": len(_KNOWN_POSITIVE_PATH.read_bytes()),
"encoding_delta": KNOWN_POSITIVE_ENCODING_DELTA,
},
},
"denominators": {
"considered": len(concepts),
"withheld": len(withheld),
"delivered": len(delivered),
},
"question": question,
"excerpts": list(delivered),
"withheld": [{"concept_id": concept_id, "rule": rule} for concept_id, rule in withheld],
}
def serialise(payload: Mapping[str, object]) -> str:
"""The payload as the bytes that are actually emitted.
`ensure_ascii=False` is load-bearing rather than cosmetic: the default
inflates this corpus by 7.1 % (1 950 745 -> 2 089 391 B), so a gate
measuring one form and a knapsack weighing the other disagree by more than
the headroom. LF only, one trailing newline.
"""
return json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=False) + "\n"