open/llm-ingestion-okf
1
0
Fork 0
Code Issues Releases 1 Activity

feat(consume): --follow-parent carries the enclosing section's text, from room the cut left

Browse source 
K3-21 B. The second form of `parent`: `okf consume --follow-parent`
(`consume.attach_parent_text`) puts the enclosing concept's text inside an
excerpt's `parent`, with that concept's own `sha256` so a claim resting on it
is cited as that concept. It runs AFTER the cut, on the room the cut left, in
rank order, so the delivered set, its order, the withheld list and the
denominators are the same with the flag as without it -- inherited text
cannot displace an excerpt, the mechanism a consumer measured when copied-in
ancestor text pushed the right section to withheld place 504 and 1 069. A
text that does not fit is cut to the longest prefix that does and marked
`truncated`; a parent the payload already holds, or one a higher-ranked
excerpt already carried, travels once. OFF; the defaults are chosen on the
measurement that follows this commit.

`delivered_text` is the one normalisation an excerpt's `text` and a parent's
share. Contract SS 8 point 6 gains the MAY; the template tells the reader
what `text`, `sha256` and `truncated` mean. README and CLAUDE.md name the
flag.

Moved on purpose: the SS 7.4 known-positive again (14 455 / 14 083 / 372 ->
14 721 / 14 346 / 375), and `skills/okf-consume/` regenerated with it.
`tests/test_parent_text.py::test_no_room_means_no_text_and_no_lost_excerpt`
changed from its red form: it asked through `build_payload` at `limit ==
spent`, where the knapsack's 500 B buckets admit nothing at all
(`budget_admits_nothing`); it now holds the rule at `attach_parent_text`.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Kjell Tore Guttormsen 2026-09-11 13:11:24 +02:00
commit 839bd61349
8 changed files with 158 additions and 25 deletions
Download patch file Download diff file Expand all files Collapse all files

113
src/llm_ingestion_okf/consume.py
View file

@ -669,14 +669,14 @@ KNOWN_POSITIVE_CASE = "docs/consumption-contract.md, encoded as a JSON string"
#: `measure()`'s own answer for that file. Vacuous ALONE -- which is why the
#: delta below exists.
KNOWN_POSITIVE_EXPECTED = 14_455
KNOWN_POSITIVE_EXPECTED = 14_721
#: The second, independent route. `wc -c` reports 14 083 raw bytes for the same
#: The second, independent route. `wc -c` reports 14 346 raw bytes for the same
#: file; the difference is this file's JSON quoting and escaping overhead. A
#: reader can derive it without running `measure()` at all, and it moves the
#: moment `measure()` changes what it counts -- which is what stops
#: `expected == measured` from proving nothing.
KNOWN_POSITIVE_ENCODING_DELTA = 372
KNOWN_POSITIVE_ENCODING_DELTA = 375
#: The two places that file can be, resolved in this order.
#:
@ -1613,6 +1613,12 @@ DEFAULT_SOURCE_QUOTA: int | None = 2
WEIGHT_BUCKET = 500
def delivered_text(body: str) -> str:
"""A concept body as a payload carries it: NFC, trailing whitespace
stripped per line. One rule for an excerpt's `text` and a `parent`'s."""
return "\n".join(line.rstrip() for line in unicodedata.normalize("NFC", body).split("\n"))
def excerpt_for(concept: Concept) -> dict[str, object] | None:
"""One concept as a payload excerpt, or `None` when it cannot be tiered.
@ -1651,9 +1657,7 @@ def excerpt_for(concept: Concept) -> dict[str, object] | None:
tier = trust_tier(concept.frontmatter.get("verified"))
if tier is None:
return None
text = "\n".join(
line.rstrip() for line in unicodedata.normalize("NFC", concept.body).split("\n")
)
text = delivered_text(concept.body)
excerpt: dict[str, object] = {
"bundle_id": concept.bundle_id,
"concept_id": concept.concept_id,
@ -1691,6 +1695,79 @@ def excerpt_weight(excerpt: Mapping[str, object]) -> int:
return len(json.dumps(excerpt, ensure_ascii=False).encode("utf-8"))
#: `--follow-parent` (K3-21 B): carry the enclosing concept's TEXT inside an
#: excerpt's `parent`, rather than the pointer alone.
DEFAULT_FOLLOW_PARENT = False
def _carrying(
excerpt: Mapping[str, object], source: Concept, text: str, *, truncated: bool
) -> dict[str, object]:
parent = excerpt["parent"]
assert isinstance(parent, dict)
carried: dict[str, object] = {**parent, "sha256": source.sha256, "text": text}
if truncated:
carried["truncated"] = True
return {**excerpt, "parent": carried}
def attach_parent_text(
delivered: Sequence[dict[str, object]],
concepts: Mapping[str, Concept],
*,
limit: int,
) -> tuple[dict[str, object], ...]:
"""The delivered excerpts, each `parent` carrying its concept's text where room allows.
Runs AFTER the cut, on the room the cut left, in rank order -- so the
delivered set, its order and the withheld list are what they were without
it, and inherited text cannot displace an excerpt. That is the mechanism a
consumer measured when it copied ancestors' text into every heading-only
section of one standard: the excerpts grew until the budget held 4-8, and
the right section fell to withheld place 504 and 1 069.
A parent the payload already holds, or one a higher-ranked excerpt already
carried, is not delivered again. A text the room cannot hold whole is cut
to the longest prefix that fits and marked `truncated`; with no room for
any of it, the pointer stays a pointer. `sha256` is the enclosing concept's
own, so a claim resting on that text is cited as that concept (SS 3.2).
"""
room = limit - sum(excerpt_weight(excerpt) for excerpt in delivered)
given = {str(excerpt["concept_id"]) for excerpt in delivered}
out: list[dict[str, object]] = []
for excerpt in delivered:
parent = excerpt.get("parent")
source = concepts.get(str(parent.get("concept_id"))) if isinstance(parent, dict) else None
if source is None or source.concept_id in given:
out.append(excerpt)
continue
text = delivered_text(source.body)
base = excerpt_weight(excerpt)
chosen = _carrying(excerpt, source, text, truncated=False)
if excerpt_weight(chosen) - base > room:
# The longest prefix that fits. The encoded weight never falls as
# a prefix grows, so the search is exact.
low, high = 0, len(text)
while low < high:
middle = (low + high + 1) // 2
trial = _carrying(excerpt, source, text[:middle], truncated=True)
if excerpt_weight(trial) - base <= room:
low = middle
else:
high = middle - 1
fits = _carrying(excerpt, source, text[:low], truncated=True)
chosen = fits if low and excerpt_weight(fits) - base <= room else dict(excerpt)
if chosen is not excerpt and "text" in _mapping_of(chosen.get("parent")):
given.add(source.concept_id)
room -= excerpt_weight(chosen) - base
out.append(chosen)
return tuple(out)
def _mapping_of(value: object) -> Mapping[str, object]:
return value if isinstance(value, Mapping) else {}
def knapsack(items: Sequence[tuple[float, int]], *, capacity: int) -> tuple[int, ...]:
"""The exact 0/1 knapsack: indices of the highest-value subset that fits.
@ -1889,6 +1966,7 @@ def build_payload(
withheld_titles: bool = False,
stem_prefix: bool = DEFAULT_STEM_PREFIX,
source_quota: int | None = DEFAULT_SOURCE_QUOTA,
follow_parent: bool = DEFAULT_FOLLOW_PARENT,
) -> dict[str, object]:
"""One bundle plus one question, cut to one contract-conformant payload.
@ -1977,6 +2055,11 @@ def build_payload(
reserve_top_rank=reserve_top_rank,
source_quota=source_quota,
)
if follow_parent:
# After the cut and never inside it: see `attach_parent_text`.
delivered = attach_parent_text(
delivered, {concept.concept_id: concept for concept in concepts}, limit=limit
)
spent = sum(excerpt_weight(excerpt) for excerpt in delivered)
if matched and not delivered:
# SS 7.3: a finding requiring a decision, never something to retry
@ -2191,6 +2274,23 @@ def parse_args(argv: list[str] | None) -> argparse.Namespace:
dest="source_quota",
help="The rule's explicit opt-out, reproducing the pre-round-11 excerpt order",
)
parser.add_argument(
"--follow-parent",
action="store_true",
default=DEFAULT_FOLLOW_PARENT,
help=(
"carry the enclosing concept's text inside an excerpt's `parent`, "
"with that concept's sha256, from the room the cut LEFT and in rank "
"order -- so it never displaces an excerpt; a text that does not fit "
"is clipped and marked `truncated` (K3-21)"
),
)
parser.add_argument(
"--no-follow-parent",
action="store_false",
dest="follow_parent",
help="The rule's explicit opt-out: `parent` is the pointer alone",
)
parser.add_argument(
"--withheld-titles",
action="store_true",
@ -2240,6 +2340,7 @@ def main(argv: list[str] | None = None) -> int:
stem_prefix=args.stem_prefix,
source_quota=args.source_quota,
withheld_titles=args.withheld_titles,
follow_parent=args.follow_parent,
)
except ConsumeError as error:
print(f"okf_consume: FAILED - {error}", file=sys.stderr)