open/llm-ingestion-okf
1
0
Fork 0
Code Issues Releases 1 Activity

feat(propose): --shell-parent points a heading-only section at the ancestor holding its text

Browse source 
A plan entry whose span holds its heading alone gets `parent_id` naming the
nearest preceding entry at a smaller level whose own span holds text, passing
over an ancestor that is empty too; the door writes the existing `parent:`
key. Nothing is copied and no boundary moves. The rule reads the plan's level
and order, never the row. Off by default: `okf consume` reads no `parent`
key, so no payload ranks differently, while the flag moves the bytes of every
bundle holding a heading-only section.

Measured before building on one 2 761-concept process code: 710 concepts are
heading-only; the level route names the ancestor the document's own <sec>
nesting names on 708 of 710 (two sit at depth 7, clipped to 6, and point one
level too high), where reading section numbers gets 686 (`12` begins with
`1`); 35 have no ancestor holding text and get none.

The red test expected only concept files to move. The index is a projection
of the frontmatter and shows the key as a facet, so the test now holds both;
the facet renders a segment id as unresolved (`p1?`), because `structure`
reads `parent` as a document number -- named in README, CLAUDE.md and the
CHANGELOG, not repaired here.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Kjell Tore Guttormsen 2026-09-11 05:14:15 +02:00
commit edd3b70a90
6 changed files with 140 additions and 4 deletions
Download patch file Download diff file Expand all files Collapse all files

32
src/llm_ingestion_okf/cli.py
View file

@ -172,6 +172,13 @@ DEFAULT_FIRST_SPAN_FROM_ZERO = True
#: rule closes each span against the next SURVIVOR. It adds no boundary, so the
#: concept count is unchanged and the cost to a consumer is a re-run.
DEFAULT_CLOSE_SPAN_GAPS = True
#: Round 20 (K3-20). A section whose body is its heading alone gets `parent:`
#: naming the nearest ancestor that holds text (`propose._link_shells`). NOT a
#: segmentation rule: it adds no boundary and copies no text, and the concept
#: count is unchanged. OFF, and the default is a measurement of what it buys:
#: `okf consume` reads no `parent` key, so no payload ranks differently with
#: it, while it moves the bytes of every bundle holding a heading-only section.
DEFAULT_SHELL_PARENT = False
#: Round 9. Clause 1 asked whether a title ENDED in a number, which is a
#: question about the number rather than about the title: a drawing's dimension
#: chain, a P&ID's schematic labels, a door schedule and a borehole log's
@ -353,6 +360,7 @@ def _propose_plans(
pdf_headings_reserve: bool = False,
ocr: bool = False,
pdf_outline: bool = DEFAULT_PDF_OUTLINE,
shell_parent: bool = DEFAULT_SHELL_PARENT,
) -> tuple[int, int, int]:
"""Propose a plan per dropped file. Returns (written, nothing, failed).
@ -393,6 +401,7 @@ def _propose_plans(
pdf_headings_reserve=pdf_headings_reserve,
ocr=ocr,
pdf_outline=pdf_outline,
shell_parent=shell_parent,
)
except ProposerError as exc:
print(f"{CLI_ID}: {relative.as_posix()}: {exc}", file=sys.stderr)
@ -431,6 +440,7 @@ def build(
pdf_headings_reserve: bool = DEFAULT_PDF_HEADINGS_RESERVE,
ocr: bool = DEFAULT_OCR,
pdf_outline: bool = DEFAULT_PDF_OUTLINE,
shell_parent: bool = DEFAULT_SHELL_PARENT,
frontmatter: Mapping[str, str] | None = None,
) -> CorpusReport:
"""Folder in, bundle out. The whole command, minus argument parsing.
@ -511,6 +521,7 @@ def build(
pdf_headings_reserve=pdf_headings_reserve,
ocr=ocr,
pdf_outline=pdf_outline,
shell_parent=shell_parent,
)
print(
f"{CLI_ID}: proposed {written} plan(s); {nothing} document(s) with no boundary; "
@ -830,6 +841,26 @@ def parse_args(argv: list[str] | None) -> argparse.Namespace:
"pre-2026-09-11 spans byte for byte"
),
)
build_parser.add_argument(
"--shell-parent",
action="store_true",
default=DEFAULT_SHELL_PARENT,
help=(
"Give a concept whose body is its heading alone a `parent:` naming "
"the nearest ancestor that holds text, by the plan's level and "
"order, passing over an empty ancestor. Nothing is copied and no "
"boundary moves. Measured on one standard: 710 of 2 761 concepts "
"are heading-only, and the route names the ancestor its own "
"nesting names on 708 of them. OFF: `okf consume` does not read "
"the key"
),
)
build_parser.add_argument(
"--no-shell-parent",
action="store_false",
dest="shell_parent",
help="The explicit form of the default",
)
build_parser.add_argument(
"--contents-name",
action="store_true",
@ -982,6 +1013,7 @@ def main(argv: list[str] | None = None) -> int:
pdf_headings_reserve=args.pdf_headings == "font-reserve",
ocr=args.ocr,
pdf_outline=args.pdf_outline,
shell_parent=args.shell_parent,
frontmatter=frontmatter_from_flags(args.frontmatter or ()),
)
except (IngestError, OSError, ValueError) as exc:

52
src/llm_ingestion_okf/propose.py
View file

@ -1461,6 +1461,49 @@ def _segment_path(candidate: Candidate, taken: set[str], prefix: str = "") -> st
return path
def _link_shells(entries: list[dict[str, Any]], levels: Sequence[int], text: str) -> None:
"""Give each HEADING-ONLY entry the nearest ancestor holding text as `parent_id`.
A process code states its lettered points once, on the section that owns
them, and every section nested below inherits them. Built faithfully, the
nested section is a concept whose body is its heading line and nothing
else -- measured on one 2 761-concept standard, **710** of them -- and the
bundle's directory tree is two levels deep, so the path does not name the
parent either. `parent_id` is the plan's existing relation: validated
against the plan's own ids and written by the door as `parent:`, naming the
ancestor's `segment_id`, which every concept of the run carries.
**An ancestor is the nearest PRECEDING entry at a smaller level**, and an
ancestor whose own span is empty too is passed over, so the pointer lands
on text in one step. A shell with no such ancestor gets none. Measured
against the standard's own `<sec>` nesting, this route names the same
ancestor on **708 of 710** shells; the two it misses sit at depth 7, which
ATX clips to 6, and point one level too high -- still an ancestor. Reading
the section NUMBER instead agreed on 686 of 710: `12` begins with `1`, and
an unnumbered document has no number to read at all.
Nothing is copied. Inheriting the text itself was measured by a consumer
through its own build of the same standard: every shell filled, and hit@1
over six scored questions fell from 6 of 6 to 2 of 6 as the inherited text
grew the excerpts past the budget.
"""
bodied = [
any(line.strip() and not line.startswith("#") for line in text[start:end].split("\n"))
for start, end in (entry["span"] for entry in entries)
]
for index, entry in enumerate(entries):
if bodied[index]:
continue
bound = levels[index]
for earlier in range(index - 1, -1, -1):
if levels[earlier] >= bound:
continue
if bodied[earlier]:
entry["parent_id"] = entries[earlier]["segment_id"]
break
bound = levels[earlier]
def build_plan(
source: Path,
text: str,
@ -1482,6 +1525,7 @@ def build_plan(
contents_name: bool = False,
bold_title: bool = False,
outline_marks: Sequence[OutlineMark] | None = None,
shell_parent: bool = False,
) -> dict[str, Any]:
"""The artifact. Every entry PROPOSED, the plan itself never adjudicated."""
taken: set[str] = set()
@ -1530,7 +1574,9 @@ def build_plan(
starts.append(offsets[mark.line])
notes.append(mark.description)
cursor = 0
levels: list[int] = []
for candidate in subdivide(text, candidates, max_segment_chars):
levels.append(candidate.level)
entries.append(
{
"segment_id": f"p{len(entries) + 1}",
@ -1572,6 +1618,10 @@ def build_plan(
description = notes[cursor]
if description is not None:
entries[-1]["description"] = description
# AFTER every entry exists, and only then: an ancestor is read off the
# final plan's level and order, so no rule above can move it afterwards.
if shell_parent:
_link_shells(entries, levels, text)
return {
"version": "1",
"source_sha256": hashlib.sha256(source_bytes).hexdigest(),
@ -1617,6 +1667,7 @@ def run(
pdf_headings_reserve: bool = False,
ocr: bool = False,
pdf_outline: bool = False,
shell_parent: bool = False,
) -> int:
if max_segment_chars < 0:
raise ProposerError(
@ -1705,6 +1756,7 @@ def run(
contents_name=contents_name,
bold_title=bold_title,
outline_marks=marks,
shell_parent=shell_parent,
)
# Nothing to propose is an OUTCOME, and it is not an artifact. An empty
# plan cannot be replayed -- `process_inbox` refuses one, because a plan