open/llm-ingestion-okf
1
0
Fork 0
Code Issues Releases 1 Activity

feat(propose): the outline rule proposes boundaries, off by default

Browse source
This commit is contained in:
Kjell Tore Guttormsen 2026-09-07 01:31:22 +02:00
commit 5080240366
2 changed files with 215 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

166
tests/test_propose_segments.py
View file

@ -763,3 +763,169 @@ def test_a_title_that_is_only_digits_is_left_alone_rather_than_emptied() -> None
make junk look like a named section.
"""
assert okf_propose_segments._strip_page_number("477") == "477"
# --- Arm D: wiring the rule into find_candidates ---------------------------
DOTTED_AND_INTEGER_SIBLINGS = """1 Innledning
Bakgrunn for prosjektet og omfanget.
2 Krav
Krav til seksjonering av bygget.
1.2 Brannkonsept
To uavhengige roemningsveier fra hver branncelle.
3 Gjennomfoering
Framdrift, faser og overlevering.
"""
ARM_B_HEADING_THEN_OUTLINE = """# Teknisk grunnlag
1 Innledning
Bakgrunn for prosjektet.
2 Krav
Krav til seksjonering.
3 Gjennomfoering
Framdrift og faser.
"""
def arm_d_plan(tmp_path: Path, text: str, run_length: int, name: str = "arm-d.md") -> dict:
"""Arm D at the `build_plan` level -- no CLI, so step 5 stands alone."""
source = write(tmp_path, text, name)
return okf_propose_segments.build_plan(
source,
text,
source.read_bytes(),
okf_type="reference",
proposed_at="2026-09-07T00:00:00Z",
outline_run=run_length,
)
def test_an_outline_entry_names_the_outline_rule_and_nothing_else() -> None:
"""Exhaustive `==`, not `in`: a second rule name would be a second claim."""
candidates = okf_propose_segments.find_candidates(ALL_THREE_INTEGER_FORMS, outline_run=3)
assert [c.rule for c in candidates] == [okf_propose_segments.RULE_OUTLINE] * 3
assert [c.title for c in candidates] == ["Innledning", "Krav", "Gjennomfoering"]
def test_a_dotted_heading_and_its_integer_sibling_carry_different_rules() -> None:
"""The paired form, and the pairing is what makes it non-vacuous.
A lone "no outline rule fired on `1.2`" assertion is green TODAY for the
opposite reason: measured, `1.2 Brannkonsept` already yields a candidate via
`_NUMBERED` with `rule:heading`. Asserting both siblings in ONE document is
what makes the integer-only claim testable.
"""
candidates = okf_propose_segments.find_candidates(DOTTED_AND_INTEGER_SIBLINGS, outline_run=3)
by_title = {c.title: c.rule for c in candidates}
assert by_title["Brannkonsept"] == okf_propose_segments.RULE_HEADING
assert by_title["Innledning"] == okf_propose_segments.RULE_OUTLINE
assert by_title["Krav"] == okf_propose_segments.RULE_OUTLINE
def test_an_outline_candidate_can_orphan_the_arm_b_candidate_it_follows() -> None:
"""Characterization, with the mechanism named rather than implied.
The orphan check at the emit loop measures a heading's body up to the NEXT
candidate of ANY rule. So an outline candidate landing immediately after an
Arm B heading empties that heading's body and deletes it. This is the
measured 34 %-deletion behaviour, pinned here so a later change to the span
bound has to confront it.
"""
without = okf_propose_segments.find_candidates(ARM_B_HEADING_THEN_OUTLINE)
assert [c.title for c in without] == ["Teknisk grunnlag"]
with_arm_d = okf_propose_segments.find_candidates(ARM_B_HEADING_THEN_OUTLINE, outline_run=3)
# The ATX heading is gone: `1 Innledning` follows it immediately, so its
# body is empty and the orphan check drops it.
assert "Teknisk grunnlag" not in [c.title for c in with_arm_d]
assert [c.rule for c in with_arm_d] == [okf_propose_segments.RULE_OUTLINE] * 3
def test_arm_d_spans_tile_and_each_anchor_quotes_its_own_span(tmp_path: Path) -> None:
plan = arm_d_plan(tmp_path, DOTTED_AND_INTEGER_SIBLINGS, 3)
spans = [tuple(entry["span"]) for entry in plan["entries"]]
assert spans == sorted(spans)
for (_, first_end), (second_start, _) in zip(spans, spans[1:]):
assert first_end == second_start, f"gap or overlap: {first_end} != {second_start}"
for entry in plan["entries"]:
start, end = entry["span"]
assert entry["anchor"]["quote"] == DOTTED_AND_INTEGER_SIBLINGS[start:end]
def test_outline_entries_get_unique_paths(tmp_path: Path) -> None:
plan = arm_d_plan(tmp_path, DOTTED_AND_INTEGER_SIBLINGS, 3)
paths = [entry["path"] for entry in plan["entries"]]
assert len(paths) == len(set(paths))
assert any(
entry["derived"]
== [
okf_propose_segments.PROPOSED_MARKER,
okf_propose_segments.RULE_OUTLINE,
]
for entry in plan["entries"]
)
def test_no_line_can_match_the_outline_grammar_and_an_arm_b_rule() -> None:
"""Why this repo ships NO coincident-boundary dedupe, pinned as a test.
The plan called for a dedupe branch guarding two candidates on one line.
Measured over 2400 synthetic lines spanning every shape the four grammars
accept: **zero** lines match `_OUTLINE` and any of `_ATX`, `_NUMBERED` or
`_TABLE_ROW`. `_ATX` needs `#`, `_TABLE_ROW` needs `|`, `_NUMBERED` needs a
dot inside the number, and `_OUTLINE` needs whitespace right after an
integer -- so the collision is impossible BY CONSTRUCTION, not merely
absent from this corpus, and a dedupe branch would be dead code with a test
that could never go red for a real reason.
This test is what keeps that argument honest: widen `_OUTLINE` until it can
overlap and this goes red, at which point the dedupe branch is owed.
"""
import itertools
collisions = []
tried = 0
for pre, num, mark, title, sep in itertools.product(
["", " ", " ", " ", " "],
["1", "9", "12", "99", "1.", "1)", "1.2", "12.3.4"],
["", "#", "##", "######", "|"],
["Innledning", "A |", "| A |", "Krav 6"],
[" ", " ", "\t"],
):
line = f"{pre}{mark}{num}{sep}{title}"
tried += 1
if okf_propose_segments._OUTLINE.match(line) and (
okf_propose_segments._ATX.match(line)
or okf_propose_segments._NUMBERED.match(line)
or okf_propose_segments._TABLE_ROW.match(line)
):
collisions.append(line)
assert tried == 2400
assert collisions == []
# Known-positive controls: the probe CAN match, so the empty result above
# is a measurement and not a broken query.
assert okf_propose_segments._OUTLINE.match("1 Innledning")
assert okf_propose_segments._ATX.match("# 1.2 Krav")
assert okf_propose_segments._NUMBERED.match("1.2 Krav")
assert okf_propose_segments._TABLE_ROW.match("| a | b |")
def test_the_outline_rule_is_off_at_the_function_level_by_default() -> None:
"""`outline_run` defaults to 0 and 0 means Arm B, exactly."""
default = okf_propose_segments.find_candidates(ALL_THREE_INTEGER_FORMS)
explicit_zero = okf_propose_segments.find_candidates(ALL_THREE_INTEGER_FORMS, outline_run=0)
assert default == explicit_zero == []
# The known-positive control: the same text DOES produce candidates when asked.
assert len(okf_propose_segments.find_candidates(ALL_THREE_INTEGER_FORMS, outline_run=3)) == 3

51
tools/okf_propose_segments.py
View file

@ -259,7 +259,7 @@ def outline_runs(
return [run for run in runs if len(run) >= minimum]
def find_candidates(text: str) -> list[Candidate]:
def find_candidates(text: str, *, outline_run: int = 0) -> list[Candidate]:
"""Every boundary the mechanical rules propose, in document order.
Two gates from Topic 2 are applied here and both REMOVE candidates:
@ -269,6 +269,11 @@ def find_candidates(text: str) -> list[Candidate]:
- the **orphan check**: a heading with no body under it proposes nothing,
because an empty concept is the silent skip this library refuses
everywhere else.
`outline_run` is Arm D's gate and it is OFF at 0: the function then behaves
exactly as it did before the rule existed. At `N >= 1` the document's own
numbered outline contributes boundaries where the integers sustain an
ascending run of at least `N`.
"""
lines = text.splitlines(keepends=True)
offsets: list[int] = []
@ -278,6 +283,25 @@ def find_candidates(text: str) -> list[Candidate]:
position += len(line)
end_of_text = position
# Computed BEFORE the loop, and that is a correctness requirement rather
# than a style choice: run selection is a whole-text decision (the LAST
# maximal run wins, because a contents listing precedes the body it lists),
# and a forward scan cannot know which run is last. Deciding it up front is
# also what keeps `marked` sorted by construction -- appending outline
# candidates in a second pass would leave `end < start` on some spans, and
# `text[start:end]` is then `""`, so the orphan check DELETES them
# silently. Silent loss, not a raise: nothing would announce it.
admitted: dict[int, str] = {}
if outline_run > 0:
runs = outline_runs(outline_lines(text), outline_run)
if runs:
# LAST run, not longest and not first. Measured against both:
# first-run opens segments inside the table of contents on 14/39
# documents; longest-run differs on 5/39 with no measured reason to
# prefer it. "Later occurrence wins" states the document's own
# ordering rather than a property of this corpus.
admitted = {index: title for index, _, title in runs[-1]}
marked: list[tuple[int, Candidate]] = []
in_table = False
for index, line in enumerate(lines):
@ -300,6 +324,25 @@ def find_candidates(text: str) -> list[Candidate]:
continue
in_table = False
outline_title = admitted.get(index)
if outline_title is not None:
outline_match = _OUTLINE.match(line)
assert outline_match is not None, "an admitted index still matches the grammar"
marked.append(
(
index,
Candidate(
title=outline_title,
level=1,
number=outline_match.group("number"),
rule=RULE_OUTLINE,
start=offsets[index],
end=end_of_text,
),
)
)
continue
atx = _ATX.match(line)
numbered = _NUMBERED.match(line)
if atx is None and numbered is None:
@ -482,12 +525,14 @@ def build_plan(
proposed_at: str,
path_prefix: str = "",
max_segment_chars: int = 0,
outline_run: int = 0,
) -> dict[str, Any]:
"""The artifact. Every entry PROPOSED, the plan itself never adjudicated."""
taken: set[str] = set()
extractor_id = source.suffix.lower().lstrip(".") or "none"
entries: list[dict[str, Any]] = []
for candidate in subdivide(text, find_candidates(text), max_segment_chars):
candidates = find_candidates(text, outline_run=outline_run)
for candidate in subdivide(text, candidates, max_segment_chars):
entries.append(
{
"segment_id": f"p{len(entries) + 1}",
@ -552,6 +597,7 @@ def run(
proposed_at: str,
path_prefix: str = "",
max_segment_chars: int = 0,
outline_run: int = 0,
) -> int:
if max_segment_chars < 0:
raise ProposerError(
@ -587,6 +633,7 @@ def run(
proposed_at=proposed_at,
path_prefix=scope,
max_segment_chars=max_segment_chars,
outline_run=outline_run,
)
# Nothing to propose is an OUTCOME, and it is not an artifact. An empty
# plan cannot be replayed -- `process_inbox` refuses one, because a plan