feat(segmentation): slice canonical extracted text by declared spans

src/llm_ingestion_okf/segmentation.py

@@ -366,3 +366,36 @@ def assert_plan_applies(
             "re-run the proposer and re-adjudicate",
             code="segmentation_extractor_mismatch",
         )
+
+
+def slice_segments(text: str, plan: SegmentationPlan) -> tuple[tuple[SegmentEntry, str], ...]:
+    """Pair every entry with the substring its declared span names, in plan order.
+
+    `text` is the CANONICAL EXTRACTED text -- whatever
+    :func:`llm_ingestion_okf.extract.extract_text` returned -- never the source
+    bytes. The distinction is not pedantry: a `.csv` is re-rendered as a table
+    and a `.pdf` has no sliceable prose at all, so an offset computed against
+    bytes would land on different characters and produce a concept body no one
+    adjudicated, with nothing failing.
+
+    Spans may OVERLAP and need not cover the whole text. Neither is asserted:
+    a preamble, a page header or a signature block is legitimately part of no
+    unit of knowledge, and forcing full coverage would make the adjudicator
+    invent a home for it. What is refused is a span reaching past the end --
+    that is not a judgement about the document but proof the plan was
+    adjudicated against a different extraction.
+    """
+    limit = len(text)
+    sliced: list[tuple[SegmentEntry, str]] = []
+    for item in plan.entries:
+        start, end = item.span
+        if end > limit:
+            raise SegmentationError(
+                f"segmentation entry {item.segment_id!r} declares span [{start}, {end}] "
+                f"but the canonical extracted text is {limit} characters — the plan was "
+                "adjudicated against a different extraction; re-run the proposer and "
+                "re-adjudicate rather than truncating to fit",
+                code="segmentation_span_invalid",
+            )
+        sliced.append((item, text[start:end]))
+    return tuple(sliced)

tests/test_segmentation.py

@@ -27,6 +27,7 @@ from llm_ingestion_okf.segmentation import (
     assert_plan_applies,
     parse_segmentation_plan,
     plan_cache_key,
+    slice_segments,
 )
 
 
@@ -343,3 +344,91 @@ def test_the_message_names_every_differing_component_not_just_the_first() -> Non
     assert "extractor_id" in str(error)
     assert "extractor_version" in str(error)
     assert "source_sha256" not in str(error)
+
+
+# --- slicing: spans are windows on the EXTRACTED text ----------------------
+#
+# Not on the source bytes. You cannot slice a PDF's bytes and recover prose,
+# and a `.csv` is re-rendered as a table before it becomes a concept body --
+# so an offset computed against bytes would land on different characters and
+# quietly produce a concept nobody adjudicated.
+
+SECTIONS = (
+    "0 Forord: bakgrunn for konseptet.\n"  # 0..34
+    "1 Brannkonsept: krav til seksjonering.\n"  # 34..73
+    "2 Roemning: to uavhengige veier.\n"  # 73..106
+    "3 Baereevne: R60 for hovedbaeresystem.\n"  # 106..145
+    "4 Slokkeanlegg: sprinkler i hele bygget.\n"  # 145..186
+)
+
+SPANS = ((0, 34), (34, 73), (73, 106), (106, 145), (145, 186))
+
+
+def five_section_plan(**overrides: Any) -> SegmentationPlan:
+    spans = overrides.pop("spans", SPANS)
+    return parse_segmentation_plan(
+        plan(
+            entries=[
+                entry(segment_id=f"s{index}", path=f"krav/{index}-del.md", span=list(span))
+                for index, span in enumerate(spans)
+            ],
+            **overrides,
+        )
+    )
+
+
+def test_five_declared_spans_yield_those_five_substrings_in_plan_order() -> None:
+    sliced = slice_segments(SECTIONS, five_section_plan())
+    assert len(sliced) == 5
+    assert [item.segment_id for item, _ in sliced] == ["s0", "s1", "s2", "s3", "s4"]
+    assert [body for _, body in sliced] == [SECTIONS[start:end] for start, end in SPANS]
+    assert sliced[1][1].startswith("1 Brannkonsept")
+
+
+def test_moving_one_span_changes_exactly_that_one_body() -> None:
+    # The S1 correspondence anchor: a concept's body is the span its OWN entry
+    # declares. If a neighbour's body moved too, the split would be an artefact
+    # of iteration rather than of the adjudication.
+    def bodies_by_id(subject: SegmentationPlan) -> dict[str, str]:
+        return {item.segment_id: body for item, body in slice_segments(SECTIONS, subject)}
+
+    baseline = bodies_by_id(five_section_plan())
+    moved_spans = list(SPANS)
+    moved_spans[2] = (73, 90)
+    moved = bodies_by_id(five_section_plan(spans=tuple(moved_spans)))
+
+    assert set(moved) == set(baseline)
+    assert {key for key in moved if moved[key] != baseline[key]} == {"s2"}
+
+
+def test_a_span_reaching_past_the_end_is_refused() -> None:
+    subject = five_section_plan(spans=((0, 34), (34, len(SECTIONS) + 1)))
+    with pytest.raises(SegmentationError) as excinfo:
+        slice_segments(SECTIONS, subject)
+    assert excinfo.value.code == "segmentation_span_invalid"
+    assert str(len(SECTIONS)) in str(excinfo.value)
+
+
+def test_a_span_ending_exactly_at_the_end_is_permitted() -> None:
+    sliced = slice_segments(SECTIONS, five_section_plan(spans=((145, len(SECTIONS)),)))
+    assert sliced[0][1] == SECTIONS[145:]
+
+
+def test_an_uncovered_gap_in_the_middle_is_permitted() -> None:
+    # A preamble, a page header or a signature block may belong to no concept.
+    # Refusing a gap would force the adjudicator to invent a home for text that
+    # is not a unit of knowledge.
+    sliced = slice_segments(SECTIONS, five_section_plan(spans=((0, 34), (106, 145))))
+    assert [body for _, body in sliced] == [SECTIONS[0:34], SECTIONS[106:145]]
+
+
+def test_overlapping_spans_are_permitted() -> None:
+    sliced = slice_segments(SECTIONS, five_section_plan(spans=((0, 73), (34, 106))))
+    assert sliced[0][1] == SECTIONS[0:73]
+    assert sliced[1][1] == SECTIONS[34:106]
+
+
+def test_slicing_an_empty_text_with_any_span_is_refused() -> None:
+    with pytest.raises(SegmentationError) as excinfo:
+        slice_segments("", five_section_plan(spans=((0, 1),)))
+    assert excinfo.value.code == "segmentation_span_invalid"