feat(segmentation): quote anchors so a re-extraction costs a re-anchor

src/llm_ingestion_okf/segmentation.py

@@ -37,7 +37,7 @@ Pure: no filesystem, no bundle, no door, no model call, no network.
 from __future__ import annotations
 
 from collections.abc import Mapping, Sequence
-from dataclasses import dataclass, field
+from dataclasses import dataclass, field, replace
 from importlib import metadata
 from pathlib import PurePosixPath
 from typing import Any
@@ -58,9 +58,10 @@ PLAN_FIELDS = (
     "entries",
 )
 
-#: The keys every entry must carry. `parent_id` and `derived` are optional --
+#: The keys every entry must carry. `parent_id`, `derived` and `anchor` are
-#: a flat plan has no parents, and an entry adjudicated from scratch derived
+#: optional -- a flat plan has no parents, an entry adjudicated from scratch
-#: nothing.
+#: derived nothing, and a plan authored before quote anchors existed carries
+#: offsets alone.
 ENTRY_FIELDS = ("segment_id", "path", "title", "okf_type", "span", "ingested_at")
 
 #: Path components refused outright, before the id grammar is consulted. An
@@ -98,6 +99,27 @@ _CONVERTED_EXTRACTOR_IDS = frozenset({"docx", "xlsx", "pptx", "odt", "rtf"})
 _PDF_DISTRIBUTION = "pdfminer.six"
 
 
+@dataclass(frozen=True)
+class SegmentAnchor:
+    """The text an entry names, plus enough context to find it again.
+
+    Offsets alone are brittle in the one direction that matters. Measured:
+    changing a single extraction flag moved a document from 15 507 to 8 290
+    characters, and every inspected span then pointed at the wrong text --
+    real prose, cut at a plausible offset, indistinguishable downstream from
+    an adjudicated body. The quote makes that recoverable rather than silent.
+
+    `prefix` and `suffix` are not decoration either: a line that occurs twice
+    is not addressable by its own text, and picking the first occurrence would
+    re-anchor a document's second section onto its first. They may be empty --
+    an entry at the very start or end of a document has no room for them.
+    """
+
+    quote: str
+    prefix: str = ""
+    suffix: str = ""
+
+
 @dataclass(frozen=True)
 class SegmentEntry:
     """One concept a document expands into.
@@ -116,6 +138,7 @@ class SegmentEntry:
     ingested_at: str
     parent_id: str | None = None
     derived: frozenset[str] = field(default_factory=frozenset)
+    anchor: SegmentAnchor | None = None
 
 
 @dataclass(frozen=True)
@@ -240,6 +263,28 @@ def _parse_derived(value: Any, *, where: str) -> frozenset[str]:
     return frozenset(value)
 
 
+def _parse_anchor(value: Any, *, where: str) -> SegmentAnchor | None:
+    if value is None:
+        return None
+    if not isinstance(value, Mapping):
+        raise SegmentationError(
+            f"{where} field 'anchor' must be a mapping with a 'quote' and optional "
+            f"'prefix'/'suffix', got {value!r}",
+            code="segmentation_plan_invalid",
+        )
+    quote = _require_str(value, "quote", where=f"{where} anchor")
+    context: dict[str, str] = {}
+    for key in ("prefix", "suffix"):
+        found = value.get(key, "")
+        if not isinstance(found, str):
+            raise SegmentationError(
+                f"{where} anchor field {key!r} must be a string, got {found!r}",
+                code="segmentation_plan_invalid",
+            )
+        context[key] = found
+    return SegmentAnchor(quote=quote, prefix=context["prefix"], suffix=context["suffix"])
+
+
 def _parse_entry(payload: Any, *, position: int) -> SegmentEntry:
     where = f"segmentation entry {position}"
     if not isinstance(payload, Mapping):
@@ -272,6 +317,7 @@ def _parse_entry(payload: Any, *, position: int) -> SegmentEntry:
         ingested_at=_require_str(payload, "ingested_at", where=where),
         parent_id=parent_id,
         derived=_parse_derived(payload.get("derived", ()), where=where),
+        anchor=_parse_anchor(payload.get("anchor"), where=where),
     )
 
 
@@ -467,10 +513,39 @@ def slice_segments(text: str, plan: SegmentationPlan) -> tuple[tuple[SegmentEntr
     that is not a judgement about the document but proof the plan was
     adjudicated against a different extraction.
     """
-    limit = len(text)
     sliced: list[tuple[SegmentEntry, str]] = []
     for item in plan.entries:
-        start, end = item.span
+        resolved = _resolve_entry(text, item)
+        start, end = resolved.span
+        sliced.append((resolved, text[start:end]))
+    return tuple(sliced)
+
+
+def _all_occurrences(text: str, needle: str) -> list[int]:
+    found: list[int] = []
+    position = text.find(needle)
+    while position != -1:
+        found.append(position)
+        position = text.find(needle, position + 1)
+    return found
+
+
+def _resolve_entry(text: str, item: SegmentEntry) -> SegmentEntry:
+    """The entry with the span that actually names its text in THIS extraction.
+
+    Offsets are a HINT the anchor may correct, and the correction is written
+    back onto the entry rather than applied only to the slice: the frontmatter
+    records `span`, so a body cut at one offset and stamped with another would
+    make the bundle disagree with itself.
+
+    An anchorless entry keeps the old behaviour exactly -- the anchor is a
+    capability a plan may carry, not a new requirement on authored plans, and
+    every golden predates it.
+    """
+    start, end = item.span
+    limit = len(text)
+    anchor = item.anchor
+    if anchor is None:
         if end > limit:
             raise SegmentationError(
                 f"segmentation entry {item.segment_id!r} declares span [{start}, {end}] "
@@ -479,5 +554,28 @@ def slice_segments(text: str, plan: SegmentationPlan) -> tuple[tuple[SegmentEntr
                 "re-adjudicate rather than truncating to fit",
                 code="segmentation_span_invalid",
             )
-        sliced.append((item, text[start:end]))
+        return item
-    return tuple(sliced)
+
+    if text[start:end] == anchor.quote:
+        return item
+
+    # Context INCLUDED in the needle, never searched for separately. A quote
+    # that occurs twice is not addressable by itself, and taking the first
+    # occurrence would re-anchor a document's second section onto its first.
+    needle = anchor.prefix + anchor.quote + anchor.suffix
+    positions = _all_occurrences(text, needle)
+    if not positions:
+        raise SegmentationError(
+            f"segmentation entry {item.segment_id!r} declares span [{start}, {end}], which "
+            "does not hold in this extraction, and its quoted anchor does not occur in "
+            "the canonical extracted text either — refusing to cut at a plausible offset "
+            "nobody adjudicated; re-run the proposer and re-adjudicate",
+            code="segmentation_span_invalid",
+        )
+    # Offsets are still worth something when the anchor is ambiguous: the
+    # nearest occurrence to the declared position is the one the adjudicator
+    # was looking at.
+    hint = start - len(anchor.prefix)
+    best = min(positions, key=lambda position: (abs(position - hint), position))
+    moved = best + len(anchor.prefix)
+    return replace(item, span=(moved, moved + len(anchor.quote)))

tests/test_propose_segments.py

@@ -142,6 +142,48 @@ def test_the_spans_it_proposes_are_slices_of_the_text_it_read(tmp_path: Path) ->
         assert text[start:end].strip()
 
 
+def test_every_entry_carries_an_anchor_quoting_its_own_span(tmp_path: Path) -> None:
+    """Written at proposal time, when text and offsets are known to agree.
+
+    Reconstructed later it would quote whatever the extraction had already
+    become, which is precisely the drift the anchor exists to survive.
+    """
+    from llm_ingestion_okf.extract import extract_text
+
+    source = write(tmp_path)
+    payload = propose(tmp_path, source=source)
+    text = extract_text(source.name, source.read_bytes())
+    assert payload["entries"]
+    for entry in payload["entries"]:
+        start, end = entry["span"]
+        anchor = entry["anchor"]
+        assert anchor["quote"] == text[start:end]
+        assert text[start - len(anchor["prefix"]) : start] == anchor["prefix"]
+        assert text[end : end + len(anchor["suffix"])] == anchor["suffix"]
+
+
+def test_a_proposed_plan_survives_a_shift_of_the_text_it_was_made_against(
+    tmp_path: Path,
+) -> None:
+    """The round trip the anchor exists for, measured end to end.
+
+    Prepend a paragraph -- exactly the benign edit that leaves every offset
+    short -- and every body must still be the text the proposer quoted. Before
+    anchors this cut each segment early and landed on real prose, with nothing
+    anywhere able to tell.
+    """
+    from llm_ingestion_okf.extract import extract_text
+    from llm_ingestion_okf.segmentation import slice_segments
+
+    source = write(tmp_path)
+    plan = parse_segmentation_plan(propose(tmp_path, source=source))
+    text = extract_text(source.name, source.read_bytes())
+    quoted = [text[start:end] for start, end in (item.span for item in plan.entries)]
+
+    shifted = "Et nytt avsnitt foran alt annet.\n\n" + text
+    assert [body for _, body in slice_segments(shifted, plan)] == quoted
+
+
 def test_the_paths_it_proposes_are_unique_and_hierarchical(tmp_path: Path) -> None:
     plan = parse_segmentation_plan(propose(tmp_path))
     paths = [entry.path for entry in plan.entries]

tests/test_segmentation.py

@@ -451,3 +451,83 @@ def test_slicing_an_empty_text_with_any_span_is_refused() -> None:
     with pytest.raises(SegmentationError) as excinfo:
         slice_segments("", five_section_plan(spans=((0, 1),)))
     assert excinfo.value.code == "segmentation_span_invalid"
+
+
+# --- S5c: the quote anchor, which survives a shift the key must not refuse --
+#
+# The key (S5b, above) refuses a FOREIGN extraction. This is the other half:
+# a benign shift INSIDE the same extraction, where the text an entry names is
+# still present but no longer at the offset written down. Measured: changing
+# one extraction flag moved a document from 15 507 to 8 290 characters and
+# every inspected span pointed at the wrong text. Offsets are therefore a HINT
+# the quote may correct -- never the last word, and never silently wrong.
+
+
+ANCHORED = "innledning\n" + SECTIONS
+
+
+def anchored_entry(index: int, **overrides: Any) -> dict[str, Any]:
+    start, end = SPANS[index]
+    quote = SECTIONS[start:end]
+    payload = entry(
+        segment_id=f"s{index}",
+        path=f"krav/{index}-del.md",
+        span=[start, end],
+        anchor={
+            "quote": quote,
+            "prefix": SECTIONS[max(0, start - 12) : start],
+            "suffix": SECTIONS[end : end + 12],
+        },
+    )
+    payload.update(overrides)
+    return payload
+
+
+def anchored_plan(*indexes: int, **overrides: Any) -> SegmentationPlan:
+    return parse_segmentation_plan(
+        plan(entries=[anchored_entry(index) for index in indexes], **overrides)
+    )
+
+
+def test_an_anchor_whose_offsets_still_hold_changes_nothing() -> None:
+    """The fast path. A correct plan must not be re-anchored into motion."""
+    sliced = slice_segments(SECTIONS, anchored_plan(1, 2))
+    assert [body for _, body in sliced] == [SECTIONS[34:73], SECTIONS[73:106]]
+    assert [item.span for item, _ in sliced] == [(34, 73), (73, 106)]
+
+
+def test_stale_offsets_re_anchor_to_the_text_the_quote_names() -> None:
+    """The whole point: the same document with eleven characters prepended.
+
+    Every declared offset is now short by exactly that much. Without the
+    anchor each body would be cut eleven characters early and land on real
+    text -- the failure no downstream assertion can see.
+    """
+    sliced = slice_segments(ANCHORED, anchored_plan(1, 2))
+    assert [body for _, body in sliced] == [SECTIONS[34:73], SECTIONS[73:106]]
+    assert [item.span for item, _ in sliced] == [(45, 84), (84, 117)]
+
+
+def test_a_quote_that_no_longer_occurs_is_refused_rather_than_cut_elsewhere() -> None:
+    """Refusing beats cutting. A body sliced at a plausible-but-wrong offset
+    reads as ordinary prose, so nothing downstream can distinguish it from an
+    adjudicated one."""
+    gone = SECTIONS.replace("1 Brannkonsept: krav til seksjonering.\n", "")
+    with pytest.raises(SegmentationError) as excinfo:
+        slice_segments(gone, anchored_plan(1))
+    assert excinfo.value.code == "segmentation_span_invalid"
+    assert "s1" in str(excinfo.value)
+
+
+def test_a_repeated_quote_re_anchors_to_the_occurrence_its_context_names() -> None:
+    """Prefix and suffix are what make a repeated line addressable at all."""
+    doubled = "xx" + SECTIONS + SECTIONS
+    sliced = slice_segments(doubled, anchored_plan(1))
+    assert sliced[0][0].span == (36, 75)
+    assert sliced[0][1] == SECTIONS[34:73]
+
+
+def test_an_entry_without_an_anchor_keeps_todays_offset_behaviour() -> None:
+    """The anchor is a capability, not a new requirement on authored plans."""
+    sliced = slice_segments(SECTIONS, five_section_plan())
+    assert [item.span for item, _ in sliced] == list(SPANS)

tools/okf_propose_segments.py

@@ -76,6 +76,12 @@ RULE_TABLE_BLOCK = "rule:table-block"
 RULE_POPPLER_SIZE_AND_BOLD = "rule:poppler-size-and-bold"
 RULE_NAMES = (RULE_HEADING, RULE_TABLE_BLOCK, RULE_POPPLER_SIZE_AND_BOLD)
 
+#: How many characters of context each side of a quote anchor carries. Enough
+#: to separate two occurrences of a repeated heading, short enough that an
+#: edit NEAR a segment does not invalidate the anchor FOR it -- the anchor
+#: exists to survive shifts, so making it fragile would defeat it.
+ANCHOR_CONTEXT = 48
+
 #: Norwegian and English function words. A heading made only of these names no
 #: unit of knowledge -- it is a connective that happened to sit on its own line.
 #: Topic 2's stop-word gate, and the only place this tool judges wording.
@@ -278,6 +284,16 @@ def build_plan(
                 "okf_type": okf_type,
                 "span": [candidate.start, candidate.end],
                 "ingested_at": proposed_at,
+                # The offsets are a hint the anchor may correct. Written at
+                # proposal time because that is the only moment the text the
+                # adjudicator will judge and the offsets naming it are known
+                # to agree -- reconstructing it later would anchor to whatever
+                # the extraction had already become.
+                "anchor": {
+                    "quote": text[candidate.start : candidate.end],
+                    "prefix": text[max(0, candidate.start - ANCHOR_CONTEXT) : candidate.start],
+                    "suffix": text[candidate.end : candidate.end + ANCHOR_CONTEXT],
+                },
                 # PROPOSED first, then the rule that proposed it. `derived` is
                 # this library's existing "which of these did we infer" marker,
                 # so a consumer that already distrusts derived fields