feat(segmentation): quote anchors so a re-extraction costs a re-anchor

src/llm_ingestion_okf/segmentation.py

@@ -37,7 +37,7 @@ Pure: no filesystem, no bundle, no door, no model call, no network.
 from __future__ import annotations
 
 from collections.abc import Mapping, Sequence
-from dataclasses import dataclass, field
+from dataclasses import dataclass, field, replace
 from importlib import metadata
 from pathlib import PurePosixPath
 from typing import Any
@@ -58,9 +58,10 @@ PLAN_FIELDS = (
     "entries",
 )
 
-#: The keys every entry must carry. `parent_id` and `derived` are optional --
-#: a flat plan has no parents, and an entry adjudicated from scratch derived
-#: nothing.
+#: The keys every entry must carry. `parent_id`, `derived` and `anchor` are
+#: optional -- a flat plan has no parents, an entry adjudicated from scratch
+#: derived nothing, and a plan authored before quote anchors existed carries
+#: offsets alone.
 ENTRY_FIELDS = ("segment_id", "path", "title", "okf_type", "span", "ingested_at")
 
 #: Path components refused outright, before the id grammar is consulted. An
@@ -98,6 +99,27 @@ _CONVERTED_EXTRACTOR_IDS = frozenset({"docx", "xlsx", "pptx", "odt", "rtf"})
 _PDF_DISTRIBUTION = "pdfminer.six"
 
 
+@dataclass(frozen=True)
+class SegmentAnchor:
+    """The text an entry names, plus enough context to find it again.
+
+    Offsets alone are brittle in the one direction that matters. Measured:
+    changing a single extraction flag moved a document from 15 507 to 8 290
+    characters, and every inspected span then pointed at the wrong text --
+    real prose, cut at a plausible offset, indistinguishable downstream from
+    an adjudicated body. The quote makes that recoverable rather than silent.
+
+    `prefix` and `suffix` are not decoration either: a line that occurs twice
+    is not addressable by its own text, and picking the first occurrence would
+    re-anchor a document's second section onto its first. They may be empty --
+    an entry at the very start or end of a document has no room for them.
+    """
+
+    quote: str
+    prefix: str = ""
+    suffix: str = ""
+
+
 @dataclass(frozen=True)
 class SegmentEntry:
     """One concept a document expands into.
@@ -116,6 +138,7 @@ class SegmentEntry:
     ingested_at: str
     parent_id: str | None = None
     derived: frozenset[str] = field(default_factory=frozenset)
+    anchor: SegmentAnchor | None = None
 
 
 @dataclass(frozen=True)
@@ -240,6 +263,28 @@ def _parse_derived(value: Any, *, where: str) -> frozenset[str]:
     return frozenset(value)
 
 
+def _parse_anchor(value: Any, *, where: str) -> SegmentAnchor | None:
+    if value is None:
+        return None
+    if not isinstance(value, Mapping):
+        raise SegmentationError(
+            f"{where} field 'anchor' must be a mapping with a 'quote' and optional "
+            f"'prefix'/'suffix', got {value!r}",
+            code="segmentation_plan_invalid",
+        )
+    quote = _require_str(value, "quote", where=f"{where} anchor")
+    context: dict[str, str] = {}
+    for key in ("prefix", "suffix"):
+        found = value.get(key, "")
+        if not isinstance(found, str):
+            raise SegmentationError(
+                f"{where} anchor field {key!r} must be a string, got {found!r}",
+                code="segmentation_plan_invalid",
+            )
+        context[key] = found
+    return SegmentAnchor(quote=quote, prefix=context["prefix"], suffix=context["suffix"])
+
+
 def _parse_entry(payload: Any, *, position: int) -> SegmentEntry:
     where = f"segmentation entry {position}"
     if not isinstance(payload, Mapping):
@@ -272,6 +317,7 @@ def _parse_entry(payload: Any, *, position: int) -> SegmentEntry:
         ingested_at=_require_str(payload, "ingested_at", where=where),
         parent_id=parent_id,
         derived=_parse_derived(payload.get("derived", ()), where=where),
+        anchor=_parse_anchor(payload.get("anchor"), where=where),
     )
 
 
@@ -467,10 +513,39 @@ def slice_segments(text: str, plan: SegmentationPlan) -> tuple[tuple[SegmentEntr
     that is not a judgement about the document but proof the plan was
     adjudicated against a different extraction.
     """
-    limit = len(text)
     sliced: list[tuple[SegmentEntry, str]] = []
     for item in plan.entries:
-        start, end = item.span
+        resolved = _resolve_entry(text, item)
+        start, end = resolved.span
+        sliced.append((resolved, text[start:end]))
+    return tuple(sliced)
+
+
+def _all_occurrences(text: str, needle: str) -> list[int]:
+    found: list[int] = []
+    position = text.find(needle)
+    while position != -1:
+        found.append(position)
+        position = text.find(needle, position + 1)
+    return found
+
+
+def _resolve_entry(text: str, item: SegmentEntry) -> SegmentEntry:
+    """The entry with the span that actually names its text in THIS extraction.
+
+    Offsets are a HINT the anchor may correct, and the correction is written
+    back onto the entry rather than applied only to the slice: the frontmatter
+    records `span`, so a body cut at one offset and stamped with another would
+    make the bundle disagree with itself.
+
+    An anchorless entry keeps the old behaviour exactly -- the anchor is a
+    capability a plan may carry, not a new requirement on authored plans, and
+    every golden predates it.
+    """
+    start, end = item.span
+    limit = len(text)
+    anchor = item.anchor
+    if anchor is None:
         if end > limit:
             raise SegmentationError(
                 f"segmentation entry {item.segment_id!r} declares span [{start}, {end}] "
@@ -479,5 +554,28 @@ def slice_segments(text: str, plan: SegmentationPlan) -> tuple[tuple[SegmentEntr
                 "re-adjudicate rather than truncating to fit",
                 code="segmentation_span_invalid",
             )
-        sliced.append((item, text[start:end]))
-    return tuple(sliced)
+        return item
+
+    if text[start:end] == anchor.quote:
+        return item
+
+    # Context INCLUDED in the needle, never searched for separately. A quote
+    # that occurs twice is not addressable by itself, and taking the first
+    # occurrence would re-anchor a document's second section onto its first.
+    needle = anchor.prefix + anchor.quote + anchor.suffix
+    positions = _all_occurrences(text, needle)
+    if not positions:
+        raise SegmentationError(
+            f"segmentation entry {item.segment_id!r} declares span [{start}, {end}], which "
+            "does not hold in this extraction, and its quoted anchor does not occur in "
+            "the canonical extracted text either — refusing to cut at a plausible offset "
+            "nobody adjudicated; re-run the proposer and re-adjudicate",
+            code="segmentation_span_invalid",
+        )
+    # Offsets are still worth something when the anchor is ambiguous: the
+    # nearest occurrence to the declared position is the one the adjudicator
+    # was looking at.
+    hint = start - len(anchor.prefix)
+    best = min(positions, key=lambda position: (abs(position - hint), position))
+    moved = best + len(anchor.prefix)
+    return replace(item, span=(moved, moved + len(anchor.quote)))