feat(segmentation): quote anchors so a re-extraction costs a re-anchor

This commit is contained in:
Kjell Tore Guttormsen 2026-09-02 14:42:16 +02:00
commit c54e8383df
4 changed files with 244 additions and 8 deletions

View file

@ -142,6 +142,48 @@ def test_the_spans_it_proposes_are_slices_of_the_text_it_read(tmp_path: Path) ->
assert text[start:end].strip()
def test_every_entry_carries_an_anchor_quoting_its_own_span(tmp_path: Path) -> None:
"""Written at proposal time, when text and offsets are known to agree.
Reconstructed later it would quote whatever the extraction had already
become, which is precisely the drift the anchor exists to survive.
"""
from llm_ingestion_okf.extract import extract_text
source = write(tmp_path)
payload = propose(tmp_path, source=source)
text = extract_text(source.name, source.read_bytes())
assert payload["entries"]
for entry in payload["entries"]:
start, end = entry["span"]
anchor = entry["anchor"]
assert anchor["quote"] == text[start:end]
assert text[start - len(anchor["prefix"]) : start] == anchor["prefix"]
assert text[end : end + len(anchor["suffix"])] == anchor["suffix"]
def test_a_proposed_plan_survives_a_shift_of_the_text_it_was_made_against(
tmp_path: Path,
) -> None:
"""The round trip the anchor exists for, measured end to end.
Prepend a paragraph -- exactly the benign edit that leaves every offset
short -- and every body must still be the text the proposer quoted. Before
anchors this cut each segment early and landed on real prose, with nothing
anywhere able to tell.
"""
from llm_ingestion_okf.extract import extract_text
from llm_ingestion_okf.segmentation import slice_segments
source = write(tmp_path)
plan = parse_segmentation_plan(propose(tmp_path, source=source))
text = extract_text(source.name, source.read_bytes())
quoted = [text[start:end] for start, end in (item.span for item in plan.entries)]
shifted = "Et nytt avsnitt foran alt annet.\n\n" + text
assert [body for _, body in slice_segments(shifted, plan)] == quoted
def test_the_paths_it_proposes_are_unique_and_hierarchical(tmp_path: Path) -> None:
plan = parse_segmentation_plan(propose(tmp_path))
paths = [entry.path for entry in plan.entries]