feat(segmentation): quote anchors so a re-extraction costs a re-anchor
This commit is contained in:
parent
9e9bb8645d
commit
c54e8383df
4 changed files with 244 additions and 8 deletions
|
|
@ -142,6 +142,48 @@ def test_the_spans_it_proposes_are_slices_of_the_text_it_read(tmp_path: Path) ->
|
|||
assert text[start:end].strip()
|
||||
|
||||
|
||||
def test_every_entry_carries_an_anchor_quoting_its_own_span(tmp_path: Path) -> None:
|
||||
"""Written at proposal time, when text and offsets are known to agree.
|
||||
|
||||
Reconstructed later it would quote whatever the extraction had already
|
||||
become, which is precisely the drift the anchor exists to survive.
|
||||
"""
|
||||
from llm_ingestion_okf.extract import extract_text
|
||||
|
||||
source = write(tmp_path)
|
||||
payload = propose(tmp_path, source=source)
|
||||
text = extract_text(source.name, source.read_bytes())
|
||||
assert payload["entries"]
|
||||
for entry in payload["entries"]:
|
||||
start, end = entry["span"]
|
||||
anchor = entry["anchor"]
|
||||
assert anchor["quote"] == text[start:end]
|
||||
assert text[start - len(anchor["prefix"]) : start] == anchor["prefix"]
|
||||
assert text[end : end + len(anchor["suffix"])] == anchor["suffix"]
|
||||
|
||||
|
||||
def test_a_proposed_plan_survives_a_shift_of_the_text_it_was_made_against(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
"""The round trip the anchor exists for, measured end to end.
|
||||
|
||||
Prepend a paragraph -- exactly the benign edit that leaves every offset
|
||||
short -- and every body must still be the text the proposer quoted. Before
|
||||
anchors this cut each segment early and landed on real prose, with nothing
|
||||
anywhere able to tell.
|
||||
"""
|
||||
from llm_ingestion_okf.extract import extract_text
|
||||
from llm_ingestion_okf.segmentation import slice_segments
|
||||
|
||||
source = write(tmp_path)
|
||||
plan = parse_segmentation_plan(propose(tmp_path, source=source))
|
||||
text = extract_text(source.name, source.read_bytes())
|
||||
quoted = [text[start:end] for start, end in (item.span for item in plan.entries)]
|
||||
|
||||
shifted = "Et nytt avsnitt foran alt annet.\n\n" + text
|
||||
assert [body for _, body in slice_segments(shifted, plan)] == quoted
|
||||
|
||||
|
||||
def test_the_paths_it_proposes_are_unique_and_hierarchical(tmp_path: Path) -> None:
|
||||
plan = parse_segmentation_plan(propose(tmp_path))
|
||||
paths = [entry.path for entry in plan.entries]
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue