feat(segmentation): quote anchors so a re-extraction costs a re-anchor
This commit is contained in:
parent
9e9bb8645d
commit
c54e8383df
4 changed files with 244 additions and 8 deletions
|
|
@ -451,3 +451,83 @@ def test_slicing_an_empty_text_with_any_span_is_refused() -> None:
|
|||
with pytest.raises(SegmentationError) as excinfo:
|
||||
slice_segments("", five_section_plan(spans=((0, 1),)))
|
||||
assert excinfo.value.code == "segmentation_span_invalid"
|
||||
|
||||
|
||||
# --- S5c: the quote anchor, which survives a shift the key must not refuse --
|
||||
#
|
||||
# The key (S5b, above) refuses a FOREIGN extraction. This is the other half:
|
||||
# a benign shift INSIDE the same extraction, where the text an entry names is
|
||||
# still present but no longer at the offset written down. Measured: changing
|
||||
# one extraction flag moved a document from 15 507 to 8 290 characters and
|
||||
# every inspected span pointed at the wrong text. Offsets are therefore a HINT
|
||||
# the quote may correct -- never the last word, and never silently wrong.
|
||||
|
||||
|
||||
ANCHORED = "innledning\n" + SECTIONS
|
||||
|
||||
|
||||
def anchored_entry(index: int, **overrides: Any) -> dict[str, Any]:
|
||||
start, end = SPANS[index]
|
||||
quote = SECTIONS[start:end]
|
||||
payload = entry(
|
||||
segment_id=f"s{index}",
|
||||
path=f"krav/{index}-del.md",
|
||||
span=[start, end],
|
||||
anchor={
|
||||
"quote": quote,
|
||||
"prefix": SECTIONS[max(0, start - 12) : start],
|
||||
"suffix": SECTIONS[end : end + 12],
|
||||
},
|
||||
)
|
||||
payload.update(overrides)
|
||||
return payload
|
||||
|
||||
|
||||
def anchored_plan(*indexes: int, **overrides: Any) -> SegmentationPlan:
|
||||
return parse_segmentation_plan(
|
||||
plan(entries=[anchored_entry(index) for index in indexes], **overrides)
|
||||
)
|
||||
|
||||
|
||||
def test_an_anchor_whose_offsets_still_hold_changes_nothing() -> None:
|
||||
"""The fast path. A correct plan must not be re-anchored into motion."""
|
||||
sliced = slice_segments(SECTIONS, anchored_plan(1, 2))
|
||||
assert [body for _, body in sliced] == [SECTIONS[34:73], SECTIONS[73:106]]
|
||||
assert [item.span for item, _ in sliced] == [(34, 73), (73, 106)]
|
||||
|
||||
|
||||
def test_stale_offsets_re_anchor_to_the_text_the_quote_names() -> None:
|
||||
"""The whole point: the same document with eleven characters prepended.
|
||||
|
||||
Every declared offset is now short by exactly that much. Without the
|
||||
anchor each body would be cut eleven characters early and land on real
|
||||
text -- the failure no downstream assertion can see.
|
||||
"""
|
||||
sliced = slice_segments(ANCHORED, anchored_plan(1, 2))
|
||||
assert [body for _, body in sliced] == [SECTIONS[34:73], SECTIONS[73:106]]
|
||||
assert [item.span for item, _ in sliced] == [(45, 84), (84, 117)]
|
||||
|
||||
|
||||
def test_a_quote_that_no_longer_occurs_is_refused_rather_than_cut_elsewhere() -> None:
|
||||
"""Refusing beats cutting. A body sliced at a plausible-but-wrong offset
|
||||
reads as ordinary prose, so nothing downstream can distinguish it from an
|
||||
adjudicated one."""
|
||||
gone = SECTIONS.replace("1 Brannkonsept: krav til seksjonering.\n", "")
|
||||
with pytest.raises(SegmentationError) as excinfo:
|
||||
slice_segments(gone, anchored_plan(1))
|
||||
assert excinfo.value.code == "segmentation_span_invalid"
|
||||
assert "s1" in str(excinfo.value)
|
||||
|
||||
|
||||
def test_a_repeated_quote_re_anchors_to_the_occurrence_its_context_names() -> None:
|
||||
"""Prefix and suffix are what make a repeated line addressable at all."""
|
||||
doubled = "xx" + SECTIONS + SECTIONS
|
||||
sliced = slice_segments(doubled, anchored_plan(1))
|
||||
assert sliced[0][0].span == (36, 75)
|
||||
assert sliced[0][1] == SECTIONS[34:73]
|
||||
|
||||
|
||||
def test_an_entry_without_an_anchor_keeps_todays_offset_behaviour() -> None:
|
||||
"""The anchor is a capability, not a new requirement on authored plans."""
|
||||
sliced = slice_segments(SECTIONS, five_section_plan())
|
||||
assert [item.span for item, _ in sliced] == list(SPANS)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue