feat(segmentation): quote anchors so a re-extraction costs a re-anchor
This commit is contained in:
parent
9e9bb8645d
commit
c54e8383df
4 changed files with 244 additions and 8 deletions
|
|
@ -37,7 +37,7 @@ Pure: no filesystem, no bundle, no door, no model call, no network.
|
|||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Mapping, Sequence
|
||||
from dataclasses import dataclass, field
|
||||
from dataclasses import dataclass, field, replace
|
||||
from importlib import metadata
|
||||
from pathlib import PurePosixPath
|
||||
from typing import Any
|
||||
|
|
@ -58,9 +58,10 @@ PLAN_FIELDS = (
|
|||
"entries",
|
||||
)
|
||||
|
||||
#: The keys every entry must carry. `parent_id` and `derived` are optional --
|
||||
#: a flat plan has no parents, and an entry adjudicated from scratch derived
|
||||
#: nothing.
|
||||
#: The keys every entry must carry. `parent_id`, `derived` and `anchor` are
|
||||
#: optional -- a flat plan has no parents, an entry adjudicated from scratch
|
||||
#: derived nothing, and a plan authored before quote anchors existed carries
|
||||
#: offsets alone.
|
||||
ENTRY_FIELDS = ("segment_id", "path", "title", "okf_type", "span", "ingested_at")
|
||||
|
||||
#: Path components refused outright, before the id grammar is consulted. An
|
||||
|
|
@ -98,6 +99,27 @@ _CONVERTED_EXTRACTOR_IDS = frozenset({"docx", "xlsx", "pptx", "odt", "rtf"})
|
|||
_PDF_DISTRIBUTION = "pdfminer.six"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SegmentAnchor:
|
||||
"""The text an entry names, plus enough context to find it again.
|
||||
|
||||
Offsets alone are brittle in the one direction that matters. Measured:
|
||||
changing a single extraction flag moved a document from 15 507 to 8 290
|
||||
characters, and every inspected span then pointed at the wrong text --
|
||||
real prose, cut at a plausible offset, indistinguishable downstream from
|
||||
an adjudicated body. The quote makes that recoverable rather than silent.
|
||||
|
||||
`prefix` and `suffix` are not decoration either: a line that occurs twice
|
||||
is not addressable by its own text, and picking the first occurrence would
|
||||
re-anchor a document's second section onto its first. They may be empty --
|
||||
an entry at the very start or end of a document has no room for them.
|
||||
"""
|
||||
|
||||
quote: str
|
||||
prefix: str = ""
|
||||
suffix: str = ""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SegmentEntry:
|
||||
"""One concept a document expands into.
|
||||
|
|
@ -116,6 +138,7 @@ class SegmentEntry:
|
|||
ingested_at: str
|
||||
parent_id: str | None = None
|
||||
derived: frozenset[str] = field(default_factory=frozenset)
|
||||
anchor: SegmentAnchor | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
|
|
@ -240,6 +263,28 @@ def _parse_derived(value: Any, *, where: str) -> frozenset[str]:
|
|||
return frozenset(value)
|
||||
|
||||
|
||||
def _parse_anchor(value: Any, *, where: str) -> SegmentAnchor | None:
|
||||
if value is None:
|
||||
return None
|
||||
if not isinstance(value, Mapping):
|
||||
raise SegmentationError(
|
||||
f"{where} field 'anchor' must be a mapping with a 'quote' and optional "
|
||||
f"'prefix'/'suffix', got {value!r}",
|
||||
code="segmentation_plan_invalid",
|
||||
)
|
||||
quote = _require_str(value, "quote", where=f"{where} anchor")
|
||||
context: dict[str, str] = {}
|
||||
for key in ("prefix", "suffix"):
|
||||
found = value.get(key, "")
|
||||
if not isinstance(found, str):
|
||||
raise SegmentationError(
|
||||
f"{where} anchor field {key!r} must be a string, got {found!r}",
|
||||
code="segmentation_plan_invalid",
|
||||
)
|
||||
context[key] = found
|
||||
return SegmentAnchor(quote=quote, prefix=context["prefix"], suffix=context["suffix"])
|
||||
|
||||
|
||||
def _parse_entry(payload: Any, *, position: int) -> SegmentEntry:
|
||||
where = f"segmentation entry {position}"
|
||||
if not isinstance(payload, Mapping):
|
||||
|
|
@ -272,6 +317,7 @@ def _parse_entry(payload: Any, *, position: int) -> SegmentEntry:
|
|||
ingested_at=_require_str(payload, "ingested_at", where=where),
|
||||
parent_id=parent_id,
|
||||
derived=_parse_derived(payload.get("derived", ()), where=where),
|
||||
anchor=_parse_anchor(payload.get("anchor"), where=where),
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -467,10 +513,39 @@ def slice_segments(text: str, plan: SegmentationPlan) -> tuple[tuple[SegmentEntr
|
|||
that is not a judgement about the document but proof the plan was
|
||||
adjudicated against a different extraction.
|
||||
"""
|
||||
limit = len(text)
|
||||
sliced: list[tuple[SegmentEntry, str]] = []
|
||||
for item in plan.entries:
|
||||
start, end = item.span
|
||||
resolved = _resolve_entry(text, item)
|
||||
start, end = resolved.span
|
||||
sliced.append((resolved, text[start:end]))
|
||||
return tuple(sliced)
|
||||
|
||||
|
||||
def _all_occurrences(text: str, needle: str) -> list[int]:
|
||||
found: list[int] = []
|
||||
position = text.find(needle)
|
||||
while position != -1:
|
||||
found.append(position)
|
||||
position = text.find(needle, position + 1)
|
||||
return found
|
||||
|
||||
|
||||
def _resolve_entry(text: str, item: SegmentEntry) -> SegmentEntry:
|
||||
"""The entry with the span that actually names its text in THIS extraction.
|
||||
|
||||
Offsets are a HINT the anchor may correct, and the correction is written
|
||||
back onto the entry rather than applied only to the slice: the frontmatter
|
||||
records `span`, so a body cut at one offset and stamped with another would
|
||||
make the bundle disagree with itself.
|
||||
|
||||
An anchorless entry keeps the old behaviour exactly -- the anchor is a
|
||||
capability a plan may carry, not a new requirement on authored plans, and
|
||||
every golden predates it.
|
||||
"""
|
||||
start, end = item.span
|
||||
limit = len(text)
|
||||
anchor = item.anchor
|
||||
if anchor is None:
|
||||
if end > limit:
|
||||
raise SegmentationError(
|
||||
f"segmentation entry {item.segment_id!r} declares span [{start}, {end}] "
|
||||
|
|
@ -479,5 +554,28 @@ def slice_segments(text: str, plan: SegmentationPlan) -> tuple[tuple[SegmentEntr
|
|||
"re-adjudicate rather than truncating to fit",
|
||||
code="segmentation_span_invalid",
|
||||
)
|
||||
sliced.append((item, text[start:end]))
|
||||
return tuple(sliced)
|
||||
return item
|
||||
|
||||
if text[start:end] == anchor.quote:
|
||||
return item
|
||||
|
||||
# Context INCLUDED in the needle, never searched for separately. A quote
|
||||
# that occurs twice is not addressable by itself, and taking the first
|
||||
# occurrence would re-anchor a document's second section onto its first.
|
||||
needle = anchor.prefix + anchor.quote + anchor.suffix
|
||||
positions = _all_occurrences(text, needle)
|
||||
if not positions:
|
||||
raise SegmentationError(
|
||||
f"segmentation entry {item.segment_id!r} declares span [{start}, {end}], which "
|
||||
"does not hold in this extraction, and its quoted anchor does not occur in "
|
||||
"the canonical extracted text either — refusing to cut at a plausible offset "
|
||||
"nobody adjudicated; re-run the proposer and re-adjudicate",
|
||||
code="segmentation_span_invalid",
|
||||
)
|
||||
# Offsets are still worth something when the anchor is ambiguous: the
|
||||
# nearest occurrence to the declared position is the one the adjudicator
|
||||
# was looking at.
|
||||
hint = start - len(anchor.prefix)
|
||||
best = min(positions, key=lambda position: (abs(position - hint), position))
|
||||
moved = best + len(anchor.prefix)
|
||||
return replace(item, span=(moved, moved + len(anchor.quote)))
|
||||
|
|
|
|||
|
|
@ -142,6 +142,48 @@ def test_the_spans_it_proposes_are_slices_of_the_text_it_read(tmp_path: Path) ->
|
|||
assert text[start:end].strip()
|
||||
|
||||
|
||||
def test_every_entry_carries_an_anchor_quoting_its_own_span(tmp_path: Path) -> None:
|
||||
"""Written at proposal time, when text and offsets are known to agree.
|
||||
|
||||
Reconstructed later it would quote whatever the extraction had already
|
||||
become, which is precisely the drift the anchor exists to survive.
|
||||
"""
|
||||
from llm_ingestion_okf.extract import extract_text
|
||||
|
||||
source = write(tmp_path)
|
||||
payload = propose(tmp_path, source=source)
|
||||
text = extract_text(source.name, source.read_bytes())
|
||||
assert payload["entries"]
|
||||
for entry in payload["entries"]:
|
||||
start, end = entry["span"]
|
||||
anchor = entry["anchor"]
|
||||
assert anchor["quote"] == text[start:end]
|
||||
assert text[start - len(anchor["prefix"]) : start] == anchor["prefix"]
|
||||
assert text[end : end + len(anchor["suffix"])] == anchor["suffix"]
|
||||
|
||||
|
||||
def test_a_proposed_plan_survives_a_shift_of_the_text_it_was_made_against(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
"""The round trip the anchor exists for, measured end to end.
|
||||
|
||||
Prepend a paragraph -- exactly the benign edit that leaves every offset
|
||||
short -- and every body must still be the text the proposer quoted. Before
|
||||
anchors this cut each segment early and landed on real prose, with nothing
|
||||
anywhere able to tell.
|
||||
"""
|
||||
from llm_ingestion_okf.extract import extract_text
|
||||
from llm_ingestion_okf.segmentation import slice_segments
|
||||
|
||||
source = write(tmp_path)
|
||||
plan = parse_segmentation_plan(propose(tmp_path, source=source))
|
||||
text = extract_text(source.name, source.read_bytes())
|
||||
quoted = [text[start:end] for start, end in (item.span for item in plan.entries)]
|
||||
|
||||
shifted = "Et nytt avsnitt foran alt annet.\n\n" + text
|
||||
assert [body for _, body in slice_segments(shifted, plan)] == quoted
|
||||
|
||||
|
||||
def test_the_paths_it_proposes_are_unique_and_hierarchical(tmp_path: Path) -> None:
|
||||
plan = parse_segmentation_plan(propose(tmp_path))
|
||||
paths = [entry.path for entry in plan.entries]
|
||||
|
|
|
|||
|
|
@ -451,3 +451,83 @@ def test_slicing_an_empty_text_with_any_span_is_refused() -> None:
|
|||
with pytest.raises(SegmentationError) as excinfo:
|
||||
slice_segments("", five_section_plan(spans=((0, 1),)))
|
||||
assert excinfo.value.code == "segmentation_span_invalid"
|
||||
|
||||
|
||||
# --- S5c: the quote anchor, which survives a shift the key must not refuse --
|
||||
#
|
||||
# The key (S5b, above) refuses a FOREIGN extraction. This is the other half:
|
||||
# a benign shift INSIDE the same extraction, where the text an entry names is
|
||||
# still present but no longer at the offset written down. Measured: changing
|
||||
# one extraction flag moved a document from 15 507 to 8 290 characters and
|
||||
# every inspected span pointed at the wrong text. Offsets are therefore a HINT
|
||||
# the quote may correct -- never the last word, and never silently wrong.
|
||||
|
||||
|
||||
ANCHORED = "innledning\n" + SECTIONS
|
||||
|
||||
|
||||
def anchored_entry(index: int, **overrides: Any) -> dict[str, Any]:
|
||||
start, end = SPANS[index]
|
||||
quote = SECTIONS[start:end]
|
||||
payload = entry(
|
||||
segment_id=f"s{index}",
|
||||
path=f"krav/{index}-del.md",
|
||||
span=[start, end],
|
||||
anchor={
|
||||
"quote": quote,
|
||||
"prefix": SECTIONS[max(0, start - 12) : start],
|
||||
"suffix": SECTIONS[end : end + 12],
|
||||
},
|
||||
)
|
||||
payload.update(overrides)
|
||||
return payload
|
||||
|
||||
|
||||
def anchored_plan(*indexes: int, **overrides: Any) -> SegmentationPlan:
|
||||
return parse_segmentation_plan(
|
||||
plan(entries=[anchored_entry(index) for index in indexes], **overrides)
|
||||
)
|
||||
|
||||
|
||||
def test_an_anchor_whose_offsets_still_hold_changes_nothing() -> None:
|
||||
"""The fast path. A correct plan must not be re-anchored into motion."""
|
||||
sliced = slice_segments(SECTIONS, anchored_plan(1, 2))
|
||||
assert [body for _, body in sliced] == [SECTIONS[34:73], SECTIONS[73:106]]
|
||||
assert [item.span for item, _ in sliced] == [(34, 73), (73, 106)]
|
||||
|
||||
|
||||
def test_stale_offsets_re_anchor_to_the_text_the_quote_names() -> None:
|
||||
"""The whole point: the same document with eleven characters prepended.
|
||||
|
||||
Every declared offset is now short by exactly that much. Without the
|
||||
anchor each body would be cut eleven characters early and land on real
|
||||
text -- the failure no downstream assertion can see.
|
||||
"""
|
||||
sliced = slice_segments(ANCHORED, anchored_plan(1, 2))
|
||||
assert [body for _, body in sliced] == [SECTIONS[34:73], SECTIONS[73:106]]
|
||||
assert [item.span for item, _ in sliced] == [(45, 84), (84, 117)]
|
||||
|
||||
|
||||
def test_a_quote_that_no_longer_occurs_is_refused_rather_than_cut_elsewhere() -> None:
|
||||
"""Refusing beats cutting. A body sliced at a plausible-but-wrong offset
|
||||
reads as ordinary prose, so nothing downstream can distinguish it from an
|
||||
adjudicated one."""
|
||||
gone = SECTIONS.replace("1 Brannkonsept: krav til seksjonering.\n", "")
|
||||
with pytest.raises(SegmentationError) as excinfo:
|
||||
slice_segments(gone, anchored_plan(1))
|
||||
assert excinfo.value.code == "segmentation_span_invalid"
|
||||
assert "s1" in str(excinfo.value)
|
||||
|
||||
|
||||
def test_a_repeated_quote_re_anchors_to_the_occurrence_its_context_names() -> None:
|
||||
"""Prefix and suffix are what make a repeated line addressable at all."""
|
||||
doubled = "xx" + SECTIONS + SECTIONS
|
||||
sliced = slice_segments(doubled, anchored_plan(1))
|
||||
assert sliced[0][0].span == (36, 75)
|
||||
assert sliced[0][1] == SECTIONS[34:73]
|
||||
|
||||
|
||||
def test_an_entry_without_an_anchor_keeps_todays_offset_behaviour() -> None:
|
||||
"""The anchor is a capability, not a new requirement on authored plans."""
|
||||
sliced = slice_segments(SECTIONS, five_section_plan())
|
||||
assert [item.span for item, _ in sliced] == list(SPANS)
|
||||
|
|
|
|||
|
|
@ -76,6 +76,12 @@ RULE_TABLE_BLOCK = "rule:table-block"
|
|||
RULE_POPPLER_SIZE_AND_BOLD = "rule:poppler-size-and-bold"
|
||||
RULE_NAMES = (RULE_HEADING, RULE_TABLE_BLOCK, RULE_POPPLER_SIZE_AND_BOLD)
|
||||
|
||||
#: How many characters of context each side of a quote anchor carries. Enough
|
||||
#: to separate two occurrences of a repeated heading, short enough that an
|
||||
#: edit NEAR a segment does not invalidate the anchor FOR it -- the anchor
|
||||
#: exists to survive shifts, so making it fragile would defeat it.
|
||||
ANCHOR_CONTEXT = 48
|
||||
|
||||
#: Norwegian and English function words. A heading made only of these names no
|
||||
#: unit of knowledge -- it is a connective that happened to sit on its own line.
|
||||
#: Topic 2's stop-word gate, and the only place this tool judges wording.
|
||||
|
|
@ -278,6 +284,16 @@ def build_plan(
|
|||
"okf_type": okf_type,
|
||||
"span": [candidate.start, candidate.end],
|
||||
"ingested_at": proposed_at,
|
||||
# The offsets are a hint the anchor may correct. Written at
|
||||
# proposal time because that is the only moment the text the
|
||||
# adjudicator will judge and the offsets naming it are known
|
||||
# to agree -- reconstructing it later would anchor to whatever
|
||||
# the extraction had already become.
|
||||
"anchor": {
|
||||
"quote": text[candidate.start : candidate.end],
|
||||
"prefix": text[max(0, candidate.start - ANCHOR_CONTEXT) : candidate.start],
|
||||
"suffix": text[candidate.end : candidate.end + ANCHOR_CONTEXT],
|
||||
},
|
||||
# PROPOSED first, then the rule that proposed it. `derived` is
|
||||
# this library's existing "which of these did we infer" marker,
|
||||
# so a consumer that already distrusts derived fields
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue