feat(segmentation): slice canonical extracted text by declared spans
This commit is contained in:
parent
832e541fb9
commit
499253e53b
2 changed files with 122 additions and 0 deletions
|
|
@ -366,3 +366,36 @@ def assert_plan_applies(
|
|||
"re-run the proposer and re-adjudicate",
|
||||
code="segmentation_extractor_mismatch",
|
||||
)
|
||||
|
||||
|
||||
def slice_segments(text: str, plan: SegmentationPlan) -> tuple[tuple[SegmentEntry, str], ...]:
|
||||
"""Pair every entry with the substring its declared span names, in plan order.
|
||||
|
||||
`text` is the CANONICAL EXTRACTED text -- whatever
|
||||
:func:`llm_ingestion_okf.extract.extract_text` returned -- never the source
|
||||
bytes. The distinction is not pedantry: a `.csv` is re-rendered as a table
|
||||
and a `.pdf` has no sliceable prose at all, so an offset computed against
|
||||
bytes would land on different characters and produce a concept body no one
|
||||
adjudicated, with nothing failing.
|
||||
|
||||
Spans may OVERLAP and need not cover the whole text. Neither is asserted:
|
||||
a preamble, a page header or a signature block is legitimately part of no
|
||||
unit of knowledge, and forcing full coverage would make the adjudicator
|
||||
invent a home for it. What is refused is a span reaching past the end --
|
||||
that is not a judgement about the document but proof the plan was
|
||||
adjudicated against a different extraction.
|
||||
"""
|
||||
limit = len(text)
|
||||
sliced: list[tuple[SegmentEntry, str]] = []
|
||||
for item in plan.entries:
|
||||
start, end = item.span
|
||||
if end > limit:
|
||||
raise SegmentationError(
|
||||
f"segmentation entry {item.segment_id!r} declares span [{start}, {end}] "
|
||||
f"but the canonical extracted text is {limit} characters — the plan was "
|
||||
"adjudicated against a different extraction; re-run the proposer and "
|
||||
"re-adjudicate rather than truncating to fit",
|
||||
code="segmentation_span_invalid",
|
||||
)
|
||||
sliced.append((item, text[start:end]))
|
||||
return tuple(sliced)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue