fix(segmentation): hash the extracted text and let the plan key fire
This commit is contained in:
parent
6dce4355be
commit
9e9bb8645d
13 changed files with 272 additions and 40 deletions
|
|
@ -41,6 +41,7 @@ from .segmentation import (
|
|||
SegmentationPlan,
|
||||
SegmentEntry,
|
||||
assert_plan_applies,
|
||||
observed_extractor_version,
|
||||
slice_segments,
|
||||
)
|
||||
from .structure import (
|
||||
|
|
@ -411,17 +412,22 @@ def _render_segments(
|
|||
A refusal is therefore reported once, for the document, rather than once
|
||||
per segment: the operator's unit of review is the document they dropped.
|
||||
"""
|
||||
extractor_id = Path(path.name).suffix.lower().lstrip(".") or "none"
|
||||
assert_plan_applies(
|
||||
plan,
|
||||
source_sha256=hashlib.sha256(source_bytes).hexdigest(),
|
||||
extractor_id=Path(path.name).suffix.lower().lstrip(".") or "none",
|
||||
# The plan's own value, passed through. Door B can observe WHICH
|
||||
# extractor ran (the suffix is what dispatches it at `extract.py`) but
|
||||
# not the version of a third-party parser -- `pdfplumber`'s transitive
|
||||
# `pdfminer.six` pin is the measured example. Naming the key and
|
||||
# leaving its value to whoever knows it is the same division D5 makes
|
||||
# for `bundle_id`; a fabricated value here would make S5b decorative.
|
||||
extractor_version=plan.extractor_version,
|
||||
# `text` is the canonical extracted text this run produced, AFTER the
|
||||
# profile's renderer -- exactly the string the plan's offsets index.
|
||||
text_sha256=hashlib.sha256(text.encode("utf-8")).hexdigest(),
|
||||
extractor_id=extractor_id,
|
||||
# OBSERVED, never the plan's own value passed back in. That is what
|
||||
# this line used to do, and comparing a value with itself made the
|
||||
# version half of S5b unable to fail: a plan adjudicated under one
|
||||
# converter replayed silently under another. The version of a
|
||||
# third-party parser is knowable here after all -- `pdfplumber`'s
|
||||
# transitive `pdfminer.six` pin is the measured example, and
|
||||
# `observed_extractor_version` is where each row names its source.
|
||||
extractor_version=observed_extractor_version(extractor_id),
|
||||
)
|
||||
sliced = slice_segments(text, plan)
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue