fix(segmentation): hash the extracted text and let the plan key fire

This commit is contained in:
Kjell Tore Guttormsen 2026-09-02 14:38:20 +02:00
commit 9e9bb8645d
13 changed files with 272 additions and 40 deletions

View file

@ -41,6 +41,7 @@ from .segmentation import (
SegmentationPlan,
SegmentEntry,
assert_plan_applies,
observed_extractor_version,
slice_segments,
)
from .structure import (
@ -411,17 +412,22 @@ def _render_segments(
A refusal is therefore reported once, for the document, rather than once
per segment: the operator's unit of review is the document they dropped.
"""
extractor_id = Path(path.name).suffix.lower().lstrip(".") or "none"
assert_plan_applies(
plan,
source_sha256=hashlib.sha256(source_bytes).hexdigest(),
extractor_id=Path(path.name).suffix.lower().lstrip(".") or "none",
# The plan's own value, passed through. Door B can observe WHICH
# extractor ran (the suffix is what dispatches it at `extract.py`) but
# not the version of a third-party parser -- `pdfplumber`'s transitive
# `pdfminer.six` pin is the measured example. Naming the key and
# leaving its value to whoever knows it is the same division D5 makes
# for `bundle_id`; a fabricated value here would make S5b decorative.
extractor_version=plan.extractor_version,
# `text` is the canonical extracted text this run produced, AFTER the
# profile's renderer -- exactly the string the plan's offsets index.
text_sha256=hashlib.sha256(text.encode("utf-8")).hexdigest(),
extractor_id=extractor_id,
# OBSERVED, never the plan's own value passed back in. That is what
# this line used to do, and comparing a value with itself made the
# version half of S5b unable to fail: a plan adjudicated under one
# converter replayed silently under another. The version of a
# third-party parser is knowable here after all -- `pdfplumber`'s
# transitive `pdfminer.six` pin is the measured example, and
# `observed_extractor_version` is where each row names its source.
extractor_version=observed_extractor_version(extractor_id),
)
sliced = slice_segments(text, plan)