feat(segmentation): key the adjudication cache on the extractor, not the hash alone

This commit is contained in:
Kjell Tore Guttormsen 2026-08-31 23:57:49 +02:00
commit 832e541fb9
2 changed files with 136 additions and 0 deletions

View file

@ -24,7 +24,9 @@ from llm_ingestion_okf.errors import SegmentationError
from llm_ingestion_okf.segmentation import (
SegmentationPlan,
SegmentEntry,
assert_plan_applies,
parse_segmentation_plan,
plan_cache_key,
)
@ -258,3 +260,86 @@ def test_two_paths_differing_only_in_normal_form_collide_as_duplicates() -> None
)
)
assert error.code == "segmentation_path_invalid"
# --- S5b: the cache key is the extractor, not the hash alone ---------------
#
# Source bytes cannot see an extractor swap or a version bump. Both invalidate
# every stored offset while `source_sha256` stays identical, so the mismatch
# has to be LOUD -- a silent re-derivation would replay a human's adjudication
# against text that human never saw.
def parsed_plan(**overrides: Any) -> SegmentationPlan:
return parse_segmentation_plan(plan(**overrides))
def applies_fails(subject: SegmentationPlan, **overrides: str) -> SegmentationError:
arguments = {
"source_sha256": subject.source_sha256,
"extractor_id": subject.extractor_id,
"extractor_version": subject.extractor_version,
}
arguments.update(overrides)
with pytest.raises(SegmentationError) as excinfo:
assert_plan_applies(subject, **arguments)
return excinfo.value
def test_the_cache_key_is_the_three_tuple() -> None:
subject = parsed_plan()
assert plan_cache_key(subject) == (
subject.source_sha256,
subject.extractor_id,
subject.extractor_version,
)
def test_two_plans_differing_only_in_extractor_id_have_different_cache_keys() -> None:
# The whole point of S5b: the hash alone would call these one cached
# adjudication, and replay the first plan's offsets against the second
# extraction.
one = parsed_plan(extractor_id="text")
other = parsed_plan(extractor_id="pdfplumber")
assert one.source_sha256 == other.source_sha256
assert plan_cache_key(one) != plan_cache_key(other)
assert plan_cache_key(one)[0] == plan_cache_key(other)[0]
def test_an_identical_triple_applies_without_raising() -> None:
subject = parsed_plan()
assert (
assert_plan_applies(
subject,
source_sha256=subject.source_sha256,
extractor_id=subject.extractor_id,
extractor_version=subject.extractor_version,
)
is None
)
def test_a_changed_extractor_version_is_refused_and_named() -> None:
error = applies_fails(parsed_plan(), extractor_version="1.0.1")
assert error.code == "segmentation_extractor_mismatch"
assert "extractor_version" in str(error)
assert "1.0.1" in str(error)
def test_a_changed_extractor_id_is_refused_and_named() -> None:
error = applies_fails(parsed_plan(), extractor_id="pdfplumber")
assert error.code == "segmentation_extractor_mismatch"
assert "extractor_id" in str(error)
def test_a_changed_source_hash_is_refused_and_named() -> None:
error = applies_fails(parsed_plan(), source_sha256="b" * 64)
assert error.code == "segmentation_extractor_mismatch"
assert "source_sha256" in str(error)
def test_the_message_names_every_differing_component_not_just_the_first() -> None:
error = applies_fails(parsed_plan(), extractor_id="pdfplumber", extractor_version="1.0.1")
assert "extractor_id" in str(error)
assert "extractor_version" in str(error)
assert "source_sha256" not in str(error)