feat(segmentation): key the adjudication cache on the extractor, not the hash alone
This commit is contained in:
parent
0a11c860ce
commit
832e541fb9
2 changed files with 136 additions and 0 deletions
|
|
@ -24,7 +24,9 @@ from llm_ingestion_okf.errors import SegmentationError
|
|||
from llm_ingestion_okf.segmentation import (
|
||||
SegmentationPlan,
|
||||
SegmentEntry,
|
||||
assert_plan_applies,
|
||||
parse_segmentation_plan,
|
||||
plan_cache_key,
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -258,3 +260,86 @@ def test_two_paths_differing_only_in_normal_form_collide_as_duplicates() -> None
|
|||
)
|
||||
)
|
||||
assert error.code == "segmentation_path_invalid"
|
||||
|
||||
|
||||
# --- S5b: the cache key is the extractor, not the hash alone ---------------
|
||||
#
|
||||
# Source bytes cannot see an extractor swap or a version bump. Both invalidate
|
||||
# every stored offset while `source_sha256` stays identical, so the mismatch
|
||||
# has to be LOUD -- a silent re-derivation would replay a human's adjudication
|
||||
# against text that human never saw.
|
||||
|
||||
|
||||
def parsed_plan(**overrides: Any) -> SegmentationPlan:
|
||||
return parse_segmentation_plan(plan(**overrides))
|
||||
|
||||
|
||||
def applies_fails(subject: SegmentationPlan, **overrides: str) -> SegmentationError:
|
||||
arguments = {
|
||||
"source_sha256": subject.source_sha256,
|
||||
"extractor_id": subject.extractor_id,
|
||||
"extractor_version": subject.extractor_version,
|
||||
}
|
||||
arguments.update(overrides)
|
||||
with pytest.raises(SegmentationError) as excinfo:
|
||||
assert_plan_applies(subject, **arguments)
|
||||
return excinfo.value
|
||||
|
||||
|
||||
def test_the_cache_key_is_the_three_tuple() -> None:
|
||||
subject = parsed_plan()
|
||||
assert plan_cache_key(subject) == (
|
||||
subject.source_sha256,
|
||||
subject.extractor_id,
|
||||
subject.extractor_version,
|
||||
)
|
||||
|
||||
|
||||
def test_two_plans_differing_only_in_extractor_id_have_different_cache_keys() -> None:
|
||||
# The whole point of S5b: the hash alone would call these one cached
|
||||
# adjudication, and replay the first plan's offsets against the second
|
||||
# extraction.
|
||||
one = parsed_plan(extractor_id="text")
|
||||
other = parsed_plan(extractor_id="pdfplumber")
|
||||
assert one.source_sha256 == other.source_sha256
|
||||
assert plan_cache_key(one) != plan_cache_key(other)
|
||||
assert plan_cache_key(one)[0] == plan_cache_key(other)[0]
|
||||
|
||||
|
||||
def test_an_identical_triple_applies_without_raising() -> None:
|
||||
subject = parsed_plan()
|
||||
assert (
|
||||
assert_plan_applies(
|
||||
subject,
|
||||
source_sha256=subject.source_sha256,
|
||||
extractor_id=subject.extractor_id,
|
||||
extractor_version=subject.extractor_version,
|
||||
)
|
||||
is None
|
||||
)
|
||||
|
||||
|
||||
def test_a_changed_extractor_version_is_refused_and_named() -> None:
|
||||
error = applies_fails(parsed_plan(), extractor_version="1.0.1")
|
||||
assert error.code == "segmentation_extractor_mismatch"
|
||||
assert "extractor_version" in str(error)
|
||||
assert "1.0.1" in str(error)
|
||||
|
||||
|
||||
def test_a_changed_extractor_id_is_refused_and_named() -> None:
|
||||
error = applies_fails(parsed_plan(), extractor_id="pdfplumber")
|
||||
assert error.code == "segmentation_extractor_mismatch"
|
||||
assert "extractor_id" in str(error)
|
||||
|
||||
|
||||
def test_a_changed_source_hash_is_refused_and_named() -> None:
|
||||
error = applies_fails(parsed_plan(), source_sha256="b" * 64)
|
||||
assert error.code == "segmentation_extractor_mismatch"
|
||||
assert "source_sha256" in str(error)
|
||||
|
||||
|
||||
def test_the_message_names_every_differing_component_not_just_the_first() -> None:
|
||||
error = applies_fails(parsed_plan(), extractor_id="pdfplumber", extractor_version="1.0.1")
|
||||
assert "extractor_id" in str(error)
|
||||
assert "extractor_version" in str(error)
|
||||
assert "source_sha256" not in str(error)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue