open/llm-ingestion-okf
1
0
Fork 0
Code Issues Releases 1 Activity

fix(segmentation): hash the extracted text and let the plan key fire

Browse source
This commit is contained in:
Kjell Tore Guttormsen 2026-09-02 14:38:20 +02:00
commit 9e9bb8645d
13 changed files with 272 additions and 40 deletions
Download patch file Download diff file Expand all files Collapse all files

23
tests/test_segmentation.py
View file

@ -48,6 +48,7 @@ def plan(**overrides: Any) -> dict[str, Any]:
payload: dict[str, Any] = {
"version": "1",
"source_sha256": "a" * 64,
"text_sha256": "c" * 64,
"extractor_id": "text",
"extractor_version": "1.0.0",
"adjudicated_at": "2026-08-31T11:00:00Z",
@ -278,6 +279,7 @@ def parsed_plan(**overrides: Any) -> SegmentationPlan:
def applies_fails(subject: SegmentationPlan, **overrides: str) -> SegmentationError:
arguments = {
"source_sha256": subject.source_sha256,
"text_sha256": subject.text_sha256,
"extractor_id": subject.extractor_id,
"extractor_version": subject.extractor_version,
}
@ -287,10 +289,11 @@ def applies_fails(subject: SegmentationPlan, **overrides: str) -> SegmentationEr
return excinfo.value
def test_the_cache_key_is_the_three_tuple() -> None:
def test_the_cache_key_is_the_four_tuple() -> None:
subject = parsed_plan()
assert plan_cache_key(subject) == (
subject.source_sha256,
subject.text_sha256,
subject.extractor_id,
subject.extractor_version,
)
@ -307,12 +310,13 @@ def test_two_plans_differing_only_in_extractor_id_have_different_cache_keys() ->
assert plan_cache_key(one)[0] == plan_cache_key(other)[0]
def test_an_identical_triple_applies_without_raising() -> None:
def test_an_identical_quadruple_applies_without_raising() -> None:
subject = parsed_plan()
assert (
assert_plan_applies(
subject,
source_sha256=subject.source_sha256,
text_sha256=subject.text_sha256,
extractor_id=subject.extractor_id,
extractor_version=subject.extractor_version,
)
@ -320,6 +324,21 @@ def test_an_identical_triple_applies_without_raising() -> None:
)
def test_a_changed_text_hash_is_refused_although_the_source_bytes_match() -> None:
"""The component the other three cannot stand in for.
Same bytes, same extractor, same version -- and a canonical text that
moved anyway, which is what a converter reshaping its output without
bumping its version looks like from here. Before this component existed
the key matched and every offset was replayed against text nobody
adjudicated, with each span still landing on real characters.
"""
error = applies_fails(parsed_plan(), text_sha256="d" * 64)
assert error.code == "segmentation_extractor_mismatch"
assert "text_sha256" in str(error)
assert "source_sha256" not in str(error)
def test_a_changed_extractor_version_is_refused_and_named() -> None:
error = applies_fails(parsed_plan(), extractor_version="1.0.1")
assert error.code == "segmentation_extractor_mismatch"