feat(segmentation): key the adjudication cache on the extractor, not the hash alone

src/llm_ingestion_okf/segmentation.py

@@ -67,6 +67,12 @@ ENTRY_FIELDS = ("segment_id", "path", "title", "okf_type", "span", "ingested_at"
 #: an authored `..` is a mistake worth naming, not a path worth normalising.
 FORBIDDEN_COMPONENTS = ("", ".", "..")
 
+#: The components of the adjudication cache key, in the order
+#: :func:`plan_cache_key` returns them. Named so a mismatch message can say
+#: WHICH one moved -- that is what tells an operator whether to re-run the
+#: proposer or re-adjudicate by hand.
+CACHE_KEY_COMPONENTS = ("source_sha256", "extractor_id", "extractor_version")
+
 
 @dataclass(frozen=True)
 class SegmentEntry:
@@ -315,3 +321,48 @@ def parse_segmentation_plan(payload: Mapping[str, Any]) -> SegmentationPlan:
         adjudicated_at=_require_str(payload, "adjudicated_at", where="the segmentation plan"),
         entries=entries,
     )
+
+
+def plan_cache_key(plan: SegmentationPlan) -> tuple[str, str, str]:
+    """The triple an adjudication is cached under: source AND extractor identity.
+
+    Not the hash alone. `source_sha256` answers "are these the same bytes?",
+    which is necessary and not sufficient: the offsets in a plan index the
+    canonical EXTRACTED text, and swapping the extractor or bumping its version
+    can re-shape that text while the source bytes are untouched. Keyed on the
+    hash alone, a stored adjudication would be replayed against text the
+    adjudicator never saw, and every span would land somewhere plausible and
+    wrong. This is design requirement S5b.
+    """
+    return (plan.source_sha256, plan.extractor_id, plan.extractor_version)
+
+
+def assert_plan_applies(
+    plan: SegmentationPlan,
+    *,
+    source_sha256: str,
+    extractor_id: str,
+    extractor_version: str,
+) -> None:
+    """Refuse loudly when a plan was adjudicated against a different extraction.
+
+    Loudly, and never by re-deriving: a silent fallback would turn "this plan
+    is stale" into "this bundle is subtly wrong", which no test downstream can
+    catch because every span still points at real text. The message names
+    which of the three components moved, because that is what tells the
+    operator whether to re-run the proposer or re-adjudicate by hand.
+    """
+    observed = (source_sha256, extractor_id, extractor_version)
+    differing = [
+        f"{name}: plan {expected!r} != run {actual!r}"
+        for name, expected, actual in zip(CACHE_KEY_COMPONENTS, plan_cache_key(plan), observed)
+        if expected != actual
+    ]
+    if differing:
+        raise SegmentationError(
+            "this segmentation plan was adjudicated against a different extraction "
+            f"({'; '.join(differing)}) — refusing to replay its offsets, which index "
+            "the canonical extracted text and would land on text no one adjudicated; "
+            "re-run the proposer and re-adjudicate",
+            code="segmentation_extractor_mismatch",
+        )

tests/test_segmentation.py

@@ -24,7 +24,9 @@ from llm_ingestion_okf.errors import SegmentationError
 from llm_ingestion_okf.segmentation import (
     SegmentationPlan,
     SegmentEntry,
+    assert_plan_applies,
     parse_segmentation_plan,
+    plan_cache_key,
 )
 
 
@@ -258,3 +260,86 @@ def test_two_paths_differing_only_in_normal_form_collide_as_duplicates() -> None
         )
     )
     assert error.code == "segmentation_path_invalid"
+
+
+# --- S5b: the cache key is the extractor, not the hash alone ---------------
+#
+# Source bytes cannot see an extractor swap or a version bump. Both invalidate
+# every stored offset while `source_sha256` stays identical, so the mismatch
+# has to be LOUD -- a silent re-derivation would replay a human's adjudication
+# against text that human never saw.
+
+
+def parsed_plan(**overrides: Any) -> SegmentationPlan:
+    return parse_segmentation_plan(plan(**overrides))
+
+
+def applies_fails(subject: SegmentationPlan, **overrides: str) -> SegmentationError:
+    arguments = {
+        "source_sha256": subject.source_sha256,
+        "extractor_id": subject.extractor_id,
+        "extractor_version": subject.extractor_version,
+    }
+    arguments.update(overrides)
+    with pytest.raises(SegmentationError) as excinfo:
+        assert_plan_applies(subject, **arguments)
+    return excinfo.value
+
+
+def test_the_cache_key_is_the_three_tuple() -> None:
+    subject = parsed_plan()
+    assert plan_cache_key(subject) == (
+        subject.source_sha256,
+        subject.extractor_id,
+        subject.extractor_version,
+    )
+
+
+def test_two_plans_differing_only_in_extractor_id_have_different_cache_keys() -> None:
+    # The whole point of S5b: the hash alone would call these one cached
+    # adjudication, and replay the first plan's offsets against the second
+    # extraction.
+    one = parsed_plan(extractor_id="text")
+    other = parsed_plan(extractor_id="pdfplumber")
+    assert one.source_sha256 == other.source_sha256
+    assert plan_cache_key(one) != plan_cache_key(other)
+    assert plan_cache_key(one)[0] == plan_cache_key(other)[0]
+
+
+def test_an_identical_triple_applies_without_raising() -> None:
+    subject = parsed_plan()
+    assert (
+        assert_plan_applies(
+            subject,
+            source_sha256=subject.source_sha256,
+            extractor_id=subject.extractor_id,
+            extractor_version=subject.extractor_version,
+        )
+        is None
+    )
+
+
+def test_a_changed_extractor_version_is_refused_and_named() -> None:
+    error = applies_fails(parsed_plan(), extractor_version="1.0.1")
+    assert error.code == "segmentation_extractor_mismatch"
+    assert "extractor_version" in str(error)
+    assert "1.0.1" in str(error)
+
+
+def test_a_changed_extractor_id_is_refused_and_named() -> None:
+    error = applies_fails(parsed_plan(), extractor_id="pdfplumber")
+    assert error.code == "segmentation_extractor_mismatch"
+    assert "extractor_id" in str(error)
+
+
+def test_a_changed_source_hash_is_refused_and_named() -> None:
+    error = applies_fails(parsed_plan(), source_sha256="b" * 64)
+    assert error.code == "segmentation_extractor_mismatch"
+    assert "source_sha256" in str(error)
+
+
+def test_the_message_names_every_differing_component_not_just_the_first() -> None:
+    error = applies_fails(parsed_plan(), extractor_id="pdfplumber", extractor_version="1.0.1")
+    assert "extractor_id" in str(error)
+    assert "extractor_version" in str(error)
+    assert "source_sha256" not in str(error)