open/llm-ingestion-okf
1
0
Fork 0
Code Issues Releases 1 Activity

fix(segmentation): hash the extracted text and let the plan key fire

Browse source
This commit is contained in:
Kjell Tore Guttormsen 2026-09-02 14:38:20 +02:00
commit 9e9bb8645d
13 changed files with 272 additions and 40 deletions
Download patch file Download diff file Expand all files Collapse all files

2
tests/test_error_codes.py
View file

@ -601,6 +601,7 @@ def segmentation_payload(**overrides: Any) -> dict[str, Any]:
payload: dict[str, Any] = {
"version": "1",
"source_sha256": "a" * 64,
"text_sha256": "c" * 64,
"extractor_id": "text",
"extractor_version": "1.0.0",
"adjudicated_at": INGESTED_AT,
@ -664,6 +665,7 @@ def test_segmentation_extractor_mismatch() -> None:
assert_plan_applies(
plan,
source_sha256="b" * 64,
text_sha256=plan.text_sha256,
extractor_id=plan.extractor_id,
extractor_version=plan.extractor_version,
)

23
tests/test_segmentation.py
View file

@ -48,6 +48,7 @@ def plan(**overrides: Any) -> dict[str, Any]:
payload: dict[str, Any] = {
"version": "1",
"source_sha256": "a" * 64,
"text_sha256": "c" * 64,
"extractor_id": "text",
"extractor_version": "1.0.0",
"adjudicated_at": "2026-08-31T11:00:00Z",
@ -278,6 +279,7 @@ def parsed_plan(**overrides: Any) -> SegmentationPlan:
def applies_fails(subject: SegmentationPlan, **overrides: str) -> SegmentationError:
arguments = {
"source_sha256": subject.source_sha256,
"text_sha256": subject.text_sha256,
"extractor_id": subject.extractor_id,
"extractor_version": subject.extractor_version,
}
@ -287,10 +289,11 @@ def applies_fails(subject: SegmentationPlan, **overrides: str) -> SegmentationEr
return excinfo.value
def test_the_cache_key_is_the_three_tuple() -> None:
def test_the_cache_key_is_the_four_tuple() -> None:
subject = parsed_plan()
assert plan_cache_key(subject) == (
subject.source_sha256,
subject.text_sha256,
subject.extractor_id,
subject.extractor_version,
)
@ -307,12 +310,13 @@ def test_two_plans_differing_only_in_extractor_id_have_different_cache_keys() ->
assert plan_cache_key(one)[0] == plan_cache_key(other)[0]
def test_an_identical_triple_applies_without_raising() -> None:
def test_an_identical_quadruple_applies_without_raising() -> None:
subject = parsed_plan()
assert (
assert_plan_applies(
subject,
source_sha256=subject.source_sha256,
text_sha256=subject.text_sha256,
extractor_id=subject.extractor_id,
extractor_version=subject.extractor_version,
)
@ -320,6 +324,21 @@ def test_an_identical_triple_applies_without_raising() -> None:
)
def test_a_changed_text_hash_is_refused_although_the_source_bytes_match() -> None:
"""The component the other three cannot stand in for.
Same bytes, same extractor, same version -- and a canonical text that
moved anyway, which is what a converter reshaping its output without
bumping its version looks like from here. Before this component existed
the key matched and every offset was replayed against text nobody
adjudicated, with each span still landing on real characters.
"""
error = applies_fails(parsed_plan(), text_sha256="d" * 64)
assert error.code == "segmentation_extractor_mismatch"
assert "text_sha256" in str(error)
assert "source_sha256" not in str(error)
def test_a_changed_extractor_version_is_refused_and_named() -> None:
error = applies_fails(parsed_plan(), extractor_version="1.0.1")
assert error.code == "segmentation_extractor_mismatch"

17
tests/test_segmented_collisions.py
View file

@ -27,7 +27,12 @@ from typing import Any
from llm_ingestion_okf.inbox import GateDecision, process_inbox
from llm_ingestion_okf.materialize import NAME_MAX_BYTES
from llm_ingestion_okf.profiles import DEFAULT, SEGMENTED_V1
from llm_ingestion_okf.segmentation import SegmentationPlan, parse_segmentation_plan
from llm_ingestion_okf.extract import extract_text
from llm_ingestion_okf.segmentation import (
SegmentationPlan,
observed_extractor_version,
parse_segmentation_plan,
)
INGESTED_AT = "2026-07-25T12:00:00Z"
PLAN_AT = "2026-08-30T09:00:00Z"
@ -49,12 +54,20 @@ def drop(inbox: Path, name: str, text: str = DOCUMENT) -> Path:
return path
def _extracted_text_sha256(source_bytes: bytes, filename: str = "n500.md") -> str:
return hashlib.sha256(extract_text(filename, source_bytes).encode("utf-8")).hexdigest()
def build_plan(source_bytes: bytes, paths: tuple[str, ...], **overrides: Any) -> SegmentationPlan:
payload: dict[str, Any] = {
"version": "1",
"source_sha256": hashlib.sha256(source_bytes).hexdigest(),
# The hash of the CANONICAL EXTRACTED text, which is what the spans
# index. Equal to the source hash on a `.md` passthrough and computed
# rather than copied, so the fixture keeps saying which one it means.
"text_sha256": _extracted_text_sha256(source_bytes),
"extractor_id": "md",
"extractor_version": "1.0.0",
"extractor_version": observed_extractor_version("md"),
"adjudicated_at": "2026-08-30T08:00:00Z",
"entries": [
{

2
tests/test_segmented_identity.py
View file

@ -196,6 +196,7 @@ def segment_entry(**overrides: object) -> SegmentEntry:
{
"version": "1",
"source_sha256": "a" * 64,
"text_sha256": "c" * 64,
"extractor_id": "text",
"extractor_version": "1.0.0",
"adjudicated_at": "2026-08-30T08:00:00Z",
@ -247,6 +248,7 @@ def test_a_declared_parent_is_mirrored_and_a_flat_segment_carries_none() -> None
{
"version": "1",
"source_sha256": "a" * 64,
"text_sha256": "c" * 64,
"extractor_id": "text",
"extractor_version": "1.0.0",
"adjudicated_at": "2026-08-30T08:00:00Z",

62
tests/test_segmented_inbox.py
View file

@ -35,7 +35,12 @@ from llm_ingestion_okf.errors import SegmentationError
from llm_ingestion_okf.extract import extract_text
from llm_ingestion_okf.inbox import GateDecision, process_inbox
from llm_ingestion_okf.profiles import DEFAULT, SEGMENTED_V1
from llm_ingestion_okf.segmentation import SegmentationPlan, parse_segmentation_plan
from llm_ingestion_okf.segmentation import (
STDLIB_EXTRACTOR_VERSION,
SegmentationPlan,
observed_extractor_version,
parse_segmentation_plan,
)
INGESTED_AT = "2026-07-25T12:00:00Z"
PLAN_AT = "2026-08-30T09:00:00Z"
@ -90,8 +95,9 @@ def build_plan(
payload: dict[str, Any] = {
"version": "1",
"source_sha256": hashlib.sha256(source_bytes).hexdigest(),
"text_sha256": hashlib.sha256(text.encode("utf-8")).hexdigest(),
"extractor_id": extractor_id,
"extractor_version": "1.0.0",
"extractor_version": observed_extractor_version(extractor_id),
"adjudicated_at": "2026-08-30T08:00:00Z",
"entries": [
{
@ -441,3 +447,55 @@ def test_without_the_capability_an_unmatched_plan_is_still_the_earlier_refusal(
with pytest.raises(SegmentationError) as excinfo:
run(tmp_path, plan=plan, profile=DEFAULT, values={})
assert excinfo.value.code == "segmentation_unsupported_profile"
# --- S5b: the cache key can actually fail ----------------------------------
#
# Both halves below were decorative before Step 11. The proposer hashed SOURCE
# BYTES only, so a converter that reshaped the extracted text left the hash
# identical and every offset moved under a key that still matched; and the run
# path passed `plan.extractor_version` straight back into `assert_plan_applies`,
# comparing the plan's value with itself. Two guards that could never fire, in
# the one place where a false pass produces a bundle nobody adjudicated and no
# downstream test can catch -- every span still lands on real text.
def test_a_plan_whose_extracted_text_hash_moved_is_refused(tmp_path: Path) -> None:
"""The signal a source-bytes hash cannot carry.
Same bytes on disk, same extractor id, same extractor version -- and a
different canonical text, which is what the offsets index. Simulated by
moving the hash rather than the converter, because the property under test
is that the component is COMPARED at all.
"""
source = drop(tmp_path / "round", "n500.md", DOCUMENT)
plan = build_plan(source.read_bytes(), DOCUMENT, text_sha256="0" * 64)
result = run(tmp_path, plan=plan)
assert {entry.error.code for entry in result.failed} == {"segmentation_extractor_mismatch"}
assert "text_sha256" in str(result.failed[0].error)
assert tree(tmp_path / "bundle") == {}
def test_a_plan_whose_extractor_version_moved_is_refused(tmp_path: Path) -> None:
"""The half of S5b that compared a value with itself."""
source = drop(tmp_path / "round", "n500.md", DOCUMENT)
plan = build_plan(source.read_bytes(), DOCUMENT, extractor_version="not-the-one-that-ran")
result = run(tmp_path, plan=plan)
assert {entry.error.code for entry in result.failed} == {"segmentation_extractor_mismatch"}
assert "extractor_version" in str(result.failed[0].error)
assert tree(tmp_path / "bundle") == {}
def test_the_observed_extractor_version_is_not_the_proposers_own(tmp_path: Path) -> None:
"""Defect (b): the proposer wrote ITS version into the extractor's field.
A stdlib row names this package's own literal because there is no third
party to name; a converter row names the pinned converter. What matters is
that the two are DIFFERENT values from different sources -- one tool
version standing in for both is exactly what made the field unable to move.
"""
assert observed_extractor_version("md") == STDLIB_EXTRACTOR_VERSION
assert observed_extractor_version("docx") != STDLIB_EXTRACTOR_VERSION
with pytest.raises(SegmentationError) as excinfo:
observed_extractor_version("nothing-registers-this")
assert excinfo.value.code == "segmentation_extractor_mismatch"

17
tests/test_segmented_index.py
View file

@ -27,7 +27,12 @@ from typing import Any
from llm_ingestion_okf.inbox import GateDecision, process_inbox
from llm_ingestion_okf.profiles import DEFAULT, SEGMENTED_V1, STRUCTURED_V1
from llm_ingestion_okf.segmentation import SegmentationPlan, parse_segmentation_plan
from llm_ingestion_okf.extract import extract_text
from llm_ingestion_okf.segmentation import (
SegmentationPlan,
observed_extractor_version,
parse_segmentation_plan,
)
INGESTED_AT = "2026-07-25T12:00:00Z"
PLAN_AT = "2026-08-30T09:00:00Z"
@ -54,12 +59,20 @@ def drop(inbox: Path, name: str, text: str = DOCUMENT) -> Path:
return path
def _extracted_text_sha256(source_bytes: bytes, filename: str = "n500.md") -> str:
return hashlib.sha256(extract_text(filename, source_bytes).encode("utf-8")).hexdigest()
def build_plan(source_bytes: bytes, paths: tuple[str, ...] = PATHS, **overrides: Any):
payload: dict[str, Any] = {
"version": "1",
"source_sha256": hashlib.sha256(source_bytes).hexdigest(),
# The hash of the CANONICAL EXTRACTED text, which is what the spans
# index. Equal to the source hash on a `.md` passthrough and computed
# rather than copied, so the fixture keeps saying which one it means.
"text_sha256": _extracted_text_sha256(source_bytes),
"extractor_id": "md",
"extractor_version": "1.0.0",
"extractor_version": observed_extractor_version("md"),
"adjudicated_at": "2026-08-30T08:00:00Z",
"entries": [
{

17
tests/test_segmented_rebuild.py
View file

@ -41,7 +41,12 @@ import pytest
from llm_ingestion_okf.errors import SegmentationError
from llm_ingestion_okf.inbox import GateDecision, process_inbox
from llm_ingestion_okf.profiles import SEGMENTED_V1
from llm_ingestion_okf.segmentation import SegmentationPlan, parse_segmentation_plan
from llm_ingestion_okf.extract import extract_text
from llm_ingestion_okf.segmentation import (
SegmentationPlan,
observed_extractor_version,
parse_segmentation_plan,
)
# THREE distinct call-level values. None of them may reach a plan-covered
# concept, and the test is worthless if they are all the same.
@ -80,6 +85,10 @@ def drop(inbox: Path, name: str = "n500.md", text: str = DOCUMENT) -> Path:
return path
def _extracted_text_sha256(source_bytes: bytes, filename: str = "n500.md") -> str:
return hashlib.sha256(extract_text(filename, source_bytes).encode("utf-8")).hexdigest()
def build_plan(
source_bytes: bytes,
entries: tuple[tuple[str, str, str | None], ...],
@ -88,8 +97,12 @@ def build_plan(
payload: dict[str, Any] = {
"version": "1",
"source_sha256": hashlib.sha256(source_bytes).hexdigest(),
# The hash of the CANONICAL EXTRACTED text, which is what the spans
# index. Equal to the source hash on a `.md` passthrough and computed
# rather than copied, so the fixture keeps saying which one it means.
"text_sha256": _extracted_text_sha256(source_bytes),
"extractor_id": "md",
"extractor_version": "1.0.0",
"extractor_version": observed_extractor_version("md"),
"adjudicated_at": "2026-08-30T08:00:00Z",
"entries": [
{

17
tests/test_segmented_rounds.py
View file

@ -38,7 +38,12 @@ from typing import Any
from llm_ingestion_okf.inbox import GateDecision, process_inbox
from llm_ingestion_okf.profiles import DEFAULT, SEGMENTED_V1
from llm_ingestion_okf.segmentation import SegmentationPlan, parse_segmentation_plan
from llm_ingestion_okf.extract import extract_text
from llm_ingestion_okf.segmentation import (
SegmentationPlan,
observed_extractor_version,
parse_segmentation_plan,
)
INGESTED_AT = "2026-07-25T12:00:00Z"
PLAN_AT = "2026-08-30T09:00:00Z"
@ -57,6 +62,10 @@ def drop(inbox: Path, name: str, text: str = DOCUMENT) -> Path:
return path
def _extracted_text_sha256(source_bytes: bytes, filename: str = "n500.md") -> str:
return hashlib.sha256(extract_text(filename, source_bytes).encode("utf-8")).hexdigest()
def build_plan(
source_bytes: bytes,
entries: tuple[tuple[str, str, str | None], ...],
@ -65,8 +74,12 @@ def build_plan(
payload: dict[str, Any] = {
"version": "1",
"source_sha256": hashlib.sha256(source_bytes).hexdigest(),
# The hash of the CANONICAL EXTRACTED text, which is what the spans
# index. Equal to the source hash on a `.md` passthrough and computed
# rather than copied, so the fixture keeps saying which one it means.
"text_sha256": _extracted_text_sha256(source_bytes),
"extractor_id": "md",
"extractor_version": "1.0.0",
"extractor_version": observed_extractor_version("md"),
"adjudicated_at": "2026-08-30T08:00:00Z",
"entries": [
{