fix(segmentation): hash the extracted text and let the plan key fire

This commit is contained in:
Kjell Tore Guttormsen 2026-09-02 14:38:20 +02:00
commit 9e9bb8645d
13 changed files with 272 additions and 40 deletions

View file

@ -41,7 +41,12 @@ import pytest
from llm_ingestion_okf.errors import SegmentationError
from llm_ingestion_okf.inbox import GateDecision, process_inbox
from llm_ingestion_okf.profiles import SEGMENTED_V1
from llm_ingestion_okf.segmentation import SegmentationPlan, parse_segmentation_plan
from llm_ingestion_okf.extract import extract_text
from llm_ingestion_okf.segmentation import (
SegmentationPlan,
observed_extractor_version,
parse_segmentation_plan,
)
# THREE distinct call-level values. None of them may reach a plan-covered
# concept, and the test is worthless if they are all the same.
@ -80,6 +85,10 @@ def drop(inbox: Path, name: str = "n500.md", text: str = DOCUMENT) -> Path:
return path
def _extracted_text_sha256(source_bytes: bytes, filename: str = "n500.md") -> str:
return hashlib.sha256(extract_text(filename, source_bytes).encode("utf-8")).hexdigest()
def build_plan(
source_bytes: bytes,
entries: tuple[tuple[str, str, str | None], ...],
@ -88,8 +97,12 @@ def build_plan(
payload: dict[str, Any] = {
"version": "1",
"source_sha256": hashlib.sha256(source_bytes).hexdigest(),
# The hash of the CANONICAL EXTRACTED text, which is what the spans
# index. Equal to the source hash on a `.md` passthrough and computed
# rather than copied, so the fixture keeps saying which one it means.
"text_sha256": _extracted_text_sha256(source_bytes),
"extractor_id": "md",
"extractor_version": "1.0.0",
"extractor_version": observed_extractor_version("md"),
"adjudicated_at": "2026-08-30T08:00:00Z",
"entries": [
{