open/llm-ingestion-okf
1
0
Fork 0
Code Issues Releases 1 Activity

fix(segmentation): hash the extracted text and let the plan key fire

Browse source
This commit is contained in:
Kjell Tore Guttormsen 2026-09-02 14:38:20 +02:00
commit 9e9bb8645d
13 changed files with 272 additions and 40 deletions
Download patch file Download diff file Expand all files Collapse all files

17
tests/test_segmented_index.py
View file

@ -27,7 +27,12 @@ from typing import Any
from llm_ingestion_okf.inbox import GateDecision, process_inbox
from llm_ingestion_okf.profiles import DEFAULT, SEGMENTED_V1, STRUCTURED_V1
from llm_ingestion_okf.segmentation import SegmentationPlan, parse_segmentation_plan
from llm_ingestion_okf.extract import extract_text
from llm_ingestion_okf.segmentation import (
SegmentationPlan,
observed_extractor_version,
parse_segmentation_plan,
)
INGESTED_AT = "2026-07-25T12:00:00Z"
PLAN_AT = "2026-08-30T09:00:00Z"
@ -54,12 +59,20 @@ def drop(inbox: Path, name: str, text: str = DOCUMENT) -> Path:
return path
def _extracted_text_sha256(source_bytes: bytes, filename: str = "n500.md") -> str:
return hashlib.sha256(extract_text(filename, source_bytes).encode("utf-8")).hexdigest()
def build_plan(source_bytes: bytes, paths: tuple[str, ...] = PATHS, **overrides: Any):
payload: dict[str, Any] = {
"version": "1",
"source_sha256": hashlib.sha256(source_bytes).hexdigest(),
# The hash of the CANONICAL EXTRACTED text, which is what the spans
# index. Equal to the source hash on a `.md` passthrough and computed
# rather than copied, so the fixture keeps saying which one it means.
"text_sha256": _extracted_text_sha256(source_bytes),
"extractor_id": "md",
"extractor_version": "1.0.0",
"extractor_version": observed_extractor_version("md"),
"adjudicated_at": "2026-08-30T08:00:00Z",
"entries": [
{