fix(segmentation): hash the extracted text and let the plan key fire
This commit is contained in:
parent
6dce4355be
commit
9e9bb8645d
13 changed files with 272 additions and 40 deletions
|
|
@ -41,7 +41,12 @@ import pytest
|
|||
from llm_ingestion_okf.errors import SegmentationError
|
||||
from llm_ingestion_okf.inbox import GateDecision, process_inbox
|
||||
from llm_ingestion_okf.profiles import SEGMENTED_V1
|
||||
from llm_ingestion_okf.segmentation import SegmentationPlan, parse_segmentation_plan
|
||||
from llm_ingestion_okf.extract import extract_text
|
||||
from llm_ingestion_okf.segmentation import (
|
||||
SegmentationPlan,
|
||||
observed_extractor_version,
|
||||
parse_segmentation_plan,
|
||||
)
|
||||
|
||||
# THREE distinct call-level values. None of them may reach a plan-covered
|
||||
# concept, and the test is worthless if they are all the same.
|
||||
|
|
@ -80,6 +85,10 @@ def drop(inbox: Path, name: str = "n500.md", text: str = DOCUMENT) -> Path:
|
|||
return path
|
||||
|
||||
|
||||
def _extracted_text_sha256(source_bytes: bytes, filename: str = "n500.md") -> str:
|
||||
return hashlib.sha256(extract_text(filename, source_bytes).encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def build_plan(
|
||||
source_bytes: bytes,
|
||||
entries: tuple[tuple[str, str, str | None], ...],
|
||||
|
|
@ -88,8 +97,12 @@ def build_plan(
|
|||
payload: dict[str, Any] = {
|
||||
"version": "1",
|
||||
"source_sha256": hashlib.sha256(source_bytes).hexdigest(),
|
||||
# The hash of the CANONICAL EXTRACTED text, which is what the spans
|
||||
# index. Equal to the source hash on a `.md` passthrough and computed
|
||||
# rather than copied, so the fixture keeps saying which one it means.
|
||||
"text_sha256": _extracted_text_sha256(source_bytes),
|
||||
"extractor_id": "md",
|
||||
"extractor_version": "1.0.0",
|
||||
"extractor_version": observed_extractor_version("md"),
|
||||
"adjudicated_at": "2026-08-30T08:00:00Z",
|
||||
"entries": [
|
||||
{
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue