fix(segmentation): hash the extracted text and let the plan key fire

This commit is contained in:
Kjell Tore Guttormsen 2026-09-02 14:38:20 +02:00
commit 9e9bb8645d
13 changed files with 272 additions and 40 deletions

View file

@ -57,6 +57,7 @@ sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
from llm_ingestion_okf.errors import IngestError # noqa: E402
from llm_ingestion_okf.extract import extract_text # noqa: E402
from llm_ingestion_okf.materialize import reduce_to_id_grammar # noqa: E402
from llm_ingestion_okf.segmentation import observed_extractor_version # noqa: E402
#: Stamped into every entry's `derived` list. The marker is what keeps a
#: proposal from being mistaken for the judgement the run path replays.
@ -266,6 +267,7 @@ def build_plan(
) -> dict[str, Any]:
"""The artifact. Every entry PROPOSED, the plan itself never adjudicated."""
taken: set[str] = set()
extractor_id = source.suffix.lower().lstrip(".") or "none"
entries: list[dict[str, Any]] = []
for candidate in find_candidates(text):
entries.append(
@ -286,8 +288,15 @@ def build_plan(
return {
"version": "1",
"source_sha256": hashlib.sha256(source_bytes).hexdigest(),
"extractor_id": source.suffix.lower().lstrip(".") or "none",
"extractor_version": PROPOSER_VERSION,
# The hash the offsets actually depend on. Source bytes alone cannot
# see a converter reshaping its output, so the staleness signal this
# plan is supposed to carry did not exist until this line did.
"text_sha256": hashlib.sha256(text.encode("utf-8")).hexdigest(),
"extractor_id": extractor_id,
# The EXTRACTOR's version, not this tool's. `PROPOSER_VERSION` sat here
# and named the wrong thing: a converter bump left the field frozen at
# the proposer's own number, so the component could not move.
"extractor_version": observed_extractor_version(extractor_id),
"adjudicated_at": proposed_at,
# NOT a timestamp question. `adjudicated_at` records when this artifact
# was produced; this records whether a human has looked at it, and it is