fix(segmentation): hash the extracted text and let the plan key fire
This commit is contained in:
parent
6dce4355be
commit
9e9bb8645d
13 changed files with 272 additions and 40 deletions
|
|
@ -57,6 +57,7 @@ sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
|
|||
from llm_ingestion_okf.errors import IngestError # noqa: E402
|
||||
from llm_ingestion_okf.extract import extract_text # noqa: E402
|
||||
from llm_ingestion_okf.materialize import reduce_to_id_grammar # noqa: E402
|
||||
from llm_ingestion_okf.segmentation import observed_extractor_version # noqa: E402
|
||||
|
||||
#: Stamped into every entry's `derived` list. The marker is what keeps a
|
||||
#: proposal from being mistaken for the judgement the run path replays.
|
||||
|
|
@ -266,6 +267,7 @@ def build_plan(
|
|||
) -> dict[str, Any]:
|
||||
"""The artifact. Every entry PROPOSED, the plan itself never adjudicated."""
|
||||
taken: set[str] = set()
|
||||
extractor_id = source.suffix.lower().lstrip(".") or "none"
|
||||
entries: list[dict[str, Any]] = []
|
||||
for candidate in find_candidates(text):
|
||||
entries.append(
|
||||
|
|
@ -286,8 +288,15 @@ def build_plan(
|
|||
return {
|
||||
"version": "1",
|
||||
"source_sha256": hashlib.sha256(source_bytes).hexdigest(),
|
||||
"extractor_id": source.suffix.lower().lstrip(".") or "none",
|
||||
"extractor_version": PROPOSER_VERSION,
|
||||
# The hash the offsets actually depend on. Source bytes alone cannot
|
||||
# see a converter reshaping its output, so the staleness signal this
|
||||
# plan is supposed to carry did not exist until this line did.
|
||||
"text_sha256": hashlib.sha256(text.encode("utf-8")).hexdigest(),
|
||||
"extractor_id": extractor_id,
|
||||
# The EXTRACTOR's version, not this tool's. `PROPOSER_VERSION` sat here
|
||||
# and named the wrong thing: a converter bump left the field frozen at
|
||||
# the proposer's own number, so the component could not move.
|
||||
"extractor_version": observed_extractor_version(extractor_id),
|
||||
"adjudicated_at": proposed_at,
|
||||
# NOT a timestamp question. `adjudicated_at` records when this artifact
|
||||
# was produced; this records whether a human has looked at it, and it is
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue