feat(inbox): materialize one document into many concepts on hierarchical paths
This commit is contained in:
parent
f6e5ec9305
commit
bb79c55f43
2 changed files with 538 additions and 40 deletions
|
|
@ -25,7 +25,7 @@ from collections.abc import Callable, Mapping
|
|||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
from .errors import IngestError, MaterializationError, SourceError
|
||||
from .errors import IngestError, MaterializationError, SegmentationError, SourceError
|
||||
from .extract import extract_text
|
||||
from .materialize import (
|
||||
_render_root_frontmatter,
|
||||
|
|
@ -37,7 +37,12 @@ from .materialize import (
|
|||
write_bytes,
|
||||
)
|
||||
from .profiles import DEFAULT, BundleProfile
|
||||
from .segmentation import SegmentEntry
|
||||
from .segmentation import (
|
||||
SegmentationPlan,
|
||||
SegmentEntry,
|
||||
assert_plan_applies,
|
||||
slice_segments,
|
||||
)
|
||||
from .structure import (
|
||||
DocumentStructure,
|
||||
_render_flow_list,
|
||||
|
|
@ -270,6 +275,103 @@ def _is_inbox_owned(path: Path) -> bool:
|
|||
return frontmatter.get("generated") == "true" and "source_file" in frontmatter
|
||||
|
||||
|
||||
def _bundle_id_key(profile: BundleProfile) -> str:
|
||||
assert profile.segmentation is not None
|
||||
return profile.segmentation.bundle_id_key
|
||||
|
||||
|
||||
def _plan_covering(plan: SegmentationPlan | None, source_bytes: bytes) -> SegmentationPlan | None:
|
||||
"""The plan for THESE bytes, or None when this file is not plan-covered.
|
||||
|
||||
Selection is by content hash, so a run may drop several documents while
|
||||
only one of them is segmented; every other file keeps today's one-concept
|
||||
rule verbatim. The hash SELECTS; :func:`assert_plan_applies` VALIDATES,
|
||||
and the two are deliberately different questions -- see S5b.
|
||||
"""
|
||||
if plan is None:
|
||||
return None
|
||||
if plan.source_sha256 != hashlib.sha256(source_bytes).hexdigest():
|
||||
return None
|
||||
return plan
|
||||
|
||||
|
||||
def _render_segments(
|
||||
plan: SegmentationPlan,
|
||||
outputs: list[tuple[str, str, tuple[str, ...]]],
|
||||
*,
|
||||
path: Path,
|
||||
text: str,
|
||||
source_bytes: bytes,
|
||||
gate: Gate,
|
||||
profile: BundleProfile,
|
||||
bundle_id: str,
|
||||
) -> BlockedFile | None:
|
||||
"""Render every segment, or refuse the WHOLE document.
|
||||
|
||||
The ordering here is the security property, not a style choice: all N
|
||||
segments are gated and all N decisions collected BEFORE a single byte is
|
||||
written. Gating and writing one at a time would leave a half-screened
|
||||
document on disk the moment segment 3 of 5 quarantines -- part of a
|
||||
document the guard refused, persisted and indexed, with the run reporting
|
||||
success for everything it managed to write first.
|
||||
|
||||
A refusal is therefore reported once, for the document, rather than once
|
||||
per segment: the operator's unit of review is the document they dropped.
|
||||
"""
|
||||
assert_plan_applies(
|
||||
plan,
|
||||
source_sha256=hashlib.sha256(source_bytes).hexdigest(),
|
||||
extractor_id=Path(path.name).suffix.lower().lstrip(".") or "none",
|
||||
# The plan's own value, passed through. Door B can observe WHICH
|
||||
# extractor ran (the suffix is what dispatches it at `extract.py`) but
|
||||
# not the version of a third-party parser -- `pdfplumber`'s transitive
|
||||
# `pdfminer.six` pin is the measured example. Naming the key and
|
||||
# leaving its value to whoever knows it is the same division D5 makes
|
||||
# for `bundle_id`; a fabricated value here would make S5b decorative.
|
||||
extractor_version=plan.extractor_version,
|
||||
)
|
||||
sliced = slice_segments(text, plan)
|
||||
|
||||
decisions = [(entry, gate(body)) for entry, body in sliced]
|
||||
refused = [
|
||||
decision for _, decision in decisions if decision.disposition != _DISPOSITION_PERSIST
|
||||
]
|
||||
if refused:
|
||||
return BlockedFile(
|
||||
source_file=path.name,
|
||||
disposition=refused[0].disposition,
|
||||
reasons=tuple(reason for decision in refused for reason in decision.reasons),
|
||||
)
|
||||
|
||||
for entry, decision in decisions:
|
||||
structure: DocumentStructure | None = None
|
||||
if profile.index.facets is not None:
|
||||
structure = derive_document_structure(decision.sanitized_text, source_file=path.name)
|
||||
_validate_facets(structure, profile)
|
||||
outputs.append(
|
||||
(
|
||||
entry.path,
|
||||
render_inbox_concept(
|
||||
decision.sanitized_text,
|
||||
okf_type=entry.okf_type,
|
||||
# DECLARED by the adjudicator, never derived from the
|
||||
# segment's own first line: the plan is the record of the
|
||||
# judgement, and a heading inside a slice is not it.
|
||||
title=entry.title,
|
||||
source_file=path.name,
|
||||
source_bytes=source_bytes,
|
||||
ingested_at=entry.ingested_at,
|
||||
profile=profile,
|
||||
structure=structure,
|
||||
segment=entry,
|
||||
bundle_id=bundle_id,
|
||||
),
|
||||
decision.reasons,
|
||||
)
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
def process_inbox(
|
||||
inbox_dir: Path,
|
||||
bundle_dir: Path,
|
||||
|
|
@ -279,6 +381,7 @@ def process_inbox(
|
|||
gate: Gate,
|
||||
profile: BundleProfile = DEFAULT,
|
||||
root_frontmatter_values: Mapping[str, str] | None = None,
|
||||
segmentation: SegmentationPlan | None = None,
|
||||
) -> InboxResult:
|
||||
"""Convert every file dropped in `inbox_dir` into an OKF concept.
|
||||
|
||||
|
|
@ -302,6 +405,24 @@ def process_inbox(
|
|||
# `materialize.py` states the same rule for Door A, and a door that half-built
|
||||
# a bundle before refusing would be worse than one that never started.
|
||||
root_head = _render_root_frontmatter(root_frontmatter_values or {}, profile=profile)
|
||||
if segmentation is not None:
|
||||
if profile.segmentation is None:
|
||||
raise SegmentationError(
|
||||
"a segmentation plan was passed to a profile that does not declare the "
|
||||
"segmentation capability — a plan would be silently ignored and the "
|
||||
"document would land as one flat concept, which is the shape this "
|
||||
"capability exists to replace",
|
||||
code="segmentation_unsupported_profile",
|
||||
)
|
||||
if profile.segmentation.bundle_id_key not in (root_frontmatter_values or {}):
|
||||
raise SegmentationError(
|
||||
f"a segmentation plan requires {profile.segmentation.bundle_id_key!r} in "
|
||||
"root_frontmatter_values — a segmented bundle's concept paths collide "
|
||||
"with any other bundle built from the same plan, so the bundle "
|
||||
"identifier is what a consumer joins on; refusing to write concepts "
|
||||
"nothing can tell apart",
|
||||
code="segmentation_plan_invalid",
|
||||
)
|
||||
run_rejection = profile.types.rejection(okf_type)
|
||||
if run_rejection is not None:
|
||||
raise MaterializationError(f"okf_type {run_rejection.reason}", code=run_rejection.code)
|
||||
|
|
@ -376,45 +497,71 @@ def process_inbox(
|
|||
)
|
||||
)
|
||||
continue
|
||||
outputs: list[tuple[str, str, tuple[str, ...]]] = []
|
||||
try:
|
||||
source_bytes = path.read_bytes()
|
||||
text = extract_text(path.name, source_bytes)
|
||||
decision = gate(text)
|
||||
if decision.disposition != _DISPOSITION_PERSIST:
|
||||
blocked = BlockedFile(
|
||||
source_file=path.name,
|
||||
disposition=decision.disposition,
|
||||
reasons=decision.reasons,
|
||||
covering = _plan_covering(segmentation, source_bytes)
|
||||
if covering is not None:
|
||||
blocked = _render_segments(
|
||||
covering,
|
||||
outputs,
|
||||
path=path,
|
||||
text=text,
|
||||
source_bytes=source_bytes,
|
||||
gate=gate,
|
||||
profile=profile,
|
||||
bundle_id=(root_frontmatter_values or {})[_bundle_id_key(profile)],
|
||||
)
|
||||
# Quarantine is a queue for the operator; everything else —
|
||||
# fail-secure, or a disposition from outside the pinned range —
|
||||
# is a refusal. Unknown values land here by construction.
|
||||
if decision.disposition == _DISPOSITION_QUARANTINE:
|
||||
quarantined.append(blocked)
|
||||
else:
|
||||
rejected.append(blocked)
|
||||
continue
|
||||
structure: DocumentStructure | None = None
|
||||
title = unicodedata.normalize("NFC", path.stem)
|
||||
if profile.index.facets is not None:
|
||||
# Derived from the SANITIZED text, never the extracted text:
|
||||
# deriving from bytes the gate rejected would put unscreened
|
||||
# content in the frontmatter and the index.
|
||||
structure = derive_document_structure(
|
||||
decision.sanitized_text, source_file=path.name
|
||||
if blocked is not None:
|
||||
if blocked.disposition == _DISPOSITION_QUARANTINE:
|
||||
quarantined.append(blocked)
|
||||
else:
|
||||
rejected.append(blocked)
|
||||
continue
|
||||
else:
|
||||
decision = gate(text)
|
||||
if decision.disposition != _DISPOSITION_PERSIST:
|
||||
blocked = BlockedFile(
|
||||
source_file=path.name,
|
||||
disposition=decision.disposition,
|
||||
reasons=decision.reasons,
|
||||
)
|
||||
# Quarantine is a queue for the operator; everything else —
|
||||
# fail-secure, or a disposition from outside the pinned range —
|
||||
# is a refusal. Unknown values land here by construction.
|
||||
if decision.disposition == _DISPOSITION_QUARANTINE:
|
||||
quarantined.append(blocked)
|
||||
else:
|
||||
rejected.append(blocked)
|
||||
continue
|
||||
structure: DocumentStructure | None = None
|
||||
title = unicodedata.normalize("NFC", path.stem)
|
||||
if profile.index.facets is not None:
|
||||
# Derived from the SANITIZED text, never the extracted text:
|
||||
# deriving from bytes the gate rejected would put unscreened
|
||||
# content in the frontmatter and the index.
|
||||
structure = derive_document_structure(
|
||||
decision.sanitized_text, source_file=path.name
|
||||
)
|
||||
title = structure.title
|
||||
_validate_facets(structure, profile)
|
||||
outputs.append(
|
||||
(
|
||||
name,
|
||||
render_inbox_concept(
|
||||
decision.sanitized_text,
|
||||
okf_type=okf_type,
|
||||
title=title,
|
||||
source_file=path.name,
|
||||
source_bytes=source_bytes,
|
||||
ingested_at=ingested_at,
|
||||
profile=profile,
|
||||
structure=structure,
|
||||
),
|
||||
decision.reasons,
|
||||
)
|
||||
)
|
||||
title = structure.title
|
||||
_validate_facets(structure, profile)
|
||||
content = render_inbox_concept(
|
||||
decision.sanitized_text,
|
||||
okf_type=okf_type,
|
||||
title=title,
|
||||
source_file=path.name,
|
||||
source_bytes=source_bytes,
|
||||
ingested_at=ingested_at,
|
||||
profile=profile,
|
||||
structure=structure,
|
||||
)
|
||||
except OSError as exc:
|
||||
failed.append(
|
||||
FailedFile(
|
||||
|
|
@ -430,10 +577,12 @@ def process_inbox(
|
|||
continue
|
||||
|
||||
bundle.mkdir(parents=True, exist_ok=True)
|
||||
written = write_bytes(bundle, name, content)
|
||||
persisted.append(
|
||||
PersistedFile(source_file=path.name, path=written, reasons=decision.reasons)
|
||||
)
|
||||
for target_name, content, reasons in outputs:
|
||||
# `write_bytes` resolves a subpath through `safe_resolve` but never
|
||||
# creates one. Without this the very first hierarchical write fails.
|
||||
(bundle / target_name).parent.mkdir(parents=True, exist_ok=True)
|
||||
written = write_bytes(bundle, target_name, content)
|
||||
persisted.append(PersistedFile(source_file=path.name, path=written, reasons=reasons))
|
||||
|
||||
# §6 index — the last disk mutation, and only when something was written.
|
||||
if persisted:
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue