feat(inbox): materialize one document into many concepts on hierarchical paths

This commit is contained in:
Kjell Tore Guttormsen 2026-09-01 00:10:13 +02:00
commit bb79c55f43
2 changed files with 538 additions and 40 deletions

View file

@ -25,7 +25,7 @@ from collections.abc import Callable, Mapping
from dataclasses import dataclass from dataclasses import dataclass
from pathlib import Path from pathlib import Path
from .errors import IngestError, MaterializationError, SourceError from .errors import IngestError, MaterializationError, SegmentationError, SourceError
from .extract import extract_text from .extract import extract_text
from .materialize import ( from .materialize import (
_render_root_frontmatter, _render_root_frontmatter,
@ -37,7 +37,12 @@ from .materialize import (
write_bytes, write_bytes,
) )
from .profiles import DEFAULT, BundleProfile from .profiles import DEFAULT, BundleProfile
from .segmentation import SegmentEntry from .segmentation import (
SegmentationPlan,
SegmentEntry,
assert_plan_applies,
slice_segments,
)
from .structure import ( from .structure import (
DocumentStructure, DocumentStructure,
_render_flow_list, _render_flow_list,
@ -270,6 +275,103 @@ def _is_inbox_owned(path: Path) -> bool:
return frontmatter.get("generated") == "true" and "source_file" in frontmatter return frontmatter.get("generated") == "true" and "source_file" in frontmatter
def _bundle_id_key(profile: BundleProfile) -> str:
assert profile.segmentation is not None
return profile.segmentation.bundle_id_key
def _plan_covering(plan: SegmentationPlan | None, source_bytes: bytes) -> SegmentationPlan | None:
"""The plan for THESE bytes, or None when this file is not plan-covered.
Selection is by content hash, so a run may drop several documents while
only one of them is segmented; every other file keeps today's one-concept
rule verbatim. The hash SELECTS; :func:`assert_plan_applies` VALIDATES,
and the two are deliberately different questions -- see S5b.
"""
if plan is None:
return None
if plan.source_sha256 != hashlib.sha256(source_bytes).hexdigest():
return None
return plan
def _render_segments(
plan: SegmentationPlan,
outputs: list[tuple[str, str, tuple[str, ...]]],
*,
path: Path,
text: str,
source_bytes: bytes,
gate: Gate,
profile: BundleProfile,
bundle_id: str,
) -> BlockedFile | None:
"""Render every segment, or refuse the WHOLE document.
The ordering here is the security property, not a style choice: all N
segments are gated and all N decisions collected BEFORE a single byte is
written. Gating and writing one at a time would leave a half-screened
document on disk the moment segment 3 of 5 quarantines -- part of a
document the guard refused, persisted and indexed, with the run reporting
success for everything it managed to write first.
A refusal is therefore reported once, for the document, rather than once
per segment: the operator's unit of review is the document they dropped.
"""
assert_plan_applies(
plan,
source_sha256=hashlib.sha256(source_bytes).hexdigest(),
extractor_id=Path(path.name).suffix.lower().lstrip(".") or "none",
# The plan's own value, passed through. Door B can observe WHICH
# extractor ran (the suffix is what dispatches it at `extract.py`) but
# not the version of a third-party parser -- `pdfplumber`'s transitive
# `pdfminer.six` pin is the measured example. Naming the key and
# leaving its value to whoever knows it is the same division D5 makes
# for `bundle_id`; a fabricated value here would make S5b decorative.
extractor_version=plan.extractor_version,
)
sliced = slice_segments(text, plan)
decisions = [(entry, gate(body)) for entry, body in sliced]
refused = [
decision for _, decision in decisions if decision.disposition != _DISPOSITION_PERSIST
]
if refused:
return BlockedFile(
source_file=path.name,
disposition=refused[0].disposition,
reasons=tuple(reason for decision in refused for reason in decision.reasons),
)
for entry, decision in decisions:
structure: DocumentStructure | None = None
if profile.index.facets is not None:
structure = derive_document_structure(decision.sanitized_text, source_file=path.name)
_validate_facets(structure, profile)
outputs.append(
(
entry.path,
render_inbox_concept(
decision.sanitized_text,
okf_type=entry.okf_type,
# DECLARED by the adjudicator, never derived from the
# segment's own first line: the plan is the record of the
# judgement, and a heading inside a slice is not it.
title=entry.title,
source_file=path.name,
source_bytes=source_bytes,
ingested_at=entry.ingested_at,
profile=profile,
structure=structure,
segment=entry,
bundle_id=bundle_id,
),
decision.reasons,
)
)
return None
def process_inbox( def process_inbox(
inbox_dir: Path, inbox_dir: Path,
bundle_dir: Path, bundle_dir: Path,
@ -279,6 +381,7 @@ def process_inbox(
gate: Gate, gate: Gate,
profile: BundleProfile = DEFAULT, profile: BundleProfile = DEFAULT,
root_frontmatter_values: Mapping[str, str] | None = None, root_frontmatter_values: Mapping[str, str] | None = None,
segmentation: SegmentationPlan | None = None,
) -> InboxResult: ) -> InboxResult:
"""Convert every file dropped in `inbox_dir` into an OKF concept. """Convert every file dropped in `inbox_dir` into an OKF concept.
@ -302,6 +405,24 @@ def process_inbox(
# `materialize.py` states the same rule for Door A, and a door that half-built # `materialize.py` states the same rule for Door A, and a door that half-built
# a bundle before refusing would be worse than one that never started. # a bundle before refusing would be worse than one that never started.
root_head = _render_root_frontmatter(root_frontmatter_values or {}, profile=profile) root_head = _render_root_frontmatter(root_frontmatter_values or {}, profile=profile)
if segmentation is not None:
if profile.segmentation is None:
raise SegmentationError(
"a segmentation plan was passed to a profile that does not declare the "
"segmentation capability — a plan would be silently ignored and the "
"document would land as one flat concept, which is the shape this "
"capability exists to replace",
code="segmentation_unsupported_profile",
)
if profile.segmentation.bundle_id_key not in (root_frontmatter_values or {}):
raise SegmentationError(
f"a segmentation plan requires {profile.segmentation.bundle_id_key!r} in "
"root_frontmatter_values — a segmented bundle's concept paths collide "
"with any other bundle built from the same plan, so the bundle "
"identifier is what a consumer joins on; refusing to write concepts "
"nothing can tell apart",
code="segmentation_plan_invalid",
)
run_rejection = profile.types.rejection(okf_type) run_rejection = profile.types.rejection(okf_type)
if run_rejection is not None: if run_rejection is not None:
raise MaterializationError(f"okf_type {run_rejection.reason}", code=run_rejection.code) raise MaterializationError(f"okf_type {run_rejection.reason}", code=run_rejection.code)
@ -376,45 +497,71 @@ def process_inbox(
) )
) )
continue continue
outputs: list[tuple[str, str, tuple[str, ...]]] = []
try: try:
source_bytes = path.read_bytes() source_bytes = path.read_bytes()
text = extract_text(path.name, source_bytes) text = extract_text(path.name, source_bytes)
decision = gate(text) covering = _plan_covering(segmentation, source_bytes)
if decision.disposition != _DISPOSITION_PERSIST: if covering is not None:
blocked = BlockedFile( blocked = _render_segments(
source_file=path.name, covering,
disposition=decision.disposition, outputs,
reasons=decision.reasons, path=path,
text=text,
source_bytes=source_bytes,
gate=gate,
profile=profile,
bundle_id=(root_frontmatter_values or {})[_bundle_id_key(profile)],
) )
# Quarantine is a queue for the operator; everything else — if blocked is not None:
# fail-secure, or a disposition from outside the pinned range — if blocked.disposition == _DISPOSITION_QUARANTINE:
# is a refusal. Unknown values land here by construction. quarantined.append(blocked)
if decision.disposition == _DISPOSITION_QUARANTINE: else:
quarantined.append(blocked) rejected.append(blocked)
else: continue
rejected.append(blocked) else:
continue decision = gate(text)
structure: DocumentStructure | None = None if decision.disposition != _DISPOSITION_PERSIST:
title = unicodedata.normalize("NFC", path.stem) blocked = BlockedFile(
if profile.index.facets is not None: source_file=path.name,
# Derived from the SANITIZED text, never the extracted text: disposition=decision.disposition,
# deriving from bytes the gate rejected would put unscreened reasons=decision.reasons,
# content in the frontmatter and the index. )
structure = derive_document_structure( # Quarantine is a queue for the operator; everything else —
decision.sanitized_text, source_file=path.name # fail-secure, or a disposition from outside the pinned range —
# is a refusal. Unknown values land here by construction.
if decision.disposition == _DISPOSITION_QUARANTINE:
quarantined.append(blocked)
else:
rejected.append(blocked)
continue
structure: DocumentStructure | None = None
title = unicodedata.normalize("NFC", path.stem)
if profile.index.facets is not None:
# Derived from the SANITIZED text, never the extracted text:
# deriving from bytes the gate rejected would put unscreened
# content in the frontmatter and the index.
structure = derive_document_structure(
decision.sanitized_text, source_file=path.name
)
title = structure.title
_validate_facets(structure, profile)
outputs.append(
(
name,
render_inbox_concept(
decision.sanitized_text,
okf_type=okf_type,
title=title,
source_file=path.name,
source_bytes=source_bytes,
ingested_at=ingested_at,
profile=profile,
structure=structure,
),
decision.reasons,
)
) )
title = structure.title
_validate_facets(structure, profile)
content = render_inbox_concept(
decision.sanitized_text,
okf_type=okf_type,
title=title,
source_file=path.name,
source_bytes=source_bytes,
ingested_at=ingested_at,
profile=profile,
structure=structure,
)
except OSError as exc: except OSError as exc:
failed.append( failed.append(
FailedFile( FailedFile(
@ -430,10 +577,12 @@ def process_inbox(
continue continue
bundle.mkdir(parents=True, exist_ok=True) bundle.mkdir(parents=True, exist_ok=True)
written = write_bytes(bundle, name, content) for target_name, content, reasons in outputs:
persisted.append( # `write_bytes` resolves a subpath through `safe_resolve` but never
PersistedFile(source_file=path.name, path=written, reasons=decision.reasons) # creates one. Without this the very first hierarchical write fails.
) (bundle / target_name).parent.mkdir(parents=True, exist_ok=True)
written = write_bytes(bundle, target_name, content)
persisted.append(PersistedFile(source_file=path.name, path=written, reasons=reasons))
# §6 index — the last disk mutation, and only when something was written. # §6 index — the last disk mutation, and only when something was written.
if persisted: if persisted:

View file

@ -0,0 +1,349 @@
"""One document becomes many concepts: Door B's 1-to-N path.
The measured defect this closes. `process_inbox` emitted exactly one flat
concept per dropped file, which is the shape OKF v0.2's Appendix A presents
v0.1 as migrating AWAY from -- and §11 could never catch it, because it checks
that every non-reserved `.md` parses with a non-empty `type`. A bundle of one
giant concept is fully conformant. Conformance is the floor, not the proof.
Two invariants are load-bearing here and are asserted rather than assumed:
- **Every segment is gated BEFORE any segment is written.** Gating and writing
one at a time would leave a half-screened document on disk the moment
segment 3 of 5 quarantines -- part of a document the guard refused, persisted
and indexed, with the run reporting success.
- **A span indexes the CANONICAL EXTRACTED TEXT.** The `.csv` fixture below is
what proves it: `render_table` re-renders those bytes, so a span computed
against the source bytes lands on different characters. On a `.md` fixture
bytes and text are identical and the assertion would pass for the wrong
reason.
`csv` is a CORE type, needing no `[extract]` extra, so that fixture can never
skip. A `pdf` would be the trap: pdfplumber is installed in this venv, so it
would pass here and skip silently in a bare consumer environment.
"""
from __future__ import annotations
import hashlib
from pathlib import Path
from typing import Any
import pytest
from llm_ingestion_okf.errors import SegmentationError
from llm_ingestion_okf.extract import extract_text
from llm_ingestion_okf.inbox import GateDecision, process_inbox
from llm_ingestion_okf.profiles import DEFAULT, SEGMENTED_V1
from llm_ingestion_okf.segmentation import SegmentationPlan, parse_segmentation_plan
INGESTED_AT = "2026-07-25T12:00:00Z"
PLAN_AT = "2026-08-30T09:00:00Z"
DOCUMENT = (
"0 Forord: bakgrunn for konseptet.\n"
"1 Brannkonsept: krav til seksjonering.\n"
"2 Roemning: to uavhengige veier.\n"
"3 Baereevne: R60 for hovedbaeresystem.\n"
"4 Slokkeanlegg: sprinkler i hele bygget.\n"
)
PATHS = (
"krav/forord.md",
"krav/3-1/brannkonsept.md",
"krav/3-1/roemning.md",
"krav/3-2/baereevne.md",
"krav/3-2/slokkeanlegg.md",
)
def gate(text: str) -> GateDecision:
return GateDecision(sanitized_text=text, disposition="warn")
def drop(inbox: Path, name: str, text: str) -> Path:
inbox.mkdir(parents=True, exist_ok=True)
path = inbox / name
path.write_text(text, encoding="utf-8", newline="")
return path
def line_spans(text: str) -> tuple[tuple[int, int], ...]:
"""Half-open spans, one per line, measured rather than hard-coded."""
spans: list[tuple[int, int]] = []
offset = 0
for line in text.splitlines(keepends=True):
spans.append((offset, offset + len(line)))
offset += len(line)
return tuple(spans)
def build_plan(
source_bytes: bytes,
text: str,
*,
paths: tuple[str, ...] = PATHS,
extractor_id: str = "md",
**overrides: Any,
) -> SegmentationPlan:
spans = line_spans(text)
payload: dict[str, Any] = {
"version": "1",
"source_sha256": hashlib.sha256(source_bytes).hexdigest(),
"extractor_id": extractor_id,
"extractor_version": "1.0.0",
"adjudicated_at": "2026-08-30T08:00:00Z",
"entries": [
{
"segment_id": f"s{index}",
"path": path,
"title": f"Del {index}",
"okf_type": "requirement",
"span": list(spans[index]),
"ingested_at": PLAN_AT,
}
for index, path in enumerate(paths)
],
}
payload.update(overrides)
return parse_segmentation_plan(payload)
def run(
tmp: Path,
*,
plan: SegmentationPlan | None,
profile=SEGMENTED_V1,
values: dict[str, str] | None = None,
round_name: str = "round",
guard=gate,
):
return process_inbox(
tmp / round_name,
tmp / "bundle",
INGESTED_AT,
okf_type="requirement",
gate=guard,
profile=profile,
root_frontmatter_values={"bundle_id": "b-1"} if values is None else values,
segmentation=plan,
)
def concepts(bundle: Path, profile=SEGMENTED_V1) -> dict[str, str]:
"""Every non-reserved concept, keyed by bundle-relative path.
Reserved names are excluded at EVERY level, not just the root -- a
per-directory index writer puts one in each directory, and counting those
as concepts would inflate the count the moment nesting appeared.
"""
if not bundle.is_dir():
return {}
return {
str(path.relative_to(bundle)): path.read_text(encoding="utf-8")
for path in sorted(bundle.rglob(f"*{profile.paths.concept_suffix}"))
if path.is_file() and path.name != profile.index.name
}
def tree(bundle: Path) -> dict[str, bytes]:
if not bundle.is_dir():
return {}
return {
str(path.relative_to(bundle)): path.read_bytes()
for path in sorted(bundle.rglob("*"))
if path.is_file()
}
def body_of(document: str) -> str:
return document.split("---\n", 2)[2].lstrip("\n")
def frontmatter_of(document: str) -> dict[str, str]:
head = document.split("---\n")[1]
return dict(
line.split(": ", 1) for line in head.splitlines() if ": " in line and line[:1] != " "
)
# --- S1: one document, exactly N concepts ---------------------------------
def test_a_five_entry_plan_yields_exactly_five_concepts(tmp_path: Path) -> None:
source = drop(tmp_path / "round", "n500.md", DOCUMENT)
run(tmp_path, plan=build_plan(source.read_bytes(), DOCUMENT))
found = concepts(tmp_path / "bundle")
assert len(found) == 5
assert len(found) > 1
assert set(found) == set(PATHS)
def test_without_a_plan_the_same_fixture_yields_exactly_one_concept(tmp_path: Path) -> None:
# The discriminating negative control. Without it, a test that counts five
# concepts proves nothing about whether the PLAN caused the split.
drop(tmp_path / "round", "n500.md", DOCUMENT)
run(tmp_path, plan=None)
found = concepts(tmp_path / "bundle")
assert len(found) == 1
assert set(found) == {"inbox-n500.md"}
def test_each_segment_id_maps_to_the_path_its_entry_declares(tmp_path: Path) -> None:
source = drop(tmp_path / "round", "n500.md", DOCUMENT)
plan = build_plan(source.read_bytes(), DOCUMENT)
run(tmp_path, plan=plan)
found = concepts(tmp_path / "bundle")
mapped = {frontmatter_of(document)["segment_id"]: path for path, document in found.items()}
assert mapped == {item.segment_id: item.path for item in plan.entries}
def test_each_body_equals_the_span_its_own_entry_declares(tmp_path: Path) -> None:
source = drop(tmp_path / "round", "n500.md", DOCUMENT)
plan = build_plan(source.read_bytes(), DOCUMENT)
run(tmp_path, plan=plan)
found = concepts(tmp_path / "bundle")
for item in plan.entries:
start, end = item.span
assert body_of(found[item.path]) == DOCUMENT[start:end]
def test_concepts_land_on_nested_paths_across_several_directories(tmp_path: Path) -> None:
source = drop(tmp_path / "round", "n500.md", DOCUMENT)
run(tmp_path, plan=build_plan(source.read_bytes(), DOCUMENT))
found = concepts(tmp_path / "bundle")
directories = {str(Path(path).parent) for path in found}
assert len(directories) >= 2
nested = [path for path in found if "/" in path]
assert nested
# The concept ID is the bundle-relative path minus the suffix -- OKF v0.2
# §2's definition, not a name we assign.
for path in nested:
assert path.endswith(".md")
assert (tmp_path / "bundle" / path).is_file()
def test_the_plan_timestamp_reaches_every_concept(tmp_path: Path) -> None:
source = drop(tmp_path / "round", "n500.md", DOCUMENT)
run(tmp_path, plan=build_plan(source.read_bytes(), DOCUMENT))
for document in concepts(tmp_path / "bundle").values():
keys = frontmatter_of(document)
assert keys["ingested_at"] == PLAN_AT
assert keys["bundle_id"] == "b-1"
# --- S9: the span indexes EXTRACTED text, not source bytes ----------------
CSV_SOURCE = "krav;beskrivelse\n3-1;seksjonering\n3-2;roemning\n"
def test_a_csv_body_equals_the_span_of_the_extracted_text(tmp_path: Path) -> None:
# `render_table` re-renders these bytes, so extracted text != source bytes.
# A span computed against the bytes would land on different characters and
# produce a concept nobody adjudicated, with nothing failing.
source = drop(tmp_path / "round", "krav.csv", CSV_SOURCE)
text = extract_text("krav.csv", source.read_bytes())
assert text != CSV_SOURCE
plan = build_plan(
source.read_bytes(),
text,
paths=tuple(f"tabell/rad-{index}.md" for index in range(len(line_spans(text)))),
extractor_id="csv",
)
run(tmp_path, plan=plan)
found = concepts(tmp_path / "bundle")
assert len(found) == len(plan.entries)
for item in plan.entries:
start, end = item.span
assert body_of(found[item.path]) == text[start:end]
# --- fail-fast misuse, before any disk mutation ---------------------------
def test_a_plan_against_a_profile_without_the_capability_is_refused(tmp_path: Path) -> None:
source = drop(tmp_path / "round", "n500.md", DOCUMENT)
with pytest.raises(SegmentationError) as excinfo:
run(tmp_path, plan=build_plan(source.read_bytes(), DOCUMENT), profile=DEFAULT, values={})
assert excinfo.value.code == "segmentation_unsupported_profile"
assert tree(tmp_path / "bundle") == {}
def test_a_plan_without_a_bundle_id_is_refused_before_any_write(tmp_path: Path) -> None:
# The assertion Step 5 deferred to here: `process_inbox` gains its
# `segmentation` parameter at THIS step, so this is the first point at
# which the required-bundle_id branch can be reached at all.
source = drop(tmp_path / "round", "n500.md", DOCUMENT)
with pytest.raises(SegmentationError) as excinfo:
run(tmp_path, plan=build_plan(source.read_bytes(), DOCUMENT), values={})
assert excinfo.value.code == "segmentation_plan_invalid"
assert "bundle_id" in str(excinfo.value)
assert tree(tmp_path / "bundle") == {}
def test_a_refused_plan_leaves_an_existing_bundle_untouched(tmp_path: Path) -> None:
source = drop(tmp_path / "one", "n500.md", DOCUMENT)
run(tmp_path, plan=build_plan(source.read_bytes(), DOCUMENT), round_name="one")
before = tree(tmp_path / "bundle")
assert before != {}
drop(tmp_path / "two", "v720.md", DOCUMENT)
with pytest.raises(SegmentationError):
run(tmp_path, plan=build_plan(source.read_bytes(), DOCUMENT), values={}, round_name="two")
assert tree(tmp_path / "bundle") == before
# --- the whole-document refusal -------------------------------------------
def test_one_quarantined_segment_persists_nothing_for_that_document(tmp_path: Path) -> None:
source = drop(tmp_path / "round", "n500.md", DOCUMENT)
plan = build_plan(source.read_bytes(), DOCUMENT)
third = DOCUMENT[plan.entries[2].span[0] : plan.entries[2].span[1]]
def quarantining(text: str) -> GateDecision:
if text == third:
return GateDecision(
sanitized_text=text, disposition="quarantine_review", reasons=("segment 3",)
)
return GateDecision(sanitized_text=text, disposition="warn")
result = run(tmp_path, plan=plan, guard=quarantining)
assert concepts(tmp_path / "bundle") == {}
assert len(result.quarantined) == 1
assert result.quarantined[0].source_file == "n500.md"
assert result.persisted == ()
def test_every_segment_is_gated_before_any_is_written(tmp_path: Path) -> None:
source = drop(tmp_path / "round", "n500.md", DOCUMENT)
plan = build_plan(source.read_bytes(), DOCUMENT)
calls: list[str] = []
written_when_gated: list[int] = []
def recording(text: str) -> GateDecision:
calls.append(text)
written_when_gated.append(len(tree(tmp_path / "bundle")))
return GateDecision(sanitized_text=text, disposition="warn")
run(tmp_path, plan=plan, guard=recording)
assert len(calls) == 5
# Nothing on disk while any gate call is still outstanding.
assert written_when_gated == [0, 0, 0, 0, 0]
assert len(concepts(tmp_path / "bundle")) == 5
def test_a_plan_covering_one_of_two_documents_leaves_the_other_flat(tmp_path: Path) -> None:
# DIFFERENT bytes, deliberately. A plan is selected by content hash, so two
# files with identical content are both covered by one plan and land every
# segment on the same path -- which is the intra-run collision Step 8's gate
# exists to refuse, not something to demonstrate here.
source = drop(tmp_path / "round", "n500.md", DOCUMENT)
drop(tmp_path / "round", "v720.md", "V720 Tunnel: egne krav.\n")
run(tmp_path, plan=build_plan(source.read_bytes(), DOCUMENT))
found = concepts(tmp_path / "bundle")
assert set(PATHS) <= set(found)
assert "inbox-v720.md" in found
assert len(found) == 6