open/llm-ingestion-okf
1
0
Fork 0
Code Issues Releases 1 Activity

feat(inbox): materialize one document into many concepts on hierarchical paths

Browse source
This commit is contained in:
Kjell Tore Guttormsen 2026-09-01 00:10:13 +02:00
commit bb79c55f43
2 changed files with 538 additions and 40 deletions
Download patch file Download diff file Expand all files Collapse all files

229
src/llm_ingestion_okf/inbox.py
View file

@ -25,7 +25,7 @@ from collections.abc import Callable, Mapping
from dataclasses import dataclass
from pathlib import Path
from .errors import IngestError, MaterializationError, SourceError
from .errors import IngestError, MaterializationError, SegmentationError, SourceError
from .extract import extract_text
from .materialize import (
_render_root_frontmatter,
@ -37,7 +37,12 @@ from .materialize import (
write_bytes,
)
from .profiles import DEFAULT, BundleProfile
from .segmentation import SegmentEntry
from .segmentation import (
SegmentationPlan,
SegmentEntry,
assert_plan_applies,
slice_segments,
)
from .structure import (
DocumentStructure,
_render_flow_list,
@ -270,6 +275,103 @@ def _is_inbox_owned(path: Path) -> bool:
return frontmatter.get("generated") == "true" and "source_file" in frontmatter
def _bundle_id_key(profile: BundleProfile) -> str:
assert profile.segmentation is not None
return profile.segmentation.bundle_id_key
def _plan_covering(plan: SegmentationPlan | None, source_bytes: bytes) -> SegmentationPlan | None:
"""The plan for THESE bytes, or None when this file is not plan-covered.
Selection is by content hash, so a run may drop several documents while
only one of them is segmented; every other file keeps today's one-concept
rule verbatim. The hash SELECTS; :func:`assert_plan_applies` VALIDATES,
and the two are deliberately different questions -- see S5b.
"""
if plan is None:
return None
if plan.source_sha256 != hashlib.sha256(source_bytes).hexdigest():
return None
return plan
def _render_segments(
plan: SegmentationPlan,
outputs: list[tuple[str, str, tuple[str, ...]]],
*,
path: Path,
text: str,
source_bytes: bytes,
gate: Gate,
profile: BundleProfile,
bundle_id: str,
) -> BlockedFile | None:
"""Render every segment, or refuse the WHOLE document.
The ordering here is the security property, not a style choice: all N
segments are gated and all N decisions collected BEFORE a single byte is
written. Gating and writing one at a time would leave a half-screened
document on disk the moment segment 3 of 5 quarantines -- part of a
document the guard refused, persisted and indexed, with the run reporting
success for everything it managed to write first.
A refusal is therefore reported once, for the document, rather than once
per segment: the operator's unit of review is the document they dropped.
"""
assert_plan_applies(
plan,
source_sha256=hashlib.sha256(source_bytes).hexdigest(),
extractor_id=Path(path.name).suffix.lower().lstrip(".") or "none",
# The plan's own value, passed through. Door B can observe WHICH
# extractor ran (the suffix is what dispatches it at `extract.py`) but
# not the version of a third-party parser -- `pdfplumber`'s transitive
# `pdfminer.six` pin is the measured example. Naming the key and
# leaving its value to whoever knows it is the same division D5 makes
# for `bundle_id`; a fabricated value here would make S5b decorative.
extractor_version=plan.extractor_version,
)
sliced = slice_segments(text, plan)
decisions = [(entry, gate(body)) for entry, body in sliced]
refused = [
decision for _, decision in decisions if decision.disposition != _DISPOSITION_PERSIST
]
if refused:
return BlockedFile(
source_file=path.name,
disposition=refused[0].disposition,
reasons=tuple(reason for decision in refused for reason in decision.reasons),
)
for entry, decision in decisions:
structure: DocumentStructure | None = None
if profile.index.facets is not None:
structure = derive_document_structure(decision.sanitized_text, source_file=path.name)
_validate_facets(structure, profile)
outputs.append(
(
entry.path,
render_inbox_concept(
decision.sanitized_text,
okf_type=entry.okf_type,
# DECLARED by the adjudicator, never derived from the
# segment's own first line: the plan is the record of the
# judgement, and a heading inside a slice is not it.
title=entry.title,
source_file=path.name,
source_bytes=source_bytes,
ingested_at=entry.ingested_at,
profile=profile,
structure=structure,
segment=entry,
bundle_id=bundle_id,
),
decision.reasons,
)
)
return None
def process_inbox(
inbox_dir: Path,
bundle_dir: Path,
@ -279,6 +381,7 @@ def process_inbox(
gate: Gate,
profile: BundleProfile = DEFAULT,
root_frontmatter_values: Mapping[str, str] | None = None,
segmentation: SegmentationPlan | None = None,
) -> InboxResult:
"""Convert every file dropped in `inbox_dir` into an OKF concept.
@ -302,6 +405,24 @@ def process_inbox(
# `materialize.py` states the same rule for Door A, and a door that half-built
# a bundle before refusing would be worse than one that never started.
root_head = _render_root_frontmatter(root_frontmatter_values or {}, profile=profile)
if segmentation is not None:
if profile.segmentation is None:
raise SegmentationError(
"a segmentation plan was passed to a profile that does not declare the "
"segmentation capability — a plan would be silently ignored and the "
"document would land as one flat concept, which is the shape this "
"capability exists to replace",
code="segmentation_unsupported_profile",
)
if profile.segmentation.bundle_id_key not in (root_frontmatter_values or {}):
raise SegmentationError(
f"a segmentation plan requires {profile.segmentation.bundle_id_key!r} in "
"root_frontmatter_values — a segmented bundle's concept paths collide "
"with any other bundle built from the same plan, so the bundle "
"identifier is what a consumer joins on; refusing to write concepts "
"nothing can tell apart",
code="segmentation_plan_invalid",
)
run_rejection = profile.types.rejection(okf_type)
if run_rejection is not None:
raise MaterializationError(f"okf_type {run_rejection.reason}", code=run_rejection.code)
@ -376,45 +497,71 @@ def process_inbox(
)
)
continue
outputs: list[tuple[str, str, tuple[str, ...]]] = []
try:
source_bytes = path.read_bytes()
text = extract_text(path.name, source_bytes)
decision = gate(text)
if decision.disposition != _DISPOSITION_PERSIST:
blocked = BlockedFile(
source_file=path.name,
disposition=decision.disposition,
reasons=decision.reasons,
covering = _plan_covering(segmentation, source_bytes)
if covering is not None:
blocked = _render_segments(
covering,
outputs,
path=path,
text=text,
source_bytes=source_bytes,
gate=gate,
profile=profile,
bundle_id=(root_frontmatter_values or {})[_bundle_id_key(profile)],
)
# Quarantine is a queue for the operator; everything else —
# fail-secure, or a disposition from outside the pinned range —
# is a refusal. Unknown values land here by construction.
if decision.disposition == _DISPOSITION_QUARANTINE:
quarantined.append(blocked)
else:
rejected.append(blocked)
continue
structure: DocumentStructure | None = None
title = unicodedata.normalize("NFC", path.stem)
if profile.index.facets is not None:
# Derived from the SANITIZED text, never the extracted text:
# deriving from bytes the gate rejected would put unscreened
# content in the frontmatter and the index.
structure = derive_document_structure(
decision.sanitized_text, source_file=path.name
if blocked is not None:
if blocked.disposition == _DISPOSITION_QUARANTINE:
quarantined.append(blocked)
else:
rejected.append(blocked)
continue
else:
decision = gate(text)
if decision.disposition != _DISPOSITION_PERSIST:
blocked = BlockedFile(
source_file=path.name,
disposition=decision.disposition,
reasons=decision.reasons,
)
# Quarantine is a queue for the operator; everything else —
# fail-secure, or a disposition from outside the pinned range —
# is a refusal. Unknown values land here by construction.
if decision.disposition == _DISPOSITION_QUARANTINE:
quarantined.append(blocked)
else:
rejected.append(blocked)
continue
structure: DocumentStructure | None = None
title = unicodedata.normalize("NFC", path.stem)
if profile.index.facets is not None:
# Derived from the SANITIZED text, never the extracted text:
# deriving from bytes the gate rejected would put unscreened
# content in the frontmatter and the index.
structure = derive_document_structure(
decision.sanitized_text, source_file=path.name
)
title = structure.title
_validate_facets(structure, profile)
outputs.append(
(
name,
render_inbox_concept(
decision.sanitized_text,
okf_type=okf_type,
title=title,
source_file=path.name,
source_bytes=source_bytes,
ingested_at=ingested_at,
profile=profile,
structure=structure,
),
decision.reasons,
)
)
title = structure.title
_validate_facets(structure, profile)
content = render_inbox_concept(
decision.sanitized_text,
okf_type=okf_type,
title=title,
source_file=path.name,
source_bytes=source_bytes,
ingested_at=ingested_at,
profile=profile,
structure=structure,
)
except OSError as exc:
failed.append(
FailedFile(
@ -430,10 +577,12 @@ def process_inbox(
continue
bundle.mkdir(parents=True, exist_ok=True)
written = write_bytes(bundle, name, content)
persisted.append(
PersistedFile(source_file=path.name, path=written, reasons=decision.reasons)
)
for target_name, content, reasons in outputs:
# `write_bytes` resolves a subpath through `safe_resolve` but never
# creates one. Without this the very first hierarchical write fails.
(bundle / target_name).parent.mkdir(parents=True, exist_ok=True)
written = write_bytes(bundle, target_name, content)
persisted.append(PersistedFile(source_file=path.name, path=written, reasons=reasons))
# §6 index — the last disk mutation, and only when something was written.
if persisted:

349
tests/test_segmented_inbox.py Normal file
View file

@ -0,0 +1,349 @@
"""One document becomes many concepts: Door B's 1-to-N path.
The measured defect this closes. `process_inbox` emitted exactly one flat
concept per dropped file, which is the shape OKF v0.2's Appendix A presents
v0.1 as migrating AWAY from -- and §11 could never catch it, because it checks
that every non-reserved `.md` parses with a non-empty `type`. A bundle of one
giant concept is fully conformant. Conformance is the floor, not the proof.
Two invariants are load-bearing here and are asserted rather than assumed:
- **Every segment is gated BEFORE any segment is written.** Gating and writing
one at a time would leave a half-screened document on disk the moment
segment 3 of 5 quarantines -- part of a document the guard refused, persisted
and indexed, with the run reporting success.
- **A span indexes the CANONICAL EXTRACTED TEXT.** The `.csv` fixture below is
what proves it: `render_table` re-renders those bytes, so a span computed
against the source bytes lands on different characters. On a `.md` fixture
bytes and text are identical and the assertion would pass for the wrong
reason.
`csv` is a CORE type, needing no `[extract]` extra, so that fixture can never
skip. A `pdf` would be the trap: pdfplumber is installed in this venv, so it
would pass here and skip silently in a bare consumer environment.
"""
from __future__ import annotations
import hashlib
from pathlib import Path
from typing import Any
import pytest
from llm_ingestion_okf.errors import SegmentationError
from llm_ingestion_okf.extract import extract_text
from llm_ingestion_okf.inbox import GateDecision, process_inbox
from llm_ingestion_okf.profiles import DEFAULT, SEGMENTED_V1
from llm_ingestion_okf.segmentation import SegmentationPlan, parse_segmentation_plan
INGESTED_AT = "2026-07-25T12:00:00Z"
PLAN_AT = "2026-08-30T09:00:00Z"
DOCUMENT = (
"0 Forord: bakgrunn for konseptet.\n"
"1 Brannkonsept: krav til seksjonering.\n"
"2 Roemning: to uavhengige veier.\n"
"3 Baereevne: R60 for hovedbaeresystem.\n"
"4 Slokkeanlegg: sprinkler i hele bygget.\n"
)
PATHS = (
"krav/forord.md",
"krav/3-1/brannkonsept.md",
"krav/3-1/roemning.md",
"krav/3-2/baereevne.md",
"krav/3-2/slokkeanlegg.md",
)
def gate(text: str) -> GateDecision:
return GateDecision(sanitized_text=text, disposition="warn")
def drop(inbox: Path, name: str, text: str) -> Path:
inbox.mkdir(parents=True, exist_ok=True)
path = inbox / name
path.write_text(text, encoding="utf-8", newline="")
return path
def line_spans(text: str) -> tuple[tuple[int, int], ...]:
"""Half-open spans, one per line, measured rather than hard-coded."""
spans: list[tuple[int, int]] = []
offset = 0
for line in text.splitlines(keepends=True):
spans.append((offset, offset + len(line)))
offset += len(line)
return tuple(spans)
def build_plan(
source_bytes: bytes,
text: str,
*,
paths: tuple[str, ...] = PATHS,
extractor_id: str = "md",
**overrides: Any,
) -> SegmentationPlan:
spans = line_spans(text)
payload: dict[str, Any] = {
"version": "1",
"source_sha256": hashlib.sha256(source_bytes).hexdigest(),
"extractor_id": extractor_id,
"extractor_version": "1.0.0",
"adjudicated_at": "2026-08-30T08:00:00Z",
"entries": [
{
"segment_id": f"s{index}",
"path": path,
"title": f"Del {index}",
"okf_type": "requirement",
"span": list(spans[index]),
"ingested_at": PLAN_AT,
}
for index, path in enumerate(paths)
],
}
payload.update(overrides)
return parse_segmentation_plan(payload)
def run(
tmp: Path,
*,
plan: SegmentationPlan | None,
profile=SEGMENTED_V1,
values: dict[str, str] | None = None,
round_name: str = "round",
guard=gate,
):
return process_inbox(
tmp / round_name,
tmp / "bundle",
INGESTED_AT,
okf_type="requirement",
gate=guard,
profile=profile,
root_frontmatter_values={"bundle_id": "b-1"} if values is None else values,
segmentation=plan,
)
def concepts(bundle: Path, profile=SEGMENTED_V1) -> dict[str, str]:
"""Every non-reserved concept, keyed by bundle-relative path.
Reserved names are excluded at EVERY level, not just the root -- a
per-directory index writer puts one in each directory, and counting those
as concepts would inflate the count the moment nesting appeared.
"""
if not bundle.is_dir():
return {}
return {
str(path.relative_to(bundle)): path.read_text(encoding="utf-8")
for path in sorted(bundle.rglob(f"*{profile.paths.concept_suffix}"))
if path.is_file() and path.name != profile.index.name
}
def tree(bundle: Path) -> dict[str, bytes]:
if not bundle.is_dir():
return {}
return {
str(path.relative_to(bundle)): path.read_bytes()
for path in sorted(bundle.rglob("*"))
if path.is_file()
}
def body_of(document: str) -> str:
return document.split("---\n", 2)[2].lstrip("\n")
def frontmatter_of(document: str) -> dict[str, str]:
head = document.split("---\n")[1]
return dict(
line.split(": ", 1) for line in head.splitlines() if ": " in line and line[:1] != " "
)
# --- S1: one document, exactly N concepts ---------------------------------
def test_a_five_entry_plan_yields_exactly_five_concepts(tmp_path: Path) -> None:
source = drop(tmp_path / "round", "n500.md", DOCUMENT)
run(tmp_path, plan=build_plan(source.read_bytes(), DOCUMENT))
found = concepts(tmp_path / "bundle")
assert len(found) == 5
assert len(found) > 1
assert set(found) == set(PATHS)
def test_without_a_plan_the_same_fixture_yields_exactly_one_concept(tmp_path: Path) -> None:
# The discriminating negative control. Without it, a test that counts five
# concepts proves nothing about whether the PLAN caused the split.
drop(tmp_path / "round", "n500.md", DOCUMENT)
run(tmp_path, plan=None)
found = concepts(tmp_path / "bundle")
assert len(found) == 1
assert set(found) == {"inbox-n500.md"}
def test_each_segment_id_maps_to_the_path_its_entry_declares(tmp_path: Path) -> None:
source = drop(tmp_path / "round", "n500.md", DOCUMENT)
plan = build_plan(source.read_bytes(), DOCUMENT)
run(tmp_path, plan=plan)
found = concepts(tmp_path / "bundle")
mapped = {frontmatter_of(document)["segment_id"]: path for path, document in found.items()}
assert mapped == {item.segment_id: item.path for item in plan.entries}
def test_each_body_equals_the_span_its_own_entry_declares(tmp_path: Path) -> None:
source = drop(tmp_path / "round", "n500.md", DOCUMENT)
plan = build_plan(source.read_bytes(), DOCUMENT)
run(tmp_path, plan=plan)
found = concepts(tmp_path / "bundle")
for item in plan.entries:
start, end = item.span
assert body_of(found[item.path]) == DOCUMENT[start:end]
def test_concepts_land_on_nested_paths_across_several_directories(tmp_path: Path) -> None:
source = drop(tmp_path / "round", "n500.md", DOCUMENT)
run(tmp_path, plan=build_plan(source.read_bytes(), DOCUMENT))
found = concepts(tmp_path / "bundle")
directories = {str(Path(path).parent) for path in found}
assert len(directories) >= 2
nested = [path for path in found if "/" in path]
assert nested
# The concept ID is the bundle-relative path minus the suffix -- OKF v0.2
# §2's definition, not a name we assign.
for path in nested:
assert path.endswith(".md")
assert (tmp_path / "bundle" / path).is_file()
def test_the_plan_timestamp_reaches_every_concept(tmp_path: Path) -> None:
source = drop(tmp_path / "round", "n500.md", DOCUMENT)
run(tmp_path, plan=build_plan(source.read_bytes(), DOCUMENT))
for document in concepts(tmp_path / "bundle").values():
keys = frontmatter_of(document)
assert keys["ingested_at"] == PLAN_AT
assert keys["bundle_id"] == "b-1"
# --- S9: the span indexes EXTRACTED text, not source bytes ----------------
CSV_SOURCE = "krav;beskrivelse\n3-1;seksjonering\n3-2;roemning\n"
def test_a_csv_body_equals_the_span_of_the_extracted_text(tmp_path: Path) -> None:
# `render_table` re-renders these bytes, so extracted text != source bytes.
# A span computed against the bytes would land on different characters and
# produce a concept nobody adjudicated, with nothing failing.
source = drop(tmp_path / "round", "krav.csv", CSV_SOURCE)
text = extract_text("krav.csv", source.read_bytes())
assert text != CSV_SOURCE
plan = build_plan(
source.read_bytes(),
text,
paths=tuple(f"tabell/rad-{index}.md" for index in range(len(line_spans(text)))),
extractor_id="csv",
)
run(tmp_path, plan=plan)
found = concepts(tmp_path / "bundle")
assert len(found) == len(plan.entries)
for item in plan.entries:
start, end = item.span
assert body_of(found[item.path]) == text[start:end]
# --- fail-fast misuse, before any disk mutation ---------------------------
def test_a_plan_against_a_profile_without_the_capability_is_refused(tmp_path: Path) -> None:
source = drop(tmp_path / "round", "n500.md", DOCUMENT)
with pytest.raises(SegmentationError) as excinfo:
run(tmp_path, plan=build_plan(source.read_bytes(), DOCUMENT), profile=DEFAULT, values={})
assert excinfo.value.code == "segmentation_unsupported_profile"
assert tree(tmp_path / "bundle") == {}
def test_a_plan_without_a_bundle_id_is_refused_before_any_write(tmp_path: Path) -> None:
# The assertion Step 5 deferred to here: `process_inbox` gains its
# `segmentation` parameter at THIS step, so this is the first point at
# which the required-bundle_id branch can be reached at all.
source = drop(tmp_path / "round", "n500.md", DOCUMENT)
with pytest.raises(SegmentationError) as excinfo:
run(tmp_path, plan=build_plan(source.read_bytes(), DOCUMENT), values={})
assert excinfo.value.code == "segmentation_plan_invalid"
assert "bundle_id" in str(excinfo.value)
assert tree(tmp_path / "bundle") == {}
def test_a_refused_plan_leaves_an_existing_bundle_untouched(tmp_path: Path) -> None:
source = drop(tmp_path / "one", "n500.md", DOCUMENT)
run(tmp_path, plan=build_plan(source.read_bytes(), DOCUMENT), round_name="one")
before = tree(tmp_path / "bundle")
assert before != {}
drop(tmp_path / "two", "v720.md", DOCUMENT)
with pytest.raises(SegmentationError):
run(tmp_path, plan=build_plan(source.read_bytes(), DOCUMENT), values={}, round_name="two")
assert tree(tmp_path / "bundle") == before
# --- the whole-document refusal -------------------------------------------
def test_one_quarantined_segment_persists_nothing_for_that_document(tmp_path: Path) -> None:
source = drop(tmp_path / "round", "n500.md", DOCUMENT)
plan = build_plan(source.read_bytes(), DOCUMENT)
third = DOCUMENT[plan.entries[2].span[0] : plan.entries[2].span[1]]
def quarantining(text: str) -> GateDecision:
if text == third:
return GateDecision(
sanitized_text=text, disposition="quarantine_review", reasons=("segment 3",)
)
return GateDecision(sanitized_text=text, disposition="warn")
result = run(tmp_path, plan=plan, guard=quarantining)
assert concepts(tmp_path / "bundle") == {}
assert len(result.quarantined) == 1
assert result.quarantined[0].source_file == "n500.md"
assert result.persisted == ()
def test_every_segment_is_gated_before_any_is_written(tmp_path: Path) -> None:
source = drop(tmp_path / "round", "n500.md", DOCUMENT)
plan = build_plan(source.read_bytes(), DOCUMENT)
calls: list[str] = []
written_when_gated: list[int] = []
def recording(text: str) -> GateDecision:
calls.append(text)
written_when_gated.append(len(tree(tmp_path / "bundle")))
return GateDecision(sanitized_text=text, disposition="warn")
run(tmp_path, plan=plan, guard=recording)
assert len(calls) == 5
# Nothing on disk while any gate call is still outstanding.
assert written_when_gated == [0, 0, 0, 0, 0]
assert len(concepts(tmp_path / "bundle")) == 5
def test_a_plan_covering_one_of_two_documents_leaves_the_other_flat(tmp_path: Path) -> None:
# DIFFERENT bytes, deliberately. A plan is selected by content hash, so two
# files with identical content are both covered by one plan and land every
# segment on the same path -- which is the intra-run collision Step 8's gate
# exists to refuse, not something to demonstrate here.
source = drop(tmp_path / "round", "n500.md", DOCUMENT)
drop(tmp_path / "round", "v720.md", "V720 Tunnel: egne krav.\n")
run(tmp_path, plan=build_plan(source.read_bytes(), DOCUMENT))
found = concepts(tmp_path / "bundle")
assert set(PATHS) <= set(found)
assert "inbox-v720.md" in found
assert len(found) == 6