open/llm-ingestion-okf
1
0
Fork 0
Code Issues Releases 1 Activity

feat(inbox): collision gate over segment paths and additive concept reporting

Browse source
This commit is contained in:
Kjell Tore Guttormsen 2026-09-01 00:14:13 +02:00
commit 34a00b746f
2 changed files with 291 additions and 26 deletions
Download patch file Download diff file Expand all files Collapse all files

115
src/llm_ingestion_okf/inbox.py
View file

@ -264,6 +264,12 @@ class InboxResult:
quarantined: tuple[BlockedFile, ...]
rejected: tuple[BlockedFile, ...]
failed: tuple[FailedFile, ...]
# One entry per CONCEPT, where `persisted` is one per SOURCE FILE. A new
# field rather than a changed meaning: under 1-to-N the two counts diverge,
# and redefining `persisted` would silently change what every existing
# consumer's number means. Without segmentation the two are equal, which is
# what makes this additive rather than a second thing to keep in step.
concepts: tuple[PersistedFile, ...] = ()
def _is_inbox_owned(path: Path) -> bool:
@ -275,6 +281,21 @@ def _is_inbox_owned(path: Path) -> bool:
return frontmatter.get("generated") == "true" and "source_file" in frontmatter
def _check_segment_path(path: str) -> str:
"""Refuse a segment path the filesystem cannot hold, measured PER COMPONENT.
`check_filename_length` measures one name against NAME_MAX, which is a
per-directory-entry limit. Measuring the JOINED path against it gets the
question backwards in both directions: a perfectly legal deep hierarchy
would be refused, and an illegal component inside a short path would be
accepted and then fail at the write with an errno that differs per platform
-- the untyped, unportable failure the length gate exists to replace.
"""
for component in path.split("/"):
check_filename_length(component, code="inbox_slug_too_long")
return path
def _bundle_id_key(profile: BundleProfile) -> str:
assert profile.segmentation is not None
return profile.segmentation.bundle_id_key
@ -435,37 +456,72 @@ def process_inbox(
dropped = sorted((path for path in inbox.iterdir() if path.is_file()), key=lambda p: p.name)
persisted: list[PersistedFile] = []
concepts: list[PersistedFile] = []
quarantined: list[BlockedFile] = []
rejected: list[BlockedFile] = []
failed: list[FailedFile] = []
# Phase 1: name every file BEFORE any gate call or write, so an intra-run
# slug collision is caught while both files can still be refused together.
named: list[tuple[Path, str]] = []
# collision is caught while both files can still be refused together. Under
# 1-to-N a document does not claim ONE name — it claims the whole set of
# paths its plan expands to, and the gate is keyed on that set. Keyed on one
# name per file, the same defect returns one level down: the second document
# would silently claim the first's concepts.
named: list[tuple[Path, tuple[str, ...], bytes]] = []
slug_owners: dict[str, list[Path]] = {}
for path in dropped:
try:
name = inbox_filename(inbox_slug(path.name), profile=profile)
except IngestError as exc:
failed.append(FailedFile(source_file=path.name, error=exc))
continue
named.append((path, name))
slug_owners.setdefault(name, []).append(path)
colliding = {name for name, owners in slug_owners.items() if len(owners) > 1}
for name in sorted(colliding):
for path in slug_owners[name]:
# Read HERE rather than in the write loop: a plan is selected by
# content hash, so the set of names a document claims is not knowable
# without its bytes, and the whole point of this phase is to know
# every name before anything happens.
source_bytes = path.read_bytes()
except OSError as exc:
failed.append(
FailedFile(
source_file=path.name,
error=MaterializationError(
f"{path.name!r} and "
f"{', '.join(repr(other.name) for other in slug_owners[name] if other != path)}"
f" both reduce to {name!r} — rename one; refusing to pick a winner",
code="inbox_slug_collision",
error=SourceError(
f"cannot read dropped file {path.name}: {exc}", code="source_file_missing"
),
)
)
continue
try:
covering = _plan_covering(segmentation, source_bytes)
targets: tuple[str, ...]
if covering is None:
targets = (inbox_filename(inbox_slug(path.name), profile=profile),)
else:
targets = tuple(_check_segment_path(item.path) for item in covering.entries)
except IngestError as exc:
failed.append(FailedFile(source_file=path.name, error=exc))
continue
named.append((path, targets, source_bytes))
for target in targets:
slug_owners.setdefault(target, []).append(path)
contested = {name for name, owners in slug_owners.items() if len(owners) > 1}
# One refusal per DOCUMENT, not per contested path: a document expanding to
# five colliding paths is one thing the operator has to fix, and five
# identical entries would report the same rename five times.
for path in sorted(
{owner for name in contested for owner in slug_owners[name]}, key=lambda item: item.name
):
claimed = sorted(name for name in contested if path in slug_owners[name])
others = sorted(
{other.name for name in claimed for other in slug_owners[name] if other != path}
)
failed.append(
FailedFile(
source_file=path.name,
error=MaterializationError(
f"{path.name!r} and {', '.join(repr(other) for other in others)}"
f" both reduce to {', '.join(repr(name) for name in claimed)}"
" — rename one; refusing to pick a winner",
code="inbox_slug_collision",
),
)
)
# Phase 2: the §3 ownership scan, evaluated against the bundle as it was
# BEFORE this run — a file written below must never be mistaken for
@ -482,16 +538,18 @@ def process_inbox(
)
owned = {name for name in pre_existing if _is_inbox_owned(bundle / name)}
for path, name in named:
if name in colliding:
for path, targets, source_bytes in named:
if any(name in contested for name in targets):
continue
if name in pre_existing and name not in owned:
unstamped = [name for name in targets if name in pre_existing and name not in owned]
if unstamped:
failed.append(
FailedFile(
source_file=path.name,
error=MaterializationError(
f"generated filename {name!r} collides with an existing file that does "
"not carry the inbox marker — refusing to overwrite curated content (§3)",
f"generated filename {unstamped[0]!r} collides with an existing file "
"that does not carry the inbox marker — refusing to overwrite curated "
"content (§3)",
code="collision_unstamped",
),
)
@ -499,7 +557,6 @@ def process_inbox(
continue
outputs: list[tuple[str, str, tuple[str, ...]]] = []
try:
source_bytes = path.read_bytes()
text = extract_text(path.name, source_bytes)
covering = _plan_covering(segmentation, source_bytes)
if covering is not None:
@ -548,7 +605,7 @@ def process_inbox(
_validate_facets(structure, profile)
outputs.append(
(
name,
targets[0],
render_inbox_concept(
decision.sanitized_text,
okf_type=okf_type,
@ -582,7 +639,12 @@ def process_inbox(
# creates one. Without this the very first hierarchical write fails.
(bundle / target_name).parent.mkdir(parents=True, exist_ok=True)
written = write_bytes(bundle, target_name, content)
persisted.append(PersistedFile(source_file=path.name, path=written, reasons=reasons))
concepts.append(PersistedFile(source_file=path.name, path=written, reasons=reasons))
# One entry per SOURCE FILE, whatever the document expanded into. That
# is what `persisted` has always meant, so an existing consumer's count
# does not change under a profile that segments.
if outputs:
persisted.append(concepts[-len(outputs)])
# §6 index — the last disk mutation, and only when something was written.
if persisted:
@ -592,7 +654,7 @@ def process_inbox(
else:
_refresh_root_frontmatter(index_path, root_head)
if profile.index.facets is None:
for entry in persisted:
for entry in concepts:
link_in_index(
bundle,
entry.path.name,
@ -607,6 +669,7 @@ def process_inbox(
quarantined=tuple(quarantined),
rejected=tuple(rejected),
failed=tuple(sorted(failed, key=lambda entry: entry.source_file)),
concepts=tuple(concepts),
)