open/llm-ingestion-okf
1
0
Fork 0
Code Issues Releases 1 Activity

feat(inbox): walk the drop directory recursively

Browse source 
Door B listed `inbox.iterdir()` and kept only top-level files. A file in a
subdirectory was neither ingested nor refused: it appeared in none of the
result's buckets, so a nested drop produced a bundle that was silently short
of what was dropped and no count said so. That broke the K1b identity for any
inbox with folders in it. Operator decision 2026-09-06.

- `walk_inbox` is the ONE walk rule, shared with `tools/okf_corpus_run.py`:
  the denominator N is now counted over exactly the set of files the door
  ingests, rather than over a second listing that happened to agree.
- Sorted on the whole relative path, not the basename, so the order is a
  function of the tree; that is what keeps rebuild-from-scratch byte-equal to
  an incremental update.
- A concept's `source_file` is the path relative to the inbox root,
  `/`-separated. The concept NAME still comes from the basename, so two
  folders holding one basename hit the existing §3 collision refusal instead
  of one silently claiming the other's concept.
- Dot-directories and a bundle directory inside the inbox are skipped with a
  CODE, in a new `InboxResult.skipped`. Recursion makes the door's own output
  reachable as its own input; a silent skip would be the same
  absence-without-a-denominator defect one level down.
- `--path-prefix` reduces per component and rejoins with `/`, so the caller
  driving a nested corpus can carry the relative directory. Reducing the whole
  string folded the separator into a `-` and flattened `sub/sub2`.

`tests/test_inbox_flow.py::test_subdirectories_are_not_walked` asserted the
opposite and is superseded in place, with the reason written down.

Measured on the K2 corpus (flat, N=43): 39/43 merged, 4 coded, K1b holds. The
bundle digest is
`1472e98aec8643c5beee540f4c42b5e437bd26e7c61d69a91bcff799f06a6d13` over 1108
files -- byte-identical to a run of the same corpus at 190086f WITHOUT this
change (`diff -r` exit 0), so recursion costs a flat inbox nothing. It differs
from the stored 2026-09-03 artifact by one line in `index.md`
(`- [Corpus run history](log.md)`), which 95eb271 added 15 hours after that
bundle was built.

Suite 1113 passed, `ruff` clean, `mypy --strict src/ tools/` clean.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Kjell Tore Guttormsen 2026-09-07 04:11:00 +02:00
commit aa87eb8818
11 changed files with 489 additions and 39 deletions
Download patch file Download diff file Expand all files Collapse all files

2
src/llm_ingestion_okf/__init__.py
View file

@ -52,6 +52,7 @@ from .inbox import (
GateDecision,
InboxResult,
PersistedFile,
SkippedPath,
process_inbox,
)
from .importer import (
@ -101,6 +102,7 @@ __all__ = [
"MergedConcept",
"NetworkGateError",
"PersistedFile",
"SkippedPath",
"RefusedConcept",
"RenderError",
"SourceError",

156
src/llm_ingestion_okf/inbox.py
View file

@ -20,6 +20,7 @@ supplies the verdict and this module only obeys it.
from __future__ import annotations
import hashlib
import os
import unicodedata
from collections.abc import Callable, Mapping, Sequence
from dataclasses import dataclass, replace
@ -275,6 +276,26 @@ class FailedFile:
error: IngestError
#: Why the walk refused to descend into a directory. A CODE rather than a log
#: line: a skipped directory is an outcome the caller has to be able to count,
#: and "we found nothing under here" is a measurement, not a fact about the
#: inbox.
SKIPPED_DOT_DIRECTORY = "skipped_dot_directory"
SKIPPED_BUNDLE_DIRECTORY = "skipped_bundle_directory"
@dataclass(frozen=True)
class SkippedPath:
"""A directory the recursive walk did not descend into, and why.
`path` is relative to the inbox root, `/`-separated, exactly like a
concept's `source_file`.
"""
path: str
code: str
@dataclass(frozen=True)
class InboxResult:
"""Every dropped file's outcome, in sorted filename order.
@ -294,6 +315,72 @@ class InboxResult:
# consumer's number means. Without segmentation the two are equal, which is
# what makes this additive rather than a second thing to keep in step.
concepts: tuple[PersistedFile, ...] = ()
# Directories the walk refused to enter, each with its code. Additive and
# last, so every existing positional construction and every existing
# consumer's four buckets keep their meaning: a skipped directory holds no
# dropped FILE outcome, it explains a set of files that were never dropped.
skipped: tuple[SkippedPath, ...] = ()
def relative_source(path: Path, inbox: Path) -> str:
"""A dropped file's name as the provenance layer records it.
Relative to the inbox root and `/`-separated, so the recorded name is the
same on every platform and two documents sharing a basename in different
folders stay distinguishable. For a flat inbox this is the bare filename,
which is why every existing bundle's bytes are unchanged.
"""
return path.relative_to(inbox).as_posix()
def walk_inbox(
inbox: Path, *, exclude: Path | None = None
) -> tuple[tuple[Path, ...], tuple[SkippedPath, ...]]:
"""Every dropped file at any depth, sorted by relative path, plus the skips.
ONE implementation, shared with `tools/okf_corpus_run.py`: the corpus
measurement counts the denominator N, and a walk that disagreed with the
door's would report a count over a different set of files than the one that
was ingested.
Sorted on the whole relative path rather than the basename -- the order has
to be a function of the TREE, or two inboxes holding the same documents
would walk them differently and their indexes would diverge.
Two directories are refused, both with a code. A dot-directory is
machinery, not documents. `exclude` is the bundle: with a flat listing a
bundle nested inside the inbox was invisible by accident, and recursion
removes the accident -- without the skip the door would extract its own
concepts and ingest them as documents on the next run.
"""
excluded = exclude.resolve() if exclude is not None else None
dropped: list[Path] = []
skipped: list[SkippedPath] = []
for parent_name, dirnames, filenames in os.walk(inbox):
parent = Path(parent_name)
kept: list[str] = []
for name in sorted(dirnames):
child = parent / name
if name.startswith("."):
skipped.append(
SkippedPath(path=relative_source(child, inbox), code=SKIPPED_DOT_DIRECTORY)
)
elif excluded is not None and child.resolve() == excluded:
skipped.append(
SkippedPath(path=relative_source(child, inbox), code=SKIPPED_BUNDLE_DIRECTORY)
)
else:
kept.append(name)
# In place: this is how `os.walk` is told not to descend, and a skipped
# directory must not be entered at all rather than entered and filtered.
dirnames[:] = kept
for name in filenames:
child = parent / name
if child.is_file():
dropped.append(child)
dropped.sort(key=lambda item: relative_source(item, inbox))
skipped.sort(key=lambda item: item.path)
return (tuple(dropped), tuple(skipped))
def _is_inbox_owned(path: Path) -> bool:
@ -454,6 +541,7 @@ def _render_segments(
gate: Gate,
profile: BundleProfile,
bundle_id: str,
source_file: str,
) -> BlockedFile | None:
"""Render every segment, or refuse the WHOLE document.
@ -467,7 +555,7 @@ def _render_segments(
A refusal is therefore reported once, for the document, rather than once
per segment: the operator's unit of review is the document they dropped.
"""
extractor_id = Path(path.name).suffix.lower().lstrip(".") or "none"
extractor_id = path.suffix.lower().lstrip(".") or "none"
assert_plan_applies(
plan,
source_sha256=hashlib.sha256(source_bytes).hexdigest(),
@ -492,7 +580,7 @@ def _render_segments(
]
if refused:
return BlockedFile(
source_file=path.name,
source_file=source_file,
disposition=refused[0].disposition,
reasons=tuple(reason for decision in refused for reason in decision.reasons),
)
@ -500,7 +588,7 @@ def _render_segments(
for entry, decision in decisions:
structure: DocumentStructure | None = None
if profile.index.facets is not None:
structure = derive_document_structure(decision.sanitized_text, source_file=path.name)
structure = derive_document_structure(decision.sanitized_text, source_file=source_file)
_validate_facets(structure, profile)
outputs.append(
(
@ -512,7 +600,7 @@ def _render_segments(
# segment's own first line: the plan is the record of the
# judgement, and a heading inside a slice is not it.
title=entry.title,
source_file=path.name,
source_file=source_file,
source_bytes=source_bytes,
ingested_at=entry.ingested_at,
profile=profile,
@ -623,10 +711,16 @@ def process_inbox(
inbox = Path(inbox_dir)
if not inbox.is_dir():
raise SourceError(f"inbox directory does not exist: {inbox}", code="source_root_missing")
bundle = Path(bundle_dir)
# Top-level only, sorted: the operator's nested structure is theirs, and a
# deterministic order is what makes a re-run comparable.
dropped = sorted((path for path in inbox.iterdir() if path.is_file()), key=lambda p: p.name)
# RECURSIVE, sorted by relative path. The operator's nested structure is
# theirs to arrange, but it is not theirs to lose: a file under a folder
# used to be neither ingested nor refused, so a bundle built from a nested
# drop was silently short of what was dropped.
dropped, skipped = walk_inbox(inbox, exclude=bundle)
def source_name(path: Path) -> str:
return relative_source(path, inbox)
persisted: list[PersistedFile] = []
concepts: list[PersistedFile] = []
@ -660,9 +754,10 @@ def process_inbox(
except OSError as exc:
failed.append(
FailedFile(
source_file=path.name,
source_file=source_name(path),
error=SourceError(
f"cannot read dropped file {path.name}: {exc}", code="source_file_missing"
f"cannot read dropped file {source_name(path)}: {exc}",
code="source_file_missing",
),
)
)
@ -673,11 +768,16 @@ def process_inbox(
matched_hashes.add(covering.source_sha256)
targets: tuple[str, ...]
if covering is None:
# From the BASENAME, not the relative path: the concept name
# is the bundle's, and folding a folder into it would rename
# every concept the moment an operator tidied their inbox. Two
# folders holding the same basename therefore collide, and the
# §3 gate below refuses both rather than picking a winner.
targets = (inbox_filename(inbox_slug(path.name), profile=profile),)
else:
targets = tuple(_check_segment_path(item.path) for item in covering.entries)
except IngestError as exc:
failed.append(FailedFile(source_file=path.name, error=exc))
failed.append(FailedFile(source_file=source_name(path), error=exc))
continue
named.append((path, targets, source_bytes, covering is not None))
for target in targets:
@ -707,17 +807,17 @@ def process_inbox(
# five colliding paths is one thing the operator has to fix, and five
# identical entries would report the same rename five times.
for path in sorted(
{owner for name in contested for owner in slug_owners[name]}, key=lambda item: item.name
{owner for name in contested for owner in slug_owners[name]}, key=source_name
):
claimed = sorted(name for name in contested if path in slug_owners[name])
others = sorted(
{other.name for name in claimed for other in slug_owners[name] if other != path}
{source_name(other) for name in claimed for other in slug_owners[name] if other != path}
)
failed.append(
FailedFile(
source_file=path.name,
source_file=source_name(path),
error=MaterializationError(
f"{path.name!r} and {', '.join(repr(other) for other in others)}"
f"{source_name(path)!r} and {', '.join(repr(other) for other in others)}"
f" both reduce to {', '.join(repr(name) for name in claimed)}"
" — rename one; refusing to pick a winner",
code="inbox_slug_collision",
@ -728,7 +828,6 @@ def process_inbox(
# Phase 2: the §3 ownership scan, evaluated against the bundle as it was
# BEFORE this run — a file written below must never be mistaken for
# pre-existing curated content by a later file's check.
bundle = Path(bundle_dir)
owned_by_source: dict[str, set[str]] = {}
if profile.segmentation is not None:
# RECURSIVE, and only here. A nested concept is invisible to a flat
@ -763,7 +862,7 @@ def process_inbox(
if unstamped:
failed.append(
FailedFile(
source_file=path.name,
source_file=source_name(path),
error=MaterializationError(
f"generated filename {unstamped[0]!r} collides with an existing file "
"that does not carry the inbox marker — refusing to overwrite curated "
@ -776,7 +875,9 @@ def process_inbox(
outputs: list[tuple[str, str, tuple[str, ...]]] = []
try:
text = extract_text(
path.name, source_bytes, renderer=_resolve_renderer(profile, path.name)
source_name(path),
source_bytes,
renderer=_resolve_renderer(profile, path.name),
)
covering = _plan_covering(plans, source_bytes)
if covering is not None:
@ -789,6 +890,7 @@ def process_inbox(
gate=gate,
profile=profile,
bundle_id=(root_frontmatter_values or {})[_bundle_id_key(profile)],
source_file=source_name(path),
)
if blocked is not None:
if blocked.disposition == _DISPOSITION_QUARANTINE:
@ -800,7 +902,7 @@ def process_inbox(
decision = gate(text)
if decision.disposition != _DISPOSITION_PERSIST:
blocked = BlockedFile(
source_file=path.name,
source_file=source_name(path),
disposition=decision.disposition,
reasons=decision.reasons,
)
@ -819,7 +921,7 @@ def process_inbox(
# deriving from bytes the gate rejected would put unscreened
# content in the frontmatter and the index.
structure = derive_document_structure(
decision.sanitized_text, source_file=path.name
decision.sanitized_text, source_file=source_name(path)
)
title = structure.title
_validate_facets(structure, profile)
@ -830,7 +932,7 @@ def process_inbox(
decision.sanitized_text,
okf_type=okf_type,
title=title,
source_file=path.name,
source_file=source_name(path),
source_bytes=source_bytes,
ingested_at=ingested_at,
profile=profile,
@ -842,15 +944,16 @@ def process_inbox(
except OSError as exc:
failed.append(
FailedFile(
source_file=path.name,
source_file=source_name(path),
error=SourceError(
f"cannot read dropped file {path.name}: {exc}", code="source_file_missing"
f"cannot read dropped file {source_name(path)}: {exc}",
code="source_file_missing",
),
)
)
continue
except IngestError as exc:
failed.append(FailedFile(source_file=path.name, error=exc))
failed.append(FailedFile(source_file=source_name(path), error=exc))
continue
bundle.mkdir(parents=True, exist_ok=True)
@ -859,7 +962,9 @@ def process_inbox(
# creates one. Without this the very first hierarchical write fails.
(bundle / target_name).parent.mkdir(parents=True, exist_ok=True)
written = write_bytes(bundle, target_name, content)
concepts.append(PersistedFile(source_file=path.name, path=written, reasons=reasons))
concepts.append(
PersistedFile(source_file=source_name(path), path=written, reasons=reasons)
)
# One entry per SOURCE FILE, whatever the document expanded into. That
# is what `persisted` has always meant, so an existing consumer's count
# does not change under a profile that segments.
@ -870,7 +975,7 @@ def process_inbox(
# document owned before and does not now is retired HERE, after the
# writes, so a failure above leaves the previous round intact.
_retire_stale_segments(
bundle, owned_by_source.get(path.name, set()) - set(targets), profile
bundle, owned_by_source.get(source_name(path), set()) - set(targets), profile
)
# §6 index — the last disk mutation, and only when something was written.
@ -907,6 +1012,7 @@ def process_inbox(
rejected=tuple(rejected),
failed=tuple(sorted(failed, key=lambda entry: entry.source_file)),
concepts=tuple(concepts),
skipped=skipped,
)