llm-ingestion-okf/tests/test_inbox_recursion.py

"""Door B walks the inbox RECURSIVELY (order 20260906T220349Z-6526414037).

Until this suite the door listed `inbox.iterdir()` and kept only top-level
files. A file in a subdirectory was neither ingested nor refused: it did not
appear in any of the result's buckets, so a bundle built from a nested drop
was silently short and no count said so. That is the absence-without-a-
denominator failure, and it broke the K1b identity (bundle == what was
dropped) for every operator whose inbox has folders in it.

What is pinned here:

- a file at any depth is ingested, and its `source_file` is the path RELATIVE
  to the inbox root, `/`-separated, so two documents that share a basename in
  different folders are still distinguishable in the provenance layer;
- the walk order is the sorted relative path, which is what keeps a
  rebuild-from-scratch byte-identical to an incremental update (the K6 gate);
- a dot-directory and a bundle directory sitting INSIDE the inbox are skipped
  with a CODE, never silently — the door's own output must not be re-ingested
  as input, and an operator has to be able to see that it was not;
- the generated concept name still comes from the BASENAME, so two nested
  files reducing to one name hit the existing §3 collision refusal rather
  than one silently claiming the other's concept.
"""

from __future__ import annotations

import hashlib
from pathlib import Path

from llm_ingestion_okf.inbox import (
    SKIPPED_BUNDLE_DIRECTORY,
    SKIPPED_DOT_DIRECTORY,
    GateDecision,
    process_inbox,
)
from llm_ingestion_okf.profiles import STRUCTURED_V1

INGESTED_AT = "2026-09-07T08:00:00Z"


def _gate(text: str) -> GateDecision:
    return GateDecision(sanitized_text=text, disposition="warn", reasons=())


def _run(inbox: Path, bundle: Path, **kwargs: object) -> object:
    return process_inbox(
        inbox,
        bundle,
        INGESTED_AT,
        okf_type="reference",
        gate=_gate,
        **kwargs,  # type: ignore[arg-type]
    )


def _drop(inbox: Path, relative: str, body: str) -> None:
    path = inbox / relative
    path.parent.mkdir(parents=True, exist_ok=True)
    path.write_text(body, encoding="utf-8")


def test_a_file_in_a_subdirectory_is_ingested_and_carries_its_relative_path(
    tmp_path: Path,
) -> None:
    inbox = tmp_path / "inbox"
    inbox.mkdir()
    _drop(inbox, "top.md", "Top level.\n")
    _drop(inbox, "sub/one.md", "One deep.\n")
    _drop(inbox, "sub/sub2/two.md", "Two deep.\n")

    result = _run(inbox, tmp_path / "bundle")

    assert [item.source_file for item in result.persisted] == [  # type: ignore[attr-defined]
        "sub/one.md",
        "sub/sub2/two.md",
        "top.md",
    ]
    assert result.failed == ()  # type: ignore[attr-defined]
    assert result.rejected == ()  # type: ignore[attr-defined]
    concept = (tmp_path / "bundle" / "inbox-two.md").read_text(encoding="utf-8")
    assert "source_file: sub/sub2/two.md" in concept


def test_the_walk_order_is_the_sorted_relative_path(tmp_path: Path) -> None:
    """Sorted on the WHOLE relative path, not the basename.

    Sorting on the basename would order `b/a.md` before `a/z.md`, which makes
    the order depend on names rather than on the tree — two inboxes holding
    the same files would walk them differently and the index would differ.
    """
    inbox = tmp_path / "inbox"
    inbox.mkdir()
    _drop(inbox, "b/aaa.md", "B then aaa.\n")
    _drop(inbox, "a/zzz.md", "A then zzz.\n")

    result = _run(inbox, tmp_path / "bundle")

    assert [item.source_file for item in result.persisted] == [  # type: ignore[attr-defined]
        "a/zzz.md",
        "b/aaa.md",
    ]


def test_a_dot_directory_is_skipped_with_a_code(tmp_path: Path) -> None:
    inbox = tmp_path / "inbox"
    inbox.mkdir()
    _drop(inbox, "kept.md", "Kept.\n")
    _drop(inbox, ".git/config.md", "Not a document.\n")
    _drop(inbox, ".git/objects/deep.md", "Not a document either.\n")

    result = _run(inbox, tmp_path / "bundle")

    assert [item.source_file for item in result.persisted] == ["kept.md"]  # type: ignore[attr-defined]
    assert [(item.path, item.code) for item in result.skipped] == [  # type: ignore[attr-defined]
        (".git", SKIPPED_DOT_DIRECTORY)
    ]


def test_a_bundle_directory_inside_the_inbox_is_skipped_with_a_code(tmp_path: Path) -> None:
    """The door's own output is not its own input.

    With a flat listing a nested bundle was invisible by accident. Recursion
    removes that accident: without this skip the second run would extract the
    concepts written by the first, ingest them as documents, and grow the
    bundle on every run.
    """
    inbox = tmp_path / "inbox"
    inbox.mkdir()
    bundle = inbox / "bundle"
    _drop(inbox, "doc.md", "A document.\n")

    first = _run(inbox, bundle)
    assert [item.source_file for item in first.persisted] == ["doc.md"]  # type: ignore[attr-defined]
    assert first.skipped == ()  # type: ignore[attr-defined]

    second = _run(inbox, bundle)

    assert [item.source_file for item in second.persisted] == ["doc.md"]  # type: ignore[attr-defined]
    assert [(item.path, item.code) for item in second.skipped] == [  # type: ignore[attr-defined]
        ("bundle", SKIPPED_BUNDLE_DIRECTORY)
    ]


def test_two_nested_files_sharing_a_basename_are_refused_not_silently_merged(
    tmp_path: Path,
) -> None:
    """The concept name still comes from the basename, so this is a collision.

    Stated as a test rather than left implicit: recursion makes it reachable
    for the first time, and the answer is the §3 refusal the door already
    gives — never a winner picked by walk order.
    """
    inbox = tmp_path / "inbox"
    inbox.mkdir()
    _drop(inbox, "a/same.md", "First.\n")
    _drop(inbox, "b/same.md", "Second.\n")

    result = _run(inbox, tmp_path / "bundle")

    assert result.persisted == ()  # type: ignore[attr-defined]
    assert [item.source_file for item in result.failed] == ["a/same.md", "b/same.md"]  # type: ignore[attr-defined]
    assert {item.error.code for item in result.failed} == {"inbox_slug_collision"}  # type: ignore[attr-defined]


def _tree_hash(root: Path) -> str:
    digest = hashlib.sha256()
    for path in sorted(root.rglob("*")):
        if not path.is_file():
            continue
        digest.update(path.relative_to(root).as_posix().encode("utf-8"))
        digest.update(b"\0")
        digest.update(path.read_bytes())
        digest.update(b"\0")
    return digest.hexdigest()


def test_a_nested_rebuild_from_scratch_equals_the_incremental_bundle(tmp_path: Path) -> None:
    """The K6 identity, over an inbox with subdirectories.

    The faceted index is a projection of the WHOLE bundle, so it is exactly
    where a walk order that depended on arrival would show up. Building one
    bundle in two rounds and another in one must produce the same bytes.
    """
    inbox = tmp_path / "inbox"
    inbox.mkdir()
    _drop(inbox, "sub/one.md", "# Ett\n\nFirst document.\n")
    _drop(inbox, "sub/sub2/two.md", "# To\n\nSecond document.\n")

    incremental = tmp_path / "incremental"
    _run(inbox, incremental, profile=STRUCTURED_V1)
    _drop(inbox, "three.md", "# Tre\n\nThird document.\n")
    _run(inbox, incremental, profile=STRUCTURED_V1)

    scratch = tmp_path / "scratch"
    _run(inbox, scratch, profile=STRUCTURED_V1)

    # The control FIRST: two empty trees hash the same, so an equality that
    # ran before recursion existed would have been a green over nothing.
    assert sorted(path.name for path in scratch.rglob("*.md")) == [
        "inbox-one.md",
        "inbox-three.md",
        "inbox-two.md",
        "index.md",
    ]
    assert _tree_hash(incremental) == _tree_hash(scratch)