feat(inbox): accept one segmentation plan per document

src/llm_ingestion_okf/inbox.py

@@ -21,7 +21,7 @@ from __future__ import annotations
 
 import hashlib
 import unicodedata
-from collections.abc import Callable, Mapping
+from collections.abc import Callable, Mapping, Sequence
 from dataclasses import dataclass, replace
 from pathlib import Path, PurePosixPath
 
@@ -387,19 +387,61 @@ def _bundle_id_key(profile: BundleProfile) -> str:
     return profile.segmentation.bundle_id_key
 
 
-def _plan_covering(plan: SegmentationPlan | None, source_bytes: bytes) -> SegmentationPlan | None:
+def _resolve_plans(
+    segmentation: SegmentationPlan | None,
+    segmentations: Mapping[str, SegmentationPlan] | None,
+) -> tuple[SegmentationPlan, ...]:
+    """The plans this run replays, from either call form, never from both.
+
+    The mapping is keyed by SOURCE FILENAME because that is the name an
+    operator reads and maintains, but the key never selects anything -- see
+    :func:`_plan_covering`. Keeping selection on content identity is what lets
+    a renamed file still find its plan, and stops a plan filed under the wrong
+    name from segmenting the wrong document.
+
+    Both forms at once is REFUSED rather than merged. They are two ways to say
+    the same thing, and merging them would let a caller hold a plan in each and
+    never learn the two disagreed.
+    """
+    if segmentation is not None and segmentations:
+        raise SegmentationError(
+            "both `segmentation` and `segmentations` were given — pass one form or the "
+            "other; merging them would hide a disagreement between two plans for the "
+            "same document",
+            code="segmentation_plan_invalid",
+        )
+    plans = tuple(segmentations.values()) if segmentations else ()
+    if segmentation is not None:
+        plans = (segmentation,)
+
+    seen: dict[str, SegmentationPlan] = {}
+    for plan in plans:
+        if plan.source_sha256 in seen and plan is not seen[plan.source_sha256]:
+            raise SegmentationError(
+                f"two segmentation plans claim source_sha256 {plan.source_sha256!r} — "
+                "selection is by content identity, so which one segmented the document "
+                "would depend on mapping order; refusing rather than picking one",
+                code="segmentation_plan_invalid",
+            )
+        seen[plan.source_sha256] = plan
+    return plans
+
+
+def _plan_covering(
+    plans: Sequence[SegmentationPlan], source_bytes: bytes
+) -> SegmentationPlan | None:
     """The plan for THESE bytes, or None when this file is not plan-covered.
 
     Selection is by content hash, so a run may drop several documents while
-    only one of them is segmented; every other file keeps today's one-concept
+    only some of them are segmented; every other file keeps today's one-concept
     rule verbatim. The hash SELECTS; :func:`assert_plan_applies` VALIDATES,
     and the two are deliberately different questions -- see S5b.
     """
-    if plan is None:
-        return None
-    if plan.source_sha256 != hashlib.sha256(source_bytes).hexdigest():
-        return None
-    return plan
+    digest = hashlib.sha256(source_bytes).hexdigest()
+    for plan in plans:
+        if plan.source_sha256 == digest:
+            return plan
+    return None
 
 
 def _render_segments(
@@ -532,6 +574,7 @@ def process_inbox(
     profile: BundleProfile = DEFAULT,
     root_frontmatter_values: Mapping[str, str] | None = None,
     segmentation: SegmentationPlan | None = None,
+    segmentations: Mapping[str, SegmentationPlan] | None = None,
 ) -> InboxResult:
     """Convert every file dropped in `inbox_dir` into an OKF concept.
 
@@ -555,7 +598,8 @@ def process_inbox(
     # `materialize.py` states the same rule for Door A, and a door that half-built
     # a bundle before refusing would be worse than one that never started.
     root_head = _render_root_frontmatter(root_frontmatter_values or {}, profile=profile)
-    if segmentation is not None:
+    plans = _resolve_plans(segmentation, segmentations)
+    if plans:
         if profile.segmentation is None:
             raise SegmentationError(
                 "a segmentation plan was passed to a profile that does not declare the "
@@ -602,7 +646,10 @@ def process_inbox(
     # hash matched a drop but whose entry paths were then refused is a covered
     # document with a bad plan, and it must keep reporting its own per-file
     # code rather than being re-reported as a plan that matched nothing.
-    plan_matched = False
+    # WHICH plans matched, not merely that one did: a corpus run where four of
+    # five plans matched would otherwise report success over four segmented
+    # documents and one silently flat one.
+    matched_hashes: set[str] = set()
     for path in dropped:
         try:
             # Read HERE rather than in the write loop: a plan is selected by
@@ -621,8 +668,9 @@ def process_inbox(
             )
             continue
         try:
-            covering = _plan_covering(segmentation, source_bytes)
-            plan_matched = plan_matched or covering is not None
+            covering = _plan_covering(plans, source_bytes)
+            if covering is not None:
+                matched_hashes.add(covering.source_sha256)
             targets: tuple[str, ...]
             if covering is None:
                 targets = (inbox_filename(inbox_slug(path.name), profile=profile),)
@@ -643,11 +691,13 @@ def process_inbox(
     # rather than "was every file examined?", so a file that could not be read
     # cannot mask the refusal. Still before any disk mutation: Phase 1 only
     # named things.
-    if segmentation is not None and not plan_matched:
+    unmatched = sorted(set(plan.source_sha256 for plan in plans) - matched_hashes)
+    if unmatched:
         raise SegmentationError(
-            f"the segmentation plan's source_sha256 {segmentation.source_sha256!r} matches "
-            f"none of the {len(dropped)} dropped file(s) — nothing would be segmented and "
-            "the run would report success over a flat bundle; check the hash against the "
+            f"{len(unmatched)} of {len(plans)} segmentation plan(s) match none of the "
+            f"{len(dropped)} dropped file(s) — first unmatched source_sha256 "
+            f"{unmatched[0]!r}; nothing would be segmented for those documents and the "
+            "run would report success over a flat bundle; check each hash against the "
             "bytes it was adjudicated over",
             code="segmentation_plan_unmatched",
         )
@@ -728,7 +778,7 @@ def process_inbox(
             text = extract_text(
                 path.name, source_bytes, renderer=_resolve_renderer(profile, path.name)
             )
-            covering = _plan_covering(segmentation, source_bytes)
+            covering = _plan_covering(plans, source_bytes)
             if covering is not None:
                 blocked = _render_segments(
                     covering,

tests/test_segmented_inbox.py

@@ -119,6 +119,7 @@ def run(
     tmp: Path,
     *,
     plan: SegmentationPlan | None,
+    plans: dict[str, SegmentationPlan] | None = None,
     profile=SEGMENTED_V1,
     values: dict[str, str] | None = None,
     round_name: str = "round",
@@ -133,6 +134,7 @@ def run(
         profile=profile,
         root_frontmatter_values={"bundle_id": "b-1"} if values is None else values,
         segmentation=plan,
+        segmentations=plans,
     )
 
 
@@ -499,3 +501,109 @@ def test_the_observed_extractor_version_is_not_the_proposers_own(tmp_path: Path)
     with pytest.raises(SegmentationError) as excinfo:
         observed_extractor_version("nothing-registers-this")
     assert excinfo.value.code == "segmentation_extractor_mismatch"
+
+
+# --- one plan per document, so a corpus can be run at all ------------------
+#
+# `process_inbox` took ONE plan, so a round over a heterogeneous corpus with
+# several segmented documents was not expressible: arm B cannot execute
+# without this. The single-plan form stays exactly as it was -- the mapping
+# arrives as a new keyword-only parameter with a default, so every existing
+# call site is source-compatible and no golden moves.
+#
+# The mapping is keyed by SOURCE FILENAME because that is what an operator
+# reads, but selection stays by CONTENT IDENTITY: `_plan_covering` matches on
+# `source_sha256`, so a renamed file still finds its plan and a plan pointed at
+# the wrong name still refuses rather than segmenting the wrong document.
+
+OTHER = "A Innledning: hva dette dokumentet dekker.\nB Virkeomraade: hvilke anlegg det gjelder.\n"
+
+OTHER_PATHS = ("annen/innledning.md", "annen/virkeomraade.md")
+
+
+def test_two_documents_with_two_plans_both_segment_in_one_run(tmp_path: Path) -> None:
+    first = drop(tmp_path / "round", "n500.md", DOCUMENT)
+    second = drop(tmp_path / "round", "n200.md", OTHER)
+    run(
+        tmp_path,
+        plan=None,
+        plans={
+            "n500.md": build_plan(first.read_bytes(), DOCUMENT),
+            "n200.md": build_plan(second.read_bytes(), OTHER, paths=OTHER_PATHS),
+        },
+    )
+    found = concepts(tmp_path / "bundle")
+    assert set(found) == set(PATHS) | set(OTHER_PATHS)
+
+
+def test_a_mapping_selects_by_content_not_by_the_name_it_is_keyed_under(
+    tmp_path: Path,
+) -> None:
+    """The key is for the operator; the hash is what decides.
+
+    Keyed under a name no dropped file carries, the plan still finds the bytes
+    it was adjudicated over. Anything else would make a rename silently produce
+    a flat bundle.
+    """
+    source = drop(tmp_path / "round", "n500.md", DOCUMENT)
+    run(
+        tmp_path,
+        plan=None,
+        plans={"whatever-i-called-it.md": build_plan(source.read_bytes(), DOCUMENT)},
+    )
+    assert set(concepts(tmp_path / "bundle")) == set(PATHS)
+
+
+def test_a_plan_in_the_mapping_that_matches_nothing_is_still_refused(
+    tmp_path: Path,
+) -> None:
+    """The silent-skip refusal has to survive the wider input.
+
+    One plan matching is not enough: a corpus run where four of five plans
+    matched would otherwise report success over four segmented documents and
+    one flat one, which is exactly the silent zero this refusal exists for.
+    """
+    source = drop(tmp_path / "round", "n500.md", DOCUMENT)
+    with pytest.raises(SegmentationError) as excinfo:
+        run(
+            tmp_path,
+            plan=None,
+            plans={
+                "n500.md": build_plan(source.read_bytes(), DOCUMENT),
+                "ghost.md": build_plan(source.read_bytes(), DOCUMENT, source_sha256="0" * 64),
+            },
+        )
+    assert excinfo.value.code == "segmentation_plan_unmatched"
+    assert tree(tmp_path / "bundle") == {}
+
+
+def test_passing_both_forms_at_once_is_refused(tmp_path: Path) -> None:
+    """Two ways to say the same thing invite a silent disagreement."""
+    source = drop(tmp_path / "round", "n500.md", DOCUMENT)
+    plan = build_plan(source.read_bytes(), DOCUMENT)
+    with pytest.raises(SegmentationError) as excinfo:
+        run(tmp_path, plan=plan, plans={"n500.md": plan})
+    assert excinfo.value.code == "segmentation_plan_invalid"
+    assert tree(tmp_path / "bundle") == {}
+
+
+def test_two_plans_claiming_the_same_bytes_are_refused(tmp_path: Path) -> None:
+    """Which one would have segmented the document is not a coin toss."""
+    source = drop(tmp_path / "round", "n500.md", DOCUMENT)
+    with pytest.raises(SegmentationError) as excinfo:
+        run(
+            tmp_path,
+            plan=None,
+            plans={
+                "a.md": build_plan(source.read_bytes(), DOCUMENT),
+                "b.md": build_plan(source.read_bytes(), DOCUMENT, paths=OTHER_PATHS + PATHS[2:]),
+            },
+        )
+    assert excinfo.value.code == "segmentation_plan_invalid"
+    assert tree(tmp_path / "bundle") == {}
+
+
+def test_the_single_plan_form_is_unchanged(tmp_path: Path) -> None:
+    source = drop(tmp_path / "round", "n500.md", DOCUMENT)
+    run(tmp_path, plan=build_plan(source.read_bytes(), DOCUMENT))
+    assert set(concepts(tmp_path / "bundle")) == set(PATHS)