feat(index): maintain existing indexes on re-materialization (spec §6)

TDD step 6: on an existing index, managed lines whose target is an
ingest-owned file removed this run are dropped, a managed label is
refreshed in place when the extraction title changes, and every other
line — curated links, promoted-verdict links, foreign line endings — is
preserved byte for byte. Load-bearing test: a promoted verdict and its
index link survive re-ingest unchanged.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01QeqhJpYQyghASjiJo5EhGg

src/llm_ingestion_okf/materialize.py

@@ -101,6 +101,36 @@ def _write_bytes(bundle_dir: Path, name: str, content: str) -> Path:
     return resolved
 
 
+def _update_index_lines(
+    index_path: Path, removed_targets: set[str], labels_by_target: dict[str, str]
+) -> None:
+    """§6 maintenance on an EXISTING index: drop managed lines whose target is
+    an ingest file removed in this run; refresh in place a managed label that
+    no longer equals the extraction title. Every other line is preserved
+    verbatim, in order — including its own line ending.
+    """
+    original = index_path.read_bytes().decode("utf-8")
+    lines = original.splitlines(keepends=True)
+    updated: list[str] = []
+    changed = False
+    for line in lines:
+        content = line.rstrip("\r\n")
+        ending = line[len(content) :]
+        match = _MANAGED_LINE_RE.match(content)
+        if match is not None:
+            target = match.group("target")
+            if target in removed_targets:
+                changed = True
+                continue
+            new_label = labels_by_target.get(target)
+            if new_label is not None and match.group("label") != new_label:
+                line = f"- [{new_label}]({target})" + ending
+                changed = True
+        updated.append(line)
+    if changed:
+        index_path.write_bytes("".join(updated).encode("utf-8"))
+
+
 def _link_in_index(bundle_dir: Path, target_name: str, label: str) -> None:
     # §6: idempotent by target — a link whose target is already present in
     # the index is never added twice.
@@ -215,8 +245,15 @@ def materialize_bundle(
     # §6 index generation — the last disk mutation. A fresh index gets
     # bundle_summary as its body; links are appended in extraction order.
     index_path = bundle / _INDEX_NAME
+    labels_by_target = {
+        generated_filename(extraction.id): extraction.title for extraction in manifest.extractions
+    }
     if not index_path.is_file():
         _write_bytes(bundle, _INDEX_NAME, manifest.bundle_summary + "\n")
+    else:
+        # Links whose target is an ingest-owned file removed this run MUST be
+        # removed; all other links — curated and promoted — are preserved.
+        _update_index_lines(index_path, owned - staged_names, labels_by_target)
     for extraction in manifest.extractions:
         _link_in_index(bundle, generated_filename(extraction.id), extraction.title)
     return IngestResult(written=written)

tests/test_index.py

@@ -0,0 +1,124 @@
+"""Index generation on an EXISTING index (ingest-spec §6).
+
+Every line the materializer does not itself manage is preserved byte for
+byte; links whose target is an ingest-owned file removed this run are
+dropped; a managed label is refreshed when the extraction title changes;
+curated and promoted links always survive — including a promoted verdict's
+(load-bearing, §11).
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any
+
+from llm_ingestion_okf.materialize import materialize_bundle
+
+INGESTED_AT = "2026-07-16T12:00:00Z"
+
+
+def make_manifest(src: Path, extraction_id: str, title: str) -> Path:
+    data: dict[str, Any] = {
+        "manifest_version": 1,
+        "source": {"type": "file", "id": "catalogue-1", "root": "data"},
+        "bundle_summary": "A test bundle.",
+        "extractions": [
+            {
+                "id": extraction_id,
+                "title": title,
+                "query": f"{extraction_id}.csv",
+                "okf_type": "dataset",
+                "max_rows": 10,
+            }
+        ],
+    }
+    src.mkdir(parents=True, exist_ok=True)
+    (src / "data").mkdir(exist_ok=True)
+    (src / "data" / f"{extraction_id}.csv").write_text("a\n1\n", encoding="utf-8", newline="")
+    path = src / "manifest.json"
+    path.write_text(json.dumps(data), encoding="utf-8")
+    return path
+
+
+def test_unmanaged_lines_preserved_and_link_appended(tmp_path: Path) -> None:
+    manifest = make_manifest(tmp_path / "src", "orders", "Orders")
+    bundle = tmp_path / "bundle"
+    bundle.mkdir()
+    original = "# My bundle\n\nSome prose.\n- [Curated](notes.md)\n"
+    (bundle / "index.md").write_text(original, encoding="utf-8", newline="")
+    materialize_bundle(manifest, bundle, INGESTED_AT)
+    expected = original + "- [Orders](ingest-orders.md)\n"
+    assert (bundle / "index.md").read_bytes() == expected.encode("utf-8")
+
+
+def test_index_without_trailing_newline_gets_one_before_links(tmp_path: Path) -> None:
+    manifest = make_manifest(tmp_path / "src", "orders", "Orders")
+    bundle = tmp_path / "bundle"
+    bundle.mkdir()
+    (bundle / "index.md").write_bytes(b"prose without newline")
+    materialize_bundle(manifest, bundle, INGESTED_AT)
+    assert (
+        bundle / "index.md"
+    ).read_bytes() == b"prose without newline\n- [Orders](ingest-orders.md)\n"
+
+
+def test_stale_ingest_link_removed_on_rematerialization(tmp_path: Path) -> None:
+    bundle = tmp_path / "bundle"
+    materialize_bundle(make_manifest(tmp_path / "a", "orders", "Orders"), bundle, INGESTED_AT)
+    materialize_bundle(make_manifest(tmp_path / "b", "refunds", "Refunds"), bundle, INGESTED_AT)
+    index = (bundle / "index.md").read_text(encoding="utf-8")
+    assert not (bundle / "ingest-orders.md").exists()
+    assert "(ingest-orders.md)" not in index
+    assert "- [Refunds](ingest-refunds.md)\n" in index
+
+
+def test_label_refreshed_when_title_changes(tmp_path: Path) -> None:
+    bundle = tmp_path / "bundle"
+    materialize_bundle(make_manifest(tmp_path / "a", "orders", "Orders"), bundle, INGESTED_AT)
+    materialize_bundle(make_manifest(tmp_path / "b", "orders", "Order lines"), bundle, INGESTED_AT)
+    index = (bundle / "index.md").read_text(encoding="utf-8")
+    assert index.count("(ingest-orders.md)") == 1
+    assert "- [Order lines](ingest-orders.md)" in index
+    assert "- [Orders](ingest-orders.md)" not in index
+
+
+def test_promoted_verdict_and_its_link_survive_reingest(tmp_path: Path) -> None:
+    """Load-bearing (spec §11): re-materialization over a bundle with a
+    promoted verdict MUST NOT delete the verdict file or its index link."""
+    bundle = tmp_path / "bundle"
+    materialize_bundle(make_manifest(tmp_path / "a", "orders", "Orders"), bundle, INGESTED_AT)
+    verdict_bytes = b"---\ntype: verdict\ndecision: approved\n---\n\nA promoted verdict.\n"
+    (bundle / "promoted-verdict-1.md").write_bytes(verdict_bytes)
+    with (bundle / "index.md").open("a", encoding="utf-8", newline="") as handle:
+        handle.write("- [Vurdering](promoted-verdict-1.md)\n")
+    # Re-ingest with a DIFFERENT extraction set so the removal path runs.
+    materialize_bundle(make_manifest(tmp_path / "b", "refunds", "Refunds"), bundle, INGESTED_AT)
+    index = (bundle / "index.md").read_text(encoding="utf-8")
+    assert (bundle / "promoted-verdict-1.md").read_bytes() == verdict_bytes
+    assert "- [Vurdering](promoted-verdict-1.md)\n" in index
+    assert "(ingest-orders.md)" not in index
+
+
+def test_curated_link_with_managed_shape_never_relabeled(tmp_path: Path) -> None:
+    """A curated line has the managed SHAPE but a non-ingest target — it is
+    never removed and never relabeled."""
+    manifest = make_manifest(tmp_path / "src", "orders", "Orders")
+    bundle = tmp_path / "bundle"
+    bundle.mkdir()
+    (bundle / "index.md").write_text("- [Old label](notes.md)\n", encoding="utf-8", newline="")
+    materialize_bundle(manifest, bundle, INGESTED_AT)
+    index = (bundle / "index.md").read_text(encoding="utf-8")
+    assert "- [Old label](notes.md)\n" in index
+
+
+def test_crlf_unmanaged_lines_preserved_verbatim(tmp_path: Path) -> None:
+    """§6 byte-for-byte preservation includes foreign line endings, also when
+    the removal path rewrites the index."""
+    bundle = tmp_path / "bundle"
+    materialize_bundle(make_manifest(tmp_path / "a", "orders", "Orders"), bundle, INGESTED_AT)
+    index_path = bundle / "index.md"
+    index_path.write_bytes(b"curated CRLF line\r\n" + index_path.read_bytes())
+    materialize_bundle(make_manifest(tmp_path / "b", "refunds", "Refunds"), bundle, INGESTED_AT)
+    assert index_path.read_bytes().startswith(b"curated CRLF line\r\n")
+    assert b"(ingest-orders.md)" not in index_path.read_bytes()