feat(index): maintain existing indexes on re-materialization (spec §6)

TDD step 6: on an existing index, managed lines whose target is an
ingest-owned file removed this run are dropped, a managed label is
refreshed in place when the extraction title changes, and every other
line — curated links, promoted-verdict links, foreign line endings — is
preserved byte for byte. Load-bearing test: a promoted verdict and its
index link survive re-ingest unchanged.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01QeqhJpYQyghASjiJo5EhGg

tests/test_index.py

@@ -0,0 +1,124 @@
+"""Index generation on an EXISTING index (ingest-spec §6).
+
+Every line the materializer does not itself manage is preserved byte for
+byte; links whose target is an ingest-owned file removed this run are
+dropped; a managed label is refreshed when the extraction title changes;
+curated and promoted links always survive — including a promoted verdict's
+(load-bearing, §11).
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any
+
+from llm_ingestion_okf.materialize import materialize_bundle
+
+INGESTED_AT = "2026-07-16T12:00:00Z"
+
+
+def make_manifest(src: Path, extraction_id: str, title: str) -> Path:
+    data: dict[str, Any] = {
+        "manifest_version": 1,
+        "source": {"type": "file", "id": "catalogue-1", "root": "data"},
+        "bundle_summary": "A test bundle.",
+        "extractions": [
+            {
+                "id": extraction_id,
+                "title": title,
+                "query": f"{extraction_id}.csv",
+                "okf_type": "dataset",
+                "max_rows": 10,
+            }
+        ],
+    }
+    src.mkdir(parents=True, exist_ok=True)
+    (src / "data").mkdir(exist_ok=True)
+    (src / "data" / f"{extraction_id}.csv").write_text("a\n1\n", encoding="utf-8", newline="")
+    path = src / "manifest.json"
+    path.write_text(json.dumps(data), encoding="utf-8")
+    return path
+
+
+def test_unmanaged_lines_preserved_and_link_appended(tmp_path: Path) -> None:
+    manifest = make_manifest(tmp_path / "src", "orders", "Orders")
+    bundle = tmp_path / "bundle"
+    bundle.mkdir()
+    original = "# My bundle\n\nSome prose.\n- [Curated](notes.md)\n"
+    (bundle / "index.md").write_text(original, encoding="utf-8", newline="")
+    materialize_bundle(manifest, bundle, INGESTED_AT)
+    expected = original + "- [Orders](ingest-orders.md)\n"
+    assert (bundle / "index.md").read_bytes() == expected.encode("utf-8")
+
+
+def test_index_without_trailing_newline_gets_one_before_links(tmp_path: Path) -> None:
+    manifest = make_manifest(tmp_path / "src", "orders", "Orders")
+    bundle = tmp_path / "bundle"
+    bundle.mkdir()
+    (bundle / "index.md").write_bytes(b"prose without newline")
+    materialize_bundle(manifest, bundle, INGESTED_AT)
+    assert (
+        bundle / "index.md"
+    ).read_bytes() == b"prose without newline\n- [Orders](ingest-orders.md)\n"
+
+
+def test_stale_ingest_link_removed_on_rematerialization(tmp_path: Path) -> None:
+    bundle = tmp_path / "bundle"
+    materialize_bundle(make_manifest(tmp_path / "a", "orders", "Orders"), bundle, INGESTED_AT)
+    materialize_bundle(make_manifest(tmp_path / "b", "refunds", "Refunds"), bundle, INGESTED_AT)
+    index = (bundle / "index.md").read_text(encoding="utf-8")
+    assert not (bundle / "ingest-orders.md").exists()
+    assert "(ingest-orders.md)" not in index
+    assert "- [Refunds](ingest-refunds.md)\n" in index
+
+
+def test_label_refreshed_when_title_changes(tmp_path: Path) -> None:
+    bundle = tmp_path / "bundle"
+    materialize_bundle(make_manifest(tmp_path / "a", "orders", "Orders"), bundle, INGESTED_AT)
+    materialize_bundle(make_manifest(tmp_path / "b", "orders", "Order lines"), bundle, INGESTED_AT)
+    index = (bundle / "index.md").read_text(encoding="utf-8")
+    assert index.count("(ingest-orders.md)") == 1
+    assert "- [Order lines](ingest-orders.md)" in index
+    assert "- [Orders](ingest-orders.md)" not in index
+
+
+def test_promoted_verdict_and_its_link_survive_reingest(tmp_path: Path) -> None:
+    """Load-bearing (spec §11): re-materialization over a bundle with a
+    promoted verdict MUST NOT delete the verdict file or its index link."""
+    bundle = tmp_path / "bundle"
+    materialize_bundle(make_manifest(tmp_path / "a", "orders", "Orders"), bundle, INGESTED_AT)
+    verdict_bytes = b"---\ntype: verdict\ndecision: approved\n---\n\nA promoted verdict.\n"
+    (bundle / "promoted-verdict-1.md").write_bytes(verdict_bytes)
+    with (bundle / "index.md").open("a", encoding="utf-8", newline="") as handle:
+        handle.write("- [Vurdering](promoted-verdict-1.md)\n")
+    # Re-ingest with a DIFFERENT extraction set so the removal path runs.
+    materialize_bundle(make_manifest(tmp_path / "b", "refunds", "Refunds"), bundle, INGESTED_AT)
+    index = (bundle / "index.md").read_text(encoding="utf-8")
+    assert (bundle / "promoted-verdict-1.md").read_bytes() == verdict_bytes
+    assert "- [Vurdering](promoted-verdict-1.md)\n" in index
+    assert "(ingest-orders.md)" not in index
+
+
+def test_curated_link_with_managed_shape_never_relabeled(tmp_path: Path) -> None:
+    """A curated line has the managed SHAPE but a non-ingest target — it is
+    never removed and never relabeled."""
+    manifest = make_manifest(tmp_path / "src", "orders", "Orders")
+    bundle = tmp_path / "bundle"
+    bundle.mkdir()
+    (bundle / "index.md").write_text("- [Old label](notes.md)\n", encoding="utf-8", newline="")
+    materialize_bundle(manifest, bundle, INGESTED_AT)
+    index = (bundle / "index.md").read_text(encoding="utf-8")
+    assert "- [Old label](notes.md)\n" in index
+
+
+def test_crlf_unmanaged_lines_preserved_verbatim(tmp_path: Path) -> None:
+    """§6 byte-for-byte preservation includes foreign line endings, also when
+    the removal path rewrites the index."""
+    bundle = tmp_path / "bundle"
+    materialize_bundle(make_manifest(tmp_path / "a", "orders", "Orders"), bundle, INGESTED_AT)
+    index_path = bundle / "index.md"
+    index_path.write_bytes(b"curated CRLF line\r\n" + index_path.read_bytes())
+    materialize_bundle(make_manifest(tmp_path / "b", "refunds", "Refunds"), bundle, INGESTED_AT)
+    assert index_path.read_bytes().startswith(b"curated CRLF line\r\n")
+    assert b"(ingest-orders.md)" not in index_path.read_bytes()