"""Index generation on an EXISTING index (ingest-spec §6). Every line the materializer does not itself manage is preserved byte for byte; links whose target is an ingest-owned file removed this run are dropped; a managed label is refreshed when the extraction title changes; curated and promoted links always survive — including a promoted verdict's (load-bearing, §11). """ from __future__ import annotations import json from pathlib import Path from typing import Any from llm_ingestion_okf.materialize import materialize_bundle INGESTED_AT = "2026-07-16T12:00:00Z" def make_manifest(src: Path, extraction_id: str, title: str) -> Path: data: dict[str, Any] = { "manifest_version": 1, "source": {"type": "file", "id": "catalogue-1", "root": "data"}, "bundle_summary": "A test bundle.", "extractions": [ { "id": extraction_id, "title": title, "query": f"{extraction_id}.csv", "okf_type": "dataset", "max_rows": 10, } ], } src.mkdir(parents=True, exist_ok=True) (src / "data").mkdir(exist_ok=True) (src / "data" / f"{extraction_id}.csv").write_text("a\n1\n", encoding="utf-8", newline="") path = src / "manifest.json" path.write_text(json.dumps(data), encoding="utf-8") return path def test_unmanaged_lines_preserved_and_link_appended(tmp_path: Path) -> None: manifest = make_manifest(tmp_path / "src", "orders", "Orders") bundle = tmp_path / "bundle" bundle.mkdir() original = "# My bundle\n\nSome prose.\n- [Curated](notes.md)\n" (bundle / "index.md").write_text(original, encoding="utf-8", newline="") materialize_bundle(manifest, bundle, INGESTED_AT) expected = original + "- [Orders](ingest-orders.md)\n" assert (bundle / "index.md").read_bytes() == expected.encode("utf-8") def test_index_without_trailing_newline_gets_one_before_links(tmp_path: Path) -> None: manifest = make_manifest(tmp_path / "src", "orders", "Orders") bundle = tmp_path / "bundle" bundle.mkdir() (bundle / "index.md").write_bytes(b"prose without newline") materialize_bundle(manifest, bundle, INGESTED_AT) assert ( bundle / "index.md" ).read_bytes() == b"prose without newline\n- [Orders](ingest-orders.md)\n" def test_stale_ingest_link_removed_on_rematerialization(tmp_path: Path) -> None: bundle = tmp_path / "bundle" materialize_bundle(make_manifest(tmp_path / "a", "orders", "Orders"), bundle, INGESTED_AT) materialize_bundle(make_manifest(tmp_path / "b", "refunds", "Refunds"), bundle, INGESTED_AT) index = (bundle / "index.md").read_text(encoding="utf-8") assert not (bundle / "ingest-orders.md").exists() assert "(ingest-orders.md)" not in index assert "- [Refunds](ingest-refunds.md)\n" in index def test_label_refreshed_when_title_changes(tmp_path: Path) -> None: bundle = tmp_path / "bundle" materialize_bundle(make_manifest(tmp_path / "a", "orders", "Orders"), bundle, INGESTED_AT) materialize_bundle(make_manifest(tmp_path / "b", "orders", "Order lines"), bundle, INGESTED_AT) index = (bundle / "index.md").read_text(encoding="utf-8") assert index.count("(ingest-orders.md)") == 1 assert "- [Order lines](ingest-orders.md)" in index assert "- [Orders](ingest-orders.md)" not in index def test_promoted_verdict_and_its_link_survive_reingest(tmp_path: Path) -> None: """Load-bearing (spec §11): re-materialization over a bundle with a promoted verdict MUST NOT delete the verdict file or its index link.""" bundle = tmp_path / "bundle" materialize_bundle(make_manifest(tmp_path / "a", "orders", "Orders"), bundle, INGESTED_AT) verdict_bytes = b"---\ntype: verdict\ndecision: approved\n---\n\nA promoted verdict.\n" (bundle / "promoted-verdict-1.md").write_bytes(verdict_bytes) with (bundle / "index.md").open("a", encoding="utf-8", newline="") as handle: handle.write("- [Vurdering](promoted-verdict-1.md)\n") # Re-ingest with a DIFFERENT extraction set so the removal path runs. materialize_bundle(make_manifest(tmp_path / "b", "refunds", "Refunds"), bundle, INGESTED_AT) index = (bundle / "index.md").read_text(encoding="utf-8") assert (bundle / "promoted-verdict-1.md").read_bytes() == verdict_bytes assert "- [Vurdering](promoted-verdict-1.md)\n" in index assert "(ingest-orders.md)" not in index def test_curated_link_with_managed_shape_never_relabeled(tmp_path: Path) -> None: """A curated line has the managed SHAPE but a non-ingest target — it is never removed and never relabeled.""" manifest = make_manifest(tmp_path / "src", "orders", "Orders") bundle = tmp_path / "bundle" bundle.mkdir() (bundle / "index.md").write_text("- [Old label](notes.md)\n", encoding="utf-8", newline="") materialize_bundle(manifest, bundle, INGESTED_AT) index = (bundle / "index.md").read_text(encoding="utf-8") assert "- [Old label](notes.md)\n" in index def test_crlf_unmanaged_lines_preserved_verbatim(tmp_path: Path) -> None: """§6 byte-for-byte preservation includes foreign line endings, also when the removal path rewrites the index.""" bundle = tmp_path / "bundle" materialize_bundle(make_manifest(tmp_path / "a", "orders", "Orders"), bundle, INGESTED_AT) index_path = bundle / "index.md" index_path.write_bytes(b"curated CRLF line\r\n" + index_path.read_bytes()) materialize_bundle(make_manifest(tmp_path / "b", "refunds", "Refunds"), bundle, INGESTED_AT) assert index_path.read_bytes().startswith(b"curated CRLF line\r\n") assert b"(ingest-orders.md)" not in index_path.read_bytes()