llm-ingestion-okf/tests/test_index.py

"""Index generation on an EXISTING index (ingest-spec §6).

Every line the materializer does not itself manage is preserved byte for
byte; links whose target is an ingest-owned file removed this run are
dropped; a managed label is refreshed when the extraction title changes;
curated and promoted links always survive — including a promoted verdict's
(load-bearing, §11).
"""

from __future__ import annotations

import json
from pathlib import Path
from typing import Any

from llm_ingestion_okf.materialize import materialize_bundle

INGESTED_AT = "2026-07-16T12:00:00Z"


def make_manifest(src: Path, extraction_id: str, title: str) -> Path:
    data: dict[str, Any] = {
        "manifest_version": 1,
        "source": {"type": "file", "id": "catalogue-1", "root": "data"},
        "bundle_summary": "A test bundle.",
        "extractions": [
            {
                "id": extraction_id,
                "title": title,
                "query": f"{extraction_id}.csv",
                "okf_type": "dataset",
                "max_rows": 10,
            }
        ],
    }
    src.mkdir(parents=True, exist_ok=True)
    (src / "data").mkdir(exist_ok=True)
    (src / "data" / f"{extraction_id}.csv").write_text("a\n1\n", encoding="utf-8", newline="")
    path = src / "manifest.json"
    path.write_text(json.dumps(data), encoding="utf-8")
    return path


def test_unmanaged_lines_preserved_and_link_appended(tmp_path: Path) -> None:
    manifest = make_manifest(tmp_path / "src", "orders", "Orders")
    bundle = tmp_path / "bundle"
    bundle.mkdir()
    original = "# My bundle\n\nSome prose.\n- [Curated](notes.md)\n"
    (bundle / "index.md").write_text(original, encoding="utf-8", newline="")
    materialize_bundle(manifest, bundle, INGESTED_AT)
    expected = original + "- [Orders](ingest-orders.md)\n"
    assert (bundle / "index.md").read_bytes() == expected.encode("utf-8")


def test_index_without_trailing_newline_gets_one_before_links(tmp_path: Path) -> None:
    manifest = make_manifest(tmp_path / "src", "orders", "Orders")
    bundle = tmp_path / "bundle"
    bundle.mkdir()
    (bundle / "index.md").write_bytes(b"prose without newline")
    materialize_bundle(manifest, bundle, INGESTED_AT)
    assert (
        bundle / "index.md"
    ).read_bytes() == b"prose without newline\n- [Orders](ingest-orders.md)\n"


def test_stale_ingest_link_removed_on_rematerialization(tmp_path: Path) -> None:
    bundle = tmp_path / "bundle"
    materialize_bundle(make_manifest(tmp_path / "a", "orders", "Orders"), bundle, INGESTED_AT)
    materialize_bundle(make_manifest(tmp_path / "b", "refunds", "Refunds"), bundle, INGESTED_AT)
    index = (bundle / "index.md").read_text(encoding="utf-8")
    assert not (bundle / "ingest-orders.md").exists()
    assert "(ingest-orders.md)" not in index
    assert "- [Refunds](ingest-refunds.md)\n" in index


def test_label_refreshed_when_title_changes(tmp_path: Path) -> None:
    bundle = tmp_path / "bundle"
    materialize_bundle(make_manifest(tmp_path / "a", "orders", "Orders"), bundle, INGESTED_AT)
    materialize_bundle(make_manifest(tmp_path / "b", "orders", "Order lines"), bundle, INGESTED_AT)
    index = (bundle / "index.md").read_text(encoding="utf-8")
    assert index.count("(ingest-orders.md)") == 1
    assert "- [Order lines](ingest-orders.md)" in index
    assert "- [Orders](ingest-orders.md)" not in index


def test_promoted_verdict_and_its_link_survive_reingest(tmp_path: Path) -> None:
    """Load-bearing (spec §11): re-materialization over a bundle with a
    promoted verdict MUST NOT delete the verdict file or its index link."""
    bundle = tmp_path / "bundle"
    materialize_bundle(make_manifest(tmp_path / "a", "orders", "Orders"), bundle, INGESTED_AT)
    verdict_bytes = b"---\ntype: verdict\ndecision: approved\n---\n\nA promoted verdict.\n"
    (bundle / "promoted-verdict-1.md").write_bytes(verdict_bytes)
    with (bundle / "index.md").open("a", encoding="utf-8", newline="") as handle:
        handle.write("- [Vurdering](promoted-verdict-1.md)\n")
    # Re-ingest with a DIFFERENT extraction set so the removal path runs.
    materialize_bundle(make_manifest(tmp_path / "b", "refunds", "Refunds"), bundle, INGESTED_AT)
    index = (bundle / "index.md").read_text(encoding="utf-8")
    assert (bundle / "promoted-verdict-1.md").read_bytes() == verdict_bytes
    assert "- [Vurdering](promoted-verdict-1.md)\n" in index
    assert "(ingest-orders.md)" not in index


def test_curated_link_with_managed_shape_never_relabeled(tmp_path: Path) -> None:
    """A curated line has the managed SHAPE but a non-ingest target — it is
    never removed and never relabeled."""
    manifest = make_manifest(tmp_path / "src", "orders", "Orders")
    bundle = tmp_path / "bundle"
    bundle.mkdir()
    (bundle / "index.md").write_text("- [Old label](notes.md)\n", encoding="utf-8", newline="")
    materialize_bundle(manifest, bundle, INGESTED_AT)
    index = (bundle / "index.md").read_text(encoding="utf-8")
    assert "- [Old label](notes.md)\n" in index


def test_crlf_unmanaged_lines_preserved_verbatim(tmp_path: Path) -> None:
    """§6 byte-for-byte preservation includes foreign line endings, also when
    the removal path rewrites the index."""
    bundle = tmp_path / "bundle"
    materialize_bundle(make_manifest(tmp_path / "a", "orders", "Orders"), bundle, INGESTED_AT)
    index_path = bundle / "index.md"
    index_path.write_bytes(b"curated CRLF line\r\n" + index_path.read_bytes())
    materialize_bundle(make_manifest(tmp_path / "b", "refunds", "Refunds"), bundle, INGESTED_AT)
    assert index_path.read_bytes().startswith(b"curated CRLF line\r\n")
    assert b"(ingest-orders.md)" not in index_path.read_bytes()