From 676c11a498c4fb15f2101ac2bfcabddd76e06eb0 Mon Sep 17 00:00:00 2001
From: Kjell Tore Guttormsen <hello@fromaitochitta.com>
Date: Fri, 3 Jul 2026 18:33:56 +0200
Subject: [PATCH] feat(ingest): index generation and stamped-replacement
 semantics (I2)

---
 src/portfolio_optimiser/ingest.py |  87 ++++++++++++++++++++++++-
 tests/test_ingest_materialize.py  | 102 ++++++++++++++++++++++++++++++
 2 files changed, 188 insertions(+), 1 deletion(-)

diff --git a/src/portfolio_optimiser/ingest.py b/src/portfolio_optimiser/ingest.py
index 4437e95..02fa2c8 100644
--- a/src/portfolio_optimiser/ingest.py
+++ b/src/portfolio_optimiser/ingest.py
@@ -50,6 +50,10 @@ _INGESTED_AT_RE = re.compile(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z$")
 
 _LOGGER = logging.getLogger("portfolio_optimiser.ingest")
 
+# §6: index.md is managed by the materializer (and reserved — the extraction-id grammar
+# keeps generated names disjoint from it by construction).
+_INDEX_NAME = "index.md"
+
 
 class IngestError(RuntimeError):
     """A materialization-time refusal (cap exceeded, curated collision, malformed source)."""
@@ -242,6 +246,49 @@ def _write_bytes(bundle_dir: Path, name: str, content: str) -> Path:
     return resolved
 
 
+def _is_ingest_owned(path: Path) -> bool:
+    # §3/§5 ownership: the ingest stamp is `generated: true` AND an `ingest_manifest`
+    # reference. parse_frontmatter returns STRINGS ("true", never booleans). Promoted
+    # verdict files carry neither key, so they can never classify as ingest-owned.
+    frontmatter = okf.parse_frontmatter(path)
+    return frontmatter.get("generated") == "true" and "ingest_manifest" in frontmatter
+
+
+# One managed index line: `- [<label>](<target>)`. Anchored full-line — removal and label
+# refresh key on this exact shape for specific ingest targets, NEVER a bare substring (a
+# promoted verdict's line has the same shape but a non-ingest target; curated prose
+# mentioning a target inline does not match the full-line form).
+_MANAGED_LINE_RE = re.compile(r"^- \[(?P<label>[^\]]*)\]\((?P<target>[^)]+)\)$")
+
+
+def _update_index_lines(
+    index_path: Path, removed_targets: set[str], labels_by_target: dict[str, str]
+) -> None:
+    """§6 index maintenance on an EXISTING index: drop managed lines whose target is an
+    ingest file removed in this run; refresh in place a managed label that no longer equals
+    the extraction title. Every other line is preserved verbatim, in order."""
+    original = index_path.read_bytes().decode("utf-8")
+    lines = original.splitlines(keepends=True)
+    updated: list[str] = []
+    changed = False
+    for line in lines:
+        content = line.rstrip("\r\n")
+        ending = line[len(content) :]
+        match = _MANAGED_LINE_RE.match(content)
+        if match is not None:
+            target = match.group("target")
+            if target in removed_targets:
+                changed = True
+                continue
+            new_label = labels_by_target.get(target)
+            if new_label is not None and match.group("label") != new_label:
+                line = f"- [{new_label}]({target})" + ending
+                changed = True
+        updated.append(line)
+    if changed:
+        index_path.write_bytes("".join(updated).encode("utf-8"))
+
+
 def materialize(
     manifest_path: str | Path, bundle_dir: str | Path, *, ingested_at: str
 ) -> list[Path]:
@@ -294,4 +341,42 @@ def materialize(
     # implicit mkdir (unlike okf.write_concept_file) — create the bundle dir explicitly.
     bundle = Path(bundle_dir)
     bundle.mkdir(parents=True, exist_ok=True)
-    return [_write_bytes(bundle, name, content) for name, content in staged]
+    staged_names = {name for name, _ in staged}
+
+    # §3 ownership scan (sorted for determinism): only files carrying the ingest stamp are
+    # ours to replace.
+    owned = {
+        path.name
+        for path in sorted(bundle.glob("*.md"))
+        if path.name != _INDEX_NAME and _is_ingest_owned(path)
+    }
+    # §3 collision gate — BEFORE any mutation: a staged filename occupied by a file WITHOUT
+    # the stamp is curated content; never overwrite it.
+    for name in sorted(staged_names):
+        if (bundle / name).is_file() and name not in owned:
+            raise IngestError(
+                f"generated filename {name!r} collides with an existing file that does not "
+                "carry the ingest stamp — refusing to overwrite curated content (ingest "
+                "spec §3)"
+            )
+
+    # §5 replacement: remove every stamped file, then write the new set.
+    for name in sorted(owned):
+        (bundle / name).unlink()
+    written = [_write_bytes(bundle, name, content) for name, content in staged]
+
+    # §6 index generation — the LAST disk mutation. Fresh index gets bundle_summary as its
+    # body (no frontmatter — spec-minimal pinned decision); an existing index keeps every
+    # unmanaged line verbatim.
+    index_path = bundle / _INDEX_NAME
+    labels_by_target = {
+        f"ingest-{extraction.id}.md": extraction.title for extraction in manifest.extractions
+    }
+    if not index_path.is_file():
+        _write_bytes(bundle, _INDEX_NAME, manifest.bundle_summary + "\n")
+    else:
+        removed_targets = owned - staged_names
+        _update_index_lines(index_path, removed_targets, labels_by_target)
+    for extraction in manifest.extractions:
+        okf.link_in_index(str(bundle), f"ingest-{extraction.id}.md", extraction.title)
+    return written
diff --git a/tests/test_ingest_materialize.py b/tests/test_ingest_materialize.py
index 735fd5c..399b199 100644
--- a/tests/test_ingest_materialize.py
+++ b/tests/test_ingest_materialize.py
@@ -250,3 +250,105 @@ def test_sql_source_has_no_connector_in_i2(tmp_path: Path) -> None:
     )
     with pytest.raises(IngestError):
         materialize(manifest_path, bundle_dir, ingested_at=_INGESTED_AT)
+
+
+# --- Step 4: index generation + stamped-replacement semantics ---
+
+_TWO_EXTRACTIONS = [
+    {"id": "costs", "title": "Project costs", "query": "costs.csv", "okf_type": "dataset", "max_rows": 100},
+    {"id": "meta", "title": "Catalogue metadata", "query": "meta.csv", "okf_type": "reference", "max_rows": 10},
+]
+_TWO_FILES = {
+    "costs.csv": b"item,cost\nled,120\n",
+    "meta.csv": b"key,value\nowner,anlegg\n",
+}
+
+
+def _bundle_bytes(bundle_dir: Path) -> dict[str, bytes]:
+    return {p.name: p.read_bytes() for p in sorted(bundle_dir.iterdir()) if p.is_file()}
+
+
+def test_fresh_dir_gets_index_with_bundle_summary_and_links_in_manifest_order(
+    tmp_path: Path,
+) -> None:
+    manifest_path, bundle_dir = _project(
+        tmp_path, extractions=copy.deepcopy(_TWO_EXTRACTIONS), files=dict(_TWO_FILES)
+    )
+    materialize(manifest_path, bundle_dir, ingested_at=_INGESTED_AT)
+    index = (bundle_dir / "index.md").read_text(encoding="utf-8")
+    assert index.startswith("Cost extracts from the project archive.\n")
+    link_lines = [line for line in index.splitlines() if line.startswith("- [")]
+    assert link_lines == [
+        "- [Project costs](ingest-costs.md)",
+        "- [Catalogue metadata](ingest-meta.md)",
+    ]
+
+
+def test_curated_index_line_preserved_byte_for_byte(tmp_path: Path) -> None:
+    manifest_path, bundle_dir = _project(tmp_path)
+    bundle_dir.mkdir()
+    curated = "Hand-written summary.\n\n- [Curated note](note.md)\n"
+    (bundle_dir / "index.md").write_bytes(curated.encode("utf-8"))
+    materialize(manifest_path, bundle_dir, ingested_at=_INGESTED_AT)
+    index = (bundle_dir / "index.md").read_text(encoding="utf-8")
+    assert index.startswith(curated)  # §6: every unmanaged line preserved verbatim
+    assert "- [Project costs](ingest-costs.md)" in index
+
+
+def test_removed_extraction_drops_file_and_index_link(tmp_path: Path) -> None:
+    manifest_path, bundle_dir = _project(
+        tmp_path, extractions=copy.deepcopy(_TWO_EXTRACTIONS), files=dict(_TWO_FILES)
+    )
+    materialize(manifest_path, bundle_dir, ingested_at=_INGESTED_AT)
+    # Re-materialize with `meta` dropped from the manifest.
+    manifest_path2, _ = _project(tmp_path, files=dict(_TWO_FILES))
+    materialize(manifest_path2, bundle_dir, ingested_at=_INGESTED_AT)
+    assert not (bundle_dir / "ingest-meta.md").exists()
+    index = (bundle_dir / "index.md").read_text(encoding="utf-8")
+    assert "](ingest-meta.md)" not in index  # §6: link to removed ingest file is gone
+    assert "- [Project costs](ingest-costs.md)" in index  # sibling link intact
+
+
+def test_changed_title_refreshes_label_in_place(tmp_path: Path) -> None:
+    manifest_path, bundle_dir = _project(
+        tmp_path, extractions=copy.deepcopy(_TWO_EXTRACTIONS), files=dict(_TWO_FILES)
+    )
+    materialize(manifest_path, bundle_dir, ingested_at=_INGESTED_AT)
+    renamed = copy.deepcopy(_TWO_EXTRACTIONS)
+    renamed[0]["title"] = "Renamed costs"
+    manifest_path2, _ = _project(tmp_path, extractions=renamed, files=dict(_TWO_FILES))
+    materialize(manifest_path2, bundle_dir, ingested_at=_INGESTED_AT)
+    link_lines = [
+        line
+        for line in (bundle_dir / "index.md").read_text(encoding="utf-8").splitlines()
+        if line.startswith("- [")
+    ]
+    # §6: the label IS the title — refreshed in place, position preserved (no stale label,
+    # no move-to-end).
+    assert link_lines == [
+        "- [Renamed costs](ingest-costs.md)",
+        "- [Catalogue metadata](ingest-meta.md)",
+    ]
+
+
+def test_unstamped_collision_fails_and_leaves_bundle_unmodified(tmp_path: Path) -> None:
+    manifest_path, bundle_dir = _project(tmp_path)
+    bundle_dir.mkdir()
+    (bundle_dir / "index.md").write_bytes(b"Curated.\n")
+    # A curated file occupying a generated filename WITHOUT the ingest stamp (§3): never
+    # overwrite curated content.
+    (bundle_dir / "ingest-costs.md").write_bytes(b"---\ntype: note\n---\n\nCurated body.\n")
+    before = _bundle_bytes(bundle_dir)
+    with pytest.raises(IngestError):
+        materialize(manifest_path, bundle_dir, ingested_at=_INGESTED_AT)
+    assert _bundle_bytes(bundle_dir) == before
+
+
+def test_rematerialization_twice_is_byte_identical_including_index(tmp_path: Path) -> None:
+    manifest_path, bundle_dir = _project(
+        tmp_path, extractions=copy.deepcopy(_TWO_EXTRACTIONS), files=dict(_TWO_FILES)
+    )
+    materialize(manifest_path, bundle_dir, ingested_at=_INGESTED_AT)
+    first = _bundle_bytes(bundle_dir)
+    materialize(manifest_path, bundle_dir, ingested_at=_INGESTED_AT)
+    assert _bundle_bytes(bundle_dir) == first  # §10, index.md included