feat(ingest): index generation and stamped-replacement semantics (I2)

src/portfolio_optimiser/ingest.py

@@ -50,6 +50,10 @@ _INGESTED_AT_RE = re.compile(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z$")
 
 _LOGGER = logging.getLogger("portfolio_optimiser.ingest")
 
+# §6: index.md is managed by the materializer (and reserved — the extraction-id grammar
+# keeps generated names disjoint from it by construction).
+_INDEX_NAME = "index.md"
+
 
 class IngestError(RuntimeError):
     """A materialization-time refusal (cap exceeded, curated collision, malformed source)."""
@@ -242,6 +246,49 @@ def _write_bytes(bundle_dir: Path, name: str, content: str) -> Path:
     return resolved
 
 
+def _is_ingest_owned(path: Path) -> bool:
+    # §3/§5 ownership: the ingest stamp is `generated: true` AND an `ingest_manifest`
+    # reference. parse_frontmatter returns STRINGS ("true", never booleans). Promoted
+    # verdict files carry neither key, so they can never classify as ingest-owned.
+    frontmatter = okf.parse_frontmatter(path)
+    return frontmatter.get("generated") == "true" and "ingest_manifest" in frontmatter
+
+
+# One managed index line: `- [<label>](<target>)`. Anchored full-line — removal and label
+# refresh key on this exact shape for specific ingest targets, NEVER a bare substring (a
+# promoted verdict's line has the same shape but a non-ingest target; curated prose
+# mentioning a target inline does not match the full-line form).
+_MANAGED_LINE_RE = re.compile(r"^- \[(?P<label>[^\]]*)\]\((?P<target>[^)]+)\)$")
+
+
+def _update_index_lines(
+    index_path: Path, removed_targets: set[str], labels_by_target: dict[str, str]
+) -> None:
+    """§6 index maintenance on an EXISTING index: drop managed lines whose target is an
+    ingest file removed in this run; refresh in place a managed label that no longer equals
+    the extraction title. Every other line is preserved verbatim, in order."""
+    original = index_path.read_bytes().decode("utf-8")
+    lines = original.splitlines(keepends=True)
+    updated: list[str] = []
+    changed = False
+    for line in lines:
+        content = line.rstrip("\r\n")
+        ending = line[len(content) :]
+        match = _MANAGED_LINE_RE.match(content)
+        if match is not None:
+            target = match.group("target")
+            if target in removed_targets:
+                changed = True
+                continue
+            new_label = labels_by_target.get(target)
+            if new_label is not None and match.group("label") != new_label:
+                line = f"- [{new_label}]({target})" + ending
+                changed = True
+        updated.append(line)
+    if changed:
+        index_path.write_bytes("".join(updated).encode("utf-8"))
+
+
 def materialize(
     manifest_path: str | Path, bundle_dir: str | Path, *, ingested_at: str
 ) -> list[Path]:
@@ -294,4 +341,42 @@ def materialize(
     # implicit mkdir (unlike okf.write_concept_file) — create the bundle dir explicitly.
     bundle = Path(bundle_dir)
     bundle.mkdir(parents=True, exist_ok=True)
-    return [_write_bytes(bundle, name, content) for name, content in staged]
+    staged_names = {name for name, _ in staged}
+
+    # §3 ownership scan (sorted for determinism): only files carrying the ingest stamp are
+    # ours to replace.
+    owned = {
+        path.name
+        for path in sorted(bundle.glob("*.md"))
+        if path.name != _INDEX_NAME and _is_ingest_owned(path)
+    }
+    # §3 collision gate — BEFORE any mutation: a staged filename occupied by a file WITHOUT
+    # the stamp is curated content; never overwrite it.
+    for name in sorted(staged_names):
+        if (bundle / name).is_file() and name not in owned:
+            raise IngestError(
+                f"generated filename {name!r} collides with an existing file that does not "
+                "carry the ingest stamp — refusing to overwrite curated content (ingest "
+                "spec §3)"
+            )
+
+    # §5 replacement: remove every stamped file, then write the new set.
+    for name in sorted(owned):
+        (bundle / name).unlink()
+    written = [_write_bytes(bundle, name, content) for name, content in staged]
+
+    # §6 index generation — the LAST disk mutation. Fresh index gets bundle_summary as its
+    # body (no frontmatter — spec-minimal pinned decision); an existing index keeps every
+    # unmanaged line verbatim.
+    index_path = bundle / _INDEX_NAME
+    labels_by_target = {
+        f"ingest-{extraction.id}.md": extraction.title for extraction in manifest.extractions
+    }
+    if not index_path.is_file():
+        _write_bytes(bundle, _INDEX_NAME, manifest.bundle_summary + "\n")
+    else:
+        removed_targets = owned - staged_names
+        _update_index_lines(index_path, removed_targets, labels_by_target)
+    for extraction in manifest.extractions:
+        okf.link_in_index(str(bundle), f"ingest-{extraction.id}.md", extraction.title)
+    return written