feat(ingest): deterministic materialization with §5/§7 provenance stamp (I2)

tests/test_ingest_materialize.py

@@ -11,14 +11,20 @@ in tests/test_ingest_golden.py; the detach-RED seam quartet in tests/test_ingest
 """
 
 import copy
+import hashlib
+import json
+import logging
 from pathlib import Path
 from typing import Any
 
 import pytest
 
-from portfolio_optimiser.ingest import IngestError, read_csv, render_table
+from portfolio_optimiser import okf
+from portfolio_optimiser.ingest import IngestError, materialize, read_csv, render_table
 from portfolio_optimiser.retrieval import PathSecurityError
 
+_INGESTED_AT = "2026-07-03T12:00:00Z"
+
 # --- local CSV-catalogue builder (conftest's LLM fixtures are irrelevant to ingest) ---
 
 
@@ -122,3 +128,125 @@ def test_bom_never_leaks_into_first_header_cell(tmp_path: Path) -> None:
     root = _catalogue(tmp_path, {"data.csv": b"\xef\xbb\xbfa,b\n1,2\n"})
     header, _rows = _read(root)
     assert header == ["a", "b"]
+
+
+# --- Step 3: materialization — provenance frontmatter, LF bytes, staging ---
+
+
+def _project(
+    tmp_path: Path,
+    *,
+    extractions: list[dict[str, Any]] | None = None,
+    source: dict[str, Any] | None = None,
+    files: dict[str, bytes] | None = None,
+) -> tuple[Path, Path]:
+    """Write a manifest + CSV catalogue under tmp_path; return (manifest_path, bundle_dir)."""
+    catalogue = tmp_path / "catalogue"
+    catalogue.mkdir(exist_ok=True)
+    for name, content in (files or {"costs.csv": b"item,cost\nled,120\nhvac,80\n"}).items():
+        (catalogue / name).write_bytes(content)
+    manifest = {
+        "manifest_version": 1,
+        # Relative root — pinned decision: resolves against the manifest file's directory.
+        "source": source or {"type": "file", "id": "prosjekt-arkiv", "root": "catalogue"},
+        "bundle_summary": "Cost extracts from the project archive.",
+        "extractions": extractions
+        or [
+            {
+                "id": "costs",
+                "title": "Project costs",
+                "query": "costs.csv",
+                "okf_type": "dataset",
+                "max_rows": 100,
+            }
+        ],
+    }
+    manifest_path = tmp_path / "manifest.json"
+    manifest_path.write_text(json.dumps(manifest), encoding="utf-8")
+    return manifest_path, tmp_path / "bundle"
+
+
+def test_frontmatter_keys_in_exact_spec_order_raw_text(tmp_path: Path) -> None:
+    manifest_path, bundle_dir = _project(tmp_path)
+    (written,) = materialize(manifest_path, bundle_dir, ingested_at=_INGESTED_AT)
+    lines = written.read_text(encoding="utf-8").splitlines()
+    keys = [line.partition(":")[0] for line in lines[1:8]]
+    # §5: exactly these keys, in exactly this order.
+    assert lines[0] == "---" and lines[8] == "---"
+    assert keys == [
+        "type",
+        "title",
+        "source_system",
+        "source_query",
+        "ingested_at",
+        "ingest_manifest",
+        "generated",
+    ]
+
+
+def test_provenance_roundtrips_via_unchanged_parse_frontmatter(tmp_path: Path) -> None:
+    manifest_path, bundle_dir = _project(tmp_path)
+    (written,) = materialize(manifest_path, bundle_dir, ingested_at=_INGESTED_AT)
+    fm = okf.parse_frontmatter(written)
+    assert fm["type"] == "dataset"
+    assert fm["title"] == "Project costs"
+    assert fm["source_system"] == "prosjekt-arkiv"
+    assert fm["source_query"] == "costs.csv"
+    assert fm["ingested_at"] == _INGESTED_AT  # verbatim (§5)
+    assert fm["generated"] == "true"  # parse_frontmatter returns strings, never booleans
+    expected = "manifest@" + hashlib.sha256(manifest_path.read_bytes()).hexdigest()[:16]
+    assert fm["ingest_manifest"] == expected
+
+
+def test_generated_file_is_lf_only_with_one_trailing_newline(tmp_path: Path) -> None:
+    manifest_path, bundle_dir = _project(tmp_path)
+    (written,) = materialize(manifest_path, bundle_dir, ingested_at=_INGESTED_AT)
+    data = written.read_bytes()
+    assert b"\r" not in data
+    assert data.endswith(b"\n") and not data.endswith(b"\n\n")
+
+
+def test_invalid_ingested_at_raises_value_error(tmp_path: Path) -> None:
+    manifest_path, bundle_dir = _project(tmp_path)
+    # §5 format is ISO-8601 UTC with a Z suffix — validated by regex (datetime.fromisoformat
+    # rejects 'Z' on Python 3.10, the repo floor).
+    for bad in ("2026-07-03 12:00:00", "2026-07-03T12:00:00+00:00", "2026-07-03", ""):
+        with pytest.raises(ValueError):
+            materialize(manifest_path, bundle_dir, ingested_at=bad)
+
+
+def test_materialize_creates_nonexistent_nested_bundle_dir(tmp_path: Path) -> None:
+    manifest_path, _ = _project(tmp_path)
+    nested = tmp_path / "deep" / "nested" / "bundle"
+    (written,) = materialize(manifest_path, nested, ingested_at=_INGESTED_AT)
+    assert written.is_file() and written.parent == nested
+
+
+def test_source_call_logged_with_id_timestamp_rowcount(
+    tmp_path: Path, caplog: pytest.LogCaptureFixture
+) -> None:
+    manifest_path, bundle_dir = _project(tmp_path)
+    with caplog.at_level(logging.INFO, logger="portfolio_optimiser.ingest"):
+        materialize(manifest_path, bundle_dir, ingested_at=_INGESTED_AT)
+    # §8: which source, when (the deterministic ingested_at argument), row count.
+    joined = " ".join(record.getMessage() for record in caplog.records)
+    assert "prosjekt-arkiv" in joined and _INGESTED_AT in joined and "rows=2" in joined
+
+
+def test_two_runs_with_identical_inputs_are_byte_identical(tmp_path: Path) -> None:
+    manifest_path, bundle_dir = _project(tmp_path)
+    materialize(manifest_path, bundle_dir, ingested_at=_INGESTED_AT)
+    first = {p.name: p.read_bytes() for p in sorted(bundle_dir.iterdir())}
+    materialize(manifest_path, bundle_dir, ingested_at=_INGESTED_AT)
+    second = {p.name: p.read_bytes() for p in sorted(bundle_dir.iterdir())}
+    assert first == second  # §10 idempotence at file level
+
+
+def test_sql_source_has_no_connector_in_i2(tmp_path: Path) -> None:
+    # The polymorphic schema VALIDATES sql (brief assumption 1); executing it is I4 —
+    # attempting to materialize is an explicit refusal, never a silent no-op.
+    manifest_path, bundle_dir = _project(
+        tmp_path, source={"type": "sql", "id": "db", "connection_ref": "PROJ_DB"}
+    )
+    with pytest.raises(IngestError):
+        materialize(manifest_path, bundle_dir, ingested_at=_INGESTED_AT)