feat(ingest): I5 — SQL D7-speil, bygget fra commons-spec alene

Speiler MAF I4 fra shared/ingest-spec.md alene: manifest → SQL-konnektor →
materialisert OKF-bundle, byte-identisk med den delte golden-fasiten. Gaten
I4→I5 verifisert løst mot ground truth (MAF-commits d7e5f2f/4f45fe6/1b7612b)
før arbeidet startet. Spec byte-identisk delt, ingen spec-endring (I4).

- ingest.py: SqlSource (type: sql, id, connection_ref); ManifestContract.source
  er nå diskriminert union FileSource | SqlSource på type (http/ukjent tag →
  fail-fast). _render_sql_cell (§5 typed: NULL→"", int→decimal, float→korteste
  round-trip, str→verbatim m/ delt _escape_cell, annet→fail — aldri stille
  coercion). _resolve_connection_ref (env-oppslag §4/§8, usatt → fail-fast).
  _read_sql (read-only sqlite file:?mode=ro, ett SELECT, max_rows §8).
  _read_extraction dispatcher på source.type; materialisering/index/replacement
  uendret fra I3.
- examples/ingest-golden-sql/: repo-lokal golden (byte-frossen kopi av I4s fasit).
- Speiltester (I4s load-bearing-sett, gjennom SQL-konnektoren, detach-bevist røde):
  sql-golden byte-fasit + mutasjonskontroller · typed-cell/NULL (NY §I5-søm) ·
  provenance/navigability/verdict-reservasjon/re-ingest-safety · SqlSource-kontrakt/
  typed-rendering/connection_ref/max_rows/read-only · spec-integritet utvidet med
  connection_ref. Stale type:"sql"-avvisningscase erstattet (sql er gyldig post-I5).
- docs/2026-07-04-I5-brief.md: brief + premiss-verifisering.

Suite 265 passed uten nøkkel/nettverk (239 + 26 nye) · ruff + mypy --strict rene.

[skip-docs] README + docs/extending.md er bevisst utsatt til I7 per sesjonsplan
(programmet batcher ingest-doc der, avgrenset til det D7 faktisk har — CSV + SQL nå,
HTTP/MCP kun pekere). Dokumentert i docs/2026-07-04-I5-brief.md.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_017MM6BWb1hWmJZuXFZ7rjxT

tests/test_ingest.py

@@ -68,7 +68,8 @@ class TestManifestValidation:
             lambda m: m.pop("bundle_summary"),
             lambda m: m.__setitem__("extractions", []),
             lambda m: m["source"].__setitem__("id", "Bad_Id"),
-            lambda m: m["source"].__setitem__("type", "sql"),
+            lambda m: m["source"].__setitem__("type", "http"),  # optional/unimplemented (§1)
+            lambda m: m["source"].__setitem__("type", "unknown"),  # bad discriminator
             lambda m: m["extractions"][0].__setitem__("id", "Bad Id"),
             lambda m: m["extractions"][0].__setitem__("title", "two\nlines"),
             lambda m: m["extractions"][0].__setitem__("max_rows", 0),

tests/test_ingest_spec_loadbearing.py

@@ -50,6 +50,7 @@ _CONTRACT_FIELDS = (
     "okf_type",
     "max_rows",
     "root",
+    "connection_ref",  # the sql source reference the D7 sql connector (I5) depends on
 )
 
 

tests/test_ingest_sql.py

@@ -0,0 +1,123 @@
+"""Ingest SQL unit + fail-fast contract tests (ingest-spec §4, §5, §8).
+
+Pins the ``SqlSource`` manifest contract, the §5 typed-cell rendering (int/float/NULL/other),
+the runtime ``connection_ref`` resolution, the §8 size cap, and read-only access enforcement.
+Everything is local: a tmp sqlite fixture, no network, no credentials.
+"""
+
+from __future__ import annotations
+
+import json
+import sqlite3
+from pathlib import Path
+
+import pytest
+from pydantic import ValidationError
+
+from portfolio_optimiser_claude.ingest import (
+    ManifestContract,
+    SqlSource,
+    _read_sql,
+    _render_sql_cell,
+    _resolve_connection_ref,
+    materialize,
+)
+
+
+def _db(tmp_path: Path, rows: list[tuple[object, ...]]) -> Path:
+    db = tmp_path / "src.sqlite"
+    con = sqlite3.connect(db)
+    con.execute("CREATE TABLE t (a INTEGER, b REAL, c TEXT, d BLOB)")
+    con.executemany("INSERT INTO t VALUES (?, ?, ?, ?)", rows)
+    con.commit()
+    con.close()
+    return db
+
+
+def _sql_manifest() -> dict:
+    return {
+        "manifest_version": 1,
+        "source": {"type": "sql", "id": "db", "connection_ref": "SRC_DSN"},
+        "bundle_summary": "s",
+        "extractions": [
+            {
+                "id": "e",
+                "title": "T",
+                "query": "SELECT a FROM t ORDER BY a",
+                "okf_type": "dataset",
+                "max_rows": 5,
+            }
+        ],
+    }
+
+
+class TestSqlSourceContract:
+    """§4: the sql source is a valid discriminated variant; the reference is required."""
+
+    def test_valid_sql_source_loads(self) -> None:
+        contract = ManifestContract(**_sql_manifest())
+        assert isinstance(contract.source, SqlSource)
+        assert contract.source.connection_ref == "SRC_DSN"
+
+    def test_connection_ref_is_required(self) -> None:
+        manifest = _sql_manifest()
+        del manifest["source"]["connection_ref"]
+        with pytest.raises(ValidationError):
+            ManifestContract(**manifest)
+
+    def test_sql_source_id_grammar_enforced(self) -> None:
+        manifest = _sql_manifest()
+        manifest["source"]["id"] = "Bad Id"
+        with pytest.raises(ValidationError):
+            ManifestContract(**manifest)
+
+
+class TestTypedCellRendering:
+    """§5: None→'', int→decimal, float→shortest round-trip, str→verbatim, other→fail."""
+
+    def test_null_is_empty_string(self) -> None:
+        assert _render_sql_cell(None) == ""
+
+    def test_int_is_plain_decimal(self) -> None:
+        assert _render_sql_cell(3) == "3"
+        assert _render_sql_cell(0) == "0"
+
+    def test_float_is_shortest_round_trip(self) -> None:
+        assert _render_sql_cell(1200.5) == "1200.5"
+        assert _render_sql_cell(89.9) == "89.9"
+        assert _render_sql_cell(4200.0) == "4200.0"  # a whole-valued REAL keeps its .0
+
+    def test_str_is_returned_raw(self) -> None:
+        # _render_sql_cell returns the RAW string; §5 escaping is _escape_cell's job.
+        assert _render_sql_cell("a|b\\c") == "a|b\\c"
+
+    def test_other_value_type_fails(self) -> None:
+        with pytest.raises(ValueError, match="not a supported value type"):
+            _render_sql_cell(b"\x00\x01")  # a BLOB — never a silent coercion
+
+
+class TestConnectorRuntime:
+    """§8: reference resolution, size cap, and read-only access — all fail-fast."""
+
+    def test_unset_connection_ref_fails_fast(self, monkeypatch: pytest.MonkeyPatch) -> None:
+        monkeypatch.delenv("SRC_DSN", raising=False)
+        with pytest.raises(ValueError, match="not set in the environment"):
+            _resolve_connection_ref("SRC_DSN")
+
+    def test_max_rows_cap_enforced_fail_fast(
+        self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+    ) -> None:
+        db = _db(tmp_path, [(1, 1.0, "x", None), (2, 2.0, "y", None)])
+        manifest = _sql_manifest()
+        manifest["extractions"][0]["max_rows"] = 1  # 2 rows > 1
+        manifest_path = tmp_path / "manifest.json"
+        manifest_path.write_text(json.dumps(manifest), encoding="utf-8")
+        monkeypatch.setenv("SRC_DSN", str(db))
+        with pytest.raises(ValueError, match="max_rows"):
+            materialize(manifest_path, tmp_path / "bundle", "2026-07-04T12:00:00Z")
+
+    def test_connection_is_read_only(self, tmp_path: Path) -> None:
+        db = _db(tmp_path, [(1, 1.0, "x", None)])
+        # The connector opens read-only (§4 SHOULD): a write statement is refused.
+        with pytest.raises(sqlite3.OperationalError):
+            _read_sql(str(db), "UPDATE t SET a = 9", 5)

tests/test_ingest_sql_golden.py

@@ -0,0 +1,105 @@
+"""Ingest SQL golden regression (ingest-spec §11) — D7 mirror of MAF I4.
+
+Consumes the repo-local ``examples/ingest-golden-sql/`` (a byte-frozen copy of MAF I4's
+golden — the SAME shared extraction, so this test proves D7's independent SQL connector
+reproduces MAF's exact bytes from the shared spec alone). ``expected-bundle/`` is compared
+file by file, byte for byte.
+
+The ``connection_ref`` env var is pointed at the LOCAL sqlite fixture — a local source: no
+network, no credentials (ingest-spec §8, §11). The mutation controls prove the net is taut:
+one changed determinism input (a source row, the timestamp, the manifest bytes) must diverge.
+"""
+
+from __future__ import annotations
+
+import shutil
+import sqlite3
+from pathlib import Path
+
+import pytest
+
+from portfolio_optimiser_claude.ingest import materialize
+
+GOLDEN = Path(__file__).resolve().parents[1] / "examples" / "ingest-golden-sql"
+CONNECTION_REF = "PORTEFOLJE_SQL_DSN"
+_FIXTURE = GOLDEN / "fixture" / "portefolje.sqlite"
+
+
+def _ingested_at() -> str:
+    return (GOLDEN / "ingested-at.txt").read_text(encoding="utf-8").strip()
+
+
+def test_materializes_golden_byte_for_byte(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setenv(CONNECTION_REF, str(_FIXTURE))
+    bundle = tmp_path / "bundle"
+    bundle.mkdir()
+    materialize(GOLDEN / "manifest.json", bundle, _ingested_at())
+
+    expected = GOLDEN / "expected-bundle"
+    expected_names = sorted(p.name for p in expected.iterdir())
+    produced_names = sorted(p.name for p in bundle.iterdir())
+    assert produced_names == expected_names  # no missing, no extra files
+    for name in expected_names:
+        assert (bundle / name).read_bytes() == (expected / name).read_bytes(), name
+
+
+def test_reingest_is_idempotent(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+    # ingest-spec §10: same source content, manifest, ingested_at → byte-identical.
+    monkeypatch.setenv(CONNECTION_REF, str(_FIXTURE))
+    bundle = tmp_path / "bundle"
+    bundle.mkdir()
+    materialize(GOLDEN / "manifest.json", bundle, _ingested_at())
+    first = {p.name: p.read_bytes() for p in bundle.iterdir()}
+    materialize(GOLDEN / "manifest.json", bundle, _ingested_at())
+    second = {p.name: p.read_bytes() for p in bundle.iterdir()}
+    assert first == second
+
+
+class TestMutationControl:
+    """One changed determinism input → divergence from the golden bytes."""
+
+    def test_changed_source_row_diverges(
+        self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+    ) -> None:
+        # Prove the connector actually reads the db: mutate a copied db, then diverge.
+        case = tmp_path / "case"
+        shutil.copytree(GOLDEN, case)
+        db = case / "fixture" / "portefolje.sqlite"
+        con = sqlite3.connect(db)
+        con.execute("UPDATE costs SET amount = 9999.9 WHERE id = 1")
+        con.commit()
+        con.close()
+        monkeypatch.setenv(CONNECTION_REF, str(db))
+        bundle = tmp_path / "bundle"
+        bundle.mkdir()
+        materialize(case / "manifest.json", bundle, _ingested_at())
+        golden_bytes = (GOLDEN / "expected-bundle" / "ingest-costs.md").read_bytes()
+        assert (bundle / "ingest-costs.md").read_bytes() != golden_bytes
+
+    def test_changed_timestamp_diverges(
+        self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+    ) -> None:
+        monkeypatch.setenv(CONNECTION_REF, str(_FIXTURE))
+        bundle = tmp_path / "bundle"
+        bundle.mkdir()
+        materialize(GOLDEN / "manifest.json", bundle, "2099-01-01T00:00:00Z")
+        golden_bytes = (GOLDEN / "expected-bundle" / "ingest-costs.md").read_bytes()
+        assert (bundle / "ingest-costs.md").read_bytes() != golden_bytes
+
+    def test_changed_manifest_bytes_change_the_stamp(
+        self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+    ) -> None:
+        # The ingest_manifest stamp is SHA-256 of the manifest's raw bytes (§5) —
+        # a whitespace-only edit still re-stamps every generated file.
+        monkeypatch.setenv(CONNECTION_REF, str(_FIXTURE))
+        case = tmp_path / "case"
+        shutil.copytree(GOLDEN, case)
+        manifest = case / "manifest.json"
+        manifest.write_bytes(manifest.read_bytes() + b"\n")
+        # The copied db lives beside the copied manifest; point the ref there too.
+        monkeypatch.setenv(CONNECTION_REF, str(case / "fixture" / "portefolje.sqlite"))
+        bundle = tmp_path / "bundle"
+        bundle.mkdir()
+        materialize(manifest, bundle, _ingested_at())
+        golden_bytes = (GOLDEN / "expected-bundle" / "ingest-costs.md").read_bytes()
+        assert (bundle / "ingest-costs.md").read_bytes() != golden_bytes

tests/test_ingest_sql_loadbearing.py

@@ -0,0 +1,156 @@
+"""Load-bearing ingest seams for the SQL source (ingest-spec §11) — D7 mirror of MAF I4.
+
+Each test goes RED when its seam detaches (the method-spec §11 regime) — a grønn-men-død test
+is the failure mode the rule exists for. The seams, proven THROUGH the SQL connector (a local
+sqlite fixture — no network, no credentials):
+
+- Typed-cell rendering (the NEW I5 seam) — SQL NULL renders as the empty string and a REAL keeps
+  its shortest round-trip form; a naive ``str()`` coercion (NULL → ``"None"``) breaks this.
+- Provenance stamping — a sql-generated file carries the §7 provenance layer, ``source_system``
+  = the sql source id.
+- Navigability — the sql bundle is consumable by the UNCHANGED ``okf`` navigation.
+- Verdict reservation — a sql manifest mapping to ``type: verdict`` is rejected fail-fast,
+  BEFORE the database is ever opened.
+- Re-ingest layer safety — re-materialization over a sql bundle carrying a promoted verdict
+  preserves the verdict file AND its index link.
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+import pytest
+from pydantic import ValidationError
+
+from portfolio_optimiser_claude import okf
+from portfolio_optimiser_claude.ingest import ManifestContract, load_manifest, materialize
+
+GOLDEN = Path(__file__).resolve().parents[1] / "examples" / "ingest-golden-sql"
+CONNECTION_REF = "PORTEFOLJE_SQL_DSN"
+INGESTED_AT = (GOLDEN / "ingested-at.txt").read_text(encoding="utf-8").strip()
+_FIXTURE = GOLDEN / "fixture" / "portefolje.sqlite"
+
+_PROVENANCE_KEYS = ("source_system", "source_query", "ingested_at", "ingest_manifest", "generated")
+
+
+def _materialized(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path:
+    monkeypatch.setenv(CONNECTION_REF, str(_FIXTURE))
+    bundle = tmp_path / "bundle"
+    bundle.mkdir()
+    materialize(GOLDEN / "manifest.json", bundle, INGESTED_AT)
+    return bundle
+
+
+class TestTypedCellSeam:
+    """NEW I5 seam: SQL typed-cell rendering (RED if it degrades to a bare ``str()``)."""
+
+    def test_null_cell_renders_as_empty_string(
+        self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+    ) -> None:
+        bundle = _materialized(tmp_path, monkeypatch)
+        costs = (bundle / "ingest-costs.md").read_text(encoding="utf-8")
+        # costs row id=2 has note = SQL NULL → an empty cell, NOT the string "None".
+        assert "| 2 | sensor | 89.9 |  |" in costs
+        assert "None" not in costs
+
+    def test_real_keeps_shortest_round_trip(
+        self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+    ) -> None:
+        bundle = _materialized(tmp_path, monkeypatch)
+        costs = (bundle / "ingest-costs.md").read_text(encoding="utf-8")
+        assert "| 3 | kabel | 4200.0 | rest |" in costs  # a whole-valued REAL keeps its .0
+
+
+class TestProvenanceStamping:
+    """Seam: a sql-generated file carries the §7 provenance layer (RED if it stops)."""
+
+    def test_generated_sql_file_carries_the_provenance_layer(
+        self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+    ) -> None:
+        bundle = _materialized(tmp_path, monkeypatch)
+        concept = okf.parse_concept_file(bundle / "ingest-costs.md")
+        for key in _PROVENANCE_KEYS:
+            assert key in concept.frontmatter, f"provenance key {key} detached"
+        assert concept.frontmatter["source_system"] == "portefolje-db"
+        assert concept.frontmatter["ingested_at"] == INGESTED_AT
+        assert concept.frontmatter["generated"] == "true"
+        stamp = load_manifest(GOLDEN / "manifest.json").stamp
+        assert concept.frontmatter["ingest_manifest"] == stamp
+
+
+class TestNavigability:
+    """Seam: the generated sql bundle is consumable by the UNCHANGED okf navigation."""
+
+    def test_sql_bundle_navigates_via_unchanged_okf(
+        self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+    ) -> None:
+        bundle = _materialized(tmp_path, monkeypatch)
+        names = [c.path.name for c in okf.navigate_bundle(bundle)]
+        # Every generated file must be REACHABLE via index cross-links (RED if linking detaches).
+        assert names == ["index.md", "ingest-costs.md", "ingest-meta.md"]
+
+    def test_sql_context_renders_the_extracted_rows(
+        self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+    ) -> None:
+        bundle = _materialized(tmp_path, monkeypatch)
+        context = okf.bundle_context(bundle)
+        assert "LED-armatur" in context  # a costs body cell
+        assert "owner" in context  # a meta body cell
+
+
+class TestVerdictReservation:
+    """Seam: a sql manifest mapping to the verdict layer is rejected fail-fast (§3)."""
+
+    def _sql_manifest(self, okf_type: str) -> dict:
+        return {
+            "manifest_version": 1,
+            "source": {"type": "sql", "id": "db", "connection_ref": CONNECTION_REF},
+            "bundle_summary": "s",
+            "extractions": [
+                {"id": "e", "title": "T", "query": "SELECT 1", "okf_type": okf_type, "max_rows": 1}
+            ],
+        }
+
+    def test_verdict_okf_type_is_rejected(self) -> None:
+        with pytest.raises(ValidationError):
+            ManifestContract(**self._sql_manifest("verdict"))
+
+    def test_rejection_is_fail_fast_before_the_db_is_opened(
+        self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+    ) -> None:
+        # With the connection_ref env var UNSET, a run that got past validation would fail with a
+        # plain ValueError trying to open the db. Rejection is a ValidationError at validation —
+        # so the db is never opened (RED if the reservation stops firing first).
+        monkeypatch.delenv(CONNECTION_REF, raising=False)
+        manifest = tmp_path / "manifest.json"
+        manifest.write_text(json.dumps(self._sql_manifest("verdict")), encoding="utf-8")
+        bundle = tmp_path / "bundle"
+        with pytest.raises(ValidationError):
+            materialize(manifest, bundle, INGESTED_AT)
+        assert not bundle.exists() or not any(bundle.iterdir())
+
+
+class TestReingestLayerSafety:
+    """Seam: re-ingest over a sql bundle preserves a promoted verdict AND its link (§3, §6)."""
+
+    def test_promoted_verdict_and_link_survive_sql_reingest(
+        self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+    ) -> None:
+        bundle = _materialized(tmp_path, monkeypatch)
+        (bundle / "promoted-verdict-led.md").write_text(
+            "---\ntype: verdict\ndecision: approved\ndescription: LED holds.\n---\n\nBody.\n",
+            encoding="utf-8",
+        )
+        index = bundle / "index.md"
+        index.write_text(
+            index.read_text(encoding="utf-8") + "- [promoted](promoted-verdict-led.md)\n",
+            encoding="utf-8",
+        )
+        materialize(GOLDEN / "manifest.json", bundle, INGESTED_AT)  # re-ingest
+
+        assert (bundle / "promoted-verdict-led.md").is_file()  # verdict file survives
+        index_text = (bundle / "index.md").read_text(encoding="utf-8")
+        assert "- [promoted](promoted-verdict-led.md)" in index_text  # verdict link survives
+        assert (bundle / "ingest-costs.md").is_file()  # ingest files still refreshed
+        assert "](ingest-costs.md)" in index_text