From 32640deffca75d37b817b732f6b1064fdb6fcbc0 Mon Sep 17 00:00:00 2001 From: Kjell Tore Guttormsen Date: Sat, 4 Jul 2026 07:18:59 +0200 Subject: [PATCH] =?UTF-8?q?feat(ingest):=20I5=20=E2=80=94=20SQL=20D7-speil?= =?UTF-8?q?,=20bygget=20fra=20commons-spec=20alene?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Speiler MAF I4 fra shared/ingest-spec.md alene: manifest → SQL-konnektor → materialisert OKF-bundle, byte-identisk med den delte golden-fasiten. Gaten I4→I5 verifisert løst mot ground truth (MAF-commits d7e5f2f/4f45fe6/1b7612b) før arbeidet startet. Spec byte-identisk delt, ingen spec-endring (I4). - ingest.py: SqlSource (type: sql, id, connection_ref); ManifestContract.source er nå diskriminert union FileSource | SqlSource på type (http/ukjent tag → fail-fast). _render_sql_cell (§5 typed: NULL→"", int→decimal, float→korteste round-trip, str→verbatim m/ delt _escape_cell, annet→fail — aldri stille coercion). _resolve_connection_ref (env-oppslag §4/§8, usatt → fail-fast). _read_sql (read-only sqlite file:?mode=ro, ett SELECT, max_rows §8). _read_extraction dispatcher på source.type; materialisering/index/replacement uendret fra I3. - examples/ingest-golden-sql/: repo-lokal golden (byte-frossen kopi av I4s fasit). - Speiltester (I4s load-bearing-sett, gjennom SQL-konnektoren, detach-bevist røde): sql-golden byte-fasit + mutasjonskontroller · typed-cell/NULL (NY §I5-søm) · provenance/navigability/verdict-reservasjon/re-ingest-safety · SqlSource-kontrakt/ typed-rendering/connection_ref/max_rows/read-only · spec-integritet utvidet med connection_ref. Stale type:"sql"-avvisningscase erstattet (sql er gyldig post-I5). - docs/2026-07-04-I5-brief.md: brief + premiss-verifisering. Suite 265 passed uten nøkkel/nettverk (239 + 26 nye) · ruff + mypy --strict rene. [skip-docs] README + docs/extending.md er bevisst utsatt til I7 per sesjonsplan (programmet batcher ingest-doc der, avgrenset til det D7 faktisk har — CSV + SQL nå, HTTP/MCP kun pekere). Dokumentert i docs/2026-07-04-I5-brief.md. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_017MM6BWb1hWmJZuXFZ7rjxT --- docs/2026-07-04-I5-brief.md | 74 +++++++++ .../expected-bundle/index.md | 3 + .../expected-bundle/ingest-costs.md | 15 ++ .../expected-bundle/ingest-meta.md | 14 ++ .../fixture/portefolje.sqlite | Bin 0 -> 12288 bytes examples/ingest-golden-sql/ingested-at.txt | 1 + examples/ingest-golden-sql/manifest.json | 21 +++ src/portfolio_optimiser_claude/ingest.py | 108 +++++++++++- tests/test_ingest.py | 3 +- tests/test_ingest_spec_loadbearing.py | 1 + tests/test_ingest_sql.py | 123 ++++++++++++++ tests/test_ingest_sql_golden.py | 105 ++++++++++++ tests/test_ingest_sql_loadbearing.py | 156 ++++++++++++++++++ 13 files changed, 615 insertions(+), 9 deletions(-) create mode 100644 docs/2026-07-04-I5-brief.md create mode 100644 examples/ingest-golden-sql/expected-bundle/index.md create mode 100644 examples/ingest-golden-sql/expected-bundle/ingest-costs.md create mode 100644 examples/ingest-golden-sql/expected-bundle/ingest-meta.md create mode 100644 examples/ingest-golden-sql/fixture/portefolje.sqlite create mode 100644 examples/ingest-golden-sql/ingested-at.txt create mode 100644 examples/ingest-golden-sql/manifest.json create mode 100644 tests/test_ingest_sql.py create mode 100644 tests/test_ingest_sql_golden.py create mode 100644 tests/test_ingest_sql_loadbearing.py diff --git a/docs/2026-07-04-I5-brief.md b/docs/2026-07-04-I5-brief.md new file mode 100644 index 0000000..2dc7630 --- /dev/null +++ b/docs/2026-07-04-I5-brief.md @@ -0,0 +1,74 @@ +# I5 — D7-speil av ingest (SQL-kilde) — brief + +> Forankret i hovedrepoets sesjonsplan +> `~/repos/portfolio-optimiser/docs/plan/2026-07-03-sesjonsplan-ingest.md` §I5 og dette +> repoets STATE. Bygget **fra `shared/ingest-spec.md` alene** — aldri ved reverse-engineering +> av MAF-koden. Golden-**bytene** (den delte fasiten fra MAF I4) er lest, siden spec §11 gjør +> dem til konformanskravet begge stacker reproduserer. Gaten I4→I5 ble verifisert løst mot +> ground truth (MAF-commits `d7e5f2f`/`4f45fe6`/`1b7612b`, «I4») før arbeidet startet. + +## Mål (§I5) + +Andre kildetype konsumert **uendret** fra samme delte kontrakt i søskenrepoet: D7s uavhengige +SQL-konnektor reproduserer MAF I4s golden **byte-identisk** fra spec alene. Scope = `sql` +(lokal SQL-database, sqlite-fixture, null nettverk). `file` var I3; `http` = ikke i D7 +(målbildet: HTTP/MCP demonstreres MAF-only, I6). + +## Premiss-verifisering (før impl) + +- **Spec byte-identisk delt:** `diff -q shared/ingest-spec.md` mot MAF = IDENTISK. Spec dekker + `sql` fullt (§4 `connection_ref`, §5 typed-cell/NULL, read-only SELECT). I4 gjorde **ingen + spec-endring** (sesjonsplan §I4) → subtree-pull er no-op (bekreftet, ikke kjørt — ingen drift). +- **Golden-stamp:** SHA-256(manifest)[:16] = `a4b891aea55df97b` reprodusert lokalt → matcher + `ingest_manifest: manifest@a4b891aea55df97b` i forventet bundle. +- **Ingen nye operatør-beslutninger:** I3s to avgjørelser (golden repo-lokal kopi; okf relaksert + for frontmatterløs index) bærer uendret inn i I5. Én lukket økt, ingen commons-runde. + +## Leveranse + +- `examples/ingest-golden-sql/` — repo-lokal golden (byte-frossen kopi av MAF I4s; `diff -r` + tomt): `manifest.json`, `fixture/portefolje.sqlite`, `ingested-at.txt`, `expected-bundle/`. +- `src/portfolio_optimiser_claude/ingest.py` — SQL-utvidelse: + - `SqlSource`-modell (`type: "sql"`, `id`, `connection_ref`); `ManifestContract.source` er nå + en **diskriminert union** `FileSource | SqlSource` på `type` (ukjent/`http`-tag → fail-fast). + - `_render_sql_cell` — §5 typed rendering: `None`(NULL)→`""`, `int`→plain decimal, `float`→ + korteste round-trip (`repr`), `str`→verbatim (§5-escaping via delt `_escape_cell`), annet + (BLOB o.l.)→**fail** (aldri stille coercion). + - `_resolve_connection_ref` — env-var-oppslag (§4/§8: referanse ved navn, secret aldri i + manifest; usatt ref → fail-fast). + - `_read_sql` — read-only sqlite (`file:…?mode=ro`), ett SELECT (driver håndhever + én-statement), `max_rows` fail-fast (§8). + - `_read_extraction` — dispatch på `source.type` (`file`→CSV, `sql`→SQL); `materialize` + kaller den (materialisering/index/replacement uendret fra I3). +- Speiltester (I4s load-bearing-sett, gjennom SQL-konnektoren): + - `test_ingest_sql_golden.py` — byte-fasit + idempotens + 3 mutasjonskontroller (endret rad / + timestamp / manifest-bytes). + - `test_ingest_sql_loadbearing.py` — typed-cell/NULL (NY §I5-søm) · provenance · navigability · + verdict-reservasjon (fail-fast før DB åpnes) · re-ingest-safety. + - `test_ingest_sql.py` — `SqlSource`-kontrakt, `_render_sql_cell`-enheter, connection_ref + fail-fast, `max_rows`-tak, read-only-håndhevelse. + - `test_ingest_spec_loadbearing.py` — `connection_ref` lagt til kontrakt-felt-guarden (D7s + sql-impl avhenger nå av at spec dokumenterer feltet). + - `test_ingest.py` — stale `type:"sql"`-avvisningscase erstattet (sql er gyldig post-I5) med + `http`/ukjent-diskriminator-avvisning (honesty-regelen). + +## SDK-kroker (STATE steg 4 — søk-først) + +I5 (SQL) introduserer **ingen SDK-flate**: ingest er deterministisk, null modellkall. Søk-først +SDK-krok-reverifisering (API-drift på `create_sdk_mcp_server`) er relevant for I6 (HTTP/MCP), +ikke I5 — utsatt dit kroken faktisk brukes. Eksisterende SDK-klient uendret og grønn i suiten. + +## Verifisering (kjørt, grønn) + +- Golden **byte-identisk** konsumert (5/5, inkl. idempotens + mutasjonskontroller). +- Speiltester **røde ved detach** — de nye sømmene bevist load-bearing (detach-bevis-skript): + typed-cell/NULL · connection_ref fail-fast · sql-dispatch · spec-integritet (`connection_ref`). +- Søskensuite **grønn uten nøkkel/nettverk**: 265 passed (239 + 26 nye). +- `ruff check` + `ruff format` rene · `mypy --strict src`: no issues (16 filer). + +## Åpent (forslag, ikke utført — utenfor I5-scope) + +- **Standard-evolusjon:** «typet index overalt» som bevisst OKF-endring forblir en egen + koordinert commons-RFC (begge specs/golden/okf). Dagens permissive okf er forover-kompatibel. +- **Åpent-speil-synk:** I3/I5-koden er ikke pushet til det offentlige `open/`-speilet + (outward-facing → egen operatør-handling). diff --git a/examples/ingest-golden-sql/expected-bundle/index.md b/examples/ingest-golden-sql/expected-bundle/index.md new file mode 100644 index 0000000..57fb21e --- /dev/null +++ b/examples/ingest-golden-sql/expected-bundle/index.md @@ -0,0 +1,3 @@ +Golden extraction case: two SQL extracts from a local project database (sql source type). +- [Project costs](ingest-costs.md) +- [Catalogue metadata](ingest-meta.md) diff --git a/examples/ingest-golden-sql/expected-bundle/ingest-costs.md b/examples/ingest-golden-sql/expected-bundle/ingest-costs.md new file mode 100644 index 0000000..8ad789a --- /dev/null +++ b/examples/ingest-golden-sql/expected-bundle/ingest-costs.md @@ -0,0 +1,15 @@ +--- +type: dataset +title: Project costs +source_system: portefolje-db +source_query: SELECT id, item, amount, note FROM costs ORDER BY id +ingested_at: 2026-07-04T12:00:00Z +ingest_manifest: manifest@a4b891aea55df97b +generated: true +--- + +| id | item | amount | note | +| --- | --- | --- | --- | +| 1 | LED-armatur | 1200.5 | godkjent | +| 2 | sensor | 89.9 | | +| 3 | kabel | 4200.0 | rest | diff --git a/examples/ingest-golden-sql/expected-bundle/ingest-meta.md b/examples/ingest-golden-sql/expected-bundle/ingest-meta.md new file mode 100644 index 0000000..c281e5a --- /dev/null +++ b/examples/ingest-golden-sql/expected-bundle/ingest-meta.md @@ -0,0 +1,14 @@ +--- +type: reference +title: Catalogue metadata +source_system: portefolje-db +source_query: SELECT k, v FROM meta ORDER BY k +ingested_at: 2026-07-04T12:00:00Z +ingest_manifest: manifest@a4b891aea55df97b +generated: true +--- + +| k | v | +| --- | --- | +| owner | anlegg \| drift | +| path | c:\\temp | diff --git a/examples/ingest-golden-sql/fixture/portefolje.sqlite b/examples/ingest-golden-sql/fixture/portefolje.sqlite new file mode 100644 index 0000000000000000000000000000000000000000..678970204bd6dcf57ee9657e38e349f3b50a69fc GIT binary patch literal 12288 zcmeI&ze>YE90%~bi?ujtS}+C?;Rh})R2RX)sa1mrMQ9D8;F7l2KekCqa#aKuAHc;; zcOStQaPkp4Ir|PSUZMpZ1ji!ZAKd-!E?h46*^bLmt=dzP+F|5-iZ)q}an8y_j4`2$ zq01za{>-GKGym59ttr^X?fblbDomf5kGeuY00Izz00bZa0SG_<0uX=z1pd3gmYB6I zi`SIb7)W0#FDb@u-Kn?^xs{!&L&*pg`s6w%Ziz0^Ts-Aj+qU^lKAqSMV-=^B_%Ssd zqC&4l2Zye+@6=1AkM{o8t2{p(1&Z`q)e;4vKJzC_6qB>LVLmbQ)qK$j0Rad=00Izz z00bZa0SG_<0uX?}vUR-RAb0{s~~X#0|R`UIsGqf`ROG=$cwludO}+Jy%<) literal 0 HcmV?d00001 diff --git a/examples/ingest-golden-sql/ingested-at.txt b/examples/ingest-golden-sql/ingested-at.txt new file mode 100644 index 0000000..509edd0 --- /dev/null +++ b/examples/ingest-golden-sql/ingested-at.txt @@ -0,0 +1 @@ +2026-07-04T12:00:00Z diff --git a/examples/ingest-golden-sql/manifest.json b/examples/ingest-golden-sql/manifest.json new file mode 100644 index 0000000..03cbc88 --- /dev/null +++ b/examples/ingest-golden-sql/manifest.json @@ -0,0 +1,21 @@ +{ + "manifest_version": 1, + "source": {"type": "sql", "id": "portefolje-db", "connection_ref": "PORTEFOLJE_SQL_DSN"}, + "bundle_summary": "Golden extraction case: two SQL extracts from a local project database (sql source type).", + "extractions": [ + { + "id": "costs", + "title": "Project costs", + "query": "SELECT id, item, amount, note FROM costs ORDER BY id", + "okf_type": "dataset", + "max_rows": 10 + }, + { + "id": "meta", + "title": "Catalogue metadata", + "query": "SELECT k, v FROM meta ORDER BY k", + "okf_type": "reference", + "max_rows": 10 + } + ] +} diff --git a/src/portfolio_optimiser_claude/ingest.py b/src/portfolio_optimiser_claude/ingest.py index d86b6e8..a785858 100644 --- a/src/portfolio_optimiser_claude/ingest.py +++ b/src/portfolio_optimiser_claude/ingest.py @@ -4,9 +4,10 @@ An addition IN FRONT of the loop (ingest-spec §1): a deterministic step that co framework to a real data source and materializes the extract as an OKF knowledge bundle, which the existing 8-step loop then consumes UNCHANGED. Zero model calls; no network. -D7 scope (I3, mirror of MAF I2): the ``file`` source type (a local CSV catalogue). The -``sql`` and ``http`` source types are later work — a manifest naming them is rejected -fail-fast at validation, never silently accepted. +D7 scope (I3–I5, mirror of MAF I2/I4): the ``file`` source type (a local CSV catalogue) and +the ``sql`` source type (a local SQL database). The ``http`` source type is a later, OPTIONAL +extension point (§1) — a manifest naming it is rejected fail-fast at validation, never +silently accepted. Contract discipline mirrors ``contracts.py``: the manifest is schema-validated fail-fast BEFORE any source call (§4), and queries are declarative configuration, never evaluated as @@ -20,10 +21,12 @@ import csv import hashlib import json import logging +import os import re +import sqlite3 from dataclasses import dataclass from pathlib import Path -from typing import Literal +from typing import Annotated, Literal from pydantic import BaseModel, Field, field_validator @@ -65,6 +68,26 @@ class FileSource(BaseModel): return value +class SqlSource(BaseModel): + """A local SQL database (``source.type == "sql"``, ingest-spec §4). + + ``connection_ref`` is the NAME of a runtime-resolved environment variable whose value is + the database path (or connection string) — the secret/location never lives in the manifest + (§4, §8), so the manifest stays versionable and shareable without credentials. + """ + + type: Literal["sql"] + id: str + connection_ref: str + + @field_validator("id") + @classmethod + def _id_grammar(cls, value: str) -> str: + if not _ID_RE.match(value): + raise ValueError(f"source.id {value!r} must match [a-z0-9][a-z0-9-]*") + return value + + class Extraction(BaseModel): """One extraction description (ingest-spec §4). ``max_rows`` is a required cap (§8).""" @@ -103,7 +126,7 @@ class ManifestContract(BaseModel): """The ingest manifest (§4) — schema-validated fail-fast before any source call.""" manifest_version: Literal[1] - source: FileSource + source: Annotated[FileSource | SqlSource, Field(discriminator="type")] bundle_summary: str extractions: list[Extraction] = Field(min_length=1) @@ -174,6 +197,77 @@ def _read_csv(csv_path: Path, max_rows: int) -> list[list[str]]: return rows +def _render_sql_cell(value: object) -> str: + """Type a SQL cell to its §5 string form (BEFORE §5 escaping via ``_escape_cell``). + + ``None`` (SQL NULL) → the empty string; ``int`` → plain decimal; ``float`` → its shortest + round-trip decimal form (Python ``repr``); ``str`` → verbatim. Any other value type (a + BLOB, say) MUST fail — never a silent coercion (§5). + """ + if value is None: + return "" + if isinstance(value, int): + return str(value) + if isinstance(value, float): + return repr(value) + if isinstance(value, str): + return value + raise ValueError(f"SQL cell of type {type(value).__name__} is not a supported value type (§5)") + + +def _resolve_connection_ref(connection_ref: str) -> str: + """Resolve the ``connection_ref`` env var to the database location at run time (§4, §8). + + Locations/credentials never live in the manifest — the reference is resolved from the + environment. An unset reference fails fast (never a silent empty connection). + """ + dsn = os.environ.get(connection_ref) + if not dsn: + raise ValueError(f"connection_ref {connection_ref!r} is not set in the environment") + return dsn + + +def _read_sql(dsn: str, query: str, max_rows: int) -> list[list[str]]: + """Run one read-only SELECT against a local sqlite database (ingest-spec §4, §5, §8). + + The connection is opened read-only (§4: the connector SHOULD enforce read-only access), and + sqlite executes exactly ONE statement per ``execute`` call — so the single-statement rule is + enforced by the driver. ``max_rows`` is enforced fail-fast on the returned rows (§8). + Returns the column-name header row followed by typed-then-stringified data rows, in source + order (§5); a stable order is the manifest query's responsibility via ORDER BY (§4). + """ + uri = Path(dsn).resolve().as_uri() + "?mode=ro" + connection = sqlite3.connect(uri, uri=True) + try: + cursor = connection.execute(query) + header = [description[0] for description in cursor.description] + data = cursor.fetchall() + finally: + connection.close() + if len(data) > max_rows: + raise ValueError(f"extraction exceeds max_rows: {len(data)} > {max_rows}") + rows: list[list[str]] = [header] + for record in data: + rows.append([_render_sql_cell(value) for value in record]) + return rows + + +def _read_extraction( + source: FileSource | SqlSource, manifest_dir: Path, extraction: Extraction +) -> list[list[str]]: + """Dispatch to the connector for ``source.type`` — header + data rows in source order (§5). + + ``file``: resolve the query path within ``root`` (boundary-checked fail-closed, §4) and read + the CSV. ``sql``: resolve ``connection_ref`` from the environment (§4, §8) and run the + read-only SELECT. Both enforce ``max_rows`` fail-fast (§8). + """ + if source.type == "file": + csv_path = _resolve_within(manifest_dir / source.root, extraction.query) + return _read_csv(csv_path, extraction.max_rows) + dsn = _resolve_connection_ref(source.connection_ref) + return _read_sql(dsn, extraction.query, extraction.max_rows) + + def _resolve_within(root: Path, relative: str) -> Path: """Resolve ``relative`` under ``root``, boundary-checked fail-closed (the OKF path rule).""" resolved = (root / relative).resolve() @@ -264,7 +358,6 @@ def materialize(manifest_path: Path, bundle_dir: Path, ingested_at: str) -> list """ loaded = load_manifest(manifest_path) source = loaded.contract.source - root = manifest_path.parent / source.root bundle_dir.mkdir(parents=True, exist_ok=True) @@ -276,8 +369,7 @@ def materialize(manifest_path: Path, bundle_dir: Path, ingested_at: str) -> list # 2. Write the new set. generated: list[Path] = [] for extraction in loaded.contract.extractions: - csv_path = _resolve_within(root, extraction.query) - rows = _read_csv(csv_path, extraction.max_rows) + rows = _read_extraction(source, manifest_path.parent, extraction) _logger.info( "ingest source=%s extraction=%s rows=%d", source.id, diff --git a/tests/test_ingest.py b/tests/test_ingest.py index adbbc78..9284ae3 100644 --- a/tests/test_ingest.py +++ b/tests/test_ingest.py @@ -68,7 +68,8 @@ class TestManifestValidation: lambda m: m.pop("bundle_summary"), lambda m: m.__setitem__("extractions", []), lambda m: m["source"].__setitem__("id", "Bad_Id"), - lambda m: m["source"].__setitem__("type", "sql"), + lambda m: m["source"].__setitem__("type", "http"), # optional/unimplemented (§1) + lambda m: m["source"].__setitem__("type", "unknown"), # bad discriminator lambda m: m["extractions"][0].__setitem__("id", "Bad Id"), lambda m: m["extractions"][0].__setitem__("title", "two\nlines"), lambda m: m["extractions"][0].__setitem__("max_rows", 0), diff --git a/tests/test_ingest_spec_loadbearing.py b/tests/test_ingest_spec_loadbearing.py index 5df1c28..21951be 100644 --- a/tests/test_ingest_spec_loadbearing.py +++ b/tests/test_ingest_spec_loadbearing.py @@ -50,6 +50,7 @@ _CONTRACT_FIELDS = ( "okf_type", "max_rows", "root", + "connection_ref", # the sql source reference the D7 sql connector (I5) depends on ) diff --git a/tests/test_ingest_sql.py b/tests/test_ingest_sql.py new file mode 100644 index 0000000..5990555 --- /dev/null +++ b/tests/test_ingest_sql.py @@ -0,0 +1,123 @@ +"""Ingest SQL unit + fail-fast contract tests (ingest-spec §4, §5, §8). + +Pins the ``SqlSource`` manifest contract, the §5 typed-cell rendering (int/float/NULL/other), +the runtime ``connection_ref`` resolution, the §8 size cap, and read-only access enforcement. +Everything is local: a tmp sqlite fixture, no network, no credentials. +""" + +from __future__ import annotations + +import json +import sqlite3 +from pathlib import Path + +import pytest +from pydantic import ValidationError + +from portfolio_optimiser_claude.ingest import ( + ManifestContract, + SqlSource, + _read_sql, + _render_sql_cell, + _resolve_connection_ref, + materialize, +) + + +def _db(tmp_path: Path, rows: list[tuple[object, ...]]) -> Path: + db = tmp_path / "src.sqlite" + con = sqlite3.connect(db) + con.execute("CREATE TABLE t (a INTEGER, b REAL, c TEXT, d BLOB)") + con.executemany("INSERT INTO t VALUES (?, ?, ?, ?)", rows) + con.commit() + con.close() + return db + + +def _sql_manifest() -> dict: + return { + "manifest_version": 1, + "source": {"type": "sql", "id": "db", "connection_ref": "SRC_DSN"}, + "bundle_summary": "s", + "extractions": [ + { + "id": "e", + "title": "T", + "query": "SELECT a FROM t ORDER BY a", + "okf_type": "dataset", + "max_rows": 5, + } + ], + } + + +class TestSqlSourceContract: + """§4: the sql source is a valid discriminated variant; the reference is required.""" + + def test_valid_sql_source_loads(self) -> None: + contract = ManifestContract(**_sql_manifest()) + assert isinstance(contract.source, SqlSource) + assert contract.source.connection_ref == "SRC_DSN" + + def test_connection_ref_is_required(self) -> None: + manifest = _sql_manifest() + del manifest["source"]["connection_ref"] + with pytest.raises(ValidationError): + ManifestContract(**manifest) + + def test_sql_source_id_grammar_enforced(self) -> None: + manifest = _sql_manifest() + manifest["source"]["id"] = "Bad Id" + with pytest.raises(ValidationError): + ManifestContract(**manifest) + + +class TestTypedCellRendering: + """§5: None→'', int→decimal, float→shortest round-trip, str→verbatim, other→fail.""" + + def test_null_is_empty_string(self) -> None: + assert _render_sql_cell(None) == "" + + def test_int_is_plain_decimal(self) -> None: + assert _render_sql_cell(3) == "3" + assert _render_sql_cell(0) == "0" + + def test_float_is_shortest_round_trip(self) -> None: + assert _render_sql_cell(1200.5) == "1200.5" + assert _render_sql_cell(89.9) == "89.9" + assert _render_sql_cell(4200.0) == "4200.0" # a whole-valued REAL keeps its .0 + + def test_str_is_returned_raw(self) -> None: + # _render_sql_cell returns the RAW string; §5 escaping is _escape_cell's job. + assert _render_sql_cell("a|b\\c") == "a|b\\c" + + def test_other_value_type_fails(self) -> None: + with pytest.raises(ValueError, match="not a supported value type"): + _render_sql_cell(b"\x00\x01") # a BLOB — never a silent coercion + + +class TestConnectorRuntime: + """§8: reference resolution, size cap, and read-only access — all fail-fast.""" + + def test_unset_connection_ref_fails_fast(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("SRC_DSN", raising=False) + with pytest.raises(ValueError, match="not set in the environment"): + _resolve_connection_ref("SRC_DSN") + + def test_max_rows_cap_enforced_fail_fast( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + db = _db(tmp_path, [(1, 1.0, "x", None), (2, 2.0, "y", None)]) + manifest = _sql_manifest() + manifest["extractions"][0]["max_rows"] = 1 # 2 rows > 1 + manifest_path = tmp_path / "manifest.json" + manifest_path.write_text(json.dumps(manifest), encoding="utf-8") + monkeypatch.setenv("SRC_DSN", str(db)) + with pytest.raises(ValueError, match="max_rows"): + materialize(manifest_path, tmp_path / "bundle", "2026-07-04T12:00:00Z") + + def test_connection_is_read_only(self, tmp_path: Path) -> None: + db = _db(tmp_path, [(1, 1.0, "x", None)]) + # The connector opens read-only (§4 SHOULD): a write statement is refused. + with pytest.raises(sqlite3.OperationalError): + _read_sql(str(db), "UPDATE t SET a = 9", 5) diff --git a/tests/test_ingest_sql_golden.py b/tests/test_ingest_sql_golden.py new file mode 100644 index 0000000..a6b722b --- /dev/null +++ b/tests/test_ingest_sql_golden.py @@ -0,0 +1,105 @@ +"""Ingest SQL golden regression (ingest-spec §11) — D7 mirror of MAF I4. + +Consumes the repo-local ``examples/ingest-golden-sql/`` (a byte-frozen copy of MAF I4's +golden — the SAME shared extraction, so this test proves D7's independent SQL connector +reproduces MAF's exact bytes from the shared spec alone). ``expected-bundle/`` is compared +file by file, byte for byte. + +The ``connection_ref`` env var is pointed at the LOCAL sqlite fixture — a local source: no +network, no credentials (ingest-spec §8, §11). The mutation controls prove the net is taut: +one changed determinism input (a source row, the timestamp, the manifest bytes) must diverge. +""" + +from __future__ import annotations + +import shutil +import sqlite3 +from pathlib import Path + +import pytest + +from portfolio_optimiser_claude.ingest import materialize + +GOLDEN = Path(__file__).resolve().parents[1] / "examples" / "ingest-golden-sql" +CONNECTION_REF = "PORTEFOLJE_SQL_DSN" +_FIXTURE = GOLDEN / "fixture" / "portefolje.sqlite" + + +def _ingested_at() -> str: + return (GOLDEN / "ingested-at.txt").read_text(encoding="utf-8").strip() + + +def test_materializes_golden_byte_for_byte(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv(CONNECTION_REF, str(_FIXTURE)) + bundle = tmp_path / "bundle" + bundle.mkdir() + materialize(GOLDEN / "manifest.json", bundle, _ingested_at()) + + expected = GOLDEN / "expected-bundle" + expected_names = sorted(p.name for p in expected.iterdir()) + produced_names = sorted(p.name for p in bundle.iterdir()) + assert produced_names == expected_names # no missing, no extra files + for name in expected_names: + assert (bundle / name).read_bytes() == (expected / name).read_bytes(), name + + +def test_reingest_is_idempotent(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + # ingest-spec §10: same source content, manifest, ingested_at → byte-identical. + monkeypatch.setenv(CONNECTION_REF, str(_FIXTURE)) + bundle = tmp_path / "bundle" + bundle.mkdir() + materialize(GOLDEN / "manifest.json", bundle, _ingested_at()) + first = {p.name: p.read_bytes() for p in bundle.iterdir()} + materialize(GOLDEN / "manifest.json", bundle, _ingested_at()) + second = {p.name: p.read_bytes() for p in bundle.iterdir()} + assert first == second + + +class TestMutationControl: + """One changed determinism input → divergence from the golden bytes.""" + + def test_changed_source_row_diverges( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + # Prove the connector actually reads the db: mutate a copied db, then diverge. + case = tmp_path / "case" + shutil.copytree(GOLDEN, case) + db = case / "fixture" / "portefolje.sqlite" + con = sqlite3.connect(db) + con.execute("UPDATE costs SET amount = 9999.9 WHERE id = 1") + con.commit() + con.close() + monkeypatch.setenv(CONNECTION_REF, str(db)) + bundle = tmp_path / "bundle" + bundle.mkdir() + materialize(case / "manifest.json", bundle, _ingested_at()) + golden_bytes = (GOLDEN / "expected-bundle" / "ingest-costs.md").read_bytes() + assert (bundle / "ingest-costs.md").read_bytes() != golden_bytes + + def test_changed_timestamp_diverges( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setenv(CONNECTION_REF, str(_FIXTURE)) + bundle = tmp_path / "bundle" + bundle.mkdir() + materialize(GOLDEN / "manifest.json", bundle, "2099-01-01T00:00:00Z") + golden_bytes = (GOLDEN / "expected-bundle" / "ingest-costs.md").read_bytes() + assert (bundle / "ingest-costs.md").read_bytes() != golden_bytes + + def test_changed_manifest_bytes_change_the_stamp( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + # The ingest_manifest stamp is SHA-256 of the manifest's raw bytes (§5) — + # a whitespace-only edit still re-stamps every generated file. + monkeypatch.setenv(CONNECTION_REF, str(_FIXTURE)) + case = tmp_path / "case" + shutil.copytree(GOLDEN, case) + manifest = case / "manifest.json" + manifest.write_bytes(manifest.read_bytes() + b"\n") + # The copied db lives beside the copied manifest; point the ref there too. + monkeypatch.setenv(CONNECTION_REF, str(case / "fixture" / "portefolje.sqlite")) + bundle = tmp_path / "bundle" + bundle.mkdir() + materialize(manifest, bundle, _ingested_at()) + golden_bytes = (GOLDEN / "expected-bundle" / "ingest-costs.md").read_bytes() + assert (bundle / "ingest-costs.md").read_bytes() != golden_bytes diff --git a/tests/test_ingest_sql_loadbearing.py b/tests/test_ingest_sql_loadbearing.py new file mode 100644 index 0000000..623a9af --- /dev/null +++ b/tests/test_ingest_sql_loadbearing.py @@ -0,0 +1,156 @@ +"""Load-bearing ingest seams for the SQL source (ingest-spec §11) — D7 mirror of MAF I4. + +Each test goes RED when its seam detaches (the method-spec §11 regime) — a grønn-men-død test +is the failure mode the rule exists for. The seams, proven THROUGH the SQL connector (a local +sqlite fixture — no network, no credentials): + +- Typed-cell rendering (the NEW I5 seam) — SQL NULL renders as the empty string and a REAL keeps + its shortest round-trip form; a naive ``str()`` coercion (NULL → ``"None"``) breaks this. +- Provenance stamping — a sql-generated file carries the §7 provenance layer, ``source_system`` + = the sql source id. +- Navigability — the sql bundle is consumable by the UNCHANGED ``okf`` navigation. +- Verdict reservation — a sql manifest mapping to ``type: verdict`` is rejected fail-fast, + BEFORE the database is ever opened. +- Re-ingest layer safety — re-materialization over a sql bundle carrying a promoted verdict + preserves the verdict file AND its index link. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest +from pydantic import ValidationError + +from portfolio_optimiser_claude import okf +from portfolio_optimiser_claude.ingest import ManifestContract, load_manifest, materialize + +GOLDEN = Path(__file__).resolve().parents[1] / "examples" / "ingest-golden-sql" +CONNECTION_REF = "PORTEFOLJE_SQL_DSN" +INGESTED_AT = (GOLDEN / "ingested-at.txt").read_text(encoding="utf-8").strip() +_FIXTURE = GOLDEN / "fixture" / "portefolje.sqlite" + +_PROVENANCE_KEYS = ("source_system", "source_query", "ingested_at", "ingest_manifest", "generated") + + +def _materialized(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: + monkeypatch.setenv(CONNECTION_REF, str(_FIXTURE)) + bundle = tmp_path / "bundle" + bundle.mkdir() + materialize(GOLDEN / "manifest.json", bundle, INGESTED_AT) + return bundle + + +class TestTypedCellSeam: + """NEW I5 seam: SQL typed-cell rendering (RED if it degrades to a bare ``str()``).""" + + def test_null_cell_renders_as_empty_string( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + bundle = _materialized(tmp_path, monkeypatch) + costs = (bundle / "ingest-costs.md").read_text(encoding="utf-8") + # costs row id=2 has note = SQL NULL → an empty cell, NOT the string "None". + assert "| 2 | sensor | 89.9 | |" in costs + assert "None" not in costs + + def test_real_keeps_shortest_round_trip( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + bundle = _materialized(tmp_path, monkeypatch) + costs = (bundle / "ingest-costs.md").read_text(encoding="utf-8") + assert "| 3 | kabel | 4200.0 | rest |" in costs # a whole-valued REAL keeps its .0 + + +class TestProvenanceStamping: + """Seam: a sql-generated file carries the §7 provenance layer (RED if it stops).""" + + def test_generated_sql_file_carries_the_provenance_layer( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + bundle = _materialized(tmp_path, monkeypatch) + concept = okf.parse_concept_file(bundle / "ingest-costs.md") + for key in _PROVENANCE_KEYS: + assert key in concept.frontmatter, f"provenance key {key} detached" + assert concept.frontmatter["source_system"] == "portefolje-db" + assert concept.frontmatter["ingested_at"] == INGESTED_AT + assert concept.frontmatter["generated"] == "true" + stamp = load_manifest(GOLDEN / "manifest.json").stamp + assert concept.frontmatter["ingest_manifest"] == stamp + + +class TestNavigability: + """Seam: the generated sql bundle is consumable by the UNCHANGED okf navigation.""" + + def test_sql_bundle_navigates_via_unchanged_okf( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + bundle = _materialized(tmp_path, monkeypatch) + names = [c.path.name for c in okf.navigate_bundle(bundle)] + # Every generated file must be REACHABLE via index cross-links (RED if linking detaches). + assert names == ["index.md", "ingest-costs.md", "ingest-meta.md"] + + def test_sql_context_renders_the_extracted_rows( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + bundle = _materialized(tmp_path, monkeypatch) + context = okf.bundle_context(bundle) + assert "LED-armatur" in context # a costs body cell + assert "owner" in context # a meta body cell + + +class TestVerdictReservation: + """Seam: a sql manifest mapping to the verdict layer is rejected fail-fast (§3).""" + + def _sql_manifest(self, okf_type: str) -> dict: + return { + "manifest_version": 1, + "source": {"type": "sql", "id": "db", "connection_ref": CONNECTION_REF}, + "bundle_summary": "s", + "extractions": [ + {"id": "e", "title": "T", "query": "SELECT 1", "okf_type": okf_type, "max_rows": 1} + ], + } + + def test_verdict_okf_type_is_rejected(self) -> None: + with pytest.raises(ValidationError): + ManifestContract(**self._sql_manifest("verdict")) + + def test_rejection_is_fail_fast_before_the_db_is_opened( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + # With the connection_ref env var UNSET, a run that got past validation would fail with a + # plain ValueError trying to open the db. Rejection is a ValidationError at validation — + # so the db is never opened (RED if the reservation stops firing first). + monkeypatch.delenv(CONNECTION_REF, raising=False) + manifest = tmp_path / "manifest.json" + manifest.write_text(json.dumps(self._sql_manifest("verdict")), encoding="utf-8") + bundle = tmp_path / "bundle" + with pytest.raises(ValidationError): + materialize(manifest, bundle, INGESTED_AT) + assert not bundle.exists() or not any(bundle.iterdir()) + + +class TestReingestLayerSafety: + """Seam: re-ingest over a sql bundle preserves a promoted verdict AND its link (§3, §6).""" + + def test_promoted_verdict_and_link_survive_sql_reingest( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + bundle = _materialized(tmp_path, monkeypatch) + (bundle / "promoted-verdict-led.md").write_text( + "---\ntype: verdict\ndecision: approved\ndescription: LED holds.\n---\n\nBody.\n", + encoding="utf-8", + ) + index = bundle / "index.md" + index.write_text( + index.read_text(encoding="utf-8") + "- [promoted](promoted-verdict-led.md)\n", + encoding="utf-8", + ) + materialize(GOLDEN / "manifest.json", bundle, INGESTED_AT) # re-ingest + + assert (bundle / "promoted-verdict-led.md").is_file() # verdict file survives + index_text = (bundle / "index.md").read_text(encoding="utf-8") + assert "- [promoted](promoted-verdict-led.md)" in index_text # verdict link survives + assert (bundle / "ingest-costs.md").is_file() # ingest files still refreshed + assert "](ingest-costs.md)" in index_text