open/portfolio-optimiser
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity Actions

test(ingest): SQL connector + load-bearing + golden, detach-proven (I4)

Browse source 
New: test_ingest_sql.py (connector mechanics + typed rendering + dispatch), test_ingest_sql_loadbearing.py (SQL seams: typed-NULL, typed-REAL, navigability via unchanged okf), test_ingest_golden_sql.py + examples/ingest-golden-sql/ (byte-frozen sqlite golden; NULL->empty, 4200.0->4200.0 and escape discriminate a typed impl from naive ones). Updated: sql-source refusal test repointed to http (I6); manifest schema-breadth comment. Each new seam proven RED at its detach point.

Refs: shared/ingest-spec.md 11 golden + load-bearing table
This commit is contained in:
Kjell Tore Guttormsen 2026-07-04 06:56:02 +02:00
commit 4f45fe6037
11 changed files with 446 additions and 5 deletions
Download patch file Download diff file Expand all files Collapse all files

3
examples/ingest-golden-sql/expected-bundle/index.md Normal file
View file

@ -0,0 +1,3 @@
Golden extraction case: two SQL extracts from a local project database (sql source type).
- [Project costs](ingest-costs.md)
- [Catalogue metadata](ingest-meta.md)

15
examples/ingest-golden-sql/expected-bundle/ingest-costs.md Normal file
View file

@ -0,0 +1,15 @@
---
type: dataset
title: Project costs
source_system: portefolje-db
source_query: SELECT id, item, amount, note FROM costs ORDER BY id
ingested_at: 2026-07-04T12:00:00Z
ingest_manifest: manifest@a4b891aea55df97b
generated: true
---
| id | item | amount | note |
| --- | --- | --- | --- |
| 1 | LED-armatur | 1200.5 | godkjent |
| 2 | sensor | 89.9 | |
| 3 | kabel | 4200.0 | rest |

14
examples/ingest-golden-sql/expected-bundle/ingest-meta.md Normal file
View file

@ -0,0 +1,14 @@
---
type: reference
title: Catalogue metadata
source_system: portefolje-db
source_query: SELECT k, v FROM meta ORDER BY k
ingested_at: 2026-07-04T12:00:00Z
ingest_manifest: manifest@a4b891aea55df97b
generated: true
---
| k | v |
| --- | --- |
| owner | anlegg \| drift |
| path | c:\\temp |

BIN
examples/ingest-golden-sql/fixture/portefolje.sqlite Normal file
View file

Binary file not shown.

1
examples/ingest-golden-sql/ingested-at.txt Normal file
View file

@ -0,0 +1 @@
2026-07-04T12:00:00Z

21
examples/ingest-golden-sql/manifest.json Normal file
View file

@ -0,0 +1,21 @@
{
"manifest_version": 1,
"source": {"type": "sql", "id": "portefolje-db", "connection_ref": "PORTEFOLJE_SQL_DSN"},
"bundle_summary": "Golden extraction case: two SQL extracts from a local project database (sql source type).",
"extractions": [
{
"id": "costs",
"title": "Project costs",
"query": "SELECT id, item, amount, note FROM costs ORDER BY id",
"okf_type": "dataset",
"max_rows": 10
},
{
"id": "meta",
"title": "Catalogue metadata",
"query": "SELECT k, v FROM meta ORDER BY k",
"okf_type": "reference",
"max_rows": 10
}
]
}

47
tests/test_ingest_golden_sql.py Normal file
View file

@ -0,0 +1,47 @@
"""I4 golden regression — the §11 golden extraction case for `sql`, byte-for-byte.
Mirrors tests/test_ingest_golden.py for the second source type. The case pins the §5 SQL type
rules AS RULES: SQL NULL → empty cell (never ``None``), REAL 4200.0 → ``4200.0`` (float form
preserved, not integer-coerced), non-integral 1200.5/89.9, and the escape order on the sql path
(``|`` → ``\\|``, ``\\`` → ``\\\\``). RED when ANY byte of the expected bundle diverges (spec
§11 "Golden regression" seam).
The fixture db is committed under ``fixture/`` (rebuild:
``.claude/projects/2026-07-04-i4-ingest-sql-maf/build_fixture.py``); the ``connection_ref`` env
var is set to it (the §4 run-time resolution). The db PATH never enters the bundle, so the
golden is checkout-location independent.
"""
from __future__ import annotations
from pathlib import Path
import pytest
from portfolio_optimiser.ingest import materialize
GOLDEN = Path(__file__).resolve().parents[1] / "examples" / "ingest-golden-sql"
def _bundle_bytes(directory: Path) -> dict[str, bytes]:
return {p.name: p.read_bytes() for p in sorted(directory.iterdir()) if p.is_file()}
def test_golden_sql_extraction_is_bit_deterministic(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
ingested_at = (GOLDEN / "ingested-at.txt").read_text(encoding="utf-8").strip()
monkeypatch.setenv("PORTEFOLJE_SQL_DSN", str(GOLDEN / "fixture" / "portefolje.sqlite"))
out = tmp_path / "bundle"
materialize(GOLDEN / "manifest.json", out, ingested_at=ingested_at)
expected = _bundle_bytes(GOLDEN / "expected-bundle")
actual = _bundle_bytes(out)
# File-SET equality first — catches extra AND missing files, not just diverging bytes.
assert actual.keys() == expected.keys()
for name, content in expected.items():
assert actual[name] == content, f"golden byte divergence in {name}"
# §10: a second run over the same output leaves every byte unchanged.
materialize(GOLDEN / "manifest.json", out, ingested_at=ingested_at)
assert _bundle_bytes(out) == expected

2
tests/test_ingest_manifest.py
View file

@ -163,7 +163,7 @@ def test_base_url_embedded_credentials_rejected() -> None:
def test_sql_and_http_variants_validate() -> None:
# Schema breadth (brief assumption 1): the polymorphic §4 schema validates all three
# source variants; only the `file` connector EXECUTES in I2 (sql → I4, http → I6).
# source variants; the `file` and `sql` connectors EXECUTE (I2, I4), http → I6.
sql = ManifestV1.model_validate(
_variant(source={"type": "sql", "id": "db", "connection_ref": "PROJ_DB"})
)

9
tests/test_ingest_materialize.py
View file

@ -242,11 +242,12 @@ def test_two_runs_with_identical_inputs_are_byte_identical(tmp_path: Path) -> No
assert first == second # §10 idempotence at file level
def test_sql_source_has_no_connector_in_i2(tmp_path: Path) -> None:
# The polymorphic schema VALIDATES sql (brief assumption 1); executing it is I4 —
# attempting to materialize is an explicit refusal, never a silent no-op.
def test_http_source_has_no_connector(tmp_path: Path) -> None:
# The polymorphic schema VALIDATES http (brief assumption 1); executing it is the gated
# I6 extension point — attempting to materialize is an explicit refusal, never a silent
# no-op. (sql now EXECUTES, I4; see tests/test_ingest_sql.py.)
manifest_path, bundle_dir = _project(
tmp_path, source={"type": "sql", "id": "db", "connection_ref": "PROJ_DB"}
tmp_path, source={"type": "http", "id": "api", "base_url": "https://host/api"}
)
with pytest.raises(IngestError):
materialize(manifest_path, bundle_dir, ingested_at=_INGESTED_AT)

199
tests/test_ingest_sql.py Normal file
View file

@ -0,0 +1,199 @@
"""I4 Step 2/3 — the ``sql`` connector (``read_sql``) + ``materialize`` dispatch.
Mirrors tests/test_ingest_materialize.py for the second source type. Covers the §4 connector
rules for ``sql`` (env-resolved ``connection_ref`` → sqlite path, read-only enforcement,
single-statement, streaming ``max_rows`` cap §8) and the §5 TYPED cell rendering the ``file``
path never exercised (all-strings): INTEGER→plain decimal, REAL→shortest round-trip, TEXT
verbatim, SQL NULL→empty string, BLOB/other→fail (never silent coercion). Offline: sqlite
fixtures under tmp_path, zero network, zero credentials, zero model calls.
Pinned SQL §5 decisions (spec-silent, delegated to I4 by the session plan; see brief):
``connection_ref`` names an env var whose value is a filesystem path to a sqlite database
opened read-only; ``float`` renders via ``repr`` (Python's shortest round-trip form).
"""
from __future__ import annotations
import json
import sqlite3
from pathlib import Path
from typing import Any
import pytest
from portfolio_optimiser import okf
from portfolio_optimiser.ingest import (
IngestError,
_sql_value_to_text,
materialize,
read_sql,
)
_INGESTED_AT = "2026-07-03T12:00:00Z"
def _make_db(path: Path, script: str) -> Path:
"""Build a sqlite fixture (read-write) from a DDL/DML script, then close it."""
conn = sqlite3.connect(path)
try:
conn.executescript(script)
conn.commit()
finally:
conn.close()
return path
def _sql_manifest(
tmp_path: Path, extractions: list[dict[str, Any]], *, connection_ref: str = "PROJ_DB"
) -> Path:
manifest = {
"manifest_version": 1,
"source": {"type": "sql", "id": "db", "connection_ref": connection_ref},
"bundle_summary": "SQL extracts from the project database.",
"extractions": extractions,
}
path = tmp_path / "manifest.json"
path.write_text(json.dumps(manifest), encoding="utf-8")
return path
# --- connector: env resolution + fail-fast (§4) --------------------------------------------------
def test_read_sql_unset_connection_ref_fails(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.delenv("PROJ_DB", raising=False)
with pytest.raises(IngestError):
read_sql("PROJ_DB", "SELECT 1", max_rows=10)
def test_read_sql_missing_db_file_fails(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setenv("PROJ_DB", str(tmp_path / "does-not-exist.sqlite"))
with pytest.raises(IngestError):
read_sql("PROJ_DB", "SELECT 1", max_rows=10)
# --- connector: §5 typed cell rendering ----------------------------------------------------------
def test_read_sql_renders_each_type_per_section5(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
"""INTEGER→plain decimal, REAL→shortest round-trip (``repr``, integral 4200.0 preserved),
TEXT verbatim, SQL NULL→empty string — all in one ordered extract."""
db = _make_db(
tmp_path / "s.sqlite",
"""
CREATE TABLE t (i INTEGER, r REAL, s TEXT, n TEXT);
INSERT INTO t VALUES (7, 1.5, 'hei', NULL);
INSERT INTO t VALUES (3, 4200.0, 'verden', 'x');
""",
)
monkeypatch.setenv("PROJ_DB", str(db))
header, rows = read_sql("PROJ_DB", "SELECT i, r, s, n FROM t ORDER BY i", max_rows=100)
assert header == ["i", "r", "s", "n"]
assert rows == [
["3", "4200.0", "verden", "x"],
["7", "1.5", "hei", ""],
]
def test_read_sql_null_is_empty_string(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
db = _make_db(
tmp_path / "n.sqlite",
"CREATE TABLE t (v TEXT); INSERT INTO t VALUES (NULL);",
)
monkeypatch.setenv("PROJ_DB", str(db))
_, rows = read_sql("PROJ_DB", "SELECT v FROM t", max_rows=10)
assert rows == [[""]] # SQL NULL → empty string, never "None"
def test_read_sql_blob_value_fails_never_silent(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
db = _make_db(
tmp_path / "b.sqlite",
"CREATE TABLE t (b BLOB); INSERT INTO t VALUES (x'00ff');",
)
monkeypatch.setenv("PROJ_DB", str(db))
with pytest.raises(IngestError): # §5: any other value type MUST fail
read_sql("PROJ_DB", "SELECT b FROM t", max_rows=10)
def test_sql_value_to_text_rejects_bool_defensively() -> None:
# sqlite3 never returns Python bool; the guard prevents str(True)->"True" silent coercion
# (bool is an int subclass, so it would otherwise slip through the integer branch).
with pytest.raises(IngestError):
_sql_value_to_text(True)
# --- connector: §8 cap + §4 read-only / single-statement -----------------------------------------
def test_read_sql_max_rows_exceeded_is_error(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
db = _make_db(
tmp_path / "m.sqlite",
"CREATE TABLE t (i INTEGER); INSERT INTO t VALUES (1),(2),(3);",
)
monkeypatch.setenv("PROJ_DB", str(db))
with pytest.raises(IngestError): # §8: error, never silent truncation
read_sql("PROJ_DB", "SELECT i FROM t ORDER BY i", max_rows=2)
def test_read_sql_is_read_only(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
db = _make_db(
tmp_path / "ro.sqlite",
"CREATE TABLE t (i INTEGER); INSERT INTO t VALUES (1);",
)
monkeypatch.setenv("PROJ_DB", str(db))
with pytest.raises(IngestError): # §4: read-only enforced at the DB (mode=ro)
read_sql("PROJ_DB", "INSERT INTO t VALUES (2)", max_rows=10)
_, rows = read_sql("PROJ_DB", "SELECT count(*) FROM t", max_rows=10)
assert rows == [["1"]] # the refused write never happened
def test_read_sql_rejects_multiple_statements(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
db = _make_db(
tmp_path / "ms.sqlite",
"CREATE TABLE t (i INTEGER); INSERT INTO t VALUES (1);",
)
monkeypatch.setenv("PROJ_DB", str(db))
with pytest.raises(IngestError): # §4: one statement
read_sql("PROJ_DB", "SELECT i FROM t; SELECT i FROM t", max_rows=10)
# --- materialize dispatch: sql executes (was refused in I2) ---------------------------------------
def test_sql_manifest_materializes_with_provenance(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
db = _make_db(
tmp_path / "s.sqlite",
"CREATE TABLE costs (item TEXT, amount REAL); INSERT INTO costs VALUES ('led', 120.0);",
)
monkeypatch.setenv("PROJ_DB", str(db))
manifest = _sql_manifest(
tmp_path,
[
{
"id": "costs",
"title": "Project costs",
"query": "SELECT item, amount FROM costs ORDER BY item",
"okf_type": "dataset",
"max_rows": 100,
}
],
)
written = materialize(manifest, tmp_path / "bundle", ingested_at=_INGESTED_AT)
assert len(written) == 1
fm = okf.parse_frontmatter(written[0])
assert fm["source_system"] == "db"
assert fm["generated"] == "true"
assert fm["source_query"] == "SELECT item, amount FROM costs ORDER BY item"
body = written[0].read_text(encoding="utf-8")
assert "| item | amount |" in body
assert "| led | 120.0 |" in body

140
tests/test_ingest_sql_loadbearing.py Normal file
View file

@ -0,0 +1,140 @@
"""I4 load-bearing seams — the ``sql`` path end-to-end (ingest spec §11's detach-RED regime).
The ``file`` seams (tests/test_ingest_loadbearing.py) are source-agnostic where they can be;
these tests prove the SEAMS the ``sql`` source introduces — typed §5 rendering (the ``file``
path is all-strings) and that a SQL-generated bundle flows through the SAME navigability and
provenance machinery. Each test is built so the seam under test is the ONLY thing preventing
the observable artifact (the Fase-2 green-but-dead trap).
- NAVIGABILITY (§2/§11): a SQL-generated bundle is consumable by the UNCHANGED
``okf.navigate_bundle``/``bundle_context`` — reachable via index cross-links, classified by
``okf_type``, provenance riding through. RED when index linking is detached.
- TYPED NULL (§5): SQL NULL renders as the empty string, NEVER ``"None"``. RED when the None
branch degrades to ``str(value)``.
- TYPED REAL (§5): a REAL value keeps its shortest round-trip float form (4200.0 → ``4200.0``),
NEVER integer-coerced. RED when the float branch degrades to ``str(int(value))``.
"""
from __future__ import annotations
import json
import sqlite3
from pathlib import Path
from typing import Any
from portfolio_optimiser import okf
from portfolio_optimiser.ingest import materialize
_INGESTED_AT = "2026-07-03T12:00:00Z"
def _make_db(path: Path, script: str) -> Path:
conn = sqlite3.connect(path)
try:
conn.executescript(script)
conn.commit()
finally:
conn.close()
return path
def _sql_manifest(tmp_path: Path, extractions: list[dict[str, Any]]) -> Path:
manifest = {
"manifest_version": 1,
"source": {"type": "sql", "id": "db", "connection_ref": "PROJ_DB"},
"bundle_summary": "SQL extracts from the project database.",
"extractions": extractions,
}
path = tmp_path / "manifest.json"
path.write_text(json.dumps(manifest), encoding="utf-8")
return path
def test_sql_generated_bundle_is_navigable_via_unchanged_okf(
tmp_path: Path, monkeypatch: Any
) -> None:
"""LOAD-BEARING NAVIGABILITY: the SQL path produces a bundle the untouched OKF navigation
reaches through index cross-links (classified by okf_type, provenance preserved) and
``bundle_context`` renders. RED when index linking is detached (file exists, unreachable)."""
db = _make_db(
tmp_path / "s.sqlite",
"CREATE TABLE costs (item TEXT, amount REAL);"
" INSERT INTO costs VALUES ('led', 120.0), ('hvac', 80.0);",
)
monkeypatch.setenv("PROJ_DB", str(db))
manifest = _sql_manifest(
tmp_path,
[
{
"id": "costs",
"title": "Project costs",
"query": "SELECT item, amount FROM costs ORDER BY item",
"okf_type": "dataset",
"max_rows": 100,
}
],
)
bundle_dir = tmp_path / "bundle"
materialize(manifest, bundle_dir, ingested_at=_INGESTED_AT)
bundle = okf.navigate_bundle(str(bundle_dir))
by_name = {f.name: f for f in bundle.files}
assert "ingest-costs.md" in by_name
assert by_name["ingest-costs.md"].type == "dataset"
assert by_name["ingest-costs.md"].frontmatter["ingest_manifest"] # provenance rides through
context = okf.bundle_context(bundle)
assert "Project costs" in context and "| hvac | 80.0 |" in context
def test_sql_null_renders_as_empty_not_none(tmp_path: Path, monkeypatch: Any) -> None:
"""LOAD-BEARING TYPED NULL: detaching the None branch to ``str(value)`` would render the
string ``None`` into the bundle; the empty cell is the ONLY correct §5 form."""
db = _make_db(
tmp_path / "s.sqlite",
"CREATE TABLE costs (item TEXT, note TEXT); INSERT INTO costs VALUES ('sensor', NULL);",
)
monkeypatch.setenv("PROJ_DB", str(db))
manifest = _sql_manifest(
tmp_path,
[
{
"id": "costs",
"title": "Project costs",
"query": "SELECT item, note FROM costs ORDER BY item",
"okf_type": "dataset",
"max_rows": 100,
}
],
)
bundle_dir = tmp_path / "bundle"
materialize(manifest, bundle_dir, ingested_at=_INGESTED_AT)
body = (bundle_dir / "ingest-costs.md").read_text(encoding="utf-8")
assert "| sensor | |" in body # NULL → empty cell
assert "None" not in body # RED if the None branch degrades to str()
def test_sql_real_preserves_float_form(tmp_path: Path, monkeypatch: Any) -> None:
"""LOAD-BEARING TYPED REAL: a REAL keeps its shortest round-trip float form; integer
coercion (``4200``) is the detach this test catches."""
db = _make_db(
tmp_path / "s.sqlite",
"CREATE TABLE costs (item TEXT, amount REAL); INSERT INTO costs VALUES ('kabel', 4200.0);",
)
monkeypatch.setenv("PROJ_DB", str(db))
manifest = _sql_manifest(
tmp_path,
[
{
"id": "costs",
"title": "Project costs",
"query": "SELECT item, amount FROM costs ORDER BY item",
"okf_type": "dataset",
"max_rows": 100,
}
],
)
bundle_dir = tmp_path / "bundle"
materialize(manifest, bundle_dir, ingested_at=_INGESTED_AT)
body = (bundle_dir / "ingest-costs.md").read_text(encoding="utf-8")
assert "| kabel | 4200.0 |" in body # float form preserved
assert "| kabel | 4200 |" not in body # RED if integer-coerced