open/portfolio-optimiser-claude
1
0
Fork 0
Code Issues Releases 1 Activity

feat(ingest): adopt llm-ingestion-okf as Door A implementation (first consumer)

Browse source 
Replace the local 391-line ingest implementation with a thin adapter over
the shared llm-ingestion-okf library (git-pinned dae0bd1a via Forgejo,
tool.uv.sources). The materialize() signature is preserved; error types are
now the library's typed hierarchy rooted in IngestError, re-exported from
the consumer seam.

- tests/test_ingest_adoption.py: new load-bearing seam tests (delegation,
  offline invariant — allow_network is never passed, error contract),
  detach-proven red twice.
- Golden suites (file + sql) pass UNCHANGED — byte-exact behaviour proven
  against the repo-local fixtures.
- 6 test files migrated to the library error hierarchy; escaping/typed-cell
  unit tests dropped (byte-bound by the ingest-edge.md golden, unit-owned by
  the library's own 189-test suite). Provenance stamp now asserted
  independently from the §5 rule.
- mypy override follow_untyped_imports for llm_ingestion_okf (no py.typed
  upstream yet — reported as a finding).

Suite: 386 passed; ruff + format + mypy --strict clean; shared/, examples/,
runs/s10/ and run_s10.py byte-untouched.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Kjell Tore Guttormsen 2026-07-16 20:46:51 +02:00
commit 5732d13369
9 changed files with 307 additions and 495 deletions
Download patch file Download diff file Expand all files Collapse all files

114
tests/test_ingest_sql.py
View file

@ -1,8 +1,13 @@
"""Ingest SQL unit + fail-fast contract tests (ingest-spec §4, §5, §8).
Pins the ``SqlSource`` manifest contract, the §5 typed-cell rendering (int/float/NULL/other),
the runtime ``connection_ref`` resolution, the §8 size cap, and read-only access enforcement.
Everything is local: a tmp sqlite fixture, no network, no credentials.
Pins the ``sql`` manifest contract, the §5 typed-cell fail-fast (a BLOB is never silently
coerced), the runtime ``connection_ref`` resolution, the §8 size cap, and read-only access
enforcement. Everything is local: a tmp sqlite fixture, no network, no credentials.
Since the adoption, every seam is proven THROUGH the consumer entry points
(``load_manifest``/``materialize``) — the connector internals are unit-owned by the
llm-ingestion-okf suite. The §5 typed-cell happy paths stay bound here in the repo by the
sql golden (int/float/text) and test_ingest_sql_loadbearing.py (NULL/REAL).
"""
from __future__ import annotations
@ -12,17 +17,18 @@ import sqlite3
from pathlib import Path
import pytest
from pydantic import ValidationError
from portfolio_optimiser_claude.ingest import (
ManifestContract,
ManifestError,
RenderError,
SourceError,
SqlSource,
_read_sql,
_render_sql_cell,
_resolve_connection_ref,
load_manifest,
materialize,
)
INGESTED_AT = "2026-07-04T12:00:00Z"
def _db(tmp_path: Path, rows: list[tuple[object, ...]]) -> Path:
db = tmp_path / "src.sqlite"
@ -34,75 +40,65 @@ def _db(tmp_path: Path, rows: list[tuple[object, ...]]) -> Path:
return db
def _sql_manifest() -> dict:
def _sql_manifest(query: str = "SELECT a FROM t ORDER BY a") -> dict:
return {
"manifest_version": 1,
"source": {"type": "sql", "id": "db", "connection_ref": "SRC_DSN"},
"bundle_summary": "s",
"extractions": [
{
"id": "e",
"title": "T",
"query": "SELECT a FROM t ORDER BY a",
"okf_type": "dataset",
"max_rows": 5,
}
{"id": "e", "title": "T", "query": query, "okf_type": "dataset", "max_rows": 5}
],
}
def _write_manifest(tmp_path: Path, manifest: dict) -> Path:
path = tmp_path / "manifest.json"
path.write_text(json.dumps(manifest), encoding="utf-8")
return path
class TestSqlSourceContract:
"""§4: the sql source is a valid discriminated variant; the reference is required."""
def test_valid_sql_source_loads(self) -> None:
contract = ManifestContract(**_sql_manifest())
assert isinstance(contract.source, SqlSource)
assert contract.source.connection_ref == "SRC_DSN"
def test_valid_sql_source_loads(self, tmp_path: Path) -> None:
manifest = load_manifest(_write_manifest(tmp_path, _sql_manifest()))
assert isinstance(manifest.source, SqlSource)
assert manifest.source.connection_ref == "SRC_DSN"
def test_connection_ref_is_required(self) -> None:
def test_connection_ref_is_required(self, tmp_path: Path) -> None:
manifest = _sql_manifest()
del manifest["source"]["connection_ref"]
with pytest.raises(ValidationError):
ManifestContract(**manifest)
with pytest.raises(ManifestError):
load_manifest(_write_manifest(tmp_path, manifest))
def test_sql_source_id_grammar_enforced(self) -> None:
def test_sql_source_id_grammar_enforced(self, tmp_path: Path) -> None:
manifest = _sql_manifest()
manifest["source"]["id"] = "Bad Id"
with pytest.raises(ValidationError):
ManifestContract(**manifest)
with pytest.raises(ManifestError):
load_manifest(_write_manifest(tmp_path, manifest))
class TestTypedCellRendering:
"""§5: None→'', int→decimal, float→shortest round-trip, str→verbatim, other→fail."""
class TestTypedCellFailFast:
"""§5: an unsupported cell type (a BLOB, say) MUST fail — never a silent coercion."""
def test_null_is_empty_string(self) -> None:
assert _render_sql_cell(None) == ""
def test_int_is_plain_decimal(self) -> None:
assert _render_sql_cell(3) == "3"
assert _render_sql_cell(0) == "0"
def test_float_is_shortest_round_trip(self) -> None:
assert _render_sql_cell(1200.5) == "1200.5"
assert _render_sql_cell(89.9) == "89.9"
assert _render_sql_cell(4200.0) == "4200.0" # a whole-valued REAL keeps its .0
def test_str_is_returned_raw(self) -> None:
# _render_sql_cell returns the RAW string; §5 escaping is _escape_cell's job.
assert _render_sql_cell("a|b\\c") == "a|b\\c"
def test_other_value_type_fails(self) -> None:
with pytest.raises(ValueError, match="not a supported value type"):
_render_sql_cell(b"\x00\x01") # a BLOB — never a silent coercion
def test_blob_cell_fails_typed(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
db = _db(tmp_path, [(1, 1.0, "x", b"\x00\x01")])
manifest_path = _write_manifest(tmp_path, _sql_manifest("SELECT d FROM t"))
monkeypatch.setenv("SRC_DSN", str(db))
with pytest.raises(RenderError, match="unsupported SQL cell type"):
materialize(manifest_path, tmp_path / "bundle", INGESTED_AT)
class TestConnectorRuntime:
"""§8: reference resolution, size cap, and read-only access — all fail-fast."""
def test_unset_connection_ref_fails_fast(self, monkeypatch: pytest.MonkeyPatch) -> None:
def test_unset_connection_ref_fails_fast(
self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.delenv("SRC_DSN", raising=False)
with pytest.raises(ValueError, match="not set in the environment"):
_resolve_connection_ref("SRC_DSN")
manifest_path = _write_manifest(tmp_path, _sql_manifest())
with pytest.raises(SourceError, match="not set in the environment"):
materialize(manifest_path, tmp_path / "bundle", INGESTED_AT)
def test_max_rows_cap_enforced_fail_fast(
self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
@ -110,14 +106,16 @@ class TestConnectorRuntime:
db = _db(tmp_path, [(1, 1.0, "x", None), (2, 2.0, "y", None)])
manifest = _sql_manifest()
manifest["extractions"][0]["max_rows"] = 1 # 2 rows > 1
manifest_path = tmp_path / "manifest.json"
manifest_path.write_text(json.dumps(manifest), encoding="utf-8")
manifest_path = _write_manifest(tmp_path, manifest)
monkeypatch.setenv("SRC_DSN", str(db))
with pytest.raises(ValueError, match="max_rows"):
materialize(manifest_path, tmp_path / "bundle", "2026-07-04T12:00:00Z")
with pytest.raises(SourceError, match="max_rows"):
materialize(manifest_path, tmp_path / "bundle", INGESTED_AT)
def test_connection_is_read_only(self, tmp_path: Path) -> None:
def test_connection_is_read_only(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
db = _db(tmp_path, [(1, 1.0, "x", None)])
# The connector opens read-only (§4 SHOULD): a write statement is refused.
with pytest.raises(sqlite3.OperationalError):
_read_sql(str(db), "UPDATE t SET a = 9", 5)
# The connector opens read-only (§4 SHOULD): a write statement is refused at the
# database and surfaces as a typed SourceError.
manifest_path = _write_manifest(tmp_path, _sql_manifest("UPDATE t SET a = 9"))
monkeypatch.setenv("SRC_DSN", str(db))
with pytest.raises(SourceError, match="readonly"):
materialize(manifest_path, tmp_path / "bundle", INGESTED_AT)