From bbdcd62ae614e65018a2212316ea60c6eb02b2a1 Mon Sep 17 00:00:00 2001 From: Kjell Tore Guttormsen Date: Sat, 4 Jul 2026 17:04:05 +0200 Subject: [PATCH] feat(ingest): gate http behind allow_network run-flag + dispatch (I6) --- src/portfolio_optimiser/ingest.py | 49 ++++++-- tests/test_ingest_http_loadbearing.py | 175 ++++++++++++++++++++++++++ tests/test_ingest_materialize.py | 9 +- 3 files changed, 219 insertions(+), 14 deletions(-) create mode 100644 tests/test_ingest_http_loadbearing.py diff --git a/src/portfolio_optimiser/ingest.py b/src/portfolio_optimiser/ingest.py index f566293..86ed63f 100644 --- a/src/portfolio_optimiser/ingest.py +++ b/src/portfolio_optimiser/ingest.py @@ -470,7 +470,12 @@ def _update_index_lines( def materialize( - manifest_path: str | Path, bundle_dir: str | Path, *, ingested_at: str + manifest_path: str | Path, + bundle_dir: str | Path, + *, + ingested_at: str, + allow_network: bool = False, + http_get: HttpGet | None = None, ) -> list[Path]: """Materialize a manifest's extractions into an OKF bundle (§5): the I2 invocation surface (a ``python -m`` CLI is deliberately deferred). @@ -478,10 +483,15 @@ def materialize( Three explicit inputs — manifest, target bundle dir, and ``ingested_at`` (REQUIRED keyword, NO wall-clock default, stamped verbatim; mirrors the promotion gate's timestamp rule). Deterministic and offline: zero model calls, zero network for the - ``file`` source type. All extractions execute and render IN MEMORY before the first - disk mutation (crash-window mitigation for the non-atomic §5 replace sequence; + ``file``/``sql`` source types. All extractions execute and render IN MEMORY before the + first disk mutation (crash-window mitigation for the non-atomic §5 replace sequence; recovery = idempotent re-run, §10). Source calls are logged per §8 (which source, - when = the ``ingested_at`` argument, row count) — never cell contents, never secrets.""" + when = the ``ingested_at`` argument, row count) — never cell contents, never secrets. + + ``allow_network`` (I6, §8) is the per-run network opt-in: an ``http`` source is refused + fail-fast unless it is set — the manifest itself cannot grant network access (local-only + default, no silent egress). ``http_get`` optionally injects the transport seam (default + ``_urllib_get``, the only socket path); both are ignored for ``file``/``sql`` sources.""" if not _INGESTED_AT_RE.match(ingested_at): raise ValueError( f"ingested_at must be ISO-8601 UTC with a Z suffix " @@ -490,10 +500,14 @@ def materialize( manifest_file = Path(manifest_path) manifest, stamp = load_manifest(manifest_file) source = manifest.source - if isinstance(source, HttpSource): + # §8 network gate (I6): refuse http fail-fast BEFORE any source access unless the per-run + # flag is set. The `and not allow_network` clause is the load-bearing seam — the manifest + # cannot grant itself network access (local-only default, no silent egress). + if isinstance(source, HttpSource) and not allow_network: raise IngestError( - f"source type {source.type!r} is a gated extension point (I6) with no connector " - "in this implementation — the schema validates it, execution defers" + f"source type {source.type!r} requires the per-run network opt-in " + "(materialize(..., allow_network=True)) — the manifest cannot grant itself network " + "access (ingest spec §8, local-only default, no silent egress)" ) # Pinned decision (file): a relative root resolves against the manifest file's directory — # never the process cwd, or the extraction would not be reproducible. (sql resolves its @@ -508,17 +522,32 @@ def materialize( for extraction in manifest.extractions: if isinstance(source, FileSource): header, rows = read_csv(root, extraction.query, max_rows=extraction.max_rows) - else: # SqlSource — http already refused above + body = render_table(header, rows) + row_count = len(rows) + elif isinstance(source, HttpSource): + # The gate above guarantees allow_network here; _urllib_get is the only socket path. + get = http_get if http_get is not None else _urllib_get + text = read_http( + source.base_url, + extraction.query, + max_rows=extraction.max_rows, + credential_ref=source.credential_ref, + get=get, + ) + body = _render_fenced_block(text) # verbatim fenced block, NOT table-escaped + row_count = len(text.splitlines()) + else: # SqlSource header, rows = read_sql( source.connection_ref, extraction.query, max_rows=extraction.max_rows ) + body = render_table(header, rows) + row_count = len(rows) _LOGGER.info( "source call: source=%s ingested_at=%s rows=%d", source.id, ingested_at, - len(rows), + row_count, ) - body = render_table(header, rows) content = _render_concept_file( manifest, extraction, body, ingested_at=ingested_at, stamp=stamp ) diff --git a/tests/test_ingest_http_loadbearing.py b/tests/test_ingest_http_loadbearing.py new file mode 100644 index 0000000..efd0017 --- /dev/null +++ b/tests/test_ingest_http_loadbearing.py @@ -0,0 +1,175 @@ +"""I6 load-bearing seams — the ``http`` path end-to-end (ingest spec §11's detach-RED regime). + +Each test is built so the seam under test is the ONLY thing preventing the observable artifact +(the Fase-2 green-but-dead trap). Every test injects a canned ``http_get`` (no socket, no +credentials), so the suite runs with the network cable pulled. + +- NETWORK GATE (§8/SC1): http is refused fail-fast unless the per-run ``allow_network`` flag is + set — and refused BEFORE any source access. Asserted on BOTH branches with the SAME manifest and + SAME recording ``get``: flag off → ``IngestError`` + ``get`` NEVER called; flag on → succeeds + + ``get`` called. Removing the whole gate → the refuse assertion goes RED (get called, no error); + reverting to the *unconditional* refusal (dropping ``and not allow_network``) → the allow + assertion goes RED (raises when it should fetch). A refuse-only test would false-green on the + second detach. +- FENCED-NOT-TABLE (§5 http seam): the body is rendered verbatim inside a fenced code block, NOT a + markdown table — a raw ``|`` and ``\`` survive un-escaped. RED if the http path degrades to + ``render_table``. +- NAVIGABILITY (§2/§11): an http-generated bundle is consumable by the UNCHANGED + ``okf.navigate_bundle``/``bundle_context`` — reachable via index cross-links, classified by + ``okf_type``, provenance riding through. RED when index linking is detached. +- OVER-CAP WRITES NOTHING (§8/SC max_rows): an over-cap response raises during in-memory staging, + before any disk mutation — no ``ingest-*.md`` is written. RED if silent truncation replaces the + raise. +""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +import pytest + +from portfolio_optimiser import okf +from portfolio_optimiser.ingest import IngestError, materialize + +_INGESTED_AT = "2026-07-04T12:00:00Z" + + +def _extraction( + ext_id: str, query: str, okf_type: str, *, title: str, max_rows: int = 10 +) -> dict[str, Any]: + return { + "id": ext_id, + "title": title, + "query": query, + "okf_type": okf_type, + "max_rows": max_rows, + } + + +def _http_manifest( + tmp_path: Path, extractions: list[dict[str, Any]], *, credential_ref: str | None = None +) -> Path: + manifest = { + "manifest_version": 1, + "source": { + "type": "http", + "id": "api", + "base_url": "https://api.example.test/v1", + "credential_ref": credential_ref, + }, + "bundle_summary": "HTTP extracts from a local mock endpoint (http source type).", + "extractions": extractions, + } + path = tmp_path / "manifest.json" + path.write_text(json.dumps(manifest), encoding="utf-8") + return path + + +def _fixed_get(body: str) -> Any: + def get(url: str, credential: str | None) -> str: + return body + + return get + + +# --- LOAD-BEARING: network gate (both branches, same manifest + same get) ------------------------ + + +def test_http_network_gate_refuses_without_flag_and_fetches_with_flag(tmp_path: Path) -> None: + manifest = _http_manifest( + tmp_path, [_extraction("status", "status", "dataset", title="Service status")] + ) + calls: list[tuple[str, str | None]] = [] + + def recording_get(url: str, credential: str | None) -> str: + calls.append((url, credential)) + return "service ok" + + # Refuse branch: flag off → IngestError AND get NEVER called (refused before any source access). + with pytest.raises(IngestError): + materialize( + manifest, + tmp_path / "b_refuse", + ingested_at=_INGESTED_AT, + allow_network=False, + http_get=recording_get, + ) + assert calls == [] # §8: no network touched when the flag is absent + + # Allow branch: flag on → succeeds (writes ingest-status.md) AND get WAS called. + written = materialize( + manifest, + tmp_path / "b_allow", + ingested_at=_INGESTED_AT, + allow_network=True, + http_get=recording_get, + ) + assert any(p.name == "ingest-status.md" for p in written) + assert len(calls) == 1 # the injected transport was exercised + + +# --- LOAD-BEARING: fenced-not-table (verbatim body, no escaping) --------------------------------- + + +def test_http_body_is_fenced_verbatim_not_table_escaped(tmp_path: Path) -> None: + manifest = _http_manifest( + tmp_path, [_extraction("status", "status", "dataset", title="Service status")] + ) + payload = r'{"state": "degraded | partial", "path": "c:\temp\cache"}' + materialize( + manifest, + tmp_path / "b", + ingested_at=_INGESTED_AT, + allow_network=True, + http_get=_fixed_get(payload), + ) + body = (tmp_path / "b" / "ingest-status.md").read_text(encoding="utf-8") + assert "```" in body # rendered inside a fenced code block + assert payload in body # verbatim: raw | and \ survive + assert "\\|" not in body # RED if the http path degraded to render_table (pipe-escaped) + + +# --- LOAD-BEARING: navigability through unchanged OKF -------------------------------------------- + + +def test_http_generated_bundle_is_navigable_via_unchanged_okf(tmp_path: Path) -> None: + manifest = _http_manifest( + tmp_path, [_extraction("status", "status", "reference", title="Service status")] + ) + bundle_dir = tmp_path / "b" + materialize( + manifest, + bundle_dir, + ingested_at=_INGESTED_AT, + allow_network=True, + http_get=_fixed_get("service ok\nall systems green"), + ) + bundle = okf.navigate_bundle(str(bundle_dir)) + by_name = {f.name: f for f in bundle.files} + assert "ingest-status.md" in by_name + assert by_name["ingest-status.md"].type == "reference" + assert by_name["ingest-status.md"].frontmatter["ingest_manifest"] # provenance rides through + context = okf.bundle_context(bundle) + assert "Service status" in context and "service ok" in context # index + body reached + + +# --- LOAD-BEARING: over-cap writes nothing (memory-staging) -------------------------------------- + + +def test_http_over_cap_raises_and_writes_nothing(tmp_path: Path) -> None: + manifest = _http_manifest( + tmp_path, [_extraction("status", "status", "dataset", title="Service status", max_rows=2)] + ) + bundle_dir = tmp_path / "b" + with pytest.raises(IngestError): # §8: error, never silent truncation + materialize( + manifest, + bundle_dir, + ingested_at=_INGESTED_AT, + allow_network=True, + http_get=_fixed_get("a\nb\nc"), # 3 lines > max_rows=2 + ) + # Over-cap raises during in-memory staging, before the first disk mutation. + assert not list(bundle_dir.glob("ingest-*.md")) diff --git a/tests/test_ingest_materialize.py b/tests/test_ingest_materialize.py index 16c72e1..48fd6dc 100644 --- a/tests/test_ingest_materialize.py +++ b/tests/test_ingest_materialize.py @@ -242,10 +242,11 @@ def test_two_runs_with_identical_inputs_are_byte_identical(tmp_path: Path) -> No assert first == second # §10 idempotence at file level -def test_http_source_has_no_connector(tmp_path: Path) -> None: - # The polymorphic schema VALIDATES http (brief assumption 1); executing it is the gated - # I6 extension point — attempting to materialize is an explicit refusal, never a silent - # no-op. (sql now EXECUTES, I4; see tests/test_ingest_sql.py.) +def test_http_source_is_refused_without_network_flag(tmp_path: Path) -> None: + # The polymorphic schema VALIDATES http (brief assumption 1). Since I6 the http connector + # EXISTS, but materialize refuses it fail-fast unless the per-run allow_network flag is set — + # the manifest cannot grant itself network access (§8). Default (no flag) → explicit refusal, + # never a silent no-op. (The allow-branch is covered in tests/test_ingest_http_loadbearing.py.) manifest_path, bundle_dir = _project( tmp_path, source={"type": "http", "id": "api", "base_url": "https://host/api"} )