feat(ingest): gate http behind allow_network run-flag + dispatch (I6)

src/portfolio_optimiser/ingest.py

@@ -470,7 +470,12 @@ def _update_index_lines(
 
 
 def materialize(
-    manifest_path: str | Path, bundle_dir: str | Path, *, ingested_at: str
+    manifest_path: str | Path,
+    bundle_dir: str | Path,
+    *,
+    ingested_at: str,
+    allow_network: bool = False,
+    http_get: HttpGet | None = None,
 ) -> list[Path]:
     """Materialize a manifest's extractions into an OKF bundle (§5): the I2 invocation
     surface (a ``python -m`` CLI is deliberately deferred).
@@ -478,10 +483,15 @@ def materialize(
     Three explicit inputs — manifest, target bundle dir, and ``ingested_at`` (REQUIRED
     keyword, NO wall-clock default, stamped verbatim; mirrors the promotion gate's
     timestamp rule). Deterministic and offline: zero model calls, zero network for the
-    ``file`` source type. All extractions execute and render IN MEMORY before the first
+    ``file``/``sql`` source types. All extractions execute and render IN MEMORY before the
-    disk mutation (crash-window mitigation for the non-atomic §5 replace sequence;
+    first disk mutation (crash-window mitigation for the non-atomic §5 replace sequence;
     recovery = idempotent re-run, §10). Source calls are logged per §8 (which source,
-    when = the ``ingested_at`` argument, row count) — never cell contents, never secrets."""
+    when = the ``ingested_at`` argument, row count) — never cell contents, never secrets.
+
+    ``allow_network`` (I6, §8) is the per-run network opt-in: an ``http`` source is refused
+    fail-fast unless it is set — the manifest itself cannot grant network access (local-only
+    default, no silent egress). ``http_get`` optionally injects the transport seam (default
+    ``_urllib_get``, the only socket path); both are ignored for ``file``/``sql`` sources."""
     if not _INGESTED_AT_RE.match(ingested_at):
         raise ValueError(
             f"ingested_at must be ISO-8601 UTC with a Z suffix "
@@ -490,10 +500,14 @@ def materialize(
     manifest_file = Path(manifest_path)
     manifest, stamp = load_manifest(manifest_file)
     source = manifest.source
-    if isinstance(source, HttpSource):
+    # §8 network gate (I6): refuse http fail-fast BEFORE any source access unless the per-run
+    # flag is set. The `and not allow_network` clause is the load-bearing seam — the manifest
+    # cannot grant itself network access (local-only default, no silent egress).
+    if isinstance(source, HttpSource) and not allow_network:
         raise IngestError(
-            f"source type {source.type!r} is a gated extension point (I6) with no connector "
+            f"source type {source.type!r} requires the per-run network opt-in "
-            "in this implementation — the schema validates it, execution defers"
+            "(materialize(..., allow_network=True)) — the manifest cannot grant itself network "
+            "access (ingest spec §8, local-only default, no silent egress)"
         )
     # Pinned decision (file): a relative root resolves against the manifest file's directory —
     # never the process cwd, or the extraction would not be reproducible. (sql resolves its
@@ -508,17 +522,32 @@ def materialize(
     for extraction in manifest.extractions:
         if isinstance(source, FileSource):
             header, rows = read_csv(root, extraction.query, max_rows=extraction.max_rows)
-        else:  # SqlSource — http already refused above
+            body = render_table(header, rows)
+            row_count = len(rows)
+        elif isinstance(source, HttpSource):
+            # The gate above guarantees allow_network here; _urllib_get is the only socket path.
+            get = http_get if http_get is not None else _urllib_get
+            text = read_http(
+                source.base_url,
+                extraction.query,
+                max_rows=extraction.max_rows,
+                credential_ref=source.credential_ref,
+                get=get,
+            )
+            body = _render_fenced_block(text)  # verbatim fenced block, NOT table-escaped
+            row_count = len(text.splitlines())
+        else:  # SqlSource
             header, rows = read_sql(
                 source.connection_ref, extraction.query, max_rows=extraction.max_rows
             )
+            body = render_table(header, rows)
+            row_count = len(rows)
         _LOGGER.info(
             "source call: source=%s ingested_at=%s rows=%d",
             source.id,
             ingested_at,
-            len(rows),
+            row_count,
         )
-        body = render_table(header, rows)
         content = _render_concept_file(
             manifest, extraction, body, ingested_at=ingested_at, stamp=stamp
         )

tests/test_ingest_http_loadbearing.py

@@ -0,0 +1,175 @@
+"""I6 load-bearing seams — the ``http`` path end-to-end (ingest spec §11's detach-RED regime).
+
+Each test is built so the seam under test is the ONLY thing preventing the observable artifact
+(the Fase-2 green-but-dead trap). Every test injects a canned ``http_get`` (no socket, no
+credentials), so the suite runs with the network cable pulled.
+
+- NETWORK GATE (§8/SC1): http is refused fail-fast unless the per-run ``allow_network`` flag is
+  set — and refused BEFORE any source access. Asserted on BOTH branches with the SAME manifest and
+  SAME recording ``get``: flag off → ``IngestError`` + ``get`` NEVER called; flag on → succeeds +
+  ``get`` called. Removing the whole gate → the refuse assertion goes RED (get called, no error);
+  reverting to the *unconditional* refusal (dropping ``and not allow_network``) → the allow
+  assertion goes RED (raises when it should fetch). A refuse-only test would false-green on the
+  second detach.
+- FENCED-NOT-TABLE (§5 http seam): the body is rendered verbatim inside a fenced code block, NOT a
+  markdown table — a raw ``|`` and ``\`` survive un-escaped. RED if the http path degrades to
+  ``render_table``.
+- NAVIGABILITY (§2/§11): an http-generated bundle is consumable by the UNCHANGED
+  ``okf.navigate_bundle``/``bundle_context`` — reachable via index cross-links, classified by
+  ``okf_type``, provenance riding through. RED when index linking is detached.
+- OVER-CAP WRITES NOTHING (§8/SC max_rows): an over-cap response raises during in-memory staging,
+  before any disk mutation — no ``ingest-*.md`` is written. RED if silent truncation replaces the
+  raise.
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any
+
+import pytest
+
+from portfolio_optimiser import okf
+from portfolio_optimiser.ingest import IngestError, materialize
+
+_INGESTED_AT = "2026-07-04T12:00:00Z"
+
+
+def _extraction(
+    ext_id: str, query: str, okf_type: str, *, title: str, max_rows: int = 10
+) -> dict[str, Any]:
+    return {
+        "id": ext_id,
+        "title": title,
+        "query": query,
+        "okf_type": okf_type,
+        "max_rows": max_rows,
+    }
+
+
+def _http_manifest(
+    tmp_path: Path, extractions: list[dict[str, Any]], *, credential_ref: str | None = None
+) -> Path:
+    manifest = {
+        "manifest_version": 1,
+        "source": {
+            "type": "http",
+            "id": "api",
+            "base_url": "https://api.example.test/v1",
+            "credential_ref": credential_ref,
+        },
+        "bundle_summary": "HTTP extracts from a local mock endpoint (http source type).",
+        "extractions": extractions,
+    }
+    path = tmp_path / "manifest.json"
+    path.write_text(json.dumps(manifest), encoding="utf-8")
+    return path
+
+
+def _fixed_get(body: str) -> Any:
+    def get(url: str, credential: str | None) -> str:
+        return body
+
+    return get
+
+
+# --- LOAD-BEARING: network gate (both branches, same manifest + same get) ------------------------
+
+
+def test_http_network_gate_refuses_without_flag_and_fetches_with_flag(tmp_path: Path) -> None:
+    manifest = _http_manifest(
+        tmp_path, [_extraction("status", "status", "dataset", title="Service status")]
+    )
+    calls: list[tuple[str, str | None]] = []
+
+    def recording_get(url: str, credential: str | None) -> str:
+        calls.append((url, credential))
+        return "service ok"
+
+    # Refuse branch: flag off → IngestError AND get NEVER called (refused before any source access).
+    with pytest.raises(IngestError):
+        materialize(
+            manifest,
+            tmp_path / "b_refuse",
+            ingested_at=_INGESTED_AT,
+            allow_network=False,
+            http_get=recording_get,
+        )
+    assert calls == []  # §8: no network touched when the flag is absent
+
+    # Allow branch: flag on → succeeds (writes ingest-status.md) AND get WAS called.
+    written = materialize(
+        manifest,
+        tmp_path / "b_allow",
+        ingested_at=_INGESTED_AT,
+        allow_network=True,
+        http_get=recording_get,
+    )
+    assert any(p.name == "ingest-status.md" for p in written)
+    assert len(calls) == 1  # the injected transport was exercised
+
+
+# --- LOAD-BEARING: fenced-not-table (verbatim body, no escaping) ---------------------------------
+
+
+def test_http_body_is_fenced_verbatim_not_table_escaped(tmp_path: Path) -> None:
+    manifest = _http_manifest(
+        tmp_path, [_extraction("status", "status", "dataset", title="Service status")]
+    )
+    payload = r'{"state": "degraded | partial", "path": "c:\temp\cache"}'
+    materialize(
+        manifest,
+        tmp_path / "b",
+        ingested_at=_INGESTED_AT,
+        allow_network=True,
+        http_get=_fixed_get(payload),
+    )
+    body = (tmp_path / "b" / "ingest-status.md").read_text(encoding="utf-8")
+    assert "```" in body  # rendered inside a fenced code block
+    assert payload in body  # verbatim: raw | and \ survive
+    assert "\\|" not in body  # RED if the http path degraded to render_table (pipe-escaped)
+
+
+# --- LOAD-BEARING: navigability through unchanged OKF --------------------------------------------
+
+
+def test_http_generated_bundle_is_navigable_via_unchanged_okf(tmp_path: Path) -> None:
+    manifest = _http_manifest(
+        tmp_path, [_extraction("status", "status", "reference", title="Service status")]
+    )
+    bundle_dir = tmp_path / "b"
+    materialize(
+        manifest,
+        bundle_dir,
+        ingested_at=_INGESTED_AT,
+        allow_network=True,
+        http_get=_fixed_get("service ok\nall systems green"),
+    )
+    bundle = okf.navigate_bundle(str(bundle_dir))
+    by_name = {f.name: f for f in bundle.files}
+    assert "ingest-status.md" in by_name
+    assert by_name["ingest-status.md"].type == "reference"
+    assert by_name["ingest-status.md"].frontmatter["ingest_manifest"]  # provenance rides through
+    context = okf.bundle_context(bundle)
+    assert "Service status" in context and "service ok" in context  # index + body reached
+
+
+# --- LOAD-BEARING: over-cap writes nothing (memory-staging) --------------------------------------
+
+
+def test_http_over_cap_raises_and_writes_nothing(tmp_path: Path) -> None:
+    manifest = _http_manifest(
+        tmp_path, [_extraction("status", "status", "dataset", title="Service status", max_rows=2)]
+    )
+    bundle_dir = tmp_path / "b"
+    with pytest.raises(IngestError):  # §8: error, never silent truncation
+        materialize(
+            manifest,
+            bundle_dir,
+            ingested_at=_INGESTED_AT,
+            allow_network=True,
+            http_get=_fixed_get("a\nb\nc"),  # 3 lines > max_rows=2
+        )
+    # Over-cap raises during in-memory staging, before the first disk mutation.
+    assert not list(bundle_dir.glob("ingest-*.md"))

tests/test_ingest_materialize.py

@@ -242,10 +242,11 @@ def test_two_runs_with_identical_inputs_are_byte_identical(tmp_path: Path) -> No
     assert first == second  # §10 idempotence at file level
 
 
-def test_http_source_has_no_connector(tmp_path: Path) -> None:
+def test_http_source_is_refused_without_network_flag(tmp_path: Path) -> None:
-    # The polymorphic schema VALIDATES http (brief assumption 1); executing it is the gated
+    # The polymorphic schema VALIDATES http (brief assumption 1). Since I6 the http connector
-    # I6 extension point — attempting to materialize is an explicit refusal, never a silent
+    # EXISTS, but materialize refuses it fail-fast unless the per-run allow_network flag is set —
-    # no-op. (sql now EXECUTES, I4; see tests/test_ingest_sql.py.)
+    # the manifest cannot grant itself network access (§8). Default (no flag) → explicit refusal,
+    # never a silent no-op. (The allow-branch is covered in tests/test_ingest_http_loadbearing.py.)
     manifest_path, bundle_dir = _project(
         tmp_path, source={"type": "http", "id": "api", "base_url": "https://host/api"}
     )