feat(ingest): http read_http connector + injectable transport seam (I6)

src/portfolio_optimiser/ingest.py

@@ -27,7 +27,15 @@ are escaped identically to data cells; empty/ragged CSV and missing ``root``/``q
 fast as ``IngestError``. For ``sql`` (I4): ``connection_ref`` names an env var whose value is a
 filesystem path to a sqlite database opened read-only; the §5 numeric rules bite the typed
 values (``_sql_value_to_text``) — INTEGER→plain decimal, REAL→``repr`` (shortest round-trip),
-TEXT verbatim, SQL NULL→empty string, BLOB/other→``IngestError`` (never silent coercion).
+TEXT verbatim, SQL NULL→empty string, BLOB/other→``IngestError`` (never silent coercion). For
+``http`` (I6): transport is an injectable ``get`` callable (default ``_urllib_get``, the only
+socket path); ``credential_ref`` names an env secret resolved at run time (Bearer), never from the
+manifest, never logged, never stamped in frontmatter; the URL is ``base_url`` (one trailing slash
+stripped) + ``/`` + ``query`` (leading slash stripped); the body is decoded UTF-8, CRLF→LF, and
+rendered verbatim inside a fenced code block (NOT table-escaped); ``max_rows`` caps the LF-only
+line count (newline count, not ``str.splitlines()``), fail-fast; a response line beginning with a
+code fence → ``IngestError`` (cannot be safely fenced — never silent corruption). Network is a
+per-run ``materialize`` argument (``allow_network``); the manifest cannot grant network (§8).
 """
 
 from __future__ import annotations
@@ -39,10 +47,13 @@ import logging
 import os
 import re
 import sqlite3
+from collections.abc import Callable
 from contextlib import closing
 from pathlib import Path
 from typing import Literal
+from urllib.error import URLError
 from urllib.parse import quote, urlsplit
+from urllib.request import Request, urlopen
 
 from pydantic import BaseModel, Field, field_validator, model_validator
 
@@ -278,6 +289,94 @@ def read_sql(
     return header, rows
 
 
+# --- http source (I6): injectable transport seam + verbatim fenced body --------------------------
+
+#: The http transport seam: ``(url, credential) -> response body text``. Injecting a canned
+#: implementation in tests keeps the suite socket-free; the default ``_urllib_get`` is the ONLY
+#: path that opens a socket, and it is reached ONLY inside the gated http branch of materialize.
+HttpGet = Callable[[str, str | None], str]
+
+
+def _urllib_get(url: str, credential: str | None) -> str:
+    """The stdlib GET — the ONLY socket path in this module (I6).
+
+    Builds a ``GET`` request, attaches ``Authorization: Bearer {credential}`` iff a credential was
+    resolved (never otherwise), reads the body and decodes it UTF-8. Transport / decode failures
+    wrap as ``IngestError`` WITHOUT the secret — the credential rides in the header, never in
+    ``url`` (base_url embedded credentials are already rejected at schema validation)."""
+    headers = {"Authorization": f"Bearer {credential}"} if credential is not None else {}
+    request = Request(url, headers=headers, method="GET")
+    try:
+        with urlopen(request) as response:
+            raw = response.read()
+        return raw.decode("utf-8")
+    except (URLError, OSError, UnicodeDecodeError) as exc:
+        raise IngestError(f"http GET failed for {url!r}: {exc}") from exc
+
+
+def read_http(
+    base_url: str,
+    query: str,
+    *,
+    max_rows: int,
+    credential_ref: str | None = None,
+    get: HttpGet = _urllib_get,
+) -> str:
+    """Execute an ``http``-source extraction: GET ``base_url`` joined with ``query`` via the
+    injectable ``get`` seam (default stdlib ``_urllib_get``), returning the response body for
+    verbatim fenced rendering (§4/§5, I6).
+
+    Pinned I6 decisions (spec-silent, mirroring the ``sql`` template): ``credential_ref`` names an
+    environment variable holding the secret, resolved at run time (present-but-unset → fail-fast,
+    like ``read_sql``; ``None`` → no auth) — never read from the manifest, never logged, never
+    stamped in frontmatter. URL = ``base_url`` with one trailing slash stripped + ``/`` + ``query``
+    with leading slashes stripped (explicit join, not ``urljoin`` — predictable path semantics).
+    The body is decoded UTF-8 (in ``get``), CRLF→LF normalized, and capped on its LF-only line
+    count — the newline count (+1 for a missing trailing newline), NOT ``str.splitlines()`` (which
+    also splits on the extended Unicode set and would over-count vs the LF-only rendered file).
+    Exceeding ``max_rows`` is an ERROR, never a truncation (§8). A body line beginning with a code
+    fence (after stripping up to 3 leading spaces — a CommonMark closing fence may be indented) →
+    ``IngestError``: a body that cannot be safely embedded in a fenced code block is an error,
+    never silently-corrupted markdown. Pure: no writes, no logging (materialization owns the §8
+    log)."""
+    if credential_ref is None:
+        credential: str | None = None
+    else:
+        credential = os.environ.get(credential_ref)
+        if not credential:
+            raise IngestError(
+                f"http source credential_ref {credential_ref!r} is not set in the environment "
+                "(credentials resolve at run time, never from the manifest — §4)"
+            )
+    url = base_url.rstrip("/") + "/" + query.lstrip("/")
+    normalized = get(url, credential).replace("\r\n", "\n").replace("\r", "\n")
+    for line in normalized.split("\n"):
+        if line.lstrip(" ").startswith("```"):
+            raise IngestError(
+                f"http extraction {query!r} contains a line that is a code-fence marker "
+                "(```) — cannot be safely embedded in a fenced code block; fail-fast, never "
+                "silent corruption (ingest spec §8)"
+            )
+    newline_count = normalized.count("\n")
+    line_count = (
+        newline_count if (normalized == "" or normalized.endswith("\n")) else newline_count + 1
+    )
+    if line_count > max_rows:
+        raise IngestError(
+            f"http extraction {query!r} exceeds max_rows={max_rows} (error, never silent "
+            "truncation — ingest spec §8)"
+        )
+    return normalized
+
+
+def _render_fenced_block(body: str) -> str:
+    """Render an http response body verbatim inside a fenced code block (§5, I6): LF-only, exactly
+    one trailing newline, the body's own trailing newlines stripped so the closing fence sits
+    flush. NOT table-escaped — pipe/backslash survive verbatim (the discriminator vs
+    ``render_table``)."""
+    return f"```\n{body.rstrip(chr(10))}\n```\n"
+
+
 def _escape_cell(value: str) -> str:
     # §5 escape order is load-bearing: `\` FIRST (else the backslash introduced by
     # pipe-escaping gets double-escaped), then `|`, then newlines → single space with