From 58dda468dc7d7eb58f71c0db522eed3392a00608 Mon Sep 17 00:00:00 2001 From: Kjell Tore Guttormsen Date: Sat, 4 Jul 2026 16:57:29 +0200 Subject: [PATCH] feat(ingest): http read_http connector + injectable transport seam (I6) --- src/portfolio_optimiser/ingest.py | 101 +++++++++++++++++++++++++++- tests/test_ingest_http.py | 108 ++++++++++++++++++++++++++++++ 2 files changed, 208 insertions(+), 1 deletion(-) create mode 100644 tests/test_ingest_http.py diff --git a/src/portfolio_optimiser/ingest.py b/src/portfolio_optimiser/ingest.py index a6411e2..f566293 100644 --- a/src/portfolio_optimiser/ingest.py +++ b/src/portfolio_optimiser/ingest.py @@ -27,7 +27,15 @@ are escaped identically to data cells; empty/ragged CSV and missing ``root``/``q fast as ``IngestError``. For ``sql`` (I4): ``connection_ref`` names an env var whose value is a filesystem path to a sqlite database opened read-only; the §5 numeric rules bite the typed values (``_sql_value_to_text``) — INTEGER→plain decimal, REAL→``repr`` (shortest round-trip), -TEXT verbatim, SQL NULL→empty string, BLOB/other→``IngestError`` (never silent coercion). +TEXT verbatim, SQL NULL→empty string, BLOB/other→``IngestError`` (never silent coercion). For +``http`` (I6): transport is an injectable ``get`` callable (default ``_urllib_get``, the only +socket path); ``credential_ref`` names an env secret resolved at run time (Bearer), never from the +manifest, never logged, never stamped in frontmatter; the URL is ``base_url`` (one trailing slash +stripped) + ``/`` + ``query`` (leading slash stripped); the body is decoded UTF-8, CRLF→LF, and +rendered verbatim inside a fenced code block (NOT table-escaped); ``max_rows`` caps the LF-only +line count (newline count, not ``str.splitlines()``), fail-fast; a response line beginning with a +code fence → ``IngestError`` (cannot be safely fenced — never silent corruption). Network is a +per-run ``materialize`` argument (``allow_network``); the manifest cannot grant network (§8). """ from __future__ import annotations @@ -39,10 +47,13 @@ import logging import os import re import sqlite3 +from collections.abc import Callable from contextlib import closing from pathlib import Path from typing import Literal +from urllib.error import URLError from urllib.parse import quote, urlsplit +from urllib.request import Request, urlopen from pydantic import BaseModel, Field, field_validator, model_validator @@ -278,6 +289,94 @@ def read_sql( return header, rows +# --- http source (I6): injectable transport seam + verbatim fenced body -------------------------- + +#: The http transport seam: ``(url, credential) -> response body text``. Injecting a canned +#: implementation in tests keeps the suite socket-free; the default ``_urllib_get`` is the ONLY +#: path that opens a socket, and it is reached ONLY inside the gated http branch of materialize. +HttpGet = Callable[[str, str | None], str] + + +def _urllib_get(url: str, credential: str | None) -> str: + """The stdlib GET — the ONLY socket path in this module (I6). + + Builds a ``GET`` request, attaches ``Authorization: Bearer {credential}`` iff a credential was + resolved (never otherwise), reads the body and decodes it UTF-8. Transport / decode failures + wrap as ``IngestError`` WITHOUT the secret — the credential rides in the header, never in + ``url`` (base_url embedded credentials are already rejected at schema validation).""" + headers = {"Authorization": f"Bearer {credential}"} if credential is not None else {} + request = Request(url, headers=headers, method="GET") + try: + with urlopen(request) as response: + raw = response.read() + return raw.decode("utf-8") + except (URLError, OSError, UnicodeDecodeError) as exc: + raise IngestError(f"http GET failed for {url!r}: {exc}") from exc + + +def read_http( + base_url: str, + query: str, + *, + max_rows: int, + credential_ref: str | None = None, + get: HttpGet = _urllib_get, +) -> str: + """Execute an ``http``-source extraction: GET ``base_url`` joined with ``query`` via the + injectable ``get`` seam (default stdlib ``_urllib_get``), returning the response body for + verbatim fenced rendering (§4/§5, I6). + + Pinned I6 decisions (spec-silent, mirroring the ``sql`` template): ``credential_ref`` names an + environment variable holding the secret, resolved at run time (present-but-unset → fail-fast, + like ``read_sql``; ``None`` → no auth) — never read from the manifest, never logged, never + stamped in frontmatter. URL = ``base_url`` with one trailing slash stripped + ``/`` + ``query`` + with leading slashes stripped (explicit join, not ``urljoin`` — predictable path semantics). + The body is decoded UTF-8 (in ``get``), CRLF→LF normalized, and capped on its LF-only line + count — the newline count (+1 for a missing trailing newline), NOT ``str.splitlines()`` (which + also splits on the extended Unicode set and would over-count vs the LF-only rendered file). + Exceeding ``max_rows`` is an ERROR, never a truncation (§8). A body line beginning with a code + fence (after stripping up to 3 leading spaces — a CommonMark closing fence may be indented) → + ``IngestError``: a body that cannot be safely embedded in a fenced code block is an error, + never silently-corrupted markdown. Pure: no writes, no logging (materialization owns the §8 + log).""" + if credential_ref is None: + credential: str | None = None + else: + credential = os.environ.get(credential_ref) + if not credential: + raise IngestError( + f"http source credential_ref {credential_ref!r} is not set in the environment " + "(credentials resolve at run time, never from the manifest — §4)" + ) + url = base_url.rstrip("/") + "/" + query.lstrip("/") + normalized = get(url, credential).replace("\r\n", "\n").replace("\r", "\n") + for line in normalized.split("\n"): + if line.lstrip(" ").startswith("```"): + raise IngestError( + f"http extraction {query!r} contains a line that is a code-fence marker " + "(```) — cannot be safely embedded in a fenced code block; fail-fast, never " + "silent corruption (ingest spec §8)" + ) + newline_count = normalized.count("\n") + line_count = ( + newline_count if (normalized == "" or normalized.endswith("\n")) else newline_count + 1 + ) + if line_count > max_rows: + raise IngestError( + f"http extraction {query!r} exceeds max_rows={max_rows} (error, never silent " + "truncation — ingest spec §8)" + ) + return normalized + + +def _render_fenced_block(body: str) -> str: + """Render an http response body verbatim inside a fenced code block (§5, I6): LF-only, exactly + one trailing newline, the body's own trailing newlines stripped so the closing fence sits + flush. NOT table-escaped — pipe/backslash survive verbatim (the discriminator vs + ``render_table``).""" + return f"```\n{body.rstrip(chr(10))}\n```\n" + + def _escape_cell(value: str) -> str: # §5 escape order is load-bearing: `\` FIRST (else the backslash introduced by # pipe-escaping gets double-escaped), then `|`, then newlines → single space with diff --git a/tests/test_ingest_http.py b/tests/test_ingest_http.py new file mode 100644 index 0000000..b1c4d6b --- /dev/null +++ b/tests/test_ingest_http.py @@ -0,0 +1,108 @@ +"""I6 Step 1 — the ``http`` connector (``read_http``) + injectable transport seam. + +Mirrors tests/test_ingest_sql.py for the third source type. Covers the §4 connector rules for +``http`` (URL join, env-resolved ``credential_ref``, streaming ``max_rows`` cap §8) and the §5 +verbatim-body rule the ``file``/``sql`` paths never exercise (body rendered inside a fenced code +block, NOT a markdown table). Every test injects a canned ``get`` callable — NO socket, NO +credentials unless monkeypatched — so the suite runs with the network cable pulled. The stdlib +``_urllib_get`` is the ONLY socket path and is never exercised here (asserted as the default). + +Pinned http §4/§5 decisions (spec-silent, delegated to I6 by the frozen spec; see plan): +transport is an injectable ``get`` (default stdlib ``urllib``); ``credential_ref`` names an env +secret resolved at run time (Bearer), never from the manifest; URL = ``base_url`` (one trailing +slash stripped) + ``/`` + ``query`` (leading slash stripped); body is decoded UTF-8, CRLF→LF, and +capped on the LF-only line count (NOT ``str.splitlines()``); a body line beginning ``` ``` ``` → +``IngestError`` fail-fast (cannot be safely fenced — never silent corruption). +""" + +from __future__ import annotations + +import inspect + +import pytest + +from portfolio_optimiser.ingest import ( + HttpGet, + IngestError, + _urllib_get, + read_http, +) + + +def _make_get(body: str, *, recorder: list[tuple[str, str | None]] | None = None) -> HttpGet: + """A canned transport seam: records ``(url, credential)`` and returns a fixed body — no + socket. Local to this file (mirrors ``_make_db`` locality; never conftest, which is + MAF/LLM-only).""" + + def get(url: str, credential: str | None) -> str: + if recorder is not None: + recorder.append((url, credential)) + return body + + return get + + +# --- connector: URL join + verbatim fetch (§4/§5) ------------------------------------------------ + + +def test_read_http_returns_body_verbatim_and_joins_url() -> None: + calls: list[tuple[str, str | None]] = [] + get = _make_get("line one\nline two", recorder=calls) + body = read_http("https://api.example.test/v1/", "status", max_rows=10, get=get) + assert body == "line one\nline two" # verbatim, no escaping + # one trailing slash stripped from base_url, leading slash stripped from query + assert calls == [("https://api.example.test/v1/status", None)] + + +# --- connector: §8 line-count cap (LF-only, boundary) -------------------------------------------- + + +def test_read_http_at_max_rows_boundary_passes() -> None: + get = _make_get("a\nb\nc") # exactly 3 lines + assert read_http("https://h.test", "q", max_rows=3, get=get) == "a\nb\nc" + + +def test_read_http_over_max_rows_is_error_never_truncated() -> None: + get = _make_get("a\nb\nc\nd") # 4 lines > max_rows + with pytest.raises(IngestError): # §8: error, never silent truncation + read_http("https://h.test", "q", max_rows=3, get=get) + + +# --- connector: fence-collision fail-fast (§8, no silent corruption) ------------------------------ + + +def test_read_http_fence_collision_is_error() -> None: + # A body line that is itself a code-fence marker cannot be safely embedded in a fenced block. + get = _make_get("safe line\n```\nrest of body") + with pytest.raises(IngestError): + read_http("https://h.test", "q", max_rows=10, get=get) + + +# --- connector: credential resolves from env, NEVER the manifest (§4) ---------------------------- + + +def test_read_http_unset_credential_ref_fails(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("API_TOKEN", raising=False) + get = _make_get("body") + with pytest.raises(IngestError): # present-but-unset → fail-fast (mirrors read_sql) + read_http("https://h.test", "q", max_rows=10, credential_ref="API_TOKEN", get=get) + + +def test_read_http_resolves_credential_from_env_not_manifest( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("API_TOKEN", "s3cret") + calls: list[tuple[str, str | None]] = [] + get = _make_get("body", recorder=calls) + read_http("https://h.test", "q", max_rows=10, credential_ref="API_TOKEN", get=get) + assert calls == [("https://h.test/q", "s3cret")] # secret comes from env, never the manifest + + +# --- transport seam: the stdlib default is the only socket path ---------------------------------- + + +def test_read_http_default_get_is_urllib() -> None: + # inspect.signature proves the stdlib default WITHOUT opening a socket — _urllib_get is + # the single socket path and is never called by the offline suite. + sig = inspect.signature(read_http) + assert sig.parameters["get"].default is _urllib_get