open/portfolio-optimiser
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity Actions

feat(ingest): http read_http connector + injectable transport seam (I6)

Browse source
This commit is contained in:
Kjell Tore Guttormsen 2026-07-04 16:57:29 +02:00
commit 58dda468dc
2 changed files with 208 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

101
src/portfolio_optimiser/ingest.py
View file

@ -27,7 +27,15 @@ are escaped identically to data cells; empty/ragged CSV and missing ``root``/``q
fast as ``IngestError``. For ``sql`` (I4): ``connection_ref`` names an env var whose value is a
filesystem path to a sqlite database opened read-only; the §5 numeric rules bite the typed
values (``_sql_value_to_text``) — INTEGER→plain decimal, REAL→``repr`` (shortest round-trip),
TEXT verbatim, SQL NULL→empty string, BLOB/other→``IngestError`` (never silent coercion).
TEXT verbatim, SQL NULL→empty string, BLOB/other→``IngestError`` (never silent coercion). For
``http`` (I6): transport is an injectable ``get`` callable (default ``_urllib_get``, the only
socket path); ``credential_ref`` names an env secret resolved at run time (Bearer), never from the
manifest, never logged, never stamped in frontmatter; the URL is ``base_url`` (one trailing slash
stripped) + ``/`` + ``query`` (leading slash stripped); the body is decoded UTF-8, CRLF→LF, and
rendered verbatim inside a fenced code block (NOT table-escaped); ``max_rows`` caps the LF-only
line count (newline count, not ``str.splitlines()``), fail-fast; a response line beginning with a
code fence → ``IngestError`` (cannot be safely fenced — never silent corruption). Network is a
per-run ``materialize`` argument (``allow_network``); the manifest cannot grant network (§8).
"""
from __future__ import annotations
@ -39,10 +47,13 @@ import logging
import os
import re
import sqlite3
from collections.abc import Callable
from contextlib import closing
from pathlib import Path
from typing import Literal
from urllib.error import URLError
from urllib.parse import quote, urlsplit
from urllib.request import Request, urlopen
from pydantic import BaseModel, Field, field_validator, model_validator
@ -278,6 +289,94 @@ def read_sql(
return header, rows
# --- http source (I6): injectable transport seam + verbatim fenced body --------------------------
#: The http transport seam: ``(url, credential) -> response body text``. Injecting a canned
#: implementation in tests keeps the suite socket-free; the default ``_urllib_get`` is the ONLY
#: path that opens a socket, and it is reached ONLY inside the gated http branch of materialize.
HttpGet = Callable[[str, str | None], str]
def _urllib_get(url: str, credential: str | None) -> str:
"""The stdlib GET — the ONLY socket path in this module (I6).
Builds a ``GET`` request, attaches ``Authorization: Bearer {credential}`` iff a credential was
resolved (never otherwise), reads the body and decodes it UTF-8. Transport / decode failures
wrap as ``IngestError`` WITHOUT the secret — the credential rides in the header, never in
``url`` (base_url embedded credentials are already rejected at schema validation)."""
headers = {"Authorization": f"Bearer {credential}"} if credential is not None else {}
request = Request(url, headers=headers, method="GET")
try:
with urlopen(request) as response:
raw = response.read()
return raw.decode("utf-8")
except (URLError, OSError, UnicodeDecodeError) as exc:
raise IngestError(f"http GET failed for {url!r}: {exc}") from exc
def read_http(
base_url: str,
query: str,
*,
max_rows: int,
credential_ref: str | None = None,
get: HttpGet = _urllib_get,
) -> str:
"""Execute an ``http``-source extraction: GET ``base_url`` joined with ``query`` via the
injectable ``get`` seam (default stdlib ``_urllib_get``), returning the response body for
verbatim fenced rendering (§4/§5, I6).
Pinned I6 decisions (spec-silent, mirroring the ``sql`` template): ``credential_ref`` names an
environment variable holding the secret, resolved at run time (present-but-unset → fail-fast,
like ``read_sql``; ``None`` → no auth) — never read from the manifest, never logged, never
stamped in frontmatter. URL = ``base_url`` with one trailing slash stripped + ``/`` + ``query``
with leading slashes stripped (explicit join, not ``urljoin`` — predictable path semantics).
The body is decoded UTF-8 (in ``get``), CRLF→LF normalized, and capped on its LF-only line
count — the newline count (+1 for a missing trailing newline), NOT ``str.splitlines()`` (which
also splits on the extended Unicode set and would over-count vs the LF-only rendered file).
Exceeding ``max_rows`` is an ERROR, never a truncation (§8). A body line beginning with a code
fence (after stripping up to 3 leading spaces — a CommonMark closing fence may be indented) →
``IngestError``: a body that cannot be safely embedded in a fenced code block is an error,
never silently-corrupted markdown. Pure: no writes, no logging (materialization owns the §8
log)."""
if credential_ref is None:
credential: str | None = None
else:
credential = os.environ.get(credential_ref)
if not credential:
raise IngestError(
f"http source credential_ref {credential_ref!r} is not set in the environment "
"(credentials resolve at run time, never from the manifest — §4)"
)
url = base_url.rstrip("/") + "/" + query.lstrip("/")
normalized = get(url, credential).replace("\r\n", "\n").replace("\r", "\n")
for line in normalized.split("\n"):
if line.lstrip(" ").startswith("```"):
raise IngestError(
f"http extraction {query!r} contains a line that is a code-fence marker "
"(```) — cannot be safely embedded in a fenced code block; fail-fast, never "
"silent corruption (ingest spec §8)"
)
newline_count = normalized.count("\n")
line_count = (
newline_count if (normalized == "" or normalized.endswith("\n")) else newline_count + 1
)
if line_count > max_rows:
raise IngestError(
f"http extraction {query!r} exceeds max_rows={max_rows} (error, never silent "
"truncation — ingest spec §8)"
)
return normalized
def _render_fenced_block(body: str) -> str:
"""Render an http response body verbatim inside a fenced code block (§5, I6): LF-only, exactly
one trailing newline, the body's own trailing newlines stripped so the closing fence sits
flush. NOT table-escaped — pipe/backslash survive verbatim (the discriminator vs
``render_table``)."""
return f"```\n{body.rstrip(chr(10))}\n```\n"
def _escape_cell(value: str) -> str:
# §5 escape order is load-bearing: `\` FIRST (else the backslash introduced by
# pipe-escaping gets double-escaped), then `|`, then newlines → single space with

108
tests/test_ingest_http.py Normal file
View file

@ -0,0 +1,108 @@
"""I6 Step 1 — the ``http`` connector (``read_http``) + injectable transport seam.
Mirrors tests/test_ingest_sql.py for the third source type. Covers the §4 connector rules for
``http`` (URL join, env-resolved ``credential_ref``, streaming ``max_rows`` cap §8) and the §5
verbatim-body rule the ``file``/``sql`` paths never exercise (body rendered inside a fenced code
block, NOT a markdown table). Every test injects a canned ``get`` callable — NO socket, NO
credentials unless monkeypatched — so the suite runs with the network cable pulled. The stdlib
``_urllib_get`` is the ONLY socket path and is never exercised here (asserted as the default).
Pinned http §4/§5 decisions (spec-silent, delegated to I6 by the frozen spec; see plan):
transport is an injectable ``get`` (default stdlib ``urllib``); ``credential_ref`` names an env
secret resolved at run time (Bearer), never from the manifest; URL = ``base_url`` (one trailing
slash stripped) + ``/`` + ``query`` (leading slash stripped); body is decoded UTF-8, CRLF→LF, and
capped on the LF-only line count (NOT ``str.splitlines()``); a body line beginning ``` ``` ``` →
``IngestError`` fail-fast (cannot be safely fenced — never silent corruption).
"""
from __future__ import annotations
import inspect
import pytest
from portfolio_optimiser.ingest import (
HttpGet,
IngestError,
_urllib_get,
read_http,
)
def _make_get(body: str, *, recorder: list[tuple[str, str | None]] | None = None) -> HttpGet:
"""A canned transport seam: records ``(url, credential)`` and returns a fixed body — no
socket. Local to this file (mirrors ``_make_db`` locality; never conftest, which is
MAF/LLM-only)."""
def get(url: str, credential: str | None) -> str:
if recorder is not None:
recorder.append((url, credential))
return body
return get
# --- connector: URL join + verbatim fetch (§4/§5) ------------------------------------------------
def test_read_http_returns_body_verbatim_and_joins_url() -> None:
calls: list[tuple[str, str | None]] = []
get = _make_get("line one\nline two", recorder=calls)
body = read_http("https://api.example.test/v1/", "status", max_rows=10, get=get)
assert body == "line one\nline two" # verbatim, no escaping
# one trailing slash stripped from base_url, leading slash stripped from query
assert calls == [("https://api.example.test/v1/status", None)]
# --- connector: §8 line-count cap (LF-only, boundary) --------------------------------------------
def test_read_http_at_max_rows_boundary_passes() -> None:
get = _make_get("a\nb\nc") # exactly 3 lines
assert read_http("https://h.test", "q", max_rows=3, get=get) == "a\nb\nc"
def test_read_http_over_max_rows_is_error_never_truncated() -> None:
get = _make_get("a\nb\nc\nd") # 4 lines > max_rows
with pytest.raises(IngestError): # §8: error, never silent truncation
read_http("https://h.test", "q", max_rows=3, get=get)
# --- connector: fence-collision fail-fast (§8, no silent corruption) ------------------------------
def test_read_http_fence_collision_is_error() -> None:
# A body line that is itself a code-fence marker cannot be safely embedded in a fenced block.
get = _make_get("safe line\n```\nrest of body")
with pytest.raises(IngestError):
read_http("https://h.test", "q", max_rows=10, get=get)
# --- connector: credential resolves from env, NEVER the manifest (§4) ----------------------------
def test_read_http_unset_credential_ref_fails(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.delenv("API_TOKEN", raising=False)
get = _make_get("body")
with pytest.raises(IngestError): # present-but-unset → fail-fast (mirrors read_sql)
read_http("https://h.test", "q", max_rows=10, credential_ref="API_TOKEN", get=get)
def test_read_http_resolves_credential_from_env_not_manifest(
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setenv("API_TOKEN", "s3cret")
calls: list[tuple[str, str | None]] = []
get = _make_get("body", recorder=calls)
read_http("https://h.test", "q", max_rows=10, credential_ref="API_TOKEN", get=get)
assert calls == [("https://h.test/q", "s3cret")] # secret comes from env, never the manifest
# --- transport seam: the stdlib default is the only socket path ----------------------------------
def test_read_http_default_get_is_urllib() -> None:
# inspect.signature proves the stdlib default WITHOUT opening a socket — _urllib_get is
# the single socket path and is never called by the offline suite.
sig = inspect.signature(read_http)
assert sig.parameters["get"].default is _urllib_get