Kjell Tore Guttormsen
ab1fa4d999
TDD step 7: read_http executes the optional http extension point behind an injectable transport seam (urllib_get is the only socket path; the suite runs socket-free against a mock). credential_ref resolves from the environment at run time, fail-fast before any transport call; the secret rides in the Authorization header only. Explicit single-slash URL join, CRLF-to-LF normalization, LF line-count cap (missing trailing newline still counts), and code-fence-marker rejection. materialize_bundle renders the body as a verbatim fenced block; the §8 gate test asserts zero transport calls without the opt-in. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01QeqhJpYQyghASjiJo5EhGg
145 lines
4.9 KiB
Python
145 lines
4.9 KiB
Python
"""The `http` connector and the §8 network gate (ingest-spec §4, §8).
|
|
|
|
The optional extension point, exercised ONLY against an injected mock
|
|
transport — the test suite never opens a socket (§11). credential_ref names
|
|
an environment variable resolved at run time; the secret rides in the
|
|
Authorization header, never in the URL and never in any output.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
import pytest
|
|
|
|
from llm_ingestion_okf.connectors import read_http
|
|
from llm_ingestion_okf.errors import NetworkGateError, SourceError
|
|
from llm_ingestion_okf.materialize import materialize_bundle
|
|
|
|
INGESTED_AT = "2026-07-16T12:00:00Z"
|
|
|
|
|
|
class MockGet:
|
|
"""A canned transport: records calls, returns a fixed body."""
|
|
|
|
def __init__(self, body: str = "hello\n") -> None:
|
|
self.body = body
|
|
self.calls: list[tuple[str, str | None]] = []
|
|
|
|
def __call__(self, url: str, credential: str | None) -> str:
|
|
self.calls.append((url, credential))
|
|
return self.body
|
|
|
|
|
|
# --- URL join and credential resolution ---
|
|
|
|
|
|
def test_url_join_is_explicit_single_slash() -> None:
|
|
get = MockGet()
|
|
read_http("https://api.test/", "/orders", max_rows=10, get=get)
|
|
assert get.calls == [("https://api.test/orders", None)]
|
|
|
|
|
|
def test_no_credential_ref_means_no_credential() -> None:
|
|
get = MockGet()
|
|
read_http("https://api.test", "orders", max_rows=10, get=get)
|
|
assert get.calls[0][1] is None
|
|
|
|
|
|
def test_credential_resolved_from_environment(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
monkeypatch.setenv("OKF_TEST_TOKEN", "s3cret")
|
|
get = MockGet()
|
|
read_http("https://api.test", "orders", max_rows=10, credential_ref="OKF_TEST_TOKEN", get=get)
|
|
url, credential = get.calls[0]
|
|
assert credential == "s3cret"
|
|
assert "s3cret" not in url
|
|
|
|
|
|
def test_unset_credential_ref_fails_before_transport(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
monkeypatch.delenv("OKF_TEST_TOKEN", raising=False)
|
|
get = MockGet()
|
|
with pytest.raises(SourceError):
|
|
read_http(
|
|
"https://api.test", "orders", max_rows=10, credential_ref="OKF_TEST_TOKEN", get=get
|
|
)
|
|
assert get.calls == []
|
|
|
|
|
|
# --- body handling ---
|
|
|
|
|
|
def test_crlf_normalized_to_lf() -> None:
|
|
assert read_http("https://x", "q", max_rows=10, get=MockGet("a\r\nb\r")) == "a\nb\n"
|
|
|
|
|
|
@pytest.mark.parametrize("body", ["```\n", "before\n```fence\n", " ```indented\n"])
|
|
def test_code_fence_line_in_body_rejected(body: str) -> None:
|
|
"""A body that cannot be safely embedded in a fenced block is an error —
|
|
never silently-corrupted markdown."""
|
|
with pytest.raises(SourceError):
|
|
read_http("https://x", "q", max_rows=10, get=MockGet(body))
|
|
|
|
|
|
def test_line_count_at_cap_is_allowed() -> None:
|
|
assert read_http("https://x", "q", max_rows=2, get=MockGet("a\nb\n")) == "a\nb\n"
|
|
|
|
|
|
def test_missing_trailing_newline_still_counts_last_line() -> None:
|
|
with pytest.raises(SourceError):
|
|
read_http("https://x", "q", max_rows=2, get=MockGet("a\nb\nc"))
|
|
|
|
|
|
def test_exceeding_cap_is_an_error() -> None:
|
|
with pytest.raises(SourceError):
|
|
read_http("https://x", "q", max_rows=1, get=MockGet("a\nb\n"))
|
|
|
|
|
|
# --- materialization end-to-end against the mock transport ---
|
|
|
|
|
|
def http_manifest(tmp_path: Path) -> Path:
|
|
data: dict[str, Any] = {
|
|
"manifest_version": 1,
|
|
"source": {"type": "http", "id": "api-1", "base_url": "https://api.test"},
|
|
"bundle_summary": "A test bundle.",
|
|
"extractions": [
|
|
{
|
|
"id": "orders",
|
|
"title": "Orders",
|
|
"query": "/orders",
|
|
"okf_type": "dataset",
|
|
"max_rows": 10,
|
|
}
|
|
],
|
|
}
|
|
src = tmp_path / "src"
|
|
src.mkdir()
|
|
path = src / "manifest.json"
|
|
path.write_text(json.dumps(data), encoding="utf-8")
|
|
return path
|
|
|
|
|
|
def test_http_materialization_renders_verbatim_fenced_body(tmp_path: Path) -> None:
|
|
manifest_path = http_manifest(tmp_path)
|
|
bundle = tmp_path / "bundle"
|
|
get = MockGet('{"orders": [1, 2]}\n')
|
|
materialize_bundle(manifest_path, bundle, INGESTED_AT, allow_network=True, http_get=get)
|
|
content = (bundle / "ingest-orders.md").read_text(encoding="utf-8")
|
|
assert content.endswith('```\n{"orders": [1, 2]}\n```\n')
|
|
assert get.calls == [("https://api.test/orders", None)]
|
|
index = (bundle / "index.md").read_text(encoding="utf-8")
|
|
assert "- [Orders](ingest-orders.md)\n" in index
|
|
|
|
|
|
def test_network_gate_blocks_before_any_transport_call(tmp_path: Path) -> None:
|
|
"""Load-bearing (spec §11): without the per-run opt-in the connector
|
|
refuses fail-fast — zero transport calls, zero disk mutation."""
|
|
manifest_path = http_manifest(tmp_path)
|
|
bundle = tmp_path / "bundle"
|
|
get = MockGet()
|
|
with pytest.raises(NetworkGateError):
|
|
materialize_bundle(manifest_path, bundle, INGESTED_AT, http_get=get)
|
|
assert get.calls == []
|
|
assert not bundle.exists()
|