"""The `http` connector and the §8 network gate (ingest-spec §4, §8). The optional extension point, exercised ONLY against an injected mock transport — the test suite never opens a socket (§11). credential_ref names an environment variable resolved at run time; the secret rides in the Authorization header, never in the URL and never in any output. """ from __future__ import annotations import json from pathlib import Path from typing import Any import pytest from llm_ingestion_okf.connectors import read_http from llm_ingestion_okf.errors import NetworkGateError, SourceError from llm_ingestion_okf.materialize import materialize_bundle INGESTED_AT = "2026-07-16T12:00:00Z" class MockGet: """A canned transport: records calls, returns a fixed body.""" def __init__(self, body: str = "hello\n") -> None: self.body = body self.calls: list[tuple[str, str | None]] = [] def __call__(self, url: str, credential: str | None) -> str: self.calls.append((url, credential)) return self.body # --- URL join and credential resolution --- def test_url_join_is_explicit_single_slash() -> None: get = MockGet() read_http("https://api.test/", "/orders", max_rows=10, get=get) assert get.calls == [("https://api.test/orders", None)] def test_no_credential_ref_means_no_credential() -> None: get = MockGet() read_http("https://api.test", "orders", max_rows=10, get=get) assert get.calls[0][1] is None def test_credential_resolved_from_environment(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setenv("OKF_TEST_TOKEN", "s3cret") get = MockGet() read_http("https://api.test", "orders", max_rows=10, credential_ref="OKF_TEST_TOKEN", get=get) url, credential = get.calls[0] assert credential == "s3cret" assert "s3cret" not in url def test_unset_credential_ref_fails_before_transport(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.delenv("OKF_TEST_TOKEN", raising=False) get = MockGet() with pytest.raises(SourceError): read_http( "https://api.test", "orders", max_rows=10, credential_ref="OKF_TEST_TOKEN", get=get ) assert get.calls == [] # --- body handling --- def test_crlf_normalized_to_lf() -> None: assert read_http("https://x", "q", max_rows=10, get=MockGet("a\r\nb\r")) == "a\nb\n" @pytest.mark.parametrize("body", ["```\n", "before\n```fence\n", " ```indented\n"]) def test_code_fence_line_in_body_rejected(body: str) -> None: """A body that cannot be safely embedded in a fenced block is an error — never silently-corrupted markdown.""" with pytest.raises(SourceError): read_http("https://x", "q", max_rows=10, get=MockGet(body)) def test_line_count_at_cap_is_allowed() -> None: assert read_http("https://x", "q", max_rows=2, get=MockGet("a\nb\n")) == "a\nb\n" def test_missing_trailing_newline_still_counts_last_line() -> None: with pytest.raises(SourceError): read_http("https://x", "q", max_rows=2, get=MockGet("a\nb\nc")) def test_exceeding_cap_is_an_error() -> None: with pytest.raises(SourceError): read_http("https://x", "q", max_rows=1, get=MockGet("a\nb\n")) # --- materialization end-to-end against the mock transport --- def http_manifest(tmp_path: Path) -> Path: data: dict[str, Any] = { "manifest_version": 1, "source": {"type": "http", "id": "api-1", "base_url": "https://api.test"}, "bundle_summary": "A test bundle.", "extractions": [ { "id": "orders", "title": "Orders", "query": "/orders", "okf_type": "dataset", "max_rows": 10, } ], } src = tmp_path / "src" src.mkdir() path = src / "manifest.json" path.write_text(json.dumps(data), encoding="utf-8") return path def test_http_materialization_renders_verbatim_fenced_body(tmp_path: Path) -> None: manifest_path = http_manifest(tmp_path) bundle = tmp_path / "bundle" get = MockGet('{"orders": [1, 2]}\n') materialize_bundle(manifest_path, bundle, INGESTED_AT, allow_network=True, http_get=get) content = (bundle / "ingest-orders.md").read_text(encoding="utf-8") assert content.endswith('```\n{"orders": [1, 2]}\n```\n') assert get.calls == [("https://api.test/orders", None)] index = (bundle / "index.md").read_text(encoding="utf-8") assert "- [Orders](ingest-orders.md)\n" in index def test_network_gate_blocks_before_any_transport_call(tmp_path: Path) -> None: """Load-bearing (spec §11): without the per-run opt-in the connector refuses fail-fast — zero transport calls, zero disk mutation.""" manifest_path = http_manifest(tmp_path) bundle = tmp_path / "bundle" get = MockGet() with pytest.raises(NetworkGateError): materialize_bundle(manifest_path, bundle, INGESTED_AT, http_get=get) assert get.calls == [] assert not bundle.exists()