test(ingest): http golden extraction, byte-deterministic (I6)

.claude/projects/2026-07-04-i6-ingest-http-maf/build_fixture.py

@@ -0,0 +1,36 @@
+"""Build the I6 http golden fixture: examples/ingest-golden-http/{fixture/status, fixture/report,
+ingested-at.txt}.
+
+Regenerable throwaway (mirrors the I4 sql builder). The http golden does NOT commit a live
+response — the fixture files ARE the canned payloads a mock ``get`` returns (no socket, no
+credentials: ``credential_ref`` is null). Payloads are authored as RAW bytes so the verbatim
+backslashes in ``c:\\temp\\cache`` and ``backslash \\`` survive exactly: a normal Python string
+literal would turn ``\\t`` into a TAB and ``\\c`` into a literal backslash-c, silently corrupting
+the fenced-not-escaped discriminator the golden test depends on. LF-only, so the golden is
+platform-stable.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+GOLDEN = Path(__file__).resolve().parents[3] / "examples" / "ingest-golden-http"
+
+
+def main() -> None:
+    fixture = GOLDEN / "fixture"
+    fixture.mkdir(parents=True, exist_ok=True)
+    # 1 line; a raw | and \ discriminate fenced-verbatim rendering from table-escaping.
+    status = rb'{"service": "billing", "state": "degraded | partial", "path": "c:\temp\cache"}'
+    (fixture / "status").write_bytes(status + b"\n")
+    # 3 lines (<= max_rows); pipe + backslash kept verbatim inside the fence.
+    report = (
+        b"baseline ok\n" + rb"pipe | and backslash \ kept verbatim" + b"\n" + b"end of report\n"
+    )
+    (fixture / "report").write_bytes(report)
+    (GOLDEN / "ingested-at.txt").write_bytes(b"2026-07-04T12:00:00Z\n")
+    print(f"built {fixture}/status, {fixture}/report + ingested-at.txt")
+
+
+if __name__ == "__main__":
+    main()

examples/ingest-golden-http/expected-bundle/index.md

@@ -0,0 +1,3 @@
+Golden extraction case: two HTTP extracts from a local mock endpoint (http source type).
+- [Service status](ingest-status.md)
+- [Ops report](ingest-report.md)

examples/ingest-golden-http/expected-bundle/ingest-report.md

@@ -0,0 +1,15 @@
+---
+type: reference
+title: Ops report
+source_system: status-api
+source_query: report
+ingested_at: 2026-07-04T12:00:00Z
+ingest_manifest: manifest@8a8ae7a7a4d1cfc2
+generated: true
+---
+
+```
+baseline ok
+pipe | and backslash \ kept verbatim
+end of report
+```

examples/ingest-golden-http/expected-bundle/ingest-status.md

@@ -0,0 +1,13 @@
+---
+type: dataset
+title: Service status
+source_system: status-api
+source_query: status
+ingested_at: 2026-07-04T12:00:00Z
+ingest_manifest: manifest@8a8ae7a7a4d1cfc2
+generated: true
+---
+
+```
+{"service": "billing", "state": "degraded | partial", "path": "c:\temp\cache"}
+```

examples/ingest-golden-http/fixture/report

@@ -0,0 +1,3 @@
+baseline ok
+pipe | and backslash \ kept verbatim
+end of report

examples/ingest-golden-http/fixture/status

@@ -0,0 +1 @@
+{"service": "billing", "state": "degraded | partial", "path": "c:\temp\cache"}

examples/ingest-golden-http/ingested-at.txt

@@ -0,0 +1 @@
+2026-07-04T12:00:00Z

examples/ingest-golden-http/manifest.json

@@ -0,0 +1,21 @@
+{
+  "manifest_version": 1,
+  "source": {"type": "http", "id": "status-api", "base_url": "https://api.example.test/v1", "credential_ref": null},
+  "bundle_summary": "Golden extraction case: two HTTP extracts from a local mock endpoint (http source type).",
+  "extractions": [
+    {
+      "id": "status",
+      "title": "Service status",
+      "query": "status",
+      "okf_type": "dataset",
+      "max_rows": 10
+    },
+    {
+      "id": "report",
+      "title": "Ops report",
+      "query": "report",
+      "okf_type": "reference",
+      "max_rows": 10
+    }
+  ]
+}

tests/test_ingest_golden_http.py

@@ -0,0 +1,64 @@
+"""I6 golden regression — the §11 golden extraction case for `http`, byte-for-byte.
+
+Mirrors tests/test_ingest_golden_sql.py for the third source type. The case pins the §5 http
+rules AS RULES: the body is rendered verbatim inside a fenced code block (NOT a markdown table),
+so a raw ``|`` and ``\`` survive un-escaped — the exact discriminator vs ``render_table``. RED when
+ANY byte of the expected bundle diverges (spec §11 "Golden regression" seam).
+
+The canned ``get`` returns the committed fixture payloads (rebuild:
+``.claude/projects/2026-07-04-i6-ingest-http-maf/build_fixture.py``) — no socket, no credentials
+(``credential_ref`` is null), so the golden runs offline. The base_url / query never enter the
+bundle, so the golden is checkout-location independent.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from portfolio_optimiser.ingest import HttpGet, materialize
+
+GOLDEN = Path(__file__).resolve().parents[1] / "examples" / "ingest-golden-http"
+
+
+def _bundle_bytes(directory: Path) -> dict[str, bytes]:
+    return {p.name: p.read_bytes() for p in sorted(directory.iterdir()) if p.is_file()}
+
+
+def _fixture_get(fixture_dir: Path) -> HttpGet:
+    """Canned transport: map the joined URL's last path segment to its committed fixture file."""
+
+    def get(url: str, credential: str | None) -> str:
+        name = url.rsplit("/", 1)[-1]
+        return (fixture_dir / name).read_text(encoding="utf-8")
+
+    return get
+
+
+def test_golden_http_extraction_is_bit_deterministic(tmp_path: Path) -> None:
+    ingested_at = (GOLDEN / "ingested-at.txt").read_text(encoding="utf-8").strip()
+    fake_get = _fixture_get(GOLDEN / "fixture")
+    out = tmp_path / "bundle"
+    materialize(
+        GOLDEN / "manifest.json",
+        out,
+        ingested_at=ingested_at,
+        allow_network=True,
+        http_get=fake_get,
+    )
+
+    expected = _bundle_bytes(GOLDEN / "expected-bundle")
+    actual = _bundle_bytes(out)
+    # File-SET equality first — catches extra AND missing files, not just diverging bytes.
+    assert actual.keys() == expected.keys()
+    for name, content in expected.items():
+        assert actual[name] == content, f"golden byte divergence in {name}"
+
+    # §10: a second run over the same output leaves every byte unchanged.
+    materialize(
+        GOLDEN / "manifest.json",
+        out,
+        ingested_at=ingested_at,
+        allow_network=True,
+        http_get=fake_get,
+    )
+    assert _bundle_bytes(out) == expected