From 24e73f47f64124378be50b47f2f44f8167f8983c Mon Sep 17 00:00:00 2001 From: Kjell Tore Guttormsen Date: Sat, 4 Jul 2026 17:12:34 +0200 Subject: [PATCH] test(ingest): http golden extraction, byte-deterministic (I6) --- .../build_fixture.py | 36 +++++++++++ .../expected-bundle/index.md | 3 + .../expected-bundle/ingest-report.md | 15 +++++ .../expected-bundle/ingest-status.md | 13 ++++ examples/ingest-golden-http/fixture/report | 3 + examples/ingest-golden-http/fixture/status | 1 + examples/ingest-golden-http/ingested-at.txt | 1 + examples/ingest-golden-http/manifest.json | 21 ++++++ tests/test_ingest_golden_http.py | 64 +++++++++++++++++++ 9 files changed, 157 insertions(+) create mode 100644 .claude/projects/2026-07-04-i6-ingest-http-maf/build_fixture.py create mode 100644 examples/ingest-golden-http/expected-bundle/index.md create mode 100644 examples/ingest-golden-http/expected-bundle/ingest-report.md create mode 100644 examples/ingest-golden-http/expected-bundle/ingest-status.md create mode 100644 examples/ingest-golden-http/fixture/report create mode 100644 examples/ingest-golden-http/fixture/status create mode 100644 examples/ingest-golden-http/ingested-at.txt create mode 100644 examples/ingest-golden-http/manifest.json create mode 100644 tests/test_ingest_golden_http.py diff --git a/.claude/projects/2026-07-04-i6-ingest-http-maf/build_fixture.py b/.claude/projects/2026-07-04-i6-ingest-http-maf/build_fixture.py new file mode 100644 index 0000000..8784087 --- /dev/null +++ b/.claude/projects/2026-07-04-i6-ingest-http-maf/build_fixture.py @@ -0,0 +1,36 @@ +"""Build the I6 http golden fixture: examples/ingest-golden-http/{fixture/status, fixture/report, +ingested-at.txt}. + +Regenerable throwaway (mirrors the I4 sql builder). The http golden does NOT commit a live +response — the fixture files ARE the canned payloads a mock ``get`` returns (no socket, no +credentials: ``credential_ref`` is null). Payloads are authored as RAW bytes so the verbatim +backslashes in ``c:\\temp\\cache`` and ``backslash \\`` survive exactly: a normal Python string +literal would turn ``\\t`` into a TAB and ``\\c`` into a literal backslash-c, silently corrupting +the fenced-not-escaped discriminator the golden test depends on. LF-only, so the golden is +platform-stable. +""" + +from __future__ import annotations + +from pathlib import Path + +GOLDEN = Path(__file__).resolve().parents[3] / "examples" / "ingest-golden-http" + + +def main() -> None: + fixture = GOLDEN / "fixture" + fixture.mkdir(parents=True, exist_ok=True) + # 1 line; a raw | and \ discriminate fenced-verbatim rendering from table-escaping. + status = rb'{"service": "billing", "state": "degraded | partial", "path": "c:\temp\cache"}' + (fixture / "status").write_bytes(status + b"\n") + # 3 lines (<= max_rows); pipe + backslash kept verbatim inside the fence. + report = ( + b"baseline ok\n" + rb"pipe | and backslash \ kept verbatim" + b"\n" + b"end of report\n" + ) + (fixture / "report").write_bytes(report) + (GOLDEN / "ingested-at.txt").write_bytes(b"2026-07-04T12:00:00Z\n") + print(f"built {fixture}/status, {fixture}/report + ingested-at.txt") + + +if __name__ == "__main__": + main() diff --git a/examples/ingest-golden-http/expected-bundle/index.md b/examples/ingest-golden-http/expected-bundle/index.md new file mode 100644 index 0000000..8cd279a --- /dev/null +++ b/examples/ingest-golden-http/expected-bundle/index.md @@ -0,0 +1,3 @@ +Golden extraction case: two HTTP extracts from a local mock endpoint (http source type). +- [Service status](ingest-status.md) +- [Ops report](ingest-report.md) diff --git a/examples/ingest-golden-http/expected-bundle/ingest-report.md b/examples/ingest-golden-http/expected-bundle/ingest-report.md new file mode 100644 index 0000000..666503d --- /dev/null +++ b/examples/ingest-golden-http/expected-bundle/ingest-report.md @@ -0,0 +1,15 @@ +--- +type: reference +title: Ops report +source_system: status-api +source_query: report +ingested_at: 2026-07-04T12:00:00Z +ingest_manifest: manifest@8a8ae7a7a4d1cfc2 +generated: true +--- + +``` +baseline ok +pipe | and backslash \ kept verbatim +end of report +``` diff --git a/examples/ingest-golden-http/expected-bundle/ingest-status.md b/examples/ingest-golden-http/expected-bundle/ingest-status.md new file mode 100644 index 0000000..12f8ef6 --- /dev/null +++ b/examples/ingest-golden-http/expected-bundle/ingest-status.md @@ -0,0 +1,13 @@ +--- +type: dataset +title: Service status +source_system: status-api +source_query: status +ingested_at: 2026-07-04T12:00:00Z +ingest_manifest: manifest@8a8ae7a7a4d1cfc2 +generated: true +--- + +``` +{"service": "billing", "state": "degraded | partial", "path": "c:\temp\cache"} +``` diff --git a/examples/ingest-golden-http/fixture/report b/examples/ingest-golden-http/fixture/report new file mode 100644 index 0000000..f5549f5 --- /dev/null +++ b/examples/ingest-golden-http/fixture/report @@ -0,0 +1,3 @@ +baseline ok +pipe | and backslash \ kept verbatim +end of report diff --git a/examples/ingest-golden-http/fixture/status b/examples/ingest-golden-http/fixture/status new file mode 100644 index 0000000..45db6d4 --- /dev/null +++ b/examples/ingest-golden-http/fixture/status @@ -0,0 +1 @@ +{"service": "billing", "state": "degraded | partial", "path": "c:\temp\cache"} diff --git a/examples/ingest-golden-http/ingested-at.txt b/examples/ingest-golden-http/ingested-at.txt new file mode 100644 index 0000000..509edd0 --- /dev/null +++ b/examples/ingest-golden-http/ingested-at.txt @@ -0,0 +1 @@ +2026-07-04T12:00:00Z diff --git a/examples/ingest-golden-http/manifest.json b/examples/ingest-golden-http/manifest.json new file mode 100644 index 0000000..26042d5 --- /dev/null +++ b/examples/ingest-golden-http/manifest.json @@ -0,0 +1,21 @@ +{ + "manifest_version": 1, + "source": {"type": "http", "id": "status-api", "base_url": "https://api.example.test/v1", "credential_ref": null}, + "bundle_summary": "Golden extraction case: two HTTP extracts from a local mock endpoint (http source type).", + "extractions": [ + { + "id": "status", + "title": "Service status", + "query": "status", + "okf_type": "dataset", + "max_rows": 10 + }, + { + "id": "report", + "title": "Ops report", + "query": "report", + "okf_type": "reference", + "max_rows": 10 + } + ] +} diff --git a/tests/test_ingest_golden_http.py b/tests/test_ingest_golden_http.py new file mode 100644 index 0000000..ec4fe26 --- /dev/null +++ b/tests/test_ingest_golden_http.py @@ -0,0 +1,64 @@ +"""I6 golden regression — the §11 golden extraction case for `http`, byte-for-byte. + +Mirrors tests/test_ingest_golden_sql.py for the third source type. The case pins the §5 http +rules AS RULES: the body is rendered verbatim inside a fenced code block (NOT a markdown table), +so a raw ``|`` and ``\`` survive un-escaped — the exact discriminator vs ``render_table``. RED when +ANY byte of the expected bundle diverges (spec §11 "Golden regression" seam). + +The canned ``get`` returns the committed fixture payloads (rebuild: +``.claude/projects/2026-07-04-i6-ingest-http-maf/build_fixture.py``) — no socket, no credentials +(``credential_ref`` is null), so the golden runs offline. The base_url / query never enter the +bundle, so the golden is checkout-location independent. +""" + +from __future__ import annotations + +from pathlib import Path + +from portfolio_optimiser.ingest import HttpGet, materialize + +GOLDEN = Path(__file__).resolve().parents[1] / "examples" / "ingest-golden-http" + + +def _bundle_bytes(directory: Path) -> dict[str, bytes]: + return {p.name: p.read_bytes() for p in sorted(directory.iterdir()) if p.is_file()} + + +def _fixture_get(fixture_dir: Path) -> HttpGet: + """Canned transport: map the joined URL's last path segment to its committed fixture file.""" + + def get(url: str, credential: str | None) -> str: + name = url.rsplit("/", 1)[-1] + return (fixture_dir / name).read_text(encoding="utf-8") + + return get + + +def test_golden_http_extraction_is_bit_deterministic(tmp_path: Path) -> None: + ingested_at = (GOLDEN / "ingested-at.txt").read_text(encoding="utf-8").strip() + fake_get = _fixture_get(GOLDEN / "fixture") + out = tmp_path / "bundle" + materialize( + GOLDEN / "manifest.json", + out, + ingested_at=ingested_at, + allow_network=True, + http_get=fake_get, + ) + + expected = _bundle_bytes(GOLDEN / "expected-bundle") + actual = _bundle_bytes(out) + # File-SET equality first — catches extra AND missing files, not just diverging bytes. + assert actual.keys() == expected.keys() + for name, content in expected.items(): + assert actual[name] == content, f"golden byte divergence in {name}" + + # §10: a second run over the same output leaves every byte unchanged. + materialize( + GOLDEN / "manifest.json", + out, + ingested_at=ingested_at, + allow_network=True, + http_get=fake_get, + ) + assert _bundle_bytes(out) == expected