diff --git a/examples/ingest-golden-file/expected-bundle/index.md b/examples/ingest-golden-file/expected-bundle/index.md new file mode 100644 index 0000000..3c7583e --- /dev/null +++ b/examples/ingest-golden-file/expected-bundle/index.md @@ -0,0 +1,3 @@ +Golden extraction case: two CSV extracts from a local project archive (file source type). +- [Project costs](ingest-costs.md) +- [Escaping edge cases](ingest-edge.md) diff --git a/examples/ingest-golden-file/expected-bundle/ingest-costs.md b/examples/ingest-golden-file/expected-bundle/ingest-costs.md new file mode 100644 index 0000000..fa542d7 --- /dev/null +++ b/examples/ingest-golden-file/expected-bundle/ingest-costs.md @@ -0,0 +1,14 @@ +--- +type: dataset +title: Project costs +source_system: prosjekt-arkiv +source_query: costs.csv +ingested_at: 2026-07-03T12:00:00Z +ingest_manifest: manifest@24109f486b9ef708 +generated: true +--- + +| item | cost_nok | note | +| --- | --- | --- | +| led-retrofit | 120000 | ROI under 2 years | +| hvac-tuning | 80000 | requires vendor quote | diff --git a/examples/ingest-golden-file/expected-bundle/ingest-edge.md b/examples/ingest-golden-file/expected-bundle/ingest-edge.md new file mode 100644 index 0000000..47da3d7 --- /dev/null +++ b/examples/ingest-golden-file/expected-bundle/ingest-edge.md @@ -0,0 +1,18 @@ +--- +type: reference +title: Escaping edge cases +source_system: prosjekt-arkiv +source_query: edge.csv +ingested_at: 2026-07-03T12:00:00Z +ingest_manifest: manifest@24109f486b9ef708 +generated: true +--- + +| id | value | +| --- | --- | +| backslash | a\\b | +| pipe | a\|b | +| newline | x y | +| both | \\\| | +| zeros | 007 | +| decimal | 1.50 | diff --git a/examples/ingest-golden-file/fixture/costs.csv b/examples/ingest-golden-file/fixture/costs.csv new file mode 100644 index 0000000..0dbc341 --- /dev/null +++ b/examples/ingest-golden-file/fixture/costs.csv @@ -0,0 +1,3 @@ +item,cost_nok,note +led-retrofit,120000,ROI under 2 years +hvac-tuning,80000,requires vendor quote diff --git a/examples/ingest-golden-file/fixture/edge.csv b/examples/ingest-golden-file/fixture/edge.csv new file mode 100644 index 0000000..e3fb623 --- /dev/null +++ b/examples/ingest-golden-file/fixture/edge.csv @@ -0,0 +1,8 @@ +id,value +backslash,a\b +pipe,a|b +newline,"x +y" +both,\| +zeros,007 +decimal,1.50 diff --git a/examples/ingest-golden-file/ingested-at.txt b/examples/ingest-golden-file/ingested-at.txt new file mode 100644 index 0000000..9f82d47 --- /dev/null +++ b/examples/ingest-golden-file/ingested-at.txt @@ -0,0 +1 @@ +2026-07-03T12:00:00Z diff --git a/examples/ingest-golden-file/manifest.json b/examples/ingest-golden-file/manifest.json new file mode 100644 index 0000000..5c3a087 --- /dev/null +++ b/examples/ingest-golden-file/manifest.json @@ -0,0 +1,21 @@ +{ + "manifest_version": 1, + "source": {"type": "file", "id": "prosjekt-arkiv", "root": "fixture"}, + "bundle_summary": "Golden extraction case: two CSV extracts from a local project archive (file source type).", + "extractions": [ + { + "id": "costs", + "title": "Project costs", + "query": "costs.csv", + "okf_type": "dataset", + "max_rows": 10 + }, + { + "id": "edge", + "title": "Escaping edge cases", + "query": "edge.csv", + "okf_type": "reference", + "max_rows": 10 + } + ] +} diff --git a/tests/test_ingest_golden.py b/tests/test_ingest_golden.py new file mode 100644 index 0000000..58e1256 --- /dev/null +++ b/tests/test_ingest_golden.py @@ -0,0 +1,39 @@ +"""I2 golden regression — the §11 golden extraction case, byte-for-byte. + +The golden case pins every determinism decision the unit tests exercise as RULES: the +text-verbatim CSV reading (the ``007``/``1.50`` cells discriminate it from a +number-normalizing implementation — their normalized forms differ byte-wise), the escape +order, the §5 frontmatter layout, and the §6 index shape. RED when ANY byte of the expected +bundle diverges (spec §11 "Golden regression" seam). The fixture layout is the shared +convention (``manifest.json``, ``fixture/``, ``ingested-at.txt``, ``expected-bundle/``); +authored LOCALLY in I2 — sharing via commons is a later gated round. +""" + +from __future__ import annotations + +from pathlib import Path + +from portfolio_optimiser.ingest import materialize + +GOLDEN = Path(__file__).resolve().parents[1] / "examples" / "ingest-golden-file" + + +def _bundle_bytes(directory: Path) -> dict[str, bytes]: + return {p.name: p.read_bytes() for p in sorted(directory.iterdir()) if p.is_file()} + + +def test_golden_extraction_is_bit_deterministic(tmp_path: Path) -> None: + ingested_at = (GOLDEN / "ingested-at.txt").read_text(encoding="utf-8").strip() + out = tmp_path / "bundle" + materialize(GOLDEN / "manifest.json", out, ingested_at=ingested_at) + + expected = _bundle_bytes(GOLDEN / "expected-bundle") + actual = _bundle_bytes(out) + # File-SET equality first — catches extra AND missing files, not just diverging bytes. + assert actual.keys() == expected.keys() + for name, content in expected.items(): + assert actual[name] == content, f"golden byte divergence in {name}" + + # §10: a second run over the same output leaves every byte unchanged. + materialize(GOLDEN / "manifest.json", out, ingested_at=ingested_at) + assert _bundle_bytes(out) == expected