test(ingest): golden extraction case examples/ingest-golden-file, byte-frozen (I2)

This commit is contained in:
Kjell Tore Guttormsen 2026-07-03 18:40:53 +02:00
commit 994f5be11c
8 changed files with 107 additions and 0 deletions

View file

@ -0,0 +1,3 @@
Golden extraction case: two CSV extracts from a local project archive (file source type).
- [Project costs](ingest-costs.md)
- [Escaping edge cases](ingest-edge.md)

View file

@ -0,0 +1,14 @@
---
type: dataset
title: Project costs
source_system: prosjekt-arkiv
source_query: costs.csv
ingested_at: 2026-07-03T12:00:00Z
ingest_manifest: manifest@24109f486b9ef708
generated: true
---
| item | cost_nok | note |
| --- | --- | --- |
| led-retrofit | 120000 | ROI under 2 years |
| hvac-tuning | 80000 | requires vendor quote |

View file

@ -0,0 +1,18 @@
---
type: reference
title: Escaping edge cases
source_system: prosjekt-arkiv
source_query: edge.csv
ingested_at: 2026-07-03T12:00:00Z
ingest_manifest: manifest@24109f486b9ef708
generated: true
---
| id | value |
| --- | --- |
| backslash | a\\b |
| pipe | a\|b |
| newline | x y |
| both | \\\| |
| zeros | 007 |
| decimal | 1.50 |

View file

@ -0,0 +1,3 @@
item,cost_nok,note
led-retrofit,120000,ROI under 2 years
hvac-tuning,80000,requires vendor quote
1 item cost_nok note
2 led-retrofit 120000 ROI under 2 years
3 hvac-tuning 80000 requires vendor quote

View file

@ -0,0 +1,8 @@
id,value
backslash,a\b
pipe,a|b
newline,"x
y"
both,\|
zeros,007
decimal,1.50
1 id value
2 backslash a\b
3 pipe a|b
4 newline x y
5 both \|
6 zeros 007
7 decimal 1.50

View file

@ -0,0 +1 @@
2026-07-03T12:00:00Z

View file

@ -0,0 +1,21 @@
{
"manifest_version": 1,
"source": {"type": "file", "id": "prosjekt-arkiv", "root": "fixture"},
"bundle_summary": "Golden extraction case: two CSV extracts from a local project archive (file source type).",
"extractions": [
{
"id": "costs",
"title": "Project costs",
"query": "costs.csv",
"okf_type": "dataset",
"max_rows": 10
},
{
"id": "edge",
"title": "Escaping edge cases",
"query": "edge.csv",
"okf_type": "reference",
"max_rows": 10
}
]
}

View file

@ -0,0 +1,39 @@
"""I2 golden regression — the §11 golden extraction case, byte-for-byte.
The golden case pins every determinism decision the unit tests exercise as RULES: the
text-verbatim CSV reading (the ``007``/``1.50`` cells discriminate it from a
number-normalizing implementation their normalized forms differ byte-wise), the escape
order, the §5 frontmatter layout, and the §6 index shape. RED when ANY byte of the expected
bundle diverges (spec §11 "Golden regression" seam). The fixture layout is the shared
convention (``manifest.json``, ``fixture/``, ``ingested-at.txt``, ``expected-bundle/``);
authored LOCALLY in I2 sharing via commons is a later gated round.
"""
from __future__ import annotations
from pathlib import Path
from portfolio_optimiser.ingest import materialize
GOLDEN = Path(__file__).resolve().parents[1] / "examples" / "ingest-golden-file"
def _bundle_bytes(directory: Path) -> dict[str, bytes]:
return {p.name: p.read_bytes() for p in sorted(directory.iterdir()) if p.is_file()}
def test_golden_extraction_is_bit_deterministic(tmp_path: Path) -> None:
ingested_at = (GOLDEN / "ingested-at.txt").read_text(encoding="utf-8").strip()
out = tmp_path / "bundle"
materialize(GOLDEN / "manifest.json", out, ingested_at=ingested_at)
expected = _bundle_bytes(GOLDEN / "expected-bundle")
actual = _bundle_bytes(out)
# File-SET equality first — catches extra AND missing files, not just diverging bytes.
assert actual.keys() == expected.keys()
for name, content in expected.items():
assert actual[name] == content, f"golden byte divergence in {name}"
# §10: a second run over the same output leaves every byte unchanged.
materialize(GOLDEN / "manifest.json", out, ingested_at=ingested_at)
assert _bundle_bytes(out) == expected