test(ingest): golden extraction case examples/ingest-golden-file, byte-frozen (I2)
This commit is contained in:
parent
e66a2b4015
commit
994f5be11c
8 changed files with 107 additions and 0 deletions
3
examples/ingest-golden-file/expected-bundle/index.md
Normal file
3
examples/ingest-golden-file/expected-bundle/index.md
Normal file
|
|
@ -0,0 +1,3 @@
|
||||||
|
Golden extraction case: two CSV extracts from a local project archive (file source type).
|
||||||
|
- [Project costs](ingest-costs.md)
|
||||||
|
- [Escaping edge cases](ingest-edge.md)
|
||||||
14
examples/ingest-golden-file/expected-bundle/ingest-costs.md
Normal file
14
examples/ingest-golden-file/expected-bundle/ingest-costs.md
Normal file
|
|
@ -0,0 +1,14 @@
|
||||||
|
---
|
||||||
|
type: dataset
|
||||||
|
title: Project costs
|
||||||
|
source_system: prosjekt-arkiv
|
||||||
|
source_query: costs.csv
|
||||||
|
ingested_at: 2026-07-03T12:00:00Z
|
||||||
|
ingest_manifest: manifest@24109f486b9ef708
|
||||||
|
generated: true
|
||||||
|
---
|
||||||
|
|
||||||
|
| item | cost_nok | note |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| led-retrofit | 120000 | ROI under 2 years |
|
||||||
|
| hvac-tuning | 80000 | requires vendor quote |
|
||||||
18
examples/ingest-golden-file/expected-bundle/ingest-edge.md
Normal file
18
examples/ingest-golden-file/expected-bundle/ingest-edge.md
Normal file
|
|
@ -0,0 +1,18 @@
|
||||||
|
---
|
||||||
|
type: reference
|
||||||
|
title: Escaping edge cases
|
||||||
|
source_system: prosjekt-arkiv
|
||||||
|
source_query: edge.csv
|
||||||
|
ingested_at: 2026-07-03T12:00:00Z
|
||||||
|
ingest_manifest: manifest@24109f486b9ef708
|
||||||
|
generated: true
|
||||||
|
---
|
||||||
|
|
||||||
|
| id | value |
|
||||||
|
| --- | --- |
|
||||||
|
| backslash | a\\b |
|
||||||
|
| pipe | a\|b |
|
||||||
|
| newline | x y |
|
||||||
|
| both | \\\| |
|
||||||
|
| zeros | 007 |
|
||||||
|
| decimal | 1.50 |
|
||||||
3
examples/ingest-golden-file/fixture/costs.csv
Normal file
3
examples/ingest-golden-file/fixture/costs.csv
Normal file
|
|
@ -0,0 +1,3 @@
|
||||||
|
item,cost_nok,note
|
||||||
|
led-retrofit,120000,ROI under 2 years
|
||||||
|
hvac-tuning,80000,requires vendor quote
|
||||||
|
8
examples/ingest-golden-file/fixture/edge.csv
Normal file
8
examples/ingest-golden-file/fixture/edge.csv
Normal file
|
|
@ -0,0 +1,8 @@
|
||||||
|
id,value
|
||||||
|
backslash,a\b
|
||||||
|
pipe,a|b
|
||||||
|
newline,"x
|
||||||
|
y"
|
||||||
|
both,\|
|
||||||
|
zeros,007
|
||||||
|
decimal,1.50
|
||||||
|
1
examples/ingest-golden-file/ingested-at.txt
Normal file
1
examples/ingest-golden-file/ingested-at.txt
Normal file
|
|
@ -0,0 +1 @@
|
||||||
|
2026-07-03T12:00:00Z
|
||||||
21
examples/ingest-golden-file/manifest.json
Normal file
21
examples/ingest-golden-file/manifest.json
Normal file
|
|
@ -0,0 +1,21 @@
|
||||||
|
{
|
||||||
|
"manifest_version": 1,
|
||||||
|
"source": {"type": "file", "id": "prosjekt-arkiv", "root": "fixture"},
|
||||||
|
"bundle_summary": "Golden extraction case: two CSV extracts from a local project archive (file source type).",
|
||||||
|
"extractions": [
|
||||||
|
{
|
||||||
|
"id": "costs",
|
||||||
|
"title": "Project costs",
|
||||||
|
"query": "costs.csv",
|
||||||
|
"okf_type": "dataset",
|
||||||
|
"max_rows": 10
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "edge",
|
||||||
|
"title": "Escaping edge cases",
|
||||||
|
"query": "edge.csv",
|
||||||
|
"okf_type": "reference",
|
||||||
|
"max_rows": 10
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
39
tests/test_ingest_golden.py
Normal file
39
tests/test_ingest_golden.py
Normal file
|
|
@ -0,0 +1,39 @@
|
||||||
|
"""I2 golden regression — the §11 golden extraction case, byte-for-byte.
|
||||||
|
|
||||||
|
The golden case pins every determinism decision the unit tests exercise as RULES: the
|
||||||
|
text-verbatim CSV reading (the ``007``/``1.50`` cells discriminate it from a
|
||||||
|
number-normalizing implementation — their normalized forms differ byte-wise), the escape
|
||||||
|
order, the §5 frontmatter layout, and the §6 index shape. RED when ANY byte of the expected
|
||||||
|
bundle diverges (spec §11 "Golden regression" seam). The fixture layout is the shared
|
||||||
|
convention (``manifest.json``, ``fixture/``, ``ingested-at.txt``, ``expected-bundle/``);
|
||||||
|
authored LOCALLY in I2 — sharing via commons is a later gated round.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from portfolio_optimiser.ingest import materialize
|
||||||
|
|
||||||
|
GOLDEN = Path(__file__).resolve().parents[1] / "examples" / "ingest-golden-file"
|
||||||
|
|
||||||
|
|
||||||
|
def _bundle_bytes(directory: Path) -> dict[str, bytes]:
|
||||||
|
return {p.name: p.read_bytes() for p in sorted(directory.iterdir()) if p.is_file()}
|
||||||
|
|
||||||
|
|
||||||
|
def test_golden_extraction_is_bit_deterministic(tmp_path: Path) -> None:
|
||||||
|
ingested_at = (GOLDEN / "ingested-at.txt").read_text(encoding="utf-8").strip()
|
||||||
|
out = tmp_path / "bundle"
|
||||||
|
materialize(GOLDEN / "manifest.json", out, ingested_at=ingested_at)
|
||||||
|
|
||||||
|
expected = _bundle_bytes(GOLDEN / "expected-bundle")
|
||||||
|
actual = _bundle_bytes(out)
|
||||||
|
# File-SET equality first — catches extra AND missing files, not just diverging bytes.
|
||||||
|
assert actual.keys() == expected.keys()
|
||||||
|
for name, content in expected.items():
|
||||||
|
assert actual[name] == content, f"golden byte divergence in {name}"
|
||||||
|
|
||||||
|
# §10: a second run over the same output leaves every byte unchanged.
|
||||||
|
materialize(GOLDEN / "manifest.json", out, ingested_at=ingested_at)
|
||||||
|
assert _bundle_bytes(out) == expected
|
||||||
Loading…
Add table
Add a link
Reference in a new issue