feat(ingest): I3 — D7-speil av ingest (filkatalog/CSV), bygget fra commons-spec alene
Speiler MAF I2 fra shared/ingest-spec.md alene: manifest → CSV-konnektor → materialisert OKF-bundle, byte-identisk med den delte golden-fasiten. - ingest.py: ManifestContract (pydantic, fail-fast, file-kilde, verdict-reservasjon §3, id-grammatikk, max_rows), CSV-konnektor (boundary-checked fail-closed), materialisering (§5-frontmatter eksakt rekkefølge, markdown-tabell m/ escaping, LF-only, SHA-256 manifest-stamp), index-generering (§6), replacement §3/§5. - okf.py: _parse_index_entry — tolererer frontmatterløs index (method-spec §3: index rendres via body = summary, ikke som typet concept-fil). Golden var spec-konform; D7-okf var strengere enn standarden. Scoped: non-index concept- filer krever fortsatt type (honesty-test). - examples/ingest-golden-file/: repo-lokal golden (byte-frossen kopi av I2s fasit). - Speiltester (I2s load-bearing-sett, alle detach-bevist røde): golden byte-fasit + mutasjonskontroller · provenance/navigability/verdict-reservasjon/re-ingest-safety · kontrakt fail-fast/max_rows/boundary/kollisjon · spec-integritet §11. - docs/2026-07-04-I3-brief.md: brief + de to operatør-avgjorte beslutningene. Suite 239 passed uten nøkkel/nettverk (189 + 50 nye) · ruff + mypy --strict rene. [skip-docs] README + docs/extending.md er bevisst utsatt til I7 per sesjonsplan (programmet batcher ingest-doc der, avgrenset til det D7 faktisk har — CSV nå, SQL/HTTP senere). Endringen er dokumentert i docs/2026-07-04-I3-brief.md. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_017MM6BWb1hWmJZuXFZ7rjxT
This commit is contained in:
parent
dc2345e134
commit
e03bd79876
15 changed files with 958 additions and 2 deletions
138
tests/test_ingest.py
Normal file
138
tests/test_ingest.py
Normal file
|
|
@ -0,0 +1,138 @@
|
|||
"""Ingest unit + fail-fast contract tests (ingest-spec §4, §5, §8).
|
||||
|
||||
The manifest is schema-validated fail-fast BEFORE any source call (§4, the startup-contract
|
||||
discipline). These tests pin the malformed-manifest rejections, the §5 cell-escaping rules,
|
||||
and the §8 size cap / fail-closed path resolution / no-overwrite collision.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from pydantic import ValidationError
|
||||
|
||||
from portfolio_optimiser_claude.ingest import (
|
||||
ManifestContract,
|
||||
_escape_cell,
|
||||
load_manifest,
|
||||
materialize,
|
||||
)
|
||||
|
||||
GOLDEN = Path(__file__).resolve().parents[1] / "examples" / "ingest-golden-file"
|
||||
INGESTED_AT = "2026-07-03T12:00:00Z"
|
||||
|
||||
|
||||
def _valid() -> dict:
|
||||
return {
|
||||
"manifest_version": 1,
|
||||
"source": {"type": "file", "id": "arkiv", "root": "fixture"},
|
||||
"bundle_summary": "s",
|
||||
"extractions": [
|
||||
{"id": "e", "title": "T", "query": "e.csv", "okf_type": "dataset", "max_rows": 5}
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def _write_case(tmp_path: Path, manifest: dict, csvs: dict[str, str]) -> Path:
|
||||
case = tmp_path / "case"
|
||||
(case / "fixture").mkdir(parents=True)
|
||||
(case / "manifest.json").write_text(json.dumps(manifest), encoding="utf-8")
|
||||
for name, text in csvs.items():
|
||||
(case / "fixture" / name).write_text(text, encoding="utf-8")
|
||||
return case
|
||||
|
||||
|
||||
class TestManifestValidation:
|
||||
"""§4: a malformed manifest never starts a run (fail-fast)."""
|
||||
|
||||
def test_valid_manifest_loads_and_stamps(self) -> None:
|
||||
contract = ManifestContract(**_valid())
|
||||
assert contract.manifest_version == 1
|
||||
assert contract.source.id == "arkiv"
|
||||
|
||||
def test_stamp_is_stem_at_sha256_16(self, tmp_path: Path) -> None:
|
||||
case = _write_case(tmp_path, _valid(), {"e.csv": "a\n1\n"})
|
||||
loaded = load_manifest(case / "manifest.json")
|
||||
stem, _, digest = loaded.stamp.partition("@")
|
||||
assert stem == "manifest"
|
||||
assert len(digest) == 16 and all(c in "0123456789abcdef" for c in digest)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"mutate",
|
||||
[
|
||||
lambda m: m.pop("manifest_version"),
|
||||
lambda m: m.__setitem__("manifest_version", 2),
|
||||
lambda m: m.pop("source"),
|
||||
lambda m: m.pop("bundle_summary"),
|
||||
lambda m: m.__setitem__("extractions", []),
|
||||
lambda m: m["source"].__setitem__("id", "Bad_Id"),
|
||||
lambda m: m["source"].__setitem__("type", "sql"),
|
||||
lambda m: m["extractions"][0].__setitem__("id", "Bad Id"),
|
||||
lambda m: m["extractions"][0].__setitem__("title", "two\nlines"),
|
||||
lambda m: m["extractions"][0].__setitem__("max_rows", 0),
|
||||
lambda m: m["extractions"][0].__setitem__("max_rows", -1),
|
||||
],
|
||||
)
|
||||
def test_malformed_manifest_is_rejected(self, mutate) -> None:
|
||||
manifest = _valid()
|
||||
mutate(manifest)
|
||||
with pytest.raises(ValidationError):
|
||||
ManifestContract(**manifest)
|
||||
|
||||
def test_duplicate_extraction_ids_are_rejected(self) -> None:
|
||||
manifest = _valid()
|
||||
manifest["extractions"].append(dict(manifest["extractions"][0]))
|
||||
with pytest.raises(ValidationError):
|
||||
ManifestContract(**manifest)
|
||||
|
||||
|
||||
class TestCellEscaping:
|
||||
"""§5: text verbatim with backslash → \\\\, pipe → \\|, newline → single space."""
|
||||
|
||||
def test_backslash_then_pipe_order(self) -> None:
|
||||
assert _escape_cell("\\|") == "\\\\\\|"
|
||||
|
||||
def test_pipe_escaped(self) -> None:
|
||||
assert _escape_cell("a|b") == "a\\|b"
|
||||
|
||||
def test_newline_becomes_single_space(self) -> None:
|
||||
assert _escape_cell("x\ny") == "x y"
|
||||
assert _escape_cell("x\r\ny") == "x y"
|
||||
|
||||
def test_plain_text_verbatim(self) -> None:
|
||||
assert _escape_cell("007") == "007"
|
||||
assert _escape_cell("1.50") == "1.50"
|
||||
|
||||
|
||||
class TestSecurityFrame:
|
||||
"""§8: size cap fail-fast, path resolution fail-closed, curated never overwritten."""
|
||||
|
||||
def test_extraction_exceeding_max_rows_fails(self, tmp_path: Path) -> None:
|
||||
manifest = _valid()
|
||||
manifest["extractions"][0]["max_rows"] = 1
|
||||
case = _write_case(tmp_path, manifest, {"e.csv": "col\n1\n2\n"}) # 2 data rows > 1
|
||||
with pytest.raises(ValueError, match="max_rows"):
|
||||
materialize(case / "manifest.json", tmp_path / "bundle", INGESTED_AT)
|
||||
|
||||
def test_query_escaping_root_is_refused(self, tmp_path: Path) -> None:
|
||||
manifest = _valid()
|
||||
manifest["extractions"][0]["query"] = "../secret.csv"
|
||||
case = _write_case(tmp_path, manifest, {"e.csv": "col\n1\n"})
|
||||
(case / "secret.csv").write_text("col\nx\n", encoding="utf-8")
|
||||
with pytest.raises(ValueError, match="escapes"):
|
||||
materialize(case / "manifest.json", tmp_path / "bundle", INGESTED_AT)
|
||||
|
||||
def test_collision_with_non_ingest_file_fails(self, tmp_path: Path) -> None:
|
||||
case = _write_case(tmp_path, _valid(), {"e.csv": "col\n1\n"})
|
||||
bundle = tmp_path / "bundle"
|
||||
bundle.mkdir()
|
||||
# A curated (non-stamped) file already occupies the generated name.
|
||||
(bundle / "ingest-e.md").write_text(
|
||||
"---\ntype: reference\ntitle: hand\n---\n\nCurated.\n", encoding="utf-8"
|
||||
)
|
||||
with pytest.raises(ValueError, match="collides"):
|
||||
materialize(case / "manifest.json", bundle, INGESTED_AT)
|
||||
# The curated file is untouched — never overwritten.
|
||||
assert "Curated." in (bundle / "ingest-e.md").read_text(encoding="utf-8")
|
||||
Loading…
Add table
Add a link
Reference in a new issue