feat(ingest): adopt llm-ingestion-okf as Door A implementation (first consumer)
Replace the local 391-line ingest implementation with a thin adapter over the shared llm-ingestion-okf library (git-pinned dae0bd1a via Forgejo, tool.uv.sources). The materialize() signature is preserved; error types are now the library's typed hierarchy rooted in IngestError, re-exported from the consumer seam. - tests/test_ingest_adoption.py: new load-bearing seam tests (delegation, offline invariant — allow_network is never passed, error contract), detach-proven red twice. - Golden suites (file + sql) pass UNCHANGED — byte-exact behaviour proven against the repo-local fixtures. - 6 test files migrated to the library error hierarchy; escaping/typed-cell unit tests dropped (byte-bound by the ingest-edge.md golden, unit-owned by the library's own 189-test suite). Provenance stamp now asserted independently from the §5 rule. - mypy override follow_untyped_imports for llm_ingestion_okf (no py.typed upstream yet — reported as a finding). Suite: 386 passed; ruff + format + mypy --strict clean; shared/, examples/, runs/s10/ and run_s10.py byte-untouched. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
parent
80a2fa1a77
commit
5732d13369
9 changed files with 307 additions and 495 deletions
|
|
@ -1,8 +1,14 @@
|
|||
"""Ingest unit + fail-fast contract tests (ingest-spec §4, §5, §8).
|
||||
|
||||
The manifest is schema-validated fail-fast BEFORE any source call (§4, the startup-contract
|
||||
discipline). These tests pin the malformed-manifest rejections, the §5 cell-escaping rules,
|
||||
and the §8 size cap / fail-closed path resolution / no-overwrite collision.
|
||||
discipline). These tests pin the malformed-manifest rejections, the §8 size cap, and the
|
||||
fail-closed path resolution / no-overwrite collision — all through the consumer seam
|
||||
(``portfolio_optimiser_claude.ingest``, backed by llm-ingestion-okf since the adoption).
|
||||
|
||||
The §5 cell-escaping rules are NOT unit-tested here anymore: every escaping case
|
||||
(backslash, pipe, backslash-then-pipe order, newline collapse, verbatim text) is bound
|
||||
byte-for-byte by the ``ingest-edge.md`` golden (test_ingest_golden.py) and unit-owned by
|
||||
the library's own suite.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
|
@ -11,11 +17,11 @@ import json
|
|||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from pydantic import ValidationError
|
||||
|
||||
from portfolio_optimiser_claude.ingest import (
|
||||
ManifestContract,
|
||||
_escape_cell,
|
||||
ManifestError,
|
||||
MaterializationError,
|
||||
SourceError,
|
||||
load_manifest,
|
||||
materialize,
|
||||
)
|
||||
|
|
@ -44,18 +50,29 @@ def _write_case(tmp_path: Path, manifest: dict, csvs: dict[str, str]) -> Path:
|
|||
return case
|
||||
|
||||
|
||||
def _load(tmp_path: Path, manifest: dict):
|
||||
path = tmp_path / "manifest.json"
|
||||
path.write_text(json.dumps(manifest), encoding="utf-8")
|
||||
return load_manifest(path)
|
||||
|
||||
|
||||
class TestManifestValidation:
|
||||
"""§4: a malformed manifest never starts a run (fail-fast)."""
|
||||
|
||||
def test_valid_manifest_loads_and_stamps(self) -> None:
|
||||
contract = ManifestContract(**_valid())
|
||||
assert contract.manifest_version == 1
|
||||
assert contract.source.id == "arkiv"
|
||||
def test_valid_manifest_loads(self, tmp_path: Path) -> None:
|
||||
manifest = _load(tmp_path, _valid())
|
||||
assert manifest.manifest_version == 1
|
||||
assert manifest.source.id == "arkiv"
|
||||
|
||||
def test_stamp_is_stem_at_sha256_16(self, tmp_path: Path) -> None:
|
||||
# The §5 provenance stamp (``{stem}@{sha256(raw)[:16]}``) is asserted from the
|
||||
# materialized frontmatter — the stamp is a property of the run, not the manifest.
|
||||
case = _write_case(tmp_path, _valid(), {"e.csv": "a\n1\n"})
|
||||
loaded = load_manifest(case / "manifest.json")
|
||||
stem, _, digest = loaded.stamp.partition("@")
|
||||
bundle = tmp_path / "bundle"
|
||||
materialize(case / "manifest.json", bundle, INGESTED_AT)
|
||||
frontmatter = (bundle / "ingest-e.md").read_text(encoding="utf-8").splitlines()
|
||||
(stamp_line,) = [ln for ln in frontmatter if ln.startswith("ingest_manifest: ")]
|
||||
stem, _, digest = stamp_line.removeprefix("ingest_manifest: ").partition("@")
|
||||
assert stem == "manifest"
|
||||
assert len(digest) == 16 and all(c in "0123456789abcdef" for c in digest)
|
||||
|
||||
|
|
@ -68,7 +85,7 @@ class TestManifestValidation:
|
|||
lambda m: m.pop("bundle_summary"),
|
||||
lambda m: m.__setitem__("extractions", []),
|
||||
lambda m: m["source"].__setitem__("id", "Bad_Id"),
|
||||
lambda m: m["source"].__setitem__("type", "http"), # optional/unimplemented (§1)
|
||||
lambda m: m["source"].__setitem__("type", "http"), # http requires base_url (§4)
|
||||
lambda m: m["source"].__setitem__("type", "unknown"), # bad discriminator
|
||||
lambda m: m["extractions"][0].__setitem__("id", "Bad Id"),
|
||||
lambda m: m["extractions"][0].__setitem__("title", "two\nlines"),
|
||||
|
|
@ -76,35 +93,17 @@ class TestManifestValidation:
|
|||
lambda m: m["extractions"][0].__setitem__("max_rows", -1),
|
||||
],
|
||||
)
|
||||
def test_malformed_manifest_is_rejected(self, mutate) -> None:
|
||||
def test_malformed_manifest_is_rejected(self, tmp_path: Path, mutate) -> None:
|
||||
manifest = _valid()
|
||||
mutate(manifest)
|
||||
with pytest.raises(ValidationError):
|
||||
ManifestContract(**manifest)
|
||||
with pytest.raises(ManifestError):
|
||||
_load(tmp_path, manifest)
|
||||
|
||||
def test_duplicate_extraction_ids_are_rejected(self) -> None:
|
||||
def test_duplicate_extraction_ids_are_rejected(self, tmp_path: Path) -> None:
|
||||
manifest = _valid()
|
||||
manifest["extractions"].append(dict(manifest["extractions"][0]))
|
||||
with pytest.raises(ValidationError):
|
||||
ManifestContract(**manifest)
|
||||
|
||||
|
||||
class TestCellEscaping:
|
||||
"""§5: text verbatim with backslash → \\\\, pipe → \\|, newline → single space."""
|
||||
|
||||
def test_backslash_then_pipe_order(self) -> None:
|
||||
assert _escape_cell("\\|") == "\\\\\\|"
|
||||
|
||||
def test_pipe_escaped(self) -> None:
|
||||
assert _escape_cell("a|b") == "a\\|b"
|
||||
|
||||
def test_newline_becomes_single_space(self) -> None:
|
||||
assert _escape_cell("x\ny") == "x y"
|
||||
assert _escape_cell("x\r\ny") == "x y"
|
||||
|
||||
def test_plain_text_verbatim(self) -> None:
|
||||
assert _escape_cell("007") == "007"
|
||||
assert _escape_cell("1.50") == "1.50"
|
||||
with pytest.raises(ManifestError):
|
||||
_load(tmp_path, manifest)
|
||||
|
||||
|
||||
class TestSecurityFrame:
|
||||
|
|
@ -114,7 +113,7 @@ class TestSecurityFrame:
|
|||
manifest = _valid()
|
||||
manifest["extractions"][0]["max_rows"] = 1
|
||||
case = _write_case(tmp_path, manifest, {"e.csv": "col\n1\n2\n"}) # 2 data rows > 1
|
||||
with pytest.raises(ValueError, match="max_rows"):
|
||||
with pytest.raises(SourceError, match="max_rows"):
|
||||
materialize(case / "manifest.json", tmp_path / "bundle", INGESTED_AT)
|
||||
|
||||
def test_query_escaping_root_is_refused(self, tmp_path: Path) -> None:
|
||||
|
|
@ -122,7 +121,7 @@ class TestSecurityFrame:
|
|||
manifest["extractions"][0]["query"] = "../secret.csv"
|
||||
case = _write_case(tmp_path, manifest, {"e.csv": "col\n1\n"})
|
||||
(case / "secret.csv").write_text("col\nx\n", encoding="utf-8")
|
||||
with pytest.raises(ValueError, match="escapes"):
|
||||
with pytest.raises(SourceError, match="escapes"):
|
||||
materialize(case / "manifest.json", tmp_path / "bundle", INGESTED_AT)
|
||||
|
||||
def test_collision_with_non_ingest_file_fails(self, tmp_path: Path) -> None:
|
||||
|
|
@ -133,7 +132,7 @@ class TestSecurityFrame:
|
|||
(bundle / "ingest-e.md").write_text(
|
||||
"---\ntype: reference\ntitle: hand\n---\n\nCurated.\n", encoding="utf-8"
|
||||
)
|
||||
with pytest.raises(ValueError, match="collides"):
|
||||
with pytest.raises(MaterializationError, match="collides"):
|
||||
materialize(case / "manifest.json", bundle, INGESTED_AT)
|
||||
# The curated file is untouched — never overwritten.
|
||||
assert "Curated." in (bundle / "ingest-e.md").read_text(encoding="utf-8")
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue