portfolio-optimiser-claude/tests/test_ingest.py

"""Ingest unit + fail-fast contract tests (ingest-spec §4, §5, §8).

The manifest is schema-validated fail-fast BEFORE any source call (§4, the startup-contract
discipline). These tests pin the malformed-manifest rejections, the §8 size cap, and the
fail-closed path resolution / no-overwrite collision — all through the consumer seam
(``portfolio_optimiser_claude.ingest``, backed by llm-ingestion-okf since the adoption).

The §5 cell-escaping rules are NOT unit-tested here anymore: every escaping case
(backslash, pipe, backslash-then-pipe order, newline collapse, verbatim text) is bound
byte-for-byte by the ``ingest-edge.md`` golden (test_ingest_golden.py) and unit-owned by
the library's own suite.
"""

from __future__ import annotations

import json
from pathlib import Path

import pytest

from portfolio_optimiser_claude.ingest import (
    ManifestError,
    MaterializationError,
    SourceError,
    load_manifest,
    materialize,
)

GOLDEN = Path(__file__).resolve().parents[1] / "examples" / "ingest-golden-file"
INGESTED_AT = "2026-07-03T12:00:00Z"


def _valid() -> dict:
    return {
        "manifest_version": 1,
        "source": {"type": "file", "id": "arkiv", "root": "fixture"},
        "bundle_summary": "s",
        "extractions": [
            {"id": "e", "title": "T", "query": "e.csv", "okf_type": "dataset", "max_rows": 5}
        ],
    }


def _write_case(tmp_path: Path, manifest: dict, csvs: dict[str, str]) -> Path:
    case = tmp_path / "case"
    (case / "fixture").mkdir(parents=True)
    (case / "manifest.json").write_text(json.dumps(manifest), encoding="utf-8")
    for name, text in csvs.items():
        (case / "fixture" / name).write_text(text, encoding="utf-8")
    return case


def _load(tmp_path: Path, manifest: dict):
    path = tmp_path / "manifest.json"
    path.write_text(json.dumps(manifest), encoding="utf-8")
    return load_manifest(path)


class TestManifestValidation:
    """§4: a malformed manifest never starts a run (fail-fast)."""

    def test_valid_manifest_loads(self, tmp_path: Path) -> None:
        manifest = _load(tmp_path, _valid())
        assert manifest.manifest_version == 1
        assert manifest.source.id == "arkiv"

    def test_stamp_is_stem_at_sha256_16(self, tmp_path: Path) -> None:
        # The §5 provenance stamp (``{stem}@{sha256(raw)[:16]}``) is asserted from the
        # materialized frontmatter — the stamp is a property of the run, not the manifest.
        case = _write_case(tmp_path, _valid(), {"e.csv": "a\n1\n"})
        bundle = tmp_path / "bundle"
        materialize(case / "manifest.json", bundle, INGESTED_AT)
        frontmatter = (bundle / "ingest-e.md").read_text(encoding="utf-8").splitlines()
        (stamp_line,) = [ln for ln in frontmatter if ln.startswith("ingest_manifest: ")]
        stem, _, digest = stamp_line.removeprefix("ingest_manifest: ").partition("@")
        assert stem == "manifest"
        assert len(digest) == 16 and all(c in "0123456789abcdef" for c in digest)

    @pytest.mark.parametrize(
        "mutate",
        [
            lambda m: m.pop("manifest_version"),
            lambda m: m.__setitem__("manifest_version", 2),
            lambda m: m.pop("source"),
            lambda m: m.pop("bundle_summary"),
            lambda m: m.__setitem__("extractions", []),
            lambda m: m["source"].__setitem__("id", "Bad_Id"),
            lambda m: m["source"].__setitem__("type", "http"),  # http requires base_url (§4)
            lambda m: m["source"].__setitem__("type", "unknown"),  # bad discriminator
            lambda m: m["extractions"][0].__setitem__("id", "Bad Id"),
            lambda m: m["extractions"][0].__setitem__("title", "two\nlines"),
            lambda m: m["extractions"][0].__setitem__("max_rows", 0),
            lambda m: m["extractions"][0].__setitem__("max_rows", -1),
        ],
    )
    def test_malformed_manifest_is_rejected(self, tmp_path: Path, mutate) -> None:
        manifest = _valid()
        mutate(manifest)
        with pytest.raises(ManifestError):
            _load(tmp_path, manifest)

    def test_duplicate_extraction_ids_are_rejected(self, tmp_path: Path) -> None:
        manifest = _valid()
        manifest["extractions"].append(dict(manifest["extractions"][0]))
        with pytest.raises(ManifestError):
            _load(tmp_path, manifest)


class TestSecurityFrame:
    """§8: size cap fail-fast, path resolution fail-closed, curated never overwritten."""

    def test_extraction_exceeding_max_rows_fails(self, tmp_path: Path) -> None:
        manifest = _valid()
        manifest["extractions"][0]["max_rows"] = 1
        case = _write_case(tmp_path, manifest, {"e.csv": "col\n1\n2\n"})  # 2 data rows > 1
        with pytest.raises(SourceError, match="max_rows"):
            materialize(case / "manifest.json", tmp_path / "bundle", INGESTED_AT)

    def test_query_escaping_root_is_refused(self, tmp_path: Path) -> None:
        manifest = _valid()
        manifest["extractions"][0]["query"] = "../secret.csv"
        case = _write_case(tmp_path, manifest, {"e.csv": "col\n1\n"})
        (case / "secret.csv").write_text("col\nx\n", encoding="utf-8")
        with pytest.raises(SourceError, match="escapes"):
            materialize(case / "manifest.json", tmp_path / "bundle", INGESTED_AT)

    def test_collision_with_non_ingest_file_fails(self, tmp_path: Path) -> None:
        case = _write_case(tmp_path, _valid(), {"e.csv": "col\n1\n"})
        bundle = tmp_path / "bundle"
        bundle.mkdir()
        # A curated (non-stamped) file already occupies the generated name.
        (bundle / "ingest-e.md").write_text(
            "---\ntype: reference\ntitle: hand\n---\n\nCurated.\n", encoding="utf-8"
        )
        with pytest.raises(MaterializationError, match="collides"):
            materialize(case / "manifest.json", bundle, INGESTED_AT)
        # The curated file is untouched — never overwritten.
        assert "Curated." in (bundle / "ingest-e.md").read_text(encoding="utf-8")