portfolio-optimiser/tests/test_ingest_manifest.py

"""I2 Step 1 tests — fail-fast ingest-manifest contract (spec §4, verdict reservation §3).

Every malformed manifest raises at validation, BEFORE any source access (the startup-contract
discipline of method spec §10 / ingest spec §4 and §9's technical gate). The verdict-layer
reservation (okf_type != verdict, case-insensitive) is enforced here — at the contract, never
downstream — closing session-plan key assumption 2 (fail-fast manifest validation without
network).

Since the ``llm-ingestion-okf`` adoption (2026-07-20) the contract is enforced by the shared
library rather than by repo-local pydantic models. The INVARIANTS are unchanged — every
malformation rejected before was verified to still be rejected — but the assertion vehicle
moved: ``ManifestV1.model_validate(dict)`` → ``load_manifest_bytes(bytes)``, and
``pydantic.ValidationError`` → ``ManifestError``. The library has zero runtime dependencies
BY DESIGN, so pydantic is not available to it. These tests now also pin the refusal ``code``,
which is the library's documented stability contract (the message text explicitly is not) —
a strictly sharper assertion than "some validation error was raised".
"""

import copy
import hashlib
import json
from pathlib import Path
from typing import Any

import pytest
from llm_ingestion_okf import FileSource, HttpSource, ManifestError, SqlSource
from llm_ingestion_okf.manifest import load_manifest_bytes

from portfolio_optimiser.ingest import ManifestV1, load_manifest

_MANIFEST: dict[str, Any] = {
    "manifest_version": 1,
    "source": {"type": "file", "id": "prosjekt-arkiv", "root": "fixture"},
    "bundle_summary": "Cost extracts from the project archive.",
    "extractions": [
        {
            "id": "costs",
            "title": "Project costs",
            "query": "costs.csv",
            "okf_type": "dataset",
            "max_rows": 100,
        },
        {
            "id": "meta",
            "title": "Catalogue metadata",
            "query": "meta.csv",
            "okf_type": "reference",
            "max_rows": 10,
        },
    ],
}


def _write(tmp_path: Path, data: dict[str, Any], name: str = "manifest.json") -> Path:
    path = tmp_path / name
    path.write_text(json.dumps(data), encoding="utf-8")
    return path


def _variant(**overrides: Any) -> dict[str, Any]:
    data = copy.deepcopy(_MANIFEST)
    data.update(overrides)
    return data


def _validate(data: dict[str, Any]) -> ManifestV1:
    """In-memory validation without touching disk — the library's equivalent of the pydantic
    ``model_validate`` these tests used before the adoption."""
    return load_manifest_bytes(json.dumps(data).encode("utf-8"))


def test_valid_file_manifest_loads_with_stamp(tmp_path: Path) -> None:
    path = _write(tmp_path, _MANIFEST)
    manifest, stamp = load_manifest(path)
    assert isinstance(manifest, ManifestV1)
    # The library's source models drop the `type` discriminator as a FIELD (it is consumed by
    # validation dispatch), so the variant is identified by class rather than by `.type`.
    assert isinstance(manifest.source, FileSource)
    assert [e.id for e in manifest.extractions] == ["costs", "meta"]
    # §5: stamp = {stem}@{first 16 hex of SHA-256 over the manifest file's RAW bytes}.
    expected = "manifest@" + hashlib.sha256(path.read_bytes()).hexdigest()[:16]
    assert stamp == expected


def test_each_missing_top_level_field_raises(tmp_path: Path) -> None:
    for field in ("manifest_version", "source", "bundle_summary", "extractions"):
        data = copy.deepcopy(_MANIFEST)
        del data[field]
        with pytest.raises(ManifestError):
            load_manifest(_write(tmp_path, data, name=f"missing-{field}.json"))


def test_manifest_version_other_than_1_rejected(tmp_path: Path) -> None:
    with pytest.raises(ManifestError) as exc:
        load_manifest(_write(tmp_path, _variant(manifest_version=2)))
    assert exc.value.code == "manifest_version_unsupported"


def test_empty_extractions_rejected(tmp_path: Path) -> None:
    with pytest.raises(ManifestError):
        load_manifest(_write(tmp_path, _variant(extractions=[])))


def test_bad_id_grammar_rejected() -> None:
    # §4 grammar for source.id and extraction.id: ^[a-z0-9][a-z0-9-]*$
    for bad in ("Upper", "-leading", "", "space id", "æøå"):
        data = copy.deepcopy(_MANIFEST)
        data["source"]["id"] = bad
        with pytest.raises(ManifestError):
            _validate(data)
        data = copy.deepcopy(_MANIFEST)
        data["extractions"][0]["id"] = bad
        with pytest.raises(ManifestError):
            _validate(data)


def test_duplicate_extraction_ids_rejected(tmp_path: Path) -> None:
    data = copy.deepcopy(_MANIFEST)
    data["extractions"][1]["id"] = data["extractions"][0]["id"]
    with pytest.raises(ManifestError) as exc:
        load_manifest(_write(tmp_path, data))
    assert exc.value.code == "extraction_id_duplicate"


def test_nonpositive_max_rows_rejected(tmp_path: Path) -> None:
    for bad in (0, -1):
        data = copy.deepcopy(_MANIFEST)
        data["extractions"][0]["max_rows"] = bad
        with pytest.raises(ManifestError):
            load_manifest(_write(tmp_path, data, name=f"rows-{bad}.json"))


def test_unknown_source_type_rejected(tmp_path: Path) -> None:
    data = copy.deepcopy(_MANIFEST)
    data["source"] = {"type": "ftp", "id": "x", "root": "fixture"}
    with pytest.raises(ManifestError) as exc:
        load_manifest(_write(tmp_path, data))
    assert exc.value.code == "source_type_unknown"


def test_file_source_missing_root_rejected(tmp_path: Path) -> None:
    data = copy.deepcopy(_MANIFEST)
    del data["source"]["root"]
    with pytest.raises(ManifestError):
        load_manifest(_write(tmp_path, data))


def test_verdict_okf_type_rejected_case_insensitively(tmp_path: Path) -> None:
    # §3: the verdict layer is RESERVED — enforced at manifest validation, before any
    # source call. The manifest is otherwise fully valid, so this validator is the ONLY
    # thing standing between an ingest manifest and machine-generated "approved" verdicts.
    for spelling in ("verdict", "Verdict", "VERDICT"):
        data = copy.deepcopy(_MANIFEST)
        data["extractions"][0]["okf_type"] = spelling
        with pytest.raises(ManifestError) as exc:
            load_manifest(_write(tmp_path, data, name=f"verdict-{spelling}.json"))
        assert exc.value.code == "okf_type_reserved"


def test_multiline_title_rejected() -> None:
    for bad in ("line1\nline2", "line1\rline2", ""):
        data = copy.deepcopy(_MANIFEST)
        data["extractions"][0]["title"] = bad
        with pytest.raises(ManifestError):
            _validate(data)


def test_title_whitespace_is_preserved_verbatim() -> None:
    """ACCEPTED DIVERGENCE (2026-07-20), recorded rather than silently dropped.

    The repo previously COLLAPSED title whitespace runs at validation, so the frontmatter
    title and the index label were guaranteed byte-identical. The library stores the title
    verbatim instead: ``_render_frontmatter`` still collapses runs when writing frontmatter,
    but the §6 index label is written raw — so for a title with irregular internal whitespace
    the two now differ. Both behaviours are spec-conformant: ``shared/ingest-spec.md`` is
    SILENT on normalization, and the old behaviour was a repo-local pinned decision, not a
    spec requirement.

    This test pins the CURRENT behaviour so the divergence cannot drift unnoticed. It goes RED
    if the library starts normalizing — which is the desired end state, and is why the point is
    queued as a commons-amendment candidate so both stacks pin the same answer in the spec.
    The golden bundles are unaffected (their titles carry no irregular whitespace)."""
    data = copy.deepcopy(_MANIFEST)
    data["extractions"][0]["title"] = "  Project   costs  "
    manifest = _validate(data)
    assert manifest.extractions[0].title == "  Project   costs  "


def test_base_url_embedded_credentials_rejected() -> None:
    # §4: base_url MUST NOT embed credentials (userinfo is the URL credential mechanism).
    data = copy.deepcopy(_MANIFEST)
    data["source"] = {"type": "http", "id": "api", "base_url": "https://user:pw@host/api"}
    with pytest.raises(ManifestError) as exc:
        _validate(data)
    assert exc.value.code == "credential_embedded"


def test_sql_and_http_variants_validate() -> None:
    # Schema breadth (brief assumption 1): the polymorphic §4 schema validates all three
    # source variants; the `file` and `sql` connectors EXECUTE (I2, I4), http → I6.
    sql = _validate(_variant(source={"type": "sql", "id": "db", "connection_ref": "PROJ_DB"}))
    assert isinstance(sql.source, SqlSource)
    assert sql.source.connection_ref == "PROJ_DB"
    http = _validate(_variant(source={"type": "http", "id": "api", "base_url": "https://host/api"}))
    assert isinstance(http.source, HttpSource)
    assert http.source.credential_ref is None


def test_failfast_before_source_access(tmp_path: Path) -> None:
    # Ordering proof (closes key assumption 2): the manifest is malformed on max_rows AND
    # points at a root that does not exist — validation raises WITHOUT touching the root
    # (load_manifest reads only the manifest file; the ingest analogue of
    # test_no_chat_client_call_on_malformed_contract).
    data = copy.deepcopy(_MANIFEST)
    data["source"]["root"] = str(tmp_path / "does-not-exist")
    data["extractions"][0]["max_rows"] = 0
    with pytest.raises(ManifestError):
        load_manifest(_write(tmp_path, data))
    assert not (tmp_path / "does-not-exist").exists()


def test_malformed_json_raises(tmp_path: Path) -> None:
    # The library wraps json decoding so EVERY manifest problem surfaces as one typed family
    # (previously this leaked a raw json.JSONDecodeError to the caller).
    path = tmp_path / "broken.json"
    path.write_text("{not json", encoding="utf-8")
    with pytest.raises(ManifestError) as exc:
        load_manifest(path)
    assert exc.value.code == "manifest_invalid_json"