portfolio-optimiser-claude/tests/test_costsim.py

"""Pre-run cost simulation (K6, S3.6-analog, D-I pkt. 3; paritetsrad 18).

The operator sees an ESTIMATED upper-bound USD cost for a (portfolio-)run BEFORE
any spend — a what-if over the models in model_map.json × effort levels. Every
seam here is load-bearing (§11): the RED-when-detached notes on each test name
the mutation that makes it fail, so a green-but-dead test can't hide.

Nothing here touches the network or a key (offline invariant): the estimate is
pure config arithmetic over schema-validated pricing config.
"""

from __future__ import annotations

import ast
import json
from pathlib import Path

import pytest
from pydantic import ValidationError

from portfolio_optimiser_claude.contracts import (
    ModelMapContract,
    ModelPriceContract,
    PricingContract,
    load_pricing,
)
from portfolio_optimiser_claude.costsim import estimate_costs, main

SRC_PKG = Path(__file__).resolve().parents[1] / "src" / "portfolio_optimiser_claude"


def _model_map(*model_ids: str) -> ModelMapContract:
    # Every profile needs a 'default' (§10); role split is irrelevant to the
    # what-if, which iterates the DISTINCT model ids configured anywhere.
    mapping = {"default": model_ids[0]}
    for i, mid in enumerate(model_ids[1:], start=1):
        mapping[f"role{i}"] = mid
    return ModelMapContract(profiles={"anthropic": mapping})


def _pricing(**rates: float) -> PricingContract:
    return PricingContract(
        prices={
            mid: ModelPriceContract(usd_per_mtok=rate, source="src", source_date="2026-01-01")
            for mid, rate in rates.items()
        }
    )


class TestPricingSchemaFailsFast:
    """§10: a malformed/missing price is a startup schema error, never a guessed rate."""

    def test_non_positive_price_is_rejected(self) -> None:
        with pytest.raises(ValidationError):
            ModelPriceContract(usd_per_mtok=0.0, source="src", source_date="2026-01-01")

    def test_a_price_entry_without_a_rate_is_rejected(self) -> None:
        with pytest.raises(ValidationError):
            PricingContract(prices={"m": {"source": "src", "source_date": "2026-01-01"}})

    def test_source_and_date_are_required(self) -> None:
        # A price with no provenance would let a stale rate pass silently (§1).
        with pytest.raises(ValidationError):
            PricingContract(prices={"m": {"usd_per_mtok": 5.0}})

    def test_empty_pricing_is_rejected(self) -> None:
        with pytest.raises(ValidationError):
            PricingContract(prices={})


class TestMissingPriceFailsFast:
    """RED (detach the price lookup guard → KeyError not ValueError): a model in
    model_map with NO price stops the estimate cold — never a guessed rate."""

    def test_a_configured_model_without_a_price_raises_named(self) -> None:
        model_map = _model_map("m-cheap", "m-unpriced")
        pricing = _pricing(**{"m-cheap": 1.0})  # m-unpriced deliberately absent
        with pytest.raises(ValueError, match="missing price for m-unpriced"):
            estimate_costs(model_map, pricing, n_projects=1, token_cap_per_project=1000)


class TestEstimateScalesAndIsDeterministic:
    """RED (detach the effort factor → efforts identical): the estimate scales
    with BOTH model price and effort, and is byte-reproducible (no clock/random)."""

    def test_cost_scales_linearly_with_model_price(self) -> None:
        model_map = _model_map("m-cheap", "m-pricey")
        pricing = _pricing(**{"m-cheap": 1.0, "m-pricey": 10.0})
        est = estimate_costs(model_map, pricing, n_projects=3, token_cap_per_project=1000)
        by_id = {m.model_id: m for m in est.models}
        # Same portfolio, same effort → 10x price ⇒ 10x cost, per effort level.
        for effort in ("low", "high", "max"):
            cheap = next(e.cost_usd for e in by_id["m-cheap"].efforts if e.effort == effort)
            pricey = next(e.cost_usd for e in by_id["m-pricey"].efforts if e.effort == effort)
            assert pricey == pytest.approx(cheap * 10.0)

    def test_cost_scales_strictly_with_effort(self) -> None:
        # DETACH POINT: make every _EFFORT_FACTORS value equal (e.g. all 1.0) and
        # this strict-increase assertion goes RED — the effort axis is dead.
        model_map = _model_map("m")
        pricing = _pricing(**{"m": 5.0})
        est = estimate_costs(model_map, pricing, n_projects=2, token_cap_per_project=1000)
        costs = [e.cost_usd for e in est.models[0].efforts]
        assert costs == sorted(costs)
        assert costs[0] < costs[-1]

    def test_estimate_is_reproducible(self) -> None:
        model_map = _model_map("m")
        pricing = _pricing(**{"m": 7.0})
        a = estimate_costs(model_map, pricing, n_projects=4, token_cap_per_project=1234)
        b = estimate_costs(model_map, pricing, n_projects=4, token_cap_per_project=1234)
        assert [(m.model_id, [(e.effort, e.cost_usd) for e in m.efforts]) for m in a.models] == [
            (m.model_id, [(e.effort, e.cost_usd) for e in m.efforts]) for m in b.models
        ]

    def test_non_positive_shape_is_rejected(self) -> None:
        pricing = _pricing(**{"m": 5.0})
        with pytest.raises(ValueError):
            estimate_costs(_model_map("m"), pricing, n_projects=0, token_cap_per_project=1000)
        with pytest.raises(ValueError):
            estimate_costs(_model_map("m"), pricing, n_projects=1, token_cap_per_project=0)


class TestNoPriceLiteralInSource:
    """RED (hardcode any example rate in costsim.py): the grep-guard proves the
    price MUST come from config — no rate literal lives in the module source."""

    def test_no_example_price_appears_as_a_literal(self) -> None:
        source = (SRC_PKG / "costsim.py").read_text(encoding="utf-8")
        pricing = load_pricing()  # the bundled data/pricing.example.json
        # Positive controls: an empty price config would make the loop below iterate
        # zero times, and an unreadable/empty source would make every substring miss —
        # both green without guarding anything.
        assert pricing.prices
        assert "usd_per_mtok" in source
        for model_id, price in pricing.prices.items():
            assert str(price.usd_per_mtok) not in source, (
                f"price for {model_id} is hardcoded in costsim.py — must come from config"
            )


def _imported_module_names(module_path: Path) -> set[str]:
    tree = ast.parse(module_path.read_text(encoding="utf-8"))
    names: set[str] = set()
    for node in ast.walk(tree):
        if isinstance(node, ast.Import):
            names.update(alias.name.split(".")[0] for alias in node.names)
        elif isinstance(node, ast.ImportFrom) and node.level == 0 and node.module:
            names.add(node.module.split(".")[0])
    return names


class TestCostsimPurity:
    """LOAD-BEARING (§11): the cost sim imports no agent toolkit — offline by design."""

    def test_costsim_never_imports_an_agent_toolkit(self) -> None:
        names = _imported_module_names(SRC_PKG / "costsim.py")
        assert not names & {"claude_agent_sdk", "anthropic"}


class TestBundledExampleAndCli:
    """Nøkkelantakelse: the bundled example prices COVER the model model_map
    configures, so the offline default path runs green — and the CLI is offline."""

    def test_bundled_pricing_validates(self) -> None:
        pricing = load_pricing()
        assert pricing.prices  # non-empty, schema-valid
        for price in pricing.prices.values():
            assert price.usd_per_mtok > 0
            assert price.source and price.source_date

    def test_the_default_estimate_runs_on_bundled_config(self) -> None:
        # The bundled model_map's model must have a bundled price — else the
        # default CLI would fail-fast. This binds the two example files together.
        from portfolio_optimiser_claude.contracts import _bundled_model_map

        est = estimate_costs(
            ModelMapContract(**_bundled_model_map()),
            load_pricing(),
            n_projects=1,
            token_cap_per_project=150_000,
        )
        assert est.models
        assert all(m.efforts for m in est.models)

    def test_cli_prints_an_estimate_marked_estimat(
        self, capsys: pytest.CaptureFixture[str]
    ) -> None:
        rc = main([])
        out = capsys.readouterr().out
        assert rc == 0
        assert "ESTIMAT" in out

    def test_cli_honours_explicit_pricing_file(
        self, tmp_path: Path, capsys: pytest.CaptureFixture[str]
    ) -> None:
        from portfolio_optimiser_claude.contracts import _bundled_model_map

        model_id = ModelMapContract(**_bundled_model_map()).profiles["anthropic"]["default"]
        pricing_file = tmp_path / "pricing.json"
        pricing_file.write_text(
            json.dumps(
                {
                    "prices": {
                        model_id: {
                            "usd_per_mtok": 3.0,
                            "source": "src",
                            "source_date": "2026-01-01",
                        }
                    }
                }
            ),
            encoding="utf-8",
        )
        rc = main(["--projects", "2", "--token-cap", "1000", "--pricing", str(pricing_file)])
        out = capsys.readouterr().out
        assert rc == 0
        assert model_id in out


def test_no_network_import_in_costsim() -> None:
    # Belt on the offline invariant: not even the stdlib socket/urllib slip in.
    names = _imported_module_names(SRC_PKG / "costsim.py")
    assert not names & {"socket", "urllib", "http", "requests", "httpx"}