portfolio-optimiser/tests/test_costsim_loadbearing.py

"""S3.6 costsim (kostnadssimulering) — offline, MAF-free, deterministic load-bearing seams.

``costsim`` estimates token/cost for a portfolio run BEFORE running (what-if over the model-map ×
effort levels), with pricing as schema-validated config. It is MAF-free (reads ``model_map.json`` as
plain data via ``__file__``-relative path, never imports ``backends``/``budget``/``contracts``) —
proven load-bearing by BOTH the direct-import AST guard (``test_okf_is_maf_free`` covering
``costsim.py``) AND the transitive import-graph seam below.
"""

from __future__ import annotations

import subprocess
import sys
from pathlib import Path

import pytest

from portfolio_optimiser import costsim

_COSTSIM = Path(__file__).resolve().parents[1] / "src" / "portfolio_optimiser" / "costsim.py"

_PRICING = {
    "source": "loadbearing-test prices",
    "date": "2026-07-15",
    "models": {
        "m-known": {
            "ore_per_1k_tokens": 20,
            "quality_guidance": {"note": "known model", "source": "test-src"},
        }
    },
}

_TWO_MODEL_PRICING = {
    "source": "loadbearing-test prices",
    "date": "2026-07-15",
    "models": {
        "m-cheap": {"ore_per_1k_tokens": 10},
        "m-dear": {"ore_per_1k_tokens": 50},
    },
}


def test_costsim_registered_maf_free() -> None:
    """Meta: costsim.py is registered in the MAF-free guard list, so ``test_okf_is_maf_free``
    actually scans it — otherwise the MAF-free claim would be green-but-dead (never checked)."""
    from tests.test_okf import _MAF_FREE_MODULES

    assert "costsim.py" in _MAF_FREE_MODULES


def test_costsim_import_is_maf_free() -> None:
    """The genuine transitive-import seam: loading costsim.py's module body in ISOLATION must NOT
    pull ``agent_framework``/``mcp`` into ``sys.modules``. Loaded standalone via
    ``spec_from_file_location`` — NOT as ``portfolio_optimiser.costsim``, since the package
    ``__init__`` imports MAF-bound ``run``/``contracts`` and would mask costsim's own cleanliness.
    The AST guard (``test_okf_is_maf_free``) only catches DIRECT imports; a transitive
    ``from portfolio_optimiser.contracts import ...`` in costsim would slip past it but trip THIS
    check — ``exec_module`` runs costsim's import statements, so any MAF-bound import lands
    ``agent_framework`` in ``sys.modules``. Detach point: add such an import → RED."""
    check = (
        "import importlib.util, sys\n"
        f"spec = importlib.util.spec_from_file_location('costsim_standalone', {str(_COSTSIM)!r})\n"
        "mod = importlib.util.module_from_spec(spec)\n"
        "spec.loader.exec_module(mod)\n"
        "assert 'agent_framework' not in sys.modules, 'agent_framework leaked into costsim import graph'\n"
        "assert 'mcp' not in sys.modules, 'mcp leaked into costsim import graph'\n"
    )
    result = subprocess.run([sys.executable, "-c", check], capture_output=True, text=True)
    assert result.returncode == 0, result.stderr


def test_missing_price_for_model_fails_fast() -> None:
    """SC1: a genuine (non-placeholder) model absent from the price map fails fast. Detach point:
    remove the presence-check raise in ``_price_ore_per_1k`` → a guessed/0 price is returned → RED."""
    pricing = costsim.PricingContract(**_PRICING)
    with pytest.raises(ValueError, match="missing price for"):
        costsim._price_ore_per_1k("m-unknown", pricing)


def test_priced_model_returns_ore_rate() -> None:
    """Control for the fail-fast seam: a priced model returns its øre rate (proves the raise fires
    only on genuine absence, not always)."""
    pricing = costsim.PricingContract(**_PRICING)
    assert costsim._price_ore_per_1k("m-known", pricing) == 20


def test_placeholder_model_is_unpriced() -> None:
    """The defined azure-placeholder path: a ``REPLACE-WITH-*`` deployment id returns ``None`` (not
    priced, not a crash) — so ``--profile azure`` (all placeholders) is well-defined."""
    pricing = costsim.PricingContract(**_PRICING)
    assert costsim._price_ore_per_1k("REPLACE-WITH-FOUNDRY-DEPLOYMENT", pricing) is None


def test_quality_guidance_is_sourced_never_number() -> None:
    """Brief Goal 4: per-model quality trade-offs are sourced GUIDANCE (prose note + provenance
    source), never a bare measured number. Detach point: ship a guidance entry without a source →
    the ``QualityGuidance.source`` (min_length=1) requirement makes load RED."""
    pricing = costsim.load_pricing()
    guided = [p for p in pricing.models.values() if p.quality_guidance is not None]
    assert guided, "bundled pricing must carry per-model quality guidance"
    for price in guided:
        assert price.quality_guidance is not None
        assert price.quality_guidance.source.strip()
        assert price.quality_guidance.note.strip()


def test_estimate_is_deterministic() -> None:
    """SC2 (determinism): identical inputs → identical integer-øre estimate, reproducibly."""
    pricing = costsim.PricingContract(**_PRICING)
    a = costsim.estimate_portfolio_ore("m-known", "high", pricing, n_projects=4)
    b = costsim.estimate_portfolio_ore("m-known", "high", pricing, n_projects=4)
    assert a == b


def test_estimate_scales_with_model_and_effort() -> None:
    """SC2 (scaling): the estimate varies with BOTH model and effort. Detach point: drop the
    ``EFFORT_FACTORS[effort]`` term (e.g. hard-code 100%) → low and high collapse to equal → the
    ``low != high`` assertion goes RED. Control: same model + same effort → identical (isolates the
    effort variable, so a pass cannot be incidental)."""
    pricing = costsim.PricingContract(**_TWO_MODEL_PRICING)
    low = costsim.estimate_run_ore("m-cheap", "low", pricing)
    high = costsim.estimate_run_ore("m-cheap", "high", pricing)
    dear_high = costsim.estimate_run_ore("m-dear", "high", pricing)
    assert low != high, "effort must scale the estimate"
    assert dear_high != high, "model must scale the estimate"
    # control: same model + same effort → identical (not incidental)
    assert costsim.estimate_run_ore("m-cheap", "high", pricing) == high