"""S3.6 costsim (kostnadssimulering) — offline, MAF-free, deterministic load-bearing seams. ``costsim`` estimates token/cost for a portfolio run BEFORE running (what-if over the model-map × effort levels), with pricing as schema-validated config. It is MAF-free (reads ``model_map.json`` as plain data via ``__file__``-relative path, never imports ``backends``/``budget``/``contracts``) — proven load-bearing by BOTH the direct-import AST guard (``test_okf_is_maf_free`` covering ``costsim.py``) AND the transitive import-graph seam below. """ from __future__ import annotations import subprocess import sys from pathlib import Path import pytest from portfolio_optimiser import costsim _COSTSIM = Path(__file__).resolve().parents[1] / "src" / "portfolio_optimiser" / "costsim.py" _PRICING = { "source": "loadbearing-test prices", "date": "2026-07-15", "models": { "m-known": { "ore_per_1k_tokens": 20, "quality_guidance": {"note": "known model", "source": "test-src"}, } }, } _TWO_MODEL_PRICING = { "source": "loadbearing-test prices", "date": "2026-07-15", "models": { "m-cheap": {"ore_per_1k_tokens": 10}, "m-dear": {"ore_per_1k_tokens": 50}, }, } def test_costsim_registered_maf_free() -> None: """Meta: costsim.py is registered in the MAF-free guard list, so ``test_okf_is_maf_free`` actually scans it — otherwise the MAF-free claim would be green-but-dead (never checked).""" from tests.test_okf import _MAF_FREE_MODULES assert "costsim.py" in _MAF_FREE_MODULES def test_costsim_import_is_maf_free() -> None: """The genuine transitive-import seam: loading costsim.py's module body in ISOLATION must NOT pull ``agent_framework``/``mcp`` into ``sys.modules``. Loaded standalone via ``spec_from_file_location`` — NOT as ``portfolio_optimiser.costsim``, since the package ``__init__`` imports MAF-bound ``run``/``contracts`` and would mask costsim's own cleanliness. The AST guard (``test_okf_is_maf_free``) only catches DIRECT imports; a transitive ``from portfolio_optimiser.contracts import ...`` in costsim would slip past it but trip THIS check — ``exec_module`` runs costsim's import statements, so any MAF-bound import lands ``agent_framework`` in ``sys.modules``. Detach point: add such an import → RED.""" check = ( "import importlib.util, sys\n" f"spec = importlib.util.spec_from_file_location('costsim_standalone', {str(_COSTSIM)!r})\n" "mod = importlib.util.module_from_spec(spec)\n" "spec.loader.exec_module(mod)\n" "assert 'agent_framework' not in sys.modules, 'agent_framework leaked into costsim import graph'\n" "assert 'mcp' not in sys.modules, 'mcp leaked into costsim import graph'\n" ) result = subprocess.run([sys.executable, "-c", check], capture_output=True, text=True) assert result.returncode == 0, result.stderr def test_missing_price_for_model_fails_fast() -> None: """SC1: a genuine (non-placeholder) model absent from the price map fails fast. Detach point: remove the presence-check raise in ``_price_ore_per_1k`` → a guessed/0 price is returned → RED.""" pricing = costsim.PricingContract(**_PRICING) with pytest.raises(ValueError, match="missing price for"): costsim._price_ore_per_1k("m-unknown", pricing) def test_priced_model_returns_ore_rate() -> None: """Control for the fail-fast seam: a priced model returns its øre rate (proves the raise fires only on genuine absence, not always).""" pricing = costsim.PricingContract(**_PRICING) assert costsim._price_ore_per_1k("m-known", pricing) == 20 def test_placeholder_model_is_unpriced() -> None: """The defined azure-placeholder path: a ``REPLACE-WITH-*`` deployment id returns ``None`` (not priced, not a crash) — so ``--profile azure`` (all placeholders) is well-defined.""" pricing = costsim.PricingContract(**_PRICING) assert costsim._price_ore_per_1k("REPLACE-WITH-FOUNDRY-DEPLOYMENT", pricing) is None def test_quality_guidance_is_sourced_never_number() -> None: """Brief Goal 4: per-model quality trade-offs are sourced GUIDANCE (prose note + provenance source), never a bare measured number. Detach point: ship a guidance entry without a source → the ``QualityGuidance.source`` (min_length=1) requirement makes load RED.""" pricing = costsim.load_pricing() guided = [p for p in pricing.models.values() if p.quality_guidance is not None] assert guided, "bundled pricing must carry per-model quality guidance" for price in guided: assert price.quality_guidance is not None assert price.quality_guidance.source.strip() assert price.quality_guidance.note.strip() def test_estimate_is_deterministic() -> None: """SC2 (determinism): identical inputs → identical integer-øre estimate, reproducibly.""" pricing = costsim.PricingContract(**_PRICING) a = costsim.estimate_portfolio_ore("m-known", "high", pricing, n_projects=4) b = costsim.estimate_portfolio_ore("m-known", "high", pricing, n_projects=4) assert a == b def test_estimate_scales_with_model_and_effort() -> None: """SC2 (scaling): the estimate varies with BOTH model and effort. Detach point: drop the ``EFFORT_FACTORS[effort]`` term (e.g. hard-code 100%) → low and high collapse to equal → the ``low != high`` assertion goes RED. Control: same model + same effort → identical (isolates the effort variable, so a pass cannot be incidental).""" pricing = costsim.PricingContract(**_TWO_MODEL_PRICING) low = costsim.estimate_run_ore("m-cheap", "low", pricing) high = costsim.estimate_run_ore("m-cheap", "high", pricing) dear_high = costsim.estimate_run_ore("m-dear", "high", pricing) assert low != high, "effort must scale the estimate" assert dear_high != high, "model must scale the estimate" # control: same model + same effort → identical (not incidental) assert costsim.estimate_run_ore("m-cheap", "high", pricing) == high