"""S3.6 costsim (kostnadssimulering) — offline, MAF-free, deterministic load-bearing seams. ``costsim`` estimates token/cost for a portfolio run BEFORE running (what-if over the model-map × effort levels), with pricing as schema-validated config. It is MAF-free (reads ``model_map.json`` as plain data via ``__file__``-relative path, never imports ``backends``/``budget``/``contracts``) — proven load-bearing by BOTH the direct-import AST guard (``test_okf_is_maf_free`` covering ``costsim.py``) AND the transitive import-graph seam below. """ from __future__ import annotations import ast import subprocess import sys from pathlib import Path import pytest from portfolio_optimiser import costsim _COSTSIM = Path(__file__).resolve().parents[1] / "src" / "portfolio_optimiser" / "costsim.py" _PRICING = { "source": "loadbearing-test prices", "date": "2026-07-15", "models": { "m-known": { "ore_per_1k_tokens": 20, "quality_guidance": {"note": "known model", "source": "test-src"}, } }, } _TWO_MODEL_PRICING = { "source": "loadbearing-test prices", "date": "2026-07-15", "models": { "m-cheap": {"ore_per_1k_tokens": 10}, "m-dear": {"ore_per_1k_tokens": 50}, }, } def test_costsim_registered_maf_free() -> None: """Meta: costsim.py is registered in the MAF-free guard list, so ``test_okf_is_maf_free`` actually scans it — otherwise the MAF-free claim would be green-but-dead (never checked).""" from tests.test_okf import _MAF_FREE_MODULES assert "costsim.py" in _MAF_FREE_MODULES def test_costsim_import_is_maf_free() -> None: """The genuine transitive-import seam: loading costsim.py's module body in ISOLATION must NOT pull ``agent_framework``/``mcp`` into ``sys.modules``. Loaded standalone via ``spec_from_file_location`` — NOT as ``portfolio_optimiser.costsim``, since the package ``__init__`` imports MAF-bound ``run``/``contracts`` and would mask costsim's own cleanliness. The AST guard (``test_okf_is_maf_free``) only catches DIRECT imports; a transitive ``from portfolio_optimiser.contracts import ...`` in costsim would slip past it but trip THIS check — ``exec_module`` runs costsim's import statements, so any MAF-bound import lands ``agent_framework`` in ``sys.modules``. Detach point: add such an import → RED.""" check = ( "import importlib.util, sys\n" f"spec = importlib.util.spec_from_file_location('costsim_standalone', {str(_COSTSIM)!r})\n" "mod = importlib.util.module_from_spec(spec)\n" "spec.loader.exec_module(mod)\n" "assert 'agent_framework' not in sys.modules, 'agent_framework leaked into costsim import graph'\n" "assert 'mcp' not in sys.modules, 'mcp leaked into costsim import graph'\n" ) result = subprocess.run([sys.executable, "-c", check], capture_output=True, text=True) assert result.returncode == 0, result.stderr def test_missing_price_for_model_fails_fast() -> None: """SC1: a genuine (non-placeholder) model absent from the price map fails fast. Detach point: remove the presence-check raise in ``_price_ore_per_1k`` → a guessed/0 price is returned → RED.""" pricing = costsim.PricingContract(**_PRICING) with pytest.raises(ValueError, match="missing price for"): costsim._price_ore_per_1k("m-unknown", pricing) def test_priced_model_returns_ore_rate() -> None: """Control for the fail-fast seam: a priced model returns its øre rate (proves the raise fires only on genuine absence, not always).""" pricing = costsim.PricingContract(**_PRICING) assert costsim._price_ore_per_1k("m-known", pricing) == 20 def test_placeholder_model_is_unpriced() -> None: """The defined azure-placeholder path: a ``REPLACE-WITH-*`` deployment id returns ``None`` (not priced, not a crash) — so ``--profile azure`` (all placeholders) is well-defined.""" pricing = costsim.PricingContract(**_PRICING) assert costsim._price_ore_per_1k("REPLACE-WITH-FOUNDRY-DEPLOYMENT", pricing) is None def test_quality_guidance_is_sourced_never_number() -> None: """Brief Goal 4: per-model quality trade-offs are sourced GUIDANCE (prose note + provenance source), never a bare measured number. Detach point: ship a guidance entry without a source → the ``QualityGuidance.source`` (min_length=1) requirement makes load RED.""" pricing = costsim.load_pricing() guided = [p for p in pricing.models.values() if p.quality_guidance is not None] assert guided, "bundled pricing must carry per-model quality guidance" for price in guided: assert price.quality_guidance is not None assert price.quality_guidance.source.strip() assert price.quality_guidance.note.strip() def test_estimate_is_deterministic() -> None: """SC2 (determinism): identical inputs → identical integer-øre estimate, reproducibly.""" pricing = costsim.PricingContract(**_PRICING) a = costsim.estimate_portfolio_ore("m-known", "high", pricing, n_projects=4) b = costsim.estimate_portfolio_ore("m-known", "high", pricing, n_projects=4) assert a == b def test_estimate_scales_with_model_and_effort() -> None: """SC2 (scaling): the estimate varies with BOTH model and effort. Detach point: drop the ``EFFORT_FACTORS[effort]`` term (e.g. hard-code 100%) → low and high collapse to equal → the ``low != high`` assertion goes RED. Control: same model + same effort → identical (isolates the effort variable, so a pass cannot be incidental).""" pricing = costsim.PricingContract(**_TWO_MODEL_PRICING) low = costsim.estimate_run_ore("m-cheap", "low", pricing) high = costsim.estimate_run_ore("m-cheap", "high", pricing) dear_high = costsim.estimate_run_ore("m-dear", "high", pricing) assert low != high, "effort must scale the estimate" assert dear_high != high, "model must scale the estimate" # control: same model + same effort → identical (not incidental) assert costsim.estimate_run_ore("m-cheap", "high", pricing) == high def test_azure_placeholder_rows_unpriced() -> None: """The defined azure path: ``--profile azure`` ships REPLACE-WITH-* placeholders → rows are unpriced (``estimat_ore`` None, placeholder label), NEVER a crash. Uses the bundled config.""" pricing = costsim.load_pricing() table = costsim.build_estimate_table("azure", ["standard"], pricing, n_projects=4) assert table["rows"], "azure profile must still produce rows" for row in table["rows"]: assert row["estimat_ore"] is None assert "placeholder" in row["estimat_type"] def test_table_carries_kost_mot_verdi_field() -> None: """SC5: the table carries a top-level ``kost_mot_verdi`` field ready for the S5.4 value report. Detach point: remove the field from ``build_estimate_table`` → RED.""" pricing = costsim.PricingContract(**_TWO_MODEL_PRICING) table = costsim.build_estimate_table( "p", ["standard"], pricing, n_projects=1, model_map={"p": {"proposer": "m-cheap"}} ) assert "kost_mot_verdi" in table assert "kjoring_kostet_ore" in table["kost_mot_verdi"] def test_row_carries_sourced_quality_guidance() -> None: """Brief Goal 4: a priced row surfaces ``kvalitets_veiledning`` with a source (guidance with kilde, never a bare number). Detach point: drop ``kvalitets_veiledning`` from the row → RED.""" pricing = costsim.PricingContract(**_PRICING) # m-known carries guidance table = costsim.build_estimate_table( "p", ["standard"], pricing, n_projects=1, model_map={"p": {"proposer": "m-known"}} ) row = table["rows"][0] assert row["kvalitets_veiledning"] is not None assert row["kvalitets_veiledning"]["source"].strip() def test_table_is_byte_deterministic() -> None: """Serialized table is byte-stable across identical inputs (sort_keys/indent/LF).""" pricing = costsim.PricingContract(**_TWO_MODEL_PRICING) mm = {"p": {"proposer": "m-cheap", "checker": "m-dear"}} t1 = costsim.build_estimate_table("p", ["low", "high"], pricing, n_projects=3, model_map=mm) t2 = costsim.build_estimate_table("p", ["low", "high"], pricing, n_projects=3, model_map=mm) assert costsim.dump_estimate_table(t1).encode() == costsim.dump_estimate_table(t2).encode() def _costsim_ast() -> ast.Module: return ast.parse(_COSTSIM.read_text(encoding="utf-8")) def test_no_network_path_in_module() -> None: """SC4: costsim imports NO network library — offline by construction. Detach point: add a ``urllib``/``http``/``socket`` import → RED.""" net = {"socket", "urllib", "http", "requests", "httpx", "ftplib", "smtplib"} imported: list[str] = [] for node in ast.walk(_costsim_ast()): if isinstance(node, ast.Import): imported += [a.name.split(".")[0] for a in node.names] elif isinstance(node, ast.ImportFrom): imported.append((node.module or "").split(".")[0]) offenders = sorted(set(imported) & net) assert offenders == [], f"costsim must have no network import, found: {offenders}" def test_costsim_source_has_no_float_literal() -> None: """SC3 supporting guard (well-defined): costsim is pure integer-øre, so ANY ``float`` literal in its source is either a stray hardcoded price or a no-float-invariant violation. Detach point: hardcode a float price (e.g. ``0.03``) → RED. The behavioral sentinel test below is the PRIMARY price-from-config seam; this literal check backs it up without the brittle allowlist a 'price-shaped number' scan would need.""" floats = [ node.value for node in ast.walk(_costsim_ast()) if isinstance(node, ast.Constant) and isinstance(node.value, float) ] assert floats == [], ( f"costsim must contain no float literal (integer øre only), found: {floats}" ) def test_price_comes_from_config() -> None: """SC3 PRIMARY seam (behavioral): a sentinel price in injected config flows into the estimate. A hardcoded price shadowing config would NOT yield the sentinel-derived value → RED.""" sentinel = { "source": "sentinel", "date": "2026-07-15", "models": {"m-x": {"ore_per_1k_tokens": 999_999}}, } pricing = costsim.PricingContract(**sentinel) expected = 100_000 * 999_999 // 1000 # standard effort (100%), default max_tokens assert costsim.estimate_run_ore("m-x", "standard", pricing) == expected