Kjell Tore Guttormsen
37625435c4
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_0145ZKPLMVeqM47z2jxxokym
170 lines
8.2 KiB
Python
170 lines
8.2 KiB
Python
"""S3.6 costsim (kostnadssimulering) — offline, MAF-free, deterministic load-bearing seams.
|
||
|
||
``costsim`` estimates token/cost for a portfolio run BEFORE running (what-if over the model-map ×
|
||
effort levels), with pricing as schema-validated config. It is MAF-free (reads ``model_map.json`` as
|
||
plain data via ``__file__``-relative path, never imports ``backends``/``budget``/``contracts``) —
|
||
proven load-bearing by BOTH the direct-import AST guard (``test_okf_is_maf_free`` covering
|
||
``costsim.py``) AND the transitive import-graph seam below.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import subprocess
|
||
import sys
|
||
from pathlib import Path
|
||
|
||
import pytest
|
||
|
||
from portfolio_optimiser import costsim
|
||
|
||
_COSTSIM = Path(__file__).resolve().parents[1] / "src" / "portfolio_optimiser" / "costsim.py"
|
||
|
||
_PRICING = {
|
||
"source": "loadbearing-test prices",
|
||
"date": "2026-07-15",
|
||
"models": {
|
||
"m-known": {
|
||
"ore_per_1k_tokens": 20,
|
||
"quality_guidance": {"note": "known model", "source": "test-src"},
|
||
}
|
||
},
|
||
}
|
||
|
||
_TWO_MODEL_PRICING = {
|
||
"source": "loadbearing-test prices",
|
||
"date": "2026-07-15",
|
||
"models": {
|
||
"m-cheap": {"ore_per_1k_tokens": 10},
|
||
"m-dear": {"ore_per_1k_tokens": 50},
|
||
},
|
||
}
|
||
|
||
|
||
def test_costsim_registered_maf_free() -> None:
|
||
"""Meta: costsim.py is registered in the MAF-free guard list, so ``test_okf_is_maf_free``
|
||
actually scans it — otherwise the MAF-free claim would be green-but-dead (never checked)."""
|
||
from tests.test_okf import _MAF_FREE_MODULES
|
||
|
||
assert "costsim.py" in _MAF_FREE_MODULES
|
||
|
||
|
||
def test_costsim_import_is_maf_free() -> None:
|
||
"""The genuine transitive-import seam: loading costsim.py's module body in ISOLATION must NOT
|
||
pull ``agent_framework``/``mcp`` into ``sys.modules``. Loaded standalone via
|
||
``spec_from_file_location`` — NOT as ``portfolio_optimiser.costsim``, since the package
|
||
``__init__`` imports MAF-bound ``run``/``contracts`` and would mask costsim's own cleanliness.
|
||
The AST guard (``test_okf_is_maf_free``) only catches DIRECT imports; a transitive
|
||
``from portfolio_optimiser.contracts import ...`` in costsim would slip past it but trip THIS
|
||
check — ``exec_module`` runs costsim's import statements, so any MAF-bound import lands
|
||
``agent_framework`` in ``sys.modules``. Detach point: add such an import → RED."""
|
||
check = (
|
||
"import importlib.util, sys\n"
|
||
f"spec = importlib.util.spec_from_file_location('costsim_standalone', {str(_COSTSIM)!r})\n"
|
||
"mod = importlib.util.module_from_spec(spec)\n"
|
||
"spec.loader.exec_module(mod)\n"
|
||
"assert 'agent_framework' not in sys.modules, 'agent_framework leaked into costsim import graph'\n"
|
||
"assert 'mcp' not in sys.modules, 'mcp leaked into costsim import graph'\n"
|
||
)
|
||
result = subprocess.run([sys.executable, "-c", check], capture_output=True, text=True)
|
||
assert result.returncode == 0, result.stderr
|
||
|
||
|
||
def test_missing_price_for_model_fails_fast() -> None:
|
||
"""SC1: a genuine (non-placeholder) model absent from the price map fails fast. Detach point:
|
||
remove the presence-check raise in ``_price_ore_per_1k`` → a guessed/0 price is returned → RED."""
|
||
pricing = costsim.PricingContract(**_PRICING)
|
||
with pytest.raises(ValueError, match="missing price for"):
|
||
costsim._price_ore_per_1k("m-unknown", pricing)
|
||
|
||
|
||
def test_priced_model_returns_ore_rate() -> None:
|
||
"""Control for the fail-fast seam: a priced model returns its øre rate (proves the raise fires
|
||
only on genuine absence, not always)."""
|
||
pricing = costsim.PricingContract(**_PRICING)
|
||
assert costsim._price_ore_per_1k("m-known", pricing) == 20
|
||
|
||
|
||
def test_placeholder_model_is_unpriced() -> None:
|
||
"""The defined azure-placeholder path: a ``REPLACE-WITH-*`` deployment id returns ``None`` (not
|
||
priced, not a crash) — so ``--profile azure`` (all placeholders) is well-defined."""
|
||
pricing = costsim.PricingContract(**_PRICING)
|
||
assert costsim._price_ore_per_1k("REPLACE-WITH-FOUNDRY-DEPLOYMENT", pricing) is None
|
||
|
||
|
||
def test_quality_guidance_is_sourced_never_number() -> None:
|
||
"""Brief Goal 4: per-model quality trade-offs are sourced GUIDANCE (prose note + provenance
|
||
source), never a bare measured number. Detach point: ship a guidance entry without a source →
|
||
the ``QualityGuidance.source`` (min_length=1) requirement makes load RED."""
|
||
pricing = costsim.load_pricing()
|
||
guided = [p for p in pricing.models.values() if p.quality_guidance is not None]
|
||
assert guided, "bundled pricing must carry per-model quality guidance"
|
||
for price in guided:
|
||
assert price.quality_guidance is not None
|
||
assert price.quality_guidance.source.strip()
|
||
assert price.quality_guidance.note.strip()
|
||
|
||
|
||
def test_estimate_is_deterministic() -> None:
|
||
"""SC2 (determinism): identical inputs → identical integer-øre estimate, reproducibly."""
|
||
pricing = costsim.PricingContract(**_PRICING)
|
||
a = costsim.estimate_portfolio_ore("m-known", "high", pricing, n_projects=4)
|
||
b = costsim.estimate_portfolio_ore("m-known", "high", pricing, n_projects=4)
|
||
assert a == b
|
||
|
||
|
||
def test_estimate_scales_with_model_and_effort() -> None:
|
||
"""SC2 (scaling): the estimate varies with BOTH model and effort. Detach point: drop the
|
||
``EFFORT_FACTORS[effort]`` term (e.g. hard-code 100%) → low and high collapse to equal → the
|
||
``low != high`` assertion goes RED. Control: same model + same effort → identical (isolates the
|
||
effort variable, so a pass cannot be incidental)."""
|
||
pricing = costsim.PricingContract(**_TWO_MODEL_PRICING)
|
||
low = costsim.estimate_run_ore("m-cheap", "low", pricing)
|
||
high = costsim.estimate_run_ore("m-cheap", "high", pricing)
|
||
dear_high = costsim.estimate_run_ore("m-dear", "high", pricing)
|
||
assert low != high, "effort must scale the estimate"
|
||
assert dear_high != high, "model must scale the estimate"
|
||
# control: same model + same effort → identical (not incidental)
|
||
assert costsim.estimate_run_ore("m-cheap", "high", pricing) == high
|
||
|
||
|
||
def test_azure_placeholder_rows_unpriced() -> None:
|
||
"""The defined azure path: ``--profile azure`` ships REPLACE-WITH-* placeholders → rows are
|
||
unpriced (``estimat_ore`` None, placeholder label), NEVER a crash. Uses the bundled config."""
|
||
pricing = costsim.load_pricing()
|
||
table = costsim.build_estimate_table("azure", ["standard"], pricing, n_projects=4)
|
||
assert table["rows"], "azure profile must still produce rows"
|
||
for row in table["rows"]:
|
||
assert row["estimat_ore"] is None
|
||
assert "placeholder" in row["estimat_type"]
|
||
|
||
|
||
def test_table_carries_kost_mot_verdi_field() -> None:
|
||
"""SC5: the table carries a top-level ``kost_mot_verdi`` field ready for the S5.4 value report.
|
||
Detach point: remove the field from ``build_estimate_table`` → RED."""
|
||
pricing = costsim.PricingContract(**_TWO_MODEL_PRICING)
|
||
table = costsim.build_estimate_table(
|
||
"p", ["standard"], pricing, n_projects=1, model_map={"p": {"proposer": "m-cheap"}}
|
||
)
|
||
assert "kost_mot_verdi" in table
|
||
assert "kjoring_kostet_ore" in table["kost_mot_verdi"]
|
||
|
||
|
||
def test_row_carries_sourced_quality_guidance() -> None:
|
||
"""Brief Goal 4: a priced row surfaces ``kvalitets_veiledning`` with a source (guidance with
|
||
kilde, never a bare number). Detach point: drop ``kvalitets_veiledning`` from the row → RED."""
|
||
pricing = costsim.PricingContract(**_PRICING) # m-known carries guidance
|
||
table = costsim.build_estimate_table(
|
||
"p", ["standard"], pricing, n_projects=1, model_map={"p": {"proposer": "m-known"}}
|
||
)
|
||
row = table["rows"][0]
|
||
assert row["kvalitets_veiledning"] is not None
|
||
assert row["kvalitets_veiledning"]["source"].strip()
|
||
|
||
|
||
def test_table_is_byte_deterministic() -> None:
|
||
"""Serialized table is byte-stable across identical inputs (sort_keys/indent/LF)."""
|
||
pricing = costsim.PricingContract(**_TWO_MODEL_PRICING)
|
||
mm = {"p": {"proposer": "m-cheap", "checker": "m-dear"}}
|
||
t1 = costsim.build_estimate_table("p", ["low", "high"], pricing, n_projects=3, model_map=mm)
|
||
t2 = costsim.build_estimate_table("p", ["low", "high"], pricing, n_projects=3, model_map=mm)
|
||
assert costsim.dump_estimate_table(t1).encode() == costsim.dump_estimate_table(t2).encode()
|