feat(portfolio): K6 — pre-run cost simulation, priced what-if (parity row 18) [skip-docs]

Before ANY spend the operator sees a deterministic UPPER-BOUND USD estimate
for a (portfolio-)run — a what-if over the models in model_map.json (Claude
models) × effort levels (S3.6-analog, D-I pkt. 3 MUST-krav). No network, no
model call, no key: pure config arithmetic (bound by an import-purity test,
mirroring okf.py).

- contracts.py: ModelPriceContract (usd_per_mtok > 0 + REQUIRED source +
  source_date so a stale rate is visible, never silent, §1) + PricingContract
  (non-empty; no hardcoded fallback rate) + load_pricing/_bundled_pricing.
- data/pricing.example.json: per-Mtok rate per model id, each with source+date.
  Example rates are Anthropic's OUTPUT price (the higher rate) so the whole cap
  billed at that single rate can only overstate — the figure is marked ESTIMAT.
  Covers the model model_map configures, so the default path runs green.
- costsim.py: estimate_costs (n_projects × cap × effort_factor tokens at the
  per-Mtok rate; a model with no price fails fast "missing price for <id>",
  never a guess) + render_estimate + `python -m …costsim`. Effort factors are
  a coarse modeling weight (not prices) — max effort = full cap = the true
  upper bound. No price literal anywhere (grep-guard proves it).
- tests/test_costsim.py: schema fail-fast, missing-price fail-fast, scales with
  model × effort + reproducible, grep-guard, import purity, bundled-example +
  CLI offline smoke. Three seams detach-proven RED (effort factor, price guard,
  price literal).

462→478 green, golden byte-exact, full gate clean (ruff+format+mypy strict,
23 src files), run_s10.py/runs/ byte-untouched. README test-count sync ×2 +
costsim.py module note. CLI run-total-cap wiring stays out of scope (planen
lists 4 files); the mechanism is complete and proven load-bearing.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01RiTwaKLesgcwXx2mDviqpt
This commit is contained in:
Kjell Tore Guttormsen 2026-07-23 23:01:01 +02:00
commit 1600c188b4
5 changed files with 495 additions and 2 deletions

219
tests/test_costsim.py Normal file
View file

@ -0,0 +1,219 @@
"""Pre-run cost simulation (K6, S3.6-analog, D-I pkt. 3; paritetsrad 18).
The operator sees an ESTIMATED upper-bound USD cost for a (portfolio-)run BEFORE
any spend a what-if over the models in model_map.json × effort levels. Every
seam here is load-bearing (§11): the RED-when-detached notes on each test name
the mutation that makes it fail, so a green-but-dead test can't hide.
Nothing here touches the network or a key (offline invariant): the estimate is
pure config arithmetic over schema-validated pricing config.
"""
from __future__ import annotations
import ast
import json
from pathlib import Path
import pytest
from pydantic import ValidationError
from portfolio_optimiser_claude.contracts import (
ModelMapContract,
ModelPriceContract,
PricingContract,
load_pricing,
)
from portfolio_optimiser_claude.costsim import estimate_costs, main
SRC_PKG = Path(__file__).resolve().parents[1] / "src" / "portfolio_optimiser_claude"
def _model_map(*model_ids: str) -> ModelMapContract:
# Every profile needs a 'default' (§10); role split is irrelevant to the
# what-if, which iterates the DISTINCT model ids configured anywhere.
mapping = {"default": model_ids[0]}
for i, mid in enumerate(model_ids[1:], start=1):
mapping[f"role{i}"] = mid
return ModelMapContract(profiles={"anthropic": mapping})
def _pricing(**rates: float) -> PricingContract:
return PricingContract(
prices={
mid: ModelPriceContract(usd_per_mtok=rate, source="src", source_date="2026-01-01")
for mid, rate in rates.items()
}
)
class TestPricingSchemaFailsFast:
"""§10: a malformed/missing price is a startup schema error, never a guessed rate."""
def test_non_positive_price_is_rejected(self) -> None:
with pytest.raises(ValidationError):
ModelPriceContract(usd_per_mtok=0.0, source="src", source_date="2026-01-01")
def test_a_price_entry_without_a_rate_is_rejected(self) -> None:
with pytest.raises(ValidationError):
PricingContract(prices={"m": {"source": "src", "source_date": "2026-01-01"}})
def test_source_and_date_are_required(self) -> None:
# A price with no provenance would let a stale rate pass silently (§1).
with pytest.raises(ValidationError):
PricingContract(prices={"m": {"usd_per_mtok": 5.0}})
def test_empty_pricing_is_rejected(self) -> None:
with pytest.raises(ValidationError):
PricingContract(prices={})
class TestMissingPriceFailsFast:
"""RED (detach the price lookup guard → KeyError not ValueError): a model in
model_map with NO price stops the estimate cold never a guessed rate."""
def test_a_configured_model_without_a_price_raises_named(self) -> None:
model_map = _model_map("m-cheap", "m-unpriced")
pricing = _pricing(**{"m-cheap": 1.0}) # m-unpriced deliberately absent
with pytest.raises(ValueError, match="missing price for m-unpriced"):
estimate_costs(model_map, pricing, n_projects=1, token_cap_per_project=1000)
class TestEstimateScalesAndIsDeterministic:
"""RED (detach the effort factor → efforts identical): the estimate scales
with BOTH model price and effort, and is byte-reproducible (no clock/random)."""
def test_cost_scales_linearly_with_model_price(self) -> None:
model_map = _model_map("m-cheap", "m-pricey")
pricing = _pricing(**{"m-cheap": 1.0, "m-pricey": 10.0})
est = estimate_costs(model_map, pricing, n_projects=3, token_cap_per_project=1000)
by_id = {m.model_id: m for m in est.models}
# Same portfolio, same effort → 10x price ⇒ 10x cost, per effort level.
for effort in ("low", "high", "max"):
cheap = next(e.cost_usd for e in by_id["m-cheap"].efforts if e.effort == effort)
pricey = next(e.cost_usd for e in by_id["m-pricey"].efforts if e.effort == effort)
assert pricey == pytest.approx(cheap * 10.0)
def test_cost_scales_strictly_with_effort(self) -> None:
# DETACH POINT: make every _EFFORT_FACTORS value equal (e.g. all 1.0) and
# this strict-increase assertion goes RED — the effort axis is dead.
model_map = _model_map("m")
pricing = _pricing(**{"m": 5.0})
est = estimate_costs(model_map, pricing, n_projects=2, token_cap_per_project=1000)
costs = [e.cost_usd for e in est.models[0].efforts]
assert costs == sorted(costs)
assert costs[0] < costs[-1]
def test_estimate_is_reproducible(self) -> None:
model_map = _model_map("m")
pricing = _pricing(**{"m": 7.0})
a = estimate_costs(model_map, pricing, n_projects=4, token_cap_per_project=1234)
b = estimate_costs(model_map, pricing, n_projects=4, token_cap_per_project=1234)
assert [(m.model_id, [(e.effort, e.cost_usd) for e in m.efforts]) for m in a.models] == [
(m.model_id, [(e.effort, e.cost_usd) for e in m.efforts]) for m in b.models
]
def test_non_positive_shape_is_rejected(self) -> None:
pricing = _pricing(**{"m": 5.0})
with pytest.raises(ValueError):
estimate_costs(_model_map("m"), pricing, n_projects=0, token_cap_per_project=1000)
with pytest.raises(ValueError):
estimate_costs(_model_map("m"), pricing, n_projects=1, token_cap_per_project=0)
class TestNoPriceLiteralInSource:
"""RED (hardcode any example rate in costsim.py): the grep-guard proves the
price MUST come from config no rate literal lives in the module source."""
def test_no_example_price_appears_as_a_literal(self) -> None:
source = (SRC_PKG / "costsim.py").read_text(encoding="utf-8")
pricing = load_pricing() # the bundled data/pricing.example.json
for model_id, price in pricing.prices.items():
assert str(price.usd_per_mtok) not in source, (
f"price for {model_id} is hardcoded in costsim.py — must come from config"
)
def _imported_module_names(module_path: Path) -> set[str]:
tree = ast.parse(module_path.read_text(encoding="utf-8"))
names: set[str] = set()
for node in ast.walk(tree):
if isinstance(node, ast.Import):
names.update(alias.name.split(".")[0] for alias in node.names)
elif isinstance(node, ast.ImportFrom) and node.level == 0 and node.module:
names.add(node.module.split(".")[0])
return names
class TestCostsimPurity:
"""LOAD-BEARING (§11): the cost sim imports no agent toolkit — offline by design."""
def test_costsim_never_imports_an_agent_toolkit(self) -> None:
names = _imported_module_names(SRC_PKG / "costsim.py")
assert not names & {"claude_agent_sdk", "anthropic"}
class TestBundledExampleAndCli:
"""Nøkkelantakelse: the bundled example prices COVER the model model_map
configures, so the offline default path runs green and the CLI is offline."""
def test_bundled_pricing_validates(self) -> None:
pricing = load_pricing()
assert pricing.prices # non-empty, schema-valid
for price in pricing.prices.values():
assert price.usd_per_mtok > 0
assert price.source and price.source_date
def test_the_default_estimate_runs_on_bundled_config(self) -> None:
# The bundled model_map's model must have a bundled price — else the
# default CLI would fail-fast. This binds the two example files together.
from portfolio_optimiser_claude.contracts import _bundled_model_map
est = estimate_costs(
ModelMapContract(**_bundled_model_map()),
load_pricing(),
n_projects=1,
token_cap_per_project=150_000,
)
assert est.models
assert all(m.efforts for m in est.models)
def test_cli_prints_an_estimate_marked_estimat(
self, capsys: pytest.CaptureFixture[str]
) -> None:
rc = main([])
out = capsys.readouterr().out
assert rc == 0
assert "ESTIMAT" in out
def test_cli_honours_explicit_pricing_file(
self, tmp_path: Path, capsys: pytest.CaptureFixture[str]
) -> None:
from portfolio_optimiser_claude.contracts import _bundled_model_map
model_id = ModelMapContract(**_bundled_model_map()).profiles["anthropic"]["default"]
pricing_file = tmp_path / "pricing.json"
pricing_file.write_text(
json.dumps(
{
"prices": {
model_id: {
"usd_per_mtok": 3.0,
"source": "src",
"source_date": "2026-01-01",
}
}
}
),
encoding="utf-8",
)
rc = main(["--projects", "2", "--token-cap", "1000", "--pricing", str(pricing_file)])
out = capsys.readouterr().out
assert rc == 0
assert model_id in out
def test_no_network_import_in_costsim() -> None:
# Belt on the offline invariant: not even the stdlib socket/urllib slip in.
names = _imported_module_names(SRC_PKG / "costsim.py")
assert not names & {"socket", "urllib", "http", "requests", "httpx"}