feat(s36): deterministic integer-ore estimate scaling with model x effort

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_0145ZKPLMVeqM47z2jxxokym
This commit is contained in:
Kjell Tore Guttormsen 2026-07-15 09:59:55 +02:00
commit eb889a7f0e
3 changed files with 99 additions and 0 deletions

View file

@ -57,3 +57,22 @@ def test_load_pricing_missing_provenance_raises(tmp_path: Path) -> None:
bad = {"models": _VALID_PRICING["models"]} # no source/date
with pytest.raises(ValidationError):
costsim.load_pricing(_write(tmp_path, bad))
def test_estimate_equals_hand_computed() -> None:
"""Anchor the integer-øre math: m-cheap = 10 øre/1k, effort standard (100%), max_tokens 100_000
100_000*100//100 * 10 // 1000 = 1000 øre/run; 3 projects 3000 øre."""
pricing = costsim.PricingContract(**_VALID_PRICING)
assert costsim.estimate_run_ore("m-cheap", "standard", pricing, max_tokens=100_000) == 1000
assert (
costsim.estimate_portfolio_ore(
"m-cheap", "standard", pricing, n_projects=3, max_tokens=100_000
)
== 3000
)
def test_unknown_effort_raises() -> None:
pricing = costsim.PricingContract(**_VALID_PRICING)
with pytest.raises(ValueError, match="unknown effort"):
costsim.estimate_run_ore("m-cheap", "turbo", pricing)

View file

@ -30,6 +30,15 @@ _PRICING = {
},
}
_TWO_MODEL_PRICING = {
"source": "loadbearing-test prices",
"date": "2026-07-15",
"models": {
"m-cheap": {"ore_per_1k_tokens": 10},
"m-dear": {"ore_per_1k_tokens": 50},
},
}
def test_costsim_registered_maf_free() -> None:
"""Meta: costsim.py is registered in the MAF-free guard list, so ``test_okf_is_maf_free``
@ -93,3 +102,26 @@ def test_quality_guidance_is_sourced_never_number() -> None:
assert price.quality_guidance is not None
assert price.quality_guidance.source.strip()
assert price.quality_guidance.note.strip()
def test_estimate_is_deterministic() -> None:
"""SC2 (determinism): identical inputs → identical integer-øre estimate, reproducibly."""
pricing = costsim.PricingContract(**_PRICING)
a = costsim.estimate_portfolio_ore("m-known", "high", pricing, n_projects=4)
b = costsim.estimate_portfolio_ore("m-known", "high", pricing, n_projects=4)
assert a == b
def test_estimate_scales_with_model_and_effort() -> None:
"""SC2 (scaling): the estimate varies with BOTH model and effort. Detach point: drop the
``EFFORT_FACTORS[effort]`` term (e.g. hard-code 100%) low and high collapse to equal the
``low != high`` assertion goes RED. Control: same model + same effort identical (isolates the
effort variable, so a pass cannot be incidental)."""
pricing = costsim.PricingContract(**_TWO_MODEL_PRICING)
low = costsim.estimate_run_ore("m-cheap", "low", pricing)
high = costsim.estimate_run_ore("m-cheap", "high", pricing)
dear_high = costsim.estimate_run_ore("m-dear", "high", pricing)
assert low != high, "effort must scale the estimate"
assert dear_high != high, "model must scale the estimate"
# control: same model + same effort → identical (not incidental)
assert costsim.estimate_run_ore("m-cheap", "high", pricing) == high