feat(s36): deterministic integer-ore estimate scaling with model x effort

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_0145ZKPLMVeqM47z2jxxokym
This commit is contained in:
Kjell Tore Guttormsen 2026-07-15 09:59:55 +02:00
commit eb889a7f0e
3 changed files with 99 additions and 0 deletions

View file

@ -102,3 +102,51 @@ def _price_ore_per_1k(model: str, pricing: PricingContract) -> int | None:
if entry is None:
raise ValueError(f"missing price for {model}")
return entry.ore_per_1k_tokens
# Effort is a costsim-LOCAL what-if scenario knob (integer percent multipliers on the token ceiling).
# HONESTY (målbilde §1): the runtime does NOT consume an effort setting today — there is no
# `effort`/`ChatOptions` counterpart in the model-map or run path — so this is an explicit scenario
# dimension for the what-if, never a measured runtime input.
EFFORT_FACTORS: dict[str, int] = {"low": 70, "standard": 100, "high": 140}
# The hard per-run token ceiling. A run cannot exceed it (mirrors run.py's default budget cap), so it
# IS the honest per-run upper bound — rounds are NOT multiplied in (that double-counts the cap).
_DEFAULT_MAX_TOKENS = 100_000
def estimate_run_ore(
model: str, effort: str, pricing: PricingContract, max_tokens: int = _DEFAULT_MAX_TOKENS
) -> int | None:
"""Deterministic INTEGER-øre upper-bound estimate for ONE run of ``model`` at ``effort``.
``max_tokens`` is the hard per-run ceiling the honest per-run upper bound (rounds are not
multiplied in that double-counts the cap). ``effort`` scales it by a scenario factor (see
``EFFORT_FACTORS``). Returns ``None`` when ``model`` is an unpriced placeholder. Pure integer
math (float NOK is banned ``ledger.py``); marked as an estimate (øvre grense), not a
prediction."""
if effort not in EFFORT_FACTORS:
raise ValueError(f"unknown effort {effort!r}; known: {sorted(EFFORT_FACTORS)}")
rate = _price_ore_per_1k(model, pricing)
if rate is None:
return None
scaled_tokens = max_tokens * EFFORT_FACTORS[effort] // 100
return scaled_tokens * rate // 1000
def estimate_portfolio_ore(
model: str,
effort: str,
pricing: PricingContract,
*,
n_projects: int,
max_tokens: int = _DEFAULT_MAX_TOKENS,
) -> int | None:
"""Deterministic portfolio upper bound = ``n_projects × estimate_run_ore``. ``None`` if the
model is an unpriced placeholder. Fail-fast on a negative project count."""
if n_projects < 0:
raise ValueError(f"n_projects must be >= 0, got {n_projects}")
per_run = estimate_run_ore(model, effort, pricing, max_tokens)
if per_run is None:
return None
return n_projects * per_run

View file

@ -57,3 +57,22 @@ def test_load_pricing_missing_provenance_raises(tmp_path: Path) -> None:
bad = {"models": _VALID_PRICING["models"]} # no source/date
with pytest.raises(ValidationError):
costsim.load_pricing(_write(tmp_path, bad))
def test_estimate_equals_hand_computed() -> None:
"""Anchor the integer-øre math: m-cheap = 10 øre/1k, effort standard (100%), max_tokens 100_000
100_000*100//100 * 10 // 1000 = 1000 øre/run; 3 projects 3000 øre."""
pricing = costsim.PricingContract(**_VALID_PRICING)
assert costsim.estimate_run_ore("m-cheap", "standard", pricing, max_tokens=100_000) == 1000
assert (
costsim.estimate_portfolio_ore(
"m-cheap", "standard", pricing, n_projects=3, max_tokens=100_000
)
== 3000
)
def test_unknown_effort_raises() -> None:
pricing = costsim.PricingContract(**_VALID_PRICING)
with pytest.raises(ValueError, match="unknown effort"):
costsim.estimate_run_ore("m-cheap", "turbo", pricing)

View file

@ -30,6 +30,15 @@ _PRICING = {
},
}
_TWO_MODEL_PRICING = {
"source": "loadbearing-test prices",
"date": "2026-07-15",
"models": {
"m-cheap": {"ore_per_1k_tokens": 10},
"m-dear": {"ore_per_1k_tokens": 50},
},
}
def test_costsim_registered_maf_free() -> None:
"""Meta: costsim.py is registered in the MAF-free guard list, so ``test_okf_is_maf_free``
@ -93,3 +102,26 @@ def test_quality_guidance_is_sourced_never_number() -> None:
assert price.quality_guidance is not None
assert price.quality_guidance.source.strip()
assert price.quality_guidance.note.strip()
def test_estimate_is_deterministic() -> None:
"""SC2 (determinism): identical inputs → identical integer-øre estimate, reproducibly."""
pricing = costsim.PricingContract(**_PRICING)
a = costsim.estimate_portfolio_ore("m-known", "high", pricing, n_projects=4)
b = costsim.estimate_portfolio_ore("m-known", "high", pricing, n_projects=4)
assert a == b
def test_estimate_scales_with_model_and_effort() -> None:
"""SC2 (scaling): the estimate varies with BOTH model and effort. Detach point: drop the
``EFFORT_FACTORS[effort]`` term (e.g. hard-code 100%) low and high collapse to equal the
``low != high`` assertion goes RED. Control: same model + same effort identical (isolates the
effort variable, so a pass cannot be incidental)."""
pricing = costsim.PricingContract(**_TWO_MODEL_PRICING)
low = costsim.estimate_run_ore("m-cheap", "low", pricing)
high = costsim.estimate_run_ore("m-cheap", "high", pricing)
dear_high = costsim.estimate_run_ore("m-dear", "high", pricing)
assert low != high, "effort must scale the estimate"
assert dear_high != high, "model must scale the estimate"
# control: same model + same effort → identical (not incidental)
assert costsim.estimate_run_ore("m-cheap", "high", pricing) == high