feat(s36): deterministic integer-ore estimate scaling with model x effort
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_0145ZKPLMVeqM47z2jxxokym
This commit is contained in:
parent
cfa93799a5
commit
eb889a7f0e
3 changed files with 99 additions and 0 deletions
|
|
@ -102,3 +102,51 @@ def _price_ore_per_1k(model: str, pricing: PricingContract) -> int | None:
|
|||
if entry is None:
|
||||
raise ValueError(f"missing price for {model}")
|
||||
return entry.ore_per_1k_tokens
|
||||
|
||||
|
||||
# Effort is a costsim-LOCAL what-if scenario knob (integer percent multipliers on the token ceiling).
|
||||
# HONESTY (målbilde §1): the runtime does NOT consume an effort setting today — there is no
|
||||
# `effort`/`ChatOptions` counterpart in the model-map or run path — so this is an explicit scenario
|
||||
# dimension for the what-if, never a measured runtime input.
|
||||
EFFORT_FACTORS: dict[str, int] = {"low": 70, "standard": 100, "high": 140}
|
||||
|
||||
# The hard per-run token ceiling. A run cannot exceed it (mirrors run.py's default budget cap), so it
|
||||
# IS the honest per-run upper bound — rounds are NOT multiplied in (that double-counts the cap).
|
||||
_DEFAULT_MAX_TOKENS = 100_000
|
||||
|
||||
|
||||
def estimate_run_ore(
|
||||
model: str, effort: str, pricing: PricingContract, max_tokens: int = _DEFAULT_MAX_TOKENS
|
||||
) -> int | None:
|
||||
"""Deterministic INTEGER-øre upper-bound estimate for ONE run of ``model`` at ``effort``.
|
||||
|
||||
``max_tokens`` is the hard per-run ceiling → the honest per-run upper bound (rounds are not
|
||||
multiplied in — that double-counts the cap). ``effort`` scales it by a scenario factor (see
|
||||
``EFFORT_FACTORS``). Returns ``None`` when ``model`` is an unpriced placeholder. Pure integer
|
||||
math (float NOK is banned — ``ledger.py``); marked as an estimate (øvre grense), not a
|
||||
prediction."""
|
||||
if effort not in EFFORT_FACTORS:
|
||||
raise ValueError(f"unknown effort {effort!r}; known: {sorted(EFFORT_FACTORS)}")
|
||||
rate = _price_ore_per_1k(model, pricing)
|
||||
if rate is None:
|
||||
return None
|
||||
scaled_tokens = max_tokens * EFFORT_FACTORS[effort] // 100
|
||||
return scaled_tokens * rate // 1000
|
||||
|
||||
|
||||
def estimate_portfolio_ore(
|
||||
model: str,
|
||||
effort: str,
|
||||
pricing: PricingContract,
|
||||
*,
|
||||
n_projects: int,
|
||||
max_tokens: int = _DEFAULT_MAX_TOKENS,
|
||||
) -> int | None:
|
||||
"""Deterministic portfolio upper bound = ``n_projects × estimate_run_ore``. ``None`` if the
|
||||
model is an unpriced placeholder. Fail-fast on a negative project count."""
|
||||
if n_projects < 0:
|
||||
raise ValueError(f"n_projects must be >= 0, got {n_projects}")
|
||||
per_run = estimate_run_ore(model, effort, pricing, max_tokens)
|
||||
if per_run is None:
|
||||
return None
|
||||
return n_projects * per_run
|
||||
|
|
|
|||
|
|
@ -57,3 +57,22 @@ def test_load_pricing_missing_provenance_raises(tmp_path: Path) -> None:
|
|||
bad = {"models": _VALID_PRICING["models"]} # no source/date
|
||||
with pytest.raises(ValidationError):
|
||||
costsim.load_pricing(_write(tmp_path, bad))
|
||||
|
||||
|
||||
def test_estimate_equals_hand_computed() -> None:
|
||||
"""Anchor the integer-øre math: m-cheap = 10 øre/1k, effort standard (100%), max_tokens 100_000
|
||||
→ 100_000*100//100 * 10 // 1000 = 1000 øre/run; 3 projects → 3000 øre."""
|
||||
pricing = costsim.PricingContract(**_VALID_PRICING)
|
||||
assert costsim.estimate_run_ore("m-cheap", "standard", pricing, max_tokens=100_000) == 1000
|
||||
assert (
|
||||
costsim.estimate_portfolio_ore(
|
||||
"m-cheap", "standard", pricing, n_projects=3, max_tokens=100_000
|
||||
)
|
||||
== 3000
|
||||
)
|
||||
|
||||
|
||||
def test_unknown_effort_raises() -> None:
|
||||
pricing = costsim.PricingContract(**_VALID_PRICING)
|
||||
with pytest.raises(ValueError, match="unknown effort"):
|
||||
costsim.estimate_run_ore("m-cheap", "turbo", pricing)
|
||||
|
|
|
|||
|
|
@ -30,6 +30,15 @@ _PRICING = {
|
|||
},
|
||||
}
|
||||
|
||||
_TWO_MODEL_PRICING = {
|
||||
"source": "loadbearing-test prices",
|
||||
"date": "2026-07-15",
|
||||
"models": {
|
||||
"m-cheap": {"ore_per_1k_tokens": 10},
|
||||
"m-dear": {"ore_per_1k_tokens": 50},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def test_costsim_registered_maf_free() -> None:
|
||||
"""Meta: costsim.py is registered in the MAF-free guard list, so ``test_okf_is_maf_free``
|
||||
|
|
@ -93,3 +102,26 @@ def test_quality_guidance_is_sourced_never_number() -> None:
|
|||
assert price.quality_guidance is not None
|
||||
assert price.quality_guidance.source.strip()
|
||||
assert price.quality_guidance.note.strip()
|
||||
|
||||
|
||||
def test_estimate_is_deterministic() -> None:
|
||||
"""SC2 (determinism): identical inputs → identical integer-øre estimate, reproducibly."""
|
||||
pricing = costsim.PricingContract(**_PRICING)
|
||||
a = costsim.estimate_portfolio_ore("m-known", "high", pricing, n_projects=4)
|
||||
b = costsim.estimate_portfolio_ore("m-known", "high", pricing, n_projects=4)
|
||||
assert a == b
|
||||
|
||||
|
||||
def test_estimate_scales_with_model_and_effort() -> None:
|
||||
"""SC2 (scaling): the estimate varies with BOTH model and effort. Detach point: drop the
|
||||
``EFFORT_FACTORS[effort]`` term (e.g. hard-code 100%) → low and high collapse to equal → the
|
||||
``low != high`` assertion goes RED. Control: same model + same effort → identical (isolates the
|
||||
effort variable, so a pass cannot be incidental)."""
|
||||
pricing = costsim.PricingContract(**_TWO_MODEL_PRICING)
|
||||
low = costsim.estimate_run_ore("m-cheap", "low", pricing)
|
||||
high = costsim.estimate_run_ore("m-cheap", "high", pricing)
|
||||
dear_high = costsim.estimate_run_ore("m-dear", "high", pricing)
|
||||
assert low != high, "effort must scale the estimate"
|
||||
assert dear_high != high, "model must scale the estimate"
|
||||
# control: same model + same effort → identical (not incidental)
|
||||
assert costsim.estimate_run_ore("m-cheap", "high", pricing) == high
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue