feat(s36): deterministic integer-ore estimate scaling with model x effort
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_0145ZKPLMVeqM47z2jxxokym
This commit is contained in:
parent
cfa93799a5
commit
eb889a7f0e
3 changed files with 99 additions and 0 deletions
|
|
@ -57,3 +57,22 @@ def test_load_pricing_missing_provenance_raises(tmp_path: Path) -> None:
|
|||
bad = {"models": _VALID_PRICING["models"]} # no source/date
|
||||
with pytest.raises(ValidationError):
|
||||
costsim.load_pricing(_write(tmp_path, bad))
|
||||
|
||||
|
||||
def test_estimate_equals_hand_computed() -> None:
|
||||
"""Anchor the integer-øre math: m-cheap = 10 øre/1k, effort standard (100%), max_tokens 100_000
|
||||
→ 100_000*100//100 * 10 // 1000 = 1000 øre/run; 3 projects → 3000 øre."""
|
||||
pricing = costsim.PricingContract(**_VALID_PRICING)
|
||||
assert costsim.estimate_run_ore("m-cheap", "standard", pricing, max_tokens=100_000) == 1000
|
||||
assert (
|
||||
costsim.estimate_portfolio_ore(
|
||||
"m-cheap", "standard", pricing, n_projects=3, max_tokens=100_000
|
||||
)
|
||||
== 3000
|
||||
)
|
||||
|
||||
|
||||
def test_unknown_effort_raises() -> None:
|
||||
pricing = costsim.PricingContract(**_VALID_PRICING)
|
||||
with pytest.raises(ValueError, match="unknown effort"):
|
||||
costsim.estimate_run_ore("m-cheap", "turbo", pricing)
|
||||
|
|
|
|||
|
|
@ -30,6 +30,15 @@ _PRICING = {
|
|||
},
|
||||
}
|
||||
|
||||
_TWO_MODEL_PRICING = {
|
||||
"source": "loadbearing-test prices",
|
||||
"date": "2026-07-15",
|
||||
"models": {
|
||||
"m-cheap": {"ore_per_1k_tokens": 10},
|
||||
"m-dear": {"ore_per_1k_tokens": 50},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def test_costsim_registered_maf_free() -> None:
|
||||
"""Meta: costsim.py is registered in the MAF-free guard list, so ``test_okf_is_maf_free``
|
||||
|
|
@ -93,3 +102,26 @@ def test_quality_guidance_is_sourced_never_number() -> None:
|
|||
assert price.quality_guidance is not None
|
||||
assert price.quality_guidance.source.strip()
|
||||
assert price.quality_guidance.note.strip()
|
||||
|
||||
|
||||
def test_estimate_is_deterministic() -> None:
|
||||
"""SC2 (determinism): identical inputs → identical integer-øre estimate, reproducibly."""
|
||||
pricing = costsim.PricingContract(**_PRICING)
|
||||
a = costsim.estimate_portfolio_ore("m-known", "high", pricing, n_projects=4)
|
||||
b = costsim.estimate_portfolio_ore("m-known", "high", pricing, n_projects=4)
|
||||
assert a == b
|
||||
|
||||
|
||||
def test_estimate_scales_with_model_and_effort() -> None:
|
||||
"""SC2 (scaling): the estimate varies with BOTH model and effort. Detach point: drop the
|
||||
``EFFORT_FACTORS[effort]`` term (e.g. hard-code 100%) → low and high collapse to equal → the
|
||||
``low != high`` assertion goes RED. Control: same model + same effort → identical (isolates the
|
||||
effort variable, so a pass cannot be incidental)."""
|
||||
pricing = costsim.PricingContract(**_TWO_MODEL_PRICING)
|
||||
low = costsim.estimate_run_ore("m-cheap", "low", pricing)
|
||||
high = costsim.estimate_run_ore("m-cheap", "high", pricing)
|
||||
dear_high = costsim.estimate_run_ore("m-dear", "high", pricing)
|
||||
assert low != high, "effort must scale the estimate"
|
||||
assert dear_high != high, "model must scale the estimate"
|
||||
# control: same model + same effort → identical (not incidental)
|
||||
assert costsim.estimate_run_ore("m-cheap", "high", pricing) == high
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue