feat(s36): PricingContract + fail-fast loader + placeholder path + quality guidance

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_0145ZKPLMVeqM47z2jxxokym
This commit is contained in:
Kjell Tore Guttormsen 2026-07-15 09:57:48 +02:00
commit cfa93799a5
4 changed files with 195 additions and 0 deletions

View file

@ -21,8 +21,15 @@ from __future__ import annotations
import json
from pathlib import Path
from pydantic import BaseModel, Field
_DATA_DIR = Path(__file__).resolve().parent / "data"
_MODEL_MAP_FILE = _DATA_DIR / "model_map.json"
_PRICING_FILE = _DATA_DIR / "pricing.example.json"
# A model-map deployment id that the operator has not yet replaced (data/model_map.json ships the
# azure roles as REPLACE-WITH-FOUNDRY-DEPLOYMENT). Such ids are UNPRICED, not an error.
PLACEHOLDER_PREFIX = "REPLACE-WITH-"
def _load_model_map(
@ -44,3 +51,54 @@ def _load_model_map(
if not isinstance(entry, dict):
raise ValueError(f"unknown profile {profile!r} in model map")
return dict(entry)
class QualityGuidance(BaseModel):
"""Per-model quality trade-off as GUIDANCE WITH SOURCE (brief Goal 4 / D-I pkt. 3): prose
``note`` + a provenance ``source`` never a bare measured number. This keeps a quality claim
honest (veiledning med kilde), never asserted as measured fact without belegg (målbilde §1)."""
note: str = Field(min_length=1)
source: str = Field(min_length=1)
class ModelPrice(BaseModel):
"""A model's blended price in INTEGER øre per 1000 total tokens (float NOK is banned — see
``ledger.py``). ``quality_guidance`` is optional sourced prose, never a number."""
ore_per_1k_tokens: int = Field(ge=0)
quality_guidance: QualityGuidance | None = None
class PricingContract(BaseModel):
"""Schema-validated pricing config (fail-fast). ``source`` + ``date`` are REQUIRED provenance
(kilde + dato) the framework ships an example; the deployer owns verified vendor prices."""
source: str = Field(min_length=1)
date: str = Field(min_length=1)
models: dict[str, ModelPrice]
def load_pricing(path: str | Path | None = None) -> PricingContract:
"""Load + validate pricing config, fail-fast. Missing file → ``FileNotFoundError``;
malformed / missing ``source``/``date`` ``pydantic.ValidationError`` (mirrors
``contracts.load_goal_config``). ``None`` loads the packaged ``data/pricing.example.json``.
Top-level ``_``-prefixed keys (e.g. ``_note``) are ignored."""
p = Path(path) if path is not None else _PRICING_FILE
if not p.is_file():
raise FileNotFoundError(f"pricing config not found: {str(p)!r}")
raw = json.loads(p.read_text(encoding="utf-8"))
data = {k: v for k, v in raw.items() if not k.startswith("_")}
return PricingContract(**data)
def _price_ore_per_1k(model: str, pricing: PricingContract) -> int | None:
"""Return ``model``'s øre-per-1k-tokens rate. Returns ``None`` for a placeholder deployment id
(``REPLACE-WITH-*`` the defined unpriced path; azure profile ships these). Fail-fast
(``ValueError``) for a genuine (non-placeholder) model absent from the price map."""
if model.startswith(PLACEHOLDER_PREFIX):
return None
entry = pricing.models.get(model)
if entry is None:
raise ValueError(f"missing price for {model}")
return entry.ore_per_1k_tokens

View file

@ -0,0 +1,28 @@
{
"_note": "S3.6 EXAMPLE prices — kr/1k tokens as INTEGER øre (blended total-token rate, matching the runtime meter's total_token_count). The framework ships this example; the DEPLOYER owns verified vendor prices (replace values, keep source+date honest). quality_guidance is sourced GUIDANCE, never a measured number. Azure ids below are forward-looking examples the deployer activates once they replace the REPLACE-WITH-FOUNDRY-DEPLOYMENT placeholder in model_map.json.",
"source": "Framework example prices (UNVERIFIED) — deployer MUST replace with vendor prices + kilde/dato",
"date": "2026-07-15",
"models": {
"qwen3:4b": {
"ore_per_1k_tokens": 0,
"quality_guidance": {
"note": "Lokal Ollama-modell: gratis, ingen egress (D6). Lavere resonnering enn frontier-modeller — egnet for billige, høy-volum utkast, ikke kritisk falsifisering.",
"source": "Ollama model card qwen3:4b (2026-07-15)"
}
},
"gpt-4o-mini": {
"ore_per_1k_tokens": 15,
"quality_guidance": {
"note": "Rimelig frontier-klasse; god kost/kvalitet for proposer-rollen. Eksempelpris — verifiser mot Azure OpenAI-prisliste før bruk.",
"source": "Azure OpenAI pricing (EXAMPLE, unverified) (2026-07-15)"
}
},
"gpt-5-mini": {
"ore_per_1k_tokens": 40,
"quality_guidance": {
"note": "Sterkere resonnering enn 4o-mini; egnet for checker-rollen der falsifisering er kritisk. Eksempelpris — verifiser før bruk.",
"source": "Azure OpenAI pricing (EXAMPLE, unverified) (2026-07-15)"
}
}
}
}

59
tests/test_costsim.py Normal file
View file

@ -0,0 +1,59 @@
"""S3.6 costsim — unit tests: pricing load/fail-fast, deterministic estimate, table, CLI.
Load-bearing detach seams live in ``test_costsim_loadbearing.py``; these are the happy-path +
schema unit tests.
"""
from __future__ import annotations
import json
from pathlib import Path
import pytest
from pydantic import ValidationError
from portfolio_optimiser import costsim
_VALID_PRICING = {
"source": "unit-test prices",
"date": "2026-07-15",
"models": {
"m-cheap": {
"ore_per_1k_tokens": 10,
"quality_guidance": {"note": "cheap draft model", "source": "test-src 2026-07-15"},
},
"m-dear": {
"ore_per_1k_tokens": 50,
"quality_guidance": {"note": "stronger reasoning", "source": "test-src 2026-07-15"},
},
},
}
def _write(tmp_path: Path, data: dict) -> Path:
p = tmp_path / "pricing.json"
p.write_text(json.dumps(data), encoding="utf-8")
return p
def test_load_pricing_valid(tmp_path: Path) -> None:
pricing = costsim.load_pricing(_write(tmp_path, _VALID_PRICING))
assert pricing.models["m-cheap"].ore_per_1k_tokens == 10
assert pricing.source == "unit-test prices"
def test_load_pricing_bundled_default() -> None:
"""No path → the packaged data/pricing.example.json loads and prices the local model."""
pricing = costsim.load_pricing()
assert "qwen3:4b" in pricing.models
def test_load_pricing_missing_file_raises(tmp_path: Path) -> None:
with pytest.raises(FileNotFoundError):
costsim.load_pricing(tmp_path / "nope.json")
def test_load_pricing_missing_provenance_raises(tmp_path: Path) -> None:
bad = {"models": _VALID_PRICING["models"]} # no source/date
with pytest.raises(ValidationError):
costsim.load_pricing(_write(tmp_path, bad))

View file

@ -13,8 +13,23 @@ import subprocess
import sys
from pathlib import Path
import pytest
from portfolio_optimiser import costsim
_COSTSIM = Path(__file__).resolve().parents[1] / "src" / "portfolio_optimiser" / "costsim.py"
_PRICING = {
"source": "loadbearing-test prices",
"date": "2026-07-15",
"models": {
"m-known": {
"ore_per_1k_tokens": 20,
"quality_guidance": {"note": "known model", "source": "test-src"},
}
},
}
def test_costsim_registered_maf_free() -> None:
"""Meta: costsim.py is registered in the MAF-free guard list, so ``test_okf_is_maf_free``
@ -43,3 +58,38 @@ def test_costsim_import_is_maf_free() -> None:
)
result = subprocess.run([sys.executable, "-c", check], capture_output=True, text=True)
assert result.returncode == 0, result.stderr
def test_missing_price_for_model_fails_fast() -> None:
"""SC1: a genuine (non-placeholder) model absent from the price map fails fast. Detach point:
remove the presence-check raise in ``_price_ore_per_1k`` a guessed/0 price is returned RED."""
pricing = costsim.PricingContract(**_PRICING)
with pytest.raises(ValueError, match="missing price for"):
costsim._price_ore_per_1k("m-unknown", pricing)
def test_priced_model_returns_ore_rate() -> None:
"""Control for the fail-fast seam: a priced model returns its øre rate (proves the raise fires
only on genuine absence, not always)."""
pricing = costsim.PricingContract(**_PRICING)
assert costsim._price_ore_per_1k("m-known", pricing) == 20
def test_placeholder_model_is_unpriced() -> None:
"""The defined azure-placeholder path: a ``REPLACE-WITH-*`` deployment id returns ``None`` (not
priced, not a crash) so ``--profile azure`` (all placeholders) is well-defined."""
pricing = costsim.PricingContract(**_PRICING)
assert costsim._price_ore_per_1k("REPLACE-WITH-FOUNDRY-DEPLOYMENT", pricing) is None
def test_quality_guidance_is_sourced_never_number() -> None:
"""Brief Goal 4: per-model quality trade-offs are sourced GUIDANCE (prose note + provenance
source), never a bare measured number. Detach point: ship a guidance entry without a source
the ``QualityGuidance.source`` (min_length=1) requirement makes load RED."""
pricing = costsim.load_pricing()
guided = [p for p in pricing.models.values() if p.quality_guidance is not None]
assert guided, "bundled pricing must carry per-model quality guidance"
for price in guided:
assert price.quality_guidance is not None
assert price.quality_guidance.source.strip()
assert price.quality_guidance.note.strip()