feat(s36): PricingContract + fail-fast loader + placeholder path + quality guidance
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_0145ZKPLMVeqM47z2jxxokym
This commit is contained in:
parent
47147e5f7c
commit
cfa93799a5
4 changed files with 195 additions and 0 deletions
|
|
@ -21,8 +21,15 @@ from __future__ import annotations
|
|||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
_DATA_DIR = Path(__file__).resolve().parent / "data"
|
||||
_MODEL_MAP_FILE = _DATA_DIR / "model_map.json"
|
||||
_PRICING_FILE = _DATA_DIR / "pricing.example.json"
|
||||
|
||||
# A model-map deployment id that the operator has not yet replaced (data/model_map.json ships the
|
||||
# azure roles as REPLACE-WITH-FOUNDRY-DEPLOYMENT). Such ids are UNPRICED, not an error.
|
||||
PLACEHOLDER_PREFIX = "REPLACE-WITH-"
|
||||
|
||||
|
||||
def _load_model_map(
|
||||
|
|
@ -44,3 +51,54 @@ def _load_model_map(
|
|||
if not isinstance(entry, dict):
|
||||
raise ValueError(f"unknown profile {profile!r} in model map")
|
||||
return dict(entry)
|
||||
|
||||
|
||||
class QualityGuidance(BaseModel):
|
||||
"""Per-model quality trade-off as GUIDANCE WITH SOURCE (brief Goal 4 / D-I pkt. 3): prose
|
||||
``note`` + a provenance ``source`` — never a bare measured number. This keeps a quality claim
|
||||
honest (veiledning med kilde), never asserted as measured fact without belegg (målbilde §1)."""
|
||||
|
||||
note: str = Field(min_length=1)
|
||||
source: str = Field(min_length=1)
|
||||
|
||||
|
||||
class ModelPrice(BaseModel):
|
||||
"""A model's blended price in INTEGER øre per 1000 total tokens (float NOK is banned — see
|
||||
``ledger.py``). ``quality_guidance`` is optional sourced prose, never a number."""
|
||||
|
||||
ore_per_1k_tokens: int = Field(ge=0)
|
||||
quality_guidance: QualityGuidance | None = None
|
||||
|
||||
|
||||
class PricingContract(BaseModel):
|
||||
"""Schema-validated pricing config (fail-fast). ``source`` + ``date`` are REQUIRED provenance
|
||||
(kilde + dato) — the framework ships an example; the deployer owns verified vendor prices."""
|
||||
|
||||
source: str = Field(min_length=1)
|
||||
date: str = Field(min_length=1)
|
||||
models: dict[str, ModelPrice]
|
||||
|
||||
|
||||
def load_pricing(path: str | Path | None = None) -> PricingContract:
|
||||
"""Load + validate pricing config, fail-fast. Missing file → ``FileNotFoundError``;
|
||||
malformed / missing ``source``/``date`` → ``pydantic.ValidationError`` (mirrors
|
||||
``contracts.load_goal_config``). ``None`` loads the packaged ``data/pricing.example.json``.
|
||||
Top-level ``_``-prefixed keys (e.g. ``_note``) are ignored."""
|
||||
p = Path(path) if path is not None else _PRICING_FILE
|
||||
if not p.is_file():
|
||||
raise FileNotFoundError(f"pricing config not found: {str(p)!r}")
|
||||
raw = json.loads(p.read_text(encoding="utf-8"))
|
||||
data = {k: v for k, v in raw.items() if not k.startswith("_")}
|
||||
return PricingContract(**data)
|
||||
|
||||
|
||||
def _price_ore_per_1k(model: str, pricing: PricingContract) -> int | None:
|
||||
"""Return ``model``'s øre-per-1k-tokens rate. Returns ``None`` for a placeholder deployment id
|
||||
(``REPLACE-WITH-*`` — the defined unpriced path; azure profile ships these). Fail-fast
|
||||
(``ValueError``) for a genuine (non-placeholder) model absent from the price map."""
|
||||
if model.startswith(PLACEHOLDER_PREFIX):
|
||||
return None
|
||||
entry = pricing.models.get(model)
|
||||
if entry is None:
|
||||
raise ValueError(f"missing price for {model}")
|
||||
return entry.ore_per_1k_tokens
|
||||
|
|
|
|||
28
src/portfolio_optimiser/data/pricing.example.json
Normal file
28
src/portfolio_optimiser/data/pricing.example.json
Normal file
|
|
@ -0,0 +1,28 @@
|
|||
{
|
||||
"_note": "S3.6 EXAMPLE prices — kr/1k tokens as INTEGER øre (blended total-token rate, matching the runtime meter's total_token_count). The framework ships this example; the DEPLOYER owns verified vendor prices (replace values, keep source+date honest). quality_guidance is sourced GUIDANCE, never a measured number. Azure ids below are forward-looking examples the deployer activates once they replace the REPLACE-WITH-FOUNDRY-DEPLOYMENT placeholder in model_map.json.",
|
||||
"source": "Framework example prices (UNVERIFIED) — deployer MUST replace with vendor prices + kilde/dato",
|
||||
"date": "2026-07-15",
|
||||
"models": {
|
||||
"qwen3:4b": {
|
||||
"ore_per_1k_tokens": 0,
|
||||
"quality_guidance": {
|
||||
"note": "Lokal Ollama-modell: gratis, ingen egress (D6). Lavere resonnering enn frontier-modeller — egnet for billige, høy-volum utkast, ikke kritisk falsifisering.",
|
||||
"source": "Ollama model card qwen3:4b (2026-07-15)"
|
||||
}
|
||||
},
|
||||
"gpt-4o-mini": {
|
||||
"ore_per_1k_tokens": 15,
|
||||
"quality_guidance": {
|
||||
"note": "Rimelig frontier-klasse; god kost/kvalitet for proposer-rollen. Eksempelpris — verifiser mot Azure OpenAI-prisliste før bruk.",
|
||||
"source": "Azure OpenAI pricing (EXAMPLE, unverified) (2026-07-15)"
|
||||
}
|
||||
},
|
||||
"gpt-5-mini": {
|
||||
"ore_per_1k_tokens": 40,
|
||||
"quality_guidance": {
|
||||
"note": "Sterkere resonnering enn 4o-mini; egnet for checker-rollen der falsifisering er kritisk. Eksempelpris — verifiser før bruk.",
|
||||
"source": "Azure OpenAI pricing (EXAMPLE, unverified) (2026-07-15)"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
59
tests/test_costsim.py
Normal file
59
tests/test_costsim.py
Normal file
|
|
@ -0,0 +1,59 @@
|
|||
"""S3.6 costsim — unit tests: pricing load/fail-fast, deterministic estimate, table, CLI.
|
||||
|
||||
Load-bearing detach seams live in ``test_costsim_loadbearing.py``; these are the happy-path +
|
||||
schema unit tests.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from pydantic import ValidationError
|
||||
|
||||
from portfolio_optimiser import costsim
|
||||
|
||||
_VALID_PRICING = {
|
||||
"source": "unit-test prices",
|
||||
"date": "2026-07-15",
|
||||
"models": {
|
||||
"m-cheap": {
|
||||
"ore_per_1k_tokens": 10,
|
||||
"quality_guidance": {"note": "cheap draft model", "source": "test-src 2026-07-15"},
|
||||
},
|
||||
"m-dear": {
|
||||
"ore_per_1k_tokens": 50,
|
||||
"quality_guidance": {"note": "stronger reasoning", "source": "test-src 2026-07-15"},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _write(tmp_path: Path, data: dict) -> Path:
|
||||
p = tmp_path / "pricing.json"
|
||||
p.write_text(json.dumps(data), encoding="utf-8")
|
||||
return p
|
||||
|
||||
|
||||
def test_load_pricing_valid(tmp_path: Path) -> None:
|
||||
pricing = costsim.load_pricing(_write(tmp_path, _VALID_PRICING))
|
||||
assert pricing.models["m-cheap"].ore_per_1k_tokens == 10
|
||||
assert pricing.source == "unit-test prices"
|
||||
|
||||
|
||||
def test_load_pricing_bundled_default() -> None:
|
||||
"""No path → the packaged data/pricing.example.json loads and prices the local model."""
|
||||
pricing = costsim.load_pricing()
|
||||
assert "qwen3:4b" in pricing.models
|
||||
|
||||
|
||||
def test_load_pricing_missing_file_raises(tmp_path: Path) -> None:
|
||||
with pytest.raises(FileNotFoundError):
|
||||
costsim.load_pricing(tmp_path / "nope.json")
|
||||
|
||||
|
||||
def test_load_pricing_missing_provenance_raises(tmp_path: Path) -> None:
|
||||
bad = {"models": _VALID_PRICING["models"]} # no source/date
|
||||
with pytest.raises(ValidationError):
|
||||
costsim.load_pricing(_write(tmp_path, bad))
|
||||
|
|
@ -13,8 +13,23 @@ import subprocess
|
|||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from portfolio_optimiser import costsim
|
||||
|
||||
_COSTSIM = Path(__file__).resolve().parents[1] / "src" / "portfolio_optimiser" / "costsim.py"
|
||||
|
||||
_PRICING = {
|
||||
"source": "loadbearing-test prices",
|
||||
"date": "2026-07-15",
|
||||
"models": {
|
||||
"m-known": {
|
||||
"ore_per_1k_tokens": 20,
|
||||
"quality_guidance": {"note": "known model", "source": "test-src"},
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def test_costsim_registered_maf_free() -> None:
|
||||
"""Meta: costsim.py is registered in the MAF-free guard list, so ``test_okf_is_maf_free``
|
||||
|
|
@ -43,3 +58,38 @@ def test_costsim_import_is_maf_free() -> None:
|
|||
)
|
||||
result = subprocess.run([sys.executable, "-c", check], capture_output=True, text=True)
|
||||
assert result.returncode == 0, result.stderr
|
||||
|
||||
|
||||
def test_missing_price_for_model_fails_fast() -> None:
|
||||
"""SC1: a genuine (non-placeholder) model absent from the price map fails fast. Detach point:
|
||||
remove the presence-check raise in ``_price_ore_per_1k`` → a guessed/0 price is returned → RED."""
|
||||
pricing = costsim.PricingContract(**_PRICING)
|
||||
with pytest.raises(ValueError, match="missing price for"):
|
||||
costsim._price_ore_per_1k("m-unknown", pricing)
|
||||
|
||||
|
||||
def test_priced_model_returns_ore_rate() -> None:
|
||||
"""Control for the fail-fast seam: a priced model returns its øre rate (proves the raise fires
|
||||
only on genuine absence, not always)."""
|
||||
pricing = costsim.PricingContract(**_PRICING)
|
||||
assert costsim._price_ore_per_1k("m-known", pricing) == 20
|
||||
|
||||
|
||||
def test_placeholder_model_is_unpriced() -> None:
|
||||
"""The defined azure-placeholder path: a ``REPLACE-WITH-*`` deployment id returns ``None`` (not
|
||||
priced, not a crash) — so ``--profile azure`` (all placeholders) is well-defined."""
|
||||
pricing = costsim.PricingContract(**_PRICING)
|
||||
assert costsim._price_ore_per_1k("REPLACE-WITH-FOUNDRY-DEPLOYMENT", pricing) is None
|
||||
|
||||
|
||||
def test_quality_guidance_is_sourced_never_number() -> None:
|
||||
"""Brief Goal 4: per-model quality trade-offs are sourced GUIDANCE (prose note + provenance
|
||||
source), never a bare measured number. Detach point: ship a guidance entry without a source →
|
||||
the ``QualityGuidance.source`` (min_length=1) requirement makes load RED."""
|
||||
pricing = costsim.load_pricing()
|
||||
guided = [p for p in pricing.models.values() if p.quality_guidance is not None]
|
||||
assert guided, "bundled pricing must carry per-model quality guidance"
|
||||
for price in guided:
|
||||
assert price.quality_guidance is not None
|
||||
assert price.quality_guidance.source.strip()
|
||||
assert price.quality_guidance.note.strip()
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue