Kjell Tore Guttormsen
1e1b7e4506
TDD from method-spec alone (§3 Step 4, §7, §9), golden.json as the only ground truth: ir.py (construction invariants, fail-fast bundle loader), validator.py (closed-form feasibility bound 0.30·Σ + Monte Carlo seed 20260624/512 samples/inclusive quantiles — reproduces every frozen golden field; Rejection as a distinct unconsumable type), provenance.py (stamp mirroring ONLY the deterministic validator). Mutation controls + seed-detach proof (§11); 45/45 green without an API key; ruff + mypy --strict clean. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01QdSfQdND84oeq2mbjueLTS
107 lines
4.5 KiB
Python
107 lines
4.5 KiB
Python
"""Golden-suite regression (method-spec §7.2, §11) — the ONLY ground truth.
|
|
|
|
Consumes ``shared/examples/bygg-energi-mikro/{validator-input,golden}.json``
|
|
UNCHANGED. The meaningful assertion is ``validates`` = true (claimed ≤ p90); the
|
|
frozen numbers are the regression net. The mutation controls prove the net is taut:
|
|
ONE changed input parameter must diverge from the golden outcome.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
import pytest
|
|
|
|
from portfolio_optimiser_claude.ir import load_validator_input
|
|
from portfolio_optimiser_claude.validator import ValidatedProposal, validate_proposal
|
|
|
|
BUNDLE = Path(__file__).resolve().parents[1] / "shared" / "examples" / "bygg-energi-mikro"
|
|
|
|
|
|
@pytest.fixture(scope="module")
|
|
def golden() -> dict[str, Any]:
|
|
raw: dict[str, Any] = json.loads((BUNDLE / "golden.json").read_text(encoding="utf-8"))
|
|
return raw
|
|
|
|
|
|
@pytest.fixture(scope="module")
|
|
def outcome() -> ValidatedProposal:
|
|
result = validate_proposal(load_validator_input(BUNDLE))
|
|
assert isinstance(result, ValidatedProposal)
|
|
return result
|
|
|
|
|
|
class TestGoldenValidator:
|
|
"""§7.2 'validator': every decided field reproduced (approx-equality on floats)."""
|
|
|
|
def test_outcome_is_the_validated_type(
|
|
self, outcome: ValidatedProposal, golden: dict[str, Any]
|
|
) -> None:
|
|
assert type(outcome).__name__ == golden["validator"]["outcome"]
|
|
|
|
def test_validates_true_claim_within_optimistic_bound(
|
|
self, outcome: ValidatedProposal, golden: dict[str, Any]
|
|
) -> None:
|
|
assert outcome.validates is golden["validator"]["validates"] is True
|
|
assert outcome.claimed_saving_nok <= outcome.p90
|
|
|
|
def test_decided_figures_match_golden(
|
|
self, outcome: ValidatedProposal, golden: dict[str, Any]
|
|
) -> None:
|
|
frozen = golden["validator"]
|
|
assert outcome.claimed_saving_nok == pytest.approx(frozen["claimed_saving_nok"])
|
|
assert outcome.nominal_feasible == pytest.approx(frozen["nominal_feasible"])
|
|
assert outcome.p10 == pytest.approx(frozen["p10"])
|
|
assert outcome.p50 == pytest.approx(frozen["p50"])
|
|
assert outcome.p90 == pytest.approx(frozen["p90"])
|
|
|
|
|
|
class TestMutationControl:
|
|
"""One changed input parameter → divergence from golden (the net is taut)."""
|
|
|
|
def test_changed_quantity_diverges(self, golden: dict[str, Any]) -> None:
|
|
proposal = load_validator_input(BUNDLE)
|
|
mutated = proposal.model_copy(
|
|
update={
|
|
"affected_items": [
|
|
proposal.affected_items[0].model_copy(update={"quantity": 310_000})
|
|
]
|
|
}
|
|
)
|
|
result = validate_proposal(mutated)
|
|
assert isinstance(result, ValidatedProposal)
|
|
assert result.nominal_feasible != pytest.approx(golden["validator"]["nominal_feasible"])
|
|
assert result.p50 != pytest.approx(golden["validator"]["p50"])
|
|
|
|
def test_changed_assumption_band_diverges(self, golden: dict[str, Any]) -> None:
|
|
# Same nominal, different uncertainty band → only the Monte Carlo percentiles
|
|
# move. Proves the golden net also covers the risk simulation, not just the
|
|
# closed-form bound.
|
|
proposal = load_validator_input(BUNDLE)
|
|
mutated = proposal.model_copy(update={"assumptions": {"ENERGI-TOTAL-EL": (0.70, 1.50)}})
|
|
result = validate_proposal(mutated)
|
|
assert isinstance(result, ValidatedProposal)
|
|
assert result.nominal_feasible == pytest.approx(golden["validator"]["nominal_feasible"])
|
|
assert result.p90 != pytest.approx(golden["validator"]["p90"])
|
|
|
|
|
|
class TestLearningSurface:
|
|
"""§7.2 'learning_surface': what the validator CANNOT compute — internally consistent."""
|
|
|
|
def test_expected_actual_is_rate_times_modelled(self, golden: dict[str, Any]) -> None:
|
|
surface = golden["learning_surface"]
|
|
assert 0 < surface["realization_rate"] < 1
|
|
assert surface["expected_actual_saving_nok"] == pytest.approx(
|
|
surface["realization_rate"] * surface["modelled_saving_nok"]
|
|
)
|
|
|
|
def test_learning_surface_is_outside_validator_reach(
|
|
self, outcome: ValidatedProposal, golden: dict[str, Any]
|
|
) -> None:
|
|
# The realization gap is encoded ONLY by the seed verdict — the validated
|
|
# outcome must not carry (and cannot compute) any realization field.
|
|
assert not hasattr(outcome, "realization_rate")
|
|
assert not hasattr(outcome, "expected_actual_saving_nok")
|
|
assert golden["learning_surface"]["modelled_saving_nok"] == outcome.claimed_saving_nok
|