feat(validator): S6 — deterministic backbone: typed IR, golden-frozen validator, provenance stamp

TDD from method-spec alone (§3 Step 4, §7, §9), golden.json as the only
ground truth: ir.py (construction invariants, fail-fast bundle loader),
validator.py (closed-form feasibility bound 0.30·Σ + Monte Carlo seed
20260624/512 samples/inclusive quantiles — reproduces every frozen golden
field; Rejection as a distinct unconsumable type), provenance.py (stamp
mirroring ONLY the deterministic validator). Mutation controls + seed-detach
proof (§11); 45/45 green without an API key; ruff + mypy --strict clean.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01QdSfQdND84oeq2mbjueLTS
This commit is contained in:
Kjell Tore Guttormsen 2026-07-03 06:27:40 +02:00
commit 1e1b7e4506
7 changed files with 529 additions and 0 deletions

View file

@ -0,0 +1,107 @@
"""Golden-suite regression (method-spec §7.2, §11) — the ONLY ground truth.
Consumes ``shared/examples/bygg-energi-mikro/{validator-input,golden}.json``
UNCHANGED. The meaningful assertion is ``validates`` = true (claimed p90); the
frozen numbers are the regression net. The mutation controls prove the net is taut:
ONE changed input parameter must diverge from the golden outcome.
"""
from __future__ import annotations
import json
from pathlib import Path
from typing import Any
import pytest
from portfolio_optimiser_claude.ir import load_validator_input
from portfolio_optimiser_claude.validator import ValidatedProposal, validate_proposal
BUNDLE = Path(__file__).resolve().parents[1] / "shared" / "examples" / "bygg-energi-mikro"
@pytest.fixture(scope="module")
def golden() -> dict[str, Any]:
raw: dict[str, Any] = json.loads((BUNDLE / "golden.json").read_text(encoding="utf-8"))
return raw
@pytest.fixture(scope="module")
def outcome() -> ValidatedProposal:
result = validate_proposal(load_validator_input(BUNDLE))
assert isinstance(result, ValidatedProposal)
return result
class TestGoldenValidator:
"""§7.2 'validator': every decided field reproduced (approx-equality on floats)."""
def test_outcome_is_the_validated_type(
self, outcome: ValidatedProposal, golden: dict[str, Any]
) -> None:
assert type(outcome).__name__ == golden["validator"]["outcome"]
def test_validates_true_claim_within_optimistic_bound(
self, outcome: ValidatedProposal, golden: dict[str, Any]
) -> None:
assert outcome.validates is golden["validator"]["validates"] is True
assert outcome.claimed_saving_nok <= outcome.p90
def test_decided_figures_match_golden(
self, outcome: ValidatedProposal, golden: dict[str, Any]
) -> None:
frozen = golden["validator"]
assert outcome.claimed_saving_nok == pytest.approx(frozen["claimed_saving_nok"])
assert outcome.nominal_feasible == pytest.approx(frozen["nominal_feasible"])
assert outcome.p10 == pytest.approx(frozen["p10"])
assert outcome.p50 == pytest.approx(frozen["p50"])
assert outcome.p90 == pytest.approx(frozen["p90"])
class TestMutationControl:
"""One changed input parameter → divergence from golden (the net is taut)."""
def test_changed_quantity_diverges(self, golden: dict[str, Any]) -> None:
proposal = load_validator_input(BUNDLE)
mutated = proposal.model_copy(
update={
"affected_items": [
proposal.affected_items[0].model_copy(update={"quantity": 310_000})
]
}
)
result = validate_proposal(mutated)
assert isinstance(result, ValidatedProposal)
assert result.nominal_feasible != pytest.approx(golden["validator"]["nominal_feasible"])
assert result.p50 != pytest.approx(golden["validator"]["p50"])
def test_changed_assumption_band_diverges(self, golden: dict[str, Any]) -> None:
# Same nominal, different uncertainty band → only the Monte Carlo percentiles
# move. Proves the golden net also covers the risk simulation, not just the
# closed-form bound.
proposal = load_validator_input(BUNDLE)
mutated = proposal.model_copy(update={"assumptions": {"ENERGI-TOTAL-EL": (0.70, 1.50)}})
result = validate_proposal(mutated)
assert isinstance(result, ValidatedProposal)
assert result.nominal_feasible == pytest.approx(golden["validator"]["nominal_feasible"])
assert result.p90 != pytest.approx(golden["validator"]["p90"])
class TestLearningSurface:
"""§7.2 'learning_surface': what the validator CANNOT compute — internally consistent."""
def test_expected_actual_is_rate_times_modelled(self, golden: dict[str, Any]) -> None:
surface = golden["learning_surface"]
assert 0 < surface["realization_rate"] < 1
assert surface["expected_actual_saving_nok"] == pytest.approx(
surface["realization_rate"] * surface["modelled_saving_nok"]
)
def test_learning_surface_is_outside_validator_reach(
self, outcome: ValidatedProposal, golden: dict[str, Any]
) -> None:
# The realization gap is encoded ONLY by the seed verdict — the validated
# outcome must not carry (and cannot compute) any realization field.
assert not hasattr(outcome, "realization_rate")
assert not hasattr(outcome, "expected_actual_saving_nok")
assert golden["learning_surface"]["modelled_saving_nok"] == outcome.claimed_saving_nok

86
tests/test_ir.py Normal file
View file

@ -0,0 +1,86 @@
"""Typed cost-IR tests (method-spec §7.1).
Schema invariants are enforced at IR construction, so a malformed proposal can never
exist as a value (§3 Step 2). Loading the IR projection from a bundle is FAIL-FAST:
a missing file raises (required input contrast the tolerant inbox, §5). Pure
config/file-layer no model client, no API key, no network.
"""
from __future__ import annotations
from pathlib import Path
from typing import Any
import pytest
from pydantic import ValidationError
from portfolio_optimiser_claude.ir import SavingsProposal, load_validator_input
BUNDLE = Path(__file__).resolve().parents[1] / "shared" / "examples" / "bygg-energi-mikro"
VALID: dict[str, Any] = {
"project_id": "P1",
"measure": "LED retrofit",
"affected_items": [{"code": "EL", "quantity": 1000, "unit_cost": 1.0}],
"claimed_saving_nok": 200,
"assumptions": {"EL": [0.8, 1.2]},
}
def build(**overrides: Any) -> SavingsProposal:
return SavingsProposal(**{**VALID, **overrides})
class TestSchemaInvariants:
"""§7.1: invariants hold at construction — a schema error, never a validator call."""
def test_valid_proposal_constructs(self) -> None:
proposal = build()
assert proposal.project_id == "P1"
assert proposal.assumptions["EL"] == (0.8, 1.2)
def test_empty_affected_items_rejected(self) -> None:
with pytest.raises(ValidationError):
build(affected_items=[])
def test_negative_quantity_rejected(self) -> None:
with pytest.raises(ValidationError):
build(affected_items=[{"code": "EL", "quantity": -1, "unit_cost": 1.0}])
@pytest.mark.parametrize("bad", [0, -0.5])
def test_nonpositive_unit_cost_rejected(self, bad: float) -> None:
with pytest.raises(ValidationError):
build(affected_items=[{"code": "EL", "quantity": 1000, "unit_cost": bad}])
@pytest.mark.parametrize("bad", [0, -100])
def test_nonpositive_claimed_saving_rejected(self, bad: float) -> None:
with pytest.raises(ValidationError):
build(claimed_saving_nok=bad)
def test_claim_above_affected_total_is_schema_error(self) -> None:
# The claimed saving MUST NOT exceed the affected items' own total
# (Σ quantity·unit_cost = 1000) — a schema error, not a validator rejection.
with pytest.raises(ValidationError):
build(claimed_saving_nok=1001)
def test_claim_equal_to_affected_total_allowed(self) -> None:
assert build(claimed_saving_nok=1000).claimed_saving_nok == 1000
def test_assumptions_default_to_empty(self) -> None:
# Empty assumptions = degenerate band, no spread (§7.1).
proposal = build(assumptions={})
assert proposal.assumptions == {}
class TestFailFastLoader:
"""§7.1: the IR projection is required input — a missing file raises."""
def test_loads_shared_bundle_projection_unchanged(self) -> None:
proposal = load_validator_input(BUNDLE)
assert proposal.project_id == "BYGG-KONTOR-NORD"
assert [item.code for item in proposal.affected_items] == ["ENERGI-TOTAL-EL"]
assert proposal.assumptions["ENERGI-TOTAL-EL"] == (0.70, 1.40)
def test_missing_projection_raises(self, tmp_path: Path) -> None:
with pytest.raises(FileNotFoundError):
load_validator_input(tmp_path)

88
tests/test_provenance.py Normal file
View file

@ -0,0 +1,88 @@
"""Provenance-stamp tests (method-spec §9).
The stamp is authoritative data, not display metadata: at least one citation,
the producing model/role, token usage, and ``validator_decision`` which mirrors
the DETERMINISTIC VALIDATOR only, stamped from the validator's outcome BEFORE any
checker override, so the two falsifiers are never conflated.
"""
from __future__ import annotations
from typing import Any
import pytest
from pydantic import ValidationError
from portfolio_optimiser_claude.ir import SavingsProposal
from portfolio_optimiser_claude.provenance import (
Citation,
Provenance,
stamp_validator_decision,
)
from portfolio_optimiser_claude.validator import Rejection, validate_proposal
CITATION: dict[str, Any] = {
"file": "tiltak-led-retrofit.md",
"span": "200 lysrorarmaturer (90 W -> 40 W)",
"snippet": "LED-retrofit av 200 lysrorarmaturer (90 W -> 40 W) i kontorlokaler",
}
def proposal_claiming(claimed: float) -> SavingsProposal:
return SavingsProposal(
project_id="P1",
measure="test measure",
affected_items=[{"code": "EL", "quantity": 1000, "unit_cost": 1.0}],
claimed_saving_nok=claimed,
)
class TestValidatorDecisionMirror:
"""§9: validator_decision mirrors ONLY the deterministic validator."""
def test_validated_outcome_stamps_validated(self) -> None:
outcome = validate_proposal(proposal_claiming(200))
assert stamp_validator_decision(outcome) == "validated"
def test_rejection_stamps_rejected(self) -> None:
outcome = validate_proposal(proposal_claiming(500))
assert isinstance(outcome, Rejection)
assert stamp_validator_decision(outcome) == "rejected"
class TestStampShape:
"""§9: at least one citation; decisions use the validator vocabulary; real usage."""
def build(self, **overrides: Any) -> Provenance:
kwargs: dict[str, Any] = {
"citations": [CITATION],
"role": "proposer",
"validator_decision": "validated",
"tokens_used": 0,
}
kwargs.update(overrides)
return Provenance(**kwargs)
def test_valid_stamp_constructs(self) -> None:
stamp = self.build()
assert stamp.citations[0] == Citation(**CITATION)
assert stamp.role == "proposer"
def test_zero_citations_rejected(self) -> None:
# A run whose context yields no citable content MUST fail fast (§9).
with pytest.raises(ValidationError):
self.build(citations=[])
def test_model_falls_back_to_neutral_unknown(self) -> None:
# Never a fabricated model name — the fallback is the neutral 'unknown' (§9).
assert self.build().model == "unknown"
def test_checker_vocabulary_rejected(self) -> None:
# 'approved' is checker/expert vocabulary (§4) — validator_decision speaks
# the validator's language only: validated | rejected.
with pytest.raises(ValidationError):
self.build(validator_decision="approved")
def test_negative_token_usage_rejected(self) -> None:
with pytest.raises(ValidationError):
self.build(tokens_used=-1)

65
tests/test_validator.py Normal file
View file

@ -0,0 +1,65 @@
"""Structural-block tests (method-spec §3 Step 4).
A claim above the optimistic feasible bound (p90) yields a ``Rejection`` that is a
DISTINCT type from ``ValidatedProposal`` carrying the claimed and feasible figures
in its reason, and NO percentiles so it can never be consumed as validated.
"""
from __future__ import annotations
import pytest
from portfolio_optimiser_claude.ir import SavingsProposal
from portfolio_optimiser_claude.validator import (
Rejection,
ValidatedProposal,
validate_proposal,
)
def proposal_claiming(claimed: float) -> SavingsProposal:
# Degenerate band (no assumptions): every Monte Carlo sample equals the fixed
# cost, so the feasible bound is exactly 0.30 × 1000 = 300 at every percentile.
return SavingsProposal(
project_id="P1",
measure="test measure",
affected_items=[{"code": "EL", "quantity": 1000, "unit_cost": 1.0}],
claimed_saving_nok=claimed,
)
class TestStructuralBlock:
"""§3 Step 4: the deterministic validator gates the numbers — blocking."""
def test_claim_above_p90_yields_rejection(self) -> None:
outcome = validate_proposal(proposal_claiming(500))
assert isinstance(outcome, Rejection)
def test_claim_within_p90_yields_validated(self) -> None:
outcome = validate_proposal(proposal_claiming(200))
assert isinstance(outcome, ValidatedProposal)
assert outcome.validates is True
# Degenerate band: no spread, all percentiles collapse to the fixed bound.
assert outcome.p10 == outcome.p50 == outcome.p90 == pytest.approx(300.0)
def test_rejection_reason_carries_claimed_and_feasible_figures(self) -> None:
outcome = validate_proposal(proposal_claiming(500))
assert isinstance(outcome, Rejection)
assert "500" in outcome.reason
assert "300" in outcome.reason
class TestRejectionIsUnconsumable:
"""§3 Step 4: a distinct type with no percentiles — never consumable as validated."""
def test_rejection_is_not_a_validated_proposal(self) -> None:
outcome = validate_proposal(proposal_claiming(500))
assert not isinstance(outcome, ValidatedProposal)
assert not issubclass(Rejection, ValidatedProposal)
assert not issubclass(ValidatedProposal, Rejection)
def test_rejection_has_no_percentiles(self) -> None:
outcome = validate_proposal(proposal_claiming(500))
assert isinstance(outcome, Rejection)
for field in ("p10", "p50", "p90", "validates"):
assert not hasattr(outcome, field)