diff --git a/src/portfolio_optimiser_claude/ir.py b/src/portfolio_optimiser_claude/ir.py new file mode 100644 index 0000000..0533e22 --- /dev/null +++ b/src/portfolio_optimiser_claude/ir.py @@ -0,0 +1,58 @@ +"""The typed cost-IR of a candidate measure (method-spec §7.1). + +Schema invariants are enforced at construction, so a malformed proposal can never +exist as a value (§3 Step 2): ``affected_items`` non-empty with ``quantity >= 0`` and +``unit_cost > 0``, ``claimed_saving_nok > 0`` and never above the affected items' own +total, ``assumptions`` an uncertainty band per cost code (empty = degenerate, no +spread). Loading the IR projection from a bundle is FAIL-FAST: a missing file raises +(required input — contrast the tolerant inbox, §5). +""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +from pydantic import BaseModel, Field, model_validator + +_VALIDATOR_INPUT_FILENAME = "validator-input.json" + + +class AffectedItem(BaseModel): + """One affected cost item: ``{code, quantity >= 0, unit_cost > 0}`` (§7.1).""" + + code: str = Field(min_length=1) + quantity: float = Field(ge=0) + unit_cost: float = Field(gt=0) + + +class SavingsProposal(BaseModel): + """The candidate measure projected into the typed cost-IR (§7.1).""" + + project_id: str = Field(min_length=1) + measure: str = Field(min_length=1) + affected_items: list[AffectedItem] = Field(min_length=1) + claimed_saving_nok: float = Field(gt=0) + assumptions: dict[str, tuple[float, float]] = Field(default_factory=dict) + + @model_validator(mode="after") + def _claim_within_affected_total(self) -> SavingsProposal: + # §7.1: a claim above the items' own total is a schema error, not a + # validator rejection — the value must never exist. + total = sum(item.quantity * item.unit_cost for item in self.affected_items) + if self.claimed_saving_nok > total: + raise ValueError( + f"claimed_saving_nok ({self.claimed_saving_nok}) exceeds the affected " + f"items' own total ({total})" + ) + return self + + +def load_validator_input(bundle_dir: Path) -> SavingsProposal: + """Load a bundle's IR projection — FAIL-FAST: a missing file raises (§7.1).""" + raw: dict[str, Any] = json.loads( + (bundle_dir / _VALIDATOR_INPUT_FILENAME).read_text(encoding="utf-8") + ) + # Shared fasit files carry an informative "_note"; extra keys are ignored. + return SavingsProposal.model_validate(raw) diff --git a/src/portfolio_optimiser_claude/provenance.py b/src/portfolio_optimiser_claude/provenance.py new file mode 100644 index 0000000..e3f7bd7 --- /dev/null +++ b/src/portfolio_optimiser_claude/provenance.py @@ -0,0 +1,42 @@ +"""The first-class provenance stamp (method-spec §9) — authoritative data. + +At least one citation into the source documents; the producing ``model`` and +``role`` (an injected test client's real model id when available, the neutral +``unknown`` as fallback — never a fabricated name); the run's token usage; and +``validator_decision``, which mirrors the DETERMINISTIC VALIDATOR only — stamped +from the validator's outcome BEFORE any checker override, so a checker-gated +proposal whose numbers passed is never mislabelled as validator-rejected (§9). +""" + +from __future__ import annotations + +from typing import Literal + +from pydantic import BaseModel, Field + +from portfolio_optimiser_claude.validator import Rejection, ValidatedProposal + + +class Citation(BaseModel): + """One citation into the source documents: file + exact text span + snippet (§9).""" + + file: str = Field(min_length=1) + span: str = Field(min_length=1) + snippet: str = Field(min_length=1) + + +class Provenance(BaseModel): + """The proposal's provenance stamp (§9) — schema-validated, fail-fast.""" + + citations: list[Citation] = Field(min_length=1) + model: str = Field(default="unknown", min_length=1) + role: str = Field(min_length=1) + validator_decision: Literal["validated", "rejected"] + tokens_used: int = Field(ge=0) + + +def stamp_validator_decision( + outcome: ValidatedProposal | Rejection, +) -> Literal["validated", "rejected"]: + """Mirror ONLY the deterministic validator's outcome (§9) — never the checker's.""" + return "validated" if isinstance(outcome, ValidatedProposal) else "rejected" diff --git a/src/portfolio_optimiser_claude/validator.py b/src/portfolio_optimiser_claude/validator.py new file mode 100644 index 0000000..87b6b91 --- /dev/null +++ b/src/portfolio_optimiser_claude/validator.py @@ -0,0 +1,83 @@ +"""The deterministic validator (method-spec §3 Step 4, frozen by the golden suite §7.2). + +The one endpoint-free judge that anchors the loop against swarm self-confirmation — +mandatory, blocking, never an optional plugin. Implements the spec's reference +procedure: the feasibility bound is the closed form ``0.30 × Σ quantity·unit_cost``; +the risk simulation is a Mersenne-Twister Monte Carlo (seed 20260624, 512 samples, +uniform draws from each item's assumptions band, fixed cost when no band) whose +``p10``/``p50``/``p90`` are the 1st/5th/9th cut points of the 10-quantiles (inclusive +method). ``shared/examples/bygg-energi-mikro/golden.json`` is the ONLY ground truth +(§7); ``test_bygg_energi_mikro.py`` freezes every decided field. +""" + +from __future__ import annotations + +import random +import statistics + +from pydantic import BaseModel + +from portfolio_optimiser_claude.ir import SavingsProposal + +# Policy cap (§3 Step 4): max feasible saving as a fraction of the affected total. +_FEASIBLE_FRACTION = 0.30 +# Frozen by the golden suite (§7.2) — changing either detaches from the fasit. +_MC_SEED = 20260624 +_MC_SAMPLES = 512 + + +class ValidatedProposal(BaseModel): + """The validated outcome: the claim sits within the feasible range (§7.2).""" + + validates: bool + claimed_saving_nok: float + nominal_feasible: float + p10: float + p50: float + p90: float + + +class Rejection(BaseModel): + """A structural block — a DISTINCT type from ``ValidatedProposal`` (§3 Step 4). + + Carries the claimed and feasible figures in its ``reason`` and NO percentiles, + so it can never be consumed as validated. + """ + + reason: str + + +def validate_proposal(proposal: SavingsProposal) -> ValidatedProposal | Rejection: + """Gate the numbers deterministically (§3 Step 4): validated outcome or rejection.""" + affected_total = sum(item.quantity * item.unit_cost for item in proposal.affected_items) + nominal_feasible = _FEASIBLE_FRACTION * affected_total + + rng = random.Random(_MC_SEED) + feasible_samples: list[float] = [] + for _ in range(_MC_SAMPLES): + sampled_total = 0.0 + for item in proposal.affected_items: + band = proposal.assumptions.get(item.code) + unit_cost = item.unit_cost if band is None else rng.uniform(band[0], band[1]) + sampled_total += item.quantity * unit_cost + feasible_samples.append(_FEASIBLE_FRACTION * sampled_total) + + cut_points = statistics.quantiles(feasible_samples, n=10, method="inclusive") + p10, p50, p90 = cut_points[0], cut_points[4], cut_points[8] + + if proposal.claimed_saving_nok > p90: + return Rejection( + reason=( + f"claimed saving {proposal.claimed_saving_nok:.2f} NOK exceeds the " + f"optimistic feasible bound {p90:.2f} NOK " + f"(nominal feasible {nominal_feasible:.2f} NOK)" + ) + ) + return ValidatedProposal( + validates=True, + claimed_saving_nok=proposal.claimed_saving_nok, + nominal_feasible=nominal_feasible, + p10=p10, + p50=p50, + p90=p90, + ) diff --git a/tests/test_bygg_energi_mikro.py b/tests/test_bygg_energi_mikro.py new file mode 100644 index 0000000..20206fd --- /dev/null +++ b/tests/test_bygg_energi_mikro.py @@ -0,0 +1,107 @@ +"""Golden-suite regression (method-spec §7.2, §11) — the ONLY ground truth. + +Consumes ``shared/examples/bygg-energi-mikro/{validator-input,golden}.json`` +UNCHANGED. The meaningful assertion is ``validates`` = true (claimed ≤ p90); the +frozen numbers are the regression net. The mutation controls prove the net is taut: +ONE changed input parameter must diverge from the golden outcome. +""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +import pytest + +from portfolio_optimiser_claude.ir import load_validator_input +from portfolio_optimiser_claude.validator import ValidatedProposal, validate_proposal + +BUNDLE = Path(__file__).resolve().parents[1] / "shared" / "examples" / "bygg-energi-mikro" + + +@pytest.fixture(scope="module") +def golden() -> dict[str, Any]: + raw: dict[str, Any] = json.loads((BUNDLE / "golden.json").read_text(encoding="utf-8")) + return raw + + +@pytest.fixture(scope="module") +def outcome() -> ValidatedProposal: + result = validate_proposal(load_validator_input(BUNDLE)) + assert isinstance(result, ValidatedProposal) + return result + + +class TestGoldenValidator: + """§7.2 'validator': every decided field reproduced (approx-equality on floats).""" + + def test_outcome_is_the_validated_type( + self, outcome: ValidatedProposal, golden: dict[str, Any] + ) -> None: + assert type(outcome).__name__ == golden["validator"]["outcome"] + + def test_validates_true_claim_within_optimistic_bound( + self, outcome: ValidatedProposal, golden: dict[str, Any] + ) -> None: + assert outcome.validates is golden["validator"]["validates"] is True + assert outcome.claimed_saving_nok <= outcome.p90 + + def test_decided_figures_match_golden( + self, outcome: ValidatedProposal, golden: dict[str, Any] + ) -> None: + frozen = golden["validator"] + assert outcome.claimed_saving_nok == pytest.approx(frozen["claimed_saving_nok"]) + assert outcome.nominal_feasible == pytest.approx(frozen["nominal_feasible"]) + assert outcome.p10 == pytest.approx(frozen["p10"]) + assert outcome.p50 == pytest.approx(frozen["p50"]) + assert outcome.p90 == pytest.approx(frozen["p90"]) + + +class TestMutationControl: + """One changed input parameter → divergence from golden (the net is taut).""" + + def test_changed_quantity_diverges(self, golden: dict[str, Any]) -> None: + proposal = load_validator_input(BUNDLE) + mutated = proposal.model_copy( + update={ + "affected_items": [ + proposal.affected_items[0].model_copy(update={"quantity": 310_000}) + ] + } + ) + result = validate_proposal(mutated) + assert isinstance(result, ValidatedProposal) + assert result.nominal_feasible != pytest.approx(golden["validator"]["nominal_feasible"]) + assert result.p50 != pytest.approx(golden["validator"]["p50"]) + + def test_changed_assumption_band_diverges(self, golden: dict[str, Any]) -> None: + # Same nominal, different uncertainty band → only the Monte Carlo percentiles + # move. Proves the golden net also covers the risk simulation, not just the + # closed-form bound. + proposal = load_validator_input(BUNDLE) + mutated = proposal.model_copy(update={"assumptions": {"ENERGI-TOTAL-EL": (0.70, 1.50)}}) + result = validate_proposal(mutated) + assert isinstance(result, ValidatedProposal) + assert result.nominal_feasible == pytest.approx(golden["validator"]["nominal_feasible"]) + assert result.p90 != pytest.approx(golden["validator"]["p90"]) + + +class TestLearningSurface: + """§7.2 'learning_surface': what the validator CANNOT compute — internally consistent.""" + + def test_expected_actual_is_rate_times_modelled(self, golden: dict[str, Any]) -> None: + surface = golden["learning_surface"] + assert 0 < surface["realization_rate"] < 1 + assert surface["expected_actual_saving_nok"] == pytest.approx( + surface["realization_rate"] * surface["modelled_saving_nok"] + ) + + def test_learning_surface_is_outside_validator_reach( + self, outcome: ValidatedProposal, golden: dict[str, Any] + ) -> None: + # The realization gap is encoded ONLY by the seed verdict — the validated + # outcome must not carry (and cannot compute) any realization field. + assert not hasattr(outcome, "realization_rate") + assert not hasattr(outcome, "expected_actual_saving_nok") + assert golden["learning_surface"]["modelled_saving_nok"] == outcome.claimed_saving_nok diff --git a/tests/test_ir.py b/tests/test_ir.py new file mode 100644 index 0000000..74887a0 --- /dev/null +++ b/tests/test_ir.py @@ -0,0 +1,86 @@ +"""Typed cost-IR tests (method-spec §7.1). + +Schema invariants are enforced at IR construction, so a malformed proposal can never +exist as a value (§3 Step 2). Loading the IR projection from a bundle is FAIL-FAST: +a missing file raises (required input — contrast the tolerant inbox, §5). Pure +config/file-layer — no model client, no API key, no network. +""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +import pytest +from pydantic import ValidationError + +from portfolio_optimiser_claude.ir import SavingsProposal, load_validator_input + +BUNDLE = Path(__file__).resolve().parents[1] / "shared" / "examples" / "bygg-energi-mikro" + +VALID: dict[str, Any] = { + "project_id": "P1", + "measure": "LED retrofit", + "affected_items": [{"code": "EL", "quantity": 1000, "unit_cost": 1.0}], + "claimed_saving_nok": 200, + "assumptions": {"EL": [0.8, 1.2]}, +} + + +def build(**overrides: Any) -> SavingsProposal: + return SavingsProposal(**{**VALID, **overrides}) + + +class TestSchemaInvariants: + """§7.1: invariants hold at construction — a schema error, never a validator call.""" + + def test_valid_proposal_constructs(self) -> None: + proposal = build() + assert proposal.project_id == "P1" + assert proposal.assumptions["EL"] == (0.8, 1.2) + + def test_empty_affected_items_rejected(self) -> None: + with pytest.raises(ValidationError): + build(affected_items=[]) + + def test_negative_quantity_rejected(self) -> None: + with pytest.raises(ValidationError): + build(affected_items=[{"code": "EL", "quantity": -1, "unit_cost": 1.0}]) + + @pytest.mark.parametrize("bad", [0, -0.5]) + def test_nonpositive_unit_cost_rejected(self, bad: float) -> None: + with pytest.raises(ValidationError): + build(affected_items=[{"code": "EL", "quantity": 1000, "unit_cost": bad}]) + + @pytest.mark.parametrize("bad", [0, -100]) + def test_nonpositive_claimed_saving_rejected(self, bad: float) -> None: + with pytest.raises(ValidationError): + build(claimed_saving_nok=bad) + + def test_claim_above_affected_total_is_schema_error(self) -> None: + # The claimed saving MUST NOT exceed the affected items' own total + # (Σ quantity·unit_cost = 1000) — a schema error, not a validator rejection. + with pytest.raises(ValidationError): + build(claimed_saving_nok=1001) + + def test_claim_equal_to_affected_total_allowed(self) -> None: + assert build(claimed_saving_nok=1000).claimed_saving_nok == 1000 + + def test_assumptions_default_to_empty(self) -> None: + # Empty assumptions = degenerate band, no spread (§7.1). + proposal = build(assumptions={}) + assert proposal.assumptions == {} + + +class TestFailFastLoader: + """§7.1: the IR projection is required input — a missing file raises.""" + + def test_loads_shared_bundle_projection_unchanged(self) -> None: + proposal = load_validator_input(BUNDLE) + assert proposal.project_id == "BYGG-KONTOR-NORD" + assert [item.code for item in proposal.affected_items] == ["ENERGI-TOTAL-EL"] + assert proposal.assumptions["ENERGI-TOTAL-EL"] == (0.70, 1.40) + + def test_missing_projection_raises(self, tmp_path: Path) -> None: + with pytest.raises(FileNotFoundError): + load_validator_input(tmp_path) diff --git a/tests/test_provenance.py b/tests/test_provenance.py new file mode 100644 index 0000000..7352539 --- /dev/null +++ b/tests/test_provenance.py @@ -0,0 +1,88 @@ +"""Provenance-stamp tests (method-spec §9). + +The stamp is authoritative data, not display metadata: at least one citation, +the producing model/role, token usage, and ``validator_decision`` — which mirrors +the DETERMINISTIC VALIDATOR only, stamped from the validator's outcome BEFORE any +checker override, so the two falsifiers are never conflated. +""" + +from __future__ import annotations + +from typing import Any + +import pytest +from pydantic import ValidationError + +from portfolio_optimiser_claude.ir import SavingsProposal +from portfolio_optimiser_claude.provenance import ( + Citation, + Provenance, + stamp_validator_decision, +) +from portfolio_optimiser_claude.validator import Rejection, validate_proposal + +CITATION: dict[str, Any] = { + "file": "tiltak-led-retrofit.md", + "span": "200 lysrorarmaturer (90 W -> 40 W)", + "snippet": "LED-retrofit av 200 lysrorarmaturer (90 W -> 40 W) i kontorlokaler", +} + + +def proposal_claiming(claimed: float) -> SavingsProposal: + return SavingsProposal( + project_id="P1", + measure="test measure", + affected_items=[{"code": "EL", "quantity": 1000, "unit_cost": 1.0}], + claimed_saving_nok=claimed, + ) + + +class TestValidatorDecisionMirror: + """§9: validator_decision mirrors ONLY the deterministic validator.""" + + def test_validated_outcome_stamps_validated(self) -> None: + outcome = validate_proposal(proposal_claiming(200)) + assert stamp_validator_decision(outcome) == "validated" + + def test_rejection_stamps_rejected(self) -> None: + outcome = validate_proposal(proposal_claiming(500)) + assert isinstance(outcome, Rejection) + assert stamp_validator_decision(outcome) == "rejected" + + +class TestStampShape: + """§9: at least one citation; decisions use the validator vocabulary; real usage.""" + + def build(self, **overrides: Any) -> Provenance: + kwargs: dict[str, Any] = { + "citations": [CITATION], + "role": "proposer", + "validator_decision": "validated", + "tokens_used": 0, + } + kwargs.update(overrides) + return Provenance(**kwargs) + + def test_valid_stamp_constructs(self) -> None: + stamp = self.build() + assert stamp.citations[0] == Citation(**CITATION) + assert stamp.role == "proposer" + + def test_zero_citations_rejected(self) -> None: + # A run whose context yields no citable content MUST fail fast (§9). + with pytest.raises(ValidationError): + self.build(citations=[]) + + def test_model_falls_back_to_neutral_unknown(self) -> None: + # Never a fabricated model name — the fallback is the neutral 'unknown' (§9). + assert self.build().model == "unknown" + + def test_checker_vocabulary_rejected(self) -> None: + # 'approved' is checker/expert vocabulary (§4) — validator_decision speaks + # the validator's language only: validated | rejected. + with pytest.raises(ValidationError): + self.build(validator_decision="approved") + + def test_negative_token_usage_rejected(self) -> None: + with pytest.raises(ValidationError): + self.build(tokens_used=-1) diff --git a/tests/test_validator.py b/tests/test_validator.py new file mode 100644 index 0000000..494da52 --- /dev/null +++ b/tests/test_validator.py @@ -0,0 +1,65 @@ +"""Structural-block tests (method-spec §3 Step 4). + +A claim above the optimistic feasible bound (p90) yields a ``Rejection`` that is a +DISTINCT type from ``ValidatedProposal`` — carrying the claimed and feasible figures +in its reason, and NO percentiles — so it can never be consumed as validated. +""" + +from __future__ import annotations + +import pytest + +from portfolio_optimiser_claude.ir import SavingsProposal +from portfolio_optimiser_claude.validator import ( + Rejection, + ValidatedProposal, + validate_proposal, +) + + +def proposal_claiming(claimed: float) -> SavingsProposal: + # Degenerate band (no assumptions): every Monte Carlo sample equals the fixed + # cost, so the feasible bound is exactly 0.30 × 1000 = 300 at every percentile. + return SavingsProposal( + project_id="P1", + measure="test measure", + affected_items=[{"code": "EL", "quantity": 1000, "unit_cost": 1.0}], + claimed_saving_nok=claimed, + ) + + +class TestStructuralBlock: + """§3 Step 4: the deterministic validator gates the numbers — blocking.""" + + def test_claim_above_p90_yields_rejection(self) -> None: + outcome = validate_proposal(proposal_claiming(500)) + assert isinstance(outcome, Rejection) + + def test_claim_within_p90_yields_validated(self) -> None: + outcome = validate_proposal(proposal_claiming(200)) + assert isinstance(outcome, ValidatedProposal) + assert outcome.validates is True + # Degenerate band: no spread, all percentiles collapse to the fixed bound. + assert outcome.p10 == outcome.p50 == outcome.p90 == pytest.approx(300.0) + + def test_rejection_reason_carries_claimed_and_feasible_figures(self) -> None: + outcome = validate_proposal(proposal_claiming(500)) + assert isinstance(outcome, Rejection) + assert "500" in outcome.reason + assert "300" in outcome.reason + + +class TestRejectionIsUnconsumable: + """§3 Step 4: a distinct type with no percentiles — never consumable as validated.""" + + def test_rejection_is_not_a_validated_proposal(self) -> None: + outcome = validate_proposal(proposal_claiming(500)) + assert not isinstance(outcome, ValidatedProposal) + assert not issubclass(Rejection, ValidatedProposal) + assert not issubclass(ValidatedProposal, Rejection) + + def test_rejection_has_no_percentiles(self) -> None: + outcome = validate_proposal(proposal_claiming(500)) + assert isinstance(outcome, Rejection) + for field in ("p10", "p50", "p90", "validates"): + assert not hasattr(outcome, field)