diff --git a/docs/fase1-spikes/findings-c.md b/docs/fase1-spikes/findings-c.md new file mode 100644 index 0000000..f2eaeb9 --- /dev/null +++ b/docs/fase1-spikes/findings-c.md @@ -0,0 +1,48 @@ +# Spike C findings — blocking deterministic hybrid validator (B1) + +**Assumption:** a blocking deterministic hybrid-validator can *structurally* block an +out-of-range proposal from ever reaching the expert — and a valid proposal yields a +risk distribution (P10/P50/P90) with a capped self-repair loop. + +## Result — CONFIRMED (fully deterministic, no MAF, no endpoint) + +Three stages over a typed Pydantic IR (`SavingsProposal`): + +1. **IR schema gate.** Field validators + a cross-field `@model_validator` reject + malformed proposals at construction: negative quantities, and a claimed saving that + exceeds the affected items' own total (both proven to raise `ValidationError`). +2. **PuLP solver-in-the-loop.** A real **CBC** solve (PuLP's bundled binary, found at + `solverdir/cbc/osx/i64/cbc` on this Intel mac) bounds the maximum feasible saving at + `MAX_SAVING_FRACTION = 0.30` of the affected items' cost. CBC absence would + **escalate** (`CbcUnavailable`) — there is no silent LP-relaxation fallback (R2). + *(PuLP 3.x deprecates `PULP_CBC_CMD`; PuLP 4.0 will require `pip install pulp[cbc]` + + `COIN_CMD`. The spike silences the cosmetic warning — a Fase 2 migration note.)* +3. **Monte Carlo.** Seeded stdlib `random` + `statistics.quantiles` over uncertain + unit-costs give **P10 ≤ P50 ≤ P90** of the feasible saving, and are reproducible under + the fixed seed. + +**Structural block (the B1 headline):** an out-of-range proposal returns a `Rejection`, +a *different type* from `ValidatedProposal` that carries **no percentiles** — accessing +`.p50` on it raises `AttributeError`, so a rejected proposal can never be consumed as +validated. A valid proposal (claim well under the feasible cap) returns a +`ValidatedProposal` with the percentiles. **`self_repair` is capped** at `max_attempts` +and hard-stops (verified it calls the generator exactly N times and no more, B4). + +### Worked example (project FV42-GSV-E1, codes 05.2 + 03.1) + +- Affected items' total ≈ **1 482 500 NOK**; 30% feasible cap ≈ **444 750 NOK**. +- Claim **200 000** → `ValidatedProposal` with ordered P10/P50/P90 around the cap. +- Claim **800 000** → constructs (≤ total) but exceeds P90 feasible → **`Rejection`**. + +## Token use + +**0 — validator is fully deterministic; live LLM generation is gated.** The +`generate_via_llm` entry point `skip`s without a LOCAL endpoint; all validator / solver / +Monte-Carlo logic is exercised on crafted IR directly, with **zero** model tokens spent. + +## Implication for Fase 2 + +The obligatory, blocking validator is realizable exactly as specified: typed IR + +real solver + risk percentiles + a type-level rejection that cannot leak downstream. +Fase 2 should keep the `Rejection`/`ValidatedProposal` type split (structural block, not +an advisory flag) and the CBC-absent escalate contract. diff --git a/spikes/c_validator.py b/spikes/c_validator.py new file mode 100644 index 0000000..8348773 --- /dev/null +++ b/spikes/c_validator.py @@ -0,0 +1,209 @@ +"""Spike C — blocking deterministic hybrid validator (B1). + +Fully deterministic; **no MAF builders**. Demonstrates that an out-of-range savings +proposal can be **structurally blocked** before it ever reaches an expert, via three +stages over a typed IR: + +1. **Pydantic IR** (`SavingsProposal`) — field validators + a cross-field + `@model_validator` reject malformed proposals at construction (negative quantities, + a claimed saving exceeding the affected items' own total). +2. **PuLP solver-in-the-loop** — a real CBC solve bounds the maximum feasible saving + (R2). CBC ships in PuLP's wheel; if it is genuinely absent the step **escalates** + (`CbcUnavailable`) — no silent LP-relaxation fallback. +3. **Monte Carlo** — stdlib `random` (seeded) + `statistics.quantiles` over uncertain + unit-costs give P10/P50/P90 of the feasible saving. To honour D6 (no heavy runs) the + CBC solve runs **once**; the MC reuses that LP's closed-form optimum, which is exact + for this model. + +The structural block (stage 4) returns a `Rejection` that is a *different type* from +`ValidatedProposal` and carries no percentiles, so it can never be consumed as validated. +""" + +from __future__ import annotations + +import random +import statistics +import warnings +from contextlib import contextmanager +from dataclasses import dataclass + +import pulp +from pydantic import BaseModel, Field, model_validator + +from portfolio_optimiser.reference_domain import Project +from spikes._harness import Budget, live_local_client_or_skip + +MAX_SAVING_FRACTION = 0.30 +"""Policy cap: at most 30% of an affected item's cost is realistically recoverable as a +saving. The LP bounds the feasible saving by this fraction.""" + +_MC_SAMPLES = 512 +_MC_SEED = 20260624 + + +class CbcUnavailable(RuntimeError): + """PuLP's bundled CBC solver is not available — escalate (no silent fallback).""" + + +@contextmanager +def _quiet_pulp(): + """Silence PuLP 3.x's `PULP_CBC_CMD` DeprecationWarning. The bundled CBC is only + reachable via `PULP_CBC_CMD`; PuLP 4.0 will require `pip install pulp[cbc]` + COIN_CMD + (a Fase 2 migration note). The warning is cosmetic here.""" + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + yield + + +class AffectedItem(BaseModel): + """One project cost line a proposal claims to save against.""" + + code: str + quantity: float = Field(ge=0) # quantities must be >= 0 + unit_cost: float = Field(gt=0) + + @property + def total(self) -> float: + return self.quantity * self.unit_cost + + +class SavingsProposal(BaseModel): + """Typed IR for a candidate cost-saving measure (B1).""" + + project_id: str + measure: str + affected_items: list[AffectedItem] = Field(min_length=1) + claimed_saving_nok: float = Field(gt=0) + # code -> (low_unit_cost, high_unit_cost) for the Monte Carlo step; empty = degenerate. + assumptions: dict[str, tuple[float, float]] = Field(default_factory=dict) + + @model_validator(mode="after") + def _claim_within_affected_total(self) -> SavingsProposal: + total = sum(item.total for item in self.affected_items) + if self.claimed_saving_nok > total: + raise ValueError( + f"claimed saving {self.claimed_saving_nok} exceeds affected items' total {total}" + ) + return self + + +@dataclass(frozen=True) +class ValidatedProposal: + """A proposal that passed every stage. Carries the Monte Carlo percentiles.""" + + proposal: SavingsProposal + p10: float + p50: float + p90: float + nominal_feasible: float + + +@dataclass(frozen=True) +class Rejection: + """A structurally-blocked proposal. Distinct type, no percentiles — it can never be + consumed as a ``ValidatedProposal``.""" + + proposal: SavingsProposal + reason: str + + +def _solve_max_feasible(items: list[AffectedItem], fraction: float) -> float: + """Real CBC solve: maximize total saving subject to a per-item upper bound and a + global fraction cap. Raises ``CbcUnavailable`` if CBC is genuinely missing.""" + with _quiet_pulp(): + solver = pulp.PULP_CBC_CMD(msg=False) + if not solver.available(): + raise CbcUnavailable("PuLP's bundled CBC solver is not available on this platform") + prob = pulp.LpProblem("max_feasible_saving", pulp.LpMaximize) + xs = [pulp.LpVariable(f"x_{i}", lowBound=0, upBound=it.total) for i, it in enumerate(items)] + prob += pulp.lpSum(xs) + prob += pulp.lpSum(xs) <= fraction * sum(it.total for it in items) + status = prob.solve(solver) + if pulp.LpStatus[status] != "Optimal": + raise CbcUnavailable(f"CBC did not reach an optimal solution (status={pulp.LpStatus[status]})") + return float(pulp.value(prob.objective)) + + +def _monte_carlo(proposal: SavingsProposal, *, fraction: float = MAX_SAVING_FRACTION) -> tuple[float, float, float]: + """Vary uncertain unit-costs (seeded) and return (P10, P50, P90) of the feasible + saving. Uses the LP's closed-form optimum (= fraction x sum of sampled totals), which + is exact here, so we do NOT spawn a CBC subprocess per sample (D6).""" + rng = random.Random(_MC_SEED) + feasibles: list[float] = [] + for _ in range(_MC_SAMPLES): + total = 0.0 + for item in proposal.affected_items: + rng_range = proposal.assumptions.get(item.code) + unit_cost = rng.uniform(*rng_range) if rng_range else item.unit_cost + total += item.quantity * unit_cost + feasibles.append(fraction * total) + deciles = statistics.quantiles(feasibles, n=10, method="inclusive") + return deciles[0], deciles[4], deciles[8] # P10, P50, P90 + + +def validate_proposal(proposal: SavingsProposal) -> ValidatedProposal | Rejection: + """Deterministic blocking validation. Returns a ``ValidatedProposal`` only when the + claim is feasible; otherwise a ``Rejection`` that cannot be consumed as validated.""" + # Stage 1 (Pydantic) already ran at construction. Stage 2: real CBC solve. + nominal = _solve_max_feasible(proposal.affected_items, MAX_SAVING_FRACTION) + # Stage 3: Monte Carlo percentiles of the feasible saving. + p10, p50, p90 = _monte_carlo(proposal) + # Stage 4: structural block — a claim above the optimistic feasible (P90) is out of range. + if proposal.claimed_saving_nok > p90: + return Rejection( + proposal=proposal, + reason=f"claimed saving {proposal.claimed_saving_nok:.0f} exceeds P90 feasible {p90:.0f}", + ) + return ValidatedProposal(proposal=proposal, p10=p10, p50=p50, p90=p90, nominal_feasible=nominal) + + +def self_repair( + generate: object, + *, + max_attempts: int = 3, +) -> ValidatedProposal | Rejection: + """Call ``generate(attempt)`` and validate; retry on rejection up to ``max_attempts``, + then hard-stop and return the last rejection. Bounded — never loops forever (B4).""" + if max_attempts <= 0: + raise ValueError(f"max_attempts must be positive, got {max_attempts}") + budget = Budget(max_tokens=10**9, max_rounds=max_attempts) # round cap == attempt cap + last: Rejection | None = None + for attempt in range(1, budget.max_rounds + 1): + result = validate_proposal(generate(attempt)) # type: ignore[operator] + if isinstance(result, ValidatedProposal): + return result + last = result + assert last is not None + return last + + +def proposal_for( + project: Project, + codes: list[str], + *, + claimed_saving_nok: float, + measure: str = "Reduce scope on selected cost codes", + assumptions: dict[str, tuple[float, float]] | None = None, +) -> SavingsProposal: + """Build a `SavingsProposal` from a real reference project's cost items (test helper).""" + items = [ + AffectedItem(code=ci.code, quantity=ci.quantity, unit_cost=ci.unit_cost) + for ci in project.cost_items + if ci.code in codes + ] + return SavingsProposal( + project_id=project.id, + measure=measure, + affected_items=items, + claimed_saving_nok=claimed_saving_nok, + assumptions=assumptions or {}, + ) + + +def generate_via_llm(project: Project) -> SavingsProposal: + """Gated LLM-generation entry (live arm). ``skip``s without a LOCAL endpoint — the + deterministic validator/solver/MC logic is what the gate proves.""" + _client = live_local_client_or_skip() # pytest.skip if PORTFOLIO_LOCAL_* unset + raise NotImplementedError( # pragma: no cover - reached only with a live endpoint + "live LLM generation is a Fase 2 concern; the spike validates crafted IR directly" + ) diff --git a/tests/spikes/test_c_validator.py b/tests/spikes/test_c_validator.py new file mode 100644 index 0000000..7565639 --- /dev/null +++ b/tests/spikes/test_c_validator.py @@ -0,0 +1,102 @@ +"""Spike C tests — blocking hybrid validator (B1). Crafted IR, deterministic. + +Checks: structural block of an out-of-range proposal (P10/P50/P90 only exist on the +validated path); a valid proposal yields P10 <= P50 <= P90; Monte Carlo reproducibility; +self-repair capped. Pattern: tests/test_reference_domain.py + pytest.raises. +""" + +import pytest +from pydantic import ValidationError + +from portfolio_optimiser.reference_domain import load_reference_projects +from spikes.c_validator import ( + Rejection, + SavingsProposal, + ValidatedProposal, + proposal_for, + self_repair, + validate_proposal, +) + +_ASSUMPTIONS = {"05.2": (200.0, 230.0), "03.1": (290.0, 330.0)} + + +@pytest.fixture(scope="module") +def project(): + return load_reference_projects()[0] # FV42-GSV-E1 + + +def _valid(project) -> SavingsProposal: + # Affected total ~1.48M NOK; 30% feasible cap ~0.44M. Claim 200k is comfortably feasible. + return proposal_for( + project, ["05.2", "03.1"], claimed_saving_nok=200_000, assumptions=_ASSUMPTIONS + ) + + +def _out_of_range(project) -> SavingsProposal: + # Constructs fine (800k <= 1.48M total) but far exceeds the ~0.44M feasible cap. + return proposal_for( + project, ["05.2", "03.1"], claimed_saving_nok=800_000, assumptions=_ASSUMPTIONS + ) + + +def test_out_of_range_is_structurally_blocked(project) -> None: + result = validate_proposal(_out_of_range(project)) + assert isinstance(result, Rejection) + # A Rejection carries no percentiles -> it can never be consumed as validated. + with pytest.raises(AttributeError): + _ = result.p50 # type: ignore[attr-defined] + + +def test_valid_proposal_yields_ordered_percentiles(project) -> None: + result = validate_proposal(_valid(project)) + assert isinstance(result, ValidatedProposal) + # P10 <= P50 <= P90 + assert result.p10 <= result.p50 <= result.p90 + assert result.nominal_feasible > 0 # the CBC solve produced a feasible bound + + +def test_monte_carlo_is_reproducible(project) -> None: + a = validate_proposal(_valid(project)) + b = validate_proposal(_valid(project)) + assert isinstance(a, ValidatedProposal) and isinstance(b, ValidatedProposal) + assert (a.p10, a.p50, a.p90) == (b.p10, b.p50, b.p90) + + +def test_pydantic_blocks_claim_above_affected_total(project) -> None: + with pytest.raises(ValidationError): + proposal_for(project, ["05.2"], claimed_saving_nok=99_000_000) + + +def test_pydantic_blocks_negative_quantity() -> None: + with pytest.raises(ValidationError): + SavingsProposal( + project_id="X", + measure="bad", + affected_items=[{"code": "A", "quantity": -1, "unit_cost": 10}], + claimed_saving_nok=1, + ) + + +def test_self_repair_stops_at_max_attempts(project) -> None: + calls = {"n": 0} + + def always_bad(_attempt: int) -> SavingsProposal: + calls["n"] += 1 + return _out_of_range(project) + + result = self_repair(always_bad, max_attempts=3) + assert isinstance(result, Rejection) + assert calls["n"] == 3 # bounded — never exceeds max_attempts + + +def test_self_repair_returns_on_first_valid(project) -> None: + calls = {"n": 0} + + def valid_on_second(attempt: int) -> SavingsProposal: + calls["n"] += 1 + return _valid(project) if attempt >= 2 else _out_of_range(project) + + result = self_repair(valid_on_second, max_attempts=3) + assert isinstance(result, ValidatedProposal) + assert calls["n"] == 2