"""Step 10 tests — LLM->IR generation + validator-as-retry (FakeChatClient, no LLM). A well-formed reply validates; a malformed/text-leaked reply is retried (not silently accepted); exhausting attempts OR crossing the meter cap is a typed failure (never a malformed proposal). Pattern: tests/spikes/test_harness.py + tests/spikes/test_c_validator.py. """ import pytest from spikes._harness import FakeChatClient from portfolio_optimiser.budget import Budget, BudgetExceeded, TokenMeter from portfolio_optimiser.generate import generate_via_llm, generate_with_validation from portfolio_optimiser.ir import SavingsProposal from portfolio_optimiser.reference_domain import load_reference_projects from portfolio_optimiser.validator import Rejection, ValidatedProposal, proposal_for # A feasible proposal for FV42-GSV-E1: 200k <= ~445k (30% of the affected total). _VALID = ( '{"project_id":"FV42-GSV-E1","measure":"Reduce scope",' '"affected_items":[{"code":"05.2","quantity":4300,"unit_cost":215},' '{"code":"03.1","quantity":1800,"unit_cost":310}],"claimed_saving_nok":200000}' ) @pytest.fixture(scope="module") def project(): return load_reference_projects()[0] # FV42-GSV-E1 def _meter() -> TokenMeter: return TokenMeter(Budget(max_tokens=10**9, max_rounds=20)) async def test_wellformed_reply_yields_validated_proposal(project) -> None: client = FakeChatClient(scripted=[_VALID], default_reply=_VALID) result = await generate_via_llm(client, project, "", _meter(), max_attempts=3) assert isinstance(result, ValidatedProposal) assert isinstance(result.proposal, SavingsProposal) async def test_malformed_reply_is_retried_not_silently_accepted(project) -> None: client = FakeChatClient(scripted=["not json at all {{{", _VALID], default_reply=_VALID) result = await generate_via_llm(client, project, "", _meter(), max_attempts=3) assert isinstance(result, ValidatedProposal) # the malformed reply was NOT accepted assert client.call_count >= 2 # it retried past the malformed reply def test_generate_with_validation_is_bounded_and_typed(project) -> None: calls = {"n": 0} def always_out_of_range(_attempt: int) -> SavingsProposal: calls["n"] += 1 return proposal_for(project, ["05.2", "03.1"], claimed_saving_nok=800_000) # Exhausting attempts -> typed Rejection (never a malformed/validated proposal); self_repair bounded. result = generate_with_validation(always_out_of_range, _meter(), max_attempts=3) assert isinstance(result, Rejection) assert calls["n"] == 3 # Crossing the meter cap -> typed failure (BudgetExceeded). tiny = TokenMeter(Budget(max_tokens=10**9, max_rounds=1)) with pytest.raises(BudgetExceeded): generate_with_validation(always_out_of_range, tiny, max_attempts=3)