63 lines
2.8 KiB
Python
63 lines
2.8 KiB
Python
"""Step 10 tests — LLM->IR generation + validator-as-retry (FakeChatClient, no LLM).
|
|
|
|
A well-formed reply validates; a malformed/text-leaked reply is retried (not silently
|
|
accepted); exhausting attempts OR crossing the meter cap is a typed failure (never a
|
|
malformed proposal). Pattern: tests/spikes/test_harness.py + tests/spikes/test_c_validator.py.
|
|
"""
|
|
|
|
import pytest
|
|
from spikes._harness import FakeChatClient
|
|
|
|
from portfolio_optimiser.budget import Budget, BudgetExceeded, TokenMeter
|
|
from portfolio_optimiser.generate import generate_via_llm, generate_with_validation
|
|
from portfolio_optimiser.ir import SavingsProposal
|
|
from portfolio_optimiser.reference_domain import load_reference_projects
|
|
from portfolio_optimiser.validator import Rejection, ValidatedProposal, proposal_for
|
|
|
|
# A feasible proposal for FV42-GSV-E1: 200k <= ~445k (30% of the affected total).
|
|
_VALID = (
|
|
'{"project_id":"FV42-GSV-E1","measure":"Reduce scope",'
|
|
'"affected_items":[{"code":"05.2","quantity":4300,"unit_cost":215},'
|
|
'{"code":"03.1","quantity":1800,"unit_cost":310}],"claimed_saving_nok":200000}'
|
|
)
|
|
|
|
|
|
@pytest.fixture(scope="module")
|
|
def project():
|
|
return load_reference_projects()[0] # FV42-GSV-E1
|
|
|
|
|
|
def _meter() -> TokenMeter:
|
|
return TokenMeter(Budget(max_tokens=10**9, max_rounds=20))
|
|
|
|
|
|
async def test_wellformed_reply_yields_validated_proposal(project) -> None:
|
|
client = FakeChatClient(scripted=[_VALID], default_reply=_VALID)
|
|
result = await generate_via_llm(client, project, "", _meter(), max_attempts=3)
|
|
assert isinstance(result, ValidatedProposal)
|
|
assert isinstance(result.proposal, SavingsProposal)
|
|
|
|
|
|
async def test_malformed_reply_is_retried_not_silently_accepted(project) -> None:
|
|
client = FakeChatClient(scripted=["not json at all {{{", _VALID], default_reply=_VALID)
|
|
result = await generate_via_llm(client, project, "", _meter(), max_attempts=3)
|
|
assert isinstance(result, ValidatedProposal) # the malformed reply was NOT accepted
|
|
assert client.call_count >= 2 # it retried past the malformed reply
|
|
|
|
|
|
def test_generate_with_validation_is_bounded_and_typed(project) -> None:
|
|
calls = {"n": 0}
|
|
|
|
def always_out_of_range(_attempt: int) -> SavingsProposal:
|
|
calls["n"] += 1
|
|
return proposal_for(project, ["05.2", "03.1"], claimed_saving_nok=800_000)
|
|
|
|
# Exhausting attempts -> typed Rejection (never a malformed/validated proposal); self_repair bounded.
|
|
result = generate_with_validation(always_out_of_range, _meter(), max_attempts=3)
|
|
assert isinstance(result, Rejection)
|
|
assert calls["n"] == 3
|
|
|
|
# Crossing the meter cap -> typed failure (BudgetExceeded).
|
|
tiny = TokenMeter(Budget(max_tokens=10**9, max_rounds=1))
|
|
with pytest.raises(BudgetExceeded):
|
|
generate_with_validation(always_out_of_range, tiny, max_attempts=3)
|