Kjell Tore Guttormsen
37547fe292
The context sets, the packaged knowledge bases and the example bundles are replaced by one fictitious example set about IT operations in an invented organisation: three context sets (serverrom-2027, driftsavtale-2027 and the two-base drift-og-avtale-2027), two synthetic knowledge bases under src/portfolio_optimiser/data/kunnskapsbaser and two example bundles under src/portfolio_optimiser/data/bundles. Numbers, codes and structural values in tests and fixtures are kept; names, ids and wording change. Dated measurement documents that only recorded runs on the replaced material are deleted. Gate figures measured on the new set are not comparable with earlier ones. The exclusion gate from the previous commit is green: 0 tracked files hit outside the shared/ subtree. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
63 lines
2.8 KiB
Python
63 lines
2.8 KiB
Python
"""Step 10 tests — LLM->IR generation + validator-as-retry (FakeChatClient, no LLM).
|
|
|
|
A well-formed reply validates; a malformed/text-leaked reply is retried (not silently
|
|
accepted); exhausting attempts OR crossing the meter cap is a typed failure (never a
|
|
malformed proposal). Pattern: tests/spikes/test_harness.py + tests/spikes/test_c_validator.py.
|
|
"""
|
|
|
|
import pytest
|
|
from spikes._harness import FakeChatClient
|
|
|
|
from portfolio_optimiser.budget import Budget, BudgetExceeded, TokenMeter
|
|
from portfolio_optimiser.generate import generate_via_llm, generate_with_validation
|
|
from portfolio_optimiser.ir import SavingsProposal
|
|
from portfolio_optimiser.reference_domain import load_reference_projects
|
|
from portfolio_optimiser.validator import Rejection, ValidatedProposal, proposal_for
|
|
|
|
# A feasible proposal for KONTOR-IT-E1: 200k <= ~445k (30% of the affected total).
|
|
_VALID = (
|
|
'{"project_id":"KONTOR-IT-E1","measure":"Reduce scope",'
|
|
'"affected_items":[{"code":"05.2","quantity":4300,"unit_cost":215},'
|
|
'{"code":"03.1","quantity":1800,"unit_cost":310}],"claimed_saving_nok":200000}'
|
|
)
|
|
|
|
|
|
@pytest.fixture(scope="module")
|
|
def project():
|
|
return load_reference_projects()[0] # KONTOR-IT-E1
|
|
|
|
|
|
def _meter() -> TokenMeter:
|
|
return TokenMeter(Budget(max_tokens=10**9, max_rounds=20))
|
|
|
|
|
|
async def test_wellformed_reply_yields_validated_proposal(project) -> None:
|
|
client = FakeChatClient(scripted=[_VALID], default_reply=_VALID)
|
|
result = await generate_via_llm(client, project, "", _meter(), max_attempts=3)
|
|
assert isinstance(result.outcome, ValidatedProposal)
|
|
assert isinstance(result.outcome.proposal, SavingsProposal)
|
|
|
|
|
|
async def test_malformed_reply_is_retried_not_silently_accepted(project) -> None:
|
|
client = FakeChatClient(scripted=["not json at all {{{", _VALID], default_reply=_VALID)
|
|
result = await generate_via_llm(client, project, "", _meter(), max_attempts=3)
|
|
assert isinstance(result.outcome, ValidatedProposal) # the malformed reply was NOT accepted
|
|
assert client.call_count >= 2 # it retried past the malformed reply
|
|
|
|
|
|
def test_generate_with_validation_is_bounded_and_typed(project) -> None:
|
|
calls = {"n": 0}
|
|
|
|
def always_out_of_range(_attempt: int) -> SavingsProposal:
|
|
calls["n"] += 1
|
|
return proposal_for(project, ["05.2", "03.1"], claimed_saving_nok=800_000)
|
|
|
|
# Exhausting attempts -> typed Rejection (never a malformed/validated proposal); self_repair bounded.
|
|
result = generate_with_validation(always_out_of_range, _meter(), max_attempts=3)
|
|
assert isinstance(result, Rejection)
|
|
assert calls["n"] == 3
|
|
|
|
# Crossing the meter cap -> typed failure (BudgetExceeded).
|
|
tiny = TokenMeter(Budget(max_tokens=10**9, max_rounds=1))
|
|
with pytest.raises(BudgetExceeded):
|
|
generate_with_validation(always_out_of_range, tiny, max_attempts=3)
|