100 lines
4.1 KiB
Python
100 lines
4.1 KiB
Python
"""Step 13 — end-to-end vertical-slice integration (deterministic, synthetic usage, no LLM).
|
|
|
|
Covers the six success criteria: (a) one valid proposal -> ValidatedProposal with populated
|
|
provenance (token usage > 0 from synthetic UsageDetails); (b) out-of-range -> Rejection with
|
|
reason; (c) Layer-2 verdict persisted; (d) a second structurally-similar run retrieves the
|
|
prior verdict via ExpeL (exercising two-arg extend_instructions); (e) tiny budget halts via
|
|
the meter/round-cap (cap not exceeded); (f) malformed contract raises before any chat call.
|
|
Pattern: tests/spikes/test_harness.py + test_c_validator.py + test_d_verdictstore.py.
|
|
"""
|
|
|
|
import pytest
|
|
from pydantic import ValidationError
|
|
|
|
from portfolio_optimiser.budget import BudgetExceeded
|
|
from portfolio_optimiser.run import run_project
|
|
from portfolio_optimiser.validator import Rejection, ValidatedProposal
|
|
|
|
_VALID = (
|
|
'{"project_id":"FV42-GSV-E1","measure":"Reduce scope",'
|
|
'"affected_items":[{"code":"05.2","quantity":4300,"unit_cost":215},'
|
|
'{"code":"03.1","quantity":1800,"unit_cost":310}],"claimed_saving_nok":200000}'
|
|
)
|
|
_OUT_OF_RANGE = _VALID.replace("200000", "800000") # parses fine, exceeds the ~445k P90
|
|
|
|
_VI = {"decision": "approved", "rationale": "feasible within range"}
|
|
|
|
|
|
async def test_a_valid_proposal_end_to_end(docs_dir, make_client_factory, fresh_store) -> None:
|
|
result = await run_project(
|
|
"FV42-GSV-E1", "local", docs_dir=docs_dir, verdict_input=_VI,
|
|
client_factory=make_client_factory(_VALID), store=fresh_store,
|
|
)
|
|
assert isinstance(result.outcome, ValidatedProposal)
|
|
stamp = result.provenance
|
|
assert len(stamp.citations) >= 1 and stamp.model and stamp.role
|
|
assert stamp.validator_decision == "validated"
|
|
assert stamp.token_usage > 0 # sourced from the synthetic UsageDetails
|
|
|
|
|
|
async def test_b_out_of_range_is_rejected(docs_dir, make_client_factory, fresh_store) -> None:
|
|
result = await run_project(
|
|
"FV42-GSV-E1", "local", docs_dir=docs_dir,
|
|
verdict_input={"decision": "rejected", "rationale": "claim too high"},
|
|
client_factory=make_client_factory(_OUT_OF_RANGE), store=fresh_store,
|
|
)
|
|
assert isinstance(result.outcome, Rejection)
|
|
assert result.outcome.reason
|
|
|
|
|
|
async def test_c_layer2_verdict_is_persisted(docs_dir, make_client_factory, fresh_store) -> None:
|
|
result = await run_project(
|
|
"FV42-GSV-E1", "local", docs_dir=docs_dir, verdict_input=_VI,
|
|
client_factory=make_client_factory(_VALID), store=fresh_store,
|
|
)
|
|
assert any(v.id == result.verdict.id for v in result.store.verdicts)
|
|
|
|
|
|
async def test_d_second_run_retrieves_prior_verdict(
|
|
docs_dir, make_client_factory, fresh_store
|
|
) -> None:
|
|
factory = make_client_factory(_VALID)
|
|
r1 = await run_project(
|
|
"FV42-GSV-E1", "local", docs_dir=docs_dir, verdict_input=_VI,
|
|
client_factory=factory, store=fresh_store,
|
|
)
|
|
r2 = await run_project(
|
|
"FV42-GSV-E1", "local", docs_dir=docs_dir, verdict_input=_VI,
|
|
client_factory=factory, store=fresh_store,
|
|
)
|
|
assert r2.retrieved # ExpeL surfaced prior verdicts on the second run
|
|
assert r2.retrieved[0].id == r1.verdict.id # structurally identical -> stable id
|
|
|
|
|
|
async def test_e_tiny_budget_halts_without_exceeding(
|
|
docs_dir, make_client_factory, fresh_store
|
|
) -> None:
|
|
with pytest.raises(BudgetExceeded):
|
|
await run_project(
|
|
"FV42-GSV-E1", "local", docs_dir=docs_dir, verdict_input=_VI,
|
|
client_factory=make_client_factory(_VALID, tokens=100),
|
|
store=fresh_store, max_tokens=10, # synthetic 100 tokens > cap 10 -> halt
|
|
)
|
|
|
|
|
|
async def test_f_malformed_contract_raises_before_any_chat(
|
|
docs_dir, make_client_factory
|
|
) -> None:
|
|
calls = {"n": 0}
|
|
base = make_client_factory(_VALID)
|
|
|
|
def spy(role: str):
|
|
calls["n"] += 1
|
|
return base(role)
|
|
|
|
with pytest.raises(ValidationError):
|
|
await run_project(
|
|
"FV42-GSV-E1", "local", docs_dir=docs_dir, verdict_input=_VI,
|
|
client_factory=spy, top_k=0, # malformed data-source contract
|
|
)
|
|
assert calls["n"] == 0 # validation fails before any chat client is touched
|