"""Step 13 — end-to-end vertical-slice integration (deterministic, synthetic usage, no LLM). Covers the six success criteria: (a) one valid proposal -> ValidatedProposal with populated provenance (token usage > 0 from synthetic UsageDetails); (b) out-of-range -> Rejection with reason; (c) Layer-2 verdict persisted; (d) a second structurally-similar run retrieves the prior verdict via ExpeL (exercising two-arg extend_instructions); (e) tiny budget halts via the meter/round-cap (cap not exceeded); (f) malformed contract raises before any chat call. Pattern: tests/spikes/test_harness.py + test_c_validator.py + test_d_verdictstore.py. """ import pytest from pydantic import ValidationError from portfolio_optimiser.budget import BudgetExceeded from portfolio_optimiser.run import run_project from portfolio_optimiser.validator import Rejection, ValidatedProposal _VALID = ( '{"project_id":"FV42-GSV-E1","measure":"Reduce scope",' '"affected_items":[{"code":"05.2","quantity":4300,"unit_cost":215},' '{"code":"03.1","quantity":1800,"unit_cost":310}],"claimed_saving_nok":200000}' ) _OUT_OF_RANGE = _VALID.replace("200000", "800000") # parses fine, exceeds the ~445k P90 _VI = {"decision": "approved", "rationale": "feasible within range"} async def test_a_valid_proposal_end_to_end(docs_dir, make_client_factory, fresh_store) -> None: result = await run_project( "FV42-GSV-E1", "local", docs_dir=docs_dir, verdict_input=_VI, client_factory=make_client_factory(_VALID), store=fresh_store, ) assert isinstance(result.outcome, ValidatedProposal) stamp = result.provenance assert len(stamp.citations) >= 1 and stamp.model and stamp.role assert stamp.validator_decision == "validated" assert stamp.token_usage > 0 # sourced from the synthetic UsageDetails async def test_a1_provenance_stamps_injected_client_model_not_sentinel( docs_dir, make_client_factory, fresh_store ) -> None: """F1 regression: an injected client_factory must stamp the injected client's REAL model (SyntheticUsageChatClient.model == 'synthetic'), never the 'fake-model' literal — the leak falsified provenance on the public deployer seam (run.py:197).""" result = await run_project( "FV42-GSV-E1", "local", docs_dir=docs_dir, verdict_input=_VI, client_factory=make_client_factory(_VALID), store=fresh_store, ) assert result.provenance.model == "synthetic" assert result.provenance.model != "fake-model" async def test_b_out_of_range_is_rejected(docs_dir, make_client_factory, fresh_store) -> None: result = await run_project( "FV42-GSV-E1", "local", docs_dir=docs_dir, verdict_input={"decision": "rejected", "rationale": "claim too high"}, client_factory=make_client_factory(_OUT_OF_RANGE), store=fresh_store, ) assert isinstance(result.outcome, Rejection) assert result.outcome.reason async def test_c_layer2_verdict_is_persisted(docs_dir, make_client_factory, fresh_store) -> None: result = await run_project( "FV42-GSV-E1", "local", docs_dir=docs_dir, verdict_input=_VI, client_factory=make_client_factory(_VALID), store=fresh_store, ) assert any(v.id == result.verdict.id for v in result.store.verdicts) async def test_d_second_run_retrieves_prior_verdict( docs_dir, make_client_factory, fresh_store ) -> None: factory = make_client_factory(_VALID) r1 = await run_project( "FV42-GSV-E1", "local", docs_dir=docs_dir, verdict_input=_VI, client_factory=factory, store=fresh_store, ) r2 = await run_project( "FV42-GSV-E1", "local", docs_dir=docs_dir, verdict_input=_VI, client_factory=factory, store=fresh_store, ) assert r2.retrieved # ExpeL surfaced prior verdicts on the second run assert r2.retrieved[0].id == r1.verdict.id # structurally identical -> stable id async def test_e_tiny_budget_halts_without_exceeding( docs_dir, make_client_factory, fresh_store ) -> None: with pytest.raises(BudgetExceeded): await run_project( "FV42-GSV-E1", "local", docs_dir=docs_dir, verdict_input=_VI, client_factory=make_client_factory(_VALID, tokens=100), store=fresh_store, max_tokens=10, # synthetic 100 tokens > cap 10 -> halt ) async def test_wiring_budget_middleware_and_retrieval_tool( docs_dir, make_client_factory, fresh_store, monkeypatch ) -> None: """Step 3 (F2/F5/F7 wiring): run_project constructs a BudgetMiddleware + a retrieval FunctionTool and passes BOTH into fresh_workflow. Spy the factory call; build a real workflow so the rest of the run completes.""" from agent_framework import FunctionTool from portfolio_optimiser import run as run_mod from portfolio_optimiser.budget import BudgetMiddleware captured: dict[str, object] = {} real_fresh = run_mod.fresh_workflow def spy_fresh(factory, **kwargs): captured.update(kwargs) return real_fresh(factory, **kwargs) monkeypatch.setattr(run_mod, "fresh_workflow", spy_fresh) await run_project( "FV42-GSV-E1", "local", docs_dir=docs_dir, verdict_input=_VI, client_factory=make_client_factory(_VALID), store=fresh_store, ) middleware = captured.get("middleware") or [] tools = captured.get("tools") or [] assert any(isinstance(m, BudgetMiddleware) for m in middleware) assert any(isinstance(t, FunctionTool) for t in tools) async def test_g_validated_proposal_derives_from_debate( docs_dir, make_client_factory, fresh_store, monkeypatch ) -> None: """F1: the candidate fed to generation derives from the DEBATE, not just retrieval. Spy the context passed to generate_via_llm; the proposer's converged output ('Reduce scope', present in _VALID but NOT in the docs_dir fixture) must reach generation. Deleting the debate->generation wiring makes this fail.""" from portfolio_optimiser import run as run_mod captured: dict[str, str] = {} real_generate = run_mod.generate_via_llm async def spy_generate(chat_client, project, context, meter, **kw): captured["context"] = context return await real_generate(chat_client, project, context, meter, **kw) monkeypatch.setattr(run_mod, "generate_via_llm", spy_generate) result = await run_project( "FV42-GSV-E1", "local", docs_dir=docs_dir, verdict_input=_VI, client_factory=make_client_factory(_VALID), store=fresh_store, ) debate_marker = "Reduce scope" # in _VALID (proposer output); NOT in docs_dir assert result.debate_output and debate_marker in result.debate_output assert debate_marker in captured["context"] # debate output reached generation assert isinstance(result.outcome, ValidatedProposal) async def test_h_tiny_budget_halts_via_debate_middleware( docs_dir, make_client_factory, fresh_store, monkeypatch ) -> None: """F8 (debate-path half): a tiny budget must short-circuit in the DEBATE middleware, not the later generate loop. Fence generation — if generate_via_llm is reached, the debate did NOT short-circuit, so fail. Because generate_via_llm independently raises BudgetExceeded, a plain pytest.raises would pass even with the debate middleware detached; the fence makes this test load-bearing (detaching the middleware fails it).""" from portfolio_optimiser import run as run_mod async def _forbidden(*args, **kwargs): raise AssertionError( "generate_via_llm reached — debate did not short-circuit via middleware" ) monkeypatch.setattr(run_mod, "generate_via_llm", _forbidden) with pytest.raises(BudgetExceeded): await run_project( "FV42-GSV-E1", "local", docs_dir=docs_dir, verdict_input=_VI, client_factory=make_client_factory( _VALID, tokens=100 ), # 100 > cap 10, on the FIRST debate call store=fresh_store, max_tokens=10, ) async def test_i_injected_meter_is_used(docs_dir, make_client_factory) -> None: """Step 1 (SC3 detach hook): a TokenMeter passed as ``meter=`` IS the meter the run uses — its pre-charged tokens flow through to ``provenance.token_usage`` (run.py:179). Detaches cleanly: reverting the run.py:151 injection makes the injected instance be ignored (a fresh internal meter is built instead), the pre-charge vanishes, and the equality below fails. Pattern: test_a (run_project call shape).""" from portfolio_optimiser.budget import Budget, TokenMeter factory = make_client_factory(_VALID) baseline = await run_project( "FV42-GSV-E1", "local", docs_dir=docs_dir, verdict_input=_VI, client_factory=factory, ) pre_charged = 5000 injected = TokenMeter(Budget(max_tokens=100_000, max_rounds=max(3 * 4, 4))) injected.charge(pre_charged) result = await run_project( "FV42-GSV-E1", "local", docs_dir=docs_dir, verdict_input=_VI, client_factory=factory, meter=injected, ) assert baseline.provenance.token_usage > 0 # the run charges a deterministic delta # The injected instance flows through: end tokens == pre-charge + the same run delta. assert result.provenance.token_usage == pre_charged + baseline.provenance.token_usage assert result.provenance.token_usage > baseline.provenance.token_usage async def test_f_malformed_contract_raises_before_any_chat(docs_dir, make_client_factory) -> None: calls = {"n": 0} base = make_client_factory(_VALID) def spy(role: str): calls["n"] += 1 return base(role) with pytest.raises(ValidationError): await run_project( "FV42-GSV-E1", "local", docs_dir=docs_dir, verdict_input=_VI, client_factory=spy, top_k=0, # malformed data-source contract ) assert calls["n"] == 0 # validation fails before any chat client is touched