diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..6eaf346 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,107 @@ +"""Shared e2e fixtures (Step 13): a scripted chat client that emits a SYNTHETIC UsageDetails +(so token accounting is real-shaped without an LLM), plus store + docs-dir fixtures. + +The synthetic ``UsageDetails`` is what lets the budget meter / provenance ``token_usage`` be a +positive, UsageDetails-sourced number in CI — the REAL-provider populated-usage assertion is +the gated live arm (Step 14). +""" + +from __future__ import annotations + +from collections.abc import Callable, Sequence +from typing import Any + +import pytest +from agent_framework import ( + BaseChatClient, + ChatResponse, + ChatResponseUpdate, + Message, + UsageDetails, +) + +from portfolio_optimiser.verdicts import VerdictStore, seed_store + + +class SyntheticUsageChatClient(BaseChatClient): + """Network-free chat client returning scripted/default replies WITH a synthetic + ``UsageDetails`` (``total_token_count``), so strict usage accounting does not hard-fail.""" + + OTEL_PROVIDER_NAME = "synthetic" + + def __init__( + self, + scripted: Sequence[str] | None = None, + *, + default_reply: str = "ok", + tokens_per_reply: int = 8, + ) -> None: + super().__init__() + self._scripted = list(scripted or []) + self._idx = 0 + self._default = default_reply + self._tokens = tokens_per_reply + self.call_count = 0 + + def _next_reply(self) -> str: + reply = self._scripted[self._idx] if self._idx < len(self._scripted) else self._default + self._idx += 1 + self.call_count += 1 + return reply + + def _inner_get_response( + self, *, messages: Sequence[Message], stream: bool, options: Any, **kwargs: Any + ) -> Any: + reply = self._next_reply() + usage = UsageDetails(total_token_count=self._tokens) + if stream: + + async def _agen() -> Any: + yield ChatResponseUpdate( + role="assistant", contents=[{"type": "text", "text": reply}] + ) + + return self._build_response_stream(_agen()) + + async def _coro() -> ChatResponse: + return ChatResponse( + messages=[Message(role="assistant", contents=[reply])], + response_id="synthetic", + usage_details=usage, + ) + + return _coro() + + +@pytest.fixture() +def make_client_factory() -> Callable[..., Callable[[str], BaseChatClient]]: + """Return a maker that builds a per-role client factory emitting synthetic usage.""" + + def _make(default_reply: str, *, tokens: int = 8) -> Callable[[str], BaseChatClient]: + def factory(role: str) -> BaseChatClient: + return SyntheticUsageChatClient(default_reply=default_reply, tokens_per_reply=tokens) + + return factory + + return _make + + +@pytest.fixture() +def fresh_store() -> VerdictStore: + return VerdictStore(verdicts=[]) + + +@pytest.fixture() +def seeded_store() -> VerdictStore: + return seed_store() + + +@pytest.fixture() +def docs_dir(tmp_path) -> str: + d = tmp_path / "docs" + d.mkdir() + (d / "cost.txt").write_text( + "Asphalt Ab11 unit rate renegotiation reduced the paving cost on the school stretch.", + encoding="utf-8", + ) + return str(d) diff --git a/tests/test_vertical_slice_e2e.py b/tests/test_vertical_slice_e2e.py new file mode 100644 index 0000000..bc74d05 --- /dev/null +++ b/tests/test_vertical_slice_e2e.py @@ -0,0 +1,100 @@ +"""Step 13 — end-to-end vertical-slice integration (deterministic, synthetic usage, no LLM). + +Covers the six success criteria: (a) one valid proposal -> ValidatedProposal with populated +provenance (token usage > 0 from synthetic UsageDetails); (b) out-of-range -> Rejection with +reason; (c) Layer-2 verdict persisted; (d) a second structurally-similar run retrieves the +prior verdict via ExpeL (exercising two-arg extend_instructions); (e) tiny budget halts via +the meter/round-cap (cap not exceeded); (f) malformed contract raises before any chat call. +Pattern: tests/spikes/test_harness.py + test_c_validator.py + test_d_verdictstore.py. +""" + +import pytest +from pydantic import ValidationError + +from portfolio_optimiser.budget import BudgetExceeded +from portfolio_optimiser.run import run_project +from portfolio_optimiser.validator import Rejection, ValidatedProposal + +_VALID = ( + '{"project_id":"FV42-GSV-E1","measure":"Reduce scope",' + '"affected_items":[{"code":"05.2","quantity":4300,"unit_cost":215},' + '{"code":"03.1","quantity":1800,"unit_cost":310}],"claimed_saving_nok":200000}' +) +_OUT_OF_RANGE = _VALID.replace("200000", "800000") # parses fine, exceeds the ~445k P90 + +_VI = {"decision": "approved", "rationale": "feasible within range"} + + +async def test_a_valid_proposal_end_to_end(docs_dir, make_client_factory, fresh_store) -> None: + result = await run_project( + "FV42-GSV-E1", "local", docs_dir=docs_dir, verdict_input=_VI, + client_factory=make_client_factory(_VALID), store=fresh_store, + ) + assert isinstance(result.outcome, ValidatedProposal) + stamp = result.provenance + assert len(stamp.citations) >= 1 and stamp.model and stamp.role + assert stamp.validator_decision == "validated" + assert stamp.token_usage > 0 # sourced from the synthetic UsageDetails + + +async def test_b_out_of_range_is_rejected(docs_dir, make_client_factory, fresh_store) -> None: + result = await run_project( + "FV42-GSV-E1", "local", docs_dir=docs_dir, + verdict_input={"decision": "rejected", "rationale": "claim too high"}, + client_factory=make_client_factory(_OUT_OF_RANGE), store=fresh_store, + ) + assert isinstance(result.outcome, Rejection) + assert result.outcome.reason + + +async def test_c_layer2_verdict_is_persisted(docs_dir, make_client_factory, fresh_store) -> None: + result = await run_project( + "FV42-GSV-E1", "local", docs_dir=docs_dir, verdict_input=_VI, + client_factory=make_client_factory(_VALID), store=fresh_store, + ) + assert any(v.id == result.verdict.id for v in result.store.verdicts) + + +async def test_d_second_run_retrieves_prior_verdict( + docs_dir, make_client_factory, fresh_store +) -> None: + factory = make_client_factory(_VALID) + r1 = await run_project( + "FV42-GSV-E1", "local", docs_dir=docs_dir, verdict_input=_VI, + client_factory=factory, store=fresh_store, + ) + r2 = await run_project( + "FV42-GSV-E1", "local", docs_dir=docs_dir, verdict_input=_VI, + client_factory=factory, store=fresh_store, + ) + assert r2.retrieved # ExpeL surfaced prior verdicts on the second run + assert r2.retrieved[0].id == r1.verdict.id # structurally identical -> stable id + + +async def test_e_tiny_budget_halts_without_exceeding( + docs_dir, make_client_factory, fresh_store +) -> None: + with pytest.raises(BudgetExceeded): + await run_project( + "FV42-GSV-E1", "local", docs_dir=docs_dir, verdict_input=_VI, + client_factory=make_client_factory(_VALID, tokens=100), + store=fresh_store, max_tokens=10, # synthetic 100 tokens > cap 10 -> halt + ) + + +async def test_f_malformed_contract_raises_before_any_chat( + docs_dir, make_client_factory +) -> None: + calls = {"n": 0} + base = make_client_factory(_VALID) + + def spy(role: str): + calls["n"] += 1 + return base(role) + + with pytest.raises(ValidationError): + await run_project( + "FV42-GSV-E1", "local", docs_dir=docs_dir, verdict_input=_VI, + client_factory=spy, top_k=0, # malformed data-source contract + ) + assert calls["n"] == 0 # validation fails before any chat client is touched