portfolio-optimiser/tests/test_vertical_slice_e2e.py

"""Step 13 — end-to-end vertical-slice integration (deterministic, synthetic usage, no LLM).

Covers the six success criteria: (a) one valid proposal -> ValidatedProposal with populated
provenance (token usage > 0 from synthetic UsageDetails); (b) out-of-range -> Rejection with
reason; (c) Layer-2 verdict persisted; (d) a second structurally-similar run retrieves the
prior verdict via ExpeL (exercising two-arg extend_instructions); (e) tiny budget halts via
the meter/round-cap (cap not exceeded); (f) malformed contract raises before any chat call.
Pattern: tests/spikes/test_harness.py + test_c_validator.py + test_d_verdictstore.py.
"""

import pytest
from pydantic import ValidationError

from portfolio_optimiser.budget import BudgetExceeded
from portfolio_optimiser.run import run_project
from portfolio_optimiser.validator import Rejection, ValidatedProposal

_VALID = (
    '{"project_id":"KONTOR-IT-E1","measure":"Reduce scope",'
    '"affected_items":[{"code":"05.2","quantity":4300,"unit_cost":215},'
    '{"code":"03.1","quantity":1800,"unit_cost":310}],"claimed_saving_nok":200000}'
)
_OUT_OF_RANGE = _VALID.replace("200000", "800000")  # parses fine, exceeds the ~445k P90

_VI = {"decision": "approved", "rationale": "feasible within range"}


async def test_a_valid_proposal_end_to_end(docs_dir, make_client_factory, fresh_store) -> None:
    result = await run_project(
        "KONTOR-IT-E1",
        "local",
        docs_dir=docs_dir,
        verdict_input=_VI,
        client_factory=make_client_factory(_VALID),
        store=fresh_store,
    )
    assert isinstance(result.outcome, ValidatedProposal)
    stamp = result.provenance
    assert len(stamp.citations) >= 1 and stamp.model and stamp.role
    assert stamp.validator_decision == "validated"
    assert stamp.token_usage > 0  # sourced from the synthetic UsageDetails


async def test_a1_provenance_stamps_injected_client_model_not_sentinel(
    docs_dir, make_client_factory, fresh_store
) -> None:
    """F1 regression: an injected client_factory must stamp the injected client's REAL model
    (SyntheticUsageChatClient.model == 'synthetic'), never the 'fake-model' literal — the leak
    falsified provenance on the public deployer seam (run.py:197)."""
    result = await run_project(
        "KONTOR-IT-E1",
        "local",
        docs_dir=docs_dir,
        verdict_input=_VI,
        client_factory=make_client_factory(_VALID),
        store=fresh_store,
    )
    assert result.provenance.model == "synthetic"
    assert result.provenance.model != "fake-model"


async def test_b_out_of_range_is_rejected(docs_dir, make_client_factory, fresh_store) -> None:
    result = await run_project(
        "KONTOR-IT-E1",
        "local",
        docs_dir=docs_dir,
        verdict_input={"decision": "rejected", "rationale": "claim too high"},
        client_factory=make_client_factory(_OUT_OF_RANGE),
        store=fresh_store,
    )
    assert isinstance(result.outcome, Rejection)
    assert result.outcome.reason


async def test_c_layer2_verdict_is_persisted(docs_dir, make_client_factory, fresh_store) -> None:
    result = await run_project(
        "KONTOR-IT-E1",
        "local",
        docs_dir=docs_dir,
        verdict_input=_VI,
        client_factory=make_client_factory(_VALID),
        store=fresh_store,
    )
    assert any(v.id == result.verdict.id for v in result.store.verdicts)


async def test_d_second_run_retrieves_prior_verdict(
    docs_dir, make_client_factory, fresh_store
) -> None:
    factory = make_client_factory(_VALID)
    r1 = await run_project(
        "KONTOR-IT-E1",
        "local",
        docs_dir=docs_dir,
        verdict_input=_VI,
        client_factory=factory,
        store=fresh_store,
    )
    r2 = await run_project(
        "KONTOR-IT-E1",
        "local",
        docs_dir=docs_dir,
        verdict_input=_VI,
        client_factory=factory,
        store=fresh_store,
    )
    assert r2.retrieved  # ExpeL surfaced prior verdicts on the second run
    assert r2.retrieved[0].id == r1.verdict.id  # structurally identical -> stable id


async def test_e_tiny_budget_halts_without_exceeding(
    docs_dir, make_client_factory, fresh_store
) -> None:
    with pytest.raises(BudgetExceeded):
        await run_project(
            "KONTOR-IT-E1",
            "local",
            docs_dir=docs_dir,
            verdict_input=_VI,
            client_factory=make_client_factory(_VALID, tokens=100),
            store=fresh_store,
            max_tokens=10,  # synthetic 100 tokens > cap 10 -> halt
        )


async def test_wiring_budget_middleware_and_retrieval_tool(
    docs_dir, make_client_factory, fresh_store, monkeypatch
) -> None:
    """Step 3 (F2/F5/F7 wiring): run_project constructs a BudgetMiddleware + a retrieval
    FunctionTool and passes BOTH into fresh_workflow. Spy the factory call; build a real
    workflow so the rest of the run completes."""
    from agent_framework import FunctionTool

    from portfolio_optimiser import run as run_mod
    from portfolio_optimiser.budget import BudgetMiddleware

    captured: dict[str, object] = {}
    real_fresh = run_mod.fresh_workflow

    def spy_fresh(factory, **kwargs):
        captured.update(kwargs)
        return real_fresh(factory, **kwargs)

    monkeypatch.setattr(run_mod, "fresh_workflow", spy_fresh)
    await run_project(
        "KONTOR-IT-E1",
        "local",
        docs_dir=docs_dir,
        verdict_input=_VI,
        client_factory=make_client_factory(_VALID),
        store=fresh_store,
    )
    middleware = captured.get("middleware") or []
    tools = captured.get("tools") or []
    assert any(isinstance(m, BudgetMiddleware) for m in middleware)
    assert any(isinstance(t, FunctionTool) for t in tools)


async def test_g_validated_proposal_derives_from_debate(
    docs_dir, make_client_factory, fresh_store, monkeypatch
) -> None:
    """F1: the candidate fed to generation derives from the DEBATE, not just retrieval. Spy
    the context passed to generate_via_llm; the proposer's converged output ('Reduce scope',
    present in _VALID but NOT in the docs_dir fixture) must reach generation. Deleting the
    debate->generation wiring makes this fail."""
    from portfolio_optimiser import run as run_mod

    captured: dict[str, str] = {}
    real_generate = run_mod.generate_via_llm

    async def spy_generate(chat_client, project, context, meter, **kw):
        captured["context"] = context
        return await real_generate(chat_client, project, context, meter, **kw)

    monkeypatch.setattr(run_mod, "generate_via_llm", spy_generate)
    result = await run_project(
        "KONTOR-IT-E1",
        "local",
        docs_dir=docs_dir,
        verdict_input=_VI,
        client_factory=make_client_factory(_VALID),
        store=fresh_store,
    )
    debate_marker = "Reduce scope"  # in _VALID (proposer output); NOT in docs_dir
    assert result.debate_output and debate_marker in result.debate_output
    assert debate_marker in captured["context"]  # debate output reached generation
    assert isinstance(result.outcome, ValidatedProposal)


async def test_h_tiny_budget_halts_via_debate_middleware(
    docs_dir, make_client_factory, fresh_store, monkeypatch
) -> None:
    """F8 (debate-path half): a tiny budget must short-circuit in the DEBATE middleware, not the
    later generate loop. Fence generation — if generate_via_llm is reached, the debate did NOT
    short-circuit, so fail. Because generate_via_llm independently raises BudgetExceeded, a plain
    pytest.raises would pass even with the debate middleware detached; the fence makes this test
    load-bearing (detaching the middleware fails it)."""
    from portfolio_optimiser import run as run_mod

    async def _forbidden(*args, **kwargs):
        raise AssertionError(
            "generate_via_llm reached — debate did not short-circuit via middleware"
        )

    monkeypatch.setattr(run_mod, "generate_via_llm", _forbidden)
    with pytest.raises(BudgetExceeded):
        await run_project(
            "KONTOR-IT-E1",
            "local",
            docs_dir=docs_dir,
            verdict_input=_VI,
            client_factory=make_client_factory(
                _VALID, tokens=100
            ),  # 100 > cap 10, on the FIRST debate call
            store=fresh_store,
            max_tokens=10,
        )


async def test_i_injected_meter_is_used(docs_dir, make_client_factory) -> None:
    """Step 1 (SC3 detach hook): a TokenMeter passed as ``meter=`` IS the meter the run uses —
    its pre-charged tokens flow through to ``provenance.token_usage`` (run.py:179). Detaches
    cleanly: reverting the run.py:151 injection makes the injected instance be ignored (a fresh
    internal meter is built instead), the pre-charge vanishes, and the equality below fails.
    Pattern: test_a (run_project call shape)."""
    from portfolio_optimiser.budget import Budget, TokenMeter

    factory = make_client_factory(_VALID)
    baseline = await run_project(
        "KONTOR-IT-E1",
        "local",
        docs_dir=docs_dir,
        verdict_input=_VI,
        client_factory=factory,
    )
    pre_charged = 5000
    injected = TokenMeter(Budget(max_tokens=100_000, max_rounds=max(3 * 4, 4)))
    injected.charge(pre_charged)
    result = await run_project(
        "KONTOR-IT-E1",
        "local",
        docs_dir=docs_dir,
        verdict_input=_VI,
        client_factory=factory,
        meter=injected,
    )
    assert baseline.provenance.token_usage > 0  # the run charges a deterministic delta
    # The injected instance flows through: end tokens == pre-charge + the same run delta.
    assert result.provenance.token_usage == pre_charged + baseline.provenance.token_usage
    assert result.provenance.token_usage > baseline.provenance.token_usage


async def test_f_malformed_contract_raises_before_any_chat(docs_dir, make_client_factory) -> None:
    calls = {"n": 0}
    base = make_client_factory(_VALID)

    def spy(role: str):
        calls["n"] += 1
        return base(role)

    with pytest.raises(ValidationError):
        await run_project(
            "KONTOR-IT-E1",
            "local",
            docs_dir=docs_dir,
            verdict_input=_VI,
            client_factory=spy,
            top_k=0,  # malformed data-source contract
        )
    assert calls["n"] == 0  # validation fails before any chat client is touched