"""Step 4 tests — token meter + budget middleware off REAL UsageDetails (no LLM). Usage is hand-built (synthetic ``UsageDetails``), so these run in CI without an endpoint. The strict-usage guard hard-fails on a missing usage; the meta-guard proves the meter is fed from ``UsageDetails`` and never a word-count proxy. Pattern: tests/spikes/test_harness.py. """ import re from pathlib import Path import pytest from agent_framework import ChatResponse, Message, UsageDetails from portfolio_optimiser.budget import ( Budget, BudgetExceeded, BudgetMiddleware, PortfolioBudget, PortfolioMeter, TokenMeter, UsageUnavailable, ) class _Ctx: """Minimal ChatContext stand-in carrying the post-call ``result``.""" def __init__(self, result: object) -> None: self.result = result async def _noop() -> None: return None def _resp(total: int | None) -> ChatResponse: usage = UsageDetails(total_token_count=total) if total is not None else None return ChatResponse(messages=[Message(role="assistant", contents=["x"])], usage_details=usage) async def test_meter_reads_total_token_count_and_accumulates() -> None: meter = TokenMeter(Budget(max_tokens=1000, max_rounds=10)) mw = BudgetMiddleware(meter) await mw.process(_Ctx(_resp(40)), _noop) # type: ignore[arg-type] await mw.process(_Ctx(_resp(50)), _noop) # type: ignore[arg-type] assert meter.tokens == 90 # read from UsageDetails, accumulated across calls async def test_cap_crossed_raises_budget_exceeded() -> None: meter = TokenMeter(Budget(max_tokens=50, max_rounds=10)) mw = BudgetMiddleware(meter) with pytest.raises(BudgetExceeded) as exc: await mw.process(_Ctx(_resp(60)), _noop) # type: ignore[arg-type] assert exc.value.kind == "tokens" assert exc.value.limit == 50 def test_non_positive_cap_rejected() -> None: with pytest.raises(ValueError): Budget(max_tokens=0, max_rounds=5) with pytest.raises(ValueError): Budget(max_tokens=5, max_rounds=0) async def test_strict_usage_none_hard_fails() -> None: meter = TokenMeter(Budget(max_tokens=100, max_rounds=10)) mw = BudgetMiddleware(meter, strict_usage=True) with pytest.raises(UsageUnavailable): await mw.process(_Ctx(_resp(None)), _noop) # type: ignore[arg-type] async def test_budget_middleware_fires_on_real_agent_chat(make_client_factory) -> None: """F8 (real-client half): registering BudgetMiddleware on a real Agent (layered chat client that carries ChatMiddlewareLayer) and running an actual chat call short-circuits with BudgetExceeded — the middleware<->client integration the prior suite never exercised (the minimal-base stand-in silently no-ops the middleware).""" from agent_framework import Agent client = make_client_factory("ok", tokens=8)("proposer") # 8 tokens/reply > cap 5 meter = TokenMeter(Budget(max_tokens=5, max_rounds=10)) agent = Agent( client, "propose a measure", name="proposer", middleware=[BudgetMiddleware(meter)] ) with pytest.raises(BudgetExceeded) as exc: await agent.run("hi") assert exc.value.kind == "tokens" assert meter.tokens == 8 # charged from the synthetic UsageDetails via the middleware async def test_pre_call_guard_does_not_await_call_next_when_exhausted() -> None: """S3.4 unit arm of the pre-call guard: the middleware refuses BEFORE ``call_next``. Counting the awaits is the whole measurement — the post-charge arm raises either way.""" calls = {"n": 0} async def _counted() -> None: calls["n"] += 1 meter = TokenMeter(Budget(max_tokens=10, max_rounds=10)) meter.charge(10) # exactly at the cap -> nothing left to fund a call with mw = BudgetMiddleware(meter) with pytest.raises(BudgetExceeded): await mw.process(_Ctx(_resp(5)), _counted) # type: ignore[arg-type] assert calls["n"] == 0 async def test_below_the_cap_still_calls_through() -> None: """Control for the guard: with budget left, the call goes through exactly as before — the guard must gate exhaustion, not traffic.""" calls = {"n": 0} async def _counted() -> None: calls["n"] += 1 meter = TokenMeter(Budget(max_tokens=10, max_rounds=10)) mw = BudgetMiddleware(meter) await mw.process(_Ctx(_resp(4)), _counted) # type: ignore[arg-type] assert calls["n"] == 1 and meter.tokens == 4 and meter.remaining() == 6 def test_portfolio_budget_rejects_unusable_configs() -> None: """Fail-fast at construction (mirroring ``Budget``): non-positive caps, a per-run cap larger than the global one (one run could then cross the pass's own ceiling), and a reserve larger than a run can ever spend (which would refuse every pass forever).""" with pytest.raises(ValueError): PortfolioBudget(max_total_tokens=0, max_tokens_per_run=10) with pytest.raises(ValueError): PortfolioBudget(max_total_tokens=100, max_tokens_per_run=0) with pytest.raises(ValueError): PortfolioBudget(max_total_tokens=100, max_tokens_per_run=200) with pytest.raises(ValueError): PortfolioBudget(max_total_tokens=100, max_tokens_per_run=50, min_run_reserve=60) with pytest.raises(ValueError): PortfolioMeter(PortfolioBudget(max_total_tokens=100, max_tokens_per_run=50), spent=-1) def test_portfolio_meter_accumulates_and_crosses() -> None: budget = PortfolioBudget(max_total_tokens=100, max_tokens_per_run=50) meter = PortfolioMeter(budget, spent=40) assert meter.remaining() == 60 and meter.required_per_run == 50 meter.record(60) meter.check() # exactly at the cap is still within it (mirrors TokenMeter's `>` boundary) meter.record(1) with pytest.raises(BudgetExceeded) as exc: meter.check() assert exc.value.kind == "portfolio_tokens" assert exc.value.limit == 100 and exc.value.observed == 101 async def test_run_meter_bound_to_portfolio_charges_both_ledgers() -> None: """A bound run meter charges the global ledger on every call, and ``remaining`` reads whichever cap binds — that is what lets one project's spend refuse another's next call.""" portfolio = PortfolioMeter(PortfolioBudget(max_total_tokens=100, max_tokens_per_run=80)) meter = TokenMeter(Budget(max_tokens=80, max_rounds=10), portfolio=portfolio) mw = BudgetMiddleware(meter) await mw.process(_Ctx(_resp(30)), _noop) # type: ignore[arg-type] assert meter.tokens == 30 and portfolio.spent == 30 assert meter.remaining() == 50 # run has 50 left, portfolio 70 -> the RUN binds await mw.process(_Ctx(_resp(45)), _noop) # type: ignore[arg-type] assert meter.remaining() == 5 # run 5, portfolio 25 -> still the run async def test_spend_is_ledgered_even_when_the_per_run_cap_raises() -> None: """Order matters: tokens the provider already billed must reach the global ledger even though the RUN's own cap raises on the same charge. Checking the run cap first and returning early would lose that spend and hand the next project a budget that was never really there.""" portfolio = PortfolioMeter(PortfolioBudget(max_total_tokens=1000, max_tokens_per_run=50)) meter = TokenMeter(Budget(max_tokens=50, max_rounds=10), portfolio=portfolio) with pytest.raises(BudgetExceeded) as exc: meter.charge(60) assert exc.value.kind == "tokens" # the RUN's cap is the one that broke assert portfolio.spent == 60 # ...and the global ledger still saw the spend def test_no_word_count_token_proxy_in_src() -> None: # The meter is fed from real UsageDetails, never a len(text.split()) word-count proxy # (research 03 Rec 3 — the Fase 1 _word_tokens anti-pattern is retired). NOTE: a bare # `.split()` grep is wrong — retrieval.py legitimately uses .split() for KEYWORD scoring # (a [0,1] overlap ratio, not a token count). So guard the specific proxy signature. proxy = re.compile(r"len\(\s*[^)]*\.split\(\)\s*\)") offenders = [ py.name for py in Path("src/portfolio_optimiser").rglob("*.py") if proxy.search(py.read_text(encoding="utf-8")) ] assert offenders == [], f"word-count token proxy found in: {offenders}"