feat(budget): enforce a global portfolio token cap before the call, not after it (S3.4/F10)

PortfolioBudget + PortfolioMeter carry ONE token ledger over a whole portfolio
pass -- and, seeded from a persisted spend file, across passes -- while the
per-run Budget/TokenMeter pair is untouched. Three enforcement points, each
doing a different job:

- startup: a remainder that cannot fund one run raises BudgetRefused before
  anything loads (a pass that can afford zero projects is a caller mistake,
  not a result);
- wave assembly: an unfundable project is NEVER STARTED and the pass stops
  structurally (budget_stop + stopped_early, completed runs preserved).
  Because every member of a wave is funded against the SAME pre-wave
  remainder, admission RESERVES each member's requirement -- otherwise a wave
  of k over-commits the cap by up to k runs;
- pre-call: BudgetMiddleware refuses a call the remainder cannot pay for
  instead of making it. The post-charge check stays: real usage is only
  knowable after the response, so the guard stops the NEXT call, never the
  one in flight.

budget_stop is its own field rather than a widened stop_reason -- a goal-stop
is success, this is resource exhaustion, and fusing them would make "we
stopped" unreadable. PortfolioMeter splits record/check so tokens the provider
already billed reach the ledger even when the same charge breaks the run's own
cap. read_spend raises on corrupt content (our own accounting state, unlike
the tolerant RAW inbox layer); write_spend takes a REQUIRED stamp with no
wall-clock default, mirroring promote_verdict.

Load-bearing MEASURED, not asserted -- 6 mutations, all red: detach the wave
check; detach the pre-call guard; detach the wave reservation; check the run
cap before crediting the global ledger; detach the startup refusal; make
read_spend tolerant. Files restored from shasum-verified copies after each.

Two findings worth keeping: the pre-call guard MASKS a detached wave check if
the test asserts on overspend (spend stays under the cap either way), so the
load-bearing assertion had to become failures == () plus never-started; and
the token arithmetic is probed (32 tokens/run at tokens=8), not guessed.

537 -> 553 tests, ruff + mypy green.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_015EaxFnaDAbMQkmTeX4u7sd
This commit is contained in:
Kjell Tore Guttormsen 2026-07-31 21:34:48 +02:00
commit a831aa1e3b
6 changed files with 733 additions and 11 deletions

View file

@ -15,6 +15,8 @@ from portfolio_optimiser.budget import (
Budget,
BudgetExceeded,
BudgetMiddleware,
PortfolioBudget,
PortfolioMeter,
TokenMeter,
UsageUnavailable,
)
@ -85,6 +87,90 @@ async def test_budget_middleware_fires_on_real_agent_chat(make_client_factory) -
assert meter.tokens == 8 # charged from the synthetic UsageDetails via the middleware
async def test_pre_call_guard_does_not_await_call_next_when_exhausted() -> None:
"""S3.4 unit arm of the pre-call guard: the middleware refuses BEFORE ``call_next``. Counting
the awaits is the whole measurement the post-charge arm raises either way."""
calls = {"n": 0}
async def _counted() -> None:
calls["n"] += 1
meter = TokenMeter(Budget(max_tokens=10, max_rounds=10))
meter.charge(10) # exactly at the cap -> nothing left to fund a call with
mw = BudgetMiddleware(meter)
with pytest.raises(BudgetExceeded):
await mw.process(_Ctx(_resp(5)), _counted) # type: ignore[arg-type]
assert calls["n"] == 0
async def test_below_the_cap_still_calls_through() -> None:
"""Control for the guard: with budget left, the call goes through exactly as before — the
guard must gate exhaustion, not traffic."""
calls = {"n": 0}
async def _counted() -> None:
calls["n"] += 1
meter = TokenMeter(Budget(max_tokens=10, max_rounds=10))
mw = BudgetMiddleware(meter)
await mw.process(_Ctx(_resp(4)), _counted) # type: ignore[arg-type]
assert calls["n"] == 1 and meter.tokens == 4 and meter.remaining() == 6
def test_portfolio_budget_rejects_unusable_configs() -> None:
"""Fail-fast at construction (mirroring ``Budget``): non-positive caps, a per-run cap larger
than the global one (one run could then cross the pass's own ceiling), and a reserve larger
than a run can ever spend (which would refuse every pass forever)."""
with pytest.raises(ValueError):
PortfolioBudget(max_total_tokens=0, max_tokens_per_run=10)
with pytest.raises(ValueError):
PortfolioBudget(max_total_tokens=100, max_tokens_per_run=0)
with pytest.raises(ValueError):
PortfolioBudget(max_total_tokens=100, max_tokens_per_run=200)
with pytest.raises(ValueError):
PortfolioBudget(max_total_tokens=100, max_tokens_per_run=50, min_run_reserve=60)
with pytest.raises(ValueError):
PortfolioMeter(PortfolioBudget(max_total_tokens=100, max_tokens_per_run=50), spent=-1)
def test_portfolio_meter_accumulates_and_crosses() -> None:
budget = PortfolioBudget(max_total_tokens=100, max_tokens_per_run=50)
meter = PortfolioMeter(budget, spent=40)
assert meter.remaining() == 60 and meter.required_per_run == 50
meter.record(60)
meter.check() # exactly at the cap is still within it (mirrors TokenMeter's `>` boundary)
meter.record(1)
with pytest.raises(BudgetExceeded) as exc:
meter.check()
assert exc.value.kind == "portfolio_tokens"
assert exc.value.limit == 100 and exc.value.observed == 101
async def test_run_meter_bound_to_portfolio_charges_both_ledgers() -> None:
"""A bound run meter charges the global ledger on every call, and ``remaining`` reads whichever
cap binds that is what lets one project's spend refuse another's next call."""
portfolio = PortfolioMeter(PortfolioBudget(max_total_tokens=100, max_tokens_per_run=80))
meter = TokenMeter(Budget(max_tokens=80, max_rounds=10), portfolio=portfolio)
mw = BudgetMiddleware(meter)
await mw.process(_Ctx(_resp(30)), _noop) # type: ignore[arg-type]
assert meter.tokens == 30 and portfolio.spent == 30
assert meter.remaining() == 50 # run has 50 left, portfolio 70 -> the RUN binds
await mw.process(_Ctx(_resp(45)), _noop) # type: ignore[arg-type]
assert meter.remaining() == 5 # run 5, portfolio 25 -> still the run
async def test_spend_is_ledgered_even_when_the_per_run_cap_raises() -> None:
"""Order matters: tokens the provider already billed must reach the global ledger even though
the RUN's own cap raises on the same charge. Checking the run cap first and returning early
would lose that spend and hand the next project a budget that was never really there."""
portfolio = PortfolioMeter(PortfolioBudget(max_total_tokens=1000, max_tokens_per_run=50))
meter = TokenMeter(Budget(max_tokens=50, max_rounds=10), portfolio=portfolio)
with pytest.raises(BudgetExceeded) as exc:
meter.charge(60)
assert exc.value.kind == "tokens" # the RUN's cap is the one that broke
assert portfolio.spent == 60 # ...and the global ledger still saw the spend
def test_no_word_count_token_proxy_in_src() -> None:
# The meter is fed from real UsageDetails, never a len(text.split()) word-count proxy
# (research 03 Rec 3 — the Fase 1 _word_tokens anti-pattern is retired). NOTE: a bare