feat(budget): enforce a global portfolio token cap before the call, not after it (S3.4/F10)
PortfolioBudget + PortfolioMeter carry ONE token ledger over a whole portfolio pass -- and, seeded from a persisted spend file, across passes -- while the per-run Budget/TokenMeter pair is untouched. Three enforcement points, each doing a different job: - startup: a remainder that cannot fund one run raises BudgetRefused before anything loads (a pass that can afford zero projects is a caller mistake, not a result); - wave assembly: an unfundable project is NEVER STARTED and the pass stops structurally (budget_stop + stopped_early, completed runs preserved). Because every member of a wave is funded against the SAME pre-wave remainder, admission RESERVES each member's requirement -- otherwise a wave of k over-commits the cap by up to k runs; - pre-call: BudgetMiddleware refuses a call the remainder cannot pay for instead of making it. The post-charge check stays: real usage is only knowable after the response, so the guard stops the NEXT call, never the one in flight. budget_stop is its own field rather than a widened stop_reason -- a goal-stop is success, this is resource exhaustion, and fusing them would make "we stopped" unreadable. PortfolioMeter splits record/check so tokens the provider already billed reach the ledger even when the same charge breaks the run's own cap. read_spend raises on corrupt content (our own accounting state, unlike the tolerant RAW inbox layer); write_spend takes a REQUIRED stamp with no wall-clock default, mirroring promote_verdict. Load-bearing MEASURED, not asserted -- 6 mutations, all red: detach the wave check; detach the pre-call guard; detach the wave reservation; check the run cap before crediting the global ledger; detach the startup refusal; make read_spend tolerant. Files restored from shasum-verified copies after each. Two findings worth keeping: the pre-call guard MASKS a detached wave check if the test asserts on overspend (spend stays under the cap either way), so the load-bearing assertion had to become failures == () plus never-started; and the token arithmetic is probed (32 tokens/run at tokens=8), not guessed. 537 -> 553 tests, ruff + mypy green. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_015EaxFnaDAbMQkmTeX4u7sd
This commit is contained in:
parent
0d50ab89d3
commit
a831aa1e3b
6 changed files with 733 additions and 11 deletions
|
|
@ -15,6 +15,8 @@ from portfolio_optimiser.budget import (
|
|||
Budget,
|
||||
BudgetExceeded,
|
||||
BudgetMiddleware,
|
||||
PortfolioBudget,
|
||||
PortfolioMeter,
|
||||
TokenMeter,
|
||||
UsageUnavailable,
|
||||
)
|
||||
|
|
@ -85,6 +87,90 @@ async def test_budget_middleware_fires_on_real_agent_chat(make_client_factory) -
|
|||
assert meter.tokens == 8 # charged from the synthetic UsageDetails via the middleware
|
||||
|
||||
|
||||
async def test_pre_call_guard_does_not_await_call_next_when_exhausted() -> None:
|
||||
"""S3.4 unit arm of the pre-call guard: the middleware refuses BEFORE ``call_next``. Counting
|
||||
the awaits is the whole measurement — the post-charge arm raises either way."""
|
||||
calls = {"n": 0}
|
||||
|
||||
async def _counted() -> None:
|
||||
calls["n"] += 1
|
||||
|
||||
meter = TokenMeter(Budget(max_tokens=10, max_rounds=10))
|
||||
meter.charge(10) # exactly at the cap -> nothing left to fund a call with
|
||||
mw = BudgetMiddleware(meter)
|
||||
with pytest.raises(BudgetExceeded):
|
||||
await mw.process(_Ctx(_resp(5)), _counted) # type: ignore[arg-type]
|
||||
assert calls["n"] == 0
|
||||
|
||||
|
||||
async def test_below_the_cap_still_calls_through() -> None:
|
||||
"""Control for the guard: with budget left, the call goes through exactly as before — the
|
||||
guard must gate exhaustion, not traffic."""
|
||||
calls = {"n": 0}
|
||||
|
||||
async def _counted() -> None:
|
||||
calls["n"] += 1
|
||||
|
||||
meter = TokenMeter(Budget(max_tokens=10, max_rounds=10))
|
||||
mw = BudgetMiddleware(meter)
|
||||
await mw.process(_Ctx(_resp(4)), _counted) # type: ignore[arg-type]
|
||||
assert calls["n"] == 1 and meter.tokens == 4 and meter.remaining() == 6
|
||||
|
||||
|
||||
def test_portfolio_budget_rejects_unusable_configs() -> None:
|
||||
"""Fail-fast at construction (mirroring ``Budget``): non-positive caps, a per-run cap larger
|
||||
than the global one (one run could then cross the pass's own ceiling), and a reserve larger
|
||||
than a run can ever spend (which would refuse every pass forever)."""
|
||||
with pytest.raises(ValueError):
|
||||
PortfolioBudget(max_total_tokens=0, max_tokens_per_run=10)
|
||||
with pytest.raises(ValueError):
|
||||
PortfolioBudget(max_total_tokens=100, max_tokens_per_run=0)
|
||||
with pytest.raises(ValueError):
|
||||
PortfolioBudget(max_total_tokens=100, max_tokens_per_run=200)
|
||||
with pytest.raises(ValueError):
|
||||
PortfolioBudget(max_total_tokens=100, max_tokens_per_run=50, min_run_reserve=60)
|
||||
with pytest.raises(ValueError):
|
||||
PortfolioMeter(PortfolioBudget(max_total_tokens=100, max_tokens_per_run=50), spent=-1)
|
||||
|
||||
|
||||
def test_portfolio_meter_accumulates_and_crosses() -> None:
|
||||
budget = PortfolioBudget(max_total_tokens=100, max_tokens_per_run=50)
|
||||
meter = PortfolioMeter(budget, spent=40)
|
||||
assert meter.remaining() == 60 and meter.required_per_run == 50
|
||||
meter.record(60)
|
||||
meter.check() # exactly at the cap is still within it (mirrors TokenMeter's `>` boundary)
|
||||
meter.record(1)
|
||||
with pytest.raises(BudgetExceeded) as exc:
|
||||
meter.check()
|
||||
assert exc.value.kind == "portfolio_tokens"
|
||||
assert exc.value.limit == 100 and exc.value.observed == 101
|
||||
|
||||
|
||||
async def test_run_meter_bound_to_portfolio_charges_both_ledgers() -> None:
|
||||
"""A bound run meter charges the global ledger on every call, and ``remaining`` reads whichever
|
||||
cap binds — that is what lets one project's spend refuse another's next call."""
|
||||
portfolio = PortfolioMeter(PortfolioBudget(max_total_tokens=100, max_tokens_per_run=80))
|
||||
meter = TokenMeter(Budget(max_tokens=80, max_rounds=10), portfolio=portfolio)
|
||||
mw = BudgetMiddleware(meter)
|
||||
await mw.process(_Ctx(_resp(30)), _noop) # type: ignore[arg-type]
|
||||
assert meter.tokens == 30 and portfolio.spent == 30
|
||||
assert meter.remaining() == 50 # run has 50 left, portfolio 70 -> the RUN binds
|
||||
await mw.process(_Ctx(_resp(45)), _noop) # type: ignore[arg-type]
|
||||
assert meter.remaining() == 5 # run 5, portfolio 25 -> still the run
|
||||
|
||||
|
||||
async def test_spend_is_ledgered_even_when_the_per_run_cap_raises() -> None:
|
||||
"""Order matters: tokens the provider already billed must reach the global ledger even though
|
||||
the RUN's own cap raises on the same charge. Checking the run cap first and returning early
|
||||
would lose that spend and hand the next project a budget that was never really there."""
|
||||
portfolio = PortfolioMeter(PortfolioBudget(max_total_tokens=1000, max_tokens_per_run=50))
|
||||
meter = TokenMeter(Budget(max_tokens=50, max_rounds=10), portfolio=portfolio)
|
||||
with pytest.raises(BudgetExceeded) as exc:
|
||||
meter.charge(60)
|
||||
assert exc.value.kind == "tokens" # the RUN's cap is the one that broke
|
||||
assert portfolio.spent == 60 # ...and the global ledger still saw the spend
|
||||
|
||||
|
||||
def test_no_word_count_token_proxy_in_src() -> None:
|
||||
# The meter is fed from real UsageDetails, never a len(text.split()) word-count proxy
|
||||
# (research 03 Rec 3 — the Fase 1 _word_tokens anti-pattern is retired). NOTE: a bare
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue