126 lines
4.2 KiB
Python
126 lines
4.2 KiB
Python
"""Step 2: shared harness invariants + the builder smoke that de-risks Spikes A/B.
|
|
|
|
The builder smoke is the one place that proves ``FakeChatClient`` can actually drive
|
|
the GA ``GroupChatBuilder`` / ``ConcurrentBuilder``. If it could not, that would itself
|
|
be a primary de-risk finding (escalate) — but it does (see findings note).
|
|
|
|
Pattern: tests/test_backends.py (parametrize + pytest.raises).
|
|
"""
|
|
|
|
import pytest
|
|
from agent_framework import Message
|
|
from agent_framework.orchestrations import ConcurrentBuilder, GroupChatBuilder
|
|
|
|
from spikes._harness import (
|
|
Budget,
|
|
BudgetExceeded,
|
|
FakeChatClient,
|
|
TokenMeter,
|
|
fake_agent,
|
|
live_local_client_or_skip,
|
|
)
|
|
|
|
|
|
# --- Budget: refuse to start without positive caps (A4 / fail-fast) ---
|
|
|
|
|
|
@pytest.mark.parametrize("caps", [(0, 5), (5, 0), (-1, 5), (5, -3)])
|
|
def test_budget_rejects_non_positive_caps(caps: tuple[int, int]) -> None:
|
|
with pytest.raises(ValueError):
|
|
Budget(*caps)
|
|
|
|
|
|
def test_budget_accepts_positive_caps() -> None:
|
|
b = Budget(max_tokens=100, max_rounds=3)
|
|
assert b.max_tokens == 100
|
|
assert b.max_rounds == 3
|
|
|
|
|
|
# --- TokenMeter: structured stop the moment a cap is crossed (B4) ---
|
|
|
|
|
|
def test_token_meter_charges_until_cap() -> None:
|
|
meter = TokenMeter(Budget(max_tokens=10, max_rounds=3))
|
|
assert meter.charge(4) == 4
|
|
assert meter.charge(6) == 10 # exactly at cap is fine
|
|
with pytest.raises(BudgetExceeded) as exc:
|
|
meter.charge(1)
|
|
assert exc.value.kind == "tokens"
|
|
assert exc.value.limit == 10
|
|
assert exc.value.observed == 11
|
|
|
|
|
|
def test_token_meter_ticks_rounds_until_cap() -> None:
|
|
meter = TokenMeter(Budget(max_tokens=100, max_rounds=2))
|
|
assert meter.tick_round() == 1
|
|
assert meter.tick_round() == 2
|
|
with pytest.raises(BudgetExceeded) as exc:
|
|
meter.tick_round()
|
|
assert exc.value.kind == "rounds"
|
|
assert exc.value.limit == 2
|
|
|
|
|
|
# --- FakeChatClient: deterministic scripted replies + word-count tokens ---
|
|
|
|
|
|
async def test_fake_client_returns_scripted_replies_in_order() -> None:
|
|
client = FakeChatClient(["one two", "three"], default_reply="fallback word")
|
|
r1 = await client.get_response([Message(role="user", contents=["hi"])])
|
|
r2 = await client.get_response([Message(role="user", contents=["hi"])])
|
|
r3 = await client.get_response([Message(role="user", contents=["hi"])]) # script exhausted
|
|
assert r1.text == "one two"
|
|
assert r2.text == "three"
|
|
assert r3.text == "fallback word"
|
|
assert client.call_count == 3
|
|
assert client.total_tokens == 2 + 1 + 2 # word counts
|
|
|
|
|
|
# --- Live gate: skip (no silent egress) when the LOCAL endpoint is unconfigured ---
|
|
|
|
|
|
def test_live_client_skips_without_env(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
monkeypatch.delenv("PORTFOLIO_LOCAL_BASE_URL", raising=False)
|
|
monkeypatch.delenv("PORTFOLIO_LOCAL_MODEL", raising=False)
|
|
with pytest.raises(pytest.skip.Exception):
|
|
live_local_client_or_skip()
|
|
|
|
|
|
# --- Builder smoke: the front-loaded de-risk for Spikes A/B ---
|
|
|
|
|
|
async def test_builder_smoke_concurrent_runs_with_fake_agents() -> None:
|
|
c1 = FakeChatClient(default_reply="alpha view")
|
|
c2 = FakeChatClient(default_reply="beta view")
|
|
workflow = ConcurrentBuilder(
|
|
participants=[fake_agent(c1, "alpha"), fake_agent(c2, "beta")]
|
|
).build()
|
|
result = await workflow.run("Evaluate this trivial task.")
|
|
assert result is not None
|
|
assert c1.call_count == 1
|
|
assert c2.call_count == 1
|
|
|
|
|
|
async def test_builder_smoke_groupchat_runs_and_respects_round_cap() -> None:
|
|
g1 = FakeChatClient(default_reply="proposer line")
|
|
g2 = FakeChatClient(default_reply="critic line")
|
|
names = ["proposer", "critic"]
|
|
counter = {"n": 0}
|
|
|
|
def select(state: object) -> str:
|
|
choice = names[counter["n"] % len(names)]
|
|
counter["n"] += 1
|
|
return choice
|
|
|
|
workflow = (
|
|
GroupChatBuilder(
|
|
participants=[fake_agent(g1, "proposer"), fake_agent(g2, "critic")],
|
|
selection_func=select,
|
|
)
|
|
.with_max_rounds(3)
|
|
.build()
|
|
)
|
|
result = await workflow.run("Debate this trivial task.")
|
|
assert result is not None
|
|
# The fake agents actually spoke; the round cap forced completion (no hang).
|
|
assert g1.call_count + g2.call_count >= 1
|
|
assert counter["n"] <= 3
|