"""Step 2: shared harness invariants + the builder smoke that de-risks Spikes A/B. The builder smoke is the one place that proves ``FakeChatClient`` can actually drive the GA ``GroupChatBuilder`` / ``ConcurrentBuilder``. If it could not, that would itself be a primary de-risk finding (escalate) — but it does (see findings note). Pattern: tests/test_backends.py (parametrize + pytest.raises). """ import pytest from agent_framework import Message from agent_framework.orchestrations import ConcurrentBuilder, GroupChatBuilder from spikes._harness import ( Budget, BudgetExceeded, FakeChatClient, TokenMeter, fake_agent, live_local_client_or_skip, ) # --- Budget: refuse to start without positive caps (A4 / fail-fast) --- @pytest.mark.parametrize("caps", [(0, 5), (5, 0), (-1, 5), (5, -3)]) def test_budget_rejects_non_positive_caps(caps: tuple[int, int]) -> None: with pytest.raises(ValueError): Budget(*caps) def test_budget_accepts_positive_caps() -> None: b = Budget(max_tokens=100, max_rounds=3) assert b.max_tokens == 100 assert b.max_rounds == 3 # --- TokenMeter: structured stop the moment a cap is crossed (B4) --- def test_token_meter_charges_until_cap() -> None: meter = TokenMeter(Budget(max_tokens=10, max_rounds=3)) assert meter.charge(4) == 4 assert meter.charge(6) == 10 # exactly at cap is fine with pytest.raises(BudgetExceeded) as exc: meter.charge(1) assert exc.value.kind == "tokens" assert exc.value.limit == 10 assert exc.value.observed == 11 def test_token_meter_ticks_rounds_until_cap() -> None: meter = TokenMeter(Budget(max_tokens=100, max_rounds=2)) assert meter.tick_round() == 1 assert meter.tick_round() == 2 with pytest.raises(BudgetExceeded) as exc: meter.tick_round() assert exc.value.kind == "rounds" assert exc.value.limit == 2 # --- FakeChatClient: deterministic scripted replies + word-count tokens --- async def test_fake_client_returns_scripted_replies_in_order() -> None: client = FakeChatClient(["one two", "three"], default_reply="fallback word") r1 = await client.get_response([Message(role="user", contents=["hi"])]) r2 = await client.get_response([Message(role="user", contents=["hi"])]) r3 = await client.get_response([Message(role="user", contents=["hi"])]) # script exhausted assert r1.text == "one two" assert r2.text == "three" assert r3.text == "fallback word" assert client.call_count == 3 assert client.total_tokens == 2 + 1 + 2 # word counts # --- Live gate: skip (no silent egress) when the LOCAL endpoint is unconfigured --- def test_live_client_skips_without_env(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.delenv("PORTFOLIO_LOCAL_BASE_URL", raising=False) monkeypatch.delenv("PORTFOLIO_LOCAL_MODEL", raising=False) with pytest.raises(pytest.skip.Exception): live_local_client_or_skip() # --- Builder smoke: the front-loaded de-risk for Spikes A/B --- async def test_builder_smoke_concurrent_runs_with_fake_agents() -> None: c1 = FakeChatClient(default_reply="alpha view") c2 = FakeChatClient(default_reply="beta view") workflow = ConcurrentBuilder( participants=[fake_agent(c1, "alpha"), fake_agent(c2, "beta")] ).build() result = await workflow.run("Evaluate this trivial task.") assert result is not None assert c1.call_count == 1 assert c2.call_count == 1 async def test_builder_smoke_groupchat_runs_and_respects_round_cap() -> None: g1 = FakeChatClient(default_reply="proposer line") g2 = FakeChatClient(default_reply="critic line") names = ["proposer", "critic"] counter = {"n": 0} def select(state: object) -> str: choice = names[counter["n"] % len(names)] counter["n"] += 1 return choice workflow = ( GroupChatBuilder( participants=[fake_agent(g1, "proposer"), fake_agent(g2, "critic")], selection_func=select, ) .with_max_rounds(3) .build() ) result = await workflow.run("Debate this trivial task.") assert result is not None # The fake agents actually spoke; the round cap forced completion (no hang). assert g1.call_count + g2.call_count >= 1 assert counter["n"] <= 3