"""Steps 2 and 3 of the loop (method-spec §3): generation + maker-checker debate. Step 2 — a reply that fails to parse into the typed IR is retried BLIND (same prompt), never silently accepted or repaired downstream, bounded by the budget meter (§8; round ticks between attempts). Step 3 — the debate is round-capped with a turn-count safety net above it, state is fresh per run, and the checker is INSTRUCTED to end with exactly one verdict line. Verdict parsing is case-insensitive, the reject marker takes precedence, trailing text is the reason, and a missing marker parses as absent (fail-open input to the gate). """ from __future__ import annotations import json import pytest from _scripted import ScriptedClient, reply from portfolio_optimiser_claude.budget import BudgetExceeded, BudgetMeter from portfolio_optimiser_claude.contracts import TerminationContract from portfolio_optimiser_claude.loop import ( generate_candidate, parse_checker_verdict, run_debate, ) # One affected item: 100 × 1000 = 100_000 NOK total; nominal feasible 30_000; # no assumptions band, so every Monte Carlo sample is 30_000 and p90 == 30_000. VALID_PROPOSAL = { "project_id": "p1", "measure": "led-retrofit", "affected_items": [{"code": "E01", "quantity": 100, "unit_cost": 1000}], "claimed_saving_nok": 25000, } def _meter(max_rounds: int = 50, max_tokens: int = 10_000) -> BudgetMeter: return BudgetMeter(TerminationContract(max_rounds=max_rounds, max_tokens=max_tokens)) class TestGenerateCandidate: def test_valid_reply_parses_into_the_typed_ir(self) -> None: client = ScriptedClient(replies=[reply(json.dumps(VALID_PROPOSAL))]) meter = _meter() proposal = generate_candidate(client, "base prompt", meter=meter) assert proposal.claimed_saving_nok == 25000 assert meter.tokens_used == 10 def test_malformed_reply_is_retried_blind_with_the_same_prompt(self) -> None: client = ScriptedClient( replies=[reply("not json at all"), reply(json.dumps(VALID_PROPOSAL))] ) proposal = generate_candidate(client, "base prompt", meter=_meter()) assert proposal.measure == "led-retrofit" # Blind retry: the SAME prompt, unchanged — never a repair instruction. assert client.prompts("proposer") == ["base prompt", "base prompt"] def test_schema_invalid_json_is_retried_never_repaired(self) -> None: # Claim above the items' own total is a schema error (§7.1) — the value # must never exist; the loop retries, it does not clamp or repair. overclaim = dict(VALID_PROPOSAL, claimed_saving_nok=999_999) client = ScriptedClient( replies=[reply(json.dumps(overclaim)), reply(json.dumps(VALID_PROPOSAL))] ) proposal = generate_candidate(client, "base prompt", meter=_meter()) assert proposal.claimed_saving_nok == 25000 def test_parse_retries_charge_round_ticks(self) -> None: # §8: round ticks are charged between attempts so the meter also # bounds parse-retries. client = ScriptedClient( replies=[reply("garbage"), reply("garbage"), reply(json.dumps(VALID_PROPOSAL))] ) meter = _meter() generate_candidate(client, "base prompt", meter=meter) assert meter.rounds_used == 2 def test_endless_garbage_is_stopped_by_the_meter(self) -> None: client = ScriptedClient(script=lambda role, prompt: reply("garbage")) with pytest.raises(BudgetExceeded) as exc_info: generate_candidate(client, "base prompt", meter=_meter(max_rounds=3)) assert exc_info.value.kind == "rounds" def test_project_id_may_be_defaulted_from_the_project(self) -> None: omitted = {k: v for k, v in VALID_PROPOSAL.items() if k != "project_id"} client = ScriptedClient(replies=[reply(json.dumps(omitted))]) proposal = generate_candidate( client, "base prompt", meter=_meter(), default_project_id="p-default" ) assert proposal.project_id == "p-default" def test_present_project_id_is_never_overwritten_by_the_default(self) -> None: client = ScriptedClient(replies=[reply(json.dumps(VALID_PROPOSAL))]) proposal = generate_candidate( client, "base prompt", meter=_meter(), default_project_id="p-default" ) assert proposal.project_id == "p1" def test_missing_usage_on_the_counting_path_fails_closed(self) -> None: client = ScriptedClient(replies=[reply(json.dumps(VALID_PROPOSAL), usage_tokens=None)]) with pytest.raises(Exception, match="usage"): generate_candidate(client, "base prompt", meter=_meter()) class TestRunDebate: def test_converges_when_the_checker_approves(self) -> None: client = ScriptedClient(replies=[reply("reasoning v1"), reply("holds. VERDICT: APPROVE")]) debate = run_debate(client, "context", max_rounds=3, meter=_meter()) assert debate.rounds == 1 assert debate.proposer_output == "reasoning v1" assert "VERDICT: APPROVE" in debate.checker_last def test_round_cap_bounds_a_never_approving_debate(self) -> None: client = ScriptedClient( script=lambda role, prompt: reply( "VERDICT: REJECT - weak numbers" if role == "checker" else "reasoning" ) ) debate = run_debate(client, "context", max_rounds=2, meter=_meter()) assert debate.rounds == 2 assert len(client.prompts("proposer")) == 2 assert len(client.prompts("checker")) == 2 assert "REJECT" in debate.checker_last def test_checker_is_instructed_to_end_with_the_verdict_line(self) -> None: # §3 Step 3: the checker MUST be instructed to end its reply with # exactly one verdict line, both marker forms spelled out. client = ScriptedClient(replies=[reply("reasoning"), reply("VERDICT: APPROVE")]) run_debate(client, "context", max_rounds=1, meter=_meter()) checker_prompt = client.prompts("checker")[0] assert "VERDICT: APPROVE" in checker_prompt assert "VERDICT: REJECT - " in checker_prompt def test_checker_critique_reaches_the_next_proposer_turn(self) -> None: client = ScriptedClient( replies=[ reply("reasoning v1"), reply("VERDICT: REJECT - unit costs are stale"), reply("reasoning v2"), reply("VERDICT: APPROVE"), ] ) debate = run_debate(client, "context", max_rounds=3, meter=_meter()) assert debate.rounds == 2 assert "unit costs are stale" in client.prompts("proposer")[1] assert debate.proposer_output == "reasoning v2" def test_debate_state_is_fresh_per_run(self) -> None: # §3 Step 3: no conversation state may survive from one run into the # next — the second run's opening proposer prompt carries nothing from # the first run's transcript. client = ScriptedClient( replies=[ reply("FIRST-RUN-MARKER reasoning"), reply("VERDICT: REJECT - FIRST-RUN-CRITIQUE"), reply("more reasoning"), reply("VERDICT: APPROVE"), reply("second-run reasoning"), reply("VERDICT: APPROVE"), ] ) run_debate(client, "context A", max_rounds=2, meter=_meter()) run_debate(client, "context B", max_rounds=2, meter=_meter()) second_run_opening = client.prompts("proposer")[2] assert "FIRST-RUN-MARKER" not in second_run_opening assert "FIRST-RUN-CRITIQUE" not in second_run_opening def test_every_turn_is_charged_on_the_meter(self) -> None: client = ScriptedClient( replies=[reply("reasoning", usage_tokens=7), reply("VERDICT: APPROVE", usage_tokens=5)] ) meter = _meter() run_debate(client, "context", max_rounds=1, meter=meter) assert meter.tokens_used == 12 assert meter.rounds_used == 1 def test_the_turn_safety_net_sits_above_the_round_cap(self) -> None: # §3 Step 3 / §8: an additional turn-count termination safety net # ABOVE the round cap — it must never fire within a round-capped run, # and must refuse a turn count beyond it. from portfolio_optimiser_claude.loop import check_turn_safety_net check_turn_safety_net(turns=2 * 3, max_rounds=3) # within: no raise with pytest.raises(RuntimeError): check_turn_safety_net(turns=2 * 3 + 3, max_rounds=3) class TestParseCheckerVerdict: def test_approve_is_parsed_case_insensitively(self) -> None: verdict = parse_checker_verdict("the numbers hold.\nverdict: approve") assert verdict.decision == "approve" def test_reject_carries_the_trailing_text_as_reason(self) -> None: verdict = parse_checker_verdict("VERDICT: REJECT - savings claim is double-counted") assert verdict.decision == "reject" assert verdict.reason == "savings claim is double-counted" def test_the_reject_marker_takes_precedence(self) -> None: verdict = parse_checker_verdict( "VERDICT: APPROVE was my first instinct, but no.\nVERDICT: REJECT - stale baseline" ) assert verdict.decision == "reject" assert verdict.reason == "stale baseline" def test_missing_marker_parses_as_absent(self) -> None: verdict = parse_checker_verdict("looks fine to me") assert verdict.decision == "absent" def test_empty_text_parses_as_absent(self) -> None: assert parse_checker_verdict("").decision == "absent"