portfolio-optimiser-claude/tests/test_loop.py

"""Steps 2 and 3 of the loop (method-spec §3): generation + maker-checker debate.

Step 2 — a reply that fails to parse into the typed IR is retried BLIND (same
prompt), never silently accepted or repaired downstream, bounded by the budget
meter (§8; round ticks between attempts). Step 3 — the debate is round-capped
with a turn-count safety net above it, state is fresh per run, and the checker
is INSTRUCTED to end with exactly one verdict line. Verdict parsing is
case-insensitive, the reject marker takes precedence, trailing text is the
reason, and a missing marker parses as absent (fail-open input to the gate).
"""

from __future__ import annotations

import json

import pytest
from _scripted import ScriptedClient, reply

from portfolio_optimiser_claude.budget import BudgetExceeded, BudgetMeter
from portfolio_optimiser_claude.contracts import TerminationContract
from portfolio_optimiser_claude.loop import (
    generate_candidate,
    parse_checker_verdict,
    run_debate,
    run_project,
)

# One affected item: 100 × 1000 = 100_000 NOK total; nominal feasible 30_000;
# no assumptions band, so every Monte Carlo sample is 30_000 and p90 == 30_000.
VALID_PROPOSAL = {
    "project_id": "p1",
    "measure": "led-retrofit",
    "affected_items": [{"code": "E01", "quantity": 100, "unit_cost": 1000}],
    "claimed_saving_nok": 25000,
}


def _meter(max_rounds: int = 50, max_tokens: int = 10_000) -> BudgetMeter:
    return BudgetMeter(TerminationContract(max_rounds=max_rounds, max_tokens=max_tokens))


class TestGenerateCandidate:
    def test_valid_reply_parses_into_the_typed_ir(self) -> None:
        client = ScriptedClient(replies=[reply(json.dumps(VALID_PROPOSAL))])
        meter = _meter()
        proposal = generate_candidate(client, "base prompt", meter=meter)
        assert proposal.claimed_saving_nok == 25000
        assert meter.tokens_used == 10

    def test_malformed_reply_is_retried_blind_with_the_same_prompt(self) -> None:
        client = ScriptedClient(
            replies=[reply("not json at all"), reply(json.dumps(VALID_PROPOSAL))]
        )
        proposal = generate_candidate(client, "base prompt", meter=_meter())
        assert proposal.measure == "led-retrofit"
        # Blind retry: the SAME prompt, unchanged — never a repair instruction.
        assert client.prompts("proposer") == ["base prompt", "base prompt"]

    def test_schema_invalid_json_is_retried_never_repaired(self) -> None:
        # Claim above the items' own total is a schema error (§7.1) — the value
        # must never exist; the loop retries, it does not clamp or repair.
        overclaim = dict(VALID_PROPOSAL, claimed_saving_nok=999_999)
        client = ScriptedClient(
            replies=[reply(json.dumps(overclaim)), reply(json.dumps(VALID_PROPOSAL))]
        )
        proposal = generate_candidate(client, "base prompt", meter=_meter())
        assert proposal.claimed_saving_nok == 25000

    def test_parse_retries_charge_round_ticks(self) -> None:
        # §8: round ticks are charged between attempts so the meter also
        # bounds parse-retries.
        client = ScriptedClient(
            replies=[reply("garbage"), reply("garbage"), reply(json.dumps(VALID_PROPOSAL))]
        )
        meter = _meter()
        generate_candidate(client, "base prompt", meter=meter)
        assert meter.rounds_used == 2

    def test_endless_garbage_is_stopped_by_the_meter(self) -> None:
        client = ScriptedClient(script=lambda role, prompt: reply("garbage"))
        with pytest.raises(BudgetExceeded) as exc_info:
            generate_candidate(client, "base prompt", meter=_meter(max_rounds=3))
        assert exc_info.value.kind == "rounds"

    def test_project_id_may_be_defaulted_from_the_project(self) -> None:
        omitted = {k: v for k, v in VALID_PROPOSAL.items() if k != "project_id"}
        client = ScriptedClient(replies=[reply(json.dumps(omitted))])
        proposal = generate_candidate(
            client, "base prompt", meter=_meter(), default_project_id="p-default"
        )
        assert proposal.project_id == "p-default"

    def test_present_project_id_is_never_overwritten_by_the_default(self) -> None:
        client = ScriptedClient(replies=[reply(json.dumps(VALID_PROPOSAL))])
        proposal = generate_candidate(
            client, "base prompt", meter=_meter(), default_project_id="p-default"
        )
        assert proposal.project_id == "p1"

    def test_missing_usage_on_the_counting_path_fails_closed(self) -> None:
        client = ScriptedClient(replies=[reply(json.dumps(VALID_PROPOSAL), usage_tokens=None)])
        with pytest.raises(Exception, match="usage"):
            generate_candidate(client, "base prompt", meter=_meter())


class TestRunDebate:
    def test_converges_when_the_checker_approves(self) -> None:
        client = ScriptedClient(replies=[reply("reasoning v1"), reply("holds. VERDICT: APPROVE")])
        debate = run_debate(client, "context", max_rounds=3, meter=_meter())
        assert debate.rounds == 1
        assert debate.proposer_output == "reasoning v1"
        assert "VERDICT: APPROVE" in debate.checker_last

    def test_round_cap_bounds_a_never_approving_debate(self) -> None:
        client = ScriptedClient(
            script=lambda role, prompt: reply(
                "VERDICT: REJECT - weak numbers" if role == "checker" else "reasoning"
            )
        )
        debate = run_debate(client, "context", max_rounds=2, meter=_meter())
        assert debate.rounds == 2
        assert len(client.prompts("proposer")) == 2
        assert len(client.prompts("checker")) == 2
        assert "REJECT" in debate.checker_last

    def test_checker_is_instructed_to_end_with_the_verdict_line(self) -> None:
        # §3 Step 3: the checker MUST be instructed to end its reply with
        # exactly one verdict line, both marker forms spelled out.
        client = ScriptedClient(replies=[reply("reasoning"), reply("VERDICT: APPROVE")])
        run_debate(client, "context", max_rounds=1, meter=_meter())
        checker_prompt = client.prompts("checker")[0]
        assert "VERDICT: APPROVE" in checker_prompt
        assert "VERDICT: REJECT - <short reason>" in checker_prompt

    def test_checker_critique_reaches_the_next_proposer_turn(self) -> None:
        client = ScriptedClient(
            replies=[
                reply("reasoning v1"),
                reply("VERDICT: REJECT - unit costs are stale"),
                reply("reasoning v2"),
                reply("VERDICT: APPROVE"),
            ]
        )
        debate = run_debate(client, "context", max_rounds=3, meter=_meter())
        assert debate.rounds == 2
        assert "unit costs are stale" in client.prompts("proposer")[1]
        assert debate.proposer_output == "reasoning v2"

    def test_debate_state_is_fresh_per_run(self) -> None:
        # §3 Step 3: no conversation state may survive from one run into the
        # next — the second run's opening proposer prompt carries nothing from
        # the first run's transcript.
        client = ScriptedClient(
            replies=[
                reply("FIRST-RUN-MARKER reasoning"),
                reply("VERDICT: REJECT - FIRST-RUN-CRITIQUE"),
                reply("more reasoning"),
                reply("VERDICT: APPROVE"),
                reply("second-run reasoning"),
                reply("VERDICT: APPROVE"),
            ]
        )
        run_debate(client, "context A", max_rounds=2, meter=_meter())
        run_debate(client, "context B", max_rounds=2, meter=_meter())
        # Positive control for the CRITIQUE negative: within the first run the critique
        # DOES reach the proposer's second prompt, in EXACTLY the form the negative
        # below searches for. That is what makes "absent from run B" a measured seam
        # rather than a string that never travels anywhere.
        first_run_second_prompt = client.prompts("proposer")[1]
        assert "FIRST-RUN-CRITIQUE" in first_run_second_prompt
        # MEASURED (this session): no such control exists for FIRST-RUN-MARKER — the
        # proposer's own prior reasoning crosses NO prompt boundary, not even within a
        # run (only the checker's critique is fed back). So the marker negative below is
        # a weaker, complementary check: it cannot go red by state leaking through the
        # transcript, because that channel carries the critique alone. Stated, not
        # dressed up as a controlled seam.
        assert "FIRST-RUN-MARKER" not in first_run_second_prompt
        second_run_opening = client.prompts("proposer")[2]
        assert "FIRST-RUN-MARKER" not in second_run_opening
        assert "FIRST-RUN-CRITIQUE" not in second_run_opening

    def test_every_turn_is_charged_on_the_meter(self) -> None:
        client = ScriptedClient(
            replies=[reply("reasoning", usage_tokens=7), reply("VERDICT: APPROVE", usage_tokens=5)]
        )
        meter = _meter()
        run_debate(client, "context", max_rounds=1, meter=meter)
        assert meter.tokens_used == 12
        assert meter.rounds_used == 1

    def test_the_turn_safety_net_sits_above_the_round_cap(self) -> None:
        # §3 Step 3 / §8: an additional turn-count termination safety net
        # ABOVE the round cap — it must never fire within a round-capped run,
        # and must refuse a turn count beyond it.
        from portfolio_optimiser_claude.loop import check_turn_safety_net

        check_turn_safety_net(turns=2 * 3, max_rounds=3)  # within: no raise
        with pytest.raises(RuntimeError):
            check_turn_safety_net(turns=2 * 3 + 3, max_rounds=3)


class TestParseCheckerVerdict:
    def test_approve_is_parsed_case_insensitively(self) -> None:
        verdict = parse_checker_verdict("the numbers hold.\nverdict: approve")
        assert verdict.decision == "approve"

    def test_reject_carries_the_trailing_text_as_reason(self) -> None:
        verdict = parse_checker_verdict("VERDICT: REJECT - savings claim is double-counted")
        assert verdict.decision == "reject"
        assert verdict.reason == "savings claim is double-counted"

    def test_the_reject_marker_takes_precedence(self) -> None:
        verdict = parse_checker_verdict(
            "VERDICT: APPROVE was my first instinct, but no.\nVERDICT: REJECT - stale baseline"
        )
        assert verdict.decision == "reject"
        assert verdict.reason == "stale baseline"

    def test_missing_marker_parses_as_absent(self) -> None:
        verdict = parse_checker_verdict("looks fine to me")
        assert verdict.decision == "absent"

    def test_empty_text_parses_as_absent(self) -> None:
        assert parse_checker_verdict("").decision == "absent"


class TestGenerationPrompt:
    def test_generation_demands_the_raw_json_object_only(self) -> None:
        # S10 post-mortem: 3 of 4 generation replies carried plausible JSON
        # wrapped in a markdown fence + commentary — each one a full-price
        # blind parse-retry (§3 Step 2). The prompt must forbid the wrapping.
        client = ScriptedClient(
            replies=[
                reply("reasoning"),
                reply("VERDICT: APPROVE"),
                reply(json.dumps(VALID_PROPOSAL)),
            ]
        )
        run_project(client, "context", meter=_meter(), max_debate_rounds=1)
        generation_prompt = client.prompts("proposer")[1]
        assert "ONLY the raw JSON object" in generation_prompt
        assert "no markdown fences" in generation_prompt