portfolio-optimiser/tests/test_explore_budget_refusal_loadbearing.py

"""Fiks-ordre 20260904T070930Z, punkt 1 — ``BudgetExceeded`` out of ``--explore`` must never
traceback.

**Measured by accident** on K2's syretest (``docs/2026-09-04-syretest-s7b-k2.md § 3.6``): "an
unparseable proposer response burned the round cap" — ``generate_via_llm``'s own retry loop
(``generate.py:_fetch_parsed``) raises ``BudgetExceeded`` from ``meter.tick_round()`` when a
proposer's reply never parses, and ``_evaluate_mandate`` (``run.py``) only swallows that mid-list:
the FIRST approach the pipeline evaluates hitting the cap re-raises by design ("nothing honest to
return"). That is ``main()``'s full single-project run dispatch (``run_project(...)``), whose
``except`` clause caught ``ValueError``/``FileNotFoundError``/``ValidationError`` but not
``BudgetExceeded`` — a ``RuntimeError`` (econ 56) — so the exception reached the interpreter as a
raw traceback.

**A second, distinct raise site reaches the same defect the same way**: the exploration loop
itself (``explore()``/``resume_exploration()``) raises the identical exception type on its OWN
round/token cap, from the ``try`` around the exploration call one block above the full-run
dispatch. Both are fixed, at their own raise sites, because a handler at only one would leave the
other tracebacking on a differently-shaped scripted run.

**Why a subprocess, not just the in-process arms this order also fixes
(``test_explore_callsites_loadbearing.py::test_the_artefact_is_written_even_when_the_exploration_was_cut_short``).**
An in-process call can only assert that ``main()`` RETURNS 1 instead of raising — it cannot see
whether a Python traceback was ever printed, because an exception that unwinds past ``capsys`` was
never captured as text in the first place (the P4 precedent: stdout/stderr is answered in a
subprocess, never in-process — ``CLAUDE.md``'s ``capsys`` note). This file is the one witness that
can tell "rc 1, one refusal line" apart from "the process crashed with an exit code that happens to
be 1 and a traceback nobody looked for".
"""

from __future__ import annotations

import json
import subprocess
import sys
from pathlib import Path
from typing import Any

_REPO = Path(__file__).resolve().parents[1]
_BUNDLE_DIR = _REPO / "shared" / "examples" / "bygg-energi-mikro"
_PID = "BYGG-KONTOR-NORD"

_VALID_PROPOSER_REPLY = json.dumps(
    {
        "measure": "LED-retrofit av kontorbelysning",
        "affected_items": [{"code": "ENERGI-TOTAL-EL", "quantity": 300000, "unit_cost": 1.0}],
        "claimed_saving_nok": 30000,
    }
)

_MANAGER_SATISFIED = json.dumps(
    {
        "is_request_satisfied": {"reason": "r", "answer": True},
        "is_in_loop": {"reason": "r", "answer": False},
        "is_progress_being_made": {"reason": "r", "answer": True},
        "next_speaker": {"reason": "r", "answer": "hypothesiser"},
        "instruction_or_question": {"reason": "r", "answer": "go"},
    }
)
_MANAGER_UNSATISFIED = json.dumps(
    {
        "is_request_satisfied": {"reason": "r", "answer": False},
        "is_in_loop": {"reason": "r", "answer": False},
        "is_progress_being_made": {"reason": "r", "answer": True},
        "next_speaker": {"reason": "r", "answer": "hypothesiser"},
        "instruction_or_question": {"reason": "r", "answer": "keep looking"},
    }
)

#: The happy-path reply set — a run against this must reach rc 0 with empty stderr. Every other
#: reply set in this file is a copy of this one with exactly ONE role swapped for an unparseable
#: or never-satisfied reply, so each test isolates the ONE cap it means to trip.
_REPLIES: dict[str, str] = {
    "proposer": _VALID_PROPOSER_REPLY,
    "checker": "VERDICT: APPROVE",
    "manager": _MANAGER_SATISFIED,
    "navigator": "NAVIGATOR: read the index.",
    "hypothesiser": "still thinking.",
}


def _config_file(tmp_path: Path, **overrides: Any) -> str:
    path = tmp_path / "exploration.json"
    path.write_text(
        json.dumps(
            {
                "max_rounds": 2,
                "max_tokens": 200_000,
                "max_stall_count": 2,
                "max_reset_count": 1,
                "max_plan_revisions": 0,
                "enable_plan_review": False,
                **overrides,
            }
        ),
        encoding="utf-8",
    )
    return str(path)


def _replies_file(tmp_path: Path, replies: dict[str, str] | None = None) -> str:
    path = tmp_path / "replies.json"
    path.write_text(json.dumps(replies if replies is not None else _REPLIES), encoding="utf-8")
    return str(path)


def _argv(tmp_path: Path, *, replies: dict[str, str] | None = None) -> list[str]:
    return [
        _PID,
        "--docs-dir",
        str(_BUNDLE_DIR),
        "--bundle-dir",
        str(_BUNDLE_DIR),
        "--explore",
        "Find the cheapest saving.",
        "--explore-config",
        _config_file(tmp_path),
        "--scripted-replies",
        _replies_file(tmp_path, replies),
    ]


def _run_in_a_fresh_process(argv: list[str]) -> subprocess.CompletedProcess[str]:
    return subprocess.run(
        [sys.executable, "-m", "portfolio_optimiser.run", *argv],
        capture_output=True,
        text=True,
        cwd=str(_REPO),
    )


def _last_line(stderr: str) -> str:
    lines = [line for line in stderr.splitlines() if line.strip()]
    assert lines, "the refusal must say something"
    return lines[-1]


# ---------------------------------------------------------------------------------------------
# 1. The exploration loop's OWN round cap (explore()/resume_exploration())
# ---------------------------------------------------------------------------------------------


def test_the_exploration_round_cap_refuses_instead_of_tracebacking(tmp_path: Path) -> None:
    """Never-satisfied manager replies burn the exploration's ``max_rounds=2``.

    Stderr also carries MAF's own ``"Magentic Orchestrator: Max round count reached"`` log line —
    unrelated noise ``run.main()`` never installs a filter for (that filter is
    ``simulation.main()``'s own — CLAUDE.md's "installeres i main(), ALDRI ved import" invariant),
    so the assertion reads the LAST line rather than the whole stream.

    Detach point: remove ``except BudgetExceeded`` around the ``explore()``/``resume_exploration()``
    call in ``run.py`` → the process exits nonzero with a traceback, and the ``"Traceback"``
    assertion goes RED (rc stays 1 either way — Python's own unhandled-exception exit code — so rc
    alone cannot gate this; the traceback text is the discriminator).
    """
    replies = dict(_REPLIES)
    replies["manager"] = _MANAGER_UNSATISFIED
    replies["hypothesiser"] = "still thinking."

    completed = _run_in_a_fresh_process(_argv(tmp_path, replies=replies))

    assert completed.returncode == 1, completed.stderr
    assert "Traceback (most recent call last)" not in completed.stderr, completed.stderr
    assert _last_line(completed.stderr).startswith(
        "run refused: budget exceeded: exploration_rounds"
    ), completed.stderr


# ---------------------------------------------------------------------------------------------
# 2. The full-run mandate evaluation's round cap (generate_via_llm's retry loop) — the exact
#    scenario measured on K2: an unparseable proposer reply burns run_project's OWN cap.
# ---------------------------------------------------------------------------------------------


def test_an_unparseable_proposer_reply_refuses_instead_of_tracebacking(tmp_path: Path) -> None:
    """The exploration concludes cleanly (manager satisfied on round 1, no hypothesis marked, so
    the mandate carries zero commissioned approaches); the pipeline then evaluates its OWN
    proposal — the only row in the plan — against an unparseable proposer reply, which burns
    ``run_project``'s default ``max_rounds=3`` inside ``generate_via_llm``. This is the shape
    ``docs/2026-09-04-syretest-s7b-k2.md`` describes verbatim.

    Detach point: revert ``BudgetExceeded`` out of the full-run dispatch's ``except`` tuple in
    ``run.py`` → traceback, RED here.
    """
    replies = dict(_REPLIES)
    replies["proposer"] = "{}"

    completed = _run_in_a_fresh_process(_argv(tmp_path, replies=replies))

    assert completed.returncode == 1, completed.stderr
    assert "Traceback (most recent call last)" not in completed.stderr, completed.stderr
    assert _last_line(completed.stderr).startswith("run refused: budget exceeded: rounds"), (
        completed.stderr
    )


# ---------------------------------------------------------------------------------------------
# 3. Control — the happy path must still reach rc 0 with nothing on stderr
# ---------------------------------------------------------------------------------------------


def test_a_run_that_never_capped_is_unaffected(tmp_path: Path) -> None:
    """Neither ``except`` clause may swallow a run that actually finishes.

    Detach point: an over-widened handler (e.g. ``except Exception`` in either spot) would also
    make this arm pass, which is why the two goal tests' traceback assertions carry the real
    weight — this control only proves the door still opens for a happy path.

    Stderr is not asserted empty: the debate's own ``GroupChatOrchestrator`` logs
    ``"reached max_rounds=…; forcing completion"`` on its OWN cap (unrelated MAF noise
    ``simulation.main()``'s filter would quiet, but ``run.main()`` never installs it) even on a
    run that otherwise succeeds — asserted absent are only the two things a refusal or a crash
    would add.
    """
    completed = _run_in_a_fresh_process(_argv(tmp_path))

    assert completed.returncode == 0, completed.stderr
    assert "Traceback (most recent call last)" not in completed.stderr, completed.stderr
    assert "run refused" not in completed.stderr, completed.stderr