portfolio-optimiser/tests/test_run_cli_chatclient_refusal.py

"""Funn 99 — a provider failure must LEAVE the CLI as one line, not as a traceback.

MEASURED FIRST, on the artefacts the paid Q5=B run left behind (``scratchpad/q5b/live/``,
never re-run here): the exploration died on

    agent_framework.exceptions.ChatClientException:
    <class 'FoundryChatClient'> service failed to complete the prompt: Error code: 400 -
    {'error': {'message': 'No tool call found for function call output with call_id
    call_AyeuYJmqvmmej9utu361Phtt.', ...}}

and that class is in NONE of ``main()``'s refusal tuples (``run.py`` catches
``FileNotFoundError, ValidationError, ValueError`` plus ``BudgetExceeded``/``PlanReviewParked``),
so the process tracebacked — the same defect class ``BudgetExceeded`` was added to that block for.

TWO seams are closed here and each one has its OWN arm, because each can regress alone:

  (A) the EXPLORATION dispatch (``run.py``'s ``explore()``/``resume_exploration()`` block), which
      is where the measured failure happened; and
  (A2) the FULL-RUN dispatch (``run_project``), which the debate's own model calls go through —
      a fix on only one of the two leaves the other tracebacking.

The KNOWN-NEGATIVE is the point of the arm, not decoration: a ``RuntimeError`` from the same seam
must still propagate. We are closing ONE named provider channel, never hiding unknown failures.

Arm (B) is the tool side, and it is authorised by the measurement rather than by symmetry: the
three failing ``quick_validate`` calls in ``Bseed-records.json`` were NOT verdicts. All three sent
``bundle_id="renholdstekniske_funksjonskrav"`` — a concept name, guessed out of the seeded cut,
never a base id — with a well-formed ``proposal_json``, so the arguments parsed against the
signature and it was ``_resolve_bundle``'s raise that MAF counted. Proof it was the raise and not
a verdict: ``q5b-Bseed-exploration.json`` records all three in ``tool_calls`` while
``quick_validations`` is EMPTY, and the sink is appended on every verdict branch.
"""

from __future__ import annotations

import json
from collections.abc import Callable
from pathlib import Path
from typing import Any

import pytest
from agent_framework import BaseChatClient
from agent_framework.exceptions import ChatClientException

from portfolio_optimiser import explore, run
from portfolio_optimiser.simulation import ScriptedChatClient

_BUNDLE_DIR = Path(__file__).resolve().parents[1] / "shared" / "examples" / "bygg-energi-mikro"
_PID = "BYGG-KONTOR-NORD"

#: The provider text VERBATIM from the measured run — truncated only where the records truncate it.
_PROVIDER_TEXT = (
    "<class 'agent_framework_foundry._chat_client.FoundryChatClient'> service failed to complete "
    "the prompt: Error code: 400 - {'error': {'message': 'No tool call found for function call "
    "output with call_id call_AyeuYJmqvmmej9utu361Phtt.', 'type': 'invalid_request_error'}}"
)

_CONTRACT_JSON: dict[str, Any] = {
    "max_rounds": 2,
    "max_tokens": 50_000,
    "max_stall_count": 2,
    "max_reset_count": 1,
    "max_plan_revisions": 0,
    "enable_plan_review": False,
}


@pytest.fixture(autouse=True)
def _isolate_model_env(monkeypatch: pytest.MonkeyPatch) -> None:
    monkeypatch.delenv("PORTFOLIO_MODEL_MAP", raising=False)
    monkeypatch.delenv("PORTFOLIO_OTEL", raising=False)


def _raising_factory(exc: BaseException) -> Callable[[str], Callable[[str], BaseChatClient]]:
    """A factory whose clients raise ``exc`` on the FIRST model call, before any reply exists.

    The raise lives in the ``reply_selector`` seam rather than in an ``_inner_get_response``
    override, and that is deliberate: the canonical body calls the selector SYNCHRONOUSLY
    (``simulation.py:443``) before it builds any coroutine, so a raise there leaves the client at
    exactly the point a provider's would — and the S2.5 consolidation guard keeps its property
    that there is no second copy of the scripted body anywhere in the tree.
    """

    def _raise(_prompt: str, _role: str) -> str:
        raise exc

    def outer(_profile: Any) -> Callable[[str], BaseChatClient]:
        def factory(role: str) -> BaseChatClient:
            return ScriptedChatClient(reply_selector=_raise, role=role)

        return factory

    return outer


def _config_file(tmp_path: Path) -> str:
    path = tmp_path / "exploration.json"
    path.write_text(json.dumps(_CONTRACT_JSON), encoding="utf-8")
    return str(path)


def _explore_argv(tmp_path: Path, *extra: str) -> list[str]:
    return [
        _PID,
        "--docs-dir",
        str(_BUNDLE_DIR),
        "--bundle-dir",
        str(_BUNDLE_DIR),
        "--explore",
        "Find the cheapest saving.",
        "--explore-config",
        _config_file(tmp_path),
        *extra,
    ]


# ---------------------------------------------------------------------------------------------
# (A) the exploration dispatch
# ---------------------------------------------------------------------------------------------


def test_a_provider_failure_in_the_exploration_leaves_the_cli_as_one_line(
    tmp_path, monkeypatch, capsys
) -> None:
    """T1 — POSITIVE. ``--explore`` against a client that raises ``ChatClientException`` must give
    rc 1 and ONE stderr line, never a traceback.

    Detach point: remove the ``except ChatClientException`` arm from the exploration block → the
    exception escapes ``main()`` and pytest reports the raise instead of an rc → RED.
    """
    monkeypatch.setattr(
        "portfolio_optimiser.run._default_factory",
        _raising_factory(ChatClientException(_PROVIDER_TEXT)),
    )
    rc = run.main(_explore_argv(tmp_path))
    err = capsys.readouterr().err
    assert rc == 1
    assert "Traceback" not in err
    assert len([line for line in err.splitlines() if line.strip()]) == 1
    assert err.startswith("run stopped:")
    assert "No tool call found for function call output" in err


def test_an_unknown_failure_from_the_same_seam_still_propagates(tmp_path, monkeypatch) -> None:
    """T2 — KNOWN-NEGATIVE. The arm is ONE named provider channel, not a blanket ``except``.

    Without this, a fix written as ``except Exception`` would pass T1 and hide every programming
    error the exploration can make. Detach point: widen the arm to ``Exception`` → RED.
    """
    monkeypatch.setattr(
        "portfolio_optimiser.run._default_factory",
        _raising_factory(RuntimeError("a defect, not a provider")),
    )
    with pytest.raises(RuntimeError, match="a defect, not a provider"):
        run.main(_explore_argv(tmp_path))


def test_the_exploration_artefact_survives_the_refusal(tmp_path, monkeypatch, capsys) -> None:
    """T3 — the evidence written BEFORE the failure must not disappear with it.

    ``run.py``'s ``finally`` writes ``{run_id}-exploration.json`` whatever ended the block; an arm
    placed so the ``finally`` is skipped (or one that returns before it) would take the one record
    of what the run had already spent with it. Detach point: catch the exception OUTSIDE the
    try/finally → RED.
    """
    outbox = tmp_path / "outbox"
    monkeypatch.setattr(
        "portfolio_optimiser.run._default_factory",
        _raising_factory(ChatClientException(_PROVIDER_TEXT)),
    )
    rc = run.main(_explore_argv(tmp_path, "--outbox-dir", str(outbox), "--run-id", "funn99"))
    assert rc == 1
    assert capsys.readouterr().err.startswith("run stopped:")
    artefact = outbox / "funn99-exploration.json"
    assert artefact.exists()
    payload = json.loads(artefact.read_text(encoding="utf-8"))
    assert payload["completed"] is False


# ---------------------------------------------------------------------------------------------
# (A2) the full-run dispatch — the debate's own model calls
# ---------------------------------------------------------------------------------------------


def test_a_provider_failure_in_the_full_run_leaves_the_cli_as_one_line(monkeypatch, capsys) -> None:
    """T4 — the SECOND seam. The debate calls the same provider, and a fix on the exploration
    block alone leaves an ordinary ``run.main([...])`` tracebacking.

    Detach point: remove the ``except ChatClientException`` arm from the full-run dispatch → RED,
    and T1 stays green — which is exactly why this is its own arm.
    """
    monkeypatch.setattr(
        "portfolio_optimiser.run._default_factory",
        _raising_factory(ChatClientException(_PROVIDER_TEXT)),
    )
    rc = run.main([_PID, "--docs-dir", str(_BUNDLE_DIR), "--bundle-dir", str(_BUNDLE_DIR)])
    err = capsys.readouterr().err
    assert rc == 1
    assert "Traceback" not in err
    assert len([line for line in err.splitlines() if line.strip()]) == 1
    assert err.startswith("run stopped:")


def test_an_unknown_failure_in_the_full_run_still_propagates(monkeypatch) -> None:
    """T5 — KNOWN-NEGATIVE for the second seam, for T2's reason."""
    monkeypatch.setattr(
        "portfolio_optimiser.run._default_factory",
        _raising_factory(RuntimeError("a defect, not a provider")),
    )
    with pytest.raises(RuntimeError, match="a defect, not a provider"):
        run.main([_PID, "--docs-dir", str(_BUNDLE_DIR), "--bundle-dir", str(_BUNDLE_DIR)])


# ---------------------------------------------------------------------------------------------
# (B) the tool — a base id the model guessed wrong is a thing it can CORRECT, not a run-ender
# ---------------------------------------------------------------------------------------------


def test_an_unknown_base_id_comes_back_as_a_refused_verdict() -> None:
    """T6 — POSITIVE. ``quick_validate`` against an id no configured base carries must RETURN a
    ``refused`` verdict naming the configured ids, not raise.

    MEASURED root, verbatim from ``Bseed-records.json``: three calls with
    ``bundle_id="renholdstekniske_funksjonskrav"``. Each raise became MAF's opaque
    ``"Error: Function failed."`` (``_tools.py:1426``; details are suppressed unless
    ``include_detailed_errors``), so the reason po had written — *which* ids exist — never reached
    the model, and the replies show it guessing at the JSON format instead. Three in a row hit
    ``DEFAULT_MAX_CONSECUTIVE_ERRORS_PER_REQUEST`` (``_tools.py:96``, value 3).

    Detach point: restore the raise → RED.
    """
    sink: list[explore.QuickValidation] = []
    tool = explore.quick_validate_tool((str(_BUNDLE_DIR),), sink=sink)
    verdict = tool.func(bundle_id="renholdstekniske_funksjonskrav", proposal_json="{}")
    assert verdict["decision"] == "refused"
    assert "renholdstekniske_funksjonskrav" in verdict["reason"]
    assert _BUNDLE_DIR.name in verdict["reason"] or "bygg" in verdict["reason"]
    assert verdict["anchored"] is False


def test_a_known_base_id_still_reaches_the_validator() -> None:
    """T7 — CONTROL. Without it, an implementation that answered ``refused`` to EVERY call would
    pass T6 while destroying the tool. Detach point: return ``refused`` unconditionally → RED."""
    sink: list[explore.QuickValidation] = []
    tool = explore.quick_validate_tool((str(_BUNDLE_DIR),), sink=sink)
    verdict = tool.func(bundle_id=_bundle_id(), proposal_json="{}")
    assert verdict["decision"] == "unparseable"


def test_the_refusal_is_recorded_in_the_sink() -> None:
    """T8 — a refused call is a thing the hypothesiser ASKED for, and the artefact is where an
    operator reads that it happened.

    While it raised, the call left ``quick_validations`` empty and was visible only in
    ``tool_calls`` — which is precisely why this session had to read two artefacts to find the
    root. Detach point: skip the sink append on the refused branch → RED.
    """
    sink: list[explore.QuickValidation] = []
    tool = explore.quick_validate_tool((str(_BUNDLE_DIR),), sink=sink)
    tool.func(bundle_id="renholdstekniske_funksjonskrav", proposal_json="{}")
    assert [entry.verdict["decision"] for entry in sink] == ["refused"]
    assert sink[0].bundle_id == "renholdstekniske_funksjonskrav"


def _bundle_id() -> str:
    from portfolio_optimiser import okf

    return okf.reconcile_bundle_id(str(_BUNDLE_DIR)).id