portfolio-optimiser/tests/test_step5_history_loadbearing.py

"""Step 5 load-bearing seam, part 2 (målbilde §5/§7): the intermediate falsification history must
ESCAPE ``generate_via_llm`` and reach the run's result — otherwise Step 5 is real but invisible.

The gap (verified in code before this file existed): the informed-refinement loop captured each
attempt's ``Rejection`` in a local ``last``, fed it into the next prompt, and then **dropped it**.
``generate_via_llm`` returned only the final ``ValidatedProposal | Rejection``, so a caller could
observe THAT a proposal validated but never that it validated *on attempt 2, after the deterministic
validator falsified attempt 1*. Step 5 was the one step of the eight with no observable output.

The seam is a typed return value (``GenerationResult``), not an out-parameter or a callback: a
returned value cannot be silently dropped by a caller that forgets to pass a collector, and mypy
forces every call site to acknowledge it. Nothing about the loop's BOUND changes — ``max_attempts``
and ``meter.tick_round`` are untouched (§6: "refine until good enough" without a cap is forbidden).

**The honesty line these tests pin** — ``refinements`` holds only the rejections that were actually
FED BACK into a later attempt's prompt. When the attempt budget runs out, the final rejection IS
``outcome``; it informed nothing and must not be double-counted as a refinement. An implementation
that simply collects every rejection it sees passes the positive test and FAILS the bounded control,
which is exactly why the control is here.

Four tests, load-bearing as a set:
- the history reaches the caller carrying the SAME rejection the next prompt received (RED if the
  seam is detached, i.e. the history is computed internally and dropped again);
- the bounded control pins fed-back-only (RED on a collect-everything implementation);
- the run-level wiring: ``RunResult.refinements`` carries it out of ``run_project`` (RED if run.py
  drops what generation returned — the seam would exist but the demo still could not show Step 5);
- the simulation genuinely exercises it (RED if the scripted proposer reverts to a constant reply,
  which would leave the seam built but the demo one step shorter, silently).

They drive the CANONICAL ``ScriptedChatClient`` through its ``reply_selector`` seam rather than
defining another ``_inner_get_response`` body (S2.5 consolidation), so the proposer used here is the
same content-keyed one the simulation uses.
"""

from __future__ import annotations

from pathlib import Path

from portfolio_optimiser.budget import Budget, TokenMeter
from portfolio_optimiser.generate import generate_via_llm
from portfolio_optimiser.reference_domain import Project, load_reference_projects
from portfolio_optimiser.run import RunResult, run_project
from portfolio_optimiser.simulation import (
    ScriptedChatClient,
    scripted_factory,
    simulate_learning_loop,
)
from portfolio_optimiser.validator import (
    Rejection,
    ValidatedProposal,
    proposal_for,
    validate_proposal,
)

# Same fixture arithmetic as test_step5_refine_loadbearing: KONTOR-IT-E1 codes 05.2 + 03.1 ->
# affected total 1_482_500 -> degenerate Monte Carlo P90 = 444_750. 800_000 parses (< affected
# total) but exceeds P90 -> rejected; 200_000 validates. Both proposals are built via proposal_for,
# so their quantity/unit_cost ARE the project's cost lines and the S4.0 baseline stage passes.
_CODES = ["05.2", "03.1"]
_BAD_CLAIM = 800_000
_CORRECTED_CLAIM = 200_000

_BUNDLE_DIR = Path(__file__).resolve().parents[1] / "shared" / "examples" / "bygg-energi-mikro"


def _meter() -> TokenMeter:
    # max_rounds well above max_attempts so max_attempts -- not BudgetExceeded -- is the bound.
    return TokenMeter(Budget(max_tokens=10**9, max_rounds=20))


def _fixture() -> tuple[Project, str, str, str, Rejection]:
    """The shared reject-then-correct fixture: the bad proposal's JSON, the corrected one's, the
    flip key (the rejected claim value, which appears ONLY once the validator's reason is fed back),
    and the rejection the validator itself produces for the bad proposal — computed with the SAME
    validator the SUT uses, so the reason is byte-identical to the one the loop feeds back."""
    project = load_reference_projects()[0]  # KONTOR-IT-E1
    bad = proposal_for(project, _CODES, claimed_saving_nok=_BAD_CLAIM)
    corrected = proposal_for(project, _CODES, claimed_saving_nok=_CORRECTED_CLAIM)
    rej = validate_proposal(bad)
    assert isinstance(rej, Rejection), "fixture invariant: the BAD claim must reject"
    return (
        project,
        bad.model_dump_json(),
        corrected.model_dump_json(),
        f"{bad.claimed_saving_nok:.0f}",
        rej,
    )


async def test_falsification_history_escapes_the_generate_loop() -> None:
    """LOAD-BEARING: the rejection that informed attempt 2 is RETURNED to the caller, carrying the
    verbatim reason attempt 2's prompt received. Goes RED when the seam is detached (the history is
    computed internally and dropped again)."""
    project, bad_json, corrected_json, flip_key, rej = _fixture()

    sink: list[str] = []
    client = ScriptedChatClient(
        sink=sink,
        reply_selector=lambda prompt, _role: corrected_json if flip_key in prompt else bad_json,
    )
    # context="" so the flip token cannot pre-exist in attempt 1's prompt.
    result = await generate_via_llm(client, project, "", _meter(), max_attempts=3)

    assert isinstance(result.outcome, ValidatedProposal), (
        "fixture invariant: the proposer corrects on attempt 2 and that proposal validates"
    )
    # The seam itself: attempt 1's falsification is observable from OUTSIDE the loop.
    assert len(result.refinements) == 1, (
        "the falsification that informed attempt 2 did not escape generate_via_llm"
    )
    # Green-but-dead guard: it is the REAL rejection (reason + the rejected proposal), not a
    # placeholder -- and it is byte-identical to what the next prompt was given.
    assert result.refinements[0].reason == rej.reason
    assert result.refinements[0].proposal.claimed_saving_nok == _BAD_CLAIM
    assert result.refinements[0].reason in sink[1], (
        "the returned rejection is not the one that was fed into the next attempt's prompt"
    )


async def test_history_holds_only_fed_back_rejections_when_budget_runs_out() -> None:
    """CONTROL + HONESTY LINE: a proposer that never fixes its claim produces ``max_attempts``
    rejections, of which only the first ``max_attempts - 1`` were ever fed into a later prompt. The
    final one IS ``outcome``. RED on a collect-everything implementation, which is the obvious wrong
    way to build this seam."""
    project, bad_json, _corrected, _flip, _rej = _fixture()

    client = ScriptedChatClient(bad_json)
    result = await generate_via_llm(client, project, "", _meter(), max_attempts=3)

    assert isinstance(result.outcome, Rejection)
    assert client.call_count == 3, "control: the loop stays bounded by max_attempts"
    assert len(result.refinements) == 2, (
        "only the rejections that INFORMED a later attempt belong in the history; the final "
        "rejection is the outcome and informed nothing"
    )
    assert all(r.reason == result.outcome.reason for r in result.refinements)


async def test_run_result_carries_the_falsification_history(tmp_path: Path) -> None:
    """RUN-LEVEL WIRING: what generation returns must survive to ``RunResult`` — otherwise the seam
    exists but Step 5 is still invisible to the demo. RED if run.py drops ``refinements``.

    Drives the REFERENCE path (a reference project, so the S4.0 baseline is always anchored) with the
    same content-keyed proposer the simulation uses."""
    project, bad_json, corrected_json, flip_key, _rej = _fixture()

    docs = tmp_path / "docs"
    docs.mkdir()
    (docs / "kilde.md").write_text("Cost saving measure candidates for the project.\n", "utf-8")

    sink: list[str] = []
    result = await run_project(
        project.id,
        "local",
        docs_dir=str(docs),
        verdict_input={"decision": "approved", "rationale": "fixture verdict"},
        client_factory=scripted_factory(
            {
                "proposer": lambda prompt, _role: (
                    corrected_json if flip_key in prompt else bad_json
                ),
                "checker": "VERDICT: APPROVE",
            },
            sink,
        ),
        max_rounds=3,
    )

    assert isinstance(result, RunResult)
    assert isinstance(result.outcome, ValidatedProposal), (
        "fixture invariant: the corrected proposal validates"
    )
    assert len(result.refinements) == 1, (
        "run_project dropped the falsification history returned by generation"
    )
    assert result.refinements[0].proposal.claimed_saving_nok == _BAD_CLAIM


async def test_simulation_actually_exercises_step_five(tmp_path: Path) -> None:
    """DEMO PROTECTION: the offline simulation must genuinely go through a falsification before it
    validates — otherwise Step 5 is buildable but unshown, which is the state this whole seam exists
    to leave behind. RED the moment the scripted proposer reverts to a constant reply: with nothing
    to correct, ``refinements`` is empty and the demo silently loses a step.

    The rejection itself is NOT scripted: the proposer only overclaims. That 250000 exceeds the P90
    of 90000 is computed by the deterministic validator, which is the part worth showing."""
    result = await simulate_learning_loop(str(_BUNDLE_DIR), str(tmp_path))

    assert isinstance(result.run_a.outcome, ValidatedProposal), (
        "the corrected proposal must still validate — Step 5 ends in a proposal, not a dead end"
    )
    assert len(result.run_a.refinements) == 1, (
        "the simulation validated on the first attempt — the scripted reject-then-correct sequence "
        "is gone, so the demo cannot show Step 5"
    )
    rejected = result.run_a.refinements[0]
    assert rejected.proposal.claimed_saving_nok == 250_000
    assert (
        result.run_a.outcome.proposal.claimed_saving_nok < rejected.proposal.claimed_saving_nok
    ), (
        "the corrected proposal must claim LESS than the falsified one — otherwise the refinement "
        "did not respond to the falsification"
    )