portfolio-optimiser/tests/test_parse_error_feedback_loadbearing.py

"""P20 DEL C — a parse failure that does not burn the round ledger, and an announcement that
names what it is about.

**C1, what was measured (P19 F4).** ``_fetch_parsed`` retried a malformed reply with the
BYTE-IDENTICAL prompt. One round-3 run, ``kontrakt-sorasen-04``, left a
``{run_id}-parse-failures.json`` with ELEVEN rows, every one of them the same failure
(``claimed_saving_nok`` ≤ 0) — eleven of the run's twelve rounds, spent re-asking a question the
model had already answered the same wrong way, because nothing ever told it what was wrong. Step
5's ``prior_rejection`` carries VALIDATOR rejections; a reply that never parsed never reaches a
validator, so no existing block could carry it.

**C2, what was measured (P17b F5).** ``--across-bundle`` takes no ``--project-id``, so the
announcement — the one thing printed before the first paid call — said "Run mandate for the
portfolio" for a commission dispatched across two named knowledge bases.

What each arm pins:

(a) the reason reaches the NEXT attempt's prompt, verbatim, asserted on what the client received;
(b) attempt 1 is byte-identical: a run whose first reply parses sends the pre-P20 prompt;
(c) the block is per-RETRY — once a reply parses, the next attempt does not carry a stale reason;
(d) the evidence artefact is unchanged: the verbatim text is still captured (funn 1 stands);
(e) the announcement names the routed bases by their DECLARED ids;
(f) an unresolvable base falls back to its directory name rather than refusing — the
    ``dimension_label`` precedent, so announcing never changes which error an operator sees;
(g) a single-project run and a portfolio pass announce exactly as before.
"""

from __future__ import annotations

import asyncio
from pathlib import Path
from typing import Any

from portfolio_optimiser.budget import Budget, TokenMeter
from portfolio_optimiser.generate import ParseFailure, _build_messages, generate_via_llm
from portfolio_optimiser.reference_domain import Project
from portfolio_optimiser.run import announced_subject
from portfolio_optimiser.simulation import ScriptedChatClient

_EXAMPLES = Path(__file__).resolve().parents[1] / "shared" / "examples"
_TUNNEL = _EXAMPLES / "tunnel-hauglia"

#: The exact failure ``kontrakt-sorasen-04`` produced eleven times: a well-formed JSON object
#: whose ``claimed_saving_nok`` is 0, refused by pydantic before any validator sees it.
_UNPARSEABLE = (
    '{"measure":"m","affected_items":[{"code":"C-1","quantity":1000,"unit_cost":100}],'
    '"claimed_saving_nok":0}'
)
#: The grounding this loop declares: P7 requires the code to occur in the input verbatim.
_CONTEXT = "The project price schedule carries cost line C-1."
_VALID = (
    '{"measure":"m","affected_items":[{"code":"C-1","quantity":1000,"unit_cost":100}],'
    '"claimed_saving_nok":5000}'
)


def _project() -> Project:
    return Project(id="p", name="n", description="d", currency="NOK", cost_items=(), docs_dir=".")


def _generate(replies: list[str]) -> tuple[Any, list[str], list[ParseFailure]]:
    """Drive the real loop with the repo's ONE scripted client, recording every prompt sent."""
    sink: list[str] = []
    failures: list[ParseFailure] = []
    client = ScriptedChatClient(script=list(replies), sink=sink, role="proposer")
    result = asyncio.run(
        generate_via_llm(
            client,
            _project(),
            _CONTEXT,
            TokenMeter(Budget(max_tokens=100_000, max_rounds=8)),
            parse_failures=failures,
        )
    )
    return result, sink, failures


# ------------------------------------------------------------------------------- C1


def test_the_parse_reason_reaches_the_next_attempts_prompt() -> None:
    """(a) The retry is no longer blind — asserted on what the CLIENT received."""
    _result, sink, _failures = _generate([_UNPARSEABLE, _VALID])
    assert len(sink) >= 2, sink
    assert "could not be PARSED" in sink[1]
    assert "claimed_saving_nok" in sink[1]


def test_the_first_prompt_is_byte_identical_to_the_pre_p20_one() -> None:
    """(b) Without this, (a) could be satisfied by always appending the block."""
    _result, sink, _failures = _generate([_VALID])
    assert "could not be PARSED" not in sink[0]
    assert sink[0] == _build_messages(_project(), _CONTEXT)[0].text


def test_a_reason_does_not_survive_a_reply_that_parsed() -> None:
    """(c) Per-RETRY, like ``prior_rejection`` is per-attempt: no stale instruction."""
    assert "could not be PARSED" not in _build_messages(_project(), "c")[0].text
    carried = _build_messages(_project(), "c", parse_error="ValueError: x")[0].text
    assert "ValueError: x" in carried


def test_the_verbatim_evidence_is_still_captured() -> None:
    """(d) Fase 1b funn 1 stands: the paid reply's own text survives the retry."""
    _result, _sink, failures = _generate([_UNPARSEABLE, _VALID])
    assert len(failures) == 1
    assert failures[0].text == _UNPARSEABLE
    assert failures[0].error.startswith("ValidationError")


# ------------------------------------------------------------------------------- C2


def test_the_announcement_names_the_routed_bases() -> None:
    """(e) Two bases, two declared ids — not "the portfolio"."""
    assert announced_subject(None, (str(_TUNNEL),)) == "tunnel-hauglia"


def test_an_unresolvable_base_falls_back_to_its_directory_name(tmp_path: Path) -> None:
    """(f) Naming never changes which error an operator sees (the dimension_label precedent)."""
    missing = tmp_path / "not-a-base"
    assert announced_subject(None, (str(missing),)) == "not-a-base"


def test_the_two_older_subjects_are_unchanged() -> None:
    """(g) A named project wins; no bases at all is still the portfolio."""
    assert announced_subject("proj-1", ()) == "proj-1"
    assert announced_subject("proj-1", ("ignored",)) == "proj-1"
    assert announced_subject(None, ()) == "the portfolio"