portfolio-optimiser/tests/conftest.py

"""Shared e2e fixtures (Step 13): a scripted chat client that emits a SYNTHETIC UsageDetails
(so token accounting is real-shaped without an LLM), plus store + docs-dir fixtures.

The synthetic ``UsageDetails`` is what lets the budget meter / provenance ``token_usage`` be a
positive, UsageDetails-sourced number in CI — the REAL-provider populated-usage assertion is
the gated live arm (Step 14).
"""

from __future__ import annotations

import json
from collections.abc import Callable, Sequence

import pytest
from agent_framework import BaseChatClient

from portfolio_optimiser.reference_domain import load_reference_projects
from portfolio_optimiser.simulation import ScriptedChatClient
from portfolio_optimiser.verdicts import VerdictStore, seed_store


class SyntheticUsageChatClient(ScriptedChatClient):
    """The scripted-list-then-default test double — now a THIN subclass of the canonical
    ``ScriptedChatClient`` (S2.5 consolidation). It keeps its full PUBLIC surface (the
    ``default_reply=`` kwarg constructor, the ``call_count`` attribute, ``model``/OTEL
    ``"synthetic"``) but delegates the shared ``_inner_get_response`` body to the canonical — the
    scripted-list-then-default behaviour lives in its selector."""

    def __init__(
        self,
        scripted: Sequence[str] | None = None,
        *,
        default_reply: str = "ok",
        tokens_per_reply: int = 8,
    ) -> None:
        scripted_list = list(scripted or [])
        counter = {"i": 0}

        def _select(_blob: str, _role: str) -> str:
            i = counter["i"]
            counter["i"] = i + 1
            return scripted_list[i] if i < len(scripted_list) else default_reply

        super().__init__(
            reply_selector=_select, default_reply=default_reply, tokens_per_reply=tokens_per_reply
        )


@pytest.fixture()
def make_client_factory() -> Callable[..., Callable[[str], BaseChatClient]]:
    """Return a maker that builds a per-role client factory emitting synthetic usage."""

    def _make(default_reply: str, *, tokens: int = 8) -> Callable[[str], BaseChatClient]:
        def factory(role: str) -> BaseChatClient:
            return SyntheticUsageChatClient(default_reply=default_reply, tokens_per_reply=tokens)

        return factory

    return _make


_DEFAULT_CLAIM = 20_000
# Last-resort reply for a prompt naming NO known reference project (the anchored per-project
# fallback below cannot be built then). Kept for that case only.
_PORTFOLIO_DEFAULT_REPLY = (
    '{"measure":"Reduce scope","affected_items":'
    '[{"code":"01.1","quantity":1,"unit_cost":100000}],"claimed_saving_nok":20000}'
)


def _anchored_default_replies() -> dict[str, str]:
    """A VALID default proposal PER reference project, quoting that project's OWN first cost line
    verbatim (S4.0): since the road path anchors the validator to ``project.cost_items``, a generic
    reply carrying an invented magnitude for code ``01.1`` is now — correctly — rejected as a
    fabricated cost line. Anchoring the fixture is the fix; weakening the gate is not.

    ``claimed_saving_nok`` stays ``20_000`` for every project, exactly as the single generic reply
    claimed before, so every ledger/goal/budget assertion built on that figure is unchanged. Each
    project's first line is ``01.1 Rigg og drift`` at >= 480 000 NOK, so P90 (>= 144 000) clears the
    claim on every project."""
    replies: dict[str, str] = {}
    for project in load_reference_projects():
        line = project.cost_items[0]
        replies[project.id] = json.dumps(
            {
                "measure": "Reduce scope",
                "affected_items": [
                    {"code": line.code, "quantity": line.quantity, "unit_cost": line.unit_cost}
                ],
                "claimed_saving_nok": _DEFAULT_CLAIM,
            }
        )
    return replies


class _ProjectAwareUsageChatClient(ScriptedChatClient):
    """Selects its reply by scanning the incoming prompt for a known ``project_id`` substring (the
    prompt embeds ``project.id`` at run.py:162 and generate.py:48), falling back to a default valid
    proposal — so ``run_portfolio``'s single ``client_factory`` stays production-shaped while tests
    vary the proposal per project. A THIN subclass: the prompt-scan lives in its selector, the shared
    ``_inner_get_response`` body in the canonical.

    The fallback is itself project-aware (S4.0): a prompt naming a reference project gets that
    project's baseline-anchored default reply, so an un-mapped project still produces a proposal the
    anchored validator admits. Only a prompt naming NO known project falls through to
    ``default_reply``."""

    def __init__(
        self, replies: dict[str, str], *, default_reply: str, tokens_per_reply: int = 8
    ) -> None:
        table = dict(replies)
        anchored = _anchored_default_replies()

        def _select(blob: str, _role: str) -> str:
            explicit = next((r for pid, r in table.items() if pid in blob), None)
            if explicit is not None:
                return explicit
            return next((r for pid, r in anchored.items() if pid in blob), default_reply)

        super().__init__(
            reply_selector=_select, default_reply=default_reply, tokens_per_reply=tokens_per_reply
        )


@pytest.fixture()
def make_portfolio_client_factory() -> Callable[..., Callable[[str], BaseChatClient]]:
    """Return a maker that builds a single project-aware client factory: every client it
    produces picks its reply from ``replies`` by scanning the prompt for the project id, so one
    factory serves the whole portfolio (matching ``run_portfolio``'s single-factory seam)."""

    def _make(
        replies: dict[str, str],
        *,
        default_reply: str = _PORTFOLIO_DEFAULT_REPLY,
        tokens: int = 8,
    ) -> Callable[[str], BaseChatClient]:
        def factory(role: str) -> BaseChatClient:
            return _ProjectAwareUsageChatClient(
                replies, default_reply=default_reply, tokens_per_reply=tokens
            )

        return factory

    return _make


class _RecordingChatClient(ScriptedChatClient):
    """Records the incoming prompt blob per call into a SHARED sink, then returns a fixed valid
    reply. Lets a test assert exactly what text reached the prompt — the probe the Step-1 ExpeL
    wiring is made load-bearing against (does a prior verdict reach the hypothesis prompt?). A THIN
    subclass: the canonical records to the ``sink`` (when given one) and returns the constant reply."""

    def __init__(self, sink: list[str], reply: str, *, tokens_per_reply: int = 8) -> None:
        super().__init__(reply, sink, tokens_per_reply=tokens_per_reply)


@pytest.fixture()
def make_recording_client_factory() -> Callable[
    [str], tuple[Callable[[str], BaseChatClient], list[str]]
]:
    """Return a maker that builds a per-role client factory recording every prompt blob into a
    shared list. Returns ``(factory, recorded_prompts)`` so the test inspects what reached the
    prompt across the whole run (debate rounds + generation)."""

    def _make(reply: str) -> tuple[Callable[[str], BaseChatClient], list[str]]:
        sink: list[str] = []

        def factory(role: str) -> BaseChatClient:
            return _RecordingChatClient(sink, reply)

        return factory, sink

    return _make


@pytest.fixture()
def fresh_store() -> VerdictStore:
    return VerdictStore(verdicts=[])


@pytest.fixture()
def seeded_store() -> VerdictStore:
    return seed_store()


@pytest.fixture()
def docs_dir(tmp_path) -> str:
    d = tmp_path / "docs"
    d.mkdir()
    (d / "cost.txt").write_text(
        "Asphalt Ab11 unit rate renegotiation reduced the paving cost on the school stretch.",
        encoding="utf-8",
    )
    return str(d)


# ------------------------------------------------------------------------------------------------
# Fase 1b — the FULL-RUN contract, in ONE copy (kø-(p): a second copy of an assertion drifts).
# ------------------------------------------------------------------------------------------------


#: What the paid run must fell, stated as an assertion rather than as prose. The open honesty limit
#: after økt 37 is narrow and specific: *"that the emitted structured schema IS accepted by the LIVE
#: endpoint is NOT verified — the tests prove conformance with the DOCUMENTED subset, not
#: acceptance."* So the contract asserts schema ACCEPTANCE, never model JUDGEMENT.
#:
#: The discriminator is an artefact this repo already owns: ``{run_id}-parse-failures.json`` is
#: written if and ONLY if some reply failed to parse (økt 35 invariant — "the file's presence is the
#: signal"). Its ABSENCE beside a RunResult therefore proves that every generation reply came back
#: in the requested shape, which is exactly what "the live endpoint honoured the schema" means.
#:
#: ``validator_decision`` is the second half: it mirrors the VALIDATOR alone (never the checker), so
#: reading it proves the deterministic gate actually ran on a parsed candidate. Both ``validated``
#: and ``rejected`` satisfy the contract — a P90 rejection is a run that CONCLUDED, and demanding
#: ``validated`` would be asserting that the model reasons well, which no schema can promise and
#: which one paid run could not establish anyway.
def assert_full_run_contract(result, outbox_dir, run_id: str) -> None:
    """Assert the Fase 1b full-run contract on a completed ``run_project`` result.

    Two things, and deliberately nothing else:

    1. **No parse-failure artefact** — every reply parsed, i.e. the live endpoint accepted the
       emitted ``response_format`` schema. This is the honesty limit being felled.
    2. **The validator was reached and decided** — ``provenance.validator_decision`` is one of the
       two decisions the deterministic gate emits.

    Deliberately NOT asserted: ``checker_verdict``, token counts, ``validated_count``, or the
    content of the proposal. Those are model-judgement claims, and one run cannot carry them.
    """
    from pathlib import Path

    artefact = Path(outbox_dir) / f"{run_id}-parse-failures.json"
    if artefact.exists():
        # Quote the evidence in the failure message: the whole point of økt 35 was that the operator
        # should never again have to guess WHY a reply did not parse.
        raise AssertionError(
            f"the endpoint did NOT honour the structured schema — {artefact.name} exists.\n"
            f"{artefact.read_text(encoding='utf-8')[:2000]}"
        )
    assert result.provenance.validator_decision in {"validated", "rejected"}, (
        "the deterministic validator never decided — the run did not reach the gate with a "
        f"parsed candidate (validator_decision={result.provenance.validator_decision!r})"
    )