portfolio-optimiser/tests/test_portfolio_concurrent_loadbearing.py

"""S3.3 concurrent fan-out — load-bearing seams for the D-D wave model.

``run_portfolio`` fans out over independent projects. S3.3 lets it run up to ``k`` projects
concurrently WITHOUT spending the property the whole suite rests on: determinism.

The design rests on one measured fact (``verdicts.py:303-304``): the only order-sensitive shared
state in the pass is ``VerdictStore.verdicts`` — ONE list with ONE append site. Retrieval ranking
is already order-independent (``verdicts.py:274-279`` ranks on ``(-similarity, id)``). So a
per-wave snapshot plus a merge barrier that sorts each wave's new verdicts on ``project_id``
restores byte-identity exactly — nothing wider is needed, and nothing narrower suffices.

This file grows across the plan's six steps. Step 1 covers the partitioning helper and the
fail-fast; the determinism contract and its probe arrive in Step 2.
"""

from __future__ import annotations

import asyncio
from collections.abc import Mapping, Sequence
from typing import Any

import pytest
from conftest import _PORTFOLIO_DEFAULT_REPLY, _ProjectAwareUsageChatClient
from test_portfolio import REPLIES

from portfolio_optimiser.run import PortfolioResult, _waves, run_portfolio
from portfolio_optimiser.verdicts import VerdictStore

# The shipped 3-project fixture, in submission order (mirrors tests/test_portfolio.py:57).
# ``REPLIES`` is reused from there deliberately: its three proposals are already verified to
# validate AND to mint three DISTINCT verdict ids, which is the precondition self-check 3 asserts.
_PORTFOLIO_IDS = ["FV42-GSV-E1", "RV13-RAS-TP", "BRU-LAKS-REHAB"]

# Scheduler yields per project, DESCENDING by submission index: the FIRST-submitted project yields
# most and therefore tends to finish LAST. This is what makes completion order depart from
# submission order — the precondition for the merge barrier to be load-bearing at all.
_YIELDS_BY_PID = {pid: (len(_PORTFOLIO_IDS) - i) * 4 for i, pid in enumerate(_PORTFOLIO_IDS)}


def test_waves_single_wave_when_k_covers_every_project() -> None:
    """k >= len(ids) is one wave — the whole portfolio runs concurrently."""
    assert _waves(["a", "b", "c"], 3) == [["a", "b", "c"]]
    assert _waves(["a", "b", "c"], 99) == [["a", "b", "c"]]


def test_waves_partitions_into_consecutive_chunks() -> None:
    """A k that does not divide the portfolio leaves a short final wave — never a dropped id."""
    assert _waves(["a", "b", "c"], 2) == [["a", "b"], ["c"]]
    assert _waves(["a", "b", "c", "d"], 2) == [["a", "b"], ["c", "d"]]


def test_waves_k1_is_the_sequential_path() -> None:
    """k=1 degenerates to one project per wave — the pre-S3.3 execution order, by construction.

    This is what makes Step 1 behaviour-preserving: with ``concurrency=1`` every wave holds a
    single project, so the wave loop visits ids in exactly the order the old ``for pid in ids``
    loop did."""
    assert _waves(["a", "b", "c"], 1) == [["a"], ["b"], ["c"]]


def test_waves_preserves_caller_order_and_loses_nothing() -> None:
    """Flattening any partition reproduces the input exactly — order preserved, no id dropped or
    duplicated. The wave model may change WHEN a project runs, never WHICH projects run."""
    ids = ["delta", "alpha", "charlie", "bravo", "echo"]
    for k in range(1, len(ids) + 2):
        flattened = [pid for wave in _waves(ids, k) for pid in wave]
        assert flattened == ids, f"k={k} did not preserve caller order"
    assert _waves([], 3) == []


async def test_concurrency_below_one_fails_fast_before_any_project_loads() -> None:
    """k < 1 raises ``ValueError`` with an explanatory message, mirroring the fail-fast idiom at
    ``run.py:547-553``. The raise precedes project loading, so the caller gets the error rather
    than a silently-empty pass — CLAUDE.md's "stoppkriterier + budsjett-tak påkrevd ved oppstart,
    fail-fast, aldri ubegrenset loop"."""
    for bad in (0, -1):
        with pytest.raises(ValueError, match="concurrency must be >= 1"):
            await run_portfolio(concurrency=bad)


# --------------------------------------------------------------------------------------------
# Step 2 — the determinism contract, and the probe that makes it non-vacuous.
# --------------------------------------------------------------------------------------------


class _Recorder:
    """Observes the pass at the ONE seam a test may touch: the chat client.

    Tracks, per client call, which project it belonged to and when it entered/left the awaited
    body. That yields three things no counter or timing could: genuine max-in-flight (two projects
    are concurrent iff their calls overlap between entry and exit), per-project completion order
    (the order of each project's LAST exit), and the raw call sequence."""

    def __init__(self) -> None:
        self.entries: list[str] = []
        self.exits: list[str] = []
        self._active: set[str] = set()
        self.max_in_flight = 0

    def enter(self, pid: str) -> None:
        self.entries.append(pid)
        self._active.add(pid)
        self.max_in_flight = max(self.max_in_flight, len(self._active))

    def leave(self, pid: str) -> None:
        self.exits.append(pid)
        self._active.discard(pid)

    def completion_order(self) -> list[str]:
        """Projects ordered by their LAST client call — the closest observable proxy for the order
        in which the projects actually finished."""
        order: list[str] = []
        for pid in self.exits:
            if pid in order:
                order.remove(pid)
            order.append(pid)
        return order


class _OrderProbeClient(_ProjectAwareUsageChatClient):
    """Perturbs completion order DETERMINISTICALLY, without a wall-clock sleep.

    ``_inner_get_response`` (``simulation.py:113-120``) is a SYNCHRONOUS method declared to return
    ``Awaitable[ChatResponse] | ResponseStream[...]``. The probe therefore returns a coroutine that
    yields to the event loop N times before delegating to the canonical body. ``asyncio.sleep(0)``
    is a pure scheduler yield, not a timed wait, so the perturbation is deterministic rather than
    timing-dependent — and it cannot make a sequential pass interleave, which is exactly why this
    probe can tell the two apart.

    N is derived from the PROMPT BLOB, never from a constructor argument: ``client_factory`` is
    ``Callable[[str], BaseChatClient]`` whose argument is the ROLE, not the project id — which is
    precisely why ``_ProjectAwareUsageChatClient`` scans the prompt in the first place.

    The stream path is delegated UNTOUCHED. The canonical returns a coroutine when
    ``stream=False`` and a stream-shaped object when ``stream=True``; wrapping both in one
    ``async def`` would hand the framework the wrong shape. The subclass chain
    (probe -> ``_ProjectAwareUsageChatClient`` -> ``ScriptedChatClient`` ->
    ``OpenAIChatCompletionClient``) is preserved so ``BudgetMiddleware`` still engages."""

    def __init__(self, replies: dict[str, str], *, default_reply: str, recorder: _Recorder) -> None:
        super().__init__(replies, default_reply=default_reply)
        self._recorder = recorder

    def _inner_get_response(
        self,
        *,
        messages: Sequence[Any],
        options: Mapping[str, Any],
        stream: bool = False,
        **kwargs: Any,
    ) -> Any:
        if stream:
            return super()._inner_get_response(
                messages=messages, options=options, stream=True, **kwargs
            )

        blob = " ".join(getattr(m, "text", "") or "" for m in messages)
        pid = next((p for p in _YIELDS_BY_PID if p in blob), "")
        yields = _YIELDS_BY_PID.get(pid, 0)
        recorder = self._recorder

        async def _probed() -> Any:
            recorder.enter(pid)
            try:
                for _ in range(yields):
                    await asyncio.sleep(0)
                return await super(_OrderProbeClient, self)._inner_get_response(
                    messages=messages, options=options, stream=False, **kwargs
                )
            finally:
                recorder.leave(pid)

        return _probed()


async def _pass(k: int, recorder: _Recorder) -> tuple[list[str], PortfolioResult]:
    """One portfolio pass at concurrency ``k`` on a FRESH store, returning the store's verdict-id
    SEQUENCE (order is the contract — never a set, never ``sorted(...)``) and the aggregate."""
    store = VerdictStore(verdicts=[])

    def factory(_role: str) -> Any:
        return _OrderProbeClient(REPLIES, default_reply=_PORTFOLIO_DEFAULT_REPLY, recorder=recorder)

    result = await run_portfolio(
        _PORTFOLIO_IDS, "local", store=store, client_factory=factory, concurrency=k
    )
    return [v.id for v in result.store.verdicts], result


def _aggregate_fields(result: PortfolioResult) -> tuple[Any, ...]:
    """The aggregate's comparable scalar surface, plus the ORDER of ``runs`` — which must follow
    ``project_ids``, never completion order."""
    return (
        result.validated_count,
        result.rejected_count,
        result.sum_claimed_saving_nok,
        result.sum_token_usage,
        result.stopped_early,
        tuple(r.verdict.id for r in result.runs),
    )


async def test_concurrent_pass_is_byte_identical_to_sequential() -> None:
    """S3.3's core contract: ``concurrency=3`` produces the SAME store-verdict id SEQUENCE as
    ``concurrency=1``, under a completion order that genuinely differs from submission order.

    **Detach point: remove the ``sorted(...)`` from ``_merge_wave`` -> this test goes RED.**
    That is the seam. Without the sort, each wave's verdicts land in completion order, which the
    probe has made differ from submission order, and the two id sequences diverge.

    **Probe mechanism.** ``_OrderProbeClient`` returns a coroutine that yields to the loop N times
    (``asyncio.sleep(0)``, a pure scheduler yield — no wall clock) before delegating to the
    canonical body, with N keyed on the project found in the prompt blob and DESCENDING by
    submission index. See the class docstring for why N cannot be a constructor argument and why
    the stream path is delegated untouched.

    **Three probe self-checks, and why the test is worthless without them.** Each one closes a way
    this assertion could pass while proving nothing:

    1. *The probe actually reordered.* If the yields had no effect, completion order would equal
       submission order, the barrier would never be exercised, and the equality would hold with
       the sort deleted. Asserted as completion order != submission order — NOT as an exact
       permutation: ``asyncio.sleep(0)`` is a scheduler yield, not an ordering primitive, and the
       total interleaving depends on every ``await`` along each ``run_project`` path, not only the
       probe's. Any non-identity permutation exercises the barrier, so demanding a specific one
       would only add a way for the test to fail for a reason that is not the contract.
    2. *Concurrency actually happened.* A ``concurrency=3`` that silently degraded to sequential
       would make both sides run the identical path — trivially equal, and the sneakiest failure
       mode of the set. Asserted as max-in-flight > 1, measured from genuine overlap between a
       client call's entry and exit, not from a counter.
    3. *There is something to order.* Ids hash candidate features; if the fixture collapsed to one
       id under first-write-wins there would be no ordering to preserve. Asserted as >= 3 distinct
       ids.

    **SC2 has a strong half and a weak half, and this test does not pretend otherwise.** Equality
    of STORE CONTENT is a real test of the barrier. Equality of the AGGREGATE is nearly free on
    this fixture: retrieval ranks on ``(-similarity, id)`` (order-independent,
    ``verdicts.py:274-279``) and the Step-1 ExpeL fold is ``bundle_dir``-gated with no reference
    project setting it. Reporting only "byte identical" would be half-vacuous — the same defect
    class as the Spike B call-counter tautology. Step 6 adds the bundle-backed case where the
    difference IS observable.

    **What the repetitions do and do not guard.** They do NOT guard hash-seed nondeterminism:
    ``PYTHONHASHSEED`` is fixed for the life of the interpreter, so every in-process repetition
    shares one seed and the second rep can never disagree with the first on that axis. What they
    DO guard is scheduler nondeterminism under real concurrency — ``asyncio.gather`` gives no
    byte-identical interleaving guarantee across repetitions, so a barrier that happened to work
    once is caught here. Claiming otherwise would be the over-read the S3.1 review corrected."""
    probe = _Recorder()
    concurrent_ids, concurrent = await _pass(3, probe)

    # Self-check 2 — concurrency actually happened.
    assert probe.max_in_flight > 1, (
        f"max in-flight was {probe.max_in_flight}: concurrency=3 never overlapped two projects, so "
        f"this pass ran the sequential path and the equality below proves nothing. "
        f"call sequence={probe.entries}"
    )

    # Self-check 1 — the probe actually reordered (non-identity, not an exact permutation).
    observed = probe.completion_order()
    assert observed != _PORTFOLIO_IDS, (
        f"completion order {observed} equals submission order {_PORTFOLIO_IDS}: the probe did not "
        "reorder anything, so the merge barrier was never exercised and this test would stay "
        "green with the sort deleted"
    )

    # Self-check 3 — there is something to order.
    assert len(set(concurrent_ids)) >= 3, (
        f"only {len(set(concurrent_ids))} distinct verdict ids ({concurrent_ids}): the fixture "
        "collapsed under first-write-wins, so no ordering exists to preserve"
    )

    # The contract itself, repeated to catch a barrier that works only by scheduling luck.
    for rep in range(20):
        sequential_ids, sequential = await _pass(1, _Recorder())
        again_ids, again = await _pass(3, _Recorder())
        assert again_ids == sequential_ids, (
            f"rep {rep}: concurrent store order {again_ids} != sequential {sequential_ids} — the "
            f"merge barrier is not restoring deterministic order (probe reordered to {observed})"
        )
        assert _aggregate_fields(again) == _aggregate_fields(sequential), (
            f"rep {rep}: aggregate diverged between k=3 and k=1 — note this is the WEAK half of "
            "SC2 (see docstring); a divergence here means something broader than ordering broke"
        )
    assert concurrent_ids == sequential_ids


async def test_runs_follow_project_ids_not_completion_order() -> None:
    """``PortfolioResult.runs`` is ordered by ``project_ids``, never by which project finished
    first — so a caller indexing ``runs[0]`` gets the project they listed first, at every ``k``.

    This holds because ``asyncio.gather`` resolves in ARGUMENT order rather than completion order,
    and waves are built in caller order. It is asserted separately from the store-content contract
    because it survives a different set of mistakes: a barrier bug reorders the STORE while leaving
    ``runs`` correct, and an executor that appended results as they completed would reorder ``runs``
    while leaving the store correct. The probe is active, so completion order is confirmed to
    differ from submission order — without that this assertion would be untested at k>1."""
    probe = _Recorder()
    _, result = await _pass(3, probe)

    assert probe.max_in_flight > 1, "the pass ran sequentially — runs ordering is untested at k>1"
    assert probe.completion_order() != _PORTFOLIO_IDS, (
        "completion order matched submission order, so this assertion would hold trivially"
    )
    # ``claimed_saving_nok`` is the per-project discriminator in REPLIES (200k / 130k / 210k) —
    # ``measure_type`` is NOT, since FV42 and BRU deliberately share "Reduce scope".
    assert [r.verdict.proposal_features.claimed_saving_nok for r in result.runs] == [
        200_000.0,
        130_000.0,
        210_000.0,
    ], (
        f"runs came back in {probe.completion_order()} (completion) rather than "
        f"{_PORTFOLIO_IDS} (submission) order"
    )