portfolio-optimiser/tests/test_portfolio.py

"""Fase 3 — sequential portfolio fan-out over N independent projects.

Covers the success criteria: SC2 (fan-out + aggregate sums), SC4 (shared store accumulates +
load-bearing cross-project retrieval), SC3 (load-bearing meter-isolation detach), SC7 (both
profiles offline + resolve_model teeth), SC1 (new project via config only + no-hardcoded-id src
guard), SC6 (extension doc exists + references the seams).

Pattern: tests/test_vertical_slice_e2e.py:28 (run_project call shape). The project-aware
synthetic client (conftest ``make_portfolio_client_factory``) selects each project's reply by
scanning the prompt for its id, so one production-shaped factory serves the whole portfolio.
"""

import pytest

from portfolio_optimiser.run import PortfolioResult, RunResult, run_portfolio, run_project
from portfolio_optimiser.validator import Rejection

# The synthetic reply IS the proposal: generate._parse_ir builds affected_items (each with its
# own quantity/unit_cost) straight from this JSON, and the validator's P90 = 0.30 x Σ(qty·unit_cost)
# ONLY when ``assumptions`` is empty (degenerate Monte Carlo, validator.py:108-113). All three
# replies therefore OMIT ``assumptions`` and carry explicit magnitudes so each
# ``claimed_saving_nok`` <= P90. Verified against validator + ir:
#   FV42-GSV-E1     Σ=1,482,500  P90=444,750  claimed 200,000 -> validates
#   RV13-RAS-TP     Σ=  756,000  P90=226,800  claimed 130,000 -> validates  (decoy)
#   BRU-LAKS-REHAB  Σ=2,580,500  P90=774,150  claimed 210,000 -> validates
# ``measure`` is byte-identical "Reduce scope" for FV42+BRU (measure-match is exact string
# equality, verdicts.py:68) and "Material substitution" for the decoy, so the BRU<->FV42 pair
# overlaps (shared code 05.2 + measure + magnitude bucket) while the decoy does not.
REPLIES = {
    "FV42-GSV-E1": (
        '{"measure":"Reduce scope","affected_items":['
        '{"code":"05.2","quantity":4300,"unit_cost":215},'
        '{"code":"03.1","quantity":1800,"unit_cost":310}],"claimed_saving_nok":200000}'
    ),
    "RV13-RAS-TP": (
        '{"measure":"Material substitution","affected_items":['
        '{"code":"88.2","quantity":180,"unit_cost":4200}],"claimed_saving_nok":130000}'
    ),
    "BRU-LAKS-REHAB": (
        '{"measure":"Reduce scope","affected_items":['
        '{"code":"05.2","quantity":4300,"unit_cost":215},'
        '{"code":"07.4","quantity":2400,"unit_cost":690}],"claimed_saving_nok":210000}'
    ),
}

_PORTFOLIO_IDS = ["FV42-GSV-E1", "RV13-RAS-TP", "BRU-LAKS-REHAB"]


async def test_a_fanout_returns_one_runresult_per_project(
    make_portfolio_client_factory, fresh_store
) -> None:
    """SC2: run_portfolio fans out sequentially, one RunResult per project, and aggregates the
    validated/rejected counts + claimed-saving + token sums."""
    result = await run_portfolio(
        _PORTFOLIO_IDS,
        "local",
        store=fresh_store,
        client_factory=make_portfolio_client_factory(REPLIES),
    )
    assert isinstance(result, PortfolioResult)
    assert len(result.runs) == 3
    assert all(isinstance(r, RunResult) for r in result.runs)
    # Aggregate pinned to the fixture constants above (all three validate).
    assert result.validated_count == 3
    assert result.rejected_count == 0
    assert result.sum_claimed_saving_nok == 540000
    # Explicit element-wise wiring check (not the field's own sum() definition).
    expected_tokens = (
        result.runs[0].provenance.token_usage
        + result.runs[1].provenance.token_usage
        + result.runs[2].provenance.token_usage
    )
    assert result.sum_token_usage == expected_tokens
    assert all(r.provenance.token_usage > 0 for r in result.runs)


async def test_a2_unknown_project_id_raises(make_portfolio_client_factory, fresh_store) -> None:
    """The unknown-id error path: an id absent from the loaded portfolio raises ValueError."""
    with pytest.raises(ValueError):
        await run_portfolio(
            ["NOPE-DOES-NOT-EXIST"],
            "local",
            store=fresh_store,
            client_factory=make_portfolio_client_factory(REPLIES),
        )


async def test_a3_rejected_proposal_excluded_from_aggregate(
    make_portfolio_client_factory, fresh_store
) -> None:
    """F2 (rejection arm, previously unexercised): one project's claim exceeds its P90, so the
    validator REJECTS it (FV42 800000 > P90 444750, <= Σ 1,482,500), while RV13+BRU validate.
    rejected_count counts it and the validated-only sum (run.py:241) EXCLUDES its claim. Also the
    F1 portfolio seam: provenance.model mirrors the injected client across all N records."""
    replies = {**REPLIES, "FV42-GSV-E1": REPLIES["FV42-GSV-E1"].replace("200000", "800000")}
    result = await run_portfolio(
        _PORTFOLIO_IDS,
        "local",
        store=fresh_store,
        client_factory=make_portfolio_client_factory(replies),
    )
    assert result.rejected_count == 1
    assert result.validated_count == 2
    # Validated-only sum EXCLUDES the rejected 800000 (run.py:241): 130000 + 210000.
    assert result.sum_claimed_saving_nok == 340000
    rejected = [r for r in result.runs if isinstance(r.outcome, Rejection)]
    assert len(rejected) == 1
    assert rejected[0].outcome.proposal.claimed_saving_nok == 800000  # carried, not summed
    assert rejected[0].provenance.validator_decision == "rejected"
    # F1 portfolio seam: the injected client's model is stamped across ALL N records.
    assert all(r.provenance.model == "synthetic" for r in result.runs)


async def test_b_shared_store_accumulates_and_surfaces_prior_verdict(
    make_portfolio_client_factory, fresh_store
) -> None:
    """SC4 (load-bearing, not mere non-emptiness): the ONE shared store accumulates a distinct
    verdict per project (3 -> 3 pairwise-distinct ids), and the cross-project ExpeL retrieval
    surfaces the STRUCTURAL match — BRU (runs[2]) retrieves FV42's verdict (runs[0]), since
    sim(BRU,FV42)=0.60 (shared code 05.2 + identical "Reduce scope" measure + same magnitude
    bucket) ranks above the RV13 decoy at sim(BRU,decoy)=0.15. If retrieval ranking breaks, or
    two proposals collide to a single minted id, this fails — it asserts the specific match, not
    store-non-empty."""
    result = await run_portfolio(
        _PORTFOLIO_IDS,
        "local",
        store=fresh_store,
        client_factory=make_portfolio_client_factory(REPLIES),
    )
    assert len(result.store.verdicts) == 3
    ids = [v.id for v in result.store.verdicts]
    assert len(set(ids)) == 3  # pairwise distinct minted ids (no collision)
    # BRU surfaces FV42 (the structural match), ranked above the decoy.
    assert result.runs[2].retrieved[0].id == result.runs[0].verdict.id


async def test_c_execution_state_isolation_is_load_bearing(
    make_client_factory, fresh_store
) -> None:
    """SC3 (cap-independent automatic detach guard): each project's execution state (the budget
    meter) is built FRESH per run, so per-project token_usage is its OWN only. Built on the
    Step-1 ``meter=`` seam + Step-3 ``meter_factory``. The factory emits a UNIFORM but VALID
    proposal (REPLIES["FV42-GSV-E1"] for every call) so both projects complete — a bare
    unparseable default would loop the generate fetch to BudgetExceeded instead. Both arms run
    every CI, so the detach is encoded automatically (no manual reviewer-revert)."""
    from portfolio_optimiser.budget import Budget, TokenMeter
    from portfolio_optimiser.reference_domain import load_reference_projects

    f = make_client_factory(REPLIES["FV42-GSV-E1"])
    rv13 = {p.id: p for p in load_reference_projects()}["RV13-RAS-TP"]

    # 1. Baseline: RV13 run standalone (its own fresh meter).
    standalone = await run_project(
        "RV13-RAS-TP",
        "local",
        docs_dir=rv13.docs_dir,
        verdict_input=rv13.verdict_input,
        client_factory=f,
    )
    # 2. Isolated (default, no meter_factory): RV13 as project 1 in the portfolio. Its usage
    #    equals the standalone baseline — independent of project 0. THE load-bearing assertion:
    #    if run_portfolio shared a meter by default, runs[1] would be cumulative and this breaks.
    iso = await run_portfolio(
        ["FV42-GSV-E1", "RV13-RAS-TP"],
        "local",
        store=fresh_store,
        client_factory=f,
    )
    assert iso.runs[1].provenance.token_usage == standalone.provenance.token_usage
    # 3. Shared (injected one meter across both): usage accumulates, proving the detach is real
    #    (and that the == arm above would redden under sharing).
    shared = TokenMeter(Budget(max_tokens=1_000_000, max_rounds=1000))
    sh = await run_portfolio(
        ["FV42-GSV-E1", "RV13-RAS-TP"],
        "local",
        client_factory=f,
        meter_factory=lambda: shared,
    )
    assert sh.runs[1].provenance.token_usage > sh.runs[0].provenance.token_usage


@pytest.mark.parametrize("profile", ["local", "azure"])
async def test_d_both_profiles_run_offline(
    make_portfolio_client_factory, fresh_store, profile
) -> None:
    """SC7: both profiles drive the portfolio contract path OFFLINE under the synthetic client
    (the path is actually executed, not just the backend instantiated). Teeth: ``resolve_model``
    is a profile-dependent seam reachable WITHOUT egress — each profile resolves its own
    configured model from the bundled model_map.json, and local != azure (deleting the azure
    entry reddens this). Honest limitation: under an injected ``client_factory``, ``profile`` is
    otherwise inert inside ``run_project`` (run.py:197 stamps the injected client's own model, not
    the profile's ``resolve_model``); the end-to-end run proves the path executes under both, and
    ``resolve_model`` is the only profile-dependent seam provable offline."""
    from portfolio_optimiser.backends import resolve_model

    result = await run_portfolio(
        _PORTFOLIO_IDS,
        profile,
        store=fresh_store,
        client_factory=make_portfolio_client_factory(REPLIES),
    )
    assert isinstance(result, PortfolioResult)
    assert len(result.runs) == 3
    assert all(isinstance(r, RunResult) for r in result.runs)
    # Profile-dependent teeth (offline, no egress): each profile resolves its own model.
    assert resolve_model(profile, "proposer")
    assert resolve_model("local", "proposer") != resolve_model("azure", "proposer")


async def test_e_new_project_flows_through_via_config_only(
    make_portfolio_client_factory, fresh_store
) -> None:
    """SC1: a brand-new project (SKOLE-VVS-OPPGR) added by CONFIG ONLY — a JSON entry + a bundled
    docs folder, zero ``src/*.py`` change — flows end-to-end through run_portfolio. Its id is not
    in REPLIES, so the default reply (which omits project_id) inherits the new project's id via
    ``_parse_ir`` setdefault, proving the loader wired the config-only project through."""
    result = await run_portfolio(
        ["SKOLE-VVS-OPPGR"],
        "local",
        store=fresh_store,
        client_factory=make_portfolio_client_factory(REPLIES),
    )
    assert len(result.runs) == 1
    assert result.runs[0].outcome.proposal.project_id == "SKOLE-VVS-OPPGR"


def test_f_no_hardcoded_project_ids_in_src() -> None:
    """SC1 load-bearing guard: reference-project ids live ONLY in the data JSON, never in code.
    If a future project is onboarded via a code-side id->path/verdict mapping, this reddens —
    backing the 'config-only' claim. Pattern: test_budget.py:88-99 (src anti-pattern grep)."""
    from pathlib import Path

    ids = ("FV42-GSV-E1", "RV13-RAS-TP", "BRU-LAKS-REHAB", "SKOLE-VVS-OPPGR")
    offenders = [
        f"{py.name}: {pid}"
        for py in Path("src/portfolio_optimiser").rglob("*.py")
        for pid in ids
        if pid in py.read_text(encoding="utf-8")
    ]
    assert offenders == [], f"hardcoded project ids in src: {offenders}"


def test_g_extension_doc_exists_and_references_seams() -> None:
    """SC6: the extension guide exists and names the concrete config seams a deployer extends —
    the per-project config (`reference_projects.json` with `docs_dir` + `verdict_input`) and the
    `model_map.json` role->deployment map. Pattern: test_budget.py:88-99 (file-content guard)."""
    from pathlib import Path

    doc = Path("docs/extending.md")
    assert doc.exists(), "docs/extending.md is missing"
    text = doc.read_text(encoding="utf-8")
    for seam in ("reference_projects.json", "docs_dir", "verdict_input", "model_map.json"):
        assert seam in text, f"extension doc does not reference seam: {seam}"