portfolio-optimiser/tests/test_portfolio.py

"""Fase 3 — sequential portfolio fan-out over N independent projects.

Covers the success criteria: SC2 (fan-out + aggregate sums), SC4 (shared store accumulates +
load-bearing cross-project retrieval), SC3 (load-bearing meter-isolation detach), SC7 (both
profiles offline + resolve_model teeth), SC1 (new project via config only + no-hardcoded-id src
guard), SC6 (extension doc exists + references the seams).

Pattern: tests/test_vertical_slice_e2e.py:28 (run_project call shape). The project-aware
synthetic client (conftest ``make_portfolio_client_factory``) selects each project's reply by
scanning the prompt for its id, so one production-shaped factory serves the whole portfolio.
"""

import json

import pytest

from portfolio_optimiser.run import PortfolioResult, RunResult, run_portfolio, run_project
from portfolio_optimiser.validator import Rejection

# S4.1: the azure block in the bundled model_map now ships REPLACE-WITH-* placeholders that
# resolve_model refuses. This test resolves azure directly (line ~224), so it points
# PORTFOLIO_MODEL_MAP at a complete, non-placeholder map (BOTH blocks — resolve_model does no
# structural validation) for BOTH parametrizations.
_OFFLINE_MODEL_MAP = {
    "local": {"default": "qwen3:4b", "proposer": "qwen3:4b", "checker": "qwen3:4b"},
    "azure": {"default": "gpt-4o-mini", "proposer": "gpt-4o-mini", "checker": "gpt-4o-mini"},
}

# The synthetic reply IS the proposal: generate._parse_ir builds affected_items (each with its
# own quantity/unit_cost) straight from this JSON, and the validator's P90 = 0.30 x Σ(qty·unit_cost)
# ONLY when ``assumptions`` is empty (degenerate Monte Carlo, validator.py). All three replies
# therefore OMIT ``assumptions`` and carry explicit magnitudes so each ``claimed_saving_nok`` <= P90.
#
# S4.0: every line below quotes a cost line the project ACTUALLY has, verbatim from
# reference_projects.json — the reference path now anchors the validator to ``project.cost_items``, so a
# reply quoting another project's code (which these fixtures used to do) is rejected as a fabricated
# cost line. Verified against reference_projects.json + validator + ir:
#   KONTOR-IT-E1     01.1 1x850,000 + 05.2 4300x215     Σ=1,774,500  P90=532,350  claimed 200,000
#   NETT-SIKR-TP     22.4 610x3,850                     Σ=2,348,500  P90=704,550  claimed 130,000 (decoy)
#   ARKIV-LAGR-MIGR  01.1 1x620,000 + 87.3 640x980      Σ=1,247,200  P90=374,160  claimed 210,000
# ``measure`` is byte-identical "Reduce scope" for KONTOR+ARKIV (measure-match is exact string
# equality, verdicts.py:68) and "Material substitution" for the decoy, so the ARKIV<->KONTOR pair still
# overlaps (shared code 01.1 + measure + magnitude bucket) while the decoy does not. 01.1 replaces
# 05.2 as the shared code because it is the only code both projects genuinely carry.
REPLIES = {
    "KONTOR-IT-E1": (
        '{"measure":"Reduce scope","affected_items":['
        '{"code":"01.1","quantity":1,"unit_cost":850000},'
        '{"code":"05.2","quantity":4300,"unit_cost":215}],"claimed_saving_nok":200000}'
    ),
    "NETT-SIKR-TP": (
        '{"measure":"Material substitution","affected_items":['
        '{"code":"22.4","quantity":610,"unit_cost":3850}],"claimed_saving_nok":130000}'
    ),
    "ARKIV-LAGR-MIGR": (
        '{"measure":"Reduce scope","affected_items":['
        '{"code":"01.1","quantity":1,"unit_cost":620000},'
        '{"code":"87.3","quantity":640,"unit_cost":980}],"claimed_saving_nok":210000}'
    ),
}

_PORTFOLIO_IDS = ["KONTOR-IT-E1", "NETT-SIKR-TP", "ARKIV-LAGR-MIGR"]


async def test_a_fanout_returns_one_runresult_per_project(
    make_portfolio_client_factory, fresh_store
) -> None:
    """SC2: run_portfolio fans out sequentially, one RunResult per project, and aggregates the
    validated/rejected counts + claimed-saving + token sums."""
    result = await run_portfolio(
        _PORTFOLIO_IDS,
        "local",
        store=fresh_store,
        client_factory=make_portfolio_client_factory(REPLIES),
    )
    assert isinstance(result, PortfolioResult)
    assert len(result.runs) == 3
    assert all(isinstance(r, RunResult) for r in result.runs)
    # Aggregate pinned to the fixture constants above (all three validate).
    assert result.validated_count == 3
    assert result.rejected_count == 0
    assert result.sum_claimed_saving_nok == 540000
    # Explicit element-wise wiring check (not the field's own sum() definition).
    expected_tokens = (
        result.runs[0].provenance.token_usage
        + result.runs[1].provenance.token_usage
        + result.runs[2].provenance.token_usage
    )
    assert result.sum_token_usage == expected_tokens
    assert all(r.provenance.token_usage > 0 for r in result.runs)


async def test_a1_default_no_goal_leaves_stop_fields_unset(
    make_portfolio_client_factory, fresh_store
) -> None:
    """Step 8 regression: with no ``goals``/``ledger`` the pass is unchanged — ``stopped_early`` is
    False and ``stop_reason`` is None, and every project still runs (the new frozen fields default,
    so the default path is byte-for-byte the prior behavior)."""
    result = await run_portfolio(
        _PORTFOLIO_IDS,
        "local",
        store=fresh_store,
        client_factory=make_portfolio_client_factory(REPLIES),
    )
    assert result.stopped_early is False
    assert result.stop_reason is None
    assert len(result.runs) == 3


async def test_a2_unknown_project_id_raises(make_portfolio_client_factory, fresh_store) -> None:
    """The unknown-id error path: an id absent from the loaded portfolio raises ValueError."""
    with pytest.raises(ValueError):
        await run_portfolio(
            ["NOPE-DOES-NOT-EXIST"],
            "local",
            store=fresh_store,
            client_factory=make_portfolio_client_factory(REPLIES),
        )


async def test_a3_rejected_proposal_excluded_from_aggregate(
    make_portfolio_client_factory, fresh_store
) -> None:
    """F2 (rejection arm, previously unexercised): one project's claim exceeds its P90, so the
    validator REJECTS it (KONTOR 800000 > P90 444750, <= Σ 1,482,500), while NETT+ARKIV validate.
    rejected_count counts it and the validated-only sum (run.py:241) EXCLUDES its claim. Also the
    F1 portfolio seam: provenance.model mirrors the injected client across all N records."""
    replies = {**REPLIES, "KONTOR-IT-E1": REPLIES["KONTOR-IT-E1"].replace("200000", "800000")}
    result = await run_portfolio(
        _PORTFOLIO_IDS,
        "local",
        store=fresh_store,
        client_factory=make_portfolio_client_factory(replies),
    )
    assert result.rejected_count == 1
    assert result.validated_count == 2
    # Validated-only sum EXCLUDES the rejected 800000 (run.py:241): 130000 + 210000.
    assert result.sum_claimed_saving_nok == 340000
    rejected = [r for r in result.runs if isinstance(r.outcome, Rejection)]
    assert len(rejected) == 1
    assert rejected[0].outcome.proposal.claimed_saving_nok == 800000  # carried, not summed
    assert rejected[0].provenance.validator_decision == "rejected"
    # F1 portfolio seam: the injected client's model is stamped across ALL N records.
    assert all(r.provenance.model == "synthetic" for r in result.runs)


async def test_b_shared_store_accumulates_and_surfaces_prior_verdict(
    make_portfolio_client_factory, fresh_store
) -> None:
    """SC4 (load-bearing, not mere non-emptiness): the ONE shared store accumulates a distinct
    verdict per project (3 -> 3 pairwise-distinct ids), and the cross-project ExpeL retrieval
    surfaces the STRUCTURAL match — ARKIV (runs[2]) retrieves KONTOR's verdict (runs[0]), since
    sim(ARKIV,KONTOR)=0.60 (shared code 05.2 + identical "Reduce scope" measure + same magnitude
    bucket) ranks above the NETT decoy at sim(ARKIV,decoy)=0.15. If retrieval ranking breaks, or
    two proposals collide to a single minted id, this fails — it asserts the specific match, not
    store-non-empty."""
    result = await run_portfolio(
        _PORTFOLIO_IDS,
        "local",
        store=fresh_store,
        client_factory=make_portfolio_client_factory(REPLIES),
    )
    assert len(result.store.verdicts) == 3
    ids = [v.id for v in result.store.verdicts]
    assert len(set(ids)) == 3  # pairwise distinct minted ids (no collision)
    # ARKIV surfaces KONTOR (the structural match), ranked above the decoy.
    assert result.runs[2].retrieved[0].id == result.runs[0].verdict.id


async def test_c_execution_state_isolation_is_load_bearing(
    make_client_factory, fresh_store
) -> None:
    """SC3 (cap-independent automatic detach guard): each project's execution state (the budget
    meter) is built FRESH per run, so per-project token_usage is its OWN only. Built on the
    Step-1 ``meter=`` seam + Step-3 ``meter_factory``. The factory emits a UNIFORM but VALID
    proposal (REPLIES["KONTOR-IT-E1"] for every call) so both projects complete — a bare
    unparseable default would loop the generate fetch to BudgetExceeded instead. Both arms run
    every CI, so the detach is encoded automatically (no manual reviewer-revert)."""
    from portfolio_optimiser.budget import Budget, TokenMeter
    from portfolio_optimiser.reference_domain import load_reference_projects

    f = make_client_factory(REPLIES["KONTOR-IT-E1"])
    nett = {p.id: p for p in load_reference_projects()}["NETT-SIKR-TP"]

    # 1. Baseline: NETT run standalone (its own fresh meter).
    standalone = await run_project(
        "NETT-SIKR-TP",
        "local",
        docs_dir=nett.docs_dir,
        verdict_input=nett.verdict_input,
        client_factory=f,
    )
    # 2. Isolated (default, no meter_factory): NETT as project 1 in the portfolio. Its usage
    #    equals the standalone baseline — independent of project 0. THE load-bearing assertion:
    #    if run_portfolio shared a meter by default, runs[1] would be cumulative and this breaks.
    iso = await run_portfolio(
        ["KONTOR-IT-E1", "NETT-SIKR-TP"],
        "local",
        store=fresh_store,
        client_factory=f,
    )
    assert iso.runs[1].provenance.token_usage == standalone.provenance.token_usage
    # 3. Shared (injected one meter across both): usage accumulates, proving the detach is real
    #    (and that the == arm above would redden under sharing).
    shared = TokenMeter(Budget(max_tokens=1_000_000, max_rounds=1000))
    sh = await run_portfolio(
        ["KONTOR-IT-E1", "NETT-SIKR-TP"],
        "local",
        client_factory=f,
        meter_factory=lambda: shared,
    )
    assert sh.runs[1].provenance.token_usage > sh.runs[0].provenance.token_usage


@pytest.mark.parametrize("profile", ["local", "azure"])
async def test_d_both_profiles_run_offline(
    make_portfolio_client_factory, fresh_store, profile, tmp_path, monkeypatch
) -> None:
    """SC7: both profiles drive the portfolio contract path OFFLINE under the synthetic client
    (the path is actually executed, not just the backend instantiated). Teeth: ``resolve_model``
    is a profile-dependent seam reachable WITHOUT egress — each profile resolves its own
    configured model from the bundled model_map.json, and local != azure (deleting the azure
    entry reddens this). Honest limitation: under an injected ``client_factory``, ``profile`` is
    otherwise inert inside ``run_project`` (run.py:197 stamps the injected client's own model, not
    the profile's ``resolve_model``); the end-to-end run proves the path executes under both, and
    ``resolve_model`` is the only profile-dependent seam provable offline."""
    from portfolio_optimiser.backends import resolve_model

    # Unconditionally (both params): the azure branch of line ~224 resolves azure even when
    # profile=="local", so the placeholder-free override must be active in both runs.
    map_path = tmp_path / "model_map.json"
    map_path.write_text(json.dumps(_OFFLINE_MODEL_MAP), encoding="utf-8")
    monkeypatch.setenv("PORTFOLIO_MODEL_MAP", str(map_path))

    result = await run_portfolio(
        _PORTFOLIO_IDS,
        profile,
        store=fresh_store,
        client_factory=make_portfolio_client_factory(REPLIES),
    )
    assert isinstance(result, PortfolioResult)
    assert len(result.runs) == 3
    assert all(isinstance(r, RunResult) for r in result.runs)
    # Profile-dependent teeth (offline, no egress): each profile resolves its own model.
    assert resolve_model(profile, "proposer")
    assert resolve_model("local", "proposer") != resolve_model("azure", "proposer")


async def test_e_new_project_flows_through_via_config_only(
    make_portfolio_client_factory, fresh_store
) -> None:
    """SC1: a brand-new project (SKOLE-VVS-OPPGR) added by CONFIG ONLY — a JSON entry + a bundled
    docs folder, zero ``src/*.py`` change — flows end-to-end through run_portfolio. Its id is not
    in REPLIES, so the default reply (which omits project_id) inherits the new project's id via
    ``_parse_ir`` setdefault, proving the loader wired the config-only project through."""
    result = await run_portfolio(
        ["SKOLE-VVS-OPPGR"],
        "local",
        store=fresh_store,
        client_factory=make_portfolio_client_factory(REPLIES),
    )
    assert len(result.runs) == 1
    assert result.runs[0].outcome.proposal.project_id == "SKOLE-VVS-OPPGR"


def test_f_no_hardcoded_project_ids_in_src() -> None:
    """SC1 load-bearing guard: reference-project ids live ONLY in the data JSON, never in code.
    If a future project is onboarded via a code-side id->path/verdict mapping, this reddens —
    backing the 'config-only' claim. Pattern: test_budget.py:88-99 (src anti-pattern grep)."""
    from pathlib import Path

    ids = ("KONTOR-IT-E1", "NETT-SIKR-TP", "ARKIV-LAGR-MIGR", "SKOLE-VVS-OPPGR")
    offenders = [
        f"{py.name}: {pid}"
        for py in Path("src/portfolio_optimiser").rglob("*.py")
        for pid in ids
        if pid in py.read_text(encoding="utf-8")
    ]
    assert offenders == [], f"hardcoded project ids in src: {offenders}"


def test_g_extension_doc_exists_and_references_seams() -> None:
    """SC6: the extension guide exists and names the concrete config seams a deployer extends —
    the per-project config (`reference_projects.json` with `docs_dir` + `verdict_input`) and the
    `model_map.json` role->deployment map. Pattern: test_budget.py:88-99 (file-content guard)."""
    from pathlib import Path

    doc = Path("docs/extending.md")
    assert doc.exists(), "docs/extending.md is missing"
    text = doc.read_text(encoding="utf-8")
    for seam in ("reference_projects.json", "docs_dir", "verdict_input", "model_map.json"):
        assert seam in text, f"extension doc does not reference seam: {seam}"