"""Fase 3 — sequential portfolio fan-out over N independent projects. Covers the success criteria: SC2 (fan-out + aggregate sums), SC4 (shared store accumulates + load-bearing cross-project retrieval), SC3 (load-bearing meter-isolation detach), SC7 (both profiles offline + resolve_model teeth), SC1 (new project via config only + no-hardcoded-id src guard), SC6 (extension doc exists + references the seams). Pattern: tests/test_vertical_slice_e2e.py:28 (run_project call shape). The project-aware synthetic client (conftest ``make_portfolio_client_factory``) selects each project's reply by scanning the prompt for its id, so one production-shaped factory serves the whole portfolio. """ import pytest from portfolio_optimiser.run import PortfolioResult, RunResult, run_portfolio, run_project from portfolio_optimiser.validator import Rejection # The synthetic reply IS the proposal: generate._parse_ir builds affected_items (each with its # own quantity/unit_cost) straight from this JSON, and the validator's P90 = 0.30 x Σ(qty·unit_cost) # ONLY when ``assumptions`` is empty (degenerate Monte Carlo, validator.py:108-113). All three # replies therefore OMIT ``assumptions`` and carry explicit magnitudes so each # ``claimed_saving_nok`` <= P90. Verified against validator + ir: # FV42-GSV-E1 Σ=1,482,500 P90=444,750 claimed 200,000 -> validates # RV13-RAS-TP Σ= 756,000 P90=226,800 claimed 130,000 -> validates (decoy) # BRU-LAKS-REHAB Σ=2,580,500 P90=774,150 claimed 210,000 -> validates # ``measure`` is byte-identical "Reduce scope" for FV42+BRU (measure-match is exact string # equality, verdicts.py:68) and "Material substitution" for the decoy, so the BRU<->FV42 pair # overlaps (shared code 05.2 + measure + magnitude bucket) while the decoy does not. REPLIES = { "FV42-GSV-E1": ( '{"measure":"Reduce scope","affected_items":[' '{"code":"05.2","quantity":4300,"unit_cost":215},' '{"code":"03.1","quantity":1800,"unit_cost":310}],"claimed_saving_nok":200000}' ), "RV13-RAS-TP": ( '{"measure":"Material substitution","affected_items":[' '{"code":"88.2","quantity":180,"unit_cost":4200}],"claimed_saving_nok":130000}' ), "BRU-LAKS-REHAB": ( '{"measure":"Reduce scope","affected_items":[' '{"code":"05.2","quantity":4300,"unit_cost":215},' '{"code":"07.4","quantity":2400,"unit_cost":690}],"claimed_saving_nok":210000}' ), } _PORTFOLIO_IDS = ["FV42-GSV-E1", "RV13-RAS-TP", "BRU-LAKS-REHAB"] async def test_a_fanout_returns_one_runresult_per_project( make_portfolio_client_factory, fresh_store ) -> None: """SC2: run_portfolio fans out sequentially, one RunResult per project, and aggregates the validated/rejected counts + claimed-saving + token sums.""" result = await run_portfolio( _PORTFOLIO_IDS, "local", store=fresh_store, client_factory=make_portfolio_client_factory(REPLIES), ) assert isinstance(result, PortfolioResult) assert len(result.runs) == 3 assert all(isinstance(r, RunResult) for r in result.runs) # Aggregate pinned to the fixture constants above (all three validate). assert result.validated_count == 3 assert result.rejected_count == 0 assert result.sum_claimed_saving_nok == 540000 # Explicit element-wise wiring check (not the field's own sum() definition). expected_tokens = ( result.runs[0].provenance.token_usage + result.runs[1].provenance.token_usage + result.runs[2].provenance.token_usage ) assert result.sum_token_usage == expected_tokens assert all(r.provenance.token_usage > 0 for r in result.runs) async def test_a2_unknown_project_id_raises(make_portfolio_client_factory, fresh_store) -> None: """The unknown-id error path: an id absent from the loaded portfolio raises ValueError.""" with pytest.raises(ValueError): await run_portfolio( ["NOPE-DOES-NOT-EXIST"], "local", store=fresh_store, client_factory=make_portfolio_client_factory(REPLIES), ) async def test_a3_rejected_proposal_excluded_from_aggregate( make_portfolio_client_factory, fresh_store ) -> None: """F2 (rejection arm, previously unexercised): one project's claim exceeds its P90, so the validator REJECTS it (FV42 800000 > P90 444750, <= Σ 1,482,500), while RV13+BRU validate. rejected_count counts it and the validated-only sum (run.py:241) EXCLUDES its claim. Also the F1 portfolio seam: provenance.model mirrors the injected client across all N records.""" replies = {**REPLIES, "FV42-GSV-E1": REPLIES["FV42-GSV-E1"].replace("200000", "800000")} result = await run_portfolio( _PORTFOLIO_IDS, "local", store=fresh_store, client_factory=make_portfolio_client_factory(replies), ) assert result.rejected_count == 1 assert result.validated_count == 2 # Validated-only sum EXCLUDES the rejected 800000 (run.py:241): 130000 + 210000. assert result.sum_claimed_saving_nok == 340000 rejected = [r for r in result.runs if isinstance(r.outcome, Rejection)] assert len(rejected) == 1 assert rejected[0].outcome.proposal.claimed_saving_nok == 800000 # carried, not summed assert rejected[0].provenance.validator_decision == "rejected" # F1 portfolio seam: the injected client's model is stamped across ALL N records. assert all(r.provenance.model == "synthetic" for r in result.runs) async def test_b_shared_store_accumulates_and_surfaces_prior_verdict( make_portfolio_client_factory, fresh_store ) -> None: """SC4 (load-bearing, not mere non-emptiness): the ONE shared store accumulates a distinct verdict per project (3 -> 3 pairwise-distinct ids), and the cross-project ExpeL retrieval surfaces the STRUCTURAL match — BRU (runs[2]) retrieves FV42's verdict (runs[0]), since sim(BRU,FV42)=0.60 (shared code 05.2 + identical "Reduce scope" measure + same magnitude bucket) ranks above the RV13 decoy at sim(BRU,decoy)=0.15. If retrieval ranking breaks, or two proposals collide to a single minted id, this fails — it asserts the specific match, not store-non-empty.""" result = await run_portfolio( _PORTFOLIO_IDS, "local", store=fresh_store, client_factory=make_portfolio_client_factory(REPLIES), ) assert len(result.store.verdicts) == 3 ids = [v.id for v in result.store.verdicts] assert len(set(ids)) == 3 # pairwise distinct minted ids (no collision) # BRU surfaces FV42 (the structural match), ranked above the decoy. assert result.runs[2].retrieved[0].id == result.runs[0].verdict.id async def test_c_execution_state_isolation_is_load_bearing( make_client_factory, fresh_store ) -> None: """SC3 (cap-independent automatic detach guard): each project's execution state (the budget meter) is built FRESH per run, so per-project token_usage is its OWN only. Built on the Step-1 ``meter=`` seam + Step-3 ``meter_factory``. The factory emits a UNIFORM but VALID proposal (REPLIES["FV42-GSV-E1"] for every call) so both projects complete — a bare unparseable default would loop the generate fetch to BudgetExceeded instead. Both arms run every CI, so the detach is encoded automatically (no manual reviewer-revert).""" from portfolio_optimiser.budget import Budget, TokenMeter from portfolio_optimiser.reference_domain import load_reference_projects f = make_client_factory(REPLIES["FV42-GSV-E1"]) rv13 = {p.id: p for p in load_reference_projects()}["RV13-RAS-TP"] # 1. Baseline: RV13 run standalone (its own fresh meter). standalone = await run_project( "RV13-RAS-TP", "local", docs_dir=rv13.docs_dir, verdict_input=rv13.verdict_input, client_factory=f, ) # 2. Isolated (default, no meter_factory): RV13 as project 1 in the portfolio. Its usage # equals the standalone baseline — independent of project 0. THE load-bearing assertion: # if run_portfolio shared a meter by default, runs[1] would be cumulative and this breaks. iso = await run_portfolio( ["FV42-GSV-E1", "RV13-RAS-TP"], "local", store=fresh_store, client_factory=f, ) assert iso.runs[1].provenance.token_usage == standalone.provenance.token_usage # 3. Shared (injected one meter across both): usage accumulates, proving the detach is real # (and that the == arm above would redden under sharing). shared = TokenMeter(Budget(max_tokens=1_000_000, max_rounds=1000)) sh = await run_portfolio( ["FV42-GSV-E1", "RV13-RAS-TP"], "local", client_factory=f, meter_factory=lambda: shared, ) assert sh.runs[1].provenance.token_usage > sh.runs[0].provenance.token_usage @pytest.mark.parametrize("profile", ["local", "azure"]) async def test_d_both_profiles_run_offline( make_portfolio_client_factory, fresh_store, profile ) -> None: """SC7: both profiles drive the portfolio contract path OFFLINE under the synthetic client (the path is actually executed, not just the backend instantiated). Teeth: ``resolve_model`` is a profile-dependent seam reachable WITHOUT egress — each profile resolves its own configured model from the bundled model_map.json, and local != azure (deleting the azure entry reddens this). Honest limitation: under an injected ``client_factory``, ``profile`` is otherwise inert inside ``run_project`` (run.py:197 stamps the injected client's own model, not the profile's ``resolve_model``); the end-to-end run proves the path executes under both, and ``resolve_model`` is the only profile-dependent seam provable offline.""" from portfolio_optimiser.backends import resolve_model result = await run_portfolio( _PORTFOLIO_IDS, profile, store=fresh_store, client_factory=make_portfolio_client_factory(REPLIES), ) assert isinstance(result, PortfolioResult) assert len(result.runs) == 3 assert all(isinstance(r, RunResult) for r in result.runs) # Profile-dependent teeth (offline, no egress): each profile resolves its own model. assert resolve_model(profile, "proposer") assert resolve_model("local", "proposer") != resolve_model("azure", "proposer") async def test_e_new_project_flows_through_via_config_only( make_portfolio_client_factory, fresh_store ) -> None: """SC1: a brand-new project (SKOLE-VVS-OPPGR) added by CONFIG ONLY — a JSON entry + a bundled docs folder, zero ``src/*.py`` change — flows end-to-end through run_portfolio. Its id is not in REPLIES, so the default reply (which omits project_id) inherits the new project's id via ``_parse_ir`` setdefault, proving the loader wired the config-only project through.""" result = await run_portfolio( ["SKOLE-VVS-OPPGR"], "local", store=fresh_store, client_factory=make_portfolio_client_factory(REPLIES), ) assert len(result.runs) == 1 assert result.runs[0].outcome.proposal.project_id == "SKOLE-VVS-OPPGR" def test_f_no_hardcoded_project_ids_in_src() -> None: """SC1 load-bearing guard: reference-project ids live ONLY in the data JSON, never in code. If a future project is onboarded via a code-side id->path/verdict mapping, this reddens — backing the 'config-only' claim. Pattern: test_budget.py:88-99 (src anti-pattern grep).""" from pathlib import Path ids = ("FV42-GSV-E1", "RV13-RAS-TP", "BRU-LAKS-REHAB", "SKOLE-VVS-OPPGR") offenders = [ f"{py.name}: {pid}" for py in Path("src/portfolio_optimiser").rglob("*.py") for pid in ids if pid in py.read_text(encoding="utf-8") ] assert offenders == [], f"hardcoded project ids in src: {offenders}" def test_g_extension_doc_exists_and_references_seams() -> None: """SC6: the extension guide exists and names the concrete config seams a deployer extends — the per-project config (`reference_projects.json` with `docs_dir` + `verdict_input`) and the `model_map.json` role->deployment map. Pattern: test_budget.py:88-99 (file-content guard).""" from pathlib import Path doc = Path("docs/extending.md") assert doc.exists(), "docs/extending.md is missing" text = doc.read_text(encoding="utf-8") for seam in ("reference_projects.json", "docs_dir", "verdict_input", "model_map.json"): assert seam in text, f"extension doc does not reference seam: {seam}"