"""S4.2 live-run drill (comparison protocol §4 pkt 2/3): the ``--live-dry-run`` cut in ``run_project`` walks the real run path — contracts, budget, eager client build — and STOPS before the first model call (``await debate.run(...)``), returning a ``DryRunReport`` with ZERO chat calls. Load-bearing pair: - T-4.2b (stop-point): a dry-run against a call-recording factory records an EMPTY sink (0 chat calls) and returns a ``DryRunReport``; detach the early ``return`` → ``debate.run`` fires → sink non-empty → RED. - T-4.2b-control (causality): the SAME factory with ``live_dry_run=False`` drives the full run and records a NON-empty sink — proving the 0 is caused by the cut, not an undriven fixture. The artefact-set completeness guard (T-4.2c, §4 pkt 2/3) lives below (Step 4). """ from __future__ import annotations import json from collections.abc import Callable from pathlib import Path from agent_framework import BaseChatClient from conftest import SyntheticUsageChatClient from portfolio_optimiser.run import DryRunReport, run_project from portfolio_optimiser.validator import ValidatedProposal BUNDLE_DIR = Path(__file__).resolve().parents[1] / "shared" / "examples" / "bygg-energi-mikro" # A VALIDATOR-VALID BYGG-KONTOR-NORD proposal so a full control run completes cleanly. _VALID_PROPOSER_REPLY = ( '{"measure":"LED-retrofit av kontorbelysning","affected_items":' '[{"code":"ENERGI-TOTAL-EL","quantity":300000,"unit_cost":1.0}],"claimed_saving_nok":30000}' ) _VERDICT_INPUT = {"decision": "approved", "rationale": "expert reviewed (sim)"} def _role_factory(proposer_reply: str, checker_reply: str) -> Callable[[str], BaseChatClient]: """Role-aware scripted factory: the checker speaks its verdict, the proposer its proposal.""" def factory(role: str) -> BaseChatClient: return SyntheticUsageChatClient( default_reply=checker_reply if role == "checker" else proposer_reply ) return factory async def test_dry_run_stops_before_first_model_call( tmp_path, make_recording_client_factory ) -> None: """T-4.2b (stop-point, load-bearing): ``live_dry_run=True`` builds contracts + clients + budget but makes ZERO chat calls — the recording sink stays empty and a ``DryRunReport`` is returned, while the run-config artefact ``r1-runconfig.json`` is written. Detach the early ``return`` (let ``debate.run`` execute) → the sink fills → RED.""" factory, sink = make_recording_client_factory(_VALID_PROPOSER_REPLY) result = await run_project( "BYGG-KONTOR-NORD", "local", docs_dir=str(BUNDLE_DIR), bundle_dir=str(BUNDLE_DIR), verdict_input=_VERDICT_INPUT, client_factory=factory, outbox_dir=str(tmp_path), run_id="r1", live_dry_run=True, ) assert isinstance(result, DryRunReport) assert sink == [] # zero chat calls — the cut held assert (tmp_path / "r1-runconfig.json").is_file() # run-config captured (§4 pkt 3) async def test_dry_run_control_full_run_makes_calls( tmp_path, make_recording_client_factory ) -> None: """T-4.2b-control (causality): the SAME factory with ``live_dry_run=False`` drives the full run and records a NON-empty sink — proving the empty sink above is caused by the dry-run cut, not by a fixture that is never driven.""" factory, sink = make_recording_client_factory(_VALID_PROPOSER_REPLY) result = await run_project( "BYGG-KONTOR-NORD", "local", docs_dir=str(BUNDLE_DIR), bundle_dir=str(BUNDLE_DIR), verdict_input=_VERDICT_INPUT, client_factory=factory, ) assert not isinstance(result, DryRunReport) # a full run returns a RunResult assert len(sink) > 0 # the debate + generation actually called the model async def test_dry_run_artefact_set_complete(tmp_path) -> None: """T-4.2c (artefact-set completeness, load-bearing · §4 pkt 2/3): a scripted FULL run with an ``outbox_dir`` + ``run_id`` writes the COMPLETE set — proposal (IR + provenance incl. token usage), outcome (percentiles + checker verdict + verdict id), AND run-config (profile + built roles + params). Drop any file or field → RED.""" factory = _role_factory(_VALID_PROPOSER_REPLY, "VERDICT: APPROVE") result = await run_project( "BYGG-KONTOR-NORD", "local", docs_dir=str(BUNDLE_DIR), bundle_dir=str(BUNDLE_DIR), verdict_input=_VERDICT_INPUT, client_factory=factory, outbox_dir=str(tmp_path), run_id="r2", ) assert isinstance(result.outcome, ValidatedProposal) proposal = json.loads((tmp_path / "r2-proposal.json").read_text(encoding="utf-8")) assert proposal["run_id"] == "r2" assert proposal["proposal"]["measure"] # the candidate IR is present assert "ENERGI-TOTAL-EL" in {a["code"] for a in proposal["proposal"]["affected_items"]} assert proposal["provenance"]["token_usage"] > 0 # §4 pkt 2: tokens captured outcome = json.loads((tmp_path / "r2-outcome.json").read_text(encoding="utf-8")) assert outcome["outcome_type"] == "validated" assert all(k in outcome for k in ("p10", "p50", "p90", "nominal_feasible")) assert outcome["checker_verdict"] == "approve" assert outcome["verdict_id"] == result.verdict.id runconfig = json.loads((tmp_path / "r2-runconfig.json").read_text(encoding="utf-8")) assert runconfig["profile"] == "local" assert set(runconfig["resolved_models"]) == {"proposer", "checker"} # built roles only assert runconfig["max_rounds"] == 3 assert runconfig["max_tokens"] == 100_000 assert runconfig["top_k"] == 3