feat(s42): live_dry_run cut in run_project + DryRunReport

src/portfolio_optimiser/run.py

@@ -28,7 +28,7 @@ from __future__ import annotations
 from collections.abc import Callable, Sequence
 from dataclasses import dataclass, replace
 from decimal import ROUND_HALF_UP, Decimal
-from typing import Any, Literal
+from typing import Any, Literal, cast
 
 from agent_framework import BaseChatClient, SessionContext
 
@@ -58,7 +58,7 @@ from portfolio_optimiser.verdicts import (
     capture_verdict,
     load_verdicts_from_dir,
 )
-from portfolio_optimiser.workflow import fresh_workflow
+from portfolio_optimiser.workflow import _MAKER_CHECKER_ROLES, fresh_workflow
 
 
 @dataclass(frozen=True)
@@ -79,6 +79,20 @@ class RunResult:
     checker_verdict: str = "absent"
 
 
+@dataclass(frozen=True)
+class DryRunReport:
+    """S4.2 offline ``--live-dry-run`` outcome (comparison protocol §4 pkt 3): everything a real run
+    would use — profile, the resolved model-id per BUILT role, and the round/token parameters —
+    captured WITHOUT a model call. A DISTINCT type from ``RunResult``, whose post-generation fields
+    (outcome/provenance/verdict) do not exist yet on a run that stopped before the first model call."""
+
+    profile: str
+    resolved_models: dict[str, str]
+    max_rounds: int
+    max_tokens: int
+    top_k: int
+
+
 @dataclass(frozen=True)
 class GoalReached:
     """A savings-goal signal VALUE (Step 8, SC6) — NOT an exception. Structured like
@@ -230,7 +244,8 @@ async def run_project(
     enable_layer1_hitl: bool = False,
     notify: Callable[[Verdict], None] | None = None,
     meter: TokenMeter | None = None,
-) -> RunResult:
+    live_dry_run: bool = False,
+) -> RunResult | DryRunReport:
     """Run the vertical slice for ONE project. ``client_factory`` is the test-injection seam
     (defaults to the real backend). ``verdict_input`` carries the expert decision/rationale
     (Layer-2). ``bundle_dir`` (Fase 2a) makes the run OKF-bundle-driven: the project is derived
@@ -247,7 +262,10 @@ async def run_project(
     outbox into a folder later read as an inbox would re-ingest raw agent output and bypass the
     Step-8 promotion gate (self-contamination) — documented here, not enforced. Raises
     ``pydantic.ValidationError`` on a bad contract and ``BudgetExceeded`` when the token/round cap is
-    crossed, and ``ValueError`` when ``outbox_dir`` is set without a ``run_id``."""
+    crossed, and ``ValueError`` when ``outbox_dir`` is set without a ``run_id``. ``live_dry_run``
+    (S4.2, comparison protocol §4 pkt 2/3) is the offline drill: it walks the whole path up to the
+    EAGER client build, writes the run-config artefact (when ``outbox_dir`` is set), and returns a
+    ``DryRunReport`` BEFORE the first model call (``debate.run``) — zero chat calls."""
     # 0. Fail-fast: an outbox write is byte-deterministic and keyed on run_id — no wall-clock default.
     if outbox_dir is not None and run_id is None:
         raise ValueError(
@@ -310,6 +328,34 @@ async def run_project(
         tools=debate_tools,
         middleware=[budget_mw],
     )
+    # S4.2 cut (comparison protocol §4 pkt 2/3): everything above is offline — contracts, budget, and
+    # the EAGER client build (fresh_workflow constructs the proposer+checker clients, workflow.py:64).
+    # Capture the run-config (resolved model per BUILT role, profile, params, token cap) and, for a
+    # ``--live-dry-run``, STOP HERE — before the first (paid) model call at ``debate.run`` below.
+    if outbox_dir is not None or live_dry_run:
+        # ``resolved_models`` reflects the configured MAP (the default factory's model-ids for the M2
+        # run). Under an injected ``client_factory`` the built clients may differ (e.g. "synthetic");
+        # ``provenance.model`` (below) stays the authority on the client actually built.
+        resolved_models = {role: resolve_model(profile, role) for role in _MAKER_CHECKER_ROLES}
+        if outbox_dir is not None:
+            assert run_id is not None  # narrowed by the step-0 guard (no wall-clock default)
+            outbox.write_run_config(
+                outbox_dir,
+                run_id,
+                profile=Profile(profile).value,
+                resolved_models=resolved_models,
+                max_rounds=max_rounds,
+                max_tokens=max_tokens,
+                top_k=top_k,
+            )
+        if live_dry_run:
+            return DryRunReport(
+                profile=Profile(profile).value,
+                resolved_models=resolved_models,
+                max_rounds=max_rounds,
+                max_tokens=max_tokens,
+                top_k=top_k,
+            )
     result = await debate.run(f"Find a cost-saving measure for {project.id}.\nContext:\n{context}")
     # F1: the candidate must derive from the DEBATE. Feed the proposer's converged output into
     # generation (retrieval context is the last-resort fallback only). The checker's verdict
@@ -532,20 +578,25 @@ async def run_portfolio(
                 if per_project_goal.mode == "hard":
                     continue  # skip THIS pid; the rest of the pass proceeds
 
-        result = await run_project(
+        # run_portfolio only drives full runs (never dry-run), so the return narrows to RunResult;
-            pid,
+        # the cast keeps the widened run_project signature honest without an @overload duplication.
-            profile,
+        result = cast(
-            docs_dir=project.docs_dir,
+            RunResult,
-            verdict_input=project.verdict_input,
+            await run_project(
-            bundle_dir=project.bundle_dir,
+                pid,
-            verdict_dir=project.verdict_dir,
+                profile,
-            dimension=dimension,
+                docs_dir=project.docs_dir,
-            store=store,
+                verdict_input=project.verdict_input,
-            client_factory=client_factory,
+                bundle_dir=project.bundle_dir,
-            max_rounds=max_rounds,
+                verdict_dir=project.verdict_dir,
-            max_tokens=max_tokens,
+                dimension=dimension,
-            top_k=top_k,
+                store=store,
-            meter=meter_factory() if meter_factory is not None else None,
+                client_factory=client_factory,
+                max_rounds=max_rounds,
+                max_tokens=max_tokens,
+                top_k=top_k,
+                meter=meter_factory() if meter_factory is not None else None,
+            ),
         )
         runs.append(result)
 
@@ -577,15 +628,20 @@ def main(argv: list[str] | None = None) -> int:
     parser.add_argument("--rationale", default="reviewed by expert")
     args = parser.parse_args(argv)
 
-    result = asyncio.run(
+    # S4.2: main() currently drives only full runs (no --live-dry-run flag yet — that is Step 3,
-        run_project(
+    # which replaces this cast with an isinstance(DryRunReport) branch + AZURE-refusal try/except).
-            args.project_id,
+    result = cast(
-            args.profile,
+        RunResult,
-            docs_dir=args.docs_dir,
+        asyncio.run(
-            bundle_dir=args.bundle_dir,
+            run_project(
-            verdict_dir=args.verdict_dir,
+                args.project_id,
-            verdict_input={"decision": args.decision, "rationale": args.rationale},
+                args.profile,
-        )
+                docs_dir=args.docs_dir,
+                bundle_dir=args.bundle_dir,
+                verdict_dir=args.verdict_dir,
+                verdict_input={"decision": args.decision, "rationale": args.rationale},
+            )
+        ),
     )
     kind = type(result.outcome).__name__
     print(f"{args.project_id}: {kind} (verdict id={result.verdict.id}, decision={args.decision})")

src/portfolio_optimiser/simulation.py

@@ -24,7 +24,7 @@ import shutil
 from collections.abc import Awaitable, Callable, Mapping, Sequence
 from dataclasses import dataclass
 from pathlib import Path
-from typing import Any
+from typing import Any, cast
 
 from agent_framework import (
     BaseChatClient,
@@ -217,15 +217,18 @@ async def simulate_learning_loop(
 
     # Run A — empty wiki isolates the persona's NEW knowledge.
     sink_a: list[str] = []
-    run_a = await run_project(
+    run_a = cast(
-        _PROJECT_ID,
+        RunResult,  # the sim only drives full runs; never dry-run (S4.2 widened run_project)
-        "local",
+        await run_project(
-        docs_dir=copy_s,
+            _PROJECT_ID,
-        bundle_dir=copy_s,
+            "local",
-        verdict_input=verdict_input,
+            docs_dir=copy_s,
-        store=VerdictStore(verdicts=[]),
+            bundle_dir=copy_s,
-        client_factory=scripted_factory(replies, sink_a),
+            verdict_input=verdict_input,
-        max_rounds=max_rounds,
+            store=VerdictStore(verdicts=[]),
+            client_factory=scripted_factory(replies, sink_a),
+            max_rounds=max_rounds,
+        ),
     )
 
     # Gate-promote the persona verdict from the raw output layer into the OKF wiki (Steg 8).
@@ -242,15 +245,18 @@ async def simulate_learning_loop(
 
     # Run B — a separate, later run reads the updated wiki.
     sink_b: list[str] = []
-    run_b = await run_project(
+    run_b = cast(
-        _PROJECT_ID,
+        RunResult,  # the sim only drives full runs; never dry-run (S4.2 widened run_project)
-        "local",
+        await run_project(
-        docs_dir=copy_s,
+            _PROJECT_ID,
-        bundle_dir=copy_s,
+            "local",
-        verdict_input=verdict_input,
+            docs_dir=copy_s,
-        store=store_b,
+            bundle_dir=copy_s,
-        client_factory=scripted_factory(replies, sink_b),
+            verdict_input=verdict_input,
-        max_rounds=max_rounds,
+            store=store_b,
+            client_factory=scripted_factory(replies, sink_b),
+            max_rounds=max_rounds,
+        ),
     )
 
     gen_a = _generation_prompts(sink_a)

tests/test_live_dry_run_loadbearing.py

@@ -0,0 +1,126 @@
+"""S4.2 live-run drill (comparison protocol §4 pkt 2/3): the ``--live-dry-run`` cut in
+``run_project`` walks the real run path — contracts, budget, eager client build — and STOPS before
+the first model call (``await debate.run(...)``), returning a ``DryRunReport`` with ZERO chat calls.
+
+Load-bearing pair:
+- T-4.2b (stop-point): a dry-run against a call-recording factory records an EMPTY sink (0 chat
+  calls) and returns a ``DryRunReport``; detach the early ``return`` → ``debate.run`` fires → sink
+  non-empty → RED.
+- T-4.2b-control (causality): the SAME factory with ``live_dry_run=False`` drives the full run and
+  records a NON-empty sink — proving the 0 is caused by the cut, not an undriven fixture.
+
+The artefact-set completeness guard (T-4.2c, §4 pkt 2/3) lives below (Step 4).
+"""
+
+from __future__ import annotations
+
+import json
+from collections.abc import Callable
+from pathlib import Path
+
+from agent_framework import BaseChatClient
+from conftest import SyntheticUsageChatClient
+
+from portfolio_optimiser.run import DryRunReport, run_project
+from portfolio_optimiser.validator import ValidatedProposal
+
+BUNDLE_DIR = Path(__file__).resolve().parents[1] / "shared" / "examples" / "bygg-energi-mikro"
+
+# A VALIDATOR-VALID BYGG-KONTOR-NORD proposal so a full control run completes cleanly.
+_VALID_PROPOSER_REPLY = (
+    '{"measure":"LED-retrofit av kontorbelysning","affected_items":'
+    '[{"code":"ENERGI-TOTAL-EL","quantity":300000,"unit_cost":1.0}],"claimed_saving_nok":30000}'
+)
+_VERDICT_INPUT = {"decision": "approved", "rationale": "expert reviewed (sim)"}
+
+
+def _role_factory(proposer_reply: str, checker_reply: str) -> Callable[[str], BaseChatClient]:
+    """Role-aware scripted factory: the checker speaks its verdict, the proposer its proposal."""
+
+    def factory(role: str) -> BaseChatClient:
+        return SyntheticUsageChatClient(
+            default_reply=checker_reply if role == "checker" else proposer_reply
+        )
+
+    return factory
+
+
+async def test_dry_run_stops_before_first_model_call(
+    tmp_path, make_recording_client_factory
+) -> None:
+    """T-4.2b (stop-point, load-bearing): ``live_dry_run=True`` builds contracts + clients + budget
+    but makes ZERO chat calls — the recording sink stays empty and a ``DryRunReport`` is returned,
+    while the run-config artefact ``r1-runconfig.json`` is written. Detach the early ``return`` (let
+    ``debate.run`` execute) → the sink fills → RED."""
+    factory, sink = make_recording_client_factory(_VALID_PROPOSER_REPLY)
+    result = await run_project(
+        "BYGG-KONTOR-NORD",
+        "local",
+        docs_dir=str(BUNDLE_DIR),
+        bundle_dir=str(BUNDLE_DIR),
+        verdict_input=_VERDICT_INPUT,
+        client_factory=factory,
+        outbox_dir=str(tmp_path),
+        run_id="r1",
+        live_dry_run=True,
+    )
+    assert isinstance(result, DryRunReport)
+    assert sink == []  # zero chat calls — the cut held
+    assert (tmp_path / "r1-runconfig.json").is_file()  # run-config captured (§4 pkt 3)
+
+
+async def test_dry_run_control_full_run_makes_calls(
+    tmp_path, make_recording_client_factory
+) -> None:
+    """T-4.2b-control (causality): the SAME factory with ``live_dry_run=False`` drives the full run
+    and records a NON-empty sink — proving the empty sink above is caused by the dry-run cut, not by
+    a fixture that is never driven."""
+    factory, sink = make_recording_client_factory(_VALID_PROPOSER_REPLY)
+    result = await run_project(
+        "BYGG-KONTOR-NORD",
+        "local",
+        docs_dir=str(BUNDLE_DIR),
+        bundle_dir=str(BUNDLE_DIR),
+        verdict_input=_VERDICT_INPUT,
+        client_factory=factory,
+    )
+    assert not isinstance(result, DryRunReport)  # a full run returns a RunResult
+    assert len(sink) > 0  # the debate + generation actually called the model
+
+
+async def test_dry_run_artefact_set_complete(tmp_path) -> None:
+    """T-4.2c (artefact-set completeness, load-bearing · §4 pkt 2/3): a scripted FULL run with an
+    ``outbox_dir`` + ``run_id`` writes the COMPLETE set — proposal (IR + provenance incl. token
+    usage), outcome (percentiles + checker verdict + verdict id), AND run-config (profile + built
+    roles + params). Drop any file or field → RED."""
+    factory = _role_factory(_VALID_PROPOSER_REPLY, "VERDICT: APPROVE")
+    result = await run_project(
+        "BYGG-KONTOR-NORD",
+        "local",
+        docs_dir=str(BUNDLE_DIR),
+        bundle_dir=str(BUNDLE_DIR),
+        verdict_input=_VERDICT_INPUT,
+        client_factory=factory,
+        outbox_dir=str(tmp_path),
+        run_id="r2",
+    )
+    assert isinstance(result.outcome, ValidatedProposal)
+
+    proposal = json.loads((tmp_path / "r2-proposal.json").read_text(encoding="utf-8"))
+    assert proposal["run_id"] == "r2"
+    assert proposal["proposal"]["measure"]  # the candidate IR is present
+    assert "ENERGI-TOTAL-EL" in {a["code"] for a in proposal["proposal"]["affected_items"]}
+    assert proposal["provenance"]["token_usage"] > 0  # §4 pkt 2: tokens captured
+
+    outcome = json.loads((tmp_path / "r2-outcome.json").read_text(encoding="utf-8"))
+    assert outcome["outcome_type"] == "validated"
+    assert all(k in outcome for k in ("p10", "p50", "p90", "nominal_feasible"))
+    assert outcome["checker_verdict"] == "approve"
+    assert outcome["verdict_id"] == result.verdict.id
+
+    runconfig = json.loads((tmp_path / "r2-runconfig.json").read_text(encoding="utf-8"))
+    assert runconfig["profile"] == "local"
+    assert set(runconfig["resolved_models"]) == {"proposer", "checker"}  # built roles only
+    assert runconfig["max_rounds"] == 3
+    assert runconfig["max_tokens"] == 100_000
+    assert runconfig["top_k"] == 3