126 lines
5.6 KiB
Python
126 lines
5.6 KiB
Python
"""S4.2 live-run drill (comparison protocol §4 pkt 2/3): the ``--live-dry-run`` cut in
|
|
``run_project`` walks the real run path — contracts, budget, eager client build — and STOPS before
|
|
the first model call (``await debate.run(...)``), returning a ``DryRunReport`` with ZERO chat calls.
|
|
|
|
Load-bearing pair:
|
|
- T-4.2b (stop-point): a dry-run against a call-recording factory records an EMPTY sink (0 chat
|
|
calls) and returns a ``DryRunReport``; detach the early ``return`` → ``debate.run`` fires → sink
|
|
non-empty → RED.
|
|
- T-4.2b-control (causality): the SAME factory with ``live_dry_run=False`` drives the full run and
|
|
records a NON-empty sink — proving the 0 is caused by the cut, not an undriven fixture.
|
|
|
|
The artefact-set completeness guard (T-4.2c, §4 pkt 2/3) lives below (Step 4).
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from collections.abc import Callable
|
|
from pathlib import Path
|
|
|
|
from agent_framework import BaseChatClient
|
|
from conftest import SyntheticUsageChatClient
|
|
|
|
from portfolio_optimiser.run import DryRunReport, run_project
|
|
from portfolio_optimiser.validator import ValidatedProposal
|
|
|
|
BUNDLE_DIR = Path(__file__).resolve().parents[1] / "shared" / "examples" / "bygg-energi-mikro"
|
|
|
|
# A VALIDATOR-VALID BYGG-KONTOR-NORD proposal so a full control run completes cleanly.
|
|
_VALID_PROPOSER_REPLY = (
|
|
'{"measure":"LED-retrofit av kontorbelysning","affected_items":'
|
|
'[{"code":"ENERGI-TOTAL-EL","quantity":300000,"unit_cost":1.0}],"claimed_saving_nok":30000}'
|
|
)
|
|
_VERDICT_INPUT = {"decision": "approved", "rationale": "expert reviewed (sim)"}
|
|
|
|
|
|
def _role_factory(proposer_reply: str, checker_reply: str) -> Callable[[str], BaseChatClient]:
|
|
"""Role-aware scripted factory: the checker speaks its verdict, the proposer its proposal."""
|
|
|
|
def factory(role: str) -> BaseChatClient:
|
|
return SyntheticUsageChatClient(
|
|
default_reply=checker_reply if role == "checker" else proposer_reply
|
|
)
|
|
|
|
return factory
|
|
|
|
|
|
async def test_dry_run_stops_before_first_model_call(
|
|
tmp_path, make_recording_client_factory
|
|
) -> None:
|
|
"""T-4.2b (stop-point, load-bearing): ``live_dry_run=True`` builds contracts + clients + budget
|
|
but makes ZERO chat calls — the recording sink stays empty and a ``DryRunReport`` is returned,
|
|
while the run-config artefact ``r1-runconfig.json`` is written. Detach the early ``return`` (let
|
|
``debate.run`` execute) → the sink fills → RED."""
|
|
factory, sink = make_recording_client_factory(_VALID_PROPOSER_REPLY)
|
|
result = await run_project(
|
|
"BYGG-KONTOR-NORD",
|
|
"local",
|
|
docs_dir=str(BUNDLE_DIR),
|
|
bundle_dir=str(BUNDLE_DIR),
|
|
verdict_input=_VERDICT_INPUT,
|
|
client_factory=factory,
|
|
outbox_dir=str(tmp_path),
|
|
run_id="r1",
|
|
live_dry_run=True,
|
|
)
|
|
assert isinstance(result, DryRunReport)
|
|
assert sink == [] # zero chat calls — the cut held
|
|
assert (tmp_path / "r1-runconfig.json").is_file() # run-config captured (§4 pkt 3)
|
|
|
|
|
|
async def test_dry_run_control_full_run_makes_calls(
|
|
tmp_path, make_recording_client_factory
|
|
) -> None:
|
|
"""T-4.2b-control (causality): the SAME factory with ``live_dry_run=False`` drives the full run
|
|
and records a NON-empty sink — proving the empty sink above is caused by the dry-run cut, not by
|
|
a fixture that is never driven."""
|
|
factory, sink = make_recording_client_factory(_VALID_PROPOSER_REPLY)
|
|
result = await run_project(
|
|
"BYGG-KONTOR-NORD",
|
|
"local",
|
|
docs_dir=str(BUNDLE_DIR),
|
|
bundle_dir=str(BUNDLE_DIR),
|
|
verdict_input=_VERDICT_INPUT,
|
|
client_factory=factory,
|
|
)
|
|
assert not isinstance(result, DryRunReport) # a full run returns a RunResult
|
|
assert len(sink) > 0 # the debate + generation actually called the model
|
|
|
|
|
|
async def test_dry_run_artefact_set_complete(tmp_path) -> None:
|
|
"""T-4.2c (artefact-set completeness, load-bearing · §4 pkt 2/3): a scripted FULL run with an
|
|
``outbox_dir`` + ``run_id`` writes the COMPLETE set — proposal (IR + provenance incl. token
|
|
usage), outcome (percentiles + checker verdict + verdict id), AND run-config (profile + built
|
|
roles + params). Drop any file or field → RED."""
|
|
factory = _role_factory(_VALID_PROPOSER_REPLY, "VERDICT: APPROVE")
|
|
result = await run_project(
|
|
"BYGG-KONTOR-NORD",
|
|
"local",
|
|
docs_dir=str(BUNDLE_DIR),
|
|
bundle_dir=str(BUNDLE_DIR),
|
|
verdict_input=_VERDICT_INPUT,
|
|
client_factory=factory,
|
|
outbox_dir=str(tmp_path),
|
|
run_id="r2",
|
|
)
|
|
assert isinstance(result.outcome, ValidatedProposal)
|
|
|
|
proposal = json.loads((tmp_path / "r2-proposal.json").read_text(encoding="utf-8"))
|
|
assert proposal["run_id"] == "r2"
|
|
assert proposal["proposal"]["measure"] # the candidate IR is present
|
|
assert "ENERGI-TOTAL-EL" in {a["code"] for a in proposal["proposal"]["affected_items"]}
|
|
assert proposal["provenance"]["token_usage"] > 0 # §4 pkt 2: tokens captured
|
|
|
|
outcome = json.loads((tmp_path / "r2-outcome.json").read_text(encoding="utf-8"))
|
|
assert outcome["outcome_type"] == "validated"
|
|
assert all(k in outcome for k in ("p10", "p50", "p90", "nominal_feasible"))
|
|
assert outcome["checker_verdict"] == "approve"
|
|
assert outcome["verdict_id"] == result.verdict.id
|
|
|
|
runconfig = json.loads((tmp_path / "r2-runconfig.json").read_text(encoding="utf-8"))
|
|
assert runconfig["profile"] == "local"
|
|
assert set(runconfig["resolved_models"]) == {"proposer", "checker"} # built roles only
|
|
assert runconfig["max_rounds"] == 3
|
|
assert runconfig["max_tokens"] == 100_000
|
|
assert runconfig["top_k"] == 3
|