open/portfolio-optimiser
1
0
Fork 0
Code Issues Releases 1 Activity

feat(s42): live_dry_run cut in run_project + DryRunReport

Browse source
This commit is contained in:
Kjell Tore Guttormsen 2026-07-15 18:05:16 +02:00
commit 0e986fe6c4
3 changed files with 234 additions and 46 deletions
Download patch file Download diff file Expand all files Collapse all files

110
src/portfolio_optimiser/run.py
View file

@ -28,7 +28,7 @@ from __future__ import annotations
from collections.abc import Callable, Sequence
from dataclasses import dataclass, replace
from decimal import ROUND_HALF_UP, Decimal
from typing import Any, Literal
from typing import Any, Literal, cast
from agent_framework import BaseChatClient, SessionContext
@ -58,7 +58,7 @@ from portfolio_optimiser.verdicts import (
capture_verdict,
load_verdicts_from_dir,
)
from portfolio_optimiser.workflow import fresh_workflow
from portfolio_optimiser.workflow import _MAKER_CHECKER_ROLES, fresh_workflow
@dataclass(frozen=True)
@ -79,6 +79,20 @@ class RunResult:
checker_verdict: str = "absent"
@dataclass(frozen=True)
class DryRunReport:
"""S4.2 offline ``--live-dry-run`` outcome (comparison protocol §4 pkt 3): everything a real run
would use — profile, the resolved model-id per BUILT role, and the round/token parameters —
captured WITHOUT a model call. A DISTINCT type from ``RunResult``, whose post-generation fields
(outcome/provenance/verdict) do not exist yet on a run that stopped before the first model call."""
profile: str
resolved_models: dict[str, str]
max_rounds: int
max_tokens: int
top_k: int
@dataclass(frozen=True)
class GoalReached:
"""A savings-goal signal VALUE (Step 8, SC6) — NOT an exception. Structured like
@ -230,7 +244,8 @@ async def run_project(
enable_layer1_hitl: bool = False,
notify: Callable[[Verdict], None] | None = None,
meter: TokenMeter | None = None,
) -> RunResult:
live_dry_run: bool = False,
) -> RunResult | DryRunReport:
"""Run the vertical slice for ONE project. ``client_factory`` is the test-injection seam
(defaults to the real backend). ``verdict_input`` carries the expert decision/rationale
(Layer-2). ``bundle_dir`` (Fase 2a) makes the run OKF-bundle-driven: the project is derived
@ -247,7 +262,10 @@ async def run_project(
outbox into a folder later read as an inbox would re-ingest raw agent output and bypass the
Step-8 promotion gate (self-contamination) — documented here, not enforced. Raises
``pydantic.ValidationError`` on a bad contract and ``BudgetExceeded`` when the token/round cap is
crossed, and ``ValueError`` when ``outbox_dir`` is set without a ``run_id``."""
crossed, and ``ValueError`` when ``outbox_dir`` is set without a ``run_id``. ``live_dry_run``
(S4.2, comparison protocol §4 pkt 2/3) is the offline drill: it walks the whole path up to the
EAGER client build, writes the run-config artefact (when ``outbox_dir`` is set), and returns a
``DryRunReport`` BEFORE the first model call (``debate.run``) — zero chat calls."""
# 0. Fail-fast: an outbox write is byte-deterministic and keyed on run_id — no wall-clock default.
if outbox_dir is not None and run_id is None:
raise ValueError(
@ -310,6 +328,34 @@ async def run_project(
tools=debate_tools,
middleware=[budget_mw],
)
# S4.2 cut (comparison protocol §4 pkt 2/3): everything above is offline — contracts, budget, and
# the EAGER client build (fresh_workflow constructs the proposer+checker clients, workflow.py:64).
# Capture the run-config (resolved model per BUILT role, profile, params, token cap) and, for a
# ``--live-dry-run``, STOP HERE — before the first (paid) model call at ``debate.run`` below.
if outbox_dir is not None or live_dry_run:
# ``resolved_models`` reflects the configured MAP (the default factory's model-ids for the M2
# run). Under an injected ``client_factory`` the built clients may differ (e.g. "synthetic");
# ``provenance.model`` (below) stays the authority on the client actually built.
resolved_models = {role: resolve_model(profile, role) for role in _MAKER_CHECKER_ROLES}
if outbox_dir is not None:
assert run_id is not None # narrowed by the step-0 guard (no wall-clock default)
outbox.write_run_config(
outbox_dir,
run_id,
profile=Profile(profile).value,
resolved_models=resolved_models,
max_rounds=max_rounds,
max_tokens=max_tokens,
top_k=top_k,
)
if live_dry_run:
return DryRunReport(
profile=Profile(profile).value,
resolved_models=resolved_models,
max_rounds=max_rounds,
max_tokens=max_tokens,
top_k=top_k,
)
result = await debate.run(f"Find a cost-saving measure for {project.id}.\nContext:\n{context}")
# F1: the candidate must derive from the DEBATE. Feed the proposer's converged output into
# generation (retrieval context is the last-resort fallback only). The checker's verdict
@ -532,20 +578,25 @@ async def run_portfolio(
if per_project_goal.mode == "hard":
continue # skip THIS pid; the rest of the pass proceeds
result = await run_project(
pid,
profile,
docs_dir=project.docs_dir,
verdict_input=project.verdict_input,
bundle_dir=project.bundle_dir,
verdict_dir=project.verdict_dir,
dimension=dimension,
store=store,
client_factory=client_factory,
max_rounds=max_rounds,
max_tokens=max_tokens,
top_k=top_k,
meter=meter_factory() if meter_factory is not None else None,
# run_portfolio only drives full runs (never dry-run), so the return narrows to RunResult;
# the cast keeps the widened run_project signature honest without an @overload duplication.
result = cast(
RunResult,
await run_project(
pid,
profile,
docs_dir=project.docs_dir,
verdict_input=project.verdict_input,
bundle_dir=project.bundle_dir,
verdict_dir=project.verdict_dir,
dimension=dimension,
store=store,
client_factory=client_factory,
max_rounds=max_rounds,
max_tokens=max_tokens,
top_k=top_k,
meter=meter_factory() if meter_factory is not None else None,
),
)
runs.append(result)
@ -577,15 +628,20 @@ def main(argv: list[str] | None = None) -> int:
parser.add_argument("--rationale", default="reviewed by expert")
args = parser.parse_args(argv)
result = asyncio.run(
run_project(
args.project_id,
args.profile,
docs_dir=args.docs_dir,
bundle_dir=args.bundle_dir,
verdict_dir=args.verdict_dir,
verdict_input={"decision": args.decision, "rationale": args.rationale},
)
# S4.2: main() currently drives only full runs (no --live-dry-run flag yet — that is Step 3,
# which replaces this cast with an isinstance(DryRunReport) branch + AZURE-refusal try/except).
result = cast(
RunResult,
asyncio.run(
run_project(
args.project_id,
args.profile,
docs_dir=args.docs_dir,
bundle_dir=args.bundle_dir,
verdict_dir=args.verdict_dir,
verdict_input={"decision": args.decision, "rationale": args.rationale},
)
),
)
kind = type(result.outcome).__name__
print(f"{args.project_id}: {kind} (verdict id={result.verdict.id}, decision={args.decision})")

44
src/portfolio_optimiser/simulation.py
View file

@ -24,7 +24,7 @@ import shutil
from collections.abc import Awaitable, Callable, Mapping, Sequence
from dataclasses import dataclass
from pathlib import Path
from typing import Any
from typing import Any, cast
from agent_framework import (
BaseChatClient,
@ -217,15 +217,18 @@ async def simulate_learning_loop(
# Run A — empty wiki isolates the persona's NEW knowledge.
sink_a: list[str] = []
run_a = await run_project(
_PROJECT_ID,
"local",
docs_dir=copy_s,
bundle_dir=copy_s,
verdict_input=verdict_input,
store=VerdictStore(verdicts=[]),
client_factory=scripted_factory(replies, sink_a),
max_rounds=max_rounds,
run_a = cast(
RunResult, # the sim only drives full runs; never dry-run (S4.2 widened run_project)
await run_project(
_PROJECT_ID,
"local",
docs_dir=copy_s,
bundle_dir=copy_s,
verdict_input=verdict_input,
store=VerdictStore(verdicts=[]),
client_factory=scripted_factory(replies, sink_a),
max_rounds=max_rounds,
),
)
# Gate-promote the persona verdict from the raw output layer into the OKF wiki (Steg 8).
@ -242,15 +245,18 @@ async def simulate_learning_loop(
# Run B — a separate, later run reads the updated wiki.
sink_b: list[str] = []
run_b = await run_project(
_PROJECT_ID,
"local",
docs_dir=copy_s,
bundle_dir=copy_s,
verdict_input=verdict_input,
store=store_b,
client_factory=scripted_factory(replies, sink_b),
max_rounds=max_rounds,
run_b = cast(
RunResult, # the sim only drives full runs; never dry-run (S4.2 widened run_project)
await run_project(
_PROJECT_ID,
"local",
docs_dir=copy_s,
bundle_dir=copy_s,
verdict_input=verdict_input,
store=store_b,
client_factory=scripted_factory(replies, sink_b),
max_rounds=max_rounds,
),
)
gen_a = _generation_prompts(sink_a)

126
tests/test_live_dry_run_loadbearing.py Normal file
View file

@ -0,0 +1,126 @@
"""S4.2 live-run drill (comparison protocol §4 pkt 2/3): the ``--live-dry-run`` cut in
``run_project`` walks the real run path — contracts, budget, eager client build — and STOPS before
the first model call (``await debate.run(...)``), returning a ``DryRunReport`` with ZERO chat calls.
Load-bearing pair:
- T-4.2b (stop-point): a dry-run against a call-recording factory records an EMPTY sink (0 chat
calls) and returns a ``DryRunReport``; detach the early ``return`` → ``debate.run`` fires → sink
non-empty → RED.
- T-4.2b-control (causality): the SAME factory with ``live_dry_run=False`` drives the full run and
records a NON-empty sink — proving the 0 is caused by the cut, not an undriven fixture.
The artefact-set completeness guard (T-4.2c, §4 pkt 2/3) lives below (Step 4).
"""
from __future__ import annotations
import json
from collections.abc import Callable
from pathlib import Path
from agent_framework import BaseChatClient
from conftest import SyntheticUsageChatClient
from portfolio_optimiser.run import DryRunReport, run_project
from portfolio_optimiser.validator import ValidatedProposal
BUNDLE_DIR = Path(__file__).resolve().parents[1] / "shared" / "examples" / "bygg-energi-mikro"
# A VALIDATOR-VALID BYGG-KONTOR-NORD proposal so a full control run completes cleanly.
_VALID_PROPOSER_REPLY = (
'{"measure":"LED-retrofit av kontorbelysning","affected_items":'
'[{"code":"ENERGI-TOTAL-EL","quantity":300000,"unit_cost":1.0}],"claimed_saving_nok":30000}'
)
_VERDICT_INPUT = {"decision": "approved", "rationale": "expert reviewed (sim)"}
def _role_factory(proposer_reply: str, checker_reply: str) -> Callable[[str], BaseChatClient]:
"""Role-aware scripted factory: the checker speaks its verdict, the proposer its proposal."""
def factory(role: str) -> BaseChatClient:
return SyntheticUsageChatClient(
default_reply=checker_reply if role == "checker" else proposer_reply
)
return factory
async def test_dry_run_stops_before_first_model_call(
tmp_path, make_recording_client_factory
) -> None:
"""T-4.2b (stop-point, load-bearing): ``live_dry_run=True`` builds contracts + clients + budget
but makes ZERO chat calls — the recording sink stays empty and a ``DryRunReport`` is returned,
while the run-config artefact ``r1-runconfig.json`` is written. Detach the early ``return`` (let
``debate.run`` execute) → the sink fills → RED."""
factory, sink = make_recording_client_factory(_VALID_PROPOSER_REPLY)
result = await run_project(
"BYGG-KONTOR-NORD",
"local",
docs_dir=str(BUNDLE_DIR),
bundle_dir=str(BUNDLE_DIR),
verdict_input=_VERDICT_INPUT,
client_factory=factory,
outbox_dir=str(tmp_path),
run_id="r1",
live_dry_run=True,
)
assert isinstance(result, DryRunReport)
assert sink == [] # zero chat calls — the cut held
assert (tmp_path / "r1-runconfig.json").is_file() # run-config captured (§4 pkt 3)
async def test_dry_run_control_full_run_makes_calls(
tmp_path, make_recording_client_factory
) -> None:
"""T-4.2b-control (causality): the SAME factory with ``live_dry_run=False`` drives the full run
and records a NON-empty sink — proving the empty sink above is caused by the dry-run cut, not by
a fixture that is never driven."""
factory, sink = make_recording_client_factory(_VALID_PROPOSER_REPLY)
result = await run_project(
"BYGG-KONTOR-NORD",
"local",
docs_dir=str(BUNDLE_DIR),
bundle_dir=str(BUNDLE_DIR),
verdict_input=_VERDICT_INPUT,
client_factory=factory,
)
assert not isinstance(result, DryRunReport) # a full run returns a RunResult
assert len(sink) > 0 # the debate + generation actually called the model
async def test_dry_run_artefact_set_complete(tmp_path) -> None:
"""T-4.2c (artefact-set completeness, load-bearing · §4 pkt 2/3): a scripted FULL run with an
``outbox_dir`` + ``run_id`` writes the COMPLETE set — proposal (IR + provenance incl. token
usage), outcome (percentiles + checker verdict + verdict id), AND run-config (profile + built
roles + params). Drop any file or field → RED."""
factory = _role_factory(_VALID_PROPOSER_REPLY, "VERDICT: APPROVE")
result = await run_project(
"BYGG-KONTOR-NORD",
"local",
docs_dir=str(BUNDLE_DIR),
bundle_dir=str(BUNDLE_DIR),
verdict_input=_VERDICT_INPUT,
client_factory=factory,
outbox_dir=str(tmp_path),
run_id="r2",
)
assert isinstance(result.outcome, ValidatedProposal)
proposal = json.loads((tmp_path / "r2-proposal.json").read_text(encoding="utf-8"))
assert proposal["run_id"] == "r2"
assert proposal["proposal"]["measure"] # the candidate IR is present
assert "ENERGI-TOTAL-EL" in {a["code"] for a in proposal["proposal"]["affected_items"]}
assert proposal["provenance"]["token_usage"] > 0 # §4 pkt 2: tokens captured
outcome = json.loads((tmp_path / "r2-outcome.json").read_text(encoding="utf-8"))
assert outcome["outcome_type"] == "validated"
assert all(k in outcome for k in ("p10", "p50", "p90", "nominal_feasible"))
assert outcome["checker_verdict"] == "approve"
assert outcome["verdict_id"] == result.verdict.id
runconfig = json.loads((tmp_path / "r2-runconfig.json").read_text(encoding="utf-8"))
assert runconfig["profile"] == "local"
assert set(runconfig["resolved_models"]) == {"proposer", "checker"} # built roles only
assert runconfig["max_rounds"] == 3
assert runconfig["max_tokens"] == 100_000
assert runconfig["top_k"] == 3