portfolio-optimiser/tests/test_scripted_cli_door_loadbearing.py
Kjell Tore Guttormsen 3abc61bac3 feat(run): a CLI door onto the offline whole-loop run (--scripted-replies)
An adopter without an API budget had two half-doors and no whole one.
`--live-dry-run` takes their own bundle but stops before the first model call
(`run_project` returns a DryRunReport), while `portfolio_optimiser.simulation`
runs the complete loop but only over ITS bundle with ITS scripted answers.
The seam for the missing third case -- the whole loop over your OWN data,
offline -- already existed as `run_project(client_factory=...)` and had zero
CLI exposure. This is the door onto that one seam, not a second implementation
of it (`scripted_factory` is imported lazily; `simulation` imports `run`, so a
module-level import would be circular).

The honesty banner is part of the feature, not decoration (maalbilde §1): a
scripted run that reads like a model run is worse than having no offline mode,
so every scripted invocation prints what is real (context navigation, debate
plumbing, deterministic validator, verdict) and what is not (the answers).

The two offline modes are mutually exclusive rather than one silently winning,
`--report` mode refuses the new flag by allowlist, and a replies file that
cannot serve the run is refused at the door rather than surfacing as a KeyError
mid-run.

Load-bearing MEASURED against the whole suite (645 -> 652), six mutations all
red: detach the wiring · detach the banner · detach the dry-run exclusivity ·
drop the flag from the --report allowlist · make the loader tolerant · control
(print the banner unconditionally).

The --report blade was measured GREEN first: with a non-existent ledger path
the load failure refused before the gate and masked it entirely. Rewritten
against a valid saved ledger, so rc 1 can only come from mode-exclusivity.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01GWsexbQjPo9rsV3aUE54ZS
2026-08-05 08:55:37 +02:00

166 lines
7 KiB
Python

"""The offline CLI door: run the WHOLE loop over your OWN bundle with zero model calls.
Before this, an adopter without an API budget had two half-doors and no whole one:
``--live-dry-run`` takes their own bundle but STOPS before the first model call
(``run.py`` returns a ``DryRunReport``), while ``portfolio_optimiser.simulation`` runs the
complete loop but only over ITS bundle with ITS scripted answers. The seam for the missing
third case already existed — ``run_project(client_factory=...)``, the test-injection seam —
with no CLI exposure at all. ``--scripted-replies <file.json>`` is that door.
**The honesty condition is part of the feature, not decoration** (målbilde §1): a scripted run
that reads like a model run is worse than no offline mode, so the banner asserted here is
load-bearing in the same sense the dataflow is. It mirrors ``simulation.py``'s honesty banner.
Load-bearing (each blade measured by detaching exactly one thing):
1. the door drives a FULL run offline over a caller-supplied bundle (RED without the flag);
2. the run is genuinely model-free (RED if a real factory is built);
3. the banner is emitted and unmistakable (RED when detached);
4. control: no banner without the flag — so blade 3 cannot pass on a constant;
5. the two offline modes are mutually exclusive rather than one silently winning.
"""
from __future__ import annotations
import json
import shutil
from pathlib import Path
import pytest
from portfolio_optimiser import run
from portfolio_optimiser.ledger import SavingsLedger
BUNDLE_DIR = Path(__file__).resolve().parents[1] / "shared" / "examples" / "bygg-energi-mikro"
# The same scripted answers the simulation uses, re-declared here as CALLER-supplied input —
# which is the whole point of the door: these arrive as a file the adopter writes, not as a
# module constant only the shipped simulation can reach.
_VALID_PROPOSAL = (
'{"measure":"LED-retrofit av kontorbelysning","affected_items":'
'[{"code":"ENERGI-TOTAL-EL","quantity":300000,"unit_cost":1.0}],"claimed_saving_nok":30000}'
)
_CHECKER_APPROVE = "Tallene er innenfor feasibelt område og resonnementet holder. VERDICT: APPROVE"
@pytest.fixture(autouse=True)
def _isolate_model_env(monkeypatch: pytest.MonkeyPatch) -> None:
"""Hermetic env (mirrors ``test_live_dry_run.py``): the operator's Foundry overrides must not
reach these assertions."""
monkeypatch.delenv("PORTFOLIO_MODEL_MAP", raising=False)
monkeypatch.delenv("PORTFOLIO_FOUNDRY_PROJECT_ENDPOINT", raising=False)
@pytest.fixture()
def bundle(tmp_path: Path) -> Path:
"""A throwaway COPY — the shared fixture is commons-owned and is never mutated by a test."""
dst = tmp_path / "bundle"
shutil.copytree(BUNDLE_DIR, dst)
return dst
@pytest.fixture()
def replies_file(tmp_path: Path) -> Path:
path = tmp_path / "replies.json"
path.write_text(
json.dumps({"proposer": _VALID_PROPOSAL, "checker": _CHECKER_APPROVE}),
encoding="utf-8",
)
return path
def _argv(bundle: Path, replies_file: Path) -> list[str]:
return [
"BYGG-KONTOR-NORD",
"--docs-dir",
str(bundle),
"--bundle-dir",
str(bundle),
"--scripted-replies",
str(replies_file),
]
def test_scripted_door_runs_the_whole_loop_offline(bundle, replies_file, capsys) -> None:
"""Blade 1 — the door exists and completes a FULL run (not a dry-run report) over the
caller's own bundle. RED before the flag: argparse exits 2 on an unrecognized argument."""
rc = run.main(_argv(bundle, replies_file))
out = capsys.readouterr().out
assert rc == 0, out
# A full run reports its outcome type + the minted verdict id; a dry-run never gets this far.
assert "BYGG-KONTOR-NORD:" in out
assert "verdict id=" in out
assert "LIVE-DRY-RUN" not in out
def test_scripted_door_makes_no_real_client(bundle, replies_file, monkeypatch, capsys) -> None:
"""Blade 2 — genuinely model-free: the production factory must never be built. Detonates if
the scripted factory is ignored and the run falls back to ``_default_factory``."""
def _boom(_profile: str): # pragma: no cover - the point is that it never runs
raise AssertionError("a real client factory was built on the scripted path")
monkeypatch.setattr(run, "_default_factory", _boom)
rc = run.main(_argv(bundle, replies_file))
assert rc == 0, capsys.readouterr().out
def test_scripted_door_says_so_unmistakably(bundle, replies_file, capsys) -> None:
"""Blade 3 — the honesty banner. A scripted run that looks like a model run is the failure
mode this asserts against; RED the moment the banner is detached."""
run.main(_argv(bundle, replies_file))
out = capsys.readouterr().out.upper()
assert "SCRIPTED" in out
assert "NO MODEL" in out or "INGEN MODELLKALL" in out
def test_no_banner_without_the_flag(bundle, capsys) -> None:
"""Blade 4 (control) — the banner must be CAUSED by the flag, not printed unconditionally.
Uses --live-dry-run so the control needs no model call of its own."""
run.main(
[
"BYGG-KONTOR-NORD",
"--docs-dir",
str(bundle),
"--bundle-dir",
str(bundle),
"--live-dry-run",
]
)
assert "SCRIPTED" not in capsys.readouterr().out.upper()
def test_the_two_offline_modes_are_exclusive(bundle, replies_file, capsys) -> None:
"""Blade 5 — ``--live-dry-run`` stops before the first call and ``--scripted-replies`` runs
every call; together they are a contradiction. Refuse, never let one silently win (mirrors
S5.3's 'refused, never ignored' partition)."""
rc = run.main([*_argv(bundle, replies_file), "--live-dry-run"])
assert rc == 1
assert "refused" in capsys.readouterr().err.lower()
def test_report_mode_refuses_the_scripted_flag(tmp_path, replies_file, capsys) -> None:
"""Blade 5b — ``--report`` is mode-exclusive by ALLOWLIST; a new flag must join the refusal
set rather than be silently dropped.
The ledger here is VALID and saved on purpose. Measured: with a non-existent ledger path this
test passed even after the allowlist entry was removed — the load failure refused first and
masked the gate entirely. With a loadable ledger the report would otherwise print and return
0, so rc 1 can only come from the mode-exclusivity check itself.
"""
ledger_path = tmp_path / "ledger.json"
SavingsLedger().save(str(ledger_path))
rc = run.main(
["--report", "--ledger", str(ledger_path), "--scripted-replies", str(replies_file)]
)
assert rc == 1
assert "mode-exclusive" in capsys.readouterr().err.lower()
def test_malformed_replies_file_is_refused(bundle, tmp_path, capsys) -> None:
"""A replies file that exists but cannot serve the run is refused with rc 1, never a
traceback and never a partial run — the same fail-fast posture the loaders take."""
bad = tmp_path / "bad.json"
bad.write_text("{not json", encoding="utf-8")
rc = run.main(_argv(bundle, bad))
assert rc == 1
assert "refused" in capsys.readouterr().err.lower()