open/portfolio-optimiser-claude
1
0
Fork 0
Code Issues Releases 1 Activity

fix(run): classify portfolio-mode flags by allowlist so a new flag fails closed

Browse source 
The portfolio entrance refused unsupported flags from a hard-coded BLOCKLIST:
--inbox, --out, --outbox, --run-id, --value-report, --live-dry-run. That
construction fails OPEN. A flag added to the parser later and forgotten in the
list is accepted, does nothing, and says nothing — the operator's flag is a
claim the run does not back (§1). MAF's report mode already used an allowlist;
the divergence was raised as an open question and the operator decided it this
session in favour of fail-closed.

unsupported_flags_given() now reports every flag GIVEN that the allowlist does
not name. "Given" is measured against the parser's own default, so it needs no
knowledge of which flags exist — that is what keeps it correct for flags added
after it was written, including store_true switches.

Load-bearing (§11), detach-proven twice (before and after ruff format, restored
from a copy): swapping the membership test back to a hard-coded refusal list
turns test_a_flag_nobody_classified_is_refused RED, while every CLI-level
refusal test stays green — they only exercise flags a blocklist already names,
so they do not cover this seam. The other direction is covered too: a run
passing all fourteen honoured flags still exits 0, and the allowlist entries
are checked against the CLI's own --help so a rename cannot leave a dead entry.

612 -> 624 passed, ruff + mypy --strict clean. README states the allowlist.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01MQu2xxwedckjU56byu1aUG
This commit is contained in:
Kjell Tore Guttormsen 2026-07-25 15:33:30 +02:00
commit fc4a536e09
3 changed files with 194 additions and 18 deletions
Download patch file Download diff file Expand all files Collapse all files

139
tests/test_cli_paritet_loadbearing.py
View file

@ -22,6 +22,7 @@ moment the two drift apart.
from __future__ import annotations
import argparse
import contextlib
import importlib
import io
@ -40,7 +41,11 @@ from portfolio_optimiser_claude.inbox import VerdictDocument, write_verdict
from portfolio_optimiser_claude.ir import load_validator_input
from portfolio_optimiser_claude.ledger import SavingsLedger
from portfolio_optimiser_claude.loop import ModelClient, ModelReply
from portfolio_optimiser_claude.run import main
from portfolio_optimiser_claude.run import (
_PORTFOLIO_SUPPORTED_DESTS,
main,
unsupported_flags_given,
)
REPO_ROOT = Path(__file__).resolve().parents[1]
BUNDLE = REPO_ROOT / "shared" / "examples" / "bygg-energi-mikro"
@ -361,6 +366,138 @@ class TestPortfolioOnTheEntrance:
assert created == []
class TestPortfolioClassifiesByAllowlist:
"""LOAD-BEARING (§1, §11): portfolio mode fails CLOSED on an unclassified flag.
The refusal used to run off a BLOCKLIST — the flags to reject, named one by
one. That construction fails OPEN: a flag added to the parser later and
forgotten in the list is accepted and then silently ignored, which is
exactly the false claim §1 forbids (the operator's flag did nothing and the
CLI said nothing). The allowlist names the flags the portfolio pass HONOURS;
everything else is refused whether or not anyone remembered it.
Detach point: replace the ``dest not in supported`` membership test with a
hard-coded list of flags to refuse → ``test_a_flag_nobody_classified_is_refused``
goes RED, because tomorrow's flag appears in no such list. The CLI-level
refusal tests stay GREEN under that mutation (they only exercise flags a
blocklist already names), so they do not cover this seam.
"""
def _parser_with(self, *flags: str) -> argparse.ArgumentParser:
parser = argparse.ArgumentParser()
for flag in flags:
parser.add_argument(flag, default=None)
return parser
def test_a_flag_nobody_classified_is_refused(self) -> None:
# "--tomorrows-flag" stands in for the next flag someone adds: it is in
# no allowlist and in no blocklist. Fail-closed means it is reported.
parser = self._parser_with("--portfolio", "--tomorrows-flag")
args = parser.parse_args(["--portfolio", "p.json", "--tomorrows-flag", "x"])
assert unsupported_flags_given(parser, args, supported=frozenset({"portfolio"})) == [
"--tomorrows-flag"
]
def test_a_supported_flag_is_not_reported(self) -> None:
parser = self._parser_with("--portfolio", "--ledger")
args = parser.parse_args(["--portfolio", "p.json", "--ledger", "l.json"])
assert (
unsupported_flags_given(parser, args, supported=frozenset({"portfolio", "ledger"}))
== []
)
def test_a_flag_left_at_its_default_was_never_given(self) -> None:
# Absence is not a claim: an unsupported flag the operator did not pass
# must not turn a valid portfolio run into an error.
parser = self._parser_with("--portfolio", "--tomorrows-flag")
args = parser.parse_args(["--portfolio", "p.json"])
assert unsupported_flags_given(parser, args, supported=frozenset({"portfolio"})) == []
def test_a_store_true_flag_counts_as_given_only_when_passed(self) -> None:
# store_true defaults to False, not None — "given" is measured against
# the parser's OWN default, so both directions are correct.
parser = argparse.ArgumentParser()
parser.add_argument("--portfolio", default=None)
parser.add_argument("--tomorrows-switch", action="store_true")
supported = frozenset({"portfolio"})
unset = parser.parse_args(["--portfolio", "p.json"])
assert unsupported_flags_given(parser, unset, supported=supported) == []
given = parser.parse_args(["--portfolio", "p.json", "--tomorrows-switch"])
assert unsupported_flags_given(parser, given, supported=supported) == ["--tomorrows-switch"]
@pytest.mark.parametrize(
"extra",
[
["--out", "out-dir"],
["--inbox", "inbox-dir"],
["--outbox", "outbox-dir"],
["--run-id", "run-1"],
["--value-report", "report.json"],
["--live-dry-run"],
],
ids=["out", "inbox", "outbox", "run-id", "value-report", "live-dry-run"],
)
def test_the_real_cli_refuses_every_unsupported_flag(
self, tmp_path: Path, extra: list[str]
) -> None:
# The wire: main() classifies with the real allowlist, before any client.
factory, created = _scripted_factory()
with pytest.raises(SystemExit):
main(
["--portfolio", str(_portfolio_file(tmp_path, ["prosjekt-a"]))] + extra,
client_factory=factory,
)
assert created == []
def test_the_real_cli_accepts_every_flag_it_honours(self, tmp_path: Path) -> None:
# The other half of fail-closed: the allowlist must not refuse the flags
# the portfolio pass genuinely uses. A goal far above the book does not
# stop the run, so this exercises the full pass.
verdict_dir, _ = _portfolio_inbox(tmp_path)
factory, created = _scripted_factory(runs=1)
code = main(
[
"--portfolio",
str(_portfolio_file(tmp_path, ["prosjekt-a"])),
"--verdict-dir",
str(verdict_dir),
"--ledger",
str(_ledger_file(tmp_path, amount_nok=1000.0)),
"--goals",
str(_goal_file(tmp_path, target_nok=10_000_000.0, mode="hard")),
"--max-rounds",
"12",
"--max-tokens",
"150000",
"--max-budget-usd-per-call",
"0.25",
"--max-debate-rounds",
"3",
"--max-attempts",
"3",
"--top-k",
"3",
"--notify-console",
"--notify-file",
str(tmp_path / "notify"),
],
client_factory=factory,
)
assert code == 0
assert created, "the portfolio pass must have run"
def test_the_allowlist_names_only_flags_the_cli_actually_has(self) -> None:
# A renamed flag would leave a dead entry behind, and the renamed flag
# would start being refused in portfolio mode without anyone saying so.
help_text = _full_help("run")
missing = sorted(
f"--{dest.replace('_', '-')}"
for dest in _PORTFOLIO_SUPPORTED_DESTS
if f"--{dest.replace('_', '-')}" not in help_text
)
assert missing == []
# --- the documentation-honesty seam (§1) -----------------------------------------------------
# Lines about third-party dev tooling are not claims about this framework's CLI.