B2's free drill earned its keep on the first command. STATE.md, docs/2026-09-12-p14-kontekstsett.md and the order all publish the same stress command ending "--max-rounds 8 --max-tokens 120000". Measured: run.py accepts neither, all four --live-dry-run drills refused with "unrecognized arguments", and main() never passed max_rounds/max_tokens to run_project at all -- so every CLI run ever made was silently bound to _DEFAULT_MAX_ROUNDS=3 / _DEFAULT_MAX_TOKENS=100_000, with no way to raise or lower the cap on a run being paid for. Three surfaces described a door that did not exist. Widening, never breaking: both flags default to exactly those values, so every existing invocation is byte-identical. Wired to BOTH dispatches -- run_portfolio takes the same two parameters and main() dropped them there too -- and refused by name in report mode, which returns above every dispatch (the F4 silent-drop gap). Load-bearing MEASURED (6 arms, ALL RED before the fix), four mutations all red, green control 1669/5 (from 1663/5, superset, 0 removed), golden BYTE-UNCHANGED (ea8c534...). MEASURED, REPORTED, NOT FIXED: single-project mode still requires --docs-dir even when --bundle-dir is given and docs_dir is unused on the bundle path, so the documented command would have refused for that reason too. The README's own form works; loosening the guard is its own call. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
145 lines
5.6 KiB
Python
145 lines
5.6 KiB
Python
"""P16 B2 - the CLI had NO door onto a paid run's cap, and the documented command proved it.
|
|
|
|
**The measured silence.** ``STATE.md``, ``docs/2026-09-12-p14-kontekstsett.md § 4.1`` and order
|
|
``20260914T091846Z`` all publish the same stress command, ending ``--max-rounds 8 --max-tokens
|
|
120000``. Measured 14.09: ``run.py`` accepts neither flag, all four free ``--live-dry-run`` drills
|
|
refused with ``unrecognized arguments``, and ``main()`` never passed ``max_rounds``/``max_tokens``
|
|
to ``run_project`` at all - so **every CLI run ever made was silently bound to the defaults**
|
|
(``max_rounds=3``, ``max_tokens=100_000``) with no way for an operator to raise or lower the cap on
|
|
a run they were paying for. Three surfaces described a door that did not exist: the Fase-3 class
|
|
(a claim the surface makes about itself), spread across the operator's own instructions.
|
|
|
|
**Widening, never breaking.** Both flags DEFAULT to the values ``run_project`` already used, so
|
|
every invocation that exists is byte-identical; what changes is that the cap can now be stated.
|
|
|
|
**Wired to BOTH dispatches, and refused in report mode.** ``run_portfolio`` takes the same two
|
|
parameters and ``main()`` dropped them there too, so a portfolio pass could not be capped either.
|
|
Report mode returns ABOVE every dispatch, so a flag left out of ``report_forbidden`` is a SILENT
|
|
DROP rather than a refusal - the gap F4 measured on ``--plan-review``.
|
|
|
|
Arms: (a) the flags parse and reach ``run_project`` * (b) they reach ``run_portfolio`` *
|
|
(c) the defaults are unchanged when the flags are absent * (d) report mode refuses each by name *
|
|
(e) the free dry-run drill accepts the documented command form.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import subprocess
|
|
import sys
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
import pytest
|
|
|
|
from portfolio_optimiser import run as run_module
|
|
|
|
_REPO = Path(__file__).resolve().parents[1]
|
|
|
|
|
|
def _base(root: Path) -> Path:
|
|
base = root / "b"
|
|
(base / "krav").mkdir(parents=True)
|
|
(base / "index.md").write_text("---\nbundle_id: b\n---\n\n- [c](krav/c.md)\n", encoding="utf-8")
|
|
(base / "krav" / "c.md").write_text(
|
|
'---\ntype: concept\ntitle: "C"\n---\n\nbody\n', encoding="utf-8"
|
|
)
|
|
return base
|
|
|
|
|
|
def _capture(monkeypatch: pytest.MonkeyPatch, target: str) -> dict[str, Any]:
|
|
"""Record the kwargs main() hands the named coroutine, then stop the run."""
|
|
seen: dict[str, Any] = {}
|
|
|
|
async def _fake(*args: Any, **kwargs: Any) -> Any:
|
|
seen.update(kwargs)
|
|
raise SystemExit(0)
|
|
|
|
monkeypatch.setattr(run_module, target, _fake)
|
|
return seen
|
|
|
|
|
|
def test_a_the_two_flags_reach_run_project(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
|
|
seen = _capture(monkeypatch, "run_project")
|
|
with pytest.raises(SystemExit):
|
|
b = str(_base(tmp_path))
|
|
run_module.main(
|
|
[
|
|
"P1",
|
|
"--profile",
|
|
"local",
|
|
"--docs-dir",
|
|
b,
|
|
"--bundle-dir",
|
|
b,
|
|
"--max-rounds",
|
|
"8",
|
|
"--max-tokens",
|
|
"120000",
|
|
]
|
|
)
|
|
assert seen["max_rounds"] == 8
|
|
assert seen["max_tokens"] == 120000
|
|
|
|
|
|
def test_b_the_two_flags_reach_run_portfolio(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
seen = _capture(monkeypatch, "run_portfolio")
|
|
with pytest.raises(SystemExit):
|
|
run_module.main(
|
|
["--portfolio", "--profile", "local", "--max-rounds", "9", "--max-tokens", "77000"]
|
|
)
|
|
assert seen["max_rounds"] == 9
|
|
assert seen["max_tokens"] == 77000
|
|
|
|
|
|
def test_c_absent_flags_keep_the_values_every_run_so_far_used(
|
|
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
|
) -> None:
|
|
"""Widening, never breaking: the defaults ARE what main() bound implicitly before."""
|
|
seen = _capture(monkeypatch, "run_project")
|
|
with pytest.raises(SystemExit):
|
|
b = str(_base(tmp_path))
|
|
run_module.main(["P1", "--profile", "local", "--docs-dir", b, "--bundle-dir", b])
|
|
assert seen["max_rounds"] == run_module._DEFAULT_MAX_ROUNDS == 3
|
|
assert seen["max_tokens"] == run_module._DEFAULT_MAX_TOKENS == 100_000
|
|
|
|
|
|
@pytest.mark.parametrize("flag,value", [("--max-rounds", "8"), ("--max-tokens", "120000")])
|
|
def test_d_report_mode_refuses_each_flag_by_name(
|
|
flag: str, value: str, tmp_path: Path, capsys: pytest.CaptureFixture[str]
|
|
) -> None:
|
|
ledger = tmp_path / "l.json"
|
|
ledger.write_text("[]", encoding="utf-8")
|
|
# The control: the SAME argv without the flag is ACCEPTED, so rc 1 below is the mutant's
|
|
# opposite outcome and not the fixture refusing for a reason of its own.
|
|
assert run_module.main(["--report", "--ledger", str(ledger)]) == 0
|
|
assert run_module.main(["--report", "--ledger", str(ledger), flag, value]) == 1
|
|
assert "mode-exclusive" in capsys.readouterr().err
|
|
|
|
|
|
def test_e_the_documented_dry_run_command_form_is_accepted(tmp_path: Path) -> None:
|
|
"""The B2 drill itself: the command STATE.md and the order publish must at least parse."""
|
|
base = _base(tmp_path)
|
|
proc = subprocess.run(
|
|
[
|
|
sys.executable,
|
|
"-m",
|
|
"portfolio_optimiser.run",
|
|
"P1",
|
|
"--profile",
|
|
"local",
|
|
"--docs-dir",
|
|
str(base),
|
|
"--bundle-dir",
|
|
str(base),
|
|
"--max-rounds",
|
|
"8",
|
|
"--max-tokens",
|
|
"120000",
|
|
"--live-dry-run",
|
|
],
|
|
capture_output=True,
|
|
text=True,
|
|
cwd=_REPO,
|
|
)
|
|
assert "unrecognized arguments" not in proc.stderr, proc.stderr
|
|
assert proc.returncode == 0, proc.stderr
|