feat(portfolio): K7 — SDK/API preflight, offline pre-spend boundary (parity row 20) [skip-docs]

Everything that CAN be validated WITHOUT a model call is validated BEFORE the
operator pays for one (S4.1-analog, SDK-native — Foundry-auth is MAF-specific,
not mirrored). The preflight IS the boundary: it never calls query(), never
validates a credential VALUE online, never touches the network. It returns a
list of structured, actionable Refusals; the CLI exits non-zero on any, so a
broken config stops cheaply instead of on the first billed call.

- preflight.py: run_preflight + `python -m …preflight --profile anthropic`.
  Four offline checks:
  * credential — ANTHROPIC_API_KEY set + not a placeholder form (the value is
    NEVER checked online, only presence, §1); the bundled CLI's own
    CLAUDE_CODE_OAUTH_TOKEN also satisfies it (run_s10 relies on it — refusing
    would be a false alarm).
  * model_map — the requested profile exists and every id it resolves to,
    THROUGH resolve_model (so the default fall-through is covered), is real,
    not a placeholder left in config.
  * sdk — claude_agent_sdk imports (a missing install is a structured refusal
    naming `uv sync`, never an ImportError out of the preflight), the run-path
    symbols exist, and the bundled Claude Code CLI is present on disk — located
    OFFLINE via the SDK package's own files, mirroring the SDK's _find_cli
    order (bundled first, then a claude on PATH). Verified against installed
    0.2.120 (bundled binary present; the STATE 0.2.110 note was stale).
  * termination — the §8 stop contract constructs with positive caps and the
    per-call USD belt is positive.
- tests/test_preflight.py: credential contract, placeholder detector, model_map
  incl. unknown-profile-without-raising, SDK + bundled-CLI offline probe (with a
  monkeypatched missing-SDK refusal), termination, run_preflight aggregation,
  CLI both paths, and the offline guards (no network import, no query() call —
  AST-based so prose stays green). THREE seams detach-proven RED: credential
  branch, placeholder model-id guard, no-network grep-guard.

478→514 green, golden byte-exact, full gate clean (ruff+format+mypy strict,
24 src files), run_s10.py/runs/ byte-untouched. README test-count sync ×2 +
preflight.py module note + load-bearing mention. IKKE-scope (held): the actual
API call (ALDRI — the preflight IS the boundary) and Foundry/Azure auth.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01RiTwaKLesgcwXx2mDviqpt
This commit is contained in:
Kjell Tore Guttormsen 2026-07-24 01:34:18 +02:00
commit c08d92a358
3 changed files with 588 additions and 3 deletions

260
tests/test_preflight.py Normal file
View file

@ -0,0 +1,260 @@
"""SDK/API preflight (K7, S4.1-analog; paritetsrad 20) — LOAD-BEARING (§11).
Everything that CAN be validated WITHOUT a model call is validated here, BEFORE
the operator pays for one. Every seam is load-bearing: the RED-when-detached
notes on the three required proofs (credential, placeholder model id, no-network
grep-guard) name the mutation that makes them fail, so a green-but-dead check
can't hide.
Offline invariant: the preflight never calls ``query()``, never validates a key
VALUE online, never touches the network. Importing ``claude_agent_sdk`` (to prove
importability + bundled CLI presence) is offline-safe the whole suite already
imports the SDK client without a key or a socket.
"""
from __future__ import annotations
import ast
import importlib
from pathlib import Path
import pytest
from portfolio_optimiser_claude.contracts import ModelMapContract
from portfolio_optimiser_claude.preflight import (
Refusal,
_check_credentials,
_check_model_map,
_check_sdk,
_check_termination,
_locate_cli,
_looks_like_placeholder,
main,
run_preflight,
)
SRC_PKG = Path(__file__).resolve().parents[1] / "src" / "portfolio_optimiser_claude"
_REAL_KEY = "sk-ant-api03-abc123def456" # shape only — never validated online
def _kinds(refusals: list[Refusal]) -> set[str]:
return {r.check for r in refusals}
class TestCredentialContract:
"""RED (detach the no-credential branch → green without a key): a run with
NO API credential set is refused BEFORE any spend the key VALUE, however,
is never checked online, only that one is present and not a placeholder."""
def test_no_credential_is_refused(self) -> None:
refusals = _check_credentials({})
assert any(r.check == "credential" for r in refusals)
def test_a_set_key_clears_the_credential_check(self) -> None:
assert _check_credentials({"ANTHROPIC_API_KEY": _REAL_KEY}) == []
def test_a_placeholder_key_is_refused(self) -> None:
refusals = _check_credentials({"ANTHROPIC_API_KEY": "your-key-here"})
assert any(r.check == "credential" for r in refusals)
assert any("placeholder" in r.detail.lower() for r in refusals)
def test_the_key_value_is_never_asserted_only_its_presence(self) -> None:
# Any non-placeholder string satisfies the contract — the preflight is
# the boundary; a bad key surfaces on the FIRST call, never here (§1).
assert _check_credentials({"ANTHROPIC_API_KEY": "obviously-not-a-real-key-42"}) == []
def test_the_bundled_cli_oauth_token_satisfies_the_credential(self) -> None:
# run_s10 relies on the bundled CLI's own credentials when no key is
# exported; refusing that would be a false alarm (§1 honesty).
assert _check_credentials({"CLAUDE_CODE_OAUTH_TOKEN": "sk-ant-oat01-real"}) == []
class TestPlaceholderModelIdRefused:
"""RED (detach the resolve_model placeholder guard → a placeholder id passes):
a model_map whose resolved id is a placeholder form is refused and the check
goes THROUGH resolve_model, so the ``default`` fall-through is covered too."""
def test_a_placeholder_default_model_id_is_refused(self) -> None:
model_map = ModelMapContract(profiles={"anthropic": {"default": "REPLACE_ME"}})
refusals = _check_model_map("anthropic", model_map=model_map)
assert any(r.check == "model_map" for r in refusals)
def test_a_placeholder_role_model_id_is_refused(self) -> None:
model_map = ModelMapContract(
profiles={"anthropic": {"default": "claude-haiku-4-5-20251001", "proposer": "<todo>"}}
)
refusals = _check_model_map("anthropic", model_map=model_map)
assert any(r.check == "model_map" for r in refusals)
def test_a_real_model_map_clears(self) -> None:
model_map = ModelMapContract(
profiles={"anthropic": {"default": "claude-haiku-4-5-20251001"}}
)
assert _check_model_map("anthropic", model_map=model_map) == []
def test_an_unknown_profile_is_refused_without_raising(self) -> None:
# resolve_model would raise on an unknown profile; the preflight turns
# that into a structured refusal, never a stack trace before spend.
model_map = ModelMapContract(
profiles={"anthropic": {"default": "claude-haiku-4-5-20251001"}}
)
refusals = _check_model_map("bedrock", model_map=model_map)
assert any(r.check == "model_map" for r in refusals)
assert any("bedrock" in r.detail for r in refusals)
def test_the_bundled_model_map_is_clear(self) -> None:
# Nøkkelantakelse: the shipped model_map configures only real ids.
assert _check_model_map("anthropic") == []
class TestPlaceholderDetector:
"""The shared placeholder detector guards BOTH the key and the model id path."""
@pytest.mark.parametrize(
"value",
["", " ", "your-key-here", "REPLACE_ME", "changeme", "<model>", "TODO", "xxxx-xxxx"],
)
def test_placeholder_forms_are_caught(self, value: str) -> None:
assert _looks_like_placeholder(value)
@pytest.mark.parametrize("value", ["claude-haiku-4-5-20251001", _REAL_KEY, "sk-ant-oat01-real"])
def test_real_values_pass(self, value: str) -> None:
assert not _looks_like_placeholder(value)
class TestSdkAndBundledCli:
"""Nøkkelantakelse (verified against installed claude-agent-sdk 0.2.120): the
SDK imports and its bundled CLI is present on disk checked OFFLINE via the
package's own files, mirroring the SDK's ``_find_cli`` order. Never a net call."""
def test_the_installed_sdk_clears(self) -> None:
assert _check_sdk() == []
def test_the_bundled_cli_is_locatable_offline(self) -> None:
sdk = importlib.import_module("claude_agent_sdk")
cli = _locate_cli(sdk)
assert cli is not None
assert cli.exists()
def test_a_missing_sdk_is_refused_not_crashed(self, monkeypatch: pytest.MonkeyPatch) -> None:
# Detach-proof for graceful degradation: an uninstallable SDK becomes a
# structured refusal naming the fix, never an ImportError at preflight.
import portfolio_optimiser_claude.preflight as pf
def _boom(name: str) -> object:
raise ImportError(f"no module named {name}")
monkeypatch.setattr(pf.importlib, "import_module", _boom)
refusals = _check_sdk()
assert any(r.check == "sdk" for r in refusals)
assert any("claude-agent-sdk" in r.detail for r in refusals)
class TestTerminationContract:
"""§8: the stop/budget contract must be SET and valid before any spend."""
def test_positive_caps_clear(self) -> None:
assert _check_termination(12, 150_000, 0.25) == []
@pytest.mark.parametrize("rounds,tokens", [(0, 150_000), (12, 0), (-1, 10)])
def test_non_positive_stop_caps_are_refused(self, rounds: int, tokens: int) -> None:
refusals = _check_termination(rounds, tokens, 0.25)
assert any(r.check == "termination" for r in refusals)
@pytest.mark.parametrize("cap", [0.0, -0.5])
def test_non_positive_usd_cap_is_refused(self, cap: float) -> None:
refusals = _check_termination(12, 150_000, cap)
assert any(r.check == "termination" for r in refusals)
class TestRunPreflightAggregates:
"""The whole preflight is the union of its checks — offline, no model call."""
def test_a_good_config_yields_no_refusals(self) -> None:
assert run_preflight(profile="anthropic", env={"ANTHROPIC_API_KEY": _REAL_KEY}) == []
def test_a_bad_config_collects_every_failing_dimension(self) -> None:
refusals = run_preflight(
profile="bedrock", # unknown profile
env={}, # no credential
max_rounds=0, # bad stop contract
max_tokens=150_000,
)
assert {"credential", "model_map", "termination"} <= _kinds(refusals)
class TestCli:
"""The thin CLI: refusals → non-zero exit + actionable lines; clear → 0.
Honesty (§1): the output states no model call was made the preflight IS
the boundary, so a green preflight never implies a validated key or a run.
"""
def test_clear_run_returns_zero(
self, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]
) -> None:
monkeypatch.setenv("ANTHROPIC_API_KEY", _REAL_KEY)
monkeypatch.delenv("CLAUDE_CODE_OAUTH_TOKEN", raising=False)
rc = main(["--profile", "anthropic"])
out = capsys.readouterr().out
assert rc == 0
assert "OK" in out
assert "no model call" in out.lower()
def test_missing_credential_returns_one_with_actionable_line(
self, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]
) -> None:
monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False)
monkeypatch.delenv("CLAUDE_CODE_OAUTH_TOKEN", raising=False)
rc = main(["--profile", "anthropic"])
out = capsys.readouterr().out
assert rc == 1
assert "ANTHROPIC_API_KEY" in out
assert "no spend" in out.lower() or "no model call" in out.lower()
def _imported_module_names(module_path: Path) -> set[str]:
tree = ast.parse(module_path.read_text(encoding="utf-8"))
names: set[str] = set()
for node in ast.walk(tree):
if isinstance(node, ast.Import):
names.update(alias.name.split(".")[0] for alias in node.names)
elif isinstance(node, ast.ImportFrom) and node.level == 0 and node.module:
names.add(node.module.split(".")[0])
return names
def _called_names(module_path: Path) -> set[str]:
"""Every function/method NAME called in the module (AST, ignores prose)."""
tree = ast.parse(module_path.read_text(encoding="utf-8"))
called: set[str] = set()
for node in ast.walk(tree):
if isinstance(node, ast.Call):
func = node.func
if isinstance(func, ast.Name):
called.add(func.id)
elif isinstance(func, ast.Attribute):
called.add(func.attr)
return called
class TestPreflightIsOffline:
"""RED (add ``import socket`` / ``import httpx`` to preflight.py): the
grep-guard proves the preflight carries NO network path it validates the
SDK by IMPORTING it, never by reaching the API."""
def test_no_network_module_is_imported(self) -> None:
# The offline seam (grep-guard, AST form): no socket/httpx path exists —
# importing any of these is the detach that turns this RED.
names = _imported_module_names(SRC_PKG / "preflight.py")
assert not names & {"socket", "urllib", "http", "requests", "httpx", "anthropic"}
def test_no_sdk_completion_is_called(self) -> None:
# The preflight IS the boundary: it may IMPORT the SDK but must never
# invoke query()/ClaudeSDKClient — that call would be the spend it guards.
# AST-based, so a docstring mentioning query() stays green; only a real
# call trips it (add ``query(prompt=...)`` → RED).
called = _called_names(SRC_PKG / "preflight.py")
assert "query" not in called
assert "ClaudeSDKClient" not in called