95 lines
4.3 KiB
Python
95 lines
4.3 KiB
Python
"""S4.1 preflight — load-bearing detach seams. Each test goes RED when the guarded mechanism is
|
|
removed; a bare happy-path pass would not catch a regression here.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import ast
|
|
import json
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from portfolio_optimiser.backends import Profile, resolve_model
|
|
|
|
_PREFLIGHT = Path(__file__).resolve().parents[1] / "src" / "portfolio_optimiser" / "preflight.py"
|
|
|
|
# Symbols preflight must never IMPORT (offline) or CALL (no auto-login). AST-based, NOT substring:
|
|
# the `https` scheme literal contains `http`, and the module prose may mention the recipe — a
|
|
# substring scan would false-positive (test_okf.py:216 lesson).
|
|
_FORBIDDEN_IMPORTS = {
|
|
"socket",
|
|
"urllib",
|
|
"http",
|
|
"requests",
|
|
"httpx",
|
|
"ftplib",
|
|
"smtplib",
|
|
"subprocess",
|
|
}
|
|
_FORBIDDEN_CALLS = {"get_token", "DefaultAzureCredential", "AzureCliCredential"}
|
|
|
|
|
|
def _preflight_ast() -> ast.Module:
|
|
return ast.parse(_PREFLIGHT.read_text(encoding="utf-8"))
|
|
|
|
|
|
def test_sc6_no_network_or_autologin_in_preflight() -> None:
|
|
"""SC6: preflight.py imports NO network/subprocess library and CALLS no credential-acquisition
|
|
symbol — so it can neither egress nor auto-log-in. Detach points: add a ``urllib``/``socket``
|
|
import, or a ``subprocess.run([...,"az","login"])`` / ``AzureCliCredential()`` call → RED. AST,
|
|
not substring: the ``https`` literal contains ``http`` and the docs recipe mentions ``az login``.
|
|
|
|
Scope honesty: this proves preflight.py's OWN direct imports are egress-free. ``preflight``
|
|
imports ``backends`` (which imports network-capable ``FoundryChatClient``), so "cannot egress"
|
|
holds precisely because preflight never CALLS client construction — not because the transitive
|
|
closure is import-free. The ``subprocess`` import ban makes shelling out to ``az login``
|
|
impossible (so the brief's literal "az login" symbol is covered transitively — a bare string
|
|
can't invoke anything without an import)."""
|
|
imported: list[str] = []
|
|
called: list[str] = []
|
|
for node in ast.walk(_preflight_ast()):
|
|
if isinstance(node, ast.Import):
|
|
imported += [a.name.split(".")[0] for a in node.names]
|
|
elif isinstance(node, ast.ImportFrom):
|
|
imported.append((node.module or "").split(".")[0])
|
|
elif isinstance(node, ast.Call):
|
|
fn = node.func
|
|
if isinstance(fn, ast.Name):
|
|
called.append(fn.id)
|
|
elif isinstance(fn, ast.Attribute):
|
|
called.append(fn.attr)
|
|
bad_imports = sorted(set(imported) & _FORBIDDEN_IMPORTS)
|
|
bad_calls = sorted(set(called) & _FORBIDDEN_CALLS)
|
|
assert bad_imports == [], f"preflight must have no network/subprocess import, found: {bad_imports}"
|
|
assert bad_calls == [], f"preflight must not acquire a credential/token, found: {bad_calls}"
|
|
|
|
|
|
def test_sc1_refusal_names_the_exact_placeholder(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
"""SC1 teeth: the refusal names the EXACT bundled placeholder deployment (value-pinned). Detach
|
|
point: make the refusal generic (drop the offending id from the message) → RED."""
|
|
from portfolio_optimiser import preflight
|
|
|
|
monkeypatch.delenv("PORTFOLIO_MODEL_MAP", raising=False)
|
|
monkeypatch.setenv("PORTFOLIO_FOUNDRY_PROJECT_ENDPOINT", "https://x.services.ai.azure.com")
|
|
result = preflight.check_azure_preflight("azure")
|
|
assert isinstance(result, preflight.PreflightRefusal)
|
|
assert "REPLACE-WITH-FOUNDRY-DEPLOYMENT" in result.reason
|
|
|
|
|
|
def test_sc4_resolve_model_guard_has_teeth(
|
|
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
|
) -> None:
|
|
"""SC4 teeth: resolve_model raises on a REPLACE-WITH-* azure deployment. Detach point: remove the
|
|
_PLACEHOLDER_PREFIX guard in resolve_model → it returns the placeholder → RED. Control: a valid
|
|
override returns the real id, so the raise fires only on the placeholder, not always."""
|
|
monkeypatch.delenv("PORTFOLIO_MODEL_MAP", raising=False)
|
|
with pytest.raises(ValueError, match="placeholder"):
|
|
resolve_model(Profile.AZURE, "proposer")
|
|
valid = tmp_path / "model_map.json"
|
|
valid.write_text(
|
|
json.dumps({"local": {"default": "qwen3:4b"}, "azure": {"default": "gpt-4o-mini"}}),
|
|
encoding="utf-8",
|
|
)
|
|
monkeypatch.setenv("PORTFOLIO_MODEL_MAP", str(valid))
|
|
assert resolve_model(Profile.AZURE, "proposer") == "gpt-4o-mini"
|