113 lines
5 KiB
Python
113 lines
5 KiB
Python
"""S4.1 preflight — unit + CLI tests: happy path, refusals, structured error handling, doc-guard.
|
|
|
|
Load-bearing detach seams (AST no-network/no-auto-login guard + refusal teeth) live in
|
|
``test_preflight_loadbearing.py``; these are the happy-path + behaviour tests.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from portfolio_optimiser import preflight
|
|
|
|
_VALID_ENDPOINT = "https://x.services.ai.azure.com/api/projects/p"
|
|
_VALID_MAP = {
|
|
"local": {"default": "qwen3:4b", "proposer": "qwen3:4b", "checker": "qwen3:4b"},
|
|
"azure": {"default": "gpt-4o-mini", "proposer": "gpt-4o-mini", "checker": "gpt-4o-mini"},
|
|
}
|
|
|
|
|
|
def _write_map(tmp_path: Path, data: dict) -> Path:
|
|
p = tmp_path / "model_map.json"
|
|
p.write_text(json.dumps(data), encoding="utf-8")
|
|
return p
|
|
|
|
|
|
def test_sc1_placeholder_refusal(
|
|
monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]
|
|
) -> None:
|
|
# SC1: bundled azure map ships REPLACE-WITH-* → refusal naming the deployment (endpoint valid,
|
|
# so the placeholder is the sole failure). Value-pinned per test_costsim.py:95-106.
|
|
monkeypatch.delenv("PORTFOLIO_MODEL_MAP", raising=False)
|
|
monkeypatch.setenv("PORTFOLIO_FOUNDRY_PROJECT_ENDPOINT", _VALID_ENDPOINT)
|
|
rc = preflight.main(["--profile", "azure"])
|
|
assert rc == 1
|
|
assert "REPLACE-WITH-FOUNDRY-DEPLOYMENT" in capsys.readouterr().err
|
|
|
|
|
|
def test_sc2_happy_path(
|
|
tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]
|
|
) -> None:
|
|
# SC2: complete valid override map + URL-shaped endpoint → rc 0, stdout carries the exact
|
|
# necessary-but-not-sufficient marker.
|
|
monkeypatch.setenv("PORTFOLIO_MODEL_MAP", str(_write_map(tmp_path, _VALID_MAP)))
|
|
monkeypatch.setenv("PORTFOLIO_FOUNDRY_PROJECT_ENDPOINT", _VALID_ENDPOINT)
|
|
rc = preflight.main(["--profile", "azure"])
|
|
assert rc == 0
|
|
assert "ikke tilstrekkelig" in capsys.readouterr().out
|
|
|
|
|
|
def test_sc3_missing_endpoint(
|
|
tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]
|
|
) -> None:
|
|
# SC3: unset endpoint → rc 1 + message names the env var.
|
|
monkeypatch.setenv("PORTFOLIO_MODEL_MAP", str(_write_map(tmp_path, _VALID_MAP)))
|
|
monkeypatch.delenv("PORTFOLIO_FOUNDRY_PROJECT_ENDPOINT", raising=False)
|
|
rc = preflight.main(["--profile", "azure"])
|
|
assert rc == 1
|
|
assert "PORTFOLIO_FOUNDRY_PROJECT_ENDPOINT" in capsys.readouterr().err
|
|
|
|
|
|
def test_wrong_host_surface_refused(
|
|
tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]
|
|
) -> None:
|
|
# A URL-valid but wrong-surface endpoint (*.openai.azure.com) is refused, naming the host.
|
|
monkeypatch.setenv("PORTFOLIO_MODEL_MAP", str(_write_map(tmp_path, _VALID_MAP)))
|
|
monkeypatch.setenv("PORTFOLIO_FOUNDRY_PROJECT_ENDPOINT", "https://x.openai.azure.com/foo")
|
|
rc = preflight.main(["--profile", "azure"])
|
|
assert rc == 1
|
|
assert "openai.azure.com" in capsys.readouterr().err
|
|
|
|
|
|
def test_bare_host_endpoint_accepted(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
# Research §3: official samples use the bare-host form (no /api/projects/) — must be accepted.
|
|
monkeypatch.setenv("PORTFOLIO_MODEL_MAP", str(_write_map(tmp_path, _VALID_MAP)))
|
|
monkeypatch.setenv("PORTFOLIO_FOUNDRY_PROJECT_ENDPOINT", "https://x.services.ai.azure.com")
|
|
assert isinstance(preflight.check_azure_preflight("azure"), preflight.PreflightOK)
|
|
|
|
|
|
def test_missing_override_file_is_structured_refusal(
|
|
monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]
|
|
) -> None:
|
|
# A bad PORTFOLIO_MODEL_MAP must NOT traceback — structured refusal, rc 1.
|
|
monkeypatch.setenv("PORTFOLIO_FOUNDRY_PROJECT_ENDPOINT", _VALID_ENDPOINT)
|
|
monkeypatch.setenv("PORTFOLIO_MODEL_MAP", "/nonexistent/model_map.json")
|
|
rc = preflight.main(["--profile", "azure"])
|
|
assert rc == 1
|
|
assert "model map not found" in capsys.readouterr().err
|
|
|
|
|
|
def test_malformed_override_is_structured_refusal(
|
|
tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]
|
|
) -> None:
|
|
# Override missing the azure block → ModelMapContract ValidationError → structured refusal.
|
|
bad = _write_map(tmp_path, {"local": {"default": "qwen3:4b"}})
|
|
monkeypatch.setenv("PORTFOLIO_FOUNDRY_PROJECT_ENDPOINT", _VALID_ENDPOINT)
|
|
monkeypatch.setenv("PORTFOLIO_MODEL_MAP", str(bad))
|
|
rc = preflight.main(["--profile", "azure"])
|
|
assert rc == 1
|
|
assert capsys.readouterr().err.strip()
|
|
|
|
|
|
def test_auth_recipe_doc_exists_and_names_facts() -> None:
|
|
# SC8 doc-guard: the verified recipe lives in docs/ (NOT the preflight docstring, so the
|
|
# AST grep-guard stays clean) and pins the load-bearing facts so it can't rot silently. The
|
|
# English "necessary-but-not-sufficient" marker is distinct from the CLI's Norwegian marker.
|
|
doc = Path(__file__).resolve().parents[1] / "docs" / "2026-07-15-foundry-auth-recipe.md"
|
|
text = doc.read_text(encoding="utf-8")
|
|
assert "Foundry User" in text
|
|
assert "services.ai.azure.com" in text
|
|
assert "necessary-but-not-sufficient" in text
|