Kjell Tore Guttormsen
e7ce6b0a31
IR schema now refuses non-finite numbers (allow_inf_nan=False on quantity/ unit_cost/claimed_saving_nok) and non-finite or negative assumption-band endpoints; json.loads accepts the bare Infinity literal, so the bundle seam is tested directly. ModelMapContract rejects empty-string model ids (min_length=1). check_turn_safety_net documented as a deliberately unreachable belt under the range-bound debate loop. 18 new tests; detach-proven (re-allow inf/nan -> 5 red, drop min_length -> 2 red). Full gate: 365 passed, ruff/format/mypy clean; golden untouched. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
126 lines
4.8 KiB
Python
126 lines
4.8 KiB
Python
"""Fail-fast startup-contract tests (method-spec §10, §4.1, §8).
|
|
|
|
The run must refuse to start on a bad config: missing/non-positive caps, a third
|
|
decision value on the run path, a profile without a ``default`` model. All of this
|
|
is pure config-layer validation — no model client, no API key, no network.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from typing import Any
|
|
|
|
import pytest
|
|
from pydantic import ValidationError
|
|
|
|
from portfolio_optimiser_claude.contracts import load_contracts
|
|
|
|
VALID_DATA_SOURCE: dict[str, Any] = {"docs_dir": "docs", "top_k": 3}
|
|
VALID_TERMINATION: dict[str, Any] = {"max_rounds": 4, "max_tokens": 20_000}
|
|
VALID_FEEDBACK: dict[str, Any] = {"decision": "approved", "rationale": "sound measure"}
|
|
VALID_MODEL_MAP: dict[str, Any] = {
|
|
"profiles": {"anthropic": {"default": "some-model-id", "proposer": "some-model-id"}}
|
|
}
|
|
|
|
|
|
def load(**overrides: dict[str, Any]) -> Any:
|
|
kwargs: dict[str, Any] = {
|
|
"data_source": VALID_DATA_SOURCE,
|
|
"termination": VALID_TERMINATION,
|
|
"feedback": VALID_FEEDBACK,
|
|
"model_map": VALID_MODEL_MAP,
|
|
}
|
|
kwargs.update(overrides)
|
|
return load_contracts(
|
|
kwargs["data_source"],
|
|
kwargs["termination"],
|
|
kwargs["feedback"],
|
|
model_map=kwargs["model_map"],
|
|
)
|
|
|
|
|
|
class TestTermination:
|
|
"""§8: required at startup, positive caps, never an unbounded loop."""
|
|
|
|
def test_missing_max_tokens_rejected(self) -> None:
|
|
with pytest.raises(ValidationError):
|
|
load(termination={"max_rounds": 4})
|
|
|
|
def test_missing_max_rounds_rejected(self) -> None:
|
|
with pytest.raises(ValidationError):
|
|
load(termination={"max_tokens": 20_000})
|
|
|
|
@pytest.mark.parametrize("bad", [0, -1])
|
|
def test_nonpositive_caps_rejected(self, bad: int) -> None:
|
|
with pytest.raises(ValidationError):
|
|
load(termination={"max_rounds": bad, "max_tokens": 20_000})
|
|
with pytest.raises(ValidationError):
|
|
load(termination={"max_rounds": 4, "max_tokens": bad})
|
|
|
|
|
|
class TestFeedback:
|
|
"""§4.1: run-path decision is BINARY — exactly two values."""
|
|
|
|
@pytest.mark.parametrize("decision", ["approved", "rejected"])
|
|
def test_binary_decisions_accepted(self, decision: str) -> None:
|
|
contracts = load(feedback={"decision": decision, "rationale": "why"})
|
|
assert contracts.feedback.decision == decision
|
|
|
|
def test_third_decision_value_rejected(self) -> None:
|
|
# approved_with_adjustment lives ONLY in bundle-seed frontmatter and the
|
|
# promotion gate's accepted set — the run path must reject it (§4).
|
|
with pytest.raises(ValidationError):
|
|
load(feedback={"decision": "approved_with_adjustment", "rationale": "why"})
|
|
|
|
def test_empty_rationale_rejected(self) -> None:
|
|
with pytest.raises(ValidationError):
|
|
load(feedback={"decision": "approved", "rationale": ""})
|
|
|
|
|
|
class TestDataSource:
|
|
"""§10: a docs directory + a positive top-k."""
|
|
|
|
def test_empty_docs_dir_rejected(self) -> None:
|
|
with pytest.raises(ValidationError):
|
|
load(data_source={"docs_dir": "", "top_k": 3})
|
|
|
|
def test_nonpositive_top_k_rejected(self) -> None:
|
|
with pytest.raises(ValidationError):
|
|
load(data_source={"docs_dir": "docs", "top_k": 0})
|
|
|
|
|
|
class TestModelMap:
|
|
"""§10: role -> model id per backend profile, each profile REQUIRING a default."""
|
|
|
|
def test_profile_without_default_rejected(self) -> None:
|
|
with pytest.raises(ValidationError):
|
|
load(model_map={"profiles": {"anthropic": {"proposer": "some-model-id"}}})
|
|
|
|
def test_empty_profiles_rejected(self) -> None:
|
|
with pytest.raises(ValidationError):
|
|
load(model_map={"profiles": {}})
|
|
|
|
def test_empty_string_model_id_rejected(self) -> None:
|
|
# C2.6: an empty-string model id satisfied dict[str, str] and slipped
|
|
# through to the client layer — it is a startup schema error.
|
|
with pytest.raises(ValidationError):
|
|
load(model_map={"profiles": {"anthropic": {"default": ""}}})
|
|
|
|
def test_empty_string_role_model_id_rejected(self) -> None:
|
|
with pytest.raises(ValidationError):
|
|
load(
|
|
model_map={"profiles": {"anthropic": {"default": "some-model-id", "proposer": ""}}}
|
|
)
|
|
|
|
def test_bundled_model_map_is_valid(self) -> None:
|
|
# model_map=None falls back to the bundled data/model_map.json, which must
|
|
# itself satisfy the contract (fail-fast on the shipped config too).
|
|
contracts = load(model_map=None) # type: ignore[arg-type]
|
|
assert "default" in contracts.model_map.profiles["anthropic"]
|
|
|
|
|
|
def test_happy_path_validates_all_four() -> None:
|
|
contracts = load()
|
|
assert contracts.data_source.top_k == 3
|
|
assert contracts.termination.max_rounds == 4
|
|
assert contracts.feedback.decision == "approved"
|
|
assert "anthropic" in contracts.model_map.profiles
|