"""S10 run-layer seams, TDD offline (§9, §10, §11): model resolution, citations, artifacts. The SDK client itself is run-path-only and NEVER imported here (the suite runs without an API key and without a network). What IS provable offline: the role -> model-id resolution against the startup contract, the §9 citation builder (fail-fast on uncitable context), and the artifact persistence layer — which must mirror the run's decisions VERBATIM (§9 non-conflation), never recompute them from the outcome. """ from __future__ import annotations import json from pathlib import Path import pytest from portfolio_optimiser_claude.artifacts import ( build_citations, persist_run_artifacts, persist_stop_artifacts, ) from portfolio_optimiser_claude.budget import BudgetExceeded from portfolio_optimiser_claude.contracts import ( ModelMapContract, TerminationContract, load_contracts, resolve_model, ) from portfolio_optimiser_claude.ir import AffectedItem, SavingsProposal from portfolio_optimiser_claude.loop import RunResult from portfolio_optimiser_claude.okf import navigate_bundle from portfolio_optimiser_claude.provenance import Citation, Provenance from portfolio_optimiser_claude.validator import Rejection, ValidatedProposal BUNDLE_DIR = Path(__file__).resolve().parent.parent / "shared" / "examples" / "bygg-energi-mikro" # --- role -> model id resolution (§10: config, fail-fast) ----------------------------------- def _model_map() -> ModelMapContract: return ModelMapContract( profiles={"anthropic": {"default": "model-default", "proposer": "model-proposer"}} ) def test_resolve_model_returns_role_entry() -> None: assert resolve_model(_model_map(), "proposer") == "model-proposer" def test_resolve_model_falls_back_to_default_for_unmapped_role() -> None: assert resolve_model(_model_map(), "checker") == "model-default" def test_resolve_model_unknown_profile_raises() -> None: with pytest.raises(ValueError): resolve_model(_model_map(), "proposer", profile="no-such-profile") def test_bundled_model_map_resolves_haiku_for_both_roles() -> None: # The S10 run path resolves from the BUNDLED map (D6: cheapest suitable model). contracts = load_contracts( data_source={"docs_dir": str(BUNDLE_DIR), "top_k": 3}, termination={"max_rounds": 5, "max_tokens": 1000}, feedback={"decision": "approved", "rationale": "startup shape check"}, ) for role in ("proposer", "checker"): assert resolve_model(contracts.model_map, role) == "claude-haiku-4-5-20251001" # --- §9 citations: at least one exact span into the source documents ------------------------ def test_build_citations_from_micro_bundle_are_exact_spans() -> None: citations = build_citations(navigate_bundle(BUNDLE_DIR)) assert citations, "the micro-bundle must yield at least one citation (§9)" for citation in citations: source = (BUNDLE_DIR / citation.file).read_text(encoding="utf-8") start, end = (int(part) for part in citation.span.removeprefix("chars ").split("-")) # The span is EXACT: the snippet is the literal text at those offsets. assert source[start:end] == citation.snippet def test_build_citations_excludes_verdict_files() -> None: cited_files = {citation.file for citation in build_citations(navigate_bundle(BUNDLE_DIR))} assert "verdict-led-fro.md" not in cited_files def test_build_citations_fails_fast_on_uncitable_context(tmp_path: Path) -> None: # §9: a run whose context yields no citable content MUST fail fast. (tmp_path / "index.md").write_text("---\ntype: index\n---\n", encoding="utf-8") with pytest.raises(ValueError): build_citations(navigate_bundle(tmp_path)) # --- artifact persistence: decisions mirrored VERBATIM, deterministic bytes ----------------- def _proposal() -> SavingsProposal: return SavingsProposal( project_id="bygg-kontor-nord", measure="LED-retrofit", affected_items=[AffectedItem(code="EL-01", quantity=100, unit_cost=250.0)], claimed_saving_nok=20000.0, ) def _overridden_run() -> RunResult: # The §9 non-conflation shape: the validator PASSED the numbers, the checker # overrode with an explicit REJECT — the artifact must carry both, verbatim. return RunResult( outcome=Rejection(reason="unit cost unsupported (checker REJECT overrode)"), validator_decision="validated", checker_decision="reject", attempts=2, proposal=_proposal(), ) def _provenance() -> Provenance: return Provenance( citations=[Citation(file="index.md", span="chars 0-5", snippet="Bygg-")], model="claude-haiku-4-5-20251001", role="proposer", validator_decision="validated", tokens_used=1234, ) def _persist(out_dir: Path) -> dict[str, Path]: return persist_run_artifacts( out_dir, run=_overridden_run(), provenance=_provenance(), termination=TerminationContract(max_rounds=10, max_tokens=150000), tokens_used=1234, rounds_used=4, cost_usd=0.0567, ) def test_persist_writes_the_four_artifacts(tmp_path: Path) -> None: paths = _persist(tmp_path / "out") assert set(paths) == {"proposal", "run_result", "provenance", "usage"} for path in paths.values(): assert path.is_file() def test_persisted_proposal_round_trips_through_the_ir(tmp_path: Path) -> None: paths = _persist(tmp_path / "out") loaded = SavingsProposal.model_validate( json.loads(paths["proposal"].read_text(encoding="utf-8")) ) assert loaded == _proposal() def test_persisted_decisions_mirror_the_run_verbatim(tmp_path: Path) -> None: # LOAD-BEARING (§9/§11): validator_decision and checker_decision are copied # from the RunResult — recomputing either from the (checker-overridden) # outcome would mislabel a validated proposal as validator-rejected. paths = _persist(tmp_path / "out") record = json.loads(paths["run_result"].read_text(encoding="utf-8")) assert record["validator_decision"] == "validated" assert record["checker_decision"] == "reject" assert record["attempts"] == 2 assert record["outcome"] == { "type": "rejected", "reason": "unit cost unsupported (checker REJECT overrode)", } def test_persisted_validated_outcome_carries_percentiles(tmp_path: Path) -> None: run = RunResult( outcome=ValidatedProposal( validates=True, claimed_saving_nok=20000.0, nominal_feasible=25000.0, p10=18000.0, p50=22000.0, p90=27000.0, ), validator_decision="validated", checker_decision="approve", attempts=1, proposal=_proposal(), ) paths = persist_run_artifacts( tmp_path / "out", run=run, provenance=_provenance(), termination=TerminationContract(max_rounds=10, max_tokens=150000), tokens_used=1234, rounds_used=4, cost_usd=None, ) record = json.loads(paths["run_result"].read_text(encoding="utf-8")) assert record["outcome"]["type"] == "validated" assert record["outcome"]["p50"] == 22000.0 def test_persisted_provenance_matches_the_stamp(tmp_path: Path) -> None: paths = _persist(tmp_path / "out") assert json.loads(paths["provenance"].read_text(encoding="utf-8")) == _provenance().model_dump() def test_persisted_usage_carries_meter_caps_and_cost(tmp_path: Path) -> None: # §8 + D6: the artifact records real usage AGAINST the caps, and the cost. paths = _persist(tmp_path / "out") usage = json.loads(paths["usage"].read_text(encoding="utf-8")) assert usage == { "cost_usd": 0.0567, "max_rounds": 10, "max_tokens": 150000, "rounds_used": 4, "tokens_used": 1234, } def test_persistence_is_deterministic(tmp_path: Path) -> None: first = {name: path.read_bytes() for name, path in _persist(tmp_path / "a").items()} second = {name: path.read_bytes() for name, path in _persist(tmp_path / "b").items()} assert first == second # --- stop artifacts: usage + cost persisted even when the budget stops the run --------------- # The S10 live run stopped structurally on the token cap (§8) and persisted # NOTHING — real spend without a record. A budget stop is a run outcome. def _persist_stop(out_dir: Path) -> dict[str, Path]: return persist_stop_artifacts( out_dir, stop=BudgetExceeded("tokens", limit=150_000, observed=162_250), termination=TerminationContract(max_rounds=12, max_tokens=150_000), tokens_used=162_250, rounds_used=5, cost_usd=0.331506, ) def test_budget_stop_persists_stop_event_and_usage(tmp_path: Path) -> None: paths = _persist_stop(tmp_path / "out") assert set(paths) == {"stop", "usage"} for path in paths.values(): assert path.is_file() def test_stop_artifact_mirrors_the_stop_event_verbatim(tmp_path: Path) -> None: # LOAD-BEARING (§8/§11): the artifact carries the breached kind, limit and # observed value exactly as raised — never recomputed, never summarised. paths = _persist_stop(tmp_path / "out") assert json.loads(paths["stop"].read_text(encoding="utf-8")) == { "kind": "tokens", "limit": 150_000, "observed": 162_250, } def test_stop_usage_artifact_has_the_completed_run_shape(tmp_path: Path) -> None: # S11 reads ONE usage format: the stop path must write the same shape # (usage against caps + cost) as the completed-run path. paths = _persist_stop(tmp_path / "out") usage = json.loads(paths["usage"].read_text(encoding="utf-8")) assert usage == { "cost_usd": 0.331506, "max_rounds": 12, "max_tokens": 150_000, "rounds_used": 5, "tokens_used": 162_250, } def test_stop_persistence_is_deterministic(tmp_path: Path) -> None: first = {name: path.read_bytes() for name, path in _persist_stop(tmp_path / "a").items()} second = {name: path.read_bytes() for name, path in _persist_stop(tmp_path / "b").items()} assert first == second