portfolio-optimiser-claude/tests/test_s10_run_layer.py

"""S10 run-layer seams, TDD offline (§9, §10, §11): model resolution, citations, artifacts.

The SDK client itself is run-path-only and NEVER imported here (the suite runs
without an API key and without a network). What IS provable offline: the
role -> model-id resolution against the startup contract, the §9 citation
builder (fail-fast on uncitable context), and the artifact persistence layer —
which must mirror the run's decisions VERBATIM (§9 non-conflation), never
recompute them from the outcome.
"""

from __future__ import annotations

import json
from pathlib import Path

import pytest

from portfolio_optimiser_claude.artifacts import (
    build_citations,
    persist_run_artifacts,
    persist_stop_artifacts,
)
from portfolio_optimiser_claude.budget import BudgetExceeded
from portfolio_optimiser_claude.contracts import (
    ModelMapContract,
    TerminationContract,
    load_contracts,
    resolve_model,
)
from portfolio_optimiser_claude.ir import AffectedItem, SavingsProposal
from portfolio_optimiser_claude.loop import RunResult
from portfolio_optimiser_claude.okf import navigate_bundle
from portfolio_optimiser_claude.provenance import Citation, Provenance
from portfolio_optimiser_claude.validator import Rejection, ValidatedProposal

BUNDLE_DIR = Path(__file__).resolve().parent.parent / "shared" / "examples" / "bygg-energi-mikro"


# --- role -> model id resolution (§10: config, fail-fast) -----------------------------------


def _model_map() -> ModelMapContract:
    return ModelMapContract(
        profiles={"anthropic": {"default": "model-default", "proposer": "model-proposer"}}
    )


def test_resolve_model_returns_role_entry() -> None:
    assert resolve_model(_model_map(), "proposer") == "model-proposer"


def test_resolve_model_falls_back_to_default_for_unmapped_role() -> None:
    assert resolve_model(_model_map(), "checker") == "model-default"


def test_resolve_model_unknown_profile_raises() -> None:
    with pytest.raises(ValueError):
        resolve_model(_model_map(), "proposer", profile="no-such-profile")


def test_bundled_model_map_resolves_haiku_for_both_roles() -> None:
    # The S10 run path resolves from the BUNDLED map (D6: cheapest suitable model).
    contracts = load_contracts(
        data_source={"docs_dir": str(BUNDLE_DIR), "top_k": 3},
        termination={"max_rounds": 5, "max_tokens": 1000},
        feedback={"decision": "approved", "rationale": "startup shape check"},
    )
    for role in ("proposer", "checker"):
        assert resolve_model(contracts.model_map, role) == "claude-haiku-4-5-20251001"


# --- §9 citations: at least one exact span into the source documents ------------------------


def test_build_citations_from_micro_bundle_are_exact_spans() -> None:
    citations = build_citations(navigate_bundle(BUNDLE_DIR))
    assert citations, "the micro-bundle must yield at least one citation (§9)"
    for citation in citations:
        source = (BUNDLE_DIR / citation.file).read_text(encoding="utf-8")
        start, end = (int(part) for part in citation.span.removeprefix("chars ").split("-"))
        # The span is EXACT: the snippet is the literal text at those offsets.
        assert source[start:end] == citation.snippet


def test_build_citations_excludes_verdict_files() -> None:
    concepts = navigate_bundle(BUNDLE_DIR)
    # Positive controls: the verdict file IS navigated (so the exclusion below does real
    # work rather than never seeing the file), and citations ARE produced — an empty set
    # would satisfy the negative without excluding anything.
    assert "verdict-led-fro.md" in {concept.path.name for concept in concepts}
    cited_files = {citation.file for citation in build_citations(concepts)}
    assert cited_files
    assert "verdict-led-fro.md" not in cited_files


def test_build_citations_fails_fast_on_uncitable_context(tmp_path: Path) -> None:
    # §9: a run whose context yields no citable content MUST fail fast.
    (tmp_path / "index.md").write_text("---\ntype: index\n---\n", encoding="utf-8")
    with pytest.raises(ValueError):
        build_citations(navigate_bundle(tmp_path))


# --- artifact persistence: decisions mirrored VERBATIM, deterministic bytes -----------------


def _proposal() -> SavingsProposal:
    return SavingsProposal(
        project_id="bygg-kontor-nord",
        measure="LED-retrofit",
        affected_items=[AffectedItem(code="EL-01", quantity=100, unit_cost=250.0)],
        claimed_saving_nok=20000.0,
    )


def _overridden_run() -> RunResult:
    # The §9 non-conflation shape: the validator PASSED the numbers, the checker
    # overrode with an explicit REJECT — the artifact must carry both, verbatim.
    return RunResult(
        outcome=Rejection(reason="unit cost unsupported (checker REJECT overrode)"),
        validator_decision="validated",
        checker_decision="reject",
        attempts=2,
        proposal=_proposal(),
    )


def _provenance() -> Provenance:
    return Provenance(
        citations=[Citation(file="index.md", span="chars 0-5", snippet="Bygg-")],
        model="claude-haiku-4-5-20251001",
        role="proposer",
        validator_decision="validated",
        tokens_used=1234,
    )


def _persist(out_dir: Path) -> dict[str, Path]:
    return persist_run_artifacts(
        out_dir,
        run=_overridden_run(),
        provenance=_provenance(),
        termination=TerminationContract(max_rounds=10, max_tokens=150000),
        tokens_used=1234,
        rounds_used=4,
        cost_usd=0.0567,
    )


def test_persist_writes_the_four_artifacts(tmp_path: Path) -> None:
    paths = _persist(tmp_path / "out")
    assert set(paths) == {"proposal", "run_result", "provenance", "usage"}
    for path in paths.values():
        assert path.is_file()


def test_persisted_proposal_round_trips_through_the_ir(tmp_path: Path) -> None:
    paths = _persist(tmp_path / "out")
    loaded = SavingsProposal.model_validate(
        json.loads(paths["proposal"].read_text(encoding="utf-8"))
    )
    assert loaded == _proposal()


def test_persisted_decisions_mirror_the_run_verbatim(tmp_path: Path) -> None:
    # LOAD-BEARING (§9/§11): validator_decision and checker_decision are copied
    # from the RunResult — recomputing either from the (checker-overridden)
    # outcome would mislabel a validated proposal as validator-rejected.
    paths = _persist(tmp_path / "out")
    record = json.loads(paths["run_result"].read_text(encoding="utf-8"))
    assert record["validator_decision"] == "validated"
    assert record["checker_decision"] == "reject"
    assert record["attempts"] == 2
    assert record["outcome"] == {
        "type": "rejected",
        "reason": "unit cost unsupported (checker REJECT overrode)",
    }


def test_persisted_validated_outcome_carries_percentiles(tmp_path: Path) -> None:
    run = RunResult(
        outcome=ValidatedProposal(
            validates=True,
            claimed_saving_nok=20000.0,
            nominal_feasible=25000.0,
            p10=18000.0,
            p50=22000.0,
            p90=27000.0,
        ),
        validator_decision="validated",
        checker_decision="approve",
        attempts=1,
        proposal=_proposal(),
    )
    paths = persist_run_artifacts(
        tmp_path / "out",
        run=run,
        provenance=_provenance(),
        termination=TerminationContract(max_rounds=10, max_tokens=150000),
        tokens_used=1234,
        rounds_used=4,
        cost_usd=None,
    )
    record = json.loads(paths["run_result"].read_text(encoding="utf-8"))
    assert record["outcome"]["type"] == "validated"
    assert record["outcome"]["p50"] == 22000.0


def test_persisted_provenance_matches_the_stamp(tmp_path: Path) -> None:
    paths = _persist(tmp_path / "out")
    assert json.loads(paths["provenance"].read_text(encoding="utf-8")) == _provenance().model_dump()


def test_persisted_usage_carries_meter_caps_and_cost(tmp_path: Path) -> None:
    # §8 + D6: the artifact records real usage AGAINST the caps, and the cost.
    paths = _persist(tmp_path / "out")
    usage = json.loads(paths["usage"].read_text(encoding="utf-8"))
    assert usage == {
        "cost_usd": 0.0567,
        "max_rounds": 10,
        "max_tokens": 150000,
        "rounds_used": 4,
        "tokens_used": 1234,
    }


def test_persistence_is_deterministic(tmp_path: Path) -> None:
    first = {name: path.read_bytes() for name, path in _persist(tmp_path / "a").items()}
    second = {name: path.read_bytes() for name, path in _persist(tmp_path / "b").items()}
    assert first == second


# --- stop artifacts: usage + cost persisted even when the budget stops the run ---------------
# The S10 live run stopped structurally on the token cap (§8) and persisted
# NOTHING — real spend without a record. A budget stop is a run outcome.


def _persist_stop(out_dir: Path) -> dict[str, Path]:
    return persist_stop_artifacts(
        out_dir,
        stop=BudgetExceeded("tokens", limit=150_000, observed=162_250),
        termination=TerminationContract(max_rounds=12, max_tokens=150_000),
        tokens_used=162_250,
        rounds_used=5,
        cost_usd=0.331506,
    )


def test_budget_stop_persists_stop_event_and_usage(tmp_path: Path) -> None:
    paths = _persist_stop(tmp_path / "out")
    assert set(paths) == {"stop", "usage"}
    for path in paths.values():
        assert path.is_file()


def test_stop_artifact_mirrors_the_stop_event_verbatim(tmp_path: Path) -> None:
    # LOAD-BEARING (§8/§11): the artifact carries the breached kind, limit and
    # observed value exactly as raised — never recomputed, never summarised.
    paths = _persist_stop(tmp_path / "out")
    assert json.loads(paths["stop"].read_text(encoding="utf-8")) == {
        "kind": "tokens",
        "limit": 150_000,
        "observed": 162_250,
    }


def test_stop_usage_artifact_has_the_completed_run_shape(tmp_path: Path) -> None:
    # S11 reads ONE usage format: the stop path must write the same shape
    # (usage against caps + cost) as the completed-run path.
    paths = _persist_stop(tmp_path / "out")
    usage = json.loads(paths["usage"].read_text(encoding="utf-8"))
    assert usage == {
        "cost_usd": 0.331506,
        "max_rounds": 12,
        "max_tokens": 150_000,
        "rounds_used": 5,
        "tokens_used": 162_250,
    }


def test_stop_persistence_is_deterministic(tmp_path: Path) -> None:
    first = {name: path.read_bytes() for name, path in _persist_stop(tmp_path / "a").items()}
    second = {name: path.read_bytes() for name, path in _persist_stop(tmp_path / "b").items()}
    assert first == second