feat(run): S10 del 1 — run-lag: §9-citations, artefakt-persistens, SDK-klient
TDD offline (RØD bekreftet før implementasjon): resolve_model (rolle->modell-id, ukjent profil feiler fail-fast), build_citations (eksakte char-spans, verdict- ekskludering, uncitable kontekst -> raise FØR spend), persist_run_artifacts (deterministiske bytes; validator/checker-avgjørelser speilet VERBATIM fra RunResult — §9 non-konflatering). Run-path-only, aldri importert av tester: SdkModelClient (claude-agent-sdk 0.2.110 verifisert mot installert pakke; max_turns=1, tools=[], max_budget_usd per kall; manglende usage -> None så §8-meteret feiler lukket) + run_s10 (kontrakter FØR klient §10, BudgetExceeded som strukturert stopp). 178/178 uten nøkkel; ruff+mypy --strict rene; fire detach-bevis røde -> revertert grønne. Live-kjøringen gjenstår (credential-gated). Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01QdSfQdND84oeq2mbjueLTS
This commit is contained in:
parent
22bfc80dda
commit
0238507df4
5 changed files with 583 additions and 0 deletions
218
tests/test_s10_run_layer.py
Normal file
218
tests/test_s10_run_layer.py
Normal file
|
|
@ -0,0 +1,218 @@
|
|||
"""S10 run-layer seams, TDD offline (§9, §10, §11): model resolution, citations, artifacts.
|
||||
|
||||
The SDK client itself is run-path-only and NEVER imported here (the suite runs
|
||||
without an API key and without a network). What IS provable offline: the
|
||||
role -> model-id resolution against the startup contract, the §9 citation
|
||||
builder (fail-fast on uncitable context), and the artifact persistence layer —
|
||||
which must mirror the run's decisions VERBATIM (§9 non-conflation), never
|
||||
recompute them from the outcome.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from portfolio_optimiser_claude.artifacts import build_citations, persist_run_artifacts
|
||||
from portfolio_optimiser_claude.contracts import (
|
||||
ModelMapContract,
|
||||
TerminationContract,
|
||||
load_contracts,
|
||||
resolve_model,
|
||||
)
|
||||
from portfolio_optimiser_claude.ir import AffectedItem, SavingsProposal
|
||||
from portfolio_optimiser_claude.loop import RunResult
|
||||
from portfolio_optimiser_claude.okf import navigate_bundle
|
||||
from portfolio_optimiser_claude.provenance import Citation, Provenance
|
||||
from portfolio_optimiser_claude.validator import Rejection, ValidatedProposal
|
||||
|
||||
BUNDLE_DIR = Path(__file__).resolve().parent.parent / "shared" / "examples" / "bygg-energi-mikro"
|
||||
|
||||
|
||||
# --- role -> model id resolution (§10: config, fail-fast) -----------------------------------
|
||||
|
||||
|
||||
def _model_map() -> ModelMapContract:
|
||||
return ModelMapContract(
|
||||
profiles={"anthropic": {"default": "model-default", "proposer": "model-proposer"}}
|
||||
)
|
||||
|
||||
|
||||
def test_resolve_model_returns_role_entry() -> None:
|
||||
assert resolve_model(_model_map(), "proposer") == "model-proposer"
|
||||
|
||||
|
||||
def test_resolve_model_falls_back_to_default_for_unmapped_role() -> None:
|
||||
assert resolve_model(_model_map(), "checker") == "model-default"
|
||||
|
||||
|
||||
def test_resolve_model_unknown_profile_raises() -> None:
|
||||
with pytest.raises(ValueError):
|
||||
resolve_model(_model_map(), "proposer", profile="no-such-profile")
|
||||
|
||||
|
||||
def test_bundled_model_map_resolves_haiku_for_both_roles() -> None:
|
||||
# The S10 run path resolves from the BUNDLED map (D6: cheapest suitable model).
|
||||
contracts = load_contracts(
|
||||
data_source={"docs_dir": str(BUNDLE_DIR), "top_k": 3},
|
||||
termination={"max_rounds": 5, "max_tokens": 1000},
|
||||
feedback={"decision": "approved", "rationale": "startup shape check"},
|
||||
)
|
||||
for role in ("proposer", "checker"):
|
||||
assert resolve_model(contracts.model_map, role) == "claude-haiku-4-5-20251001"
|
||||
|
||||
|
||||
# --- §9 citations: at least one exact span into the source documents ------------------------
|
||||
|
||||
|
||||
def test_build_citations_from_micro_bundle_are_exact_spans() -> None:
|
||||
citations = build_citations(navigate_bundle(BUNDLE_DIR))
|
||||
assert citations, "the micro-bundle must yield at least one citation (§9)"
|
||||
for citation in citations:
|
||||
source = (BUNDLE_DIR / citation.file).read_text(encoding="utf-8")
|
||||
start, end = (int(part) for part in citation.span.removeprefix("chars ").split("-"))
|
||||
# The span is EXACT: the snippet is the literal text at those offsets.
|
||||
assert source[start:end] == citation.snippet
|
||||
|
||||
|
||||
def test_build_citations_excludes_verdict_files() -> None:
|
||||
cited_files = {citation.file for citation in build_citations(navigate_bundle(BUNDLE_DIR))}
|
||||
assert "verdict-led-fro.md" not in cited_files
|
||||
|
||||
|
||||
def test_build_citations_fails_fast_on_uncitable_context(tmp_path: Path) -> None:
|
||||
# §9: a run whose context yields no citable content MUST fail fast.
|
||||
(tmp_path / "index.md").write_text("---\ntype: index\n---\n", encoding="utf-8")
|
||||
with pytest.raises(ValueError):
|
||||
build_citations(navigate_bundle(tmp_path))
|
||||
|
||||
|
||||
# --- artifact persistence: decisions mirrored VERBATIM, deterministic bytes -----------------
|
||||
|
||||
|
||||
def _proposal() -> SavingsProposal:
|
||||
return SavingsProposal(
|
||||
project_id="bygg-kontor-nord",
|
||||
measure="LED-retrofit",
|
||||
affected_items=[AffectedItem(code="EL-01", quantity=100, unit_cost=250.0)],
|
||||
claimed_saving_nok=20000.0,
|
||||
)
|
||||
|
||||
|
||||
def _overridden_run() -> RunResult:
|
||||
# The §9 non-conflation shape: the validator PASSED the numbers, the checker
|
||||
# overrode with an explicit REJECT — the artifact must carry both, verbatim.
|
||||
return RunResult(
|
||||
outcome=Rejection(reason="unit cost unsupported (checker REJECT overrode)"),
|
||||
validator_decision="validated",
|
||||
checker_decision="reject",
|
||||
attempts=2,
|
||||
proposal=_proposal(),
|
||||
)
|
||||
|
||||
|
||||
def _provenance() -> Provenance:
|
||||
return Provenance(
|
||||
citations=[Citation(file="index.md", span="chars 0-5", snippet="Bygg-")],
|
||||
model="claude-haiku-4-5-20251001",
|
||||
role="proposer",
|
||||
validator_decision="validated",
|
||||
tokens_used=1234,
|
||||
)
|
||||
|
||||
|
||||
def _persist(out_dir: Path) -> dict[str, Path]:
|
||||
return persist_run_artifacts(
|
||||
out_dir,
|
||||
run=_overridden_run(),
|
||||
provenance=_provenance(),
|
||||
termination=TerminationContract(max_rounds=10, max_tokens=150000),
|
||||
tokens_used=1234,
|
||||
rounds_used=4,
|
||||
cost_usd=0.0567,
|
||||
)
|
||||
|
||||
|
||||
def test_persist_writes_the_four_artifacts(tmp_path: Path) -> None:
|
||||
paths = _persist(tmp_path / "out")
|
||||
assert set(paths) == {"proposal", "run_result", "provenance", "usage"}
|
||||
for path in paths.values():
|
||||
assert path.is_file()
|
||||
|
||||
|
||||
def test_persisted_proposal_round_trips_through_the_ir(tmp_path: Path) -> None:
|
||||
paths = _persist(tmp_path / "out")
|
||||
loaded = SavingsProposal.model_validate(
|
||||
json.loads(paths["proposal"].read_text(encoding="utf-8"))
|
||||
)
|
||||
assert loaded == _proposal()
|
||||
|
||||
|
||||
def test_persisted_decisions_mirror_the_run_verbatim(tmp_path: Path) -> None:
|
||||
# LOAD-BEARING (§9/§11): validator_decision and checker_decision are copied
|
||||
# from the RunResult — recomputing either from the (checker-overridden)
|
||||
# outcome would mislabel a validated proposal as validator-rejected.
|
||||
paths = _persist(tmp_path / "out")
|
||||
record = json.loads(paths["run_result"].read_text(encoding="utf-8"))
|
||||
assert record["validator_decision"] == "validated"
|
||||
assert record["checker_decision"] == "reject"
|
||||
assert record["attempts"] == 2
|
||||
assert record["outcome"] == {
|
||||
"type": "rejected",
|
||||
"reason": "unit cost unsupported (checker REJECT overrode)",
|
||||
}
|
||||
|
||||
|
||||
def test_persisted_validated_outcome_carries_percentiles(tmp_path: Path) -> None:
|
||||
run = RunResult(
|
||||
outcome=ValidatedProposal(
|
||||
validates=True,
|
||||
claimed_saving_nok=20000.0,
|
||||
nominal_feasible=25000.0,
|
||||
p10=18000.0,
|
||||
p50=22000.0,
|
||||
p90=27000.0,
|
||||
),
|
||||
validator_decision="validated",
|
||||
checker_decision="approve",
|
||||
attempts=1,
|
||||
proposal=_proposal(),
|
||||
)
|
||||
paths = persist_run_artifacts(
|
||||
tmp_path / "out",
|
||||
run=run,
|
||||
provenance=_provenance(),
|
||||
termination=TerminationContract(max_rounds=10, max_tokens=150000),
|
||||
tokens_used=1234,
|
||||
rounds_used=4,
|
||||
cost_usd=None,
|
||||
)
|
||||
record = json.loads(paths["run_result"].read_text(encoding="utf-8"))
|
||||
assert record["outcome"]["type"] == "validated"
|
||||
assert record["outcome"]["p50"] == 22000.0
|
||||
|
||||
|
||||
def test_persisted_provenance_matches_the_stamp(tmp_path: Path) -> None:
|
||||
paths = _persist(tmp_path / "out")
|
||||
assert json.loads(paths["provenance"].read_text(encoding="utf-8")) == _provenance().model_dump()
|
||||
|
||||
|
||||
def test_persisted_usage_carries_meter_caps_and_cost(tmp_path: Path) -> None:
|
||||
# §8 + D6: the artifact records real usage AGAINST the caps, and the cost.
|
||||
paths = _persist(tmp_path / "out")
|
||||
usage = json.loads(paths["usage"].read_text(encoding="utf-8"))
|
||||
assert usage == {
|
||||
"cost_usd": 0.0567,
|
||||
"max_rounds": 10,
|
||||
"max_tokens": 150000,
|
||||
"rounds_used": 4,
|
||||
"tokens_used": 1234,
|
||||
}
|
||||
|
||||
|
||||
def test_persistence_is_deterministic(tmp_path: Path) -> None:
|
||||
first = {name: path.read_bytes() for name, path in _persist(tmp_path / "a").items()}
|
||||
second = {name: path.read_bytes() for name, path in _persist(tmp_path / "b").items()}
|
||||
assert first == second
|
||||
Loading…
Add table
Add a link
Reference in a new issue