feat(run): S10 del 1 — run-lag: §9-citations, artefakt-persistens, SDK-klient

TDD offline (RØD bekreftet før implementasjon): resolve_model (rolle->modell-id,
ukjent profil feiler fail-fast), build_citations (eksakte char-spans, verdict-
ekskludering, uncitable kontekst -> raise FØR spend), persist_run_artifacts
(deterministiske bytes; validator/checker-avgjørelser speilet VERBATIM fra
RunResult — §9 non-konflatering). Run-path-only, aldri importert av tester:
SdkModelClient (claude-agent-sdk 0.2.110 verifisert mot installert pakke;
max_turns=1, tools=[], max_budget_usd per kall; manglende usage -> None så
§8-meteret feiler lukket) + run_s10 (kontrakter FØR klient §10, BudgetExceeded
som strukturert stopp). 178/178 uten nøkkel; ruff+mypy --strict rene; fire
detach-bevis røde -> revertert grønne. Live-kjøringen gjenstår (credential-gated).

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01QdSfQdND84oeq2mbjueLTS
This commit is contained in:
Kjell Tore Guttormsen 2026-07-03 07:51:05 +02:00
commit 0238507df4
5 changed files with 583 additions and 0 deletions

View file

@ -0,0 +1,114 @@
"""The S10 output layer: §9 citations + deterministic run-artifact persistence.
``build_citations`` cites the navigated read-context (never ``type: verdict``
files the verdict layer must not leak, §3 Step 1) with EXACT character spans
into the source files; a context that yields no citable content fails fast (§9).
``persist_run_artifacts`` writes the captured run for S11: the decisions are
mirrored VERBATIM from the ``RunResult`` (§9 non-conflation never recomputed
from a checker-overridden outcome), and the bytes are deterministic (sorted
keys, 2-space indent the house JSON convention).
Pure file layer by design imports no agent toolkit; the SDK client stays
run-path-only (§11: the suite runs without a key).
"""
from __future__ import annotations
import json
from pathlib import Path
from typing import Any
from portfolio_optimiser_claude.contracts import TerminationContract
from portfolio_optimiser_claude.loop import RunResult
from portfolio_optimiser_claude.okf import ConceptFile
from portfolio_optimiser_claude.provenance import Citation, Provenance
from portfolio_optimiser_claude.validator import Rejection
_VERDICT_TYPE = "verdict"
def build_citations(concepts: list[ConceptFile]) -> list[Citation]:
"""One citation per citable concept: file + exact char span + snippet (§9).
The snippet is the concept body's first non-empty line; the span locates it
verbatim in the SOURCE file (frontmatter included), so every citation is
independently verifiable. Verdict files and empty bodies are skipped; a
context with no citable content raises (fail fast, §9).
"""
citations: list[Citation] = []
for concept in concepts:
if concept.type == _VERDICT_TYPE or not concept.body:
continue
snippet = next((line for line in concept.body.splitlines() if line.strip()), "")
if not snippet:
continue
source = concept.path.read_text(encoding="utf-8")
start = source.find(snippet)
if start < 0:
continue
citations.append(
Citation(
file=concept.path.name,
span=f"chars {start}-{start + len(snippet)}",
snippet=snippet,
)
)
if not citations:
raise ValueError("context yields no citable content — a run must fail fast (§9)")
return citations
def _dump_json(path: Path, payload: dict[str, Any]) -> None:
path.write_text(json.dumps(payload, sort_keys=True, indent=2) + "\n", encoding="utf-8")
def persist_run_artifacts(
out_dir: Path,
*,
run: RunResult,
provenance: Provenance,
termination: TerminationContract,
tokens_used: int,
rounds_used: int,
cost_usd: float | None,
) -> dict[str, Path]:
"""Persist the run for S11: proposal, result, provenance stamp, usage-vs-caps.
``validator_decision`` and ``checker_decision`` are copied from the
``RunResult`` fields the two falsifiers were recorded separately there
(§9) and recomputing either from the outcome would conflate them.
"""
out_dir.mkdir(parents=True, exist_ok=True)
outcome: dict[str, Any] = (
{"type": "rejected", "reason": run.outcome.reason}
if isinstance(run.outcome, Rejection)
else {"type": "validated", **run.outcome.model_dump()}
)
paths = {
"proposal": out_dir / "proposal.json",
"run_result": out_dir / "run_result.json",
"provenance": out_dir / "provenance.json",
"usage": out_dir / "usage.json",
}
_dump_json(paths["proposal"], run.proposal.model_dump())
_dump_json(
paths["run_result"],
{
"validator_decision": run.validator_decision,
"checker_decision": run.checker_decision,
"attempts": run.attempts,
"outcome": outcome,
},
)
_dump_json(paths["provenance"], provenance.model_dump())
_dump_json(
paths["usage"],
{
"tokens_used": tokens_used,
"rounds_used": rounds_used,
"max_tokens": termination.max_tokens,
"max_rounds": termination.max_rounds,
"cost_usd": cost_usd,
},
)
return paths

View file

@ -73,6 +73,19 @@ class Contracts(BaseModel):
feedback: FeedbackContract
def resolve_model(model_map: ModelMapContract, role: str, *, profile: str = "anthropic") -> str:
"""Resolve role -> model id from the validated map; unmapped roles use ``default``.
The profile itself is config (§10) an unknown profile is a startup error,
never a silent fallback.
"""
mapping = model_map.profiles.get(profile)
if mapping is None:
raise ValueError(f"model_map has no profile '{profile}'")
# Contract validation (§10) guarantees every profile carries a 'default'.
return mapping.get(role, mapping["default"])
def _bundled_model_map() -> dict[str, Any]:
raw: dict[str, Any] = json.loads(
files("portfolio_optimiser_claude")

View file

@ -0,0 +1,133 @@
"""S10 — the ONE live model run of the programme (D6). RUN-PATH ONLY.
Wires the offline-proven pipeline (S6S9) to the real Claude Agent SDK client
for a single controlled run on the micro-bundle: startup contracts validated
fail-fast BEFORE the client is built (§10), citations proven citable BEFORE any
spend (§9), the read-context navigated + experience-folded exactly as the
offline suite proves (§3 Step 1), the loop bounded by the §8 meter AND the
per-call USD cap, and the captured artifacts persisted for S11 with the cost
logged. Never imported by the test suite.
Run: uv run python -m portfolio_optimiser_claude.run_s10
"""
from __future__ import annotations
import argparse
import os
from pathlib import Path
from portfolio_optimiser_claude.artifacts import build_citations, persist_run_artifacts
from portfolio_optimiser_claude.budget import BudgetExceeded, BudgetMeter
from portfolio_optimiser_claude.contracts import load_contracts
from portfolio_optimiser_claude.experience import (
CandidateFeatures,
VerdictStore,
fold_experience,
seed_store_from_bundle,
)
from portfolio_optimiser_claude.ir import load_validator_input
from portfolio_optimiser_claude.loop import run_project
from portfolio_optimiser_claude.okf import bundle_context, navigate_bundle
from portfolio_optimiser_claude.provenance import Provenance
from portfolio_optimiser_claude.sdk_client import SdkModelClient
from portfolio_optimiser_claude.validator import Rejection
_DEFAULT_BUNDLE = Path(__file__).resolve().parents[2] / "shared" / "examples" / "bygg-energi-mikro"
_PROPOSER_ROLE = "proposer"
def main(argv: list[str] | None = None) -> int:
parser = argparse.ArgumentParser(description="S10: the single live run (hard caps, D6).")
parser.add_argument("--bundle", type=Path, default=_DEFAULT_BUNDLE)
parser.add_argument("--out", type=Path, default=Path("runs") / "s10")
parser.add_argument("--max-rounds", type=int, default=12)
parser.add_argument("--max-tokens", type=int, default=150_000)
parser.add_argument("--max-budget-usd-per-call", type=float, default=0.25)
parser.add_argument("--max-debate-rounds", type=int, default=3)
parser.add_argument("--max-attempts", type=int, default=3)
parser.add_argument("--top-k", type=int, default=3)
args = parser.parse_args(argv)
if not os.environ.get("ANTHROPIC_API_KEY"):
# The bundled CLI resolves its own credentials (Claude Code login) when
# no key is exported; an unauthenticated run fails on the FIRST call.
print("note: ANTHROPIC_API_KEY not set — relying on the CLI's own credentials.")
# §10: ALL startup contracts schema-validated BEFORE any model client exists.
contracts = load_contracts(
data_source={"docs_dir": str(args.bundle), "top_k": args.top_k},
termination={"max_rounds": args.max_rounds, "max_tokens": args.max_tokens},
feedback={"decision": "approved", "rationale": "startup shape check (§10)"},
)
# §9: prove the context citable BEFORE any spend; §3 Step 1: navigate + fold.
concepts = navigate_bundle(args.bundle)
citations = build_citations(concepts)
ir_projection = load_validator_input(args.bundle)
store = VerdictStore()
seeded = seed_store_from_bundle(store, args.bundle)
context = fold_experience(
store,
CandidateFeatures.from_proposal(ir_projection),
bundle_context(args.bundle),
contracts.data_source.top_k,
)
meter = BudgetMeter(contracts.termination)
client = SdkModelClient(
contracts.model_map, max_budget_usd_per_call=args.max_budget_usd_per_call
)
print(
f"S10 live run: bundle={args.bundle.name} seeded_verdicts={seeded} "
f"caps: max_rounds={args.max_rounds} max_tokens={args.max_tokens} "
f"max_budget_usd_per_call={args.max_budget_usd_per_call}"
)
try:
result = run_project(
client,
context,
meter=meter,
max_debate_rounds=args.max_debate_rounds,
max_attempts=args.max_attempts,
default_project_id=ir_projection.project_id,
)
except BudgetExceeded as stop:
# §8: the structured stop event — report it, never a silent hang.
print(f"STOPPED by budget: {stop.kind} observed {stop.observed} > limit {stop.limit}")
print(f"cost so far: {client.total_cost_usd:.6f} USD")
return 3
provenance = Provenance(
citations=citations,
model=client.last_model or "unknown", # §9: the REAL id, neutral fallback
role=_PROPOSER_ROLE,
validator_decision=result.validator_decision,
tokens_used=meter.tokens_used,
)
paths = persist_run_artifacts(
args.out,
run=result,
provenance=provenance,
termination=contracts.termination,
tokens_used=meter.tokens_used,
rounds_used=meter.rounds_used,
cost_usd=round(client.total_cost_usd, 6),
)
outcome_kind = "rejected" if isinstance(result.outcome, Rejection) else "validated"
print(
f"result: validator={result.validator_decision} checker={result.checker_decision} "
f"attempts={result.attempts} outcome={outcome_kind}"
)
print(
f"usage: tokens={meter.tokens_used}/{contracts.termination.max_tokens} "
f"rounds={meter.rounds_used}/{contracts.termination.max_rounds}"
)
print(f"model: {client.last_model} cost: {client.total_cost_usd:.6f} USD")
for name, path in sorted(paths.items()):
print(f"artifact: {name} -> {path}")
return 0
if __name__ == "__main__":
raise SystemExit(main())

View file

@ -0,0 +1,105 @@
"""The Claude Agent SDK model client — RUN-PATH ONLY (honesty rule §1, §11).
This module is the ONE place the programme touches a real model (S10, D6). It
is never imported by the test suite the offline suite proves the loop with
the scripted stand-in, and this client slots into the same ``ModelClient``
protocol seam. Each ``complete()`` is one bounded ``query()`` call: no tools,
one turn, a first-class USD cap (``ClaudeAgentOptions.max_budget_usd``) ON TOP
of the §8 token/round meter that the loop already charges.
Verified against claude-agent-sdk 0.2.110: ``query()`` yields
``AssistantMessage`` (text blocks + real model id) and a closing
``ResultMessage`` (provider-reported ``usage`` + ``total_cost_usd``). A reply
without usage is passed through as ``None`` so the meter fails CLOSED (§8)
this client never invents a count.
"""
from __future__ import annotations
import asyncio
from typing import Any
from claude_agent_sdk import (
AssistantMessage,
ClaudeAgentOptions,
ResultMessage,
TextBlock,
query,
)
from portfolio_optimiser_claude.contracts import ModelMapContract, resolve_model
from portfolio_optimiser_claude.loop import ModelReply
# The provider-reported usage fields that make up the TOTAL token count (§8).
_USAGE_TOKEN_FIELDS = (
"input_tokens",
"output_tokens",
"cache_creation_input_tokens",
"cache_read_input_tokens",
)
def _total_tokens(usage: dict[str, Any] | None) -> int | None:
"""Sum the provider-reported token fields; no usage stays ``None`` (§8)."""
if usage is None:
return None
counts = [usage[field] for field in _USAGE_TOKEN_FIELDS if isinstance(usage.get(field), int)]
return sum(counts) if counts else None
class SdkModelClient:
"""``ModelClient`` over ``claude_agent_sdk.query()`` — one bounded call each.
``total_cost_usd`` accumulates the provider-reported cost across calls so
the run can log it (D6); ``last_model`` carries the REAL model id from the
latest reply for the §9 provenance stamp.
"""
def __init__(
self,
model_map: ModelMapContract,
*,
profile: str = "anthropic",
max_budget_usd_per_call: float = 0.25,
) -> None:
if max_budget_usd_per_call <= 0:
raise ValueError(
f"max_budget_usd_per_call must be positive, got {max_budget_usd_per_call}"
)
self._model_map = model_map
self._profile = profile
self._max_budget_usd_per_call = max_budget_usd_per_call
self.total_cost_usd = 0.0
self.last_model: str | None = None
def complete(self, prompt: str, *, role: str) -> ModelReply:
model_id = resolve_model(self._model_map, role, profile=self._profile)
return asyncio.run(self._complete_async(prompt, model_id))
async def _complete_async(self, prompt: str, model_id: str) -> ModelReply:
options = ClaudeAgentOptions(
model=model_id,
max_turns=1, # a single completion — the agentic loop lives in loop.py, not here
tools=[], # pure text completion: no tool surface, no silent egress
max_budget_usd=self._max_budget_usd_per_call,
)
text_parts: list[str] = []
reply_model: str | None = None
usage_tokens: int | None = None
async for message in query(prompt=prompt, options=options):
if isinstance(message, AssistantMessage):
if message.error is not None:
raise RuntimeError(f"model call failed: {message.error}")
reply_model = message.model
text_parts.extend(
block.text for block in message.content if isinstance(block, TextBlock)
)
elif isinstance(message, ResultMessage):
if message.is_error:
raise RuntimeError(f"model call failed: {message.subtype}: {message.errors}")
usage_tokens = _total_tokens(message.usage)
if message.total_cost_usd is not None:
self.total_cost_usd += message.total_cost_usd
if reply_model is not None:
self.last_model = reply_model
return ModelReply(text="".join(text_parts), usage_tokens=usage_tokens, model=reply_model)

218
tests/test_s10_run_layer.py Normal file
View file

@ -0,0 +1,218 @@
"""S10 run-layer seams, TDD offline (§9, §10, §11): model resolution, citations, artifacts.
The SDK client itself is run-path-only and NEVER imported here (the suite runs
without an API key and without a network). What IS provable offline: the
role -> model-id resolution against the startup contract, the §9 citation
builder (fail-fast on uncitable context), and the artifact persistence layer
which must mirror the run's decisions VERBATIM (§9 non-conflation), never
recompute them from the outcome.
"""
from __future__ import annotations
import json
from pathlib import Path
import pytest
from portfolio_optimiser_claude.artifacts import build_citations, persist_run_artifacts
from portfolio_optimiser_claude.contracts import (
ModelMapContract,
TerminationContract,
load_contracts,
resolve_model,
)
from portfolio_optimiser_claude.ir import AffectedItem, SavingsProposal
from portfolio_optimiser_claude.loop import RunResult
from portfolio_optimiser_claude.okf import navigate_bundle
from portfolio_optimiser_claude.provenance import Citation, Provenance
from portfolio_optimiser_claude.validator import Rejection, ValidatedProposal
BUNDLE_DIR = Path(__file__).resolve().parent.parent / "shared" / "examples" / "bygg-energi-mikro"
# --- role -> model id resolution (§10: config, fail-fast) -----------------------------------
def _model_map() -> ModelMapContract:
return ModelMapContract(
profiles={"anthropic": {"default": "model-default", "proposer": "model-proposer"}}
)
def test_resolve_model_returns_role_entry() -> None:
assert resolve_model(_model_map(), "proposer") == "model-proposer"
def test_resolve_model_falls_back_to_default_for_unmapped_role() -> None:
assert resolve_model(_model_map(), "checker") == "model-default"
def test_resolve_model_unknown_profile_raises() -> None:
with pytest.raises(ValueError):
resolve_model(_model_map(), "proposer", profile="no-such-profile")
def test_bundled_model_map_resolves_haiku_for_both_roles() -> None:
# The S10 run path resolves from the BUNDLED map (D6: cheapest suitable model).
contracts = load_contracts(
data_source={"docs_dir": str(BUNDLE_DIR), "top_k": 3},
termination={"max_rounds": 5, "max_tokens": 1000},
feedback={"decision": "approved", "rationale": "startup shape check"},
)
for role in ("proposer", "checker"):
assert resolve_model(contracts.model_map, role) == "claude-haiku-4-5-20251001"
# --- §9 citations: at least one exact span into the source documents ------------------------
def test_build_citations_from_micro_bundle_are_exact_spans() -> None:
citations = build_citations(navigate_bundle(BUNDLE_DIR))
assert citations, "the micro-bundle must yield at least one citation (§9)"
for citation in citations:
source = (BUNDLE_DIR / citation.file).read_text(encoding="utf-8")
start, end = (int(part) for part in citation.span.removeprefix("chars ").split("-"))
# The span is EXACT: the snippet is the literal text at those offsets.
assert source[start:end] == citation.snippet
def test_build_citations_excludes_verdict_files() -> None:
cited_files = {citation.file for citation in build_citations(navigate_bundle(BUNDLE_DIR))}
assert "verdict-led-fro.md" not in cited_files
def test_build_citations_fails_fast_on_uncitable_context(tmp_path: Path) -> None:
# §9: a run whose context yields no citable content MUST fail fast.
(tmp_path / "index.md").write_text("---\ntype: index\n---\n", encoding="utf-8")
with pytest.raises(ValueError):
build_citations(navigate_bundle(tmp_path))
# --- artifact persistence: decisions mirrored VERBATIM, deterministic bytes -----------------
def _proposal() -> SavingsProposal:
return SavingsProposal(
project_id="bygg-kontor-nord",
measure="LED-retrofit",
affected_items=[AffectedItem(code="EL-01", quantity=100, unit_cost=250.0)],
claimed_saving_nok=20000.0,
)
def _overridden_run() -> RunResult:
# The §9 non-conflation shape: the validator PASSED the numbers, the checker
# overrode with an explicit REJECT — the artifact must carry both, verbatim.
return RunResult(
outcome=Rejection(reason="unit cost unsupported (checker REJECT overrode)"),
validator_decision="validated",
checker_decision="reject",
attempts=2,
proposal=_proposal(),
)
def _provenance() -> Provenance:
return Provenance(
citations=[Citation(file="index.md", span="chars 0-5", snippet="Bygg-")],
model="claude-haiku-4-5-20251001",
role="proposer",
validator_decision="validated",
tokens_used=1234,
)
def _persist(out_dir: Path) -> dict[str, Path]:
return persist_run_artifacts(
out_dir,
run=_overridden_run(),
provenance=_provenance(),
termination=TerminationContract(max_rounds=10, max_tokens=150000),
tokens_used=1234,
rounds_used=4,
cost_usd=0.0567,
)
def test_persist_writes_the_four_artifacts(tmp_path: Path) -> None:
paths = _persist(tmp_path / "out")
assert set(paths) == {"proposal", "run_result", "provenance", "usage"}
for path in paths.values():
assert path.is_file()
def test_persisted_proposal_round_trips_through_the_ir(tmp_path: Path) -> None:
paths = _persist(tmp_path / "out")
loaded = SavingsProposal.model_validate(
json.loads(paths["proposal"].read_text(encoding="utf-8"))
)
assert loaded == _proposal()
def test_persisted_decisions_mirror_the_run_verbatim(tmp_path: Path) -> None:
# LOAD-BEARING (§9/§11): validator_decision and checker_decision are copied
# from the RunResult — recomputing either from the (checker-overridden)
# outcome would mislabel a validated proposal as validator-rejected.
paths = _persist(tmp_path / "out")
record = json.loads(paths["run_result"].read_text(encoding="utf-8"))
assert record["validator_decision"] == "validated"
assert record["checker_decision"] == "reject"
assert record["attempts"] == 2
assert record["outcome"] == {
"type": "rejected",
"reason": "unit cost unsupported (checker REJECT overrode)",
}
def test_persisted_validated_outcome_carries_percentiles(tmp_path: Path) -> None:
run = RunResult(
outcome=ValidatedProposal(
validates=True,
claimed_saving_nok=20000.0,
nominal_feasible=25000.0,
p10=18000.0,
p50=22000.0,
p90=27000.0,
),
validator_decision="validated",
checker_decision="approve",
attempts=1,
proposal=_proposal(),
)
paths = persist_run_artifacts(
tmp_path / "out",
run=run,
provenance=_provenance(),
termination=TerminationContract(max_rounds=10, max_tokens=150000),
tokens_used=1234,
rounds_used=4,
cost_usd=None,
)
record = json.loads(paths["run_result"].read_text(encoding="utf-8"))
assert record["outcome"]["type"] == "validated"
assert record["outcome"]["p50"] == 22000.0
def test_persisted_provenance_matches_the_stamp(tmp_path: Path) -> None:
paths = _persist(tmp_path / "out")
assert json.loads(paths["provenance"].read_text(encoding="utf-8")) == _provenance().model_dump()
def test_persisted_usage_carries_meter_caps_and_cost(tmp_path: Path) -> None:
# §8 + D6: the artifact records real usage AGAINST the caps, and the cost.
paths = _persist(tmp_path / "out")
usage = json.loads(paths["usage"].read_text(encoding="utf-8"))
assert usage == {
"cost_usd": 0.0567,
"max_rounds": 10,
"max_tokens": 150000,
"rounds_used": 4,
"tokens_used": 1234,
}
def test_persistence_is_deterministic(tmp_path: Path) -> None:
first = {name: path.read_bytes() for name, path in _persist(tmp_path / "a").items()}
second = {name: path.read_bytes() for name, path in _persist(tmp_path / "b").items()}
assert first == second