diff --git a/src/portfolio_optimiser_claude/artifacts.py b/src/portfolio_optimiser_claude/artifacts.py new file mode 100644 index 0000000..bf99242 --- /dev/null +++ b/src/portfolio_optimiser_claude/artifacts.py @@ -0,0 +1,114 @@ +"""The S10 output layer: §9 citations + deterministic run-artifact persistence. + +``build_citations`` cites the navigated read-context (never ``type: verdict`` +files — the verdict layer must not leak, §3 Step 1) with EXACT character spans +into the source files; a context that yields no citable content fails fast (§9). +``persist_run_artifacts`` writes the captured run for S11: the decisions are +mirrored VERBATIM from the ``RunResult`` (§9 non-conflation — never recomputed +from a checker-overridden outcome), and the bytes are deterministic (sorted +keys, 2-space indent — the house JSON convention). + +Pure file layer by design — imports no agent toolkit; the SDK client stays +run-path-only (§11: the suite runs without a key). +""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +from portfolio_optimiser_claude.contracts import TerminationContract +from portfolio_optimiser_claude.loop import RunResult +from portfolio_optimiser_claude.okf import ConceptFile +from portfolio_optimiser_claude.provenance import Citation, Provenance +from portfolio_optimiser_claude.validator import Rejection + +_VERDICT_TYPE = "verdict" + + +def build_citations(concepts: list[ConceptFile]) -> list[Citation]: + """One citation per citable concept: file + exact char span + snippet (§9). + + The snippet is the concept body's first non-empty line; the span locates it + verbatim in the SOURCE file (frontmatter included), so every citation is + independently verifiable. Verdict files and empty bodies are skipped; a + context with no citable content raises (fail fast, §9). + """ + citations: list[Citation] = [] + for concept in concepts: + if concept.type == _VERDICT_TYPE or not concept.body: + continue + snippet = next((line for line in concept.body.splitlines() if line.strip()), "") + if not snippet: + continue + source = concept.path.read_text(encoding="utf-8") + start = source.find(snippet) + if start < 0: + continue + citations.append( + Citation( + file=concept.path.name, + span=f"chars {start}-{start + len(snippet)}", + snippet=snippet, + ) + ) + if not citations: + raise ValueError("context yields no citable content — a run must fail fast (§9)") + return citations + + +def _dump_json(path: Path, payload: dict[str, Any]) -> None: + path.write_text(json.dumps(payload, sort_keys=True, indent=2) + "\n", encoding="utf-8") + + +def persist_run_artifacts( + out_dir: Path, + *, + run: RunResult, + provenance: Provenance, + termination: TerminationContract, + tokens_used: int, + rounds_used: int, + cost_usd: float | None, +) -> dict[str, Path]: + """Persist the run for S11: proposal, result, provenance stamp, usage-vs-caps. + + ``validator_decision`` and ``checker_decision`` are copied from the + ``RunResult`` fields — the two falsifiers were recorded separately there + (§9) and recomputing either from the outcome would conflate them. + """ + out_dir.mkdir(parents=True, exist_ok=True) + outcome: dict[str, Any] = ( + {"type": "rejected", "reason": run.outcome.reason} + if isinstance(run.outcome, Rejection) + else {"type": "validated", **run.outcome.model_dump()} + ) + paths = { + "proposal": out_dir / "proposal.json", + "run_result": out_dir / "run_result.json", + "provenance": out_dir / "provenance.json", + "usage": out_dir / "usage.json", + } + _dump_json(paths["proposal"], run.proposal.model_dump()) + _dump_json( + paths["run_result"], + { + "validator_decision": run.validator_decision, + "checker_decision": run.checker_decision, + "attempts": run.attempts, + "outcome": outcome, + }, + ) + _dump_json(paths["provenance"], provenance.model_dump()) + _dump_json( + paths["usage"], + { + "tokens_used": tokens_used, + "rounds_used": rounds_used, + "max_tokens": termination.max_tokens, + "max_rounds": termination.max_rounds, + "cost_usd": cost_usd, + }, + ) + return paths diff --git a/src/portfolio_optimiser_claude/contracts.py b/src/portfolio_optimiser_claude/contracts.py index f7ea1cd..46ee8f6 100644 --- a/src/portfolio_optimiser_claude/contracts.py +++ b/src/portfolio_optimiser_claude/contracts.py @@ -73,6 +73,19 @@ class Contracts(BaseModel): feedback: FeedbackContract +def resolve_model(model_map: ModelMapContract, role: str, *, profile: str = "anthropic") -> str: + """Resolve role -> model id from the validated map; unmapped roles use ``default``. + + The profile itself is config (§10) — an unknown profile is a startup error, + never a silent fallback. + """ + mapping = model_map.profiles.get(profile) + if mapping is None: + raise ValueError(f"model_map has no profile '{profile}'") + # Contract validation (§10) guarantees every profile carries a 'default'. + return mapping.get(role, mapping["default"]) + + def _bundled_model_map() -> dict[str, Any]: raw: dict[str, Any] = json.loads( files("portfolio_optimiser_claude") diff --git a/src/portfolio_optimiser_claude/run_s10.py b/src/portfolio_optimiser_claude/run_s10.py new file mode 100644 index 0000000..364484b --- /dev/null +++ b/src/portfolio_optimiser_claude/run_s10.py @@ -0,0 +1,133 @@ +"""S10 — the ONE live model run of the programme (D6). RUN-PATH ONLY. + +Wires the offline-proven pipeline (S6–S9) to the real Claude Agent SDK client +for a single controlled run on the micro-bundle: startup contracts validated +fail-fast BEFORE the client is built (§10), citations proven citable BEFORE any +spend (§9), the read-context navigated + experience-folded exactly as the +offline suite proves (§3 Step 1), the loop bounded by the §8 meter AND the +per-call USD cap, and the captured artifacts persisted for S11 with the cost +logged. Never imported by the test suite. + +Run: uv run python -m portfolio_optimiser_claude.run_s10 +""" + +from __future__ import annotations + +import argparse +import os +from pathlib import Path + +from portfolio_optimiser_claude.artifacts import build_citations, persist_run_artifacts +from portfolio_optimiser_claude.budget import BudgetExceeded, BudgetMeter +from portfolio_optimiser_claude.contracts import load_contracts +from portfolio_optimiser_claude.experience import ( + CandidateFeatures, + VerdictStore, + fold_experience, + seed_store_from_bundle, +) +from portfolio_optimiser_claude.ir import load_validator_input +from portfolio_optimiser_claude.loop import run_project +from portfolio_optimiser_claude.okf import bundle_context, navigate_bundle +from portfolio_optimiser_claude.provenance import Provenance +from portfolio_optimiser_claude.sdk_client import SdkModelClient +from portfolio_optimiser_claude.validator import Rejection + +_DEFAULT_BUNDLE = Path(__file__).resolve().parents[2] / "shared" / "examples" / "bygg-energi-mikro" +_PROPOSER_ROLE = "proposer" + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description="S10: the single live run (hard caps, D6).") + parser.add_argument("--bundle", type=Path, default=_DEFAULT_BUNDLE) + parser.add_argument("--out", type=Path, default=Path("runs") / "s10") + parser.add_argument("--max-rounds", type=int, default=12) + parser.add_argument("--max-tokens", type=int, default=150_000) + parser.add_argument("--max-budget-usd-per-call", type=float, default=0.25) + parser.add_argument("--max-debate-rounds", type=int, default=3) + parser.add_argument("--max-attempts", type=int, default=3) + parser.add_argument("--top-k", type=int, default=3) + args = parser.parse_args(argv) + + if not os.environ.get("ANTHROPIC_API_KEY"): + # The bundled CLI resolves its own credentials (Claude Code login) when + # no key is exported; an unauthenticated run fails on the FIRST call. + print("note: ANTHROPIC_API_KEY not set — relying on the CLI's own credentials.") + + # §10: ALL startup contracts schema-validated BEFORE any model client exists. + contracts = load_contracts( + data_source={"docs_dir": str(args.bundle), "top_k": args.top_k}, + termination={"max_rounds": args.max_rounds, "max_tokens": args.max_tokens}, + feedback={"decision": "approved", "rationale": "startup shape check (§10)"}, + ) + + # §9: prove the context citable BEFORE any spend; §3 Step 1: navigate + fold. + concepts = navigate_bundle(args.bundle) + citations = build_citations(concepts) + ir_projection = load_validator_input(args.bundle) + store = VerdictStore() + seeded = seed_store_from_bundle(store, args.bundle) + context = fold_experience( + store, + CandidateFeatures.from_proposal(ir_projection), + bundle_context(args.bundle), + contracts.data_source.top_k, + ) + + meter = BudgetMeter(contracts.termination) + client = SdkModelClient( + contracts.model_map, max_budget_usd_per_call=args.max_budget_usd_per_call + ) + print( + f"S10 live run: bundle={args.bundle.name} seeded_verdicts={seeded} " + f"caps: max_rounds={args.max_rounds} max_tokens={args.max_tokens} " + f"max_budget_usd_per_call={args.max_budget_usd_per_call}" + ) + try: + result = run_project( + client, + context, + meter=meter, + max_debate_rounds=args.max_debate_rounds, + max_attempts=args.max_attempts, + default_project_id=ir_projection.project_id, + ) + except BudgetExceeded as stop: + # §8: the structured stop event — report it, never a silent hang. + print(f"STOPPED by budget: {stop.kind} observed {stop.observed} > limit {stop.limit}") + print(f"cost so far: {client.total_cost_usd:.6f} USD") + return 3 + + provenance = Provenance( + citations=citations, + model=client.last_model or "unknown", # §9: the REAL id, neutral fallback + role=_PROPOSER_ROLE, + validator_decision=result.validator_decision, + tokens_used=meter.tokens_used, + ) + paths = persist_run_artifacts( + args.out, + run=result, + provenance=provenance, + termination=contracts.termination, + tokens_used=meter.tokens_used, + rounds_used=meter.rounds_used, + cost_usd=round(client.total_cost_usd, 6), + ) + outcome_kind = "rejected" if isinstance(result.outcome, Rejection) else "validated" + print( + f"result: validator={result.validator_decision} checker={result.checker_decision} " + f"attempts={result.attempts} outcome={outcome_kind}" + ) + print( + f"usage: tokens={meter.tokens_used}/{contracts.termination.max_tokens} " + f"rounds={meter.rounds_used}/{contracts.termination.max_rounds}" + ) + print(f"model: {client.last_model} cost: {client.total_cost_usd:.6f} USD") + for name, path in sorted(paths.items()): + print(f"artifact: {name} -> {path}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/portfolio_optimiser_claude/sdk_client.py b/src/portfolio_optimiser_claude/sdk_client.py new file mode 100644 index 0000000..f879404 --- /dev/null +++ b/src/portfolio_optimiser_claude/sdk_client.py @@ -0,0 +1,105 @@ +"""The Claude Agent SDK model client — RUN-PATH ONLY (honesty rule §1, §11). + +This module is the ONE place the programme touches a real model (S10, D6). It +is never imported by the test suite — the offline suite proves the loop with +the scripted stand-in, and this client slots into the same ``ModelClient`` +protocol seam. Each ``complete()`` is one bounded ``query()`` call: no tools, +one turn, a first-class USD cap (``ClaudeAgentOptions.max_budget_usd``) ON TOP +of the §8 token/round meter that the loop already charges. + +Verified against claude-agent-sdk 0.2.110: ``query()`` yields +``AssistantMessage`` (text blocks + real model id) and a closing +``ResultMessage`` (provider-reported ``usage`` + ``total_cost_usd``). A reply +without usage is passed through as ``None`` so the meter fails CLOSED (§8) — +this client never invents a count. +""" + +from __future__ import annotations + +import asyncio +from typing import Any + +from claude_agent_sdk import ( + AssistantMessage, + ClaudeAgentOptions, + ResultMessage, + TextBlock, + query, +) + +from portfolio_optimiser_claude.contracts import ModelMapContract, resolve_model +from portfolio_optimiser_claude.loop import ModelReply + +# The provider-reported usage fields that make up the TOTAL token count (§8). +_USAGE_TOKEN_FIELDS = ( + "input_tokens", + "output_tokens", + "cache_creation_input_tokens", + "cache_read_input_tokens", +) + + +def _total_tokens(usage: dict[str, Any] | None) -> int | None: + """Sum the provider-reported token fields; no usage stays ``None`` (§8).""" + if usage is None: + return None + counts = [usage[field] for field in _USAGE_TOKEN_FIELDS if isinstance(usage.get(field), int)] + return sum(counts) if counts else None + + +class SdkModelClient: + """``ModelClient`` over ``claude_agent_sdk.query()`` — one bounded call each. + + ``total_cost_usd`` accumulates the provider-reported cost across calls so + the run can log it (D6); ``last_model`` carries the REAL model id from the + latest reply for the §9 provenance stamp. + """ + + def __init__( + self, + model_map: ModelMapContract, + *, + profile: str = "anthropic", + max_budget_usd_per_call: float = 0.25, + ) -> None: + if max_budget_usd_per_call <= 0: + raise ValueError( + f"max_budget_usd_per_call must be positive, got {max_budget_usd_per_call}" + ) + self._model_map = model_map + self._profile = profile + self._max_budget_usd_per_call = max_budget_usd_per_call + self.total_cost_usd = 0.0 + self.last_model: str | None = None + + def complete(self, prompt: str, *, role: str) -> ModelReply: + model_id = resolve_model(self._model_map, role, profile=self._profile) + return asyncio.run(self._complete_async(prompt, model_id)) + + async def _complete_async(self, prompt: str, model_id: str) -> ModelReply: + options = ClaudeAgentOptions( + model=model_id, + max_turns=1, # a single completion — the agentic loop lives in loop.py, not here + tools=[], # pure text completion: no tool surface, no silent egress + max_budget_usd=self._max_budget_usd_per_call, + ) + text_parts: list[str] = [] + reply_model: str | None = None + usage_tokens: int | None = None + async for message in query(prompt=prompt, options=options): + if isinstance(message, AssistantMessage): + if message.error is not None: + raise RuntimeError(f"model call failed: {message.error}") + reply_model = message.model + text_parts.extend( + block.text for block in message.content if isinstance(block, TextBlock) + ) + elif isinstance(message, ResultMessage): + if message.is_error: + raise RuntimeError(f"model call failed: {message.subtype}: {message.errors}") + usage_tokens = _total_tokens(message.usage) + if message.total_cost_usd is not None: + self.total_cost_usd += message.total_cost_usd + if reply_model is not None: + self.last_model = reply_model + return ModelReply(text="".join(text_parts), usage_tokens=usage_tokens, model=reply_model) diff --git a/tests/test_s10_run_layer.py b/tests/test_s10_run_layer.py new file mode 100644 index 0000000..2b5b7e6 --- /dev/null +++ b/tests/test_s10_run_layer.py @@ -0,0 +1,218 @@ +"""S10 run-layer seams, TDD offline (§9, §10, §11): model resolution, citations, artifacts. + +The SDK client itself is run-path-only and NEVER imported here (the suite runs +without an API key and without a network). What IS provable offline: the +role -> model-id resolution against the startup contract, the §9 citation +builder (fail-fast on uncitable context), and the artifact persistence layer — +which must mirror the run's decisions VERBATIM (§9 non-conflation), never +recompute them from the outcome. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from portfolio_optimiser_claude.artifacts import build_citations, persist_run_artifacts +from portfolio_optimiser_claude.contracts import ( + ModelMapContract, + TerminationContract, + load_contracts, + resolve_model, +) +from portfolio_optimiser_claude.ir import AffectedItem, SavingsProposal +from portfolio_optimiser_claude.loop import RunResult +from portfolio_optimiser_claude.okf import navigate_bundle +from portfolio_optimiser_claude.provenance import Citation, Provenance +from portfolio_optimiser_claude.validator import Rejection, ValidatedProposal + +BUNDLE_DIR = Path(__file__).resolve().parent.parent / "shared" / "examples" / "bygg-energi-mikro" + + +# --- role -> model id resolution (§10: config, fail-fast) ----------------------------------- + + +def _model_map() -> ModelMapContract: + return ModelMapContract( + profiles={"anthropic": {"default": "model-default", "proposer": "model-proposer"}} + ) + + +def test_resolve_model_returns_role_entry() -> None: + assert resolve_model(_model_map(), "proposer") == "model-proposer" + + +def test_resolve_model_falls_back_to_default_for_unmapped_role() -> None: + assert resolve_model(_model_map(), "checker") == "model-default" + + +def test_resolve_model_unknown_profile_raises() -> None: + with pytest.raises(ValueError): + resolve_model(_model_map(), "proposer", profile="no-such-profile") + + +def test_bundled_model_map_resolves_haiku_for_both_roles() -> None: + # The S10 run path resolves from the BUNDLED map (D6: cheapest suitable model). + contracts = load_contracts( + data_source={"docs_dir": str(BUNDLE_DIR), "top_k": 3}, + termination={"max_rounds": 5, "max_tokens": 1000}, + feedback={"decision": "approved", "rationale": "startup shape check"}, + ) + for role in ("proposer", "checker"): + assert resolve_model(contracts.model_map, role) == "claude-haiku-4-5-20251001" + + +# --- §9 citations: at least one exact span into the source documents ------------------------ + + +def test_build_citations_from_micro_bundle_are_exact_spans() -> None: + citations = build_citations(navigate_bundle(BUNDLE_DIR)) + assert citations, "the micro-bundle must yield at least one citation (§9)" + for citation in citations: + source = (BUNDLE_DIR / citation.file).read_text(encoding="utf-8") + start, end = (int(part) for part in citation.span.removeprefix("chars ").split("-")) + # The span is EXACT: the snippet is the literal text at those offsets. + assert source[start:end] == citation.snippet + + +def test_build_citations_excludes_verdict_files() -> None: + cited_files = {citation.file for citation in build_citations(navigate_bundle(BUNDLE_DIR))} + assert "verdict-led-fro.md" not in cited_files + + +def test_build_citations_fails_fast_on_uncitable_context(tmp_path: Path) -> None: + # §9: a run whose context yields no citable content MUST fail fast. + (tmp_path / "index.md").write_text("---\ntype: index\n---\n", encoding="utf-8") + with pytest.raises(ValueError): + build_citations(navigate_bundle(tmp_path)) + + +# --- artifact persistence: decisions mirrored VERBATIM, deterministic bytes ----------------- + + +def _proposal() -> SavingsProposal: + return SavingsProposal( + project_id="bygg-kontor-nord", + measure="LED-retrofit", + affected_items=[AffectedItem(code="EL-01", quantity=100, unit_cost=250.0)], + claimed_saving_nok=20000.0, + ) + + +def _overridden_run() -> RunResult: + # The §9 non-conflation shape: the validator PASSED the numbers, the checker + # overrode with an explicit REJECT — the artifact must carry both, verbatim. + return RunResult( + outcome=Rejection(reason="unit cost unsupported (checker REJECT overrode)"), + validator_decision="validated", + checker_decision="reject", + attempts=2, + proposal=_proposal(), + ) + + +def _provenance() -> Provenance: + return Provenance( + citations=[Citation(file="index.md", span="chars 0-5", snippet="Bygg-")], + model="claude-haiku-4-5-20251001", + role="proposer", + validator_decision="validated", + tokens_used=1234, + ) + + +def _persist(out_dir: Path) -> dict[str, Path]: + return persist_run_artifacts( + out_dir, + run=_overridden_run(), + provenance=_provenance(), + termination=TerminationContract(max_rounds=10, max_tokens=150000), + tokens_used=1234, + rounds_used=4, + cost_usd=0.0567, + ) + + +def test_persist_writes_the_four_artifacts(tmp_path: Path) -> None: + paths = _persist(tmp_path / "out") + assert set(paths) == {"proposal", "run_result", "provenance", "usage"} + for path in paths.values(): + assert path.is_file() + + +def test_persisted_proposal_round_trips_through_the_ir(tmp_path: Path) -> None: + paths = _persist(tmp_path / "out") + loaded = SavingsProposal.model_validate( + json.loads(paths["proposal"].read_text(encoding="utf-8")) + ) + assert loaded == _proposal() + + +def test_persisted_decisions_mirror_the_run_verbatim(tmp_path: Path) -> None: + # LOAD-BEARING (§9/§11): validator_decision and checker_decision are copied + # from the RunResult — recomputing either from the (checker-overridden) + # outcome would mislabel a validated proposal as validator-rejected. + paths = _persist(tmp_path / "out") + record = json.loads(paths["run_result"].read_text(encoding="utf-8")) + assert record["validator_decision"] == "validated" + assert record["checker_decision"] == "reject" + assert record["attempts"] == 2 + assert record["outcome"] == { + "type": "rejected", + "reason": "unit cost unsupported (checker REJECT overrode)", + } + + +def test_persisted_validated_outcome_carries_percentiles(tmp_path: Path) -> None: + run = RunResult( + outcome=ValidatedProposal( + validates=True, + claimed_saving_nok=20000.0, + nominal_feasible=25000.0, + p10=18000.0, + p50=22000.0, + p90=27000.0, + ), + validator_decision="validated", + checker_decision="approve", + attempts=1, + proposal=_proposal(), + ) + paths = persist_run_artifacts( + tmp_path / "out", + run=run, + provenance=_provenance(), + termination=TerminationContract(max_rounds=10, max_tokens=150000), + tokens_used=1234, + rounds_used=4, + cost_usd=None, + ) + record = json.loads(paths["run_result"].read_text(encoding="utf-8")) + assert record["outcome"]["type"] == "validated" + assert record["outcome"]["p50"] == 22000.0 + + +def test_persisted_provenance_matches_the_stamp(tmp_path: Path) -> None: + paths = _persist(tmp_path / "out") + assert json.loads(paths["provenance"].read_text(encoding="utf-8")) == _provenance().model_dump() + + +def test_persisted_usage_carries_meter_caps_and_cost(tmp_path: Path) -> None: + # §8 + D6: the artifact records real usage AGAINST the caps, and the cost. + paths = _persist(tmp_path / "out") + usage = json.loads(paths["usage"].read_text(encoding="utf-8")) + assert usage == { + "cost_usd": 0.0567, + "max_rounds": 10, + "max_tokens": 150000, + "rounds_used": 4, + "tokens_used": 1234, + } + + +def test_persistence_is_deterministic(tmp_path: Path) -> None: + first = {name: path.read_bytes() for name, path in _persist(tmp_path / "a").items()} + second = {name: path.read_bytes() for name, path in _persist(tmp_path / "b").items()} + assert first == second