From 7637c6feae251bc7865787ed2b51efb34334b100 Mon Sep 17 00:00:00 2001 From: Kjell Tore Guttormsen Date: Fri, 3 Jul 2026 10:49:51 +0200 Subject: [PATCH] =?UTF-8?q?fix(run):=20S10=20del=202=20=E2=80=94=20post-mo?= =?UTF-8?q?rtem:=20stopp-artefakt,=20SDK-isolasjon,=20raw-JSON-direktiv?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Transkript-analyse av den stoppede live-kjøringen (10 kall, 162 250 tokens, $0.331506): konfig-lekkasjen (setting_sources=None laster ALLE filsystem- settings) injiserte operatørens Claude-konfig i hvert kall — ~10-15k uncachede tokens, en påtvunget bekreftelses-preamble som gjorde ren-JSON-svar umulige, og en checker kapret av lekkede instrukser (debatt konvergerte aldri). - persist_stop_artifacts: stopp-event verbatim + usage/kost persisteres ALLTID ved BudgetExceeded (delt usage-shape med fullført-run-stien) - build_call_options: setting_sources=[] (SDK isolation mode, verifisert mot installert 0.2.110-kilde), system_prompt=None → tom system-prompt; detach- bevis via monkeypatchet query - _generation_prompt: krever ONLY the raw JSON object (fence-innpakning ga 4 fullpris parse-retries) 187/187 uten nøkkel · ruff + mypy --strict rene · tre detach-bevis RØDE → grønn Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01QdSfQdND84oeq2mbjueLTS --- src/portfolio_optimiser_claude/artifacts.py | 58 ++++++++++++--- src/portfolio_optimiser_claude/loop.py | 3 +- src/portfolio_optimiser_claude/run_s10.py | 19 ++++- src/portfolio_optimiser_claude/sdk_client.py | 45 ++++++++---- tests/test_loop.py | 19 +++++ tests/test_s10_run_layer.py | 61 +++++++++++++++- tests/test_sdk_isolation.py | 77 ++++++++++++++++++++ 7 files changed, 255 insertions(+), 27 deletions(-) create mode 100644 tests/test_sdk_isolation.py diff --git a/src/portfolio_optimiser_claude/artifacts.py b/src/portfolio_optimiser_claude/artifacts.py index bf99242..118d353 100644 --- a/src/portfolio_optimiser_claude/artifacts.py +++ b/src/portfolio_optimiser_claude/artifacts.py @@ -18,6 +18,7 @@ import json from pathlib import Path from typing import Any +from portfolio_optimiser_claude.budget import BudgetExceeded from portfolio_optimiser_claude.contracts import TerminationContract from portfolio_optimiser_claude.loop import RunResult from portfolio_optimiser_claude.okf import ConceptFile @@ -62,6 +63,23 @@ def _dump_json(path: Path, payload: dict[str, Any]) -> None: path.write_text(json.dumps(payload, sort_keys=True, indent=2) + "\n", encoding="utf-8") +def _usage_payload( + termination: TerminationContract, + tokens_used: int, + rounds_used: int, + cost_usd: float | None, +) -> dict[str, Any]: + # ONE usage shape for both outcomes (completed run and budget stop) — + # S11 must never branch on which path wrote the artifact. + return { + "tokens_used": tokens_used, + "rounds_used": rounds_used, + "max_tokens": termination.max_tokens, + "max_rounds": termination.max_rounds, + "cost_usd": cost_usd, + } + + def persist_run_artifacts( out_dir: Path, *, @@ -101,14 +119,34 @@ def persist_run_artifacts( }, ) _dump_json(paths["provenance"], provenance.model_dump()) - _dump_json( - paths["usage"], - { - "tokens_used": tokens_used, - "rounds_used": rounds_used, - "max_tokens": termination.max_tokens, - "max_rounds": termination.max_rounds, - "cost_usd": cost_usd, - }, - ) + _dump_json(paths["usage"], _usage_payload(termination, tokens_used, rounds_used, cost_usd)) + return paths + + +def persist_stop_artifacts( + out_dir: Path, + *, + stop: BudgetExceeded, + termination: TerminationContract, + tokens_used: int, + rounds_used: int, + cost_usd: float | None, +) -> dict[str, Path]: + """Persist a budget-stopped run: the stop event verbatim + usage-vs-caps. + + A budget stop is a run outcome, not an absence of one (§8): the S10 live + run stopped structurally on the token cap and persisted NOTHING — real + spend without a record. The stop event mirrors the raised ``BudgetExceeded`` + fields exactly; the usage artifact keeps the completed-run shape. + """ + out_dir.mkdir(parents=True, exist_ok=True) + paths = { + "stop": out_dir / "stop.json", + "usage": out_dir / "usage.json", + } + _dump_json( + paths["stop"], + {"kind": stop.kind, "limit": stop.limit, "observed": stop.observed}, + ) + _dump_json(paths["usage"], _usage_payload(termination, tokens_used, rounds_used, cost_usd)) return paths diff --git a/src/portfolio_optimiser_claude/loop.py b/src/portfolio_optimiser_claude/loop.py index 2de2265..e0ba00e 100644 --- a/src/portfolio_optimiser_claude/loop.py +++ b/src/portfolio_optimiser_claude/loop.py @@ -293,7 +293,8 @@ def _generation_prompt(context: str, converged_reasoning: str) -> str: f"Converged reasoning from the maker-checker debate:\n{converged_reasoning}\n\n" "Reply with exactly one candidate measure as a JSON object with the " "fields: project_id, measure, affected_items (list of {code, quantity, " - "unit_cost}), claimed_saving_nok, and optionally assumptions." + "unit_cost}), claimed_saving_nok, and optionally assumptions. " + "Output ONLY the raw JSON object — no markdown fences, no commentary." ) diff --git a/src/portfolio_optimiser_claude/run_s10.py b/src/portfolio_optimiser_claude/run_s10.py index 364484b..c1900ad 100644 --- a/src/portfolio_optimiser_claude/run_s10.py +++ b/src/portfolio_optimiser_claude/run_s10.py @@ -17,7 +17,11 @@ import argparse import os from pathlib import Path -from portfolio_optimiser_claude.artifacts import build_citations, persist_run_artifacts +from portfolio_optimiser_claude.artifacts import ( + build_citations, + persist_run_artifacts, + persist_stop_artifacts, +) from portfolio_optimiser_claude.budget import BudgetExceeded, BudgetMeter from portfolio_optimiser_claude.contracts import load_contracts from portfolio_optimiser_claude.experience import ( @@ -93,9 +97,20 @@ def main(argv: list[str] | None = None) -> int: default_project_id=ir_projection.project_id, ) except BudgetExceeded as stop: - # §8: the structured stop event — report it, never a silent hang. + # §8: the structured stop event — report it, never a silent hang, and + # persist the spend (the first live run stopped here with NO record). print(f"STOPPED by budget: {stop.kind} observed {stop.observed} > limit {stop.limit}") print(f"cost so far: {client.total_cost_usd:.6f} USD") + stop_paths = persist_stop_artifacts( + args.out, + stop=stop, + termination=contracts.termination, + tokens_used=meter.tokens_used, + rounds_used=meter.rounds_used, + cost_usd=round(client.total_cost_usd, 6), + ) + for name, path in sorted(stop_paths.items()): + print(f"artifact: {name} -> {path}") return 3 provenance = Provenance( diff --git a/src/portfolio_optimiser_claude/sdk_client.py b/src/portfolio_optimiser_claude/sdk_client.py index f879404..3d16bc9 100644 --- a/src/portfolio_optimiser_claude/sdk_client.py +++ b/src/portfolio_optimiser_claude/sdk_client.py @@ -1,11 +1,13 @@ -"""The Claude Agent SDK model client — RUN-PATH ONLY (honesty rule §1, §11). +"""The Claude Agent SDK model client (honesty rule §1, §11). -This module is the ONE place the programme touches a real model (S10, D6). It -is never imported by the test suite — the offline suite proves the loop with -the scripted stand-in, and this client slots into the same ``ModelClient`` -protocol seam. Each ``complete()`` is one bounded ``query()`` call: no tools, -one turn, a first-class USD cap (``ClaudeAgentOptions.max_budget_usd``) ON TOP -of the §8 token/round meter that the loop already charges. +This module is the ONE place the programme touches a real model (S10, D6). +The network path (``query()``) is run-path-only; the offline suite proves the +loop with the scripted stand-in, imports this module WITHOUT touching a key +or the network, and pins the call options (``tests/test_sdk_isolation.py``). +Each ``complete()`` is one bounded, ISOLATED ``query()`` call: no tools, one +turn, a first-class USD cap (``ClaudeAgentOptions.max_budget_usd``) ON TOP of +the §8 token/round meter that the loop already charges, and NO filesystem +settings (``setting_sources=[]``). Verified against claude-agent-sdk 0.2.110: ``query()`` yields ``AssistantMessage`` (text blocks + real model id) and a closing @@ -39,6 +41,28 @@ _USAGE_TOKEN_FIELDS = ( ) +def build_call_options(model_id: str, *, max_budget_usd: float) -> ClaudeAgentOptions: + """One bounded, ISOLATED completion call (§8 + S10 post-mortem). + + ``setting_sources=[]`` is the SDK's documented isolation mode (verified + against 0.2.110): the spawned CLI loads NO filesystem settings — no + session hooks, no CLAUDE.md, no operator instructions. The default + (``None``) loads ALL sources: in the S10 live run that injected the + operator's config into every call (~10-15k uncached tokens each) and + mandated a confirmation preamble that made pure-JSON replies impossible. + ``system_prompt=None`` serializes to an EMPTY system prompt, not the + Claude Code preset. + """ + return ClaudeAgentOptions( + model=model_id, + max_turns=1, # a single completion — the agentic loop lives in loop.py, not here + tools=[], # pure text completion: no tool surface, no silent egress + max_budget_usd=max_budget_usd, + setting_sources=[], + system_prompt=None, + ) + + def _total_tokens(usage: dict[str, Any] | None) -> int | None: """Sum the provider-reported token fields; no usage stays ``None`` (§8).""" if usage is None: @@ -77,12 +101,7 @@ class SdkModelClient: return asyncio.run(self._complete_async(prompt, model_id)) async def _complete_async(self, prompt: str, model_id: str) -> ModelReply: - options = ClaudeAgentOptions( - model=model_id, - max_turns=1, # a single completion — the agentic loop lives in loop.py, not here - tools=[], # pure text completion: no tool surface, no silent egress - max_budget_usd=self._max_budget_usd_per_call, - ) + options = build_call_options(model_id, max_budget_usd=self._max_budget_usd_per_call) text_parts: list[str] = [] reply_model: str | None = None usage_tokens: int | None = None diff --git a/tests/test_loop.py b/tests/test_loop.py index b7d4661..daedc72 100644 --- a/tests/test_loop.py +++ b/tests/test_loop.py @@ -22,6 +22,7 @@ from portfolio_optimiser_claude.loop import ( generate_candidate, parse_checker_verdict, run_debate, + run_project, ) # One affected item: 100 × 1000 = 100_000 NOK total; nominal feasible 30_000; @@ -208,3 +209,21 @@ class TestParseCheckerVerdict: def test_empty_text_parses_as_absent(self) -> None: assert parse_checker_verdict("").decision == "absent" + + +class TestGenerationPrompt: + def test_generation_demands_the_raw_json_object_only(self) -> None: + # S10 post-mortem: 3 of 4 generation replies carried plausible JSON + # wrapped in a markdown fence + commentary — each one a full-price + # blind parse-retry (§3 Step 2). The prompt must forbid the wrapping. + client = ScriptedClient( + replies=[ + reply("reasoning"), + reply("VERDICT: APPROVE"), + reply(json.dumps(VALID_PROPOSAL)), + ] + ) + run_project(client, "context", meter=_meter(), max_debate_rounds=1) + generation_prompt = client.prompts("proposer")[1] + assert "ONLY the raw JSON object" in generation_prompt + assert "no markdown fences" in generation_prompt diff --git a/tests/test_s10_run_layer.py b/tests/test_s10_run_layer.py index 2b5b7e6..f4efde4 100644 --- a/tests/test_s10_run_layer.py +++ b/tests/test_s10_run_layer.py @@ -15,7 +15,12 @@ from pathlib import Path import pytest -from portfolio_optimiser_claude.artifacts import build_citations, persist_run_artifacts +from portfolio_optimiser_claude.artifacts import ( + build_citations, + persist_run_artifacts, + persist_stop_artifacts, +) +from portfolio_optimiser_claude.budget import BudgetExceeded from portfolio_optimiser_claude.contracts import ( ModelMapContract, TerminationContract, @@ -216,3 +221,57 @@ def test_persistence_is_deterministic(tmp_path: Path) -> None: first = {name: path.read_bytes() for name, path in _persist(tmp_path / "a").items()} second = {name: path.read_bytes() for name, path in _persist(tmp_path / "b").items()} assert first == second + + +# --- stop artifacts: usage + cost persisted even when the budget stops the run --------------- +# The S10 live run stopped structurally on the token cap (§8) and persisted +# NOTHING — real spend without a record. A budget stop is a run outcome. + + +def _persist_stop(out_dir: Path) -> dict[str, Path]: + return persist_stop_artifacts( + out_dir, + stop=BudgetExceeded("tokens", limit=150_000, observed=162_250), + termination=TerminationContract(max_rounds=12, max_tokens=150_000), + tokens_used=162_250, + rounds_used=5, + cost_usd=0.331506, + ) + + +def test_budget_stop_persists_stop_event_and_usage(tmp_path: Path) -> None: + paths = _persist_stop(tmp_path / "out") + assert set(paths) == {"stop", "usage"} + for path in paths.values(): + assert path.is_file() + + +def test_stop_artifact_mirrors_the_stop_event_verbatim(tmp_path: Path) -> None: + # LOAD-BEARING (§8/§11): the artifact carries the breached kind, limit and + # observed value exactly as raised — never recomputed, never summarised. + paths = _persist_stop(tmp_path / "out") + assert json.loads(paths["stop"].read_text(encoding="utf-8")) == { + "kind": "tokens", + "limit": 150_000, + "observed": 162_250, + } + + +def test_stop_usage_artifact_has_the_completed_run_shape(tmp_path: Path) -> None: + # S11 reads ONE usage format: the stop path must write the same shape + # (usage against caps + cost) as the completed-run path. + paths = _persist_stop(tmp_path / "out") + usage = json.loads(paths["usage"].read_text(encoding="utf-8")) + assert usage == { + "cost_usd": 0.331506, + "max_rounds": 12, + "max_tokens": 150_000, + "rounds_used": 5, + "tokens_used": 162_250, + } + + +def test_stop_persistence_is_deterministic(tmp_path: Path) -> None: + first = {name: path.read_bytes() for name, path in _persist_stop(tmp_path / "a").items()} + second = {name: path.read_bytes() for name, path in _persist_stop(tmp_path / "b").items()} + assert first == second diff --git a/tests/test_sdk_isolation.py b/tests/test_sdk_isolation.py new file mode 100644 index 0000000..e11b0a0 --- /dev/null +++ b/tests/test_sdk_isolation.py @@ -0,0 +1,77 @@ +"""Prompt-isolation proof for the SDK call options (§11, S10 post-mortem). + +The S10 live run leaked the operator's Claude Code configuration into every +spawned CLI session: ``ClaudeAgentOptions.setting_sources`` defaults to +``None``, which loads ALL filesystem settings (verified against SDK 0.2.110) +— session-start hooks injected STATE.md into the model's context, every reply +opened with a mandated confirmation line (so a reply was NEVER pure JSON), +and each call paid ~10-15k uncached context tokens. ``[]`` is the SDK's +documented isolation mode: no filesystem settings, no hooks, no CLAUDE.md. + +Importing the client here is offline-safe: constructing options touches no +network and needs no API key; ``query()`` is replaced with a recording fake. +""" + +from __future__ import annotations + +from typing import Any, AsyncIterator + +import pytest + +from portfolio_optimiser_claude import sdk_client +from portfolio_optimiser_claude.contracts import ModelMapContract +from portfolio_optimiser_claude.sdk_client import SdkModelClient, build_call_options + + +class TestBuildCallOptions: + def test_all_filesystem_settings_are_disabled(self) -> None: + # LOAD-BEARING (§11): [] is SDK isolation mode. The default (None) + # loads user+project+local settings — hooks and CLAUDE.md leak into + # the model's context, exactly the observed S10 failure. + options = build_call_options("model-x", max_budget_usd=0.25) + assert options.setting_sources == [] + + def test_the_system_prompt_is_empty(self) -> None: + # None serializes to --system-prompt "" (verified against 0.2.110): + # no Claude Code preset, no appended operator instructions. + options = build_call_options("model-x", max_budget_usd=0.25) + assert options.system_prompt is None + + def test_the_call_stays_bounded_single_turn_no_tools(self) -> None: + options = build_call_options("model-x", max_budget_usd=0.25) + assert options.max_turns == 1 + assert options.tools == [] + assert options.max_budget_usd == 0.25 + assert options.model == "model-x" + + +class TestCompleteThreadsIsolatedOptions: + def test_complete_passes_the_isolated_options_to_query( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + # Detach-proof: if complete() ever builds its options inline again + # (dropping the isolation), this goes RED — grønn-men-død guard. + captured: dict[str, Any] = {} + + def fake_query(*, prompt: str, options: Any) -> AsyncIterator[Any]: + captured["prompt"] = prompt + captured["options"] = options + + async def _empty() -> AsyncIterator[Any]: + return + yield + + return _empty() + + monkeypatch.setattr(sdk_client, "query", fake_query) + client = SdkModelClient( + ModelMapContract(profiles={"anthropic": {"default": "model-default"}}), + max_budget_usd_per_call=0.10, + ) + reply = client.complete("the prompt", role="proposer") + assert captured["prompt"] == "the prompt" + assert captured["options"].setting_sources == [] + assert captured["options"].max_budget_usd == 0.10 + assert captured["options"].model == "model-default" + # No usage surfaced by the fake → the reply fails CLOSED (§8). + assert reply.usage_tokens is None