1
0
Fork 0

fix(run): S10 del 2 — post-mortem: stopp-artefakt, SDK-isolasjon, raw-JSON-direktiv

Transkript-analyse av den stoppede live-kjøringen (10 kall, 162 250 tokens,
$0.331506): konfig-lekkasjen (setting_sources=None laster ALLE filsystem-
settings) injiserte operatørens Claude-konfig i hvert kall — ~10-15k uncachede
tokens, en påtvunget bekreftelses-preamble som gjorde ren-JSON-svar umulige,
og en checker kapret av lekkede instrukser (debatt konvergerte aldri).

- persist_stop_artifacts: stopp-event verbatim + usage/kost persisteres ALLTID
  ved BudgetExceeded (delt usage-shape med fullført-run-stien)
- build_call_options: setting_sources=[] (SDK isolation mode, verifisert mot
  installert 0.2.110-kilde), system_prompt=None → tom system-prompt; detach-
  bevis via monkeypatchet query
- _generation_prompt: krever ONLY the raw JSON object (fence-innpakning ga
  4 fullpris parse-retries)

187/187 uten nøkkel · ruff + mypy --strict rene · tre detach-bevis RØDE → grønn

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01QdSfQdND84oeq2mbjueLTS
This commit is contained in:
Kjell Tore Guttormsen 2026-07-03 10:49:51 +02:00
commit 7637c6feae
7 changed files with 253 additions and 25 deletions

View file

@ -22,6 +22,7 @@ from portfolio_optimiser_claude.loop import (
generate_candidate,
parse_checker_verdict,
run_debate,
run_project,
)
# One affected item: 100 × 1000 = 100_000 NOK total; nominal feasible 30_000;
@ -208,3 +209,21 @@ class TestParseCheckerVerdict:
def test_empty_text_parses_as_absent(self) -> None:
assert parse_checker_verdict("").decision == "absent"
class TestGenerationPrompt:
def test_generation_demands_the_raw_json_object_only(self) -> None:
# S10 post-mortem: 3 of 4 generation replies carried plausible JSON
# wrapped in a markdown fence + commentary — each one a full-price
# blind parse-retry (§3 Step 2). The prompt must forbid the wrapping.
client = ScriptedClient(
replies=[
reply("reasoning"),
reply("VERDICT: APPROVE"),
reply(json.dumps(VALID_PROPOSAL)),
]
)
run_project(client, "context", meter=_meter(), max_debate_rounds=1)
generation_prompt = client.prompts("proposer")[1]
assert "ONLY the raw JSON object" in generation_prompt
assert "no markdown fences" in generation_prompt

View file

@ -15,7 +15,12 @@ from pathlib import Path
import pytest
from portfolio_optimiser_claude.artifacts import build_citations, persist_run_artifacts
from portfolio_optimiser_claude.artifacts import (
build_citations,
persist_run_artifacts,
persist_stop_artifacts,
)
from portfolio_optimiser_claude.budget import BudgetExceeded
from portfolio_optimiser_claude.contracts import (
ModelMapContract,
TerminationContract,
@ -216,3 +221,57 @@ def test_persistence_is_deterministic(tmp_path: Path) -> None:
first = {name: path.read_bytes() for name, path in _persist(tmp_path / "a").items()}
second = {name: path.read_bytes() for name, path in _persist(tmp_path / "b").items()}
assert first == second
# --- stop artifacts: usage + cost persisted even when the budget stops the run ---------------
# The S10 live run stopped structurally on the token cap (§8) and persisted
# NOTHING — real spend without a record. A budget stop is a run outcome.
def _persist_stop(out_dir: Path) -> dict[str, Path]:
return persist_stop_artifacts(
out_dir,
stop=BudgetExceeded("tokens", limit=150_000, observed=162_250),
termination=TerminationContract(max_rounds=12, max_tokens=150_000),
tokens_used=162_250,
rounds_used=5,
cost_usd=0.331506,
)
def test_budget_stop_persists_stop_event_and_usage(tmp_path: Path) -> None:
paths = _persist_stop(tmp_path / "out")
assert set(paths) == {"stop", "usage"}
for path in paths.values():
assert path.is_file()
def test_stop_artifact_mirrors_the_stop_event_verbatim(tmp_path: Path) -> None:
# LOAD-BEARING (§8/§11): the artifact carries the breached kind, limit and
# observed value exactly as raised — never recomputed, never summarised.
paths = _persist_stop(tmp_path / "out")
assert json.loads(paths["stop"].read_text(encoding="utf-8")) == {
"kind": "tokens",
"limit": 150_000,
"observed": 162_250,
}
def test_stop_usage_artifact_has_the_completed_run_shape(tmp_path: Path) -> None:
# S11 reads ONE usage format: the stop path must write the same shape
# (usage against caps + cost) as the completed-run path.
paths = _persist_stop(tmp_path / "out")
usage = json.loads(paths["usage"].read_text(encoding="utf-8"))
assert usage == {
"cost_usd": 0.331506,
"max_rounds": 12,
"max_tokens": 150_000,
"rounds_used": 5,
"tokens_used": 162_250,
}
def test_stop_persistence_is_deterministic(tmp_path: Path) -> None:
first = {name: path.read_bytes() for name, path in _persist_stop(tmp_path / "a").items()}
second = {name: path.read_bytes() for name, path in _persist_stop(tmp_path / "b").items()}
assert first == second

View file

@ -0,0 +1,77 @@
"""Prompt-isolation proof for the SDK call options (§11, S10 post-mortem).
The S10 live run leaked the operator's Claude Code configuration into every
spawned CLI session: ``ClaudeAgentOptions.setting_sources`` defaults to
``None``, which loads ALL filesystem settings (verified against SDK 0.2.110)
session-start hooks injected STATE.md into the model's context, every reply
opened with a mandated confirmation line (so a reply was NEVER pure JSON),
and each call paid ~10-15k uncached context tokens. ``[]`` is the SDK's
documented isolation mode: no filesystem settings, no hooks, no CLAUDE.md.
Importing the client here is offline-safe: constructing options touches no
network and needs no API key; ``query()`` is replaced with a recording fake.
"""
from __future__ import annotations
from typing import Any, AsyncIterator
import pytest
from portfolio_optimiser_claude import sdk_client
from portfolio_optimiser_claude.contracts import ModelMapContract
from portfolio_optimiser_claude.sdk_client import SdkModelClient, build_call_options
class TestBuildCallOptions:
def test_all_filesystem_settings_are_disabled(self) -> None:
# LOAD-BEARING (§11): [] is SDK isolation mode. The default (None)
# loads user+project+local settings — hooks and CLAUDE.md leak into
# the model's context, exactly the observed S10 failure.
options = build_call_options("model-x", max_budget_usd=0.25)
assert options.setting_sources == []
def test_the_system_prompt_is_empty(self) -> None:
# None serializes to --system-prompt "" (verified against 0.2.110):
# no Claude Code preset, no appended operator instructions.
options = build_call_options("model-x", max_budget_usd=0.25)
assert options.system_prompt is None
def test_the_call_stays_bounded_single_turn_no_tools(self) -> None:
options = build_call_options("model-x", max_budget_usd=0.25)
assert options.max_turns == 1
assert options.tools == []
assert options.max_budget_usd == 0.25
assert options.model == "model-x"
class TestCompleteThreadsIsolatedOptions:
def test_complete_passes_the_isolated_options_to_query(
self, monkeypatch: pytest.MonkeyPatch
) -> None:
# Detach-proof: if complete() ever builds its options inline again
# (dropping the isolation), this goes RED — grønn-men-død guard.
captured: dict[str, Any] = {}
def fake_query(*, prompt: str, options: Any) -> AsyncIterator[Any]:
captured["prompt"] = prompt
captured["options"] = options
async def _empty() -> AsyncIterator[Any]:
return
yield
return _empty()
monkeypatch.setattr(sdk_client, "query", fake_query)
client = SdkModelClient(
ModelMapContract(profiles={"anthropic": {"default": "model-default"}}),
max_budget_usd_per_call=0.10,
)
reply = client.complete("the prompt", role="proposer")
assert captured["prompt"] == "the prompt"
assert captured["options"].setting_sources == []
assert captured["options"].max_budget_usd == 0.10
assert captured["options"].model == "model-default"
# No usage surfaced by the fake → the reply fails CLOSED (§8).
assert reply.usage_tokens is None