Kjell Tore Guttormsen
7637c6feae
Transkript-analyse av den stoppede live-kjøringen (10 kall, 162 250 tokens, $0.331506): konfig-lekkasjen (setting_sources=None laster ALLE filsystem- settings) injiserte operatørens Claude-konfig i hvert kall — ~10-15k uncachede tokens, en påtvunget bekreftelses-preamble som gjorde ren-JSON-svar umulige, og en checker kapret av lekkede instrukser (debatt konvergerte aldri). - persist_stop_artifacts: stopp-event verbatim + usage/kost persisteres ALLTID ved BudgetExceeded (delt usage-shape med fullført-run-stien) - build_call_options: setting_sources=[] (SDK isolation mode, verifisert mot installert 0.2.110-kilde), system_prompt=None → tom system-prompt; detach- bevis via monkeypatchet query - _generation_prompt: krever ONLY the raw JSON object (fence-innpakning ga 4 fullpris parse-retries) 187/187 uten nøkkel · ruff + mypy --strict rene · tre detach-bevis RØDE → grønn Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01QdSfQdND84oeq2mbjueLTS
229 lines
10 KiB
Python
229 lines
10 KiB
Python
"""Steps 2 and 3 of the loop (method-spec §3): generation + maker-checker debate.
|
||
|
||
Step 2 — a reply that fails to parse into the typed IR is retried BLIND (same
|
||
prompt), never silently accepted or repaired downstream, bounded by the budget
|
||
meter (§8; round ticks between attempts). Step 3 — the debate is round-capped
|
||
with a turn-count safety net above it, state is fresh per run, and the checker
|
||
is INSTRUCTED to end with exactly one verdict line. Verdict parsing is
|
||
case-insensitive, the reject marker takes precedence, trailing text is the
|
||
reason, and a missing marker parses as absent (fail-open input to the gate).
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import json
|
||
|
||
import pytest
|
||
from _scripted import ScriptedClient, reply
|
||
|
||
from portfolio_optimiser_claude.budget import BudgetExceeded, BudgetMeter
|
||
from portfolio_optimiser_claude.contracts import TerminationContract
|
||
from portfolio_optimiser_claude.loop import (
|
||
generate_candidate,
|
||
parse_checker_verdict,
|
||
run_debate,
|
||
run_project,
|
||
)
|
||
|
||
# One affected item: 100 × 1000 = 100_000 NOK total; nominal feasible 30_000;
|
||
# no assumptions band, so every Monte Carlo sample is 30_000 and p90 == 30_000.
|
||
VALID_PROPOSAL = {
|
||
"project_id": "p1",
|
||
"measure": "led-retrofit",
|
||
"affected_items": [{"code": "E01", "quantity": 100, "unit_cost": 1000}],
|
||
"claimed_saving_nok": 25000,
|
||
}
|
||
|
||
|
||
def _meter(max_rounds: int = 50, max_tokens: int = 10_000) -> BudgetMeter:
|
||
return BudgetMeter(TerminationContract(max_rounds=max_rounds, max_tokens=max_tokens))
|
||
|
||
|
||
class TestGenerateCandidate:
|
||
def test_valid_reply_parses_into_the_typed_ir(self) -> None:
|
||
client = ScriptedClient(replies=[reply(json.dumps(VALID_PROPOSAL))])
|
||
meter = _meter()
|
||
proposal = generate_candidate(client, "base prompt", meter=meter)
|
||
assert proposal.claimed_saving_nok == 25000
|
||
assert meter.tokens_used == 10
|
||
|
||
def test_malformed_reply_is_retried_blind_with_the_same_prompt(self) -> None:
|
||
client = ScriptedClient(
|
||
replies=[reply("not json at all"), reply(json.dumps(VALID_PROPOSAL))]
|
||
)
|
||
proposal = generate_candidate(client, "base prompt", meter=_meter())
|
||
assert proposal.measure == "led-retrofit"
|
||
# Blind retry: the SAME prompt, unchanged — never a repair instruction.
|
||
assert client.prompts("proposer") == ["base prompt", "base prompt"]
|
||
|
||
def test_schema_invalid_json_is_retried_never_repaired(self) -> None:
|
||
# Claim above the items' own total is a schema error (§7.1) — the value
|
||
# must never exist; the loop retries, it does not clamp or repair.
|
||
overclaim = dict(VALID_PROPOSAL, claimed_saving_nok=999_999)
|
||
client = ScriptedClient(
|
||
replies=[reply(json.dumps(overclaim)), reply(json.dumps(VALID_PROPOSAL))]
|
||
)
|
||
proposal = generate_candidate(client, "base prompt", meter=_meter())
|
||
assert proposal.claimed_saving_nok == 25000
|
||
|
||
def test_parse_retries_charge_round_ticks(self) -> None:
|
||
# §8: round ticks are charged between attempts so the meter also
|
||
# bounds parse-retries.
|
||
client = ScriptedClient(
|
||
replies=[reply("garbage"), reply("garbage"), reply(json.dumps(VALID_PROPOSAL))]
|
||
)
|
||
meter = _meter()
|
||
generate_candidate(client, "base prompt", meter=meter)
|
||
assert meter.rounds_used == 2
|
||
|
||
def test_endless_garbage_is_stopped_by_the_meter(self) -> None:
|
||
client = ScriptedClient(script=lambda role, prompt: reply("garbage"))
|
||
with pytest.raises(BudgetExceeded) as exc_info:
|
||
generate_candidate(client, "base prompt", meter=_meter(max_rounds=3))
|
||
assert exc_info.value.kind == "rounds"
|
||
|
||
def test_project_id_may_be_defaulted_from_the_project(self) -> None:
|
||
omitted = {k: v for k, v in VALID_PROPOSAL.items() if k != "project_id"}
|
||
client = ScriptedClient(replies=[reply(json.dumps(omitted))])
|
||
proposal = generate_candidate(
|
||
client, "base prompt", meter=_meter(), default_project_id="p-default"
|
||
)
|
||
assert proposal.project_id == "p-default"
|
||
|
||
def test_present_project_id_is_never_overwritten_by_the_default(self) -> None:
|
||
client = ScriptedClient(replies=[reply(json.dumps(VALID_PROPOSAL))])
|
||
proposal = generate_candidate(
|
||
client, "base prompt", meter=_meter(), default_project_id="p-default"
|
||
)
|
||
assert proposal.project_id == "p1"
|
||
|
||
def test_missing_usage_on_the_counting_path_fails_closed(self) -> None:
|
||
client = ScriptedClient(replies=[reply(json.dumps(VALID_PROPOSAL), usage_tokens=None)])
|
||
with pytest.raises(Exception, match="usage"):
|
||
generate_candidate(client, "base prompt", meter=_meter())
|
||
|
||
|
||
class TestRunDebate:
|
||
def test_converges_when_the_checker_approves(self) -> None:
|
||
client = ScriptedClient(replies=[reply("reasoning v1"), reply("holds. VERDICT: APPROVE")])
|
||
debate = run_debate(client, "context", max_rounds=3, meter=_meter())
|
||
assert debate.rounds == 1
|
||
assert debate.proposer_output == "reasoning v1"
|
||
assert "VERDICT: APPROVE" in debate.checker_last
|
||
|
||
def test_round_cap_bounds_a_never_approving_debate(self) -> None:
|
||
client = ScriptedClient(
|
||
script=lambda role, prompt: reply(
|
||
"VERDICT: REJECT - weak numbers" if role == "checker" else "reasoning"
|
||
)
|
||
)
|
||
debate = run_debate(client, "context", max_rounds=2, meter=_meter())
|
||
assert debate.rounds == 2
|
||
assert len(client.prompts("proposer")) == 2
|
||
assert len(client.prompts("checker")) == 2
|
||
assert "REJECT" in debate.checker_last
|
||
|
||
def test_checker_is_instructed_to_end_with_the_verdict_line(self) -> None:
|
||
# §3 Step 3: the checker MUST be instructed to end its reply with
|
||
# exactly one verdict line, both marker forms spelled out.
|
||
client = ScriptedClient(replies=[reply("reasoning"), reply("VERDICT: APPROVE")])
|
||
run_debate(client, "context", max_rounds=1, meter=_meter())
|
||
checker_prompt = client.prompts("checker")[0]
|
||
assert "VERDICT: APPROVE" in checker_prompt
|
||
assert "VERDICT: REJECT - <short reason>" in checker_prompt
|
||
|
||
def test_checker_critique_reaches_the_next_proposer_turn(self) -> None:
|
||
client = ScriptedClient(
|
||
replies=[
|
||
reply("reasoning v1"),
|
||
reply("VERDICT: REJECT - unit costs are stale"),
|
||
reply("reasoning v2"),
|
||
reply("VERDICT: APPROVE"),
|
||
]
|
||
)
|
||
debate = run_debate(client, "context", max_rounds=3, meter=_meter())
|
||
assert debate.rounds == 2
|
||
assert "unit costs are stale" in client.prompts("proposer")[1]
|
||
assert debate.proposer_output == "reasoning v2"
|
||
|
||
def test_debate_state_is_fresh_per_run(self) -> None:
|
||
# §3 Step 3: no conversation state may survive from one run into the
|
||
# next — the second run's opening proposer prompt carries nothing from
|
||
# the first run's transcript.
|
||
client = ScriptedClient(
|
||
replies=[
|
||
reply("FIRST-RUN-MARKER reasoning"),
|
||
reply("VERDICT: REJECT - FIRST-RUN-CRITIQUE"),
|
||
reply("more reasoning"),
|
||
reply("VERDICT: APPROVE"),
|
||
reply("second-run reasoning"),
|
||
reply("VERDICT: APPROVE"),
|
||
]
|
||
)
|
||
run_debate(client, "context A", max_rounds=2, meter=_meter())
|
||
run_debate(client, "context B", max_rounds=2, meter=_meter())
|
||
second_run_opening = client.prompts("proposer")[2]
|
||
assert "FIRST-RUN-MARKER" not in second_run_opening
|
||
assert "FIRST-RUN-CRITIQUE" not in second_run_opening
|
||
|
||
def test_every_turn_is_charged_on_the_meter(self) -> None:
|
||
client = ScriptedClient(
|
||
replies=[reply("reasoning", usage_tokens=7), reply("VERDICT: APPROVE", usage_tokens=5)]
|
||
)
|
||
meter = _meter()
|
||
run_debate(client, "context", max_rounds=1, meter=meter)
|
||
assert meter.tokens_used == 12
|
||
assert meter.rounds_used == 1
|
||
|
||
def test_the_turn_safety_net_sits_above_the_round_cap(self) -> None:
|
||
# §3 Step 3 / §8: an additional turn-count termination safety net
|
||
# ABOVE the round cap — it must never fire within a round-capped run,
|
||
# and must refuse a turn count beyond it.
|
||
from portfolio_optimiser_claude.loop import check_turn_safety_net
|
||
|
||
check_turn_safety_net(turns=2 * 3, max_rounds=3) # within: no raise
|
||
with pytest.raises(RuntimeError):
|
||
check_turn_safety_net(turns=2 * 3 + 3, max_rounds=3)
|
||
|
||
|
||
class TestParseCheckerVerdict:
|
||
def test_approve_is_parsed_case_insensitively(self) -> None:
|
||
verdict = parse_checker_verdict("the numbers hold.\nverdict: approve")
|
||
assert verdict.decision == "approve"
|
||
|
||
def test_reject_carries_the_trailing_text_as_reason(self) -> None:
|
||
verdict = parse_checker_verdict("VERDICT: REJECT - savings claim is double-counted")
|
||
assert verdict.decision == "reject"
|
||
assert verdict.reason == "savings claim is double-counted"
|
||
|
||
def test_the_reject_marker_takes_precedence(self) -> None:
|
||
verdict = parse_checker_verdict(
|
||
"VERDICT: APPROVE was my first instinct, but no.\nVERDICT: REJECT - stale baseline"
|
||
)
|
||
assert verdict.decision == "reject"
|
||
assert verdict.reason == "stale baseline"
|
||
|
||
def test_missing_marker_parses_as_absent(self) -> None:
|
||
verdict = parse_checker_verdict("looks fine to me")
|
||
assert verdict.decision == "absent"
|
||
|
||
def test_empty_text_parses_as_absent(self) -> None:
|
||
assert parse_checker_verdict("").decision == "absent"
|
||
|
||
|
||
class TestGenerationPrompt:
|
||
def test_generation_demands_the_raw_json_object_only(self) -> None:
|
||
# S10 post-mortem: 3 of 4 generation replies carried plausible JSON
|
||
# wrapped in a markdown fence + commentary — each one a full-price
|
||
# blind parse-retry (§3 Step 2). The prompt must forbid the wrapping.
|
||
client = ScriptedClient(
|
||
replies=[
|
||
reply("reasoning"),
|
||
reply("VERDICT: APPROVE"),
|
||
reply(json.dumps(VALID_PROPOSAL)),
|
||
]
|
||
)
|
||
run_project(client, "context", meter=_meter(), max_debate_rounds=1)
|
||
generation_prompt = client.prompts("proposer")[1]
|
||
assert "ONLY the raw JSON object" in generation_prompt
|
||
assert "no markdown fences" in generation_prompt
|