feat(loop): S8 — D7 agentic loop: budget meter, maker-checker gate, informed refinement

Spec §3 steps 2–5 + §8, TDD-ed offline (scripted, honesty-marked stand-in):
- budget.py: BudgetMeter over TerminationContract — provider-reported usage
  only (missing usage fails closed), structured BudgetExceeded stop event.
- loop.py: ModelClient protocol; blind parse-retry generation (never silent
  repair); round-capped debate with turn safety net and mandated VERDICT
  line; opt-in-reject checker gate (explicit REJECT overrides a validated
  outcome, validator rejection stands); most-recent-reason-verbatim informed
  refinement under max_attempts; validator_decision stamped BEFORE override,
  checker_decision as its own result field (§9, never conflated).
- 45 new tests (121 total, no API key); four detach proofs run RED and
  reverted green: checker override, informed block, surfaced checker output,
  stamp-before-override.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01QdSfQdND84oeq2mbjueLTS
This commit is contained in:
Kjell Tore Guttormsen 2026-07-03 07:21:02 +02:00
commit 9a4caeb419
7 changed files with 1003 additions and 0 deletions

View file

@ -0,0 +1,162 @@
"""Load-bearing: the checker gate (method-spec §3 Step 4, §11).
Either the checker actually gates, or the debate must not be called
maker-checker. RED-conditions proved here: the checker's surfaced output is
detached from the gate, OR its explicit REJECT no longer overrides an
otherwise-validated outcome. The gate is opt-in-reject (fail-open): APPROVE or
a missing marker never blocks; a validator rejection stands regardless of the
checker; the two falsifiers are recorded SEPARATELY (§9) the provenance
field mirrors only the validator, the checker's decision is its own field.
"""
from __future__ import annotations
import json
from _scripted import ScriptedClient, reply
from portfolio_optimiser_claude.budget import BudgetMeter
from portfolio_optimiser_claude.contracts import TerminationContract
from portfolio_optimiser_claude.loop import (
CheckerVerdict,
apply_checker_gate,
run_project,
)
from portfolio_optimiser_claude.validator import Rejection, ValidatedProposal
# 100 × 1000 = 100_000 NOK total; nominal feasible 30_000 == p90 (no band).
VALID_PROPOSAL = {
"project_id": "p1",
"measure": "led-retrofit",
"affected_items": [{"code": "E01", "quantity": 100, "unit_cost": 1000}],
"claimed_saving_nok": 25000,
}
# IR-valid (≤ total) but above p90 → the validator rejects it.
OVERCLAIMED_PROPOSAL = dict(VALID_PROPOSAL, claimed_saving_nok=90000)
VALIDATED = ValidatedProposal(
validates=True,
claimed_saving_nok=25000,
nominal_feasible=30000,
p10=30000,
p50=30000,
p90=30000,
)
def _meter() -> BudgetMeter:
return BudgetMeter(TerminationContract(max_rounds=50, max_tokens=10_000))
class TestApplyCheckerGate:
def test_explicit_reject_overrides_a_validated_outcome(self) -> None:
# THE load-bearing seam: red if the override is detached.
verdict = CheckerVerdict(decision="reject", reason="double-counted savings")
outcome = apply_checker_gate(VALIDATED, verdict)
assert isinstance(outcome, Rejection)
assert outcome.reason.startswith("double-counted savings")
def test_approve_never_blocks(self) -> None:
verdict = CheckerVerdict(decision="approve", reason=None)
assert apply_checker_gate(VALIDATED, verdict) is VALIDATED
def test_absent_marker_never_blocks(self) -> None:
# Fail-open: the validator remains the sole gate on such runs.
verdict = CheckerVerdict(decision="absent", reason=None)
assert apply_checker_gate(VALIDATED, verdict) is VALIDATED
def test_a_validator_rejection_stands_regardless_of_the_checker(self) -> None:
validator_rejection = Rejection(reason="claimed saving exceeds the feasible bound")
verdict = CheckerVerdict(decision="approve", reason=None)
assert apply_checker_gate(validator_rejection, verdict) is validator_rejection
class TestRunProjectGating:
def test_checker_reject_flips_an_otherwise_validated_run(self) -> None:
# Speiltest (sesjonsplan S8): numbers pass the validator, the checker's
# surfaced REJECT flips the run outcome — red without the override.
client = ScriptedClient(
replies=[
reply("reasoning"),
reply("VERDICT: REJECT - assumptions unsupported"),
reply(json.dumps(VALID_PROPOSAL)),
]
)
result = run_project(client, "context", meter=_meter(), max_debate_rounds=1)
assert isinstance(result.outcome, Rejection)
assert result.outcome.reason.startswith("assumptions unsupported")
# §9: the two falsifiers are never conflated — the provenance mirror
# says the NUMBERS passed; the checker's decision is its own field.
assert result.validator_decision == "validated"
assert result.checker_decision == "reject"
def test_the_gate_consumes_the_checkers_last_surfaced_output(self) -> None:
# Two debate rounds, two distinct REJECT reasons: the gate must carry
# the LAST one — red if the surfaced-output seam is detached.
client = ScriptedClient(
replies=[
reply("reasoning v1"),
reply("VERDICT: REJECT - EARLY-REASON"),
reply("reasoning v2"),
reply("VERDICT: REJECT - FINAL-REASON"),
reply(json.dumps(VALID_PROPOSAL)),
]
)
result = run_project(client, "context", meter=_meter(), max_debate_rounds=2)
assert isinstance(result.outcome, Rejection)
assert result.outcome.reason.startswith("FINAL-REASON")
assert "EARLY-REASON" not in result.outcome.reason
def test_an_approved_run_passes_through_ungated(self) -> None:
client = ScriptedClient(
replies=[
reply("reasoning"),
reply("sound. VERDICT: APPROVE"),
reply(json.dumps(VALID_PROPOSAL)),
]
)
result = run_project(client, "context", meter=_meter(), max_debate_rounds=1)
assert isinstance(result.outcome, ValidatedProposal)
assert result.validator_decision == "validated"
assert result.checker_decision == "approve"
def test_a_missing_marker_is_fail_open(self) -> None:
client = ScriptedClient(
replies=[
reply("reasoning"),
reply("no explicit verdict line here"),
reply(json.dumps(VALID_PROPOSAL)),
]
)
result = run_project(client, "context", meter=_meter(), max_debate_rounds=1)
assert isinstance(result.outcome, ValidatedProposal)
assert result.checker_decision == "absent"
def test_a_validator_rejection_stands_even_when_the_checker_approves(self) -> None:
client = ScriptedClient(
replies=[
reply("reasoning"),
reply("VERDICT: APPROVE"),
reply(json.dumps(OVERCLAIMED_PROPOSAL)),
reply(json.dumps(OVERCLAIMED_PROPOSAL)),
reply(json.dumps(OVERCLAIMED_PROPOSAL)),
]
)
result = run_project(client, "context", meter=_meter(), max_debate_rounds=1, max_attempts=3)
assert isinstance(result.outcome, Rejection)
assert result.validator_decision == "rejected"
assert result.checker_decision == "approve"
def test_the_debates_converged_output_feeds_generation(self) -> None:
# §3 Step 3: the debate's converged proposer output feeds Step 2's
# generation context.
client = ScriptedClient(
replies=[
reply("CONVERGED-REASONING-MARKER"),
reply("VERDICT: APPROVE"),
reply(json.dumps(VALID_PROPOSAL)),
]
)
run_project(client, "context", meter=_meter(), max_debate_rounds=1)
generation_prompt = client.prompts("proposer")[1]
assert "CONVERGED-REASONING-MARKER" in generation_prompt