feat(portfolio): K11 — per-run value report, pure projection over the three layers (parity row 25) [skip-docs]
The S5.4 analog: every run/portfolio pass can produce a deterministic value report — modelled → expert-corrected → realized, goal progress, a quantified learning effect, cost against value — with no model call, no clock and no new state. It is a PURE PROJECTION over what is already persisted (K5 outbox pairs, §4.2 inbox verdicts, K1 ledger), joined on the verdict_id K5 mints. The honesty rule (§1) sets the shape, not the layout: - approved -> the claim stands (quantified) - rejected -> the claim is void, 0 (quantified — an earned zero) - approved_with_adjustment -> §4.2 carries NO adjusted amount, so the corrected value is UNQUANTIFIED, never back-filled with the claim - no verdict -> realized is UNMARKED, never zero-that-reads-as-judged and never the modelled figure Partial quantification is counted in the output (2 of 4 …, 2 UNMARKED) rather than summed into a full-looking total. Learning is measured, not asserted: a rising approval share is reported only alongside the modelled→corrected gap that shrank behind it, over cohorts split by run_id order. Cost (USD, a K6 upper bound) and value (NOK) sit side by side and are never divided — no sourced FX rate exists here, and a ratio would invent one. Surfaces: standalone CLI (valuereport) and an opt-in --value-report on run.py, which requires --outbox and is refused BEFORE any spend without one; the report is written on both run outcomes and never rewrites the run's exit code (a budget stop stays a budget stop). Six seams detach-proven RED: honesty boundary (corrected + realized), gap arithmetic, projection purity, run-seam wiring, pre-spend fail-fast, both-outcome reporting. Fixtures are COMMITTED and generated with the real primitives, so the id-join under test is genuine. Note on the purity test: it was green-but-dead in its first form. Snapshotting the committed fixture tree in place let an earlier test's stray write pre-seed the 'before' snapshot, so the detached write reproduced it byte-for-byte. Every test now projects from a per-test copy, and a pinned file-set test guards the committed tree. Found by running the detach proof — which is what §11 is for. portfolio.py is deliberately NOT wired: run_portfolio persists nothing, so there is nothing for a projection to read. Its docstring now says that instead of promising the wiring it did not get. 562 -> 584 tests green; ruff + mypy --strict clean over 27 src files. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01MQu2xxwedckjU56byu1aUG
This commit is contained in:
parent
d0107e7f8c
commit
4dcdd8017a
18 changed files with 1477 additions and 6 deletions
164
tests/test_valuereport_seam_loadbearing.py
Normal file
164
tests/test_valuereport_seam_loadbearing.py
Normal file
|
|
@ -0,0 +1,164 @@
|
|||
"""Value-report seam in run.py — LOAD-BEARING (S5.4-analog; §11; paritetsrad 25; K11).
|
||||
|
||||
The seam this file keeps alive: the deliverable entrance can produce the value
|
||||
report as an OPT-IN side product of a run, projecting the layers that are on
|
||||
disk when the run finishes — including the outbox pair the run itself just
|
||||
filed. The report is a projection, never a second source of truth, so it can
|
||||
neither change the run's verdict nor be produced without an outbox to project.
|
||||
|
||||
Detach proofs:
|
||||
|
||||
* Report wired: an opted-in run writes the JSON, and it carries THIS run's
|
||||
proposal as modelled value. Detach point: drop the ``write_value_report`` call
|
||||
in ``main`` → no file → ``test_run_writes_the_value_report`` RED.
|
||||
* Opt-in requires an outbox, refused BEFORE any spend: ``--value-report`` without
|
||||
``--outbox`` exits via ``parser.error`` and the scripted client is never
|
||||
constructed. Detach point: remove the fail-fast → the run spends and only then
|
||||
discovers it has nothing to project → ``test_value_report_without_outbox_
|
||||
refuses_before_any_spend`` RED.
|
||||
* Both outcomes: a budget-stopped run still writes the report AND still exits 3.
|
||||
Detach point: produce the report only on success, or let a report failure
|
||||
overwrite the run's exit code → ``test_budget_stop_still_reports_and_keeps_
|
||||
exit_code`` RED.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from _scripted import ScriptedClient, reply
|
||||
|
||||
from portfolio_optimiser_claude.contracts import Contracts
|
||||
from portfolio_optimiser_claude.ir import load_validator_input
|
||||
from portfolio_optimiser_claude.loop import ModelClient
|
||||
from portfolio_optimiser_claude.run import main as run_main
|
||||
|
||||
BUNDLE = Path(__file__).resolve().parents[1] / "shared" / "examples" / "bygg-energi-mikro"
|
||||
|
||||
|
||||
def _scripted_factory(replies: list[object]) -> tuple[object, list[ScriptedClient]]:
|
||||
created: list[ScriptedClient] = []
|
||||
|
||||
def factory(contracts: Contracts, max_budget_usd_per_call: float) -> ModelClient:
|
||||
client = ScriptedClient(replies=list(replies)) # type: ignore[arg-type]
|
||||
created.append(client)
|
||||
return client
|
||||
|
||||
return factory, created
|
||||
|
||||
|
||||
def _happy_replies() -> list[object]:
|
||||
return [
|
||||
reply("debate reasoning"),
|
||||
reply("VERDICT: APPROVE"),
|
||||
reply(json.dumps(load_validator_input(BUNDLE).model_dump())),
|
||||
]
|
||||
|
||||
|
||||
def _run_args(tmp_path: Path, *extra: str) -> list[str]:
|
||||
return [
|
||||
"--bundle",
|
||||
str(BUNDLE),
|
||||
"--out",
|
||||
str(tmp_path / "out"),
|
||||
"--outbox",
|
||||
str(tmp_path / "outbox"),
|
||||
"--run-id",
|
||||
"r-001",
|
||||
*extra,
|
||||
]
|
||||
|
||||
|
||||
def test_run_writes_the_value_report(tmp_path: Path) -> None:
|
||||
"""LOAD-BEARING (report wired): the run's own pair shows up as modelled value."""
|
||||
destination = tmp_path / "reports" / "value.json"
|
||||
factory, _ = _scripted_factory(_happy_replies())
|
||||
|
||||
code = run_main(_run_args(tmp_path, "--value-report", str(destination)), client_factory=factory)
|
||||
|
||||
assert code == 0
|
||||
payload = json.loads(destination.read_text(encoding="utf-8"))
|
||||
assert payload["n_proposals"] == 1
|
||||
assert payload["modelled_nok"] == load_validator_input(BUNDLE).claimed_saving_nok
|
||||
# No inbox, no ledger → the expert stages are UNMARKED, never mirrored from modelled.
|
||||
assert payload["expert_corrected_nok"] is None
|
||||
assert payload["n_pending"] == 1
|
||||
assert payload["projects"][0]["realized_nok"] is None
|
||||
|
||||
|
||||
def test_value_report_without_outbox_refuses_before_any_spend(tmp_path: Path) -> None:
|
||||
"""LOAD-BEARING (fail-fast): nothing to project → refuse BEFORE constructing a client."""
|
||||
factory, created = _scripted_factory(_happy_replies())
|
||||
|
||||
with pytest.raises(SystemExit):
|
||||
run_main(
|
||||
[
|
||||
"--bundle",
|
||||
str(BUNDLE),
|
||||
"--out",
|
||||
str(tmp_path / "out"),
|
||||
"--value-report",
|
||||
str(tmp_path / "value.json"),
|
||||
],
|
||||
client_factory=factory,
|
||||
)
|
||||
|
||||
assert created == [] # refused before the client existed → no spend
|
||||
assert not (tmp_path / "value.json").exists()
|
||||
|
||||
|
||||
def test_budget_stop_still_reports_and_keeps_exit_code(tmp_path: Path) -> None:
|
||||
"""LOAD-BEARING (both outcomes): a stop is still reported, and stays exit 3."""
|
||||
destination = tmp_path / "value.json"
|
||||
factory, _ = _scripted_factory([reply("debate reasoning", usage_tokens=10)])
|
||||
|
||||
code = run_main(
|
||||
_run_args(tmp_path, "--max-tokens", "5", "--value-report", str(destination)),
|
||||
client_factory=factory,
|
||||
)
|
||||
|
||||
assert code == 3 # the run's own verdict is untouched by reporting
|
||||
payload = json.loads(destination.read_text(encoding="utf-8"))
|
||||
assert payload["n_proposals"] == 0 # a stop files no pair — an honest empty picture
|
||||
|
||||
|
||||
def test_report_failure_never_rewrites_a_budget_stop(tmp_path: Path) -> None:
|
||||
"""A malformed ledger fails the report, but a budget stop still reports as a stop."""
|
||||
broken_ledger = tmp_path / "ledger.json"
|
||||
broken_ledger.write_text('{"entries": {}}', encoding="utf-8")
|
||||
factory, _ = _scripted_factory([reply("debate reasoning", usage_tokens=10)])
|
||||
|
||||
code = run_main(
|
||||
_run_args(
|
||||
tmp_path,
|
||||
"--max-tokens",
|
||||
"5",
|
||||
"--value-report",
|
||||
str(tmp_path / "value.json"),
|
||||
"--ledger",
|
||||
str(broken_ledger),
|
||||
),
|
||||
client_factory=factory,
|
||||
)
|
||||
|
||||
assert code == 3 # NOT rewritten to 1 — the budget stop is the bigger news
|
||||
assert not (tmp_path / "value.json").exists()
|
||||
|
||||
|
||||
def test_report_failure_fails_an_otherwise_clean_run(tmp_path: Path) -> None:
|
||||
"""The operator asked for a report and did not get one → the command is non-zero."""
|
||||
broken_ledger = tmp_path / "ledger.json"
|
||||
broken_ledger.write_text("[]", encoding="utf-8")
|
||||
factory, _ = _scripted_factory(_happy_replies())
|
||||
|
||||
code = run_main(
|
||||
_run_args(
|
||||
tmp_path, "--value-report", str(tmp_path / "value.json"), "--ledger", str(broken_ledger)
|
||||
),
|
||||
client_factory=factory,
|
||||
)
|
||||
|
||||
assert code == 1
|
||||
assert (tmp_path / "outbox" / "r-001-proposal.json").exists() # the run itself still landed
|
||||
Loading…
Add table
Add a link
Reference in a new issue