"""Load-bearing: EVERY evaluated approach gets its own judgeable artefacts (Trekk A5). Trekk A3/A4 made a run *evaluate* every commissioned approach and *report* what became of each. That closes the reporting half of krav 1 but not the learning half: ``write_outbox`` still wrote ONE proposal per run, so of three evaluated approaches only the selected one could ever receive an expert verdict. The other two taught the system nothing — the coverage report said they happened, and the learning loop never saw them. The defect class is a KEY COLLAPSE, and it has two independent halves, each pinned here: * the WRITER — one artefact pair per run means the non-selected approaches are never written; * the READER — ``hitl._read_outbox_proposals`` joins proposal to outcome on the ``run_id`` FIELD read from file content. Per-approach files that all carry the same ``run_id`` collapse onto one dict key (last write wins), so writing three pairs while joining on ``run_id`` alone still yields one pending row. This is the S3.2 key-collision class: two rows sharing one key silently become one. A third property is what makes the artefacts genuinely judgeable rather than merely present: each one must carry the verdict id THAT approach's proposal would be judged under (``verdicts.verdict_key``, the S3.2 content hash). Sharing one run-level verdict id would mean a single expert verdict marked all three approaches judged — the collapse again, one layer down. Detach points, each RED on its own: * write one artefact pair per run instead of one per evaluated approach; * key the outbox join on ``run_id`` alone -> ``hitl pending`` reports one row for three approaches; * stamp every per-approach artefact with the run's single verdict id -> judging one clears all. The control (``test_without_a_mandate_the_outbox_is_byte_unchanged``) proves the addition is inert on the no-mandate path: the same two filenames as before, carrying no ``approach_id`` key at all. """ from __future__ import annotations import json from collections.abc import Callable from pathlib import Path from portfolio_optimiser import hitl from portfolio_optimiser.mandate import OWN_PROPOSAL_ID, Approach, Mandate from portfolio_optimiser.run import run_project from portfolio_optimiser.simulation import ScriptedChatClient from portfolio_optimiser.verdicts import ( ProposalFeatures, VerdictStore, capture_verdict, write_verdict, ) BUNDLE_DIR = Path(__file__).resolve().parents[1] / "shared" / "examples" / "bygg-energi-mikro" _VERDICT_INPUT = {"decision": "approved", "rationale": "expert reviewed (sim)"} _RUN_ID = "run-a5" # BYGG-KONTOR-NORD: affected total = 300000 x 1.0 -> degenerate Monte Carlo P90 = 0.30 x 300000 # = 90000. A claim <= 90000 validates; a claim above it is REJECTED by the deterministic validator. def _reply(measure: str, claimed: int) -> str: return ( f'{{"measure":"{measure}","affected_items":' f'[{{"code":"ENERGI-TOTAL-EL","quantity":300000,"unit_cost":1.0}}],' f'"claimed_saving_nok":{claimed}}}' ) # Labels ABSENT from the bundle's own prose (the "LED-retrofit" trap: it appears in 6 bundle files, # so a client keyed on it would match every prompt through the context and prove nothing). _LED = Approach(id="led-retrofit", label="Behovsstyrt belysning i fellesarealer") _HVAC = Approach(id="hvac-swap", label="Utskifting av ventilasjonsaggregat") _LED_MEASURE = "Behovsstyrt belysning i fellesarealer" _HVAC_MEASURE = "Utskifting av ventilasjonsaggregat" _OWN_MEASURE = "Systemets eget forslag" #: LED validates (30k <= cap); HVAC is above the cap -> the validator rejects it. The three claims #: are DISTINCT, so each approach mints a distinct verdict key — a test where two approaches shared #: a claim could not tell per-approach keying from one shared key. #: #: Written as FLOATS where a verdict key is minted from them: ``SavingsProposal.claimed_saving_nok`` #: is typed ``float``, so pydantic coerces the JSON ``30000`` to ``30000.0`` — and ``_mint_id`` #: hashes the raw value, where ``30000`` and ``30000.0`` are different keys (the S3.2 magnitude #: rule). An expert judging the artefact reads the same coerced value back out of it. _LED_CLAIM, _HVAC_CLAIM, _OWN_CLAIM = 30_000, 200_000, 20_000 _REPLY_BY_LABEL = { _LED.label: _reply(_LED_MEASURE, _LED_CLAIM), _HVAC.label: _reply(_HVAC_MEASURE, _HVAC_CLAIM), } _DEFAULT_REPLY = _reply(_OWN_MEASURE, _OWN_CLAIM) def _select_reply(blob: str, _role: str) -> str: """Reply according to WHICH approach the prompt carries (canonical ``reply_selector`` seam — never a copied ``_inner_get_response`` body, S2.5 consolidation guard).""" return next((r for label, r in _REPLY_BY_LABEL.items() if label in blob), _DEFAULT_REPLY) def _factory(sink: list[str]) -> Callable[[str], ScriptedChatClient]: def factory(role: str) -> ScriptedChatClient: return ScriptedChatClient( sink=sink, role=role, reply_selector=_select_reply, default_reply=_DEFAULT_REPLY ) return factory _MANDATE = Mandate( objective="Cut energy cost without rebuilding.", approaches=(_LED, _HVAC), allow_own_proposals=True, ) async def _run(mandate: Mandate | None, outbox_dir: Path): sink: list[str] = [] return await run_project( "BYGG-KONTOR-NORD", "local", docs_dir=str(BUNDLE_DIR), bundle_dir=str(BUNDLE_DIR), verdict_input=_VERDICT_INPUT, store=VerdictStore(verdicts=[]), client_factory=_factory(sink), mandate=mandate, outbox_dir=str(outbox_dir), run_id=_RUN_ID, ) def _payload(path: Path) -> dict: return json.loads(path.read_text(encoding="utf-8")) def _verdict_key_for(measure: str, claimed: float) -> str: """The id an expert verdict on THAT proposal would key under — minted through the SAME public primitive the run uses, so the test cannot pass against a private copy of the hash.""" return capture_verdict( ProposalFeatures( affected_codes=frozenset({"ENERGI-TOTAL-EL"}), measure_type=measure, claimed_saving_nok=claimed, description=measure, ), "approved", "irrelevant to the id", ).id async def test_each_evaluated_approach_gets_its_own_artefacts(tmp_path: Path) -> None: """Three evaluated approaches -> three proposal artefacts, each carrying ITS OWN candidate. RED when the writer stays one-pair-per-run: only the selected approach's file exists, and the two the expert also commissioned cannot be judged. """ outbox = tmp_path / "outbox" await _run(_MANDATE, outbox) proposals = sorted(p.name for p in outbox.glob("*-proposal.json")) assert proposals == [ f"{_RUN_ID}-hvac-swap-proposal.json", f"{_RUN_ID}-led-retrofit-proposal.json", f"{_RUN_ID}-{OWN_PROPOSAL_ID}-proposal.json", ] by_approach = { _payload(p)["approach_id"]: _payload(p)["proposal"] for p in outbox.glob("*-proposal.json") } assert by_approach["led-retrofit"]["measure"] == _LED_MEASURE assert by_approach["hvac-swap"]["measure"] == _HVAC_MEASURE assert by_approach[OWN_PROPOSAL_ID]["measure"] == _OWN_MEASURE async def test_each_approach_outcome_is_written_with_its_own_status(tmp_path: Path) -> None: """The outcome half must follow its own approach too: the rejected approach's artefact carries the rejection, not the selected approach's success.""" outbox = tmp_path / "outbox" await _run(_MANDATE, outbox) by_approach = {_payload(p)["approach_id"]: _payload(p) for p in outbox.glob("*-outcome.json")} assert by_approach["led-retrofit"]["outcome_type"] == "validated" assert by_approach["hvac-swap"]["outcome_type"] == "rejected" assert by_approach["hvac-swap"]["reason"], "a rejected approach must carry the reason" assert by_approach[OWN_PROPOSAL_ID]["outcome_type"] == "validated" async def test_each_artefact_stamps_its_own_validator_decision(tmp_path: Path) -> None: """Provenance follows the artefact it stamps. RED when every per-approach artefact reuses the run's stamp: the rejected approach's file would then carry ``validator_decision="validated"`` — the artefact would claim the deterministic gate admitted a candidate it actually refused, which is the one field in the stamp nothing else can correct. """ outbox = tmp_path / "outbox" await _run(_MANDATE, outbox) decisions = { _payload(p)["approach_id"]: _payload(p)["provenance"]["validator_decision"] for p in outbox.glob("*-proposal.json") } assert decisions == { "led-retrofit": "validated", "hvac-swap": "rejected", OWN_PROPOSAL_ID: "validated", } async def test_hitl_pending_lists_each_approach_separately(tmp_path: Path) -> None: """The READER half. RED while the outbox join keys on ``run_id`` alone: three files sharing one ``run_id`` collapse to a single pending row, and two commissioned approaches disappear from the expert's queue even though their artefacts are on disk.""" outbox, inbox = tmp_path / "outbox", tmp_path / "inbox" inbox.mkdir() await _run(_MANDATE, outbox) rows = hitl.pending(str(outbox), str(inbox)) assert {r.approach_id for r in rows} == {"led-retrofit", "hvac-swap", OWN_PROPOSAL_ID} assert len({r.verdict_id for r in rows}) == 3, "each approach must be judgeable on its own key" async def test_judging_one_approach_leaves_the_others_pending(tmp_path: Path) -> None: """The keys must be the approaches' OWN verdict keys, not one run-level id. RED when every per-approach artefact is stamped with the run's single verdict id: one delivered verdict would then clear all three from the queue, and the two unjudged approaches would be reported as judged. """ outbox, inbox = tmp_path / "outbox", tmp_path / "inbox" await _run(_MANDATE, outbox) judged = _verdict_key_for(_LED_MEASURE, float(_LED_CLAIM)) write_verdict( str(inbox), capture_verdict( ProposalFeatures( affected_codes=frozenset({"ENERGI-TOTAL-EL"}), measure_type=_LED_MEASURE, claimed_saving_nok=float(_LED_CLAIM), description=_LED_MEASURE, ), "approved", "the expert judged this approach and only this one", ), ) rows = hitl.pending(str(outbox), str(inbox)) assert judged not in {r.verdict_id for r in rows} assert {r.approach_id for r in rows} == {"hvac-swap", OWN_PROPOSAL_ID} async def test_without_a_mandate_the_outbox_is_byte_unchanged(tmp_path: Path) -> None: """Control: the no-mandate path keeps today's two filenames and carries NO ``approach_id`` key. A run nobody commissioned has no approaches to key on, and adding a null field would change the bytes of every existing artefact (the writers are byte-deterministic by contract). """ outbox = tmp_path / "outbox" await _run(None, outbox) written = sorted(p.name for p in outbox.glob("*.json")) assert written == [ f"{_RUN_ID}-outcome.json", f"{_RUN_ID}-proposal.json", f"{_RUN_ID}-runconfig.json", ] assert "approach_id" not in _payload(outbox / f"{_RUN_ID}-proposal.json") assert "approach_id" not in _payload(outbox / f"{_RUN_ID}-outcome.json")