feat(run): every commissioned approach is evaluated, and every one is reported

Carrying an approach into the prompt only half-answers krav 1. The expert asked
for their approaches to be CONCRETELY EVALUATED, which means each must reach a
verdict and each verdict must be visible. `run_project(mandate=...)` evaluates
every commissioned approach in turn — the run's own proposal last, when allowed —
each under the SAME meter. No new loop: the caps already in force are the bound.

`RunResult.coverage` is the settlement, one row per approach: validated (with the
figure), rejected (with the validator's reason verbatim), or not_evaluated (with
why). `not_evaluated` is the row that earns the type its keep — an approach the
run never reached must be reported as unreached, because an omitted row is
indistinguishable from an approach nobody ordered. That silence is the defect
class krav 1 is asking us to remove.

Budget exhaustion mid-list is reported, not swallowed. But if the FIRST approach
exhausts it there is nothing honest to return, so BudgetExceeded propagates
exactly as before — a run that produced nothing must still fail loudly.

RunResult stays single-outcome (portfolio aggregation, outbox artefacts and HITL
keying all rest on that). The choice is deterministic: highest validated saving,
ties by mandate order — never whichever ran last.

Load-bearing MEASURED against the whole 702-test suite. FIVE mutations red:
evaluate only the first approach (5 red) · drop the rejected rows (3) · ignore
allow_own_proposals (1) · select produced[-1] (1) · select produced[0] (1).

The sixth measurement is why this commit exists in this shape: the ordering
mutation FIRST STAYED GREEN. The test had placed the bigger approach last, where
"highest saving" and "whichever ran last" give the same answer, so an
order-dependent implementation passed it. A scenario that cannot separate two
implementations proves nothing about either — the test now pins BOTH orderings,
and each mutation direction fails one of them.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01ULCqjLF61rehj5cZmdUoR3

src/portfolio_optimiser/mandate.py

@@ -29,7 +29,9 @@ Two refusals are load-bearing, both at construction time:
 
 from __future__ import annotations
 
+from dataclasses import dataclass
 from pathlib import Path
+from typing import Literal
 
 from pydantic import BaseModel, Field, model_validator
 
@@ -86,6 +88,26 @@ class Mandate(BaseModel):
         return self
 
 
+@dataclass(frozen=True)
+class ApproachOutcome:
+    """What became of ONE commissioned approach — one row of the run's coverage report.
+
+    ``not_evaluated`` is the row that earns this type its keep: an approach the run never got to
+    (budget exhausted, pass stopped) must be reported as *not evaluated*, never omitted. An omitted
+    row is indistinguishable from an approach nobody ordered, which is exactly the silence krav 1
+    is asking us to remove.
+
+    ``detail`` carries the validator's reason on a rejection, or why an approach went unevaluated;
+    ``saving_nok`` is set only for a validated row (the claimed figure the validator admitted).
+    """
+
+    id: str
+    label: str
+    status: Literal["validated", "rejected", "not_evaluated"]
+    detail: str = ""
+    saving_nok: float | None = None
+
+
 def load_mandate(path: str | Path) -> Mandate:
     """Fail-fast standalone loader for a run mandate (mirrors ``dimension.load_dimension``).
 

src/portfolio_optimiser/run.py

@@ -27,7 +27,7 @@ from __future__ import annotations
 
 import asyncio
 import json
-from collections.abc import Callable, Iterable, Sequence
+from collections.abc import Awaitable, Callable, Iterable, Sequence
 from dataclasses import dataclass, replace
 from pathlib import Path
 from typing import Any, Literal, cast
@@ -38,6 +38,7 @@ from pydantic import ValidationError
 from portfolio_optimiser.backends import Profile, get_backend, resolve_model
 from portfolio_optimiser.budget import (
     Budget,
+    BudgetExceeded,
     BudgetMiddleware,
     BudgetRefused,
     PortfolioMeter,
@@ -54,6 +55,12 @@ from portfolio_optimiser.datasource import (
 from portfolio_optimiser.dimension import Dimension, admits, load_dimension
 from portfolio_optimiser.generate import generate_via_llm
 from portfolio_optimiser.ir import SavingsProposal
+from portfolio_optimiser.mandate import (
+    OWN_PROPOSAL_ID,
+    Approach,
+    ApproachOutcome,
+    Mandate,
+)
 from portfolio_optimiser.provenance import ProvenanceStamp
 from portfolio_optimiser.reference_domain import Project, load_reference_projects
 from portfolio_optimiser.validator import Rejection, ValidatedProposal, baseline_from_project
@@ -100,6 +107,10 @@ class RunResult:
     store: VerdictStore
     debate_output: str
     checker_verdict: str = "absent"
+    #: One row per commissioned approach (Trekk A4). EMPTY when the run had no mandate — an empty
+    #: report is honest there, because nothing was ordered. It defaults so every existing
+    #: constructor call and every frozen aggregate over ``RunResult`` is unaffected.
+    coverage: tuple[ApproachOutcome, ...] = ()
 
 
 @dataclass(frozen=True)
@@ -199,6 +210,79 @@ class PortfolioResult:
     budget_stop: BudgetStop | None = None
 
 
+def _coverage_row(
+    row_id: str, label: str, outcome: ValidatedProposal | Rejection
+) -> ApproachOutcome:
+    """One coverage row from one evaluated approach. A rejection carries the validator's reason
+    verbatim — a bare status would tell the expert their approach failed without telling them why,
+    which is the part they can actually act on."""
+    if isinstance(outcome, ValidatedProposal):
+        return ApproachOutcome(
+            id=row_id,
+            label=label,
+            status="validated",
+            saving_nok=outcome.proposal.claimed_saving_nok,
+        )
+    return ApproachOutcome(id=row_id, label=label, status="rejected", detail=outcome.reason)
+
+
+def _select_outcome(
+    produced: list[tuple[int, ValidatedProposal | Rejection]],
+) -> ValidatedProposal | Rejection:
+    """Pick the ONE outcome a ``RunResult`` carries out of everything the mandate produced.
+
+    ``RunResult`` stays single-outcome on purpose: portfolio aggregation, the outbox artefacts and
+    the HITL verdict keying all rest on there being exactly one. The choice is deterministic —
+    highest validated saving, ties broken by mandate order — so it can never depend on which
+    approach happened to run last. When nothing validated, the FIRST rejection stands, which keeps
+    a fully-rejected mandate reporting a typed ``Rejection`` rather than a fabricated success.
+    """
+    validated = [(i, o) for i, o in produced if isinstance(o, ValidatedProposal)]
+    if validated:
+        return min(validated, key=lambda t: (-t[1].proposal.claimed_saving_nok, t[0]))[1]
+    return produced[0][1]
+
+
+async def _evaluate_mandate(
+    mandate: Mandate,
+    evaluate: Callable[[Approach | None], Awaitable[ValidatedProposal | Rejection]],
+) -> tuple[ValidatedProposal | Rejection, tuple[ApproachOutcome, ...]]:
+    """Evaluate every commissioned approach, then the run's own proposal when allowed, and report
+    what became of each (Trekk A3/A4).
+
+    Budget exhaustion mid-list is REPORTED, not swallowed: the approaches that were never reached
+    become ``not_evaluated`` rows. But if the very first approach exhausts the budget there is
+    nothing honest to return, so ``BudgetExceeded`` propagates exactly as it did before — a run
+    that produced nothing must still fail loudly rather than hand back an empty report.
+    """
+    plan: list[tuple[str, str, Approach | None]] = [(a.id, a.label, a) for a in mandate.approaches]
+    if mandate.allow_own_proposals:
+        plan.append((OWN_PROPOSAL_ID, "the system's own proposal", None))
+
+    rows: list[ApproachOutcome] = []
+    produced: list[tuple[int, ValidatedProposal | Rejection]] = []
+    for index, (row_id, label, approach) in enumerate(plan):
+        try:
+            outcome = await evaluate(approach)
+        except BudgetExceeded:
+            if not produced:
+                raise
+            rows.extend(
+                ApproachOutcome(
+                    id=rid,
+                    label=lbl,
+                    status="not_evaluated",
+                    detail="budget exhausted before this approach was evaluated",
+                )
+                for rid, lbl, _ in plan[index:]
+            )
+            break
+        produced.append((index, outcome))
+        rows.append(_coverage_row(row_id, label, outcome))
+
+    return _select_outcome(produced), tuple(rows)
+
+
 def _authored_texts(result: Any, name: str) -> list[str]:
     """The texts of ``get_outputs()`` entries authored by participant ``name`` (proposer/checker),
     in surfaced order. MAF surfaces ``author_name`` on each output's ``messages`` — NOT on the
@@ -317,6 +401,7 @@ async def run_project(
     live_dry_run: bool = False,
     semantic_retrieval: bool = False,
     embedder: Embedder | None = None,
+    mandate: Mandate | None = None,
 ) -> RunResult | DryRunReport:
     """Run the vertical slice for ONE project. ``client_factory`` is the test-injection seam
     (defaults to the real backend). ``verdict_input`` carries the expert decision/rationale
@@ -487,11 +572,22 @@ async def run_project(
         ).format_fewshot()
         gen_context = f"{fewshot}\n\n{gen_context}"
 
-    # 5. Structured candidate -> blocking validation on the NUMBERS; token bound = the meter.
+    # 5. Structured candidate(s) -> blocking validation on the NUMBERS; token bound = the meter.
+    # Without a mandate this is the single pre-Trekk-A call, unchanged. With one, every
+    # commissioned approach is evaluated in turn (and the run's own proposal last, when allowed),
+    # each under the SAME meter — no new unbounded loop; the caps already in force are the bound.
     proposer_client = factory("proposer")
-    validator_outcome = await generate_via_llm(
-        proposer_client, project, gen_context, meter, baseline=baseline
-    )
+
+    async def _evaluate(approach: Approach | None) -> ValidatedProposal | Rejection:
+        return await generate_via_llm(
+            proposer_client, project, gen_context, meter, baseline=baseline, approach=approach
+        )
+
+    coverage: tuple[ApproachOutcome, ...] = ()
+    if mandate is None:
+        validator_outcome = await _evaluate(None)
+    else:
+        validator_outcome, coverage = await _evaluate_mandate(mandate, _evaluate)
     proposal = validator_outcome.proposal
 
     # 6. First-class provenance stamp (authoritative; independent of MAF Annotation).
@@ -579,6 +675,7 @@ async def run_project(
         store=store,
         debate_output=debate_output,
         checker_verdict=checker_decision,
+        coverage=coverage,
     )
 
 

tests/test_mandate_run_loadbearing.py

@@ -0,0 +1,202 @@
+"""Load-bearing: a run EVALUATES every commissioned approach, and REPORTS what happened to each
+(Trekk A3/A4 — krav 1 and 2).
+
+Krav 1 is only half-met by carrying an approach into the prompt (``test_mandate_generation_
+loadbearing.py``): the expert asked for their approaches to be *concretely evaluated*, which means
+each one must reach a verdict and each verdict must be visible. The defect class this pins is
+silence — an approach that was commissioned but never evaluated, or evaluated and quietly dropped
+because a different one won, is indistinguishable from one that was never ordered.
+
+Detach points, each RED on its own:
+
+* run one generation instead of one per approach -> the coverage report loses rows;
+* drop the rejected row (report only what validated) -> the expert's approach vanishes silently;
+* drop the own-proposal row -> "og/eller" becomes "or".
+
+The control (``test_no_mandate_runs_exactly_one_generation_with_empty_coverage``) proves the whole
+addition is inert without a mandate: today's single-shot path, unchanged.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+from pathlib import Path
+
+import pytest
+
+from portfolio_optimiser.mandate import OWN_PROPOSAL_ID, Approach, Mandate
+from portfolio_optimiser.simulation import ScriptedChatClient
+from portfolio_optimiser.run import run_project
+from portfolio_optimiser.validator import Rejection, ValidatedProposal
+from portfolio_optimiser.verdicts import VerdictStore
+
+BUNDLE_DIR = Path(__file__).resolve().parents[1] / "shared" / "examples" / "bygg-energi-mikro"
+_VERDICT_INPUT = {"decision": "approved", "rationale": "expert reviewed (sim)"}
+
+# BYGG-KONTOR-NORD: affected total = 300000 x 1.0 -> degenerate Monte Carlo P90 = 0.30 x 300000
+# = 90000. A claim <= 90000 validates; a claim above it is REJECTED by the deterministic validator.
+
+
+def _reply(measure: str, claimed: int) -> str:
+    return (
+        f'{{"measure":"{measure}","affected_items":'
+        f'[{{"code":"ENERGI-TOTAL-EL","quantity":300000,"unit_cost":1.0}}],'
+        f'"claimed_saving_nok":{claimed}}}'
+    )
+
+
+# Labels are chosen to be ABSENT from the bundle's own prose: "LED-retrofit" appears in 6 of the
+# bundle's files, so a client keyed on it would match every prompt through the context and prove
+# nothing about which approach was bound to which call.
+_LED = Approach(id="led-retrofit", label="Behovsstyrt belysning i fellesarealer")
+_HVAC = Approach(id="hvac-swap", label="Utskifting av ventilasjonsaggregat")
+
+#: LED validates (30k <= cap); HVAC is above the cap -> the validator rejects it.
+_REPLY_BY_LABEL = {
+    _LED.label: _reply("Behovsstyrt belysning i fellesarealer", 30_000),
+    _HVAC.label: _reply("Utskifting av ventilasjonsaggregat", 200_000),
+}
+_DEFAULT_REPLY = _reply("Systemets eget forslag", 20_000)
+
+
+def _select_reply(blob: str, _role: str) -> str:
+    """Reply according to WHICH approach the prompt carries — so a per-approach outcome can only
+    differ if the loop really bound that approach to that call. Plugged into the CANONICAL
+    ``ScriptedChatClient`` selector seam rather than a copied ``_inner_get_response`` body (S2.5
+    consolidation guard)."""
+    return next((r for label, r in _REPLY_BY_LABEL.items() if label in blob), _DEFAULT_REPLY)
+
+
+def _factory(sink: list[str]) -> Callable[[str], ScriptedChatClient]:
+    def factory(role: str) -> ScriptedChatClient:
+        return ScriptedChatClient(
+            sink=sink, role=role, reply_selector=_select_reply, default_reply=_DEFAULT_REPLY
+        )
+
+    return factory
+
+
+def _generation_prompts(sink: list[str]) -> list[str]:
+    """Generation-call prompts only (``_build_messages`` embeds 'SavingsProposal'), isolated from
+    the debate-round prompts sharing the sink."""
+    return [p for p in sink if "SavingsProposal" in p]
+
+
+async def _run(mandate: Mandate | None, sink: list[str]):
+    return await run_project(
+        "BYGG-KONTOR-NORD",
+        "local",
+        docs_dir=str(BUNDLE_DIR),
+        bundle_dir=str(BUNDLE_DIR),
+        verdict_input=_VERDICT_INPUT,
+        store=VerdictStore(verdicts=[]),
+        client_factory=_factory(sink),
+        mandate=mandate,
+    )
+
+
+def _row(result, approach_id: str):
+    return next((c for c in result.coverage if c.id == approach_id), None)
+
+
+async def test_every_commissioned_approach_is_evaluated() -> None:
+    """Both commissioned approaches are generated for — one prompt each, each bound to its own
+    approach. One generation for two approaches means only one was ever evaluated."""
+    sink: list[str] = []
+    mandate = Mandate(
+        objective="Cut energy cost without rebuilding.",
+        approaches=(_LED, _HVAC),
+        allow_own_proposals=False,
+    )
+    result = await _run(mandate, sink)
+
+    prompts = _generation_prompts(sink)
+    assert sum(_LED.label in p for p in prompts) >= 1
+    assert sum(_HVAC.label in p for p in prompts) >= 1
+    assert {c.id for c in result.coverage} == {"led-retrofit", "hvac-swap"}
+
+
+async def test_a_rejected_approach_is_reported_not_dropped() -> None:
+    """The approach the validator rejected is STILL on the report, with the reason — this is the
+    silence the coverage report exists to prevent."""
+    sink: list[str] = []
+    mandate = Mandate(
+        objective="Cut energy cost without rebuilding.",
+        approaches=(_LED, _HVAC),
+        allow_own_proposals=False,
+    )
+    result = await _run(mandate, sink)
+
+    led, hvac = _row(result, "led-retrofit"), _row(result, "hvac-swap")
+    assert led is not None and hvac is not None
+    assert led.status == "validated"
+    assert hvac.status == "rejected"
+    assert hvac.detail, "a rejected approach must carry the validator's reason, not a bare status"
+
+
+async def test_own_proposal_is_reported_alongside_commissioned_ones() -> None:
+    """'og/eller': with ``allow_own_proposals`` the system's own candidate is evaluated too, and
+    reported under its own reserved row rather than merged into an expert's."""
+    sink: list[str] = []
+    mandate = Mandate(
+        objective="Cut energy cost without rebuilding.",
+        approaches=(_LED,),
+        allow_own_proposals=True,
+    )
+    result = await _run(mandate, sink)
+
+    assert {c.id for c in result.coverage} == {"led-retrofit", OWN_PROPOSAL_ID}
+
+
+@pytest.mark.parametrize("big_first", [True, False])
+async def test_outcome_is_the_best_validated_candidate_deterministically(big_first: bool) -> None:
+    """With more than one validated approach the run still returns ONE outcome (portfolio
+    aggregation, outbox and HITL keying rest on that), chosen by highest validated saving.
+
+    BOTH orderings are exercised, and that is the whole point: a first run of this test placed the
+    bigger approach LAST, where "highest saving" and "whichever ran last" give the same answer — an
+    order-dependent implementation passed it (MEASURED: the mutation stayed green). A test whose
+    scenario cannot separate the two implementations proves nothing about either. With both
+    orderings pinned, ``produced[-1]`` fails one case and ``produced[0]`` fails the other.
+    """
+    sink: list[str] = []
+    big = Approach(id="big", label="Behovsstyrt belysning i fellesarealer")  # 30k, validates
+    small = Approach(id="small", label="Nattsenking av temperatur")  # default reply, 20k
+    mandate = Mandate(
+        objective="Cut energy cost without rebuilding.",
+        approaches=(big, small) if big_first else (small, big),
+        allow_own_proposals=False,
+    )
+    result = await _run(mandate, sink)
+
+    assert isinstance(result.outcome, ValidatedProposal)
+    assert result.outcome.proposal.claimed_saving_nok == 30_000
+    assert all(c.status == "validated" for c in result.coverage)
+
+
+async def test_all_rejected_still_returns_a_rejection_and_full_coverage() -> None:
+    """When nothing validates the run still reports every approach — and the outcome stays a
+    typed ``Rejection`` (never an empty or fabricated success)."""
+    sink: list[str] = []
+    hvac2 = Approach(id="hvac-2", label="Utskifting av ventilasjonsaggregat")
+    mandate = Mandate(
+        objective="Cut energy cost without rebuilding.",
+        approaches=(_HVAC, hvac2),
+        allow_own_proposals=False,
+    )
+    result = await _run(mandate, sink)
+
+    assert isinstance(result.outcome, Rejection)
+    assert {c.id for c in result.coverage} == {"hvac-swap", "hvac-2"}
+    assert all(c.status == "rejected" for c in result.coverage)
+
+
+async def test_no_mandate_runs_exactly_one_generation_with_empty_coverage() -> None:
+    """CONTROL: without a mandate the run is byte-for-byte the pre-Trekk-A one — a single
+    generation, and no coverage report claiming approaches nobody commissioned."""
+    sink: list[str] = []
+    result = await _run(None, sink)
+
+    assert len(_generation_prompts(sink)) == 1
+    assert result.coverage == ()
+    assert isinstance(result.outcome, ValidatedProposal)