portfolio-optimiser/tests/test_mandate.py
Kjell Tore Guttormsen 30bcdd3544 docs(mandate): how a domain expert commissions a run — and one honesty fix the run itself exposed
`docs/bestille-en-kjoring.md` is the commissioning half of the expert-facing pair
(`ekspert-svar.md` is the judging half): the mandate file field by field, how to
run it, and — separated deliberately — what a commission does NOT do. It directs
what is evaluated, never what is approved.

Registered in _LIVE_DOCS, so it cannot silently fall behind the code.

The example output in it is COPIED FROM A REAL RUN, not composed, and running
that run is what found the defect fixed here: three approaches against the same
cost line each validated at 30000 NOK, and the settlement printed
"Validated total: 90000 NOK". Commissioned approaches are ALTERNATIVES — they
usually attack the same line — so summing them reports money the project cannot
realise. A domain expert reading that total would reasonably believe the run
found 90k.

The settlement now reports how many approaches held and which one the run
carries: a selection, not an arithmetic claim. That also removes the last money
addition from this module, which is the right place for it not to be.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01ULCqjLF61rehj5cZmdUoR3
2026-08-05 16:31:27 +02:00

327 lines
13 KiB
Python

"""Unit tests for the run MANDATE IR + its fail-fast loader (Trekk A1).
The mandate is what a domain expert commissions a run with: what shall be evaluated
(named approaches, and/or the system's own proposals) and what the run is meant to
achieve (``objective`` / ``success_criteria``). It is *authoritative startup config* —
so loading mirrors ``dimension.load_dimension`` / ``contracts.load_goal_config``
(missing -> ``FileNotFoundError``, malformed -> ``ValidationError``), NOT the tolerant
RAW verdict-inbox layer.
Two refusals carry real defect classes and are tested explicitly:
* an EMPTY commission (no approaches AND no own proposals) — a run with nothing to do
is a caller error, not a result (mirrors ``BudgetRefused``'s startup refusal);
* a DUPLICATE approach id — ``id`` is the coverage-report key, so two rows sharing one
key would silently collapse into one (the S3.2 key-collision class), and the reserved
``OWN_PROPOSAL_ID`` would collide with the system's own row the same way.
"""
from __future__ import annotations
import pytest
from pydantic import ValidationError
from portfolio_optimiser.mandate import (
OWN_PROPOSAL_ID,
Approach,
ApproachOutcome,
Mandate,
announce,
load_mandate,
settle,
)
def _mandate(**overrides: object) -> Mandate:
kwargs: dict[str, object] = {
"objective": "Find operating measures that can be executed in 2026 without rebuilding.",
"approaches": (
Approach(id="led-retrofit", label="LED retrofit of office lighting"),
Approach(id="service-contract", label="Renegotiate the service contract"),
),
}
kwargs.update(overrides)
return Mandate(**kwargs) # type: ignore[arg-type]
def test_mandate_defaults_allow_own_proposals() -> None:
"""``allow_own_proposals`` defaults to True: naming approaches never silently forbids
the system from adding its own (the operator's 'og/eller')."""
assert _mandate().allow_own_proposals is True
def test_mandate_accepts_named_approaches_only() -> None:
"""'Evaluate ONLY these' is a legal commission."""
m = _mandate(allow_own_proposals=False)
assert [a.id for a in m.approaches] == ["led-retrofit", "service-contract"]
assert m.allow_own_proposals is False
def test_mandate_accepts_own_proposals_only() -> None:
"""'Find your own' with no named approaches is a legal commission (today's behaviour,
now stated rather than implied)."""
m = _mandate(approaches=())
assert m.approaches == ()
assert m.allow_own_proposals is True
def test_mandate_empty_commission_refused() -> None:
"""No approaches AND no own proposals = a run with nothing to do -> refused at
construction, never started."""
with pytest.raises(ValidationError):
_mandate(approaches=(), allow_own_proposals=False)
def test_mandate_duplicate_approach_id_refused() -> None:
"""``id`` keys the coverage report; two rows on one key would collapse silently."""
with pytest.raises(ValidationError):
_mandate(
approaches=(
Approach(id="led-retrofit", label="LED retrofit"),
Approach(id="led-retrofit", label="LED retrofit, second take"),
)
)
def test_mandate_reserved_own_proposal_id_refused() -> None:
"""The system's own row is reported under ``OWN_PROPOSAL_ID``; an expert approach
claiming that id would collide with it."""
with pytest.raises(ValidationError):
_mandate(approaches=(Approach(id=OWN_PROPOSAL_ID, label="Sneaky"),))
def test_mandate_requires_an_objective() -> None:
"""An empty objective defeats the whole point of the mandate (krav 2): a run must say
what it is for."""
with pytest.raises(ValidationError):
_mandate(objective="")
def test_approach_requires_id_and_label() -> None:
"""Both the coverage key and its human label must be non-empty."""
with pytest.raises(ValidationError):
Approach(id="", label="No id")
with pytest.raises(ValidationError):
Approach(id="no-label", label="")
# --- fail-fast loader (mirrors load_dimension / load_goal_config) --------------------------------
def test_load_mandate_round_trip(tmp_path) -> None:
"""A valid mandate JSON round-trips through ``load_mandate`` (accepts str | Path)."""
m = _mandate()
p = tmp_path / "mandate.json"
p.write_text(m.model_dump_json(), encoding="utf-8")
assert load_mandate(p) == m
assert load_mandate(str(p)) == m
def test_load_mandate_missing_file_raises(tmp_path) -> None:
"""Missing file fails fast — authoritative startup config, not a tolerant RAW inbox."""
with pytest.raises(FileNotFoundError):
load_mandate(tmp_path / "does-not-exist.json")
def test_load_mandate_malformed_shape_raises(tmp_path) -> None:
"""Malformed content (missing ``objective``) fails fast with ``ValidationError``."""
bad = tmp_path / "mandate.json"
bad.write_text('{"approaches": []}', encoding="utf-8")
with pytest.raises(ValidationError):
load_mandate(bad)
def test_load_mandate_not_json_raises(tmp_path) -> None:
"""Non-JSON content fails fast too (never read as 'no mandate')."""
bad = tmp_path / "mandate.json"
bad.write_text("this is not json", encoding="utf-8")
with pytest.raises(ValidationError):
load_mandate(bad)
# --- the run announcement (Trekk A2): what this run will do, before the first paid call ----------
def _announce(m: Mandate | None = None, **overrides: object) -> str:
kwargs: dict[str, object] = {
"project_id": "BYGG-KONTOR-NORD",
"max_rounds": 3,
"max_tokens": 100_000,
}
kwargs.update(overrides)
return announce(m if m is not None else _mandate(), **kwargs) # type: ignore[arg-type]
def test_announce_names_every_commissioned_approach() -> None:
"""Every approach the expert commissioned is named BEFORE the run starts — the operator must
be able to see what was ordered without reading the config file back."""
text = _announce()
assert "led-retrofit" in text
assert "service-contract" in text
assert "LED retrofit of office lighting" in text
def test_announce_states_the_objective_and_success_criteria() -> None:
"""Krav 2: the run says what it is for, and what would count as success."""
m = _mandate(success_criteria="At least one measure that passes the validator.")
text = _announce(m)
assert "without rebuilding" in text
assert "At least one measure that passes the validator." in text
def test_announce_distinguishes_only_these_from_these_plus_own() -> None:
"""The 'og/eller' choice is visible in the announcement, not buried in the config."""
both = _announce(_mandate(allow_own_proposals=True))
only = _announce(_mandate(allow_own_proposals=False))
assert "own proposals" in both
assert "only these" in only
assert both != only
def test_announce_without_approaches_says_so() -> None:
"""A 'find your own' commission announces that too — never a blank line where the list was."""
text = _announce(_mandate(approaches=()))
assert "own proposals" in text
def test_announce_omits_optional_lines_when_absent() -> None:
"""Dimension and target lines appear ONLY when the run actually has them — an announcement
must not imply a scope or a target that was never configured."""
text = _announce()
assert "Scoped to" not in text
assert "Target" not in text
def test_announce_includes_scope_and_target_when_given() -> None:
"""...and they ARE stated when configured (the target is restated from the goal config, which
remains its one home)."""
text = _announce(dimension_label="energi (cost codes ENERGI-*)", goal_nok=150_000.0)
assert "energi (cost codes ENERGI-*)" in text
assert "150000" in text
def test_announce_declares_no_egress_by_default() -> None:
"""The egress declaration is present and explicit even when nothing is contacted — silence
would read the same as 'not checked' (repo invariant: no silent egress)."""
assert "no external services" in _announce()
def test_announce_names_every_external_service() -> None:
"""Trekk B's egress declaration: every server that may be contacted is named up front."""
text = _announce(external_services=("prisregister", "maalerdata"))
assert "prisregister" in text
assert "maalerdata" in text
assert "no external services" not in text
def test_announce_states_the_caps() -> None:
"""Stop criteria + budget cap are part of 'what this run will do' (fail-fast invariant)."""
text = _announce()
assert "3 rounds" in text
assert "100000 tokens" in text
def test_announce_is_deterministic() -> None:
"""Byte-stable: no wall clock, no set-iteration ordering — so it can be golden-tested."""
assert _announce() == _announce()
# --- the settlement (Trekk A4): what the run actually did about each approach -------------------
_ROWS = (
ApproachOutcome(id="led-retrofit", label="LED", status="validated", saving_nok=30_000.0),
ApproachOutcome(
id="service-contract",
label="Contract",
status="rejected",
detail="claimed 200000 exceeds feasible 90000",
),
ApproachOutcome(
id="night-setback",
label="Setback",
status="not_evaluated",
detail="budget exhausted before this approach was evaluated",
),
)
def test_settle_reports_every_row_including_the_unevaluated_one() -> None:
"""All three statuses reach the report. The unevaluated row is the point: an approach the run
never got to must be visibly unreached, not absent."""
text = settle(_ROWS)
assert "led-retrofit" in text
assert "service-contract" in text
assert "night-setback" in text
assert "NOT EVALUATED" in text
def test_settle_carries_the_rejection_reason() -> None:
"""A rejected approach without its reason tells the expert nothing they can act on."""
assert "claimed 200000 exceeds feasible 90000" in settle(_ROWS)
def test_settle_never_sums_alternative_approaches() -> None:
"""Commissioned approaches are ALTERNATIVES, not additive savings — several of them usually
attack the same cost line. Summing them would report a figure the project cannot realise.
MEASURED on a real run: three approaches against one cost line each validated at 30000 and the
settlement claimed a 90000 total. What is honest is how many passed and which one the run
carries — a selection, not an arithmetic claim.
"""
rows = (
ApproachOutcome(id="a", label="A", status="validated", saving_nok=30_000.0),
ApproachOutcome(id="b", label="B", status="validated", saving_nok=20_000.0),
)
text = settle(rows)
assert "50000" not in text # the sum is never formed
assert "30000" in text # the best one is named
assert "2 of 2" in text
def test_settle_counts_the_rejected_out_of_the_validated_tally() -> None:
"""A rejected approach counts toward how many were commissioned, never toward how many held."""
text = settle(_ROWS)
assert "1 of 3" in text
assert "200000 NOK" not in text # the rejected claim is never presented as a saving
def test_settle_states_whether_the_target_was_reached() -> None:
"""Krav 2's second half: the run answers, in its own output, whether it achieved what it was
commissioned to achieve."""
missed = settle(_ROWS, goal_nok=150_000.0, goal_reached=False)
hit = settle(_ROWS, goal_nok=10_000.0, goal_reached=True)
assert "not reached" in missed
assert "reached" in hit and "not reached" not in hit
def test_settle_renders_the_goal_verdict_it_is_GIVEN_and_never_decides_it() -> None:
"""The renderer must not form a SECOND opinion on a money question.
``ledger.to_ore`` is the framework's one NOK->øre conversion and the goal comparison already
runs on quantised integers; this module cannot import it without dragging ``verdicts`` — and
therefore ``agent_framework`` — into a deliberately framework-neutral file, and a private copy
of a money conversion is exactly the ``(p)`` defect. So the caller decides and this renders.
Told the opposite of what its own float sum suggests, it prints what it was told.
"""
text = settle(_ROWS, goal_nok=150_000.0, goal_reached=True)
assert "not reached" not in text
def test_settle_omits_the_target_line_when_no_goal_was_configured() -> None:
"""No goal configured -> no verdict on a goal. An absent target must not read as a missed one."""
assert "Target" not in settle(_ROWS)
# ...and a goal figure without a decided verdict renders no claim either.
assert "Target" not in settle(_ROWS, goal_nok=150_000.0)
def test_settle_is_empty_without_coverage() -> None:
"""No mandate -> nothing to settle. An empty block beats a header over zero rows, which would
imply a commission that never existed."""
assert settle(()) == ""
def test_settle_is_deterministic() -> None:
"""Byte-stable, like the announcement."""
assert settle(_ROWS, goal_nok=150_000.0, goal_reached=False) == settle(
_ROWS, goal_nok=150_000.0, goal_reached=False
)