Krav 2: a run must be clear about what it shall do and achieve. `--mandate` makes both ends explicit. BEFORE the first (paid) model call the run prints what it was commissioned to do — objective, every named approach, whether its own proposals are allowed, the scope, the caps, and which external services it will contact. AFTER the run it settles: one row per approach, validated with the figure, rejected with the validator's reason, or not evaluated with why. The announcement is placed with the other refusals and above the scripted banner, for the reason the required-args guard was hoisted there: a refused run must not first print a banner about work it never did. `--live-dry-run` announces without contacting anything, so a commission can be inspected before it costs money. `settle` renders the goal verdict it is GIVEN and never decides it. `ledger.to_ore` is the framework's one NOK->øre conversion and the goal comparison already runs on quantised integers, but `mandate.py` cannot import it without dragging `verdicts` — and therefore agent_framework — into a deliberately framework-neutral module, while a private copy of a money conversion is exactly the (p) defect. So the caller decides and this renders; a goal figure without a decided verdict makes no claim at all. The caps the announcement prints come from named constants shared with `run_project`/`run_portfolio`'s defaults — a second copy could drift and make the announcement describe a run that never happened. Load-bearing MEASURED against the whole 717-test suite, four mutations all red: detach the announcement (4) · detach the settlement print (2) · make the CLI loader tolerant (2) · announce the commission but never hand it to the run (2). The last one is the one that matters: without it, a run could print a commission it had no intention of executing. DEVIATION from the approved plan, stated rather than quietly dropped: --goals is still refused outside portfolio mode. Accepting it in single-project mode would have admitted a flag whose documented function (the goal-stop against the ledger) still does nothing there — the same accepted-but-inert defect --embedder-config was just fixed for. The mandate's success_criteria carries "what shall this run achieve" in the expert's own words instead; the numeric target stays portfolio-level. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01ULCqjLF61rehj5cZmdUoR3
309 lines
12 KiB
Python
309 lines
12 KiB
Python
"""Unit tests for the run MANDATE IR + its fail-fast loader (Trekk A1).
|
|
|
|
The mandate is what a domain expert commissions a run with: what shall be evaluated
|
|
(named approaches, and/or the system's own proposals) and what the run is meant to
|
|
achieve (``objective`` / ``success_criteria``). It is *authoritative startup config* —
|
|
so loading mirrors ``dimension.load_dimension`` / ``contracts.load_goal_config``
|
|
(missing -> ``FileNotFoundError``, malformed -> ``ValidationError``), NOT the tolerant
|
|
RAW verdict-inbox layer.
|
|
|
|
Two refusals carry real defect classes and are tested explicitly:
|
|
|
|
* an EMPTY commission (no approaches AND no own proposals) — a run with nothing to do
|
|
is a caller error, not a result (mirrors ``BudgetRefused``'s startup refusal);
|
|
* a DUPLICATE approach id — ``id`` is the coverage-report key, so two rows sharing one
|
|
key would silently collapse into one (the S3.2 key-collision class), and the reserved
|
|
``OWN_PROPOSAL_ID`` would collide with the system's own row the same way.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import pytest
|
|
from pydantic import ValidationError
|
|
|
|
from portfolio_optimiser.mandate import (
|
|
OWN_PROPOSAL_ID,
|
|
Approach,
|
|
ApproachOutcome,
|
|
Mandate,
|
|
announce,
|
|
load_mandate,
|
|
settle,
|
|
)
|
|
|
|
|
|
def _mandate(**overrides: object) -> Mandate:
|
|
kwargs: dict[str, object] = {
|
|
"objective": "Find operating measures that can be executed in 2026 without rebuilding.",
|
|
"approaches": (
|
|
Approach(id="led-retrofit", label="LED retrofit of office lighting"),
|
|
Approach(id="service-contract", label="Renegotiate the service contract"),
|
|
),
|
|
}
|
|
kwargs.update(overrides)
|
|
return Mandate(**kwargs) # type: ignore[arg-type]
|
|
|
|
|
|
def test_mandate_defaults_allow_own_proposals() -> None:
|
|
"""``allow_own_proposals`` defaults to True: naming approaches never silently forbids
|
|
the system from adding its own (the operator's 'og/eller')."""
|
|
assert _mandate().allow_own_proposals is True
|
|
|
|
|
|
def test_mandate_accepts_named_approaches_only() -> None:
|
|
"""'Evaluate ONLY these' is a legal commission."""
|
|
m = _mandate(allow_own_proposals=False)
|
|
assert [a.id for a in m.approaches] == ["led-retrofit", "service-contract"]
|
|
assert m.allow_own_proposals is False
|
|
|
|
|
|
def test_mandate_accepts_own_proposals_only() -> None:
|
|
"""'Find your own' with no named approaches is a legal commission (today's behaviour,
|
|
now stated rather than implied)."""
|
|
m = _mandate(approaches=())
|
|
assert m.approaches == ()
|
|
assert m.allow_own_proposals is True
|
|
|
|
|
|
def test_mandate_empty_commission_refused() -> None:
|
|
"""No approaches AND no own proposals = a run with nothing to do -> refused at
|
|
construction, never started."""
|
|
with pytest.raises(ValidationError):
|
|
_mandate(approaches=(), allow_own_proposals=False)
|
|
|
|
|
|
def test_mandate_duplicate_approach_id_refused() -> None:
|
|
"""``id`` keys the coverage report; two rows on one key would collapse silently."""
|
|
with pytest.raises(ValidationError):
|
|
_mandate(
|
|
approaches=(
|
|
Approach(id="led-retrofit", label="LED retrofit"),
|
|
Approach(id="led-retrofit", label="LED retrofit, second take"),
|
|
)
|
|
)
|
|
|
|
|
|
def test_mandate_reserved_own_proposal_id_refused() -> None:
|
|
"""The system's own row is reported under ``OWN_PROPOSAL_ID``; an expert approach
|
|
claiming that id would collide with it."""
|
|
with pytest.raises(ValidationError):
|
|
_mandate(approaches=(Approach(id=OWN_PROPOSAL_ID, label="Sneaky"),))
|
|
|
|
|
|
def test_mandate_requires_an_objective() -> None:
|
|
"""An empty objective defeats the whole point of the mandate (krav 2): a run must say
|
|
what it is for."""
|
|
with pytest.raises(ValidationError):
|
|
_mandate(objective="")
|
|
|
|
|
|
def test_approach_requires_id_and_label() -> None:
|
|
"""Both the coverage key and its human label must be non-empty."""
|
|
with pytest.raises(ValidationError):
|
|
Approach(id="", label="No id")
|
|
with pytest.raises(ValidationError):
|
|
Approach(id="no-label", label="")
|
|
|
|
|
|
# --- fail-fast loader (mirrors load_dimension / load_goal_config) --------------------------------
|
|
|
|
|
|
def test_load_mandate_round_trip(tmp_path) -> None:
|
|
"""A valid mandate JSON round-trips through ``load_mandate`` (accepts str | Path)."""
|
|
m = _mandate()
|
|
p = tmp_path / "mandate.json"
|
|
p.write_text(m.model_dump_json(), encoding="utf-8")
|
|
assert load_mandate(p) == m
|
|
assert load_mandate(str(p)) == m
|
|
|
|
|
|
def test_load_mandate_missing_file_raises(tmp_path) -> None:
|
|
"""Missing file fails fast — authoritative startup config, not a tolerant RAW inbox."""
|
|
with pytest.raises(FileNotFoundError):
|
|
load_mandate(tmp_path / "does-not-exist.json")
|
|
|
|
|
|
def test_load_mandate_malformed_shape_raises(tmp_path) -> None:
|
|
"""Malformed content (missing ``objective``) fails fast with ``ValidationError``."""
|
|
bad = tmp_path / "mandate.json"
|
|
bad.write_text('{"approaches": []}', encoding="utf-8")
|
|
with pytest.raises(ValidationError):
|
|
load_mandate(bad)
|
|
|
|
|
|
def test_load_mandate_not_json_raises(tmp_path) -> None:
|
|
"""Non-JSON content fails fast too (never read as 'no mandate')."""
|
|
bad = tmp_path / "mandate.json"
|
|
bad.write_text("this is not json", encoding="utf-8")
|
|
with pytest.raises(ValidationError):
|
|
load_mandate(bad)
|
|
|
|
|
|
# --- the run announcement (Trekk A2): what this run will do, before the first paid call ----------
|
|
|
|
|
|
def _announce(m: Mandate | None = None, **overrides: object) -> str:
|
|
kwargs: dict[str, object] = {
|
|
"project_id": "BYGG-KONTOR-NORD",
|
|
"max_rounds": 3,
|
|
"max_tokens": 100_000,
|
|
}
|
|
kwargs.update(overrides)
|
|
return announce(m if m is not None else _mandate(), **kwargs) # type: ignore[arg-type]
|
|
|
|
|
|
def test_announce_names_every_commissioned_approach() -> None:
|
|
"""Every approach the expert commissioned is named BEFORE the run starts — the operator must
|
|
be able to see what was ordered without reading the config file back."""
|
|
text = _announce()
|
|
assert "led-retrofit" in text
|
|
assert "service-contract" in text
|
|
assert "LED retrofit of office lighting" in text
|
|
|
|
|
|
def test_announce_states_the_objective_and_success_criteria() -> None:
|
|
"""Krav 2: the run says what it is for, and what would count as success."""
|
|
m = _mandate(success_criteria="At least one measure that passes the validator.")
|
|
text = _announce(m)
|
|
assert "without rebuilding" in text
|
|
assert "At least one measure that passes the validator." in text
|
|
|
|
|
|
def test_announce_distinguishes_only_these_from_these_plus_own() -> None:
|
|
"""The 'og/eller' choice is visible in the announcement, not buried in the config."""
|
|
both = _announce(_mandate(allow_own_proposals=True))
|
|
only = _announce(_mandate(allow_own_proposals=False))
|
|
assert "own proposals" in both
|
|
assert "only these" in only
|
|
assert both != only
|
|
|
|
|
|
def test_announce_without_approaches_says_so() -> None:
|
|
"""A 'find your own' commission announces that too — never a blank line where the list was."""
|
|
text = _announce(_mandate(approaches=()))
|
|
assert "own proposals" in text
|
|
|
|
|
|
def test_announce_omits_optional_lines_when_absent() -> None:
|
|
"""Dimension and target lines appear ONLY when the run actually has them — an announcement
|
|
must not imply a scope or a target that was never configured."""
|
|
text = _announce()
|
|
assert "Scoped to" not in text
|
|
assert "Target" not in text
|
|
|
|
|
|
def test_announce_includes_scope_and_target_when_given() -> None:
|
|
"""...and they ARE stated when configured (the target is restated from the goal config, which
|
|
remains its one home)."""
|
|
text = _announce(dimension_label="energi (cost codes ENERGI-*)", goal_nok=150_000.0)
|
|
assert "energi (cost codes ENERGI-*)" in text
|
|
assert "150000" in text
|
|
|
|
|
|
def test_announce_declares_no_egress_by_default() -> None:
|
|
"""The egress declaration is present and explicit even when nothing is contacted — silence
|
|
would read the same as 'not checked' (repo invariant: no silent egress)."""
|
|
assert "no external services" in _announce()
|
|
|
|
|
|
def test_announce_names_every_external_service() -> None:
|
|
"""Trekk B's egress declaration: every server that may be contacted is named up front."""
|
|
text = _announce(external_services=("prisregister", "maalerdata"))
|
|
assert "prisregister" in text
|
|
assert "maalerdata" in text
|
|
assert "no external services" not in text
|
|
|
|
|
|
def test_announce_states_the_caps() -> None:
|
|
"""Stop criteria + budget cap are part of 'what this run will do' (fail-fast invariant)."""
|
|
text = _announce()
|
|
assert "3 rounds" in text
|
|
assert "100000 tokens" in text
|
|
|
|
|
|
def test_announce_is_deterministic() -> None:
|
|
"""Byte-stable: no wall clock, no set-iteration ordering — so it can be golden-tested."""
|
|
assert _announce() == _announce()
|
|
|
|
|
|
# --- the settlement (Trekk A4): what the run actually did about each approach -------------------
|
|
|
|
_ROWS = (
|
|
ApproachOutcome(id="led-retrofit", label="LED", status="validated", saving_nok=30_000.0),
|
|
ApproachOutcome(
|
|
id="service-contract",
|
|
label="Contract",
|
|
status="rejected",
|
|
detail="claimed 200000 exceeds feasible 90000",
|
|
),
|
|
ApproachOutcome(
|
|
id="night-setback",
|
|
label="Setback",
|
|
status="not_evaluated",
|
|
detail="budget exhausted before this approach was evaluated",
|
|
),
|
|
)
|
|
|
|
|
|
def test_settle_reports_every_row_including_the_unevaluated_one() -> None:
|
|
"""All three statuses reach the report. The unevaluated row is the point: an approach the run
|
|
never got to must be visibly unreached, not absent."""
|
|
text = settle(_ROWS)
|
|
assert "led-retrofit" in text
|
|
assert "service-contract" in text
|
|
assert "night-setback" in text
|
|
assert "NOT EVALUATED" in text
|
|
|
|
|
|
def test_settle_carries_the_rejection_reason() -> None:
|
|
"""A rejected approach without its reason tells the expert nothing they can act on."""
|
|
assert "claimed 200000 exceeds feasible 90000" in settle(_ROWS)
|
|
|
|
|
|
def test_settle_totals_only_the_validated_savings() -> None:
|
|
"""The total is what passed the validator — never the claimed sum of everything attempted."""
|
|
text = settle(_ROWS)
|
|
assert "30000" in text
|
|
assert "200000 NOK" not in text # the rejected claim is never folded into a total
|
|
|
|
|
|
def test_settle_states_whether_the_target_was_reached() -> None:
|
|
"""Krav 2's second half: the run answers, in its own output, whether it achieved what it was
|
|
commissioned to achieve."""
|
|
missed = settle(_ROWS, goal_nok=150_000.0, goal_reached=False)
|
|
hit = settle(_ROWS, goal_nok=10_000.0, goal_reached=True)
|
|
assert "not reached" in missed
|
|
assert "reached" in hit and "not reached" not in hit
|
|
|
|
|
|
def test_settle_renders_the_goal_verdict_it_is_GIVEN_and_never_decides_it() -> None:
|
|
"""The renderer must not form a SECOND opinion on a money question.
|
|
|
|
``ledger.to_ore`` is the framework's one NOK->øre conversion and the goal comparison already
|
|
runs on quantised integers; this module cannot import it without dragging ``verdicts`` — and
|
|
therefore ``agent_framework`` — into a deliberately framework-neutral file, and a private copy
|
|
of a money conversion is exactly the ``(p)`` defect. So the caller decides and this renders.
|
|
Told the opposite of what its own float sum suggests, it prints what it was told.
|
|
"""
|
|
text = settle(_ROWS, goal_nok=150_000.0, goal_reached=True)
|
|
assert "not reached" not in text
|
|
|
|
|
|
def test_settle_omits_the_target_line_when_no_goal_was_configured() -> None:
|
|
"""No goal configured -> no verdict on a goal. An absent target must not read as a missed one."""
|
|
assert "Target" not in settle(_ROWS)
|
|
# ...and a goal figure without a decided verdict renders no claim either.
|
|
assert "Target" not in settle(_ROWS, goal_nok=150_000.0)
|
|
|
|
|
|
def test_settle_is_empty_without_coverage() -> None:
|
|
"""No mandate -> nothing to settle. An empty block beats a header over zero rows, which would
|
|
imply a commission that never existed."""
|
|
assert settle(()) == ""
|
|
|
|
|
|
def test_settle_is_deterministic() -> None:
|
|
"""Byte-stable, like the announcement."""
|
|
assert settle(_ROWS, goal_nok=150_000.0, goal_reached=False) == settle(
|
|
_ROWS, goal_nok=150_000.0, goal_reached=False
|
|
)
|