test(v1-gate): harden the gate against a handwritten green
An independent review made rows 1, 2 and 4 green from a handwritten directory in a minute, and 10 of 20 mutants survived the gate's tests. Rounds now need a new point and their own ids, a timezone-aware given_at in order, and a report the feedback was given on; every outcome must name a run whose own coverage confirms (a)-(d), the feedback must fall between the two runs, and a NOK change under 1 % is noise. Row 4 counts content lines kept unchanged and in order, shows the expert's additions, and calls a byte-identical copy untouched unless round 3 acknowledges it. Row 6 counts the runs' own proposals. Types 3 and 7 are proven through the real flags with the action in the result (still 3 of 8). The contract numbers and the evidence register are pinned to their source. Every run prints that rows 1-2 cannot prove who wrote the feedback. A rounds directory inside the repo that git would commit, and a missing stress or bundle root, are usage errors. The review's 20 mutants, re-run: 20 of 20 killed. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
parent
938a1ca30e
commit
9825b2677c
5 changed files with 979 additions and 213 deletions
|
|
@ -31,7 +31,7 @@ from typing import Any
|
|||
import pytest
|
||||
|
||||
from portfolio_optimiser import okf, run
|
||||
from portfolio_optimiser.mandate import Approach, Mandate
|
||||
from portfolio_optimiser.mandate import Approach, BindingRequirement, Mandate
|
||||
from portfolio_optimiser.run import run_project
|
||||
from portfolio_optimiser.simulation import scripted_factory
|
||||
from portfolio_optimiser.verdicts import VerdictStore
|
||||
|
|
@ -67,6 +67,143 @@ def _surface_or_fail(type_no: int, what: str, keywords: tuple[str, ...]) -> None
|
|||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------------------------
|
||||
# Row 3 — types 3 and 7: the way in is the REAL flag, and the action shows in the RESULT
|
||||
# ---------------------------------------------------------------------------------------------
|
||||
|
||||
_REQUIREMENT = {"path": "tiltak-led-retrofit.md", "ref": "Krav 1"}
|
||||
|
||||
|
||||
def _cli_run(tmp_path: Path, replies: dict[str, Any], *extra: str) -> tuple[int, Path, str]:
|
||||
"""One in-process CLI run on the micro base with scripted replies, writing an outbox."""
|
||||
replies_file = tmp_path / "replies.json"
|
||||
replies_file.write_text(json.dumps(replies), encoding="utf-8")
|
||||
out = tmp_path / "out"
|
||||
buffer = io.StringIO()
|
||||
with contextlib.redirect_stdout(buffer):
|
||||
rc = run.main(
|
||||
[
|
||||
_PID,
|
||||
"--bundle-dir",
|
||||
str(_BUNDLE),
|
||||
"--scripted-replies",
|
||||
str(replies_file),
|
||||
"--outbox-dir",
|
||||
str(out),
|
||||
"--run-id",
|
||||
"probe",
|
||||
*extra,
|
||||
]
|
||||
)
|
||||
return rc, out, buffer.getvalue()
|
||||
|
||||
|
||||
def test_type_3_a_commissioned_angle_is_evaluated_through_the_cli(tmp_path: Path) -> None:
|
||||
"""``--mandate`` is the way in; the action is a VALIDATED outcome for that angle in the
|
||||
outbox. A settlement line alone is not enough — a ``NOT EVALUATED`` row prints the id too."""
|
||||
mandate = tmp_path / "mandate.json"
|
||||
mandate.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"objective": "Kutt energikostnad",
|
||||
"approaches": [
|
||||
{"id": "ny-vinkling", "label": "LED-retrofit", "requirement": _REQUIREMENT}
|
||||
],
|
||||
"allow_own_proposals": False,
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
rc, out, stdout = _cli_run(
|
||||
tmp_path, {"proposer": _VALID_REPLY, "checker": _CHECKER_REPLY}, "--mandate", str(mandate)
|
||||
)
|
||||
assert rc == 0, stdout
|
||||
coverage = json.loads((out / "probe-coverage.json").read_text(encoding="utf-8"))
|
||||
assert [(r["id"], r["status"]) for r in coverage["rows"]] == [("ny-vinkling", "validated")]
|
||||
outcome = json.loads((out / "probe-ny-vinkling-outcome.json").read_text(encoding="utf-8"))
|
||||
assert outcome["outcome_type"] == "validated"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_type_3_a_new_angle_changes_the_outcome() -> None:
|
||||
"""Adding an angle in a later round changes what the run carries: the new angle's own reply
|
||||
(reachable only if its label reached the prompt) becomes the selected, validated outcome."""
|
||||
labels = {"Behovsstyrt belysning": 30_000, "Nattsenking av temperatur": 60_000}
|
||||
|
||||
def select(prompt: str, _role: str) -> str:
|
||||
claimed = next((v for k, v in labels.items() if k in prompt), 10_000)
|
||||
return _VALID_REPLY.replace("30000}", f"{claimed}}}")
|
||||
|
||||
requirement = BindingRequirement(**_REQUIREMENT)
|
||||
first = Approach(id="a1", label="Behovsstyrt belysning", requirement=requirement)
|
||||
second = Approach(id="a2", label="Nattsenking av temperatur", requirement=requirement)
|
||||
|
||||
async def outcome(*approaches: Approach) -> Any:
|
||||
return await run_project(
|
||||
_PID,
|
||||
"local",
|
||||
docs_dir=str(_BUNDLE),
|
||||
bundle_dir=str(_BUNDLE),
|
||||
store=VerdictStore(verdicts=[]),
|
||||
client_factory=scripted_factory({"proposer": select, "checker": _CHECKER_REPLY}, []),
|
||||
mandate=Mandate(objective="o", approaches=approaches, allow_own_proposals=False),
|
||||
)
|
||||
|
||||
before = await outcome(first)
|
||||
after = await outcome(first, second)
|
||||
assert before.outcome.proposal.claimed_saving_nok == 30_000
|
||||
assert {r.id: r.status for r in after.coverage} == {"a1": "validated", "a2": "validated"}
|
||||
assert after.outcome.proposal.claimed_saving_nok == 60_000
|
||||
|
||||
|
||||
def test_type_7_the_mcp_flag_puts_a_service_the_run_calls_into_the_result(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
"""``--mcp-config`` is the way in; the action is the RESULT recording that the debate called
|
||||
the configured service. Without the flag the same script leaves no external call."""
|
||||
from tests.test_b4_mcp_call_trace_loadbearing import _as_context_manager, _lookup_unit_price
|
||||
|
||||
monkeypatch.setattr(
|
||||
run, "build_mcp_tools", lambda _c: [_as_context_manager(_lookup_unit_price)]
|
||||
)
|
||||
config = tmp_path / "mcp.json"
|
||||
config.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"servers": [
|
||||
{
|
||||
"name": "prisregister",
|
||||
"transport": "http",
|
||||
"url": "https://intern.example/mcp",
|
||||
"allowed_tools": ["lookup_unit_price"],
|
||||
"timeout_seconds": 15,
|
||||
}
|
||||
]
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
replies = {
|
||||
"proposer": [
|
||||
{"call": "lookup_unit_price", "args": {"code": "ENERGI-TOTAL-EL"}},
|
||||
*([_VALID_REPLY] * 4),
|
||||
],
|
||||
"checker": _CHECKER_REPLY,
|
||||
}
|
||||
|
||||
def calls(sub: str, *extra: str) -> list[Any]:
|
||||
(tmp_path / sub).mkdir()
|
||||
rc, out, stdout = _cli_run(tmp_path / sub, replies, *extra)
|
||||
assert rc == 0, stdout
|
||||
proposal = json.loads((out / "probe-proposal.json").read_text(encoding="utf-8"))
|
||||
return list(proposal["provenance"]["external_calls"])
|
||||
|
||||
assert calls("with", "--mcp-config", str(config)) == [
|
||||
{"server": "prisregister", "tool": "lookup_unit_price"}
|
||||
]
|
||||
assert calls("without") == []
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------------------------
|
||||
# Row 3 — the five types without a complete surface
|
||||
# ---------------------------------------------------------------------------------------------
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue