portfolio-optimiser/tests/test_v1_probes.py

"""v1 gate probes — the named tests the v1 gate (``python -m portfolio_optimiser.evals.v1_gate``)
runs to decide two of its rows. Every test here that is RED today carries
``xfail(strict=True)``, so the ordinary suite stays green while the gap is real, and the gate runs
the file with ``--runxfail`` so the gap shows as red there. ``strict`` is the other half: the day a
capability makes one of these pass, the suite goes RED on the XPASS until the marker is removed —
a closed gap cannot stay labelled open.

**Row 3 (feedback types with a way in AND an action).** Types 1, 3 and 7 are proven by EXISTING
tests elsewhere in the suite (registered by node id in ``evals/v1_gate.json``). The five types with
no complete surface (2, 4, 5, 6, 8) get a probe here that is red BECAUSE the surface is missing,
never a missing test. Each probe measures the absence (the CLI's own ``--help``); if a matching
option appears it STILL fails, naming the option — "partial is no", and a door with no observed
action is exactly partial. Such a probe goes green only when it is rewritten to drive the new door
and observe what it does.

**Row 6 (a validated proposal whose approach declared no requirement).** Two probes against the
real ``run_project``: no declaration anywhere, and a declaration made by the RUN (the debate) but
not by the approach. The second is the reading the gate measures the stress outboxes with: a
run-level declaration cannot be attributed to one approach (the judge labels it ``run``), so it
does not count as the approach having declared anything.
"""

from __future__ import annotations

import contextlib
import io
import json
import re
from pathlib import Path
from typing import Any

import pytest

from portfolio_optimiser import okf, run
from portfolio_optimiser.mandate import Approach, Mandate
from portfolio_optimiser.run import run_project
from portfolio_optimiser.simulation import scripted_factory
from portfolio_optimiser.verdicts import VerdictStore

_BUNDLE = Path(__file__).resolve().parents[1] / "shared" / "examples" / "bygg-energi-mikro"
_BASE_ID = "bygg-energi-mikro"
_PID = "BYGG-KONTOR-NORD"
_VALID_REPLY = (
    '{"measure":"LED-retrofit av kontorbelysning","affected_items":'
    '[{"code":"ENERGI-TOTAL-EL","quantity":300000,"unit_cost":1.0}],"claimed_saving_nok":30000}'
)
_CHECKER_REPLY = "Reasoning holds.\nVERDICT: APPROVE"

_NO_SURFACE = "v1 probe: no surface"
_PARTIAL = "v1 probe: surface without an observed action"


def _cli_options() -> set[str]:
    """Every option string the CLI's own ``--help`` prints — the surface, measured."""
    buffer = io.StringIO()
    with contextlib.redirect_stdout(buffer), pytest.raises(SystemExit):
        run.main(["--help"])
    return set(re.findall(r"--[a-z][a-z-]*", buffer.getvalue()))


def _surface_or_fail(type_no: int, what: str, keywords: tuple[str, ...]) -> None:
    hits = sorted(o for o in _cli_options() if any(k in o for k in keywords))
    if not hits:
        pytest.fail(f"{_NO_SURFACE}: type {type_no} ({what}) — no CLI option matches {keywords}")
    pytest.fail(
        f"{_PARTIAL}: type {type_no} ({what}) — {hits} appeared; rewrite this probe to drive it "
        "and observe the action"
    )


# ---------------------------------------------------------------------------------------------
# Row 3 — the five types without a complete surface
# ---------------------------------------------------------------------------------------------


@pytest.mark.xfail(strict=True, reason="type 2: no typed removal; only revise free text")
def test_type_2_remove_a_direction_has_a_typed_door() -> None:
    _surface_or_fail(2, "take a direction away", ("drop", "remove", "exclude", "withdraw"))


@pytest.mark.xfail(strict=True, reason="type 4: no surface relaxes a requirement")
def test_type_4_relax_a_requirement_has_a_door() -> None:
    _surface_or_fail(4, "relax a requirement", ("relax", "waive", "loosen"))


@pytest.mark.xfail(strict=True, reason="type 5: concept graph edits have no CLI door")
def test_type_5_edit_the_concept_graph_has_a_door() -> None:
    _surface_or_fail(5, "edit the concept graph", ("promote", "concept", "graph"))


@pytest.mark.xfail(strict=True, reason="type 6: no skills flag")
def test_type_6_skills_per_analysis_has_a_door() -> None:
    _surface_or_fail(6, "skills per analysis", ("skill",))


@pytest.mark.xfail(strict=True, reason="type 8: no door for inline context such as meeting notes")
def test_type_8_inline_context_has_a_door() -> None:
    _surface_or_fail(8, "inline context", ("note", "minutes", "inline", "attach"))


# ---------------------------------------------------------------------------------------------
# Row 6 — a validated proposal must rest on a declaration its approach made
# ---------------------------------------------------------------------------------------------


def _mandate() -> Mandate:
    return Mandate(
        objective="Kutt energikostnad",
        approaches=(Approach(id="a1", label="LED-retrofit", description="expert's reason"),),
    )


async def _statuses(script: dict[str, Any], tmp_path: Path) -> dict[str, str]:
    result = await run_project(
        _PID,
        "local",
        docs_dir=str(_BUNDLE),
        bundle_dir=str(_BUNDLE),
        store=VerdictStore(verdicts=[]),
        client_factory=scripted_factory(script, []),
        mandate=_mandate(),
        outbox_dir=str(tmp_path),
        run_id="v1-row6",
    )
    return {row.id: row.status for row in result.coverage}


@pytest.mark.xfail(strict=True, reason="row 6: no stage refuses a validation with no declaration")
@pytest.mark.asyncio
async def test_row6_an_approach_that_declared_nothing_cannot_be_validated(tmp_path: Path) -> None:
    statuses = await _statuses({"proposer": _VALID_REPLY, "checker": _CHECKER_REPLY}, tmp_path)
    debate = json.loads((tmp_path / "v1-row6-debate.json").read_text(encoding="utf-8"))
    assert debate["requirements"] == []  # precondition: nothing was declared anywhere
    assert statuses["a1"] != "validated", "validated without any declared requirement"


@pytest.mark.xfail(strict=True, reason="row 6: a run-level declaration still stands in")
@pytest.mark.asyncio
async def test_row6_a_run_level_declaration_does_not_stand_in_for_the_approach(
    tmp_path: Path,
) -> None:
    concepts = [f.name for f in okf.navigate_bundle(str(_BUNDLE)).context_files][:3]
    script = {
        "proposer": [
            *({"call": "read_file", "args": {"bundle_id": _BASE_ID, "path": n}} for n in concepts),
            {
                "call": "declare_requirement",
                "args": {"bundle_id": _BASE_ID, "path": concepts[0], "ref": "probe"},
            },
            _VALID_REPLY,
            _VALID_REPLY,
            _VALID_REPLY,
            _VALID_REPLY,
        ],
        "checker": _CHECKER_REPLY,
    }
    statuses = await _statuses(script, tmp_path)
    debate = json.loads((tmp_path / "v1-row6-debate.json").read_text(encoding="utf-8"))
    assert [r["path"] for r in debate["requirements"]] == [concepts[0]]  # precondition
    assert statuses["a1"] != "validated", "validated on a declaration the approach never made"