306 lines
14 KiB
Python
306 lines
14 KiB
Python
"""Load-bearing CLI tests: the observable GoalReached stop (S5.3 Step 3, SC1) and the
|
|
``--semantic-retrieval`` wiring proof (S3.1 remediation, review finding ``c23e87ee``).
|
|
|
|
DETACH THAT TURNS THE GOAL ARMS RED: route ``--goals``/``--ledger`` away from ``run_portfolio``
|
|
(drop the wiring in ``main()``'s portfolio dispatch) → no goal-stop line reaches stdout → the
|
|
positive arm's marker assertion fails. The positive+control pair proves the printed scope/øre
|
|
fields flow from ``run_portfolio``'s actual ``GoalReached``/``PortfolioResult`` values — a canned
|
|
print could not produce BOTH the portfolio-scope and the project-scope variant from the same
|
|
wiring.
|
|
|
|
DETACH POINTS THAT TURN THE ``--semantic-retrieval`` ARMS RED — two, because the review's finding
|
|
was that the flag was green-but-dead at exactly these joints:
|
|
1. Hardcode ``semantic_retrieval=False`` at ``main()``'s full-run ``run_project(...)`` call → the
|
|
single-project positive loses its marker → RED.
|
|
2. Drop ``semantic_retrieval=semantic_retrieval`` from the ``run_project(...)`` call inside
|
|
``run_portfolio`` → the portfolio positive loses its marker → RED.
|
|
Both were previously covered ONLY below the CLI (``run_project`` driven directly), so either
|
|
mutation left the suite green. These arms drive ``run.main([...])`` and nothing else.
|
|
|
|
Why the fixture looks the way it does: ``main()`` exposes no ``--top-k``, so retrieval returns 3
|
|
verdicts. The inbox therefore holds FOUR structurally tied verdicts — three decoys whose minted ids
|
|
sort before the marker's, plus the marker. Structurally (score tied, ``id`` ascending) the marker is
|
|
the FOURTH and never reaches the prompt; only the cosine term lifts it into the top 3. A two-verdict
|
|
fixture would put the marker in the prompt either way and the control would prove nothing.
|
|
|
|
Both goal arms are provably OFFLINE: the goal is already met before any project builds a chat client
|
|
(portfolio-hard ``break``s at ``run.py:560-567``; per-project-hard ``continue``s past the only pid at
|
|
``run.py:571-579`` — no client is ever constructed). The ``--semantic-retrieval`` arms DO run a full
|
|
pass, but every model call goes to a recording stand-in injected by monkeypatching
|
|
``run._default_factory`` — the seam ``main()`` resolves through, since it never passes
|
|
``client_factory``. No socket/network is exercised (brief NFR). Marker value 13731 øre and the
|
|
``SENTINEL-CLI-45f9a2`` string each appear nowhere else in the codebase (repo marker convention).
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from portfolio_optimiser import run
|
|
from portfolio_optimiser.ledger import LedgerEntry, SavingsLedger
|
|
from portfolio_optimiser.reference_domain import Project
|
|
from portfolio_optimiser.verdicts import ProposalFeatures, capture_verdict, write_verdict
|
|
|
|
_MARKER_ORE = 13731 # marker: appears nowhere else in the codebase
|
|
|
|
_BUNDLE_DIR = Path(__file__).resolve().parents[1] / "shared" / "examples" / "bygg-energi-mikro"
|
|
_PID = "BYGG-KONTOR-NORD"
|
|
|
|
# The learning signal rides in the RATIONALE — that is what ``ExpeLContextProvider.format_fewshot``
|
|
# emits into the prompt. It cannot be leaked by bundle context: the string appears nowhere in the
|
|
# bundle (grep-verified), so its presence in a generation prompt can only have come via the fold.
|
|
_CLI_MARKER = "SENTINEL-CLI-45f9a2 realiseringsgrad=0.5391"
|
|
|
|
# Extra cost codes appended to the bundle's own code. Every combination ties on the structural
|
|
# score (equal Jaccard, same measure, same magnitude bucket); they differ only in the code set, so
|
|
# the canonical embedding string differs and cosine has something to separate. Chosen empirically:
|
|
# the three decoys' minted ids all sort BEFORE the marker's, and the marker's cosine is the highest.
|
|
_MARKER_CODE = "45.9"
|
|
_DECOY_CODES = ("58.9", "15.5", "36.8")
|
|
|
|
_ENERGY_REPLY = (
|
|
'{"measure":"LED-retrofit av kontorbelysning","affected_items":'
|
|
'[{"code":"ENERGI-TOTAL-EL","quantity":300000,"unit_cost":1.0}],"claimed_saving_nok":30000}'
|
|
)
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def _isolate_model_env(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
"""Hermetic env (verbatim from ``test_live_dry_run.py``) — irrelevant here (no client is built),
|
|
kept for parity so an accidental client construction can never read the operator's Foundry env."""
|
|
monkeypatch.delenv("PORTFOLIO_MODEL_MAP", raising=False)
|
|
monkeypatch.delenv("PORTFOLIO_FOUNDRY_PROJECT_ENDPOINT", raising=False)
|
|
|
|
|
|
def _marker_ledger(tmp_path: Path) -> Path:
|
|
"""A ledger with exactly one realized entry totalling the marker øre against FV42-GSV-E1 —
|
|
reused by both arms (portfolio_total == per_project_total('FV42-GSV-E1') == 13731)."""
|
|
led = SavingsLedger()
|
|
led.add_realized(
|
|
LedgerEntry(
|
|
project_id="FV42-GSV-E1",
|
|
dimension="energi",
|
|
candidate_identity="c-marker",
|
|
amount_ore=_MARKER_ORE,
|
|
verdict_id="v-marker",
|
|
provenance="prior-hitl",
|
|
)
|
|
)
|
|
p = tmp_path / "ledger.json"
|
|
led.save(str(p))
|
|
return p
|
|
|
|
|
|
def _write_goals(tmp_path: Path, data: dict) -> Path:
|
|
p = tmp_path / "goals.json"
|
|
p.write_text(json.dumps(data), encoding="utf-8")
|
|
return p
|
|
|
|
|
|
def test_portfolio_hard_goal_reached_is_observable_in_cli(tmp_path, capsys) -> None:
|
|
"""POSITIVE: a portfolio-hard goal already met by the ledger stops the whole pass offline and
|
|
prints an observable goal-stop line carrying scope=portfolio + the marker øre + stopped_early."""
|
|
goals = _write_goals(tmp_path, {"portfolio": {"absolute_ore": _MARKER_ORE, "mode": "hard"}})
|
|
ledger = _marker_ledger(tmp_path)
|
|
rc = run.main(["--portfolio", "--goals", str(goals), "--ledger", str(ledger)])
|
|
assert rc == 0
|
|
out = capsys.readouterr().out
|
|
assert "goal reached: scope=portfolio" in out
|
|
assert f"observed_ore={_MARKER_ORE}" in out
|
|
assert "stopped_early=True" in out
|
|
|
|
|
|
def test_per_project_hard_goal_control_distinguishes_scope(tmp_path, capsys) -> None:
|
|
"""CONTROL (causality): a per-project-hard goal met on the ONLY selected pid skips it (continue,
|
|
no client built) → scope=project, stopped_early=False, and NO portfolio-scope line. Proves the
|
|
printed fields are the actual run_portfolio values, not a canned string."""
|
|
goals = _write_goals(
|
|
tmp_path, {"per_project": {"FV42-GSV-E1": {"absolute_ore": _MARKER_ORE, "mode": "hard"}}}
|
|
)
|
|
ledger = _marker_ledger(tmp_path)
|
|
rc = run.main(["FV42-GSV-E1", "--portfolio", "--goals", str(goals), "--ledger", str(ledger)])
|
|
assert rc == 0
|
|
out = capsys.readouterr().out
|
|
assert "scope=project" in out
|
|
assert "stopped_early=False" in out
|
|
assert "scope=portfolio" not in out
|
|
|
|
|
|
# --- S3.1: the --semantic-retrieval wiring, proved at main() level (finding c23e87ee) -----------
|
|
|
|
|
|
def _generation_prompts(sink: list[str]) -> list[str]:
|
|
"""The generation-call prompts (``generate._build_messages`` embeds 'SavingsProposal'),
|
|
isolated from the debate-round prompts also captured in the shared sink."""
|
|
return [p for p in sink if "SavingsProposal" in p]
|
|
|
|
|
|
def _tied_inbox(tmp_path: Path) -> str:
|
|
"""A populated async inbox: three structurally tied decoys plus the marker verdict.
|
|
|
|
Written through the real authoring primitives — ``write_verdict(capture_verdict(...))`` — so
|
|
every id is minted the way the framework mints them, never hand-assigned. ``--verdict-dir`` is
|
|
the ONLY route by which ``main()`` can hand ``run_project`` a non-empty store."""
|
|
from portfolio_optimiser.verdicts import bundle_candidate_features
|
|
|
|
query = bundle_candidate_features(str(_BUNDLE_DIR))
|
|
inbox = tmp_path / "inbox"
|
|
|
|
def tied(extra_code: str, decision: str, rationale: str) -> None:
|
|
write_verdict(
|
|
str(inbox),
|
|
capture_verdict(
|
|
ProposalFeatures(
|
|
affected_codes=query.affected_codes | {extra_code},
|
|
measure_type=query.measure_type,
|
|
claimed_saving_nok=query.claimed_saving_nok,
|
|
# ``description == measure`` is the shape both live minting paths emit.
|
|
description=query.measure_type,
|
|
),
|
|
decision,
|
|
rationale,
|
|
),
|
|
)
|
|
|
|
for code in _DECOY_CODES:
|
|
tied(code, "rejected", f"ingen realiseringsdata for tiltak {code}")
|
|
tied(_MARKER_CODE, "approved", f"tidligere LED-dom [{_CLI_MARKER}]")
|
|
return str(inbox)
|
|
|
|
|
|
@pytest.fixture()
|
|
def _recorded_main(monkeypatch):
|
|
"""Inject a recording chat client into the seam ``main()`` actually resolves through.
|
|
|
|
``main()`` never passes ``client_factory``, so ``run_project`` falls back to the module-level
|
|
``_default_factory``. Patching it is therefore the only observation point available to a
|
|
main()-level test — and patching it (rather than passing a factory) is what keeps this test
|
|
honest about the CLI path."""
|
|
from tests.conftest import _RecordingChatClient
|
|
|
|
sink: list[str] = []
|
|
monkeypatch.setattr(
|
|
"portfolio_optimiser.run._default_factory",
|
|
lambda profile: lambda role: _RecordingChatClient(sink, _ENERGY_REPLY),
|
|
)
|
|
return sink
|
|
|
|
|
|
def test_cli_semantic_retrieval_reaches_the_hypothesis_prompt(tmp_path, _recorded_main) -> None:
|
|
"""POSITIVE — ``run.main([... --semantic-retrieval])`` must carry the cosine-surfaced verdict's
|
|
rationale into the hypothesis prompt.
|
|
|
|
This is the arm the review's ``c23e87ee`` demanded: every prior proof drove ``run_project``
|
|
directly, so the CLI could stop forwarding the flag without a single test noticing.
|
|
|
|
Detach point: hardcode ``semantic_retrieval=False`` at ``main()``'s ``run_project(...)`` call
|
|
→ RED."""
|
|
rc = run.main(
|
|
[
|
|
_PID,
|
|
"--docs-dir",
|
|
str(_BUNDLE_DIR),
|
|
"--bundle-dir",
|
|
str(_BUNDLE_DIR),
|
|
"--verdict-dir",
|
|
_tied_inbox(tmp_path),
|
|
"--semantic-retrieval",
|
|
]
|
|
)
|
|
assert rc == 0
|
|
|
|
gen_prompts = _generation_prompts(_recorded_main)
|
|
assert gen_prompts, "the generation call must have happened"
|
|
assert any(_CLI_MARKER in p for p in gen_prompts), (
|
|
"the cosine-surfaced verdict did not reach the hypothesis prompt — main() is not "
|
|
"forwarding --semantic-retrieval into the Step-1 fold"
|
|
)
|
|
|
|
|
|
def test_cli_without_the_flag_leaves_the_marker_out(tmp_path, _recorded_main) -> None:
|
|
"""CAUSALITY CONTROL — the identical CLI invocation WITHOUT the flag must not carry the marker.
|
|
|
|
Without this the positive proves nothing: it would pass merely because the inbox contains the
|
|
marker. Structurally the marker is the fourth of four tied verdicts and retrieval returns three,
|
|
so only the cosine term can reach it."""
|
|
rc = run.main(
|
|
[
|
|
_PID,
|
|
"--docs-dir",
|
|
str(_BUNDLE_DIR),
|
|
"--bundle-dir",
|
|
str(_BUNDLE_DIR),
|
|
"--verdict-dir",
|
|
_tied_inbox(tmp_path),
|
|
]
|
|
)
|
|
assert rc == 0
|
|
|
|
assert all(_CLI_MARKER not in p for p in _recorded_main), (
|
|
"the marker reached a prompt with --semantic-retrieval OFF — the default CLI path is not "
|
|
"the structural ranking, so the positive arm is not load-bearing"
|
|
)
|
|
|
|
|
|
def test_cli_portfolio_mode_forwards_semantic_retrieval(tmp_path, monkeypatch, _recorded_main):
|
|
"""POSITIVE (portfolio arm) — ``main(["--portfolio", "--semantic-retrieval"])`` must reach each
|
|
project's Step-1 fold.
|
|
|
|
Portfolio mode refuses ``--verdict-dir`` (it is single-project-only), so the inbox arrives on
|
|
the ``Project`` record instead — the same topology
|
|
``tests/test_portfolio_learning_loadbearing.py`` uses.
|
|
|
|
Detach point: drop ``semantic_retrieval=semantic_retrieval`` from the ``run_project(...)`` call
|
|
inside ``run_portfolio`` → RED."""
|
|
project = _bundle_project(tmp_path, verdict_dir=_tied_inbox(tmp_path))
|
|
monkeypatch.setattr("portfolio_optimiser.run.load_reference_projects", lambda: (project,))
|
|
|
|
rc = run.main(["--portfolio", "--semantic-retrieval"])
|
|
assert rc == 0
|
|
|
|
gen_prompts = _generation_prompts(_recorded_main)
|
|
assert gen_prompts, "the generation call must have happened"
|
|
assert any(_CLI_MARKER in p for p in gen_prompts), (
|
|
"the marker did not reach the hypothesis prompt in portfolio mode — run_portfolio is not "
|
|
"forwarding semantic_retrieval to run_project"
|
|
)
|
|
|
|
|
|
def test_cli_portfolio_mode_without_the_flag_leaves_the_marker_out(
|
|
tmp_path, monkeypatch, _recorded_main
|
|
):
|
|
"""CAUSALITY CONTROL for the portfolio arm."""
|
|
project = _bundle_project(tmp_path, verdict_dir=_tied_inbox(tmp_path))
|
|
monkeypatch.setattr("portfolio_optimiser.run.load_reference_projects", lambda: (project,))
|
|
|
|
rc = run.main(["--portfolio"])
|
|
assert rc == 0
|
|
|
|
assert all(_CLI_MARKER not in p for p in _recorded_main), (
|
|
"the marker reached a prompt without the flag — the portfolio default is not structural"
|
|
)
|
|
|
|
|
|
def _bundle_project(tmp_path: Path, *, verdict_dir: str) -> Project:
|
|
"""One bundle-backed project pointed at the SAME bundle the tied set was derived from — a
|
|
different bundle would yield a different query and untie the set."""
|
|
docs = tmp_path / "portfolio-docs"
|
|
docs.mkdir(exist_ok=True)
|
|
(docs / "cost.txt").write_text(
|
|
"LED-retrofit av lysrorarmaturer i kontorlokaler reduserte energikostnaden.",
|
|
encoding="utf-8",
|
|
)
|
|
return Project(
|
|
id=_PID,
|
|
name="Bundle-backed portfolio project",
|
|
description="bundle-backed project for the CLI semantic-retrieval forwarding proof",
|
|
currency="NOK",
|
|
cost_items=(),
|
|
docs_dir=str(docs),
|
|
verdict_input={"decision": "approved", "rationale": "expert reviewed (sim)"},
|
|
bundle_dir=str(_BUNDLE_DIR),
|
|
verdict_dir=verdict_dir,
|
|
)
|