portfolio-optimiser/tests/test_content_keyed_script_loadbearing.py

"""The scripted demo proposer is DATA, keyed on the project the prompt names (demo-week plan §3
Monday, §4 risk 2): adding a project to the walkthrough must be a registry entry, never a
hand-written second script under time pressure.

**The open decision this file closes — measured, not assumed.** The plan (§6) stated explicitly that
"the candidate is uniquely identifiable in the blob" was UNVERIFIED. Dumping every blob that reaches
the ``reply_selector`` across a full two-run simulation shows two prompt shapes:

* the DEBATE prompt (``run.py``: ``"Find a cost-saving measure for {project.id}.\\nContext:\\n..."``)
  — the whole bundle context, ~12k chars;
* the GENERATION prompt (``generate._build_messages``: ``"Project: {id} - {name}"`` plus, as its
  context, the DEBATE OUTPUT).

The cost code and the measure name reach the *generation* prompt only because the scripted reply
itself is echoed back as ``debate_output`` — keying on them would key the script on its own output.
The project id is the one identifier that BOTH shapes carry and that the FRAMEWORK stamps. So the
project id is the key, and these tests pin that decision.

The load-bearing set:
- per-project keying (RED on any proposer that ignores the prompt);
- the flip key is scoped to its own candidate (RED if one global flip token is shared, which would
  make one project's falsification correct another project's proposal);
- unknown and ambiguous prompts FAIL LOUD (RED on a first-match/default-fallback implementation —
  the silent-wrong-script failure this whole seam exists to prevent);
- the wiring: the simulation's own proposer is the registry-driven one (RED the moment it reverts to
  two hard-coded constants — the seam would exist while the demo still ran off a hand-written
  script);
- the data-entry rule: a candidate's flip key must be ABSENT from the bundle it is demoed against,
  or attempt 1's prompt already contains it and the correction proves nothing.
"""

from __future__ import annotations

from pathlib import Path

import pytest

from portfolio_optimiser import simulation
from portfolio_optimiser.simulation import (
    ScriptedCandidate,
    ScriptedCandidateError,
    scripted_proposer,
    simulate_learning_loop,
)

_BUNDLE_DIR = Path(__file__).resolve().parents[1] / "shared" / "examples" / "bygg-energi-mikro"

# A two-entry registry: the whole point is that a SECOND project is data. The claims are distinct
# per candidate so an assert can tell whose script answered, and the flip keys are distinct so a
# leaked global flip token is observable.
_ALFA = ScriptedCandidate(
    project_id="DEMO-ALFA",
    overclaimed='{"measure":"alfa","affected_items":[],"claimed_saving_nok":111111}',
    corrected='{"measure":"alfa","affected_items":[],"claimed_saving_nok":11}',
    flip_key="111111",
)
_BETA = ScriptedCandidate(
    project_id="DEMO-BETA",
    overclaimed='{"measure":"beta","affected_items":[],"claimed_saving_nok":222222}',
    corrected='{"measure":"beta","affected_items":[],"claimed_saving_nok":22}',
    flip_key="222222",
)
_REGISTRY = (_ALFA, _BETA)


def _generation_prompt(project_id: str, *, tail: str = "") -> str:
    """A prompt in the measured shape of ``generate._build_messages`` — the framework stamps the
    project id; ``tail`` stands in for the appended rejection block."""
    return (
        "Propose ONE concrete cost-saving measure for this project.\n"
        f"Project: {project_id} - Et prosjekt\n"
        "Context (prior verdicts / cited cost docs):\n\n" + tail
    )


def test_the_reply_is_keyed_on_the_project_the_prompt_names() -> None:
    """LOAD-BEARING: one registry, two projects, two different scripts — selected by the project id
    the framework stamped into the prompt. RED on a proposer that returns a constant, which is
    exactly the hand-written-script state this replaces."""
    proposer = scripted_proposer(_REGISTRY)

    assert proposer(_generation_prompt("DEMO-ALFA"), "proposer") == _ALFA.overclaimed
    assert proposer(_generation_prompt("DEMO-BETA"), "proposer") == _BETA.overclaimed


def test_the_falsification_flips_only_its_own_candidate() -> None:
    """LOAD-BEARING: the flip key belongs to the candidate, not to the module. A prompt that names
    BETA but carries ALFA's rejected figure must still get BETA's *overclaimed* reply — otherwise one
    project's falsification would silently correct another project's proposal, and Step 5 would show
    a correction nothing caused. RED on a single shared flip token.

    **The prompt names the SECOND registry entry on purpose** (measured): the obvious wrong
    implementation reads ``candidates[0].flip_key``, and asserting on the first entry cannot tell
    that apart from reading the matched candidate's — the two coincide there. A test that cannot
    separate two implementations proves nothing, so the assert is made where they diverge."""
    proposer = scripted_proposer(_REGISTRY)

    assert proposer(_generation_prompt("DEMO-BETA", tail=_ALFA.flip_key), "proposer") == (
        _BETA.overclaimed
    )
    # Control: BETA's OWN key does flip it, so the assert above is not merely observing a proposer
    # that never corrects at all.
    assert proposer(_generation_prompt("DEMO-BETA", tail=_BETA.flip_key), "proposer") == (
        _BETA.corrected
    )


def test_an_unregistered_project_fails_loud() -> None:
    """CONTROL: no default, no first-match fallback. A project without a registry entry must raise —
    a demo that silently answers with ANOTHER project's numbers is worse than one that stops, because
    the numbers would look plausible on screen. RED on a fallback implementation."""
    proposer = scripted_proposer(_REGISTRY)

    with pytest.raises(ScriptedCandidateError, match="DEMO-GAMMA|no scripted candidate"):
        proposer(_generation_prompt("DEMO-GAMMA"), "proposer")


def test_an_ambiguous_prompt_fails_loud() -> None:
    """CONTROL: two registered ids in one blob is undecidable, so it must raise rather than pick.
    This is reachable for real — a bundle's context can mention a sibling project — and the fix is
    the DATA (distinct ids), which is why the failure must be visible at rehearsal, not at the demo."""
    proposer = scripted_proposer(_REGISTRY)
    both = _generation_prompt("DEMO-ALFA") + "\nSe også DEMO-BETA.\n"

    with pytest.raises(ScriptedCandidateError, match="ambiguous|DEMO-BETA"):
        proposer(both, "proposer")


def test_the_simulation_proposer_is_registry_driven() -> None:
    """WIRING: the seam must be the one the DEMO runs on. RED the moment ``_proposer_reply`` reverts
    to two hard-coded constants — a constant proposer answers an unknown project happily."""
    assert any(c.project_id == simulation._PROJECT_ID for c in simulation._CANDIDATES), (
        "the demo project has no registry entry — the walkthrough would raise at the first turn"
    )

    with pytest.raises(ScriptedCandidateError):
        simulation._proposer_reply(_generation_prompt("IKKE-REGISTRERT"), "proposer")


def test_every_candidates_flip_key_is_absent_from_the_demo_bundle() -> None:
    """DATA-ENTRY RULE: the flip key is what tells the scripted proposer that the validator's
    rejection came back. If it already occurs in the bundle, attempt 1's prompt carries it, the
    proposer 'corrects' before anything was falsified, and Step 5 shows a correction with no cause.
    Checked against the demo bundle for every registered candidate, so adding an entry with a
    colliding key is caught here rather than on stage."""
    corpus = "\n".join(p.read_text("utf-8") for p in sorted(_BUNDLE_DIR.rglob("*")) if p.is_file())
    assert corpus, "control: the bundle was read, so an absence assert below means something"

    for candidate in simulation._CANDIDATES:
        assert candidate.flip_key not in corpus, (
            f"{candidate.project_id}: flip key {candidate.flip_key!r} occurs in the demo bundle, so "
            "attempt 1's prompt already contains it and the correction proves nothing"
        )


async def test_the_project_id_is_data_too(tmp_path: Path) -> None:
    """A new bundle is pointed at by ARGUMENT, project id included — otherwise Tuesday's content
    swap still needs a code edit. RED if ``project_id`` is ignored and the module constant is used:
    the bundle's own IR projection would then match and no error would surface."""
    with pytest.raises(ValueError, match="project_id"):
        await simulate_learning_loop(
            str(_BUNDLE_DIR), str(tmp_path), project_id="ET-ANNET-PROSJEKT"
        )