portfolio-optimiser/tests/test_identifier_grounding_loadbearing.py

"""P7 — an identifier the proposal builds on must appear VERBATIM in the input, or the verdict
falls.

The hole this closes was MEASURED (økt 108/109). ``_reconcile_against_baseline`` already carries
the sentence "the cost code is absent from the baseline — a fabricated line", but it is reached
only through ``if baseline is not None`` (``validator.py``), and an unanchored run has no baseline.
So the paid K2 run of økt 108 ended in ``ValidatedProposal`` (verdict ``5fd6272e3725fe68``) on two
cost codes — ``M-04-01`` / ``M-04-03`` — that appear in NO prompt of that run. The falsifier was
tied to whether a cost baseline happened to exist. **The input always exists; the baseline does
not.**

Every fixture under ``tests/fixtures/p7-grounding/`` is the VERBATIM generation prompt of a free
recording — the entire input the proposer saw on the attempt that produced its candidate. They are
tracked test data on purpose: the recordings live in ``scratchpad/``, which ``git archive HEAD``
does not carry, so a test reading them would pass here and fail the handover gate.
"""

from __future__ import annotations

from pathlib import Path

import pytest

from portfolio_optimiser.ir import (
    AffectedItem,
    CostBaseline,
    CostBaselineLine,
    SavingsProposal,
)
from portfolio_optimiser.validator import Rejection, ValidatedProposal, validate_proposal

FIXTURES = Path(__file__).parent / "fixtures" / "p7-grounding"

#: The known positive (premiss (vii), `docs/2026-09-08-n-bundlene-hypoteseform.md` § 11.4): the
#: model quoted this requirement number VERBATIM in 6 of 6 replies, and it stands in 6 of 6
#: prompts. Note the EM-DASH (U+2014): the hyphen variant scores 0 of 6 in both.
KNOWN_POSITIVE = "Krav 3.3.1—13"


def _prompt(name: str) -> str:
    return (FIXTURES / name).read_text(encoding="utf-8")


def _proposal(*codes: str, unit_cost: float = 100.0) -> SavingsProposal:
    """A minimal proposal carrying exactly ``codes``, cheap enough that every LATER stage passes —
    so a rejection can only have come from the grounding stage."""
    return SavingsProposal(
        project_id="K2",
        measure="reduce marking scope",
        affected_items=[
            AffectedItem(code=c, quantity=10.0, unit_cost=unit_cost) for c in codes
        ],
        claimed_saving_nok=1.0,
    )


def _project():
    """The reference project, used only as the prompt's header — the grounding text is the rendered
    prompt, so the project's own cost items play no part here."""
    from portfolio_optimiser.reference_domain import load_reference_projects

    return load_reference_projects()[0]


# --------------------------------------------------------------------------------------------
# (a)/(b) the two recordings that reached a verdict on fabricated codes
# --------------------------------------------------------------------------------------------


@pytest.mark.parametrize(
    ("fixture", "codes"),
    [
        ("p6-k2-generation-prompt.txt", ("M-04-01", "M-04-03")),
        ("s7c-k2-generation-prompt.txt", ("PRD-001", "PRD-002")),
    ],
)
def test_a_fabricated_code_from_a_real_recording_falls(fixture: str, codes: tuple[str, ...]) -> None:
    """(a)+(b). Both recordings' codes are absent from the prompt that produced them (denominator
    2 of 2 in each), and the verdict must now fall — with ``baseline=None``, which is exactly the
    configuration under which økt 108 said ``validated``."""
    text = _prompt(fixture)
    for code in codes:
        assert code not in text, f"fixture drifted: {code!r} is IN the prompt"
    ruling = validate_proposal(_proposal(*codes), baseline=None, grounding=text)
    assert isinstance(ruling, Rejection), f"{codes} cleared the gate on an input that names neither"
    for code in codes:
        assert repr(code) in ruling.reason


def test_b_the_reason_names_every_ungrounded_identifier_in_the_proposal_s_own_order() -> None:
    """Completeness, for ``_reconcile_against_baseline``'s own reason (økt 94, measured live): a
    message naming only the FIRST violation reads as an instruction to fix that one field, and the
    proposer fixes one and rebreaks the other. Same ``"; "`` joiner, same PROPOSAL order."""
    text = _prompt("p6-k2-generation-prompt.txt")
    ruling = validate_proposal(_proposal("M-04-03", "M-04-01"), baseline=None, grounding=text)
    assert isinstance(ruling, Rejection)
    parts = ruling.reason.split("; ")
    assert len(parts) == 2, f"expected one sentence per violation, got {ruling.reason!r}"
    assert "'M-04-03'" in parts[0] and "'M-04-01'" in parts[1], ruling.reason


# --------------------------------------------------------------------------------------------
# (c) THE KNOWN POSITIVE — with its control on the SAME input
# --------------------------------------------------------------------------------------------


def test_c_the_known_positive_is_not_flagged() -> None:
    """``Krav 3.3.1—13`` stands VERBATIM in the N100 generation prompt, so a proposal that cites it
    is grounded and the grounding stage must stay silent. A rule that flags everything code-shaped
    is red here."""
    text = _prompt("p4-n100-generation-prompt.txt")
    assert KNOWN_POSITIVE in text, "fixture drifted: the known positive is not in the prompt"
    ruling = validate_proposal(_proposal(KNOWN_POSITIVE), baseline=None, grounding=text)
    assert isinstance(ruling, ValidatedProposal), getattr(ruling, "reason", "")


def test_c_control_the_rule_can_still_flag_on_that_same_recording() -> None:
    """THE CONTROL that keeps the arm above from proving nothing. ``CRS-01`` is the one identifier
    the model produced on the N100 recording that no prompt carries (1 of 2 ungrounded). Same
    fixture, same call — only the identifier differs, and this one must fall."""
    text = _prompt("p4-n100-generation-prompt.txt")
    assert "CRS-01" not in text
    ruling = validate_proposal(_proposal("CRS-01"), baseline=None, grounding=text)
    assert isinstance(ruling, Rejection)
    assert "'CRS-01'" in ruling.reason


# --------------------------------------------------------------------------------------------
# (d) a code that IS in the input is never flagged, however fabricated it looks
# --------------------------------------------------------------------------------------------


def test_d_a_code_quoted_verbatim_from_the_input_is_not_flagged() -> None:
    """The discriminator between this rule and "flag anything that looks like a code". The token is
    deliberately shaped like the fabrications above; the ONLY difference is that the input says it."""
    text = "Context:\nPrice schedule line ZZZ-999-01 covers technical marking.\n"
    ruling = validate_proposal(_proposal("ZZZ-999-01"), baseline=None, grounding=text)
    assert isinstance(ruling, ValidatedProposal), getattr(ruling, "reason", "")


# --------------------------------------------------------------------------------------------
# (e) THE HOLE ITSELF: the stage does not depend on a baseline
# --------------------------------------------------------------------------------------------


def test_e_the_stage_fires_with_no_baseline_at_all() -> None:
    """Premiss (iv). The pre-P7 tree returns ``ValidatedProposal`` here — measured on the real P6
    proposal — because stage 0 sits behind ``if baseline is not None``."""
    text = _prompt("p6-k2-generation-prompt.txt")
    assert isinstance(
        validate_proposal(_proposal("M-04-01"), baseline=None, grounding=text), Rejection
    )


def test_e_the_stage_is_not_gated_on_the_absence_of_a_baseline() -> None:
    """Independence in the other direction, and a statement about the STAGE rather than about the
    product: an anchored call whose baseline carries the code still reaches stage 0b, and stage 0b
    judges only the text it was handed. What counts as "the input" is the CALLER's decision — and
    the caller (``generate._grounding_text``) folds the baseline's own codes in, so no real run
    rejects a line its baseline proves. The pair is deliberate: putting the baseline inside this
    stage would make the stage untestable in isolation, and putting it nowhere would turn an
    anchored run into a wall (see ``test_g_...`` below for the product-level half)."""
    text = _prompt("p6-k2-generation-prompt.txt")
    baseline = CostBaseline(
        project_id="K2", items={"M-04-01": CostBaselineLine(quantity=10.0, unit_cost=100.0)}
    )
    assert isinstance(
        validate_proposal(_proposal("M-04-01"), baseline=baseline, grounding=text), Rejection
    )


def test_e_control_no_grounding_argument_is_the_pre_P7_behaviour() -> None:
    """``grounding=None`` must be byte-for-byte the old gate — that is what keeps every existing
    caller, every golden and the demo transcript unchanged."""
    ruling = validate_proposal(_proposal("M-04-01", "M-04-03"), baseline=None)
    assert isinstance(ruling, ValidatedProposal)


# --------------------------------------------------------------------------------------------
# (f) THE WIRING — the seam where a model-authored proposal meets its own input
# --------------------------------------------------------------------------------------------


async def test_f_generate_via_llm_grounds_the_candidate_in_the_prompt_it_sent() -> None:
    """LOAD-BEARING WIRING, behavioural. ``generate_via_llm`` is the ONE place a model-authored
    proposal and the exact text the model was given exist at the same moment (premiss (viii)).
    The rendered ``messages`` ARE the proposer's entire input for that call, so an identifier the
    proposal carries but the messages do not came from the model's weights, not from this run.

    Driven through the canonical ``ScriptedChatClient`` (S2.5) so the seam is exercised, not
    inspected. RED when ``grounding=`` is detached at the call site: the outcome flips to
    ``ValidatedProposal`` — which is precisely the økt-108 defect."""
    from portfolio_optimiser.budget import Budget, TokenMeter
    from portfolio_optimiser.generate import generate_via_llm
    from portfolio_optimiser.simulation import ScriptedChatClient

    project = _project()
    reply = _proposal("GHOST-77-01").model_dump_json()
    meter = TokenMeter(Budget(max_tokens=10**9, max_rounds=20))
    result = await generate_via_llm(
        ScriptedChatClient(reply=reply, tokens_per_reply=8),
        project,
        "The price schedule carries line REAL-11-02 for technical marking.",
        meter,
        max_attempts=1,
    )
    assert isinstance(result.outcome, Rejection), "an identifier no prompt carried was validated"
    assert "'GHOST-77-01'" in result.outcome.reason


async def test_f_control_a_code_the_context_names_survives_the_same_call() -> None:
    """THE CONTROL. Same client, same call, same meter — only the identifier changes, and this one
    the context states verbatim. Without it the arm above would also pass on a rule that rejects
    every proposal ``generate_via_llm`` ever produces."""
    from portfolio_optimiser.budget import Budget, TokenMeter
    from portfolio_optimiser.generate import generate_via_llm
    from portfolio_optimiser.simulation import ScriptedChatClient

    project = _project()
    reply = _proposal("REAL-11-02").model_dump_json()
    meter = TokenMeter(Budget(max_tokens=10**9, max_rounds=20))
    result = await generate_via_llm(
        ScriptedChatClient(reply=reply, tokens_per_reply=8),
        project,
        "The price schedule carries line REAL-11-02 for technical marking.",
        meter,
        max_attempts=1,
    )
    assert isinstance(result.outcome, ValidatedProposal), getattr(result.outcome, "reason", "")


# --------------------------------------------------------------------------------------------
# (g) THE SEAM'S OTHER TWO SOURCES — what only the caller can prove the run was given
# --------------------------------------------------------------------------------------------


def test_g_the_seam_grounds_a_code_the_baseline_proves_even_when_no_prompt_repeats_it() -> None:
    """The product-level half of the arm above. Stage 0 has ALREADY ruled this code a real line of
    the project; the weaker stage must not overrule the stronger falsifier just because the debate's
    prose summary did not repeat it. RED on a composer that drops the baseline: every anchored S2c
    run would then reject its own real codes — a wall, not a gate."""
    from portfolio_optimiser.generate import _grounding_text

    project = _project()
    baseline = CostBaseline(
        project_id="K2", items={"M-04-01": CostBaselineLine(quantity=10.0, unit_cost=100.0)}
    )
    text = _grounding_text(project, baseline, "the debate summarised this in prose, naming no code")
    assert isinstance(
        validate_proposal(_proposal("M-04-01"), baseline=baseline, grounding=text),
        ValidatedProposal,
    )


def test_g_the_seam_grounds_a_code_the_delivered_base_carries() -> None:
    """``delivered`` is the source only ``run_project`` can supply: on the S2c bundle path the
    debate NAVIGATES, so the base's own text never reaches the generation prompt. RED when the
    threading is detached — the run would then reject every real code of every unanchored base."""
    from portfolio_optimiser.generate import _grounding_text

    project = _project()
    text = _grounding_text(project, None, "the price schedule line BASE-77-01 is real")
    assert isinstance(
        validate_proposal(_proposal("BASE-77-01"), baseline=None, grounding=text),
        ValidatedProposal,
    )
    assert isinstance(
        validate_proposal(_proposal("BASE-77-02"), baseline=None, grounding=text), Rejection
    ), "CONTROL: the delivered base must not ground a code it does not carry"


async def test_g_run_project_grounds_a_candidate_in_the_base_it_delivered(tmp_path) -> None:
    """BEHAVIOURAL, over the real ``run_project`` bundle arm — the ONE place the delivered base's
    text is composed. Two arms over the same crafted copy of the fixture base:

    * a code whose ONLY occurrence is inside a ``type: verdict`` document must NOT ground the
      proposal. That is MAJOR-3 N2 / S7a-3's ``context_files``-not-``files`` rule applied to
      grounding: the verdict layer reaches a hypothesis through the GATED ExpeL fold alone, and
      grounding a proposal in a prior verdict routes that material around its own gate. RED when
      the grounding text is built from ``bundle.files``;
    * the CONTROL, an ordinary concept-file code on the same base, must survive — otherwise the arm
      above would also pass on a run that rejects everything, and RED when the threading from
      ``run_project`` is detached at all.
    """
    import shutil

    from portfolio_optimiser.run import run_project
    from portfolio_optimiser.simulation import ScriptedChatClient
    from portfolio_optimiser.validator import ValidatedProposal as VP

    base = tmp_path / "base"
    shutil.copytree(
        Path(__file__).resolve().parents[1] / "shared" / "examples" / "bygg-energi-mikro", base
    )
    (base / "dom-tidligere.md").write_text(
        "---\ntype: verdict\ntitle: Prior judgement\ndecision: approved\n---\n\n"
        "The measure on cost line VERDICT-ONLY-9 was approved.\n",
        encoding="utf-8",
    )
    index = base / "index.md"
    index.write_text(
        index.read_text(encoding="utf-8") + "\n- [Prior judgement](dom-tidligere.md)\n",
        encoding="utf-8",
    )

    async def _outcome(code: str):
        reply = (
            f'{{"measure":"LED-retrofit","affected_items":'
            f'[{{"code":"{code}","quantity":300000,"unit_cost":1.0}}],'
            f'"claimed_saving_nok":30000}}'
        )
        result = await run_project(
            "BYGG-KONTOR-NORD",
            "local",
            docs_dir=str(base),
            bundle_dir=str(base),
            client_factory=lambda role: ScriptedChatClient(
                "Reasoning holds.\nVERDICT: APPROVE" if role == "checker" else reply, role=role
            ),
        )
        return result.outcome

    control = await _outcome("ENERGI-TOTAL-EL")
    assert isinstance(control, VP), (
        f"CONTROL: a code the delivered base carries was not grounded — {control}"
    )
    leaked = await _outcome("VERDICT-ONLY-9")
    assert isinstance(leaked, Rejection), "a verdict-layer code grounded a proposal"
    assert "'VERDICT-ONLY-9'" in leaked.reason


# --------------------------------------------------------------------------------------------
# (h) THE GATE MUST NOT DISARM ITSELF ON ITS SECOND ROUND
# --------------------------------------------------------------------------------------------


async def test_h_the_refusal_does_not_ground_the_next_attempt_that_repeats_the_code() -> None:
    """MEASURED while building this rule, and the reason the rendered PROMPT is not evidence.

    Step 5 feeds the previous ``Rejection.reason`` VERBATIM into the next attempt's prompt — and
    this stage's reason QUOTES the identifier it just refused. An implementation that grounded a
    candidate in its own prompt would therefore accept on attempt 2 exactly what it refused on
    attempt 1: a falsifier that disarms itself. Both attempts here repeat the same ungrounded code,
    and both must fall.
    """
    from portfolio_optimiser.budget import Budget, TokenMeter
    from portfolio_optimiser.generate import generate_via_llm
    from portfolio_optimiser.simulation import ScriptedChatClient

    reply = _proposal("GHOST-77-01").model_dump_json()
    client = ScriptedChatClient(reply=reply, tokens_per_reply=8)
    result = await generate_via_llm(
        client,
        _project(),
        "The price schedule carries line REAL-11-02 for technical marking.",
        TokenMeter(Budget(max_tokens=10**9, max_rounds=20)),
        max_attempts=2,
    )
    assert client.call_count == 2, "the informed-refinement loop did not reach a second attempt"
    assert isinstance(result.outcome, Rejection), (
        "attempt 2 was accepted after attempt 1 refused the same identifier — the refusal grounded it"
    )
    assert "'GHOST-77-01'" in result.outcome.reason