portfolio-optimiser/tests/test_prepass_excerpt_fields_loadbearing.py

"""Load-bearing gate for order 20260908T141941Z (P3): the excerpt's new fields reach the prompt.

P2 (its ledger row is in `docs/invarianter.md`) measured that llm-ingestion-okf's producer
now emits ``title``, ``req_number``, ``sources`` and per-excerpt ``source_*`` locators on every
excerpt (14 members, was 9) -- and that ``po`` drops all five TWICE: ``PrepassExcerpt`` is
``extra="ignore"`` so parsing throws them away, and ``_data_blocks`` renders only ``concept_id``,
``adjudication`` and ``trust_tier`` so even a field that survived parsing would not reach the
DATA block. (b') -- "did the model name the concept" -- was therefore a ``po`` verdict, never a
model verdict: no implementation of the model could have passed it while the field never arrived.

**Field-form decision, measured, not guessed.** The producer's OWN message (cited in P2: K2
carries five ``source_*`` locators where the requirement bundles carry two, and the rule is a
prefix rule, not an allowlist) states that ``source_*`` is an open-ended FAMILY, not a fixed
two-member allowlist. Two forms were compared against "does a future producer's new ``source_foo``
key reach the prompt without a code change here":

- **Named fields** (``source_element_id: str | None``, ``source_sha256: str | None``, ...): a
  producer adding a THIRD ``source_*`` member requires a new field declared on ``PrepassExcerpt``
  and a new line in the renderer -- a code change, exactly what the prefix rule says not to expect.
- **``model_extra``** (``extra="allow"`` on ``PrepassExcerpt`` alone, read back via a
  ``source_``-prefix scan): a new ``source_*`` key lands in ``model_extra`` unnamed and is picked
  up by the SAME scan with zero code change.

The second form is what P3 implements, for ``source_*`` locators only. ``title``, ``req_number``
and ``sources`` are SS-8-declared, singular, fixed members -- not a growing family -- so they are
named fields.

**Known-negative, gated separately below:** a payload without the five new fields (the P1 form,
9 members) must render BYTE-IDENTICAL to before this change. The header line's format is the
thing that could drift, so it is pinned literally, not diffed against a recorded golden string.
"""

from __future__ import annotations

import json
from pathlib import Path
from typing import Any

from portfolio_optimiser import prepass

FIXTURE = Path(__file__).parent / "fixtures" / "prepass" / "bygg-energi-mikro-fixture.payload.json"


def _raw() -> dict[str, Any]:
    return json.loads(FIXTURE.read_text(encoding="utf-8"))


def _raw_with_new_fields() -> dict[str, Any]:
    """The P1-form fixture's first excerpt, upgraded to the shape P2 measured on a real payload."""
    raw = _raw()
    excerpt = raw["excerpts"][0]
    assert excerpt["concept_id"] == "bygg-kontor-nord"
    excerpt["title"] = "Kontorbygg Nord (BYGG-KONTOR-NORD)"
    excerpt["req_number"] = "Krav 3.3.1–13"
    excerpt["sources"] = [
        {"resource": "https://example.invalid/driftskrav/859984", "title": "D100:2027"}
    ]
    excerpt["source_element_id"] = "id-4b61eee9-a149-42b3-863d-293b8320c15a"
    excerpt["source_sha256"] = "c58e8bbc5fa9a5400c111e51b04c05f2cfd9edabd884ef5352a486fdab2cb5ab"
    return raw


# --- the fields survive parsing ---------------------------------------------------------------


def test_the_excerpt_carries_title_req_number_and_sources() -> None:
    payload = prepass.PrepassPayload.model_validate(_raw_with_new_fields())
    excerpt = payload.excerpts[0]
    assert excerpt.title == "Kontorbygg Nord (BYGG-KONTOR-NORD)"
    assert excerpt.req_number == "Krav 3.3.1–13"
    assert excerpt.sources is not None
    assert excerpt.sources[0].resource == "https://example.invalid/driftskrav/859984"


def test_a_future_source_star_key_survives_without_a_code_change() -> None:
    """The measured discriminator: a THIRD locator a future producer adds, never declared as a
    named field anywhere in this module, still reaches the excerpt through the prefix scan."""
    raw = _raw_with_new_fields()
    raw["excerpts"][0]["source_page_number"] = "12"
    payload = prepass.PrepassPayload.model_validate(raw)
    excerpt = payload.excerpts[0]
    locators = dict(excerpt.source_locators())
    assert locators["source_element_id"] == "id-4b61eee9-a149-42b3-863d-293b8320c15a"
    assert locators["source_sha256"] == (
        "c58e8bbc5fa9a5400c111e51b04c05f2cfd9edabd884ef5352a486fdab2cb5ab"
    )
    assert locators["source_page_number"] == "12"


def test_source_locators_excludes_the_named_sources_field() -> None:
    """``sources`` (the list) is a named field and must not double up as its own locator entry --
    it is not itself a ``source_*`` SCALAR."""
    payload = prepass.PrepassPayload.model_validate(_raw_with_new_fields())
    keys = {key for key, _ in payload.excerpts[0].source_locators()}
    assert "sources" not in keys


def test_a_p1_form_excerpt_has_no_source_locators() -> None:
    """The absence control: an excerpt carrying none of the new fields reports an empty tuple,
    never a fabricated entry."""
    payload = prepass.load_prepass_payload(str(FIXTURE))
    assert payload.excerpts[0].source_locators() == ()
    assert payload.excerpts[0].title is None
    assert payload.excerpts[0].req_number is None
    assert payload.excerpts[0].sources is None


# --- the fields reach the DATA block ------------------------------------------------------------


def _header_for(rendering: str, concept_id: str) -> str:
    for line in rendering.split("\n"):
        if line.startswith(f"--- BEGIN DATA {concept_id} "):
            return line
    raise AssertionError(f"no BEGIN DATA line for {concept_id!r} in rendering")


def test_the_data_block_renders_req_number_title_and_the_address() -> None:
    payload = prepass.PrepassPayload.model_validate(_raw_with_new_fields())
    header = _header_for(prepass.render_context(payload), "bygg-kontor-nord")
    assert "Krav 3.3.1–13" in header
    assert "Kontorbygg Nord (BYGG-KONTOR-NORD)" in header
    assert "https://example.invalid/driftskrav/859984" in header
    assert "id-4b61eee9-a149-42b3-863d-293b8320c15a" in header


def test_the_data_block_renders_a_future_locator_too() -> None:
    """The renderer reads the SAME ``source_locators()`` scan the parsing test proved is
    code-change-free, so a new locator reaches the prompt without touching ``_data_blocks``."""
    raw = _raw_with_new_fields()
    raw["excerpts"][0]["source_page_number"] = "12"
    payload = prepass.PrepassPayload.model_validate(raw)
    header = _header_for(prepass.render_context(payload), "bygg-kontor-nord")
    assert "source_page_number: 12" in header


def test_render_seed_also_carries_the_new_fields() -> None:
    """``render_seed`` shares ``_data_blocks`` with ``render_context`` (ko-(p)); a fix that patched
    only one arm would leave the exploration door silently behind the debate door."""
    payload = prepass.PrepassPayload.model_validate(_raw_with_new_fields())
    header = _header_for(prepass.render_seed(payload), "bygg-kontor-nord")
    assert "Krav 3.3.1–13" in header
    assert "Kontorbygg Nord (BYGG-KONTOR-NORD)" in header


# --- known-negative: a P1-form payload renders byte-identical to before ------------------------


def test_a_p1_form_payload_renders_the_header_exactly_as_before() -> None:
    """The known-negative. Every excerpt in the shipped fixture carries none of the five new
    fields, so the header must be EXACTLY the old two-field form -- pinned literally, because a
    change to the conditional append logic that leaked an empty clause (e.g. a trailing ``, ``)
    would not show up in a substring assert."""
    payload = prepass.load_prepass_payload(str(FIXTURE))
    rendering = prepass.render_context(payload)
    for excerpt in payload.excerpts:
        expected = (
            f"--- BEGIN DATA {excerpt.concept_id} "
            f"(adjudication: {excerpt.adjudication}, trust_tier: {excerpt.trust_tier}) ---"
        )
        assert expected in rendering, rendering


def test_a_p1_form_payload_render_seed_also_unchanged() -> None:
    payload = prepass.load_prepass_payload(str(FIXTURE))
    rendering = prepass.render_seed(payload)
    for excerpt in payload.excerpts:
        expected = (
            f"--- BEGIN DATA {excerpt.concept_id} "
            f"(adjudication: {excerpt.adjudication}, trust_tier: {excerpt.trust_tier}) ---"
        )
        assert expected in rendering, rendering


def test_a_partially_upgraded_payload_only_appends_the_fields_it_has() -> None:
    """One excerpt carries the new fields, a sibling does not (a real corpus mixes eras of
    ingestion) -- the sibling's header must stay in the old two-field form."""
    raw = _raw_with_new_fields()
    payload = prepass.PrepassPayload.model_validate(raw)
    rendering = prepass.render_context(payload)
    sibling = payload.excerpts[1]
    assert sibling.title is None
    expected_sibling = (
        f"--- BEGIN DATA {sibling.concept_id} "
        f"(adjudication: {sibling.adjudication}, trust_tier: {sibling.trust_tier}) ---"
    )
    assert expected_sibling in rendering, rendering