portfolio-optimiser/tests/test_prepass_padding_collapse_loadbearing.py

"""A spreadsheet render's PADDING is dropped on the way into the prompt; nothing else is.

**Finding 5 of the S7 acid test** (``docs/2026-09-07-syretest-s7-prepass-k2.md`` § 10 pkt. 5), after
S7c had already felled the obvious cure. Opening both locks DID deliver K2's price schedule -- it
came back at rank 10 and its bytes stood in 2 of 11 prompts -- and the model still cited
``5647500`` / ``prissammenstilling`` / ``prisskjema`` / ``pris`` in **0 of 4** and **0 of 11**
answers (``docs/2026-09-08-syretest-s7c-begge-laaser-k2.md`` § 5). The blind spot had moved off the
ranking and onto the READING, and what po owns there is the rendering of the excerpt that reaches
the prompt.

**MEASURED FIRST, on the actual rendering path** (``prepass.concept_text``, the local re-derivation
that ``verify_against_bundle`` binds a payload to -- not the raw file), against
``~/repos/portfolio-optimiser/scratchpad/s7-prepass/k2-bundle-s7/del-ii-bilag-7-prisskjema/prissammenstilling-sheet-1.md``:

* raw on disk 119 lines / 101 188 B; delivered excerpt **104 lines / 67 245 characters** -- which
  reconciles the two denominators the order carried: the same document, before and after the
  frontmatter split and the per-line ``rstrip`` the producer does.
* the delivered text still holds **208** interior whitespace runs of >= 2 characters, **117** of
  them >= 100 and the longest **887**;
* those >= 100 runs are **56 806 of 67 245 characters = 84.5 %** of the excerpt, spread over
  **72 of 104 lines**.

So a label and its amount reach the model hundreds of characters apart -- ``82`` then 593 spaces
then ``Prosjektering (tiltransport av prosjekterende)`` then 250 spaces then ``5647500.0`` -- and
84.5 % of what the run pays for is a column width pandoc chose.

**The collapse is DETERMINISTIC and carries no judgement**, and each half of that is gated below:

* interior runs of spaces/tabs collapse to ONE space -- K = 2, i.e. there is no threshold to argue
  about; "two or more" is the whole rule.
* LEADING whitespace is untouched. Indentation is markdown structure (nested lists, indented code),
  not padding, and a rule that flattened it would rewrite documents rather than unpad them.
* no newline is touched, so the line count is invariant and no row is merged with its neighbour.
* no non-whitespace character is touched, so no number can change -- asserted as the per-line
  non-whitespace sequence, not merely as "the digits are still somewhere".

**WHERE it lives is the load-bearing choice.** It is in ``_data_blocks`` -- the ONE renderer both
arms share (kø-(p)) -- and therefore AFTER ``verify_against_bundle``. ``concept_text`` is the
re-derivation a payload's bytes are checked against, so collapsing there would make every payload
ever written fail its own digest, and would defeat the gate that stops a payload delivering bytes
the base does not hold. Arm (f) is that separation.

**Scope limit, stated.** That a model then USES the numbers cannot be shown here -- it would take a
paid run, and none is ordered. What is delivered is the deterministic property; a paid confirmation
is a later, separate order.

**Honesty limit, stated.** Collapsing to a single space loses the CELL BOUNDARY: after the collapse
``Post SUM`` is indistinguishable from prose that happens to contain those two words. That is a
real loss. Inventing a delimiter to carry the boundary would be exactly the judgement this rule is
written to avoid, and no measurement here says which delimiter a model reads better; so the loss is
recorded rather than papered over.
"""

from __future__ import annotations

import json
import re
from pathlib import Path
from typing import Any

from portfolio_optimiser import prepass

FIXTURE = Path(__file__).parent / "fixtures" / "prepass" / "bygg-energi-mikro-fixture.payload.json"

#: A K2-shaped row: code, 593 spaces, description, 250 spaces, amount. The widths are the ones
#: measured on line 34 of the real price schedule, so the fixture is the measurement rather than a
#: round number chosen for the test.
_CODE = "82"
_DESC = "Prosjektering (tiltransport av prosjekterende)"
_AMOUNT = "5647500.0"
_PADDED_ROW = f"  {_CODE}{' ' * 593}{_DESC}{' ' * 250}{_AMOUNT}"
#: An indented fenced block: leading whitespace is structure and must survive untouched.
_INDENTED = "    indented code line"
_PADDED_TEXT = "\n".join(["## Prissammenstilling", "", _PADDED_ROW, _INDENTED, ""])


def _payload_with(text: str) -> prepass.PrepassPayload:
    raw: dict[str, Any] = json.loads(FIXTURE.read_text(encoding="utf-8"))
    raw["excerpts"] = [raw["excerpts"][0]]
    raw["excerpts"][0]["text"] = text
    return prepass.PrepassPayload.model_validate(raw)


def _longest_interior_run(text: str) -> int:
    """The longest run of horizontal whitespace that is NOT a line's leading indentation."""
    longest = 0
    for line in text.split("\n"):
        body = line.lstrip(" \t")
        for match in re.finditer(r"[ \t]{2,}", body):
            longest = max(longest, len(match.group(0)))
    return longest


# --- (a) the delivered excerpt reaches the prompt with a BOUNDED separator ---------------------


def test_the_rendered_excerpt_separates_label_and_amount_by_a_bounded_separator() -> None:
    """(a) The property finding 5 is about. Detach point: remove the collapse from
    ``_data_blocks`` -> RED, because the 593- and 250-character runs ride into the prompt."""
    rendered = prepass.render_context(_payload_with(_PADDED_TEXT))
    assert _longest_interior_run(rendered) <= 1, (
        "the rendered excerpt still carries a padding run; a label and its amount reach the "
        "model hundreds of characters apart"
    )
    assert f"{_CODE} {_DESC} {_AMOUNT}" in rendered, (
        "the row's three cells no longer read as one row"
    )


def test_a_text_that_needs_no_collapse_renders_unchanged() -> None:
    """(a) CONTROL. An excerpt with no padding must render exactly as before, or arm (a) would be
    satisfied by a renderer that mangles every payload equally."""
    plain = "## Heading\n\nOne ordinary sentence with single spaces.\n"
    assert plain in prepass.render_context(_payload_with(plain))


# --- (b) known-negative: no number, and no other character, is ever changed --------------------


def test_no_non_whitespace_character_is_changed() -> None:
    """(b) KNOWN-NEGATIVE. The per-line sequence of non-whitespace characters is identical before
    and after. Stronger than "the digits are still somewhere": a rule that stripped a trailing
    ``.0``, or reordered two cells, would pass a looser assertion and fail this one."""
    collapsed = prepass.collapse_padding(_PADDED_TEXT)
    for before, after in zip(_PADDED_TEXT.split("\n"), collapsed.split("\n"), strict=True):
        assert re.sub(r"\s+", "", before) == re.sub(r"\s+", "", after)
    assert _AMOUNT in collapsed


# --- (c) known-negative: no line is merged with its neighbour ----------------------------------


def test_no_line_is_merged_with_its_neighbour() -> None:
    """(c) KNOWN-NEGATIVE. Newlines are not whitespace this rule may touch: merging rows is how a
    naive ``re.sub(r"\\s+", " ", text)`` would silently turn a table into a paragraph."""
    collapsed = prepass.collapse_padding(_PADDED_TEXT)
    assert collapsed.split("\n") != []
    assert len(collapsed.split("\n")) == len(_PADDED_TEXT.split("\n"))


# --- (d) leading indentation is structure, not padding -----------------------------------------


def test_leading_indentation_survives() -> None:
    """(d) An indented line keeps its indentation: markdown reads it as structure, and collapsing
    it would rewrite the document rather than unpad it."""
    collapsed = prepass.collapse_padding(_PADDED_TEXT)
    assert _INDENTED in collapsed.split("\n")


# --- (e) ONE seam, both arms --------------------------------------------------------------------


def test_both_arms_collapse_because_they_share_one_renderer() -> None:
    """(e) ``render_seed`` is the exploration's arm and ``render_context`` the debate's; they share
    ``_data_blocks``. A fix in one only would leave the other reading padding."""
    payload = _payload_with(_PADDED_TEXT)
    for rendered in (prepass.render_context(payload), prepass.render_seed(payload)):
        assert _longest_interior_run(rendered) <= 1


# --- (f) the gate is untouched: verification reads the UNCOLLAPSED text -------------------------


def test_the_local_re_derivation_is_not_collapsed() -> None:
    """(f) ``concept_text`` is what ``verify_against_bundle`` binds a payload's bytes to. Collapsing
    THERE would make every payload ever written fail its own digest and would break the gate that
    stops a payload delivering bytes the base does not hold. Detach point: move the collapse into
    ``concept_text`` -> RED."""
    padded = Path(__file__).parent / "fixtures" / "prepass" / "_padded-concept.md"
    padded.parent.mkdir(parents=True, exist_ok=True)
    padded.write_text(f"---\ntype: reference\n---\n\n{_PADDED_ROW}\n", encoding="utf-8")
    try:
        derived = prepass.concept_text(padded)
        assert " " * 593 in derived, (
            "the local re-derivation collapsed padding; a payload's own digest can no longer match"
        )
    finally:
        padded.unlink()