portfolio-optimiser/tests/test_golden_transcript_loadbearing.py

"""P4 pkt. 3 — the demo transcript is pinned as a checked-in fasit, and the pin is also the
demo's abort path.

**What this adds over criterion 6.** Criterion 6 is self-identity: run the demo twice, diff the two
outputs. It catches non-determinism, and nothing else — two runs of a *regressed* demo agree with
each other just as happily as two runs of a correct one. Between the Wednesday freeze and the
Thursday stage there is no second run to compare against, so the fasit has to leave the process:
the transcript is checked in, and the demo is diffed against the file.

**Why stdout is literal and stderr is not.** stdout was measured byte-identical across runs AND
across a fresh clone (P4 pkt. 1), so it is pinned verbatim — the file under ``golden/`` is exactly
what the operator sees, which is what makes it usable as the abort path: if the live run fails on
stage, that file IS the transcript. stderr carries two spans that are environment state rather than
program output, and both were measured to differ between a fresh clone and the working copy:

1. the absolute ``.../site-packages/`` prefix in the two ``ExperimentalWarning`` lines, and
2. the temp directory behind ``(arbeidskopi: ...)`` — a fresh ``mkdtemp`` suffix every run, under a
   ``TMPDIR`` that is per-user.

Both are masked; everything else is compared byte for byte. **Pinned stderr is two lines: a blank
line and the ``arbeidskopi:`` line — it was four until core 1.16.0 (F15, measured).** MAF moved the
``ExperimentalWarning`` emission from decoration time to first USE (``_feature_stage.py``'s
``_add_runtime_warning``), and the demo instantiates neither ``SkillResource`` nor ``MemoryStore``,
so the two warning lines stopped being emitted upstream — they were never suppressed here. Span 1's
``site-packages`` masking is KEPT for that reason: the span is currently unoccupied, not retired,
and a warning that returns must still trip this pin with its path masked.

**The narrowness is the point, and it is under test.** P4 pkt. 2 damped the round-cap notices with a
filter keyed on the MESSAGE precisely so that a NEW warning — from a MAF bump, or a subtree pull —
would still reach stderr and trip this pin. A normalisation that masked whole lines would undo that
in one step and leave a pin that can no longer fail for the reason it exists. So the masking is
measured, not asserted: ``test_normalisation_does_not_mask_a_new_warning`` feeds a stderr with one
extra line through the same normaliser and requires it to STOP matching the fasit.

**Honesty about what a golden proves.** The fasit was generated from a run, so it cannot by itself
prove that run was right — it pins the output the P3 criteria were measured against (eight step
lines; hypothesis #1 REJECTED on the P90 stage and the corrected one VALIDATED for the same
candidate; both learning paths reaching Run B). Regenerating it is therefore a decision, never
housekeeping: whoever regenerates re-takes those measurements.
"""

from __future__ import annotations

import os
import re
import subprocess
import sys
from pathlib import Path

import pytest

_GOLDEN_DIR = Path(__file__).resolve().parent / "golden"
_STDOUT_FASIT = _GOLDEN_DIR / "demo-transcript.stdout"
_STDERR_FASIT = _GOLDEN_DIR / "demo-transcript.stderr"

# Span 1: the interpreter's site-packages prefix, up to and including the separator. Anchored on
# the literal ``/site-packages/`` rather than on a home directory, so a checkout anywhere (or a
# different Python minor version) normalises to the same text.
_SITE_PACKAGES = re.compile(r"\S*/site-packages/")
# Span 2: the throwaway working copy. The ``po-sim-`` prefix is kept VISIBLE — it is a property of
# the program (``tempfile.mkdtemp(prefix="po-sim-")``), not of the environment — while the TMPDIR
# root and the random suffix, which are both, are masked.
_WORKING_COPY = re.compile(r"\(arbeidskopi: \S*/po-sim-\w+\)")


def normalise_stderr(text: str) -> str:
    """Mask the two measured environment-dependent spans, and nothing else."""
    text = _SITE_PACKAGES.sub("<SITE-PACKAGES>/", text)
    return _WORKING_COPY.sub("(arbeidskopi: <TMPDIR>/po-sim-<SUFFIKS>)", text)


@pytest.fixture(scope="module")
def demo() -> subprocess.CompletedProcess[str]:
    """Run the real demo once for this module.

    Runs the ``-m`` form rather than the console script: it needs no assumption about PATH, and the
    two forms were measured to write identical stdout when the entry point was added (P4 pkt. 5).
    ``PYTHONIOENCODING`` is pinned because the fasit is a file of UTF-8 bytes — without it the
    comparison would measure the operator's locale instead of the program.
    """
    proc = subprocess.run(
        [sys.executable, "-m", "portfolio_optimiser.simulation"],
        capture_output=True,
        text=True,
        encoding="utf-8",
        env={**os.environ, "PYTHONIOENCODING": "utf-8"},
        check=False,
    )
    assert proc.returncode == 0, proc.stderr
    return proc


def test_demo_stdout_matches_the_checked_in_transcript(
    demo: subprocess.CompletedProcess[str],
) -> None:
    """T-P4.3a: stdout is byte-identical to the fasit — no normalisation, no tolerance.

    RED on any change to a step line, a number, a label or the ordering. That is the whole point:
    on demo day the operator has already read this file, and anything the program says that the
    file does not is a surprise on stage.
    """
    assert _STDOUT_FASIT.exists(), (
        f"the pinned transcript is missing: {_STDOUT_FASIT}. It is generated by re-taking the P3 "
        "measurements, not by copying whatever the demo currently prints."
    )
    assert demo.stdout == _STDOUT_FASIT.read_text(encoding="utf-8")


def test_demo_stderr_matches_the_normalised_transcript(
    demo: subprocess.CompletedProcess[str],
) -> None:
    """T-P4.3b: stderr, with the two environment spans masked, is identical to the fasit.

    RED when a new warning appears (a MAF bump, a subtree pull) and RED when the round-cap damping
    is detached — the two lines it drops would land right back in this comparison.
    """
    assert _STDERR_FASIT.exists(), f"the pinned stderr is missing: {_STDERR_FASIT}"
    assert normalise_stderr(demo.stderr) == _STDERR_FASIT.read_text(encoding="utf-8")


def test_normalisation_is_not_a_no_op(demo: subprocess.CompletedProcess[str]) -> None:
    """T-P4.3c (RED-proof for the test above): the raw stderr really does differ from the fasit.

    Without this, T-P4.3b would be indistinguishable from a run where the environment spans happened
    to be stable and the normaliser did nothing at all — and a masking that never fires cannot be
    said to be narrow.
    """
    fasit = _STDERR_FASIT.read_text(encoding="utf-8")
    assert demo.stderr != fasit
    assert normalise_stderr(demo.stderr) == fasit


def test_normalisation_does_not_mask_a_new_warning(
    demo: subprocess.CompletedProcess[str],
) -> None:
    """T-P4.3d (control): the masking is span-scoped, so an EXTRA stderr line still trips the pin.

    This is the property P4 pkt. 2 paid for by keying the damping on the message rather than on the
    logger. Measured here on a synthetic stderr rather than by provoking a real warning, because the
    thing under test is the normaliser's reach — the real-warning path is what T-P4.3b covers.
    """
    intruder = (
        "/somewhere/lib/python3.12/site-packages/agent_framework/_new.py:1: DeprecationWarning: "
        "something changed under us.\n"
    )
    polluted = intruder + demo.stderr
    assert normalise_stderr(polluted) != _STDERR_FASIT.read_text(encoding="utf-8")