portfolio-optimiser/tests/test_prose_code_form_loadbearing.py

"""P19 DEL B — a "cost code" must have a FORM when the input offers forms.

**The measured defect.** P18's round 2 ended with two ``validated`` proposals whose
``affected_item`` codes were ordinary words out of a road standard's prose:
``impulsventilator`` (in 4 of 270 N500 documents) and ``bituminøst bærelag`` (4 of 1 133 N200).
Both are GROUNDED in P7's sense — they appear verbatim in the input, which is all that stage asks —
and neither is INERT in P18/B1's sense, because neither is anywhere near the 5 % document share.
They are simply not identifiers of a cost line. The gate had no stage that could say so.

**Replayed offline against the very bases those runs were given** (the known positive, measured
15.09 rather than asserted): both come back ``Rejection`` naming the denominator —
``…offers 391 identifiers of its own`` / ``…offers 1359``.

**The generality guard is the half that makes this a rule rather than a preference.** The gate
fires ONLY where the input demonstrably offers identifier forms. A corpus that carries none cannot
be answered in a form it does not use, and refusing there would be a rule about SHAPES rather than
about grounding — exactly what ``_ground_against_input``'s own docstring refuses for the stage it
sits inside.

**B1 — the third and fourth forms, transcribed from a measurement.** R761's requirement numbers are
bare dotted numbers (``12.1``, ``52.11``); ALL SIX ``ref`` values in
``contexts/kontrakt-sorasen-2027/fasit.json`` are of that shape and NEITHER pre-P19 form matched
one, so r761's whole offer was **3 identifiers over 6.5 MB**. Measured after: **2 332**. And the
first form was WIDENED in the same pass, because P19/B2 made these forms decide ``prose`` as well
as count an offer: this repo's own ``ENERGI-TOTAL-EL`` matched neither, so the classifier called a
real cost code prose and the new gate refused it. A gate may only be wrong in the direction that
admits too much.

**Two exemptions, both load-bearing:**

* a code the BASELINE carries is never refused here — stage 0 has already ruled it a real line of
  this project, and the weaker stage must not overrule the stronger falsifier (the sentence
  ``_grounding_text`` already carries about its third source). A derived schedule whose codes are
  bare section numbers would otherwise be refused wholesale by the gate meant to protect it;
* ``has_identifier_form`` FULL-matches. ``impulsventilator 12.1`` carrying a process number does
  not make the word a cost code, and a substring rule would let any prose code smuggle one along.

What each arm pins:

(a) the two MEASURED defects are refused, with the denominator in the reason (Step 5 feeds that
    reason verbatim into the next attempt, and "ungrounded" alone teaches nothing);
(b) the generality guard: a base offering no identifier form leaves the gate OFF;
(c) an anchored code is exempt — the stronger falsifier wins;
(d) the 26 known negatives: every ``ref`` in all four fasit files still classifies as an identifier,
    and so does this repo's own ``ENERGI-TOTAL-EL``;
(e) B1's known positives and negatives, one by one;
(f) B2: the classification is reported, and it is the SAME classifier the gate uses.
"""

from __future__ import annotations

import glob
import json

import pytest

from portfolio_optimiser.ir import CostBaseline, CostBaselineLine, SavingsProposal
from portfolio_optimiser.validator import (
    Grounding,
    Rejection,
    ValidatedProposal,
    classify_codes,
    has_identifier_form,
    identifier_tokens,
    validate_proposal,
)

#: A base that OFFERS identifier forms: eleven documents carrying requirement numbers, one of which
#: also mentions the prose word. Eleven because ``_GROUNDING_MIN_INERT_DOCUMENTS`` is 10 — with
#: fewer, P18/B1's share rule cannot fire at all and this file would be measuring that instead.
_OFFERING = tuple(f"Krav 10.4.3—{i} om ventilasjon i tunnelen." for i in range(1, 12)) + (
    "Krav 8.4.2—1 sier at impulsventilator skal dimensjoneres for brannlast.",
)

#: The same corpus with every identifier removed — prose only.
_FORMLESS = ("ingen koder her, bare tekst om ventilasjon", "og enda mer prosa om tunnelen")


def _proposal(code: str, *, quantity: float = 4.0, unit_cost: float = 250_000.0) -> SavingsProposal:
    return SavingsProposal(
        project_id="t",
        measure="m",
        affected_items=[{"code": code, "quantity": quantity, "unit_cost": unit_cost}],
        claimed_saving_nok=100_000.0,
    )


# ---------------------------------------------------------------------------------------------
# (a) the measured defect
# ---------------------------------------------------------------------------------------------


def test_a_word_from_the_prose_is_not_a_cost_code() -> None:
    """(a) P18's ``impulsventilator``, in the shape that reached ``validated``."""
    verdict = validate_proposal(_proposal("impulsventilator"), grounding=Grounding(_OFFERING))
    assert isinstance(verdict, Rejection)
    assert "has no identifier form" in verdict.reason
    # The DENOMINATOR, not just a complaint: Step 5 feeds this reason into the next attempt.
    assert "offers 14 identifiers of its own" in verdict.reason


def test_the_same_input_still_admits_a_real_reference() -> None:
    """(a), the other half. A gate that can only refuse proves nothing."""
    assert isinstance(
        validate_proposal(_proposal("Krav 8.4.2—1"), grounding=Grounding(_OFFERING)),
        ValidatedProposal,
    )


# ---------------------------------------------------------------------------------------------
# (b)-(c) the two exemptions
# ---------------------------------------------------------------------------------------------


def test_a_base_that_offers_no_form_leaves_the_gate_off() -> None:
    """(b) The generality guard. A corpus with no identifiers cannot be answered in one."""
    assert isinstance(
        validate_proposal(_proposal("ventilasjon"), grounding=Grounding(_FORMLESS)),
        ValidatedProposal,
    )


def test_a_code_the_baseline_carries_is_never_refused_for_its_shape() -> None:
    """(c) Stage 0 has already ruled it a real line; the weaker stage must not overrule it."""
    baseline = CostBaseline(
        project_id="t",
        items={"impulsventilator": CostBaselineLine(quantity=4.0, unit_cost=250_000.0)},
    )
    assert isinstance(
        validate_proposal(
            _proposal("impulsventilator"), baseline=baseline, grounding=Grounding(_OFFERING)
        ),
        ValidatedProposal,
    )


# ---------------------------------------------------------------------------------------------
# (d)-(e) known negatives and positives
# ---------------------------------------------------------------------------------------------


def test_every_fasit_reference_in_every_context_set_is_an_identifier() -> None:
    """(d) Every fasit reference across every context set, with the denominator.

    One of them — ``Krav 3.3.2—1_1`` — is why the second form grew an optional ``_<n>`` suffix.
    Measured, not anticipated: before that it was the single reference the classifier called prose.

    **The denominator MOVED 26 -> 32 with P17b's fifth context set**, and it is asserted rather
    than dropped for the reason it was written down in the first place: a list comprehension over
    ``contexts/*/fasit.json`` that quietly found fewer rows would make this arm weaker without
    making it red. The six new ones are four ``Krav x.y.z—n`` from n200-2024 and TWO bare
    ``prosessnr`` from r761-2025 (``12.11``, ``12.12``) — the punctuation-and-digits form B1 added,
    now exercised by a fasit and not only by a known-positive.
    """
    refs = [
        concept["ref"]
        for path in sorted(glob.glob("contexts/*/fasit.json"))
        for row in json.loads(open(path, encoding="utf-8").read())["must_cite"]
        for concept in row["concepts"]
    ]
    assert len(refs) == 32, f"denominator moved: {len(refs)}"
    assert [r for r in refs if not has_identifier_form(r)] == []
    assert has_identifier_form("ENERGI-TOTAL-EL"), "this repo's own reference cost code"


@pytest.mark.parametrize(
    "token",
    ["12.1", "12.12", "22.1", "52.1", "52.11", "51.1", "65 ASFALTDEKKER", "SHA-01", "B-20-00-00"],
)
def test_b1_known_positives(token: str) -> None:
    """(e) The forms the delivered corpora carry."""
    assert has_identifier_form(token)


@pytest.mark.parametrize(
    "token",
    [
        "2027",
        "250000",
        "15.09.2026",
        "2026-09-15",
        "impulsventilator",
        "bituminøst bærelag",
        "0.70",
    ],
)
def test_b1_known_negatives(token: str) -> None:
    """(e) Bare numbers and dates are not identifiers — the K2-measured inert class."""
    assert not identifier_tokens(token), f"{token!r} was counted as an identifier"


# ---------------------------------------------------------------------------------------------
# (f) B2 — the report
# ---------------------------------------------------------------------------------------------


def test_the_classification_is_reported_and_is_the_gates_own() -> None:
    """(f) One classifier, two consumers (kø-(p)): a report that disagreed with the gate about one
    proposal would be evidence about nothing."""
    forms = classify_codes(["impulsventilator", "Krav 8.4.2—1", "12.1", "ENERGI-TOTAL-EL"])
    assert forms == {
        "impulsventilator": "prose",
        "Krav 8.4.2—1": "identifier",
        "12.1": "identifier",
        "ENERGI-TOTAL-EL": "identifier",
    }
    # And the gate agrees, code for code, on the SAME input.
    for code, kind in forms.items():
        verdict = validate_proposal(_proposal(code), grounding=Grounding(_OFFERING + (code,)))
        refused = isinstance(verdict, Rejection) and "has no identifier form" in verdict.reason
        assert refused == (kind == "prose"), (code, kind, verdict)


# --------------------------------------------------------------------------------------------
# P21 B3: ``rejection_stage`` — which falsifier wrote a reason. A REPORT, never a gate: nothing
# branches on it, so an unrecognised sentence costs a label rather than a verdict. It lives beside
# the sentences it keys on, so the classifier and the wordings cannot drift apart (kø-(p)).
# --------------------------------------------------------------------------------------------


def test_p21_rejection_stage_names_each_stage_from_its_own_sentence() -> None:
    from portfolio_optimiser.validator import rejection_stage

    assert (
        rejection_stage("unknown cost code 'X': not in project p's cost baseline (3 known codes)")
        == "stage0-baseline"
    )
    assert (
        rejection_stage(
            "quantity 5 for cost code 'X' is outside the 5.0% tolerance around the baseline "
            "quantity 9"
        )
        == "stage0-baseline"
    )
    assert (
        rejection_stage("ungrounded identifier 'X': it appears nowhere in the input (9 chars)")
        == "stage0b-grounding"
    )
    assert rejection_stage("claimed saving 9 exceeds P90 feasible 4") == "stage4-p90"
    assert (
        rejection_stage("claimed saving 9 exceeds the nominal feasible 4 at the items' stated")
        == "stage4b-nominal"
    )
    assert (
        rejection_stage("claimed 9 exceeds the energy_efficiency method cap 4 (stricter)")
        == "stage5-method-cap"
    )
    # The honest answer for a sentence this module did not write — a label, never a verdict.
    assert rejection_stage("something else entirely") == "other"


def test_p21_rejection_stage_is_keyed_on_the_sentences_the_validator_emits() -> None:
    """The control: the markers are not a private paraphrase but the text stage 0 really writes.

    Without it the classifier could key on wording nothing emits and every arm above would still
    be green — the vacuous-gate class, on a reporter.
    """
    from portfolio_optimiser.ir import AffectedItem, CostBaseline, CostBaselineLine, SavingsProposal
    from portfolio_optimiser.validator import Rejection, rejection_stage, validate_proposal

    baseline = CostBaseline(
        project_id="proj", items={"REAL-1": CostBaselineLine(quantity=10.0, unit_cost=100.0)}
    )
    proposal = SavingsProposal(
        project_id="proj",
        measure="m",
        affected_items=[AffectedItem(code="FAKE-1", quantity=10.0, unit_cost=100.0)],
        claimed_saving_nok=100.0,
    )
    outcome = validate_proposal(proposal, baseline=baseline)
    assert isinstance(outcome, Rejection)
    assert rejection_stage(outcome.reason) == "stage0-baseline"