portfolio-optimiser-claude/tests/test_step8_promotion_loadbearing.py
Kjell Tore Guttormsen 30ba68a703 test(loadbearing): close the vacuous-negative class across the whole suite
Oekt 17 found the class on four named files. This sweep ENUMERATES it: 42 negative
substring assertions across 21 test files (STATE's "~34 across 23" was a premise --
measured, it is 42/21). Sixteen of them measured an absence without ever having
shown presence; all sixteen now carry a positive control asserting the searched-for
string PRESENT in the source artifact, in EXACTLY the form the negative looks for.

Files touched: test_costsim, test_loop, test_okf (3 sites), test_preflight,
test_run_entrance, test_s10_run_layer, test_sdk_version_guard, test_simulation
(2 sites), test_step1_expel, test_step5_refine, test_step7_async_loop,
test_step8_promotion, test_valuereport.

VALUE-PROOF (green-without / red-with, per the oekt-17 rule that a detach proof is
not a value proof). Seven source/fixture mutations, each making the negative vacuous:

  M1 verdict fixture loses the realization signal        VALUE-PROVEN
  M2 decoy fixture loses its text                        VALUE-PROVEN
  M3 renderer stops emitting typed section headings      VALUE-PROVEN
  M4 promotion stops writing the marker                  VALUE-PROVEN (pass 2)
  M5 fold stops rendering the realization surface        VALUE-PROVEN
  M6 report stops labelling the cost section             VALUE-PROVEN
  M7 preflight stops importing the SDK                   VALUE-PROVEN

M4 needed pass 2: a PRECEDING assertion caught the same mutation, hiding the new
control behind it -- the oekt-17 lesson reproduced. The remaining nine controls are
vacuity guards (non-emptiness / form-presence) whose mutation would have to break
the source artificially; they are stated as guards, not claimed as value-proven.

MEASURED FINDING (test_loop): the FIRST-RUN-MARKER negative cannot be given a
positive control at all. Within a run only the CHECKER's critique is fed back --
the proposer's own prior reasoning crosses no prompt boundary, not even within a
run. So that negative holds trivially. Left in place with the limitation stated in
the test rather than dressed up as a controlled seam; the CRITIQUE negative beside
it IS controlled and is the real seam.

Mutations were in-place on src/ and shared/ with original bytes restored and
sha-verified; git status clean before and after. Suite 688 -> 688 (assertions added
inside existing tests, no new test cases). ruff + mypy --strict green.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_017Vc5PmZGjwuJypdhzKnJa5
2026-07-31 21:39:28 +02:00

235 lines
10 KiB
Python

"""Step-8 promotion gate — LOAD-BEARING (method-spec §3 Step 8, §6, §11).
The seam this file keeps alive: only APPROVED knowledge enters the wiki
(fail-closed), the promoted file is NAVIGABLE (index-linked — an unlinked file
is unreachable), and the index label is NEUTRAL (the index body flows verbatim
into the rendered read-context, so a descriptive label would leak the learning
signal around the gated fold). RED when a non-approved verdict reaches the
wiki, when an approved one is not navigable, or when the label leaks.
"""
from __future__ import annotations
import shutil
from pathlib import Path
import pytest
from portfolio_optimiser_claude.experience import (
CandidateFeatures,
VerdictStore,
fold_experience,
seed_store_from_bundle,
)
from portfolio_optimiser_claude.inbox import VerdictDocument
from portfolio_optimiser_claude.ir import load_validator_input
from portfolio_optimiser_claude.okf import bundle_context, navigate_bundle, parse_concept_file
from portfolio_optimiser_claude.promotion import PromotionError, promote
SHARED_BUNDLE = Path(__file__).resolve().parents[1] / "shared" / "examples" / "bygg-energi-mikro"
MARKER = "realiseringsgrad=0.79"
RATIONALE = f"Godkjent med korreksjon: i drift realiseres ~79% ({MARKER})."
BASE_PROMPT = "Propose exactly one cost-saving measure for this project."
# Structured learning fields a promoted file MUST NOT reproduce (§6) — the raw
# verdict model carries the signal only as rationale prose.
_SEED_ONLY_FIELDS = (
"realization_rate",
"expected_actual_saving_nok",
"modelled_saving_nok",
"gap_source",
"context_key",
)
def _document(
decision: str = "approved",
rationale: str = RATIONALE,
codes: frozenset[str] = frozenset({"E01"}),
verdict_id: str | None = None,
) -> VerdictDocument:
doc = VerdictDocument.from_candidate(
CandidateFeatures(
affected_codes=codes, measure_type="led-retrofit", claimed_saving_nok=25000.0
),
decision=decision,
rationale=rationale,
description="LED retrofit",
)
if verdict_id is not None:
doc = doc.model_copy(update={"id": verdict_id})
return doc
def _promote(doc: VerdictDocument, bundle: Path) -> Path:
return promote(
doc,
bundle,
approved_by="persona:expert-reviewer",
experiment="S9-offline-sim",
timestamp="2026-07-03T12:00:00Z",
)
@pytest.fixture()
def bundle(tmp_path: Path) -> Path:
target = tmp_path / "bundle"
shutil.copytree(SHARED_BUNDLE, target)
return target
class TestFailClosed:
"""§6: a non-approved verdict is refused — writing and linking NOTHING."""
@pytest.mark.parametrize("decision", ["rejected", "maybe"])
def test_non_approved_decisions_are_refused_writing_nothing(
self, bundle: Path, decision: str
) -> None:
index_before = (bundle / "index.md").read_bytes()
files_before = sorted(p.name for p in bundle.iterdir())
with pytest.raises(PromotionError):
_promote(_document(decision=decision), bundle)
assert (bundle / "index.md").read_bytes() == index_before
assert sorted(p.name for p in bundle.iterdir()) == files_before
@pytest.mark.parametrize("decision", ["approved", "approved_with_adjustment"])
def test_the_accepted_set_is_exactly_the_two_approval_forms(
self, bundle: Path, decision: str
) -> None:
assert _promote(_document(decision=decision), bundle).is_file()
class TestPromotedFile:
"""§6: minimal promoted file — signal as rationale prose, provenance-stamped."""
def test_frontmatter_carries_the_contract_fields(self, bundle: Path) -> None:
doc = _document()
concept = parse_concept_file(_promote(doc, bundle))
assert concept.type == "verdict"
assert concept.frontmatter["decision"] == "approved"
assert concept.frontmatter["description"] == RATIONALE
assert concept.frontmatter["verdict_id"] == doc.id # verbatim
provenance = concept.frontmatter["provenance"]
for part in ("persona:expert-reviewer", "S9-offline-sim", "2026-07-03T12:00:00Z"):
assert part in provenance
def test_no_structured_learning_fields_are_reproduced(self, bundle: Path) -> None:
concept = parse_concept_file(_promote(_document(), bundle))
# Positive control: the frontmatter IS populated, so the absences below are
# withheld fields rather than an empty mapping that excludes everything.
assert concept.frontmatter
for field in _SEED_ONLY_FIELDS:
assert field not in concept.frontmatter
def test_timestamp_is_an_explicit_required_argument(self, bundle: Path) -> None:
# No wall-clock default — promotion is deterministic and reproducible.
with pytest.raises(TypeError):
promote( # type: ignore[call-arg]
_document(), bundle, approved_by="x", experiment="y"
)
def test_same_candidate_id_is_last_write_wins_per_file(self, bundle: Path) -> None:
first = _promote(_document(rationale=f"first ({MARKER})"), bundle)
second = _promote(_document(rationale=f"second ({MARKER})"), bundle)
assert first == second
assert parse_concept_file(second).frontmatter["description"] == f"second ({MARKER})"
class TestNavigability:
"""LOAD-BEARING (§11): an approved verdict must be navigable — and fold back."""
def test_promoted_file_is_reachable_via_index_navigation(self, bundle: Path) -> None:
path = _promote(_document(), bundle)
assert path.name in {c.path.name for c in navigate_bundle(bundle)}
def test_promoted_verdict_closes_the_loop_through_seeding(self, bundle: Path) -> None:
# Promotion → navigation → seeding → fold: the expert's rationale (the
# learning signal as prose) reaches the next run's hypothesis prompt.
_promote(_document(), bundle)
store = VerdictStore()
seed_store_from_bundle(store, bundle)
features = CandidateFeatures.from_proposal(load_validator_input(bundle))
prompt = fold_experience(store, features, BASE_PROMPT, k=3)
assert MARKER in prompt
def test_two_promoted_candidates_both_seed_verbatim_ids(self, bundle: Path) -> None:
# §4.2 read-VERBATIM at the seeding layer: distinct candidates get
# distinct store entries — re-minting from bundle features would
# collide them (first-write-wins) and silently drop one rationale.
_promote(_document(codes=frozenset({"E01"}), rationale=f"one ({MARKER})"), bundle)
_promote(_document(codes=frozenset({"E02"}), rationale=f"two ({MARKER})"), bundle)
store = VerdictStore()
seed_store_from_bundle(store, bundle)
features = CandidateFeatures.from_proposal(load_validator_input(bundle))
rationales = {r.rationale for r in store.retrieve(features, k=10)}
assert f"one ({MARKER})" in rationales
assert f"two ({MARKER})" in rationales
def test_linking_is_idempotent(self, bundle: Path) -> None:
path = _promote(_document(), bundle)
_promote(_document(), bundle)
index_text = (bundle / "index.md").read_text(encoding="utf-8")
assert index_text.count(f"]({path.name})") == 1
class TestNeutralLabel:
"""LOAD-BEARING (§11): the index label carries NO verdict signal."""
def test_index_never_carries_the_rationale_or_marker(self, bundle: Path) -> None:
path = _promote(_document(), bundle)
# Positive control: the signal IS live in the promoted file, in exactly the form
# the index assertions search for. Without it both `not in` checks would also
# pass if the marker stopped being written at all — green for the wrong reason.
promoted_text = path.read_text(encoding="utf-8")
assert MARKER in promoted_text
assert RATIONALE in promoted_text
index_text = (bundle / "index.md").read_text(encoding="utf-8")
assert MARKER not in index_text
assert RATIONALE not in index_text
def test_label_is_fixed_across_verdicts(self, bundle: Path) -> None:
a = _promote(_document(codes=frozenset({"E01"})), bundle)
b = _promote(_document(codes=frozenset({"E02"})), bundle)
index_lines = (bundle / "index.md").read_text(encoding="utf-8").splitlines()
line_a = next(line for line in index_lines if a.name in line)
line_b = next(line for line in index_lines if b.name in line)
assert line_a.replace(a.name, "") == line_b.replace(b.name, "")
def test_rendered_read_context_never_carries_the_marker(self, bundle: Path) -> None:
# Belt and braces: the index body flows verbatim into the rendered
# read-context — after promotion it must still exclude the signal.
path = _promote(_document(), bundle)
# Positive control (same reason as above): the marker is live on disk, so the
# exclusion below measures the RENDERING boundary and not a missing marker.
assert MARKER in path.read_text(encoding="utf-8")
assert MARKER not in bundle_context(bundle)
class TestPathSafety:
"""§6: path-safe, fail-closed against escaping names."""
def test_escaping_id_is_sanitised_into_the_bundle(self, bundle: Path, tmp_path: Path) -> None:
path = _promote(_document(verdict_id="../../evil"), bundle)
assert path.parent == bundle
assert not (tmp_path / "evil").exists()
assert "/" not in path.name and "\\" not in path.name
def test_sanitised_ids_never_collide_with_a_distinct_id_owning_the_name(
self, bundle: Path
) -> None:
# C2.5 restarbeid-funn 2: 'e/vil' used to sanitise to 'evil' — the
# SAME promoted filename as the distinct id 'evil', so the second
# promotion silently overwrote the first curated verdict.
first = _promote(_document(verdict_id="e/vil", rationale=f"one ({MARKER})"), bundle)
second = _promote(_document(verdict_id="evil", rationale=f"two ({MARKER})"), bundle)
assert first != second
assert parse_concept_file(first).frontmatter["verdict_id"] == "e/vil"
assert parse_concept_file(second).frontmatter["verdict_id"] == "evil"
def test_degenerate_token_falls_back_to_a_content_hash(self, bundle: Path) -> None:
path = _promote(_document(verdict_id="///"), bundle)
assert path.parent == bundle
token = path.name.removeprefix("promoted-verdict-").removesuffix(".md")
assert token, "degenerate token must fall back to a non-empty content hash"
assert set(token) <= set("0123456789abcdef")