"""The green-before / red-after procedure, made a tool instead of a memory. WHY THIS EXISTS. This repo proves a test is load-bearing by mutating what it rests on and watching it go red. That procedure has been hand-built ten times (2x oekt 23, 4x oekt 31, 2x oekt 32, 1x oekt 33, 1x oekt 37) and rewritten from memory in a scratchpad every time. Four of its controls fail SILENTLY, which is exactly the class STATE's operating model calls oensket 4 of the verification law -- a negative result from a broken query, consumed as a positive fact: 1. the anchor is not unique, so the mutation lands somewhere else too; 2. the node id was mistyped, so pytest never ran the test and the non-zero exit MIMICS red (MEASURED 2026-08-25: rc=4, not rc=1 -- distinguishable, but only if something looks at the number); 3. the restore is not verified, so the tree keeps the mutation; 4. the test goes red on a NameError or a broken import rather than on the assertion the proof is about (oekt 33: "not on a NameError, not on the control"). Prose cannot enforce any of those. That is the whole argument for a tool. WHAT THIS IS NOT. The other mutation class -- mutate a COPY held in memory and call the guard directly -- already has a home in this suite and needs no tool (``test_guard_red_when_*``, five files). This tool is for the class that mutates the REAL tree and runs pytest as a subprocess, which is why it must NOT live inside the suite as a fixture: the suite runs every session, and a fixture that writes to disk makes every interrupted run a mutated tree. So every test below runs against a SANDBOX built in ``tmp_path``. Nothing here mutates this repo. That is a property of the design, not a precaution. """ from __future__ import annotations import hashlib import importlib.util import sys from pathlib import Path from types import ModuleType import pytest HARNESS_PATH = Path(__file__).resolve().parents[1] / "scripts" / "mutation_harness.py" def _load_harness() -> ModuleType: """Import the tool from its published path, not from a package alias. The path is part of the contract: the order asked for a standalone script, and a test that imported it through some other name would stay green if the script moved out from under the operator running it by hand. """ assert HARNESS_PATH.is_file(), f"the harness is not at its documented path: {HARNESS_PATH}" spec = importlib.util.spec_from_file_location("mutation_harness", HARNESS_PATH) assert spec is not None and spec.loader is not None module = importlib.util.module_from_spec(spec) sys.modules["mutation_harness"] = module spec.loader.exec_module(module) return module mh = _load_harness() # -------------------------------------------------------------------------- # The sandbox: a seam, a test that rests on it, and a control that does not. # -------------------------------------------------------------------------- SEAM_SOURCE = '''\ """A stand-in seam with one uniquely anchored behaviour.""" PREFIX = "mcp__" def qualify(server: str, tool: str) -> str: return f"{PREFIX}{server}__{tool}" def population() -> list[str]: return ["record_call", "flush_calls"] ''' SEAM_TESTS = """\ from seam import population, qualify def test_the_population_is_readable_and_is_two() -> None: # CONTROL: green before AND after. If a mutation reds this, it landed wider # than the seam and the target's redness proves nothing. assert len(population()) == 2 def test_the_name_carries_the_server(){ANNOT} -> None: # TARGET: green before, red after. assert qualify("recorder", "record_call") == "mcp__recorder__record_call" """.replace("{ANNOT}", "") TARGET_ID = "test_seam.py::test_the_name_carries_the_server" CONTROL_ID = "test_seam.py::test_the_population_is_readable_and_is_two" @pytest.fixture() def sandbox(tmp_path: Path) -> Path: (tmp_path / "seam.py").write_text(SEAM_SOURCE, encoding="utf-8") (tmp_path / "test_seam.py").write_text(SEAM_TESTS, encoding="utf-8") return tmp_path def _sha256(path: Path) -> str: return hashlib.sha256(path.read_bytes()).hexdigest() # -------------------------------------------------------------------------- # 1. Classification: the layer where oensket 4 actually bites. # -------------------------------------------------------------------------- class TestExitCodesAreClassifiedNotJustTestedForZero: def test_the_four_measured_exit_codes_map_to_three_distinct_verdicts(self) -> None: # MEASURED 2026-08-25 against pytest 8 in this repo's .venv. Kept as a # table so the mapping reads as the measurement, not as a belief. assert mh.classify(0, "1 passed").verdict == "green" assert mh.classify(1, "E AssertionError: nope").verdict == "red" assert mh.classify(4, "no tests ran").verdict == "not-collected" assert mh.classify(5, "no tests collected").verdict == "not-collected" def test_a_mistyped_node_id_is_never_read_as_red(self) -> None: # THE control this tool exists for. rc=4 is non-zero, so any harness # that asks "rc != 0?" calls a typo a proof. This one cannot. outcome = mh.classify(4, "no tests ran in 0.59s") assert outcome.verdict == "not-collected" assert outcome.verdict != "red" def test_the_error_type_is_read_off_the_output_not_assumed(self) -> None: assert mh.classify(1, "E AssertionError: x").error_types == ("AssertionError",) assert mh.classify(1, "E NameError: name 'q' is not defined").error_types == ( "NameError", ) def test_dotted_and_non_error_suffixed_failures_are_read_too(self) -> None: # MEASURED, and it cost a rewrite. The first version of this tool gated # on "is it an AssertionError?" and REJECTED three of the four real # proofs run against itself, because a legitimately red test can die as # `Failed: DID NOT RAISE`, or as a dotted custom exception. A gate that # refuses real evidence gets switched off, which is worse than none -- # so the type is REPORTED, never used to model what pytest may emit # (oekt 28: the fixture is the tool's emission, not a model of it). assert mh.classify(1, "E Failed: DID NOT RAISE").error_types == ("Failed",) assert mh.classify(1, "E mutation_harness.NotAValueProof: stayed green").error_types == ( "NotAValueProof", ) assert mh.classify(1, "E assert 1 == 2").error_types == () # -------------------------------------------------------------------------- # 2. The anchor, checked before anything is written. # -------------------------------------------------------------------------- class TestTheAnchorIsProvenUniqueBeforeTheFileIsTouched: def test_an_ambiguous_anchor_is_refused_and_the_file_is_untouched(self, sandbox: Path) -> None: target = sandbox / "seam.py" before = _sha256(target) with pytest.raises(mh.AnchorNotUnique) as excinfo: mh.prove( mh.Mutation(target=target, anchor="server", replacement="srv"), expect_red=[TARGET_ID], cwd=sandbox, ) assert "3" in str(excinfo.value) or "occurrence" in str(excinfo.value).lower() assert _sha256(target) == before, "refusal must happen BEFORE the write" def test_an_absent_anchor_is_refused(self, sandbox: Path) -> None: target = sandbox / "seam.py" before = _sha256(target) with pytest.raises(mh.AnchorNotUnique): mh.prove( mh.Mutation(target=target, anchor="not in this file", replacement="x"), expect_red=[TARGET_ID], cwd=sandbox, ) assert _sha256(target) == before def test_a_proof_with_no_target_is_vacuous_and_refused(self, sandbox: Path) -> None: # A harness run that reds nothing proves nothing, and would otherwise # report success. Refuse it rather than emit an empty proof. with pytest.raises(ValueError): mh.prove( mh.Mutation(target=sandbox / "seam.py", anchor='PREFIX = "mcp__"', replacement=""), expect_red=[], cwd=sandbox, ) # -------------------------------------------------------------------------- # 3. The value proof of the tool itself: it catches a known-red mutation and # refuses to call a known-harmless one a proof. # -------------------------------------------------------------------------- DETACHING = ('PREFIX = "mcp__"', 'PREFIX = ""') HARMLESS = ("A stand-in seam", "A stand-in seam (comment touched)") class TestTheToolIsValueProvedInBothDirections: def test_a_detaching_mutation_is_caught_and_reported_as_a_value_proof( self, sandbox: Path ) -> None: target = sandbox / "seam.py" report = mh.prove( mh.Mutation(target=target, anchor=DETACHING[0], replacement=DETACHING[1]), expect_red=[TARGET_ID], expect_green=[CONTROL_ID], cwd=sandbox, ) assert report.value_proved assert all(o.verdict == "green" for o in report.before) assert report.after_by_id[TARGET_ID].verdict == "red" assert report.after_by_id[TARGET_ID].error_types == ("AssertionError",) assert report.after_by_id[CONTROL_ID].verdict == "green" def test_a_harmless_mutation_is_not_flagged_as_a_proof(self, sandbox: Path) -> None: # The other half of the value proof, and the one that is easy to skip: # a tool that reports success on everything reports nothing. target = sandbox / "seam.py" with pytest.raises(mh.NotAValueProof) as excinfo: mh.prove( mh.Mutation(target=target, anchor=HARMLESS[0], replacement=HARMLESS[1]), expect_red=[TARGET_ID], expect_green=[CONTROL_ID], cwd=sandbox, ) assert TARGET_ID in str(excinfo.value) assert "green" in str(excinfo.value).lower() def test_the_target_must_be_green_before_or_it_is_not_a_measurement( self, sandbox: Path ) -> None: # POSITIVE CONTROL OF THE MEASURING APPARATUS, and it runs FIRST. If the # target is already red, "red after" is not caused by the mutation. (sandbox / "test_seam.py").write_text( SEAM_TESTS.replace('== "mcp__recorder__record_call"', '== "already wrong"'), encoding="utf-8", ) with pytest.raises(mh.NotGreenBefore): mh.prove( mh.Mutation( target=sandbox / "seam.py", anchor=DETACHING[0], replacement=DETACHING[1] ), expect_red=[TARGET_ID], cwd=sandbox, ) def test_a_mistyped_node_id_raises_instead_of_proving_anything(self, sandbox: Path) -> None: with pytest.raises(mh.NodeNotCollected) as excinfo: mh.prove( mh.Mutation( target=sandbox / "seam.py", anchor=DETACHING[0], replacement=DETACHING[1] ), expect_red=["test_seam.py::test_this_id_does_not_exist"], cwd=sandbox, ) assert "test_this_id_does_not_exist" in str(excinfo.value) def test_a_mutation_that_breaks_the_import_is_not_a_value_proof(self, sandbox: Path) -> None: # rc=4 AFTER the mutation means collection broke, which any garbage # edit achieves. It is the loudest possible way to prove nothing. with pytest.raises(mh.NodeNotCollected): mh.prove( mh.Mutation( target=sandbox / "seam.py", anchor='PREFIX = "mcp__"', replacement="import definitely_not_a_module", ), expect_red=[TARGET_ID], cwd=sandbox, ) def test_a_pinned_red_location_is_verified_against_the_real_output(self, sandbox: Path) -> None: # oekt 33 said the redness must land on the line the proof is about # ("not on a NameError, not on the control"). That is not derivable # from the exception type -- it is derivable from WHERE it died. So the # caller pins it and the tool checks the pin against pytest's actual # output, rather than guessing from a type it does not control. report = mh.prove( mh.Mutation(target=sandbox / "seam.py", anchor=DETACHING[0], replacement=DETACHING[1]), expect_red=[TARGET_ID], red_at={TARGET_ID: "mcp__recorder__record_call"}, cwd=sandbox, ) assert report.value_proved def test_a_red_that_misses_the_pinned_location_is_refused(self, sandbox: Path) -> None: with pytest.raises(mh.NotAValueProof) as excinfo: mh.prove( mh.Mutation( target=sandbox / "seam.py", anchor=DETACHING[0], replacement=DETACHING[1] ), expect_red=[TARGET_ID], red_at={TARGET_ID: "a line this failure never prints"}, cwd=sandbox, ) assert "a line this failure never prints" in str(excinfo.value) def test_a_red_on_a_broken_module_is_still_reported_with_its_type(self, sandbox: Path) -> None: # The honest limit, stated rather than papered over: a NameError inside # a test that RAN is red, and the tool does not silently bless or # reject it -- it names the type so the operator can see what died. # (A mutation that breaks COLLECTION is a different case and is refused # outright above: rc=4 is not red.) report = mh.prove( mh.Mutation( target=sandbox / "seam.py", anchor=' return f"{PREFIX}{server}__{tool}"', replacement=" return undefined_name", ), expect_red=[TARGET_ID], cwd=sandbox, ) assert report.after_by_id[TARGET_ID].error_types == ("NameError",) assert "NameError" in mh._format(report) def test_a_control_that_goes_red_fails_the_proof(self, sandbox: Path) -> None: # The mutation lands wider than the seam: the population control reds # too, so the target's redness is not attributable to the seam. with pytest.raises(mh.ControlWentRed) as excinfo: mh.prove( mh.Mutation( target=sandbox / "seam.py", anchor='return ["record_call", "flush_calls"]', replacement='return ["record_call"]', ), expect_red=[CONTROL_ID], expect_green=[CONTROL_ID], cwd=sandbox, ) assert CONTROL_ID in str(excinfo.value) # -------------------------------------------------------------------------- # 4. The restore, verified in both directions, git or no git. # -------------------------------------------------------------------------- class TestTheRestoreIsVerifiedNotAssumed: def test_the_target_is_restored_byte_identical_after_a_successful_proof( self, sandbox: Path ) -> None: target = sandbox / "seam.py" before = _sha256(target) report = mh.prove( mh.Mutation(target=target, anchor=DETACHING[0], replacement=DETACHING[1]), expect_red=[TARGET_ID], cwd=sandbox, ) assert _sha256(target) == before assert report.sha256_before == before == report.sha256_after def test_the_target_is_restored_even_when_the_proof_raises(self, sandbox: Path) -> None: # The restore lives in `finally` or it does not exist. Every failure # path above would otherwise leave the tree mutated. target = sandbox / "seam.py" before = _sha256(target) with pytest.raises(mh.NotAValueProof): mh.prove( mh.Mutation(target=target, anchor=HARMLESS[0], replacement=HARMLESS[1]), expect_red=[TARGET_ID], cwd=sandbox, ) assert _sha256(target) == before, "a failed proof must not leave a mutated tree" def test_the_restore_does_not_go_through_git(self, sandbox: Path) -> None: # The .venv case (oekt 37): the target is not tracked, so `git status` # would never catch the mutation and `git checkout --` cannot undo it. # The harness holds the original bytes itself, which is why the same # mechanism covers tracked and untracked targets alike. assert not (sandbox / ".git").exists() target = sandbox / "seam.py" before = _sha256(target) mh.prove( mh.Mutation(target=target, anchor=DETACHING[0], replacement=DETACHING[1]), expect_red=[TARGET_ID], cwd=sandbox, ) assert _sha256(target) == before def test_a_corrupted_restore_is_reported_rather_than_passed_over(self, sandbox: Path) -> None: # Force the one failure the sha256 check exists to catch. Without it, # a partial write leaves a silently wrong tree and the run still says # "restored". target = sandbox / "seam.py" with pytest.raises(mh.RestoreFailed): mh._verify_restore(target, b"not what was written", "0" * 64) # -------------------------------------------------------------------------- # 5. The command line, wired to the same code path the tests prove. # -------------------------------------------------------------------------- class TestTheCommandLineReachesTheProvenCodePath: def test_a_successful_proof_exits_zero_and_says_what_it_measured( self, sandbox: Path, capsys: pytest.CaptureFixture[str] ) -> None: code = mh.main( [ "--target", str(sandbox / "seam.py"), "--anchor", DETACHING[0], "--replacement", DETACHING[1], "--red", TARGET_ID, "--green", CONTROL_ID, "--cwd", str(sandbox), ] ) out = capsys.readouterr().out assert code == 0 assert "VALUE-PROVED" in out assert TARGET_ID in out assert _sha256(sandbox / "seam.py") in out, "the verified restore is part of the report" def test_a_failed_proof_exits_non_zero_and_names_the_reason( self, sandbox: Path, capsys: pytest.CaptureFixture[str] ) -> None: code = mh.main( [ "--target", str(sandbox / "seam.py"), "--anchor", HARMLESS[0], "--replacement", HARMLESS[1], "--red", TARGET_ID, "--cwd", str(sandbox), ] ) out = capsys.readouterr().out assert code != 0 assert "NOT a value proof" in out or "NotAValueProof" in out