okf-wiki/gates/tests/test_mutant_gate.py
Kjell Tore Guttormsen 0fd86a23a3
okf-wiki 1.1.1
First public release, published as one snapshot. What changed between
versions is in CHANGELOG.md.

Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
2026-11-14 21:03:20 +01:00

376 lines
17 KiB
Python

"""Tests for gates/mutant_gate.py, the fail-closed runner of the auditor's mutant gate.
Every fixture here is synthetic and neutral text. The real mutants live in gates-local/ (CLAUDE.md rule 3).
Run: python3 -m unittest discover -s gates/tests
"""
from __future__ import annotations
import io
import json
import re
import sys
import tempfile
import textwrap
import unittest
from contextlib import redirect_stdout
from pathlib import Path
from typing import Any
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
import mutant_gate # noqa: E402
from okfwiki.audit import GROUP_OF # noqa: E402
PY = sys.executable
# Reads {bundle}/EXPECT: if it names a class, report exactly that class; otherwise report clean.
PERFECT = """
import json, sys, pathlib
p = pathlib.Path(sys.argv[1]) / "EXPECT"
cls = p.read_text().strip() if p.exists() else ""
if cls:
print(json.dumps({"findings": [{"check": cls}]})); sys.exit(1)
print(json.dumps({"findings": []})); sys.exit(0)
"""
REJECT_ALL = """
import json, sys
print(json.dumps({"findings": [{"check": "reject-all"}]})); sys.exit(1)
"""
CRASH = """
raise RuntimeError("boom")
"""
WRONG_CLASS = """
import json, sys, pathlib
p = pathlib.Path(sys.argv[1]) / "EXPECT"
if p.exists():
print(json.dumps({"findings": [{"check": "broken-link"}]})); sys.exit(1)
print(json.dumps({"findings": []})); sys.exit(0)
"""
NOT_JSON = """
import sys
print("findings: lots"); sys.exit(1)
"""
EXIT0_WITH_FINDINGS = """
import json
print(json.dumps({"findings": [{"check": "x"}]}))
"""
EXIT1_NO_FINDINGS = """
import json, sys
print(json.dumps({"findings": []})); sys.exit(1)
"""
SLEEPER = """
import time
time.sleep(10)
"""
VANDAL = """
import json, sys, pathlib
for f in pathlib.Path(sys.argv[1]).iterdir():
f.write_text("vandalised")
print(json.dumps({"findings": []})); sys.exit(0)
"""
SOURCES_ECHO = """
import json, sys, pathlib
ok = (pathlib.Path(sys.argv[2]) / "src" / "a.txt").read_text() == "alpha\\n"
print(json.dumps({"findings": [] if ok else [{"check": "sources-missing"}]}))
sys.exit(0 if ok else 1)
"""
CASES: dict[str, tuple[str, str | None, dict[str, str]]] = {
"baseline": ("known-positive", None, {"page.md": "---\ntype: Note\n---\nPlain text.\n"}),
"m01-alpha": ("mutant", "class-alpha", {"page.md": "---\ntype: Note\n---\nA.\n", "EXPECT": "class-alpha\n"}),
"m02-beta": ("mutant", "class-beta", {"page.md": "---\ntype: Note\n---\nB.\n", "EXPECT": "class-beta\n"}),
"m03-beta": ("mutant", "class-beta", {"page.md": "---\ntype: Note\n---\nC.\n", "EXPECT": "class-beta\n"}),
}
class GateFixture:
"""A synthetic manifest, case tree and sources tree in a temporary directory."""
def __init__(self, root: Path) -> None:
self.root = root
self.cases = root / "cases"
self.sources = root / "sources"
(self.sources / "src").mkdir(parents=True)
(self.sources / "src" / "a.txt").write_text("alpha\n")
entries = []
for name, (kind, cls, files) in CASES.items():
d = self.cases / name
d.mkdir(parents=True)
for rel, content in files.items():
(d / rel).write_text(content)
entries.append({"name": name, "kind": kind, "class": cls, "mutation": "synthetic",
"files": mutant_gate.hash_tree(d)})
self.manifest = root / "manifest.json"
self.manifest.write_text(json.dumps({
"schema": 1,
"sources": [{"path": "src/a.txt", "sha256": mutant_gate.sha256_file(self.sources / "src" / "a.txt")}],
"cases": entries,
}))
def script(self, name: str, body: str) -> Path:
p = self.root / f"{name}.py"
p.write_text(textwrap.dedent(body))
return p
def run(self, command: list[str], timeout: float = 5.0) -> tuple[int, str]:
out = io.StringIO()
with redirect_stdout(out):
code = mutant_gate.main([
"--manifest", str(self.manifest), "--cases", str(self.cases),
"--sources", str(self.sources), "--timeout", str(timeout), "--", *command,
])
return code, out.getvalue()
class MutantGateTest(unittest.TestCase):
def setUp(self) -> None:
self._tmp = tempfile.TemporaryDirectory()
self.fx = GateFixture(Path(self._tmp.name))
def tearDown(self) -> None:
self._tmp.cleanup()
# --- the three outcomes the order names ---------------------------------------------------------
def test_missing_auditor_catches_nothing_and_says_so(self) -> None:
code, out = self.fx.run([str(self.fx.root / "no-such-auditor"), "{bundle}"])
self.assertEqual(code, 1)
self.assertIn("caught 0 of 3", out)
self.assertIn("errors 4 of 4", out)
self.assertIn("auditor not found", out)
self.assertIn("verdict: RED", out)
def test_reject_all_catches_everything_but_raises_a_false_alarm(self) -> None:
code, out = self.fx.run([PY, str(self.fx.script("reject", REJECT_ALL)), "{bundle}"])
self.assertEqual(code, 1)
self.assertIn("caught 3 of 3", out)
self.assertIn("false alarms 1 of 1", out)
self.assertIn("class matched 0 of 3", out)
self.assertIn("verdict: RED", out)
def test_perfect_auditor_turns_the_gate_green(self) -> None:
code, out = self.fx.run([PY, str(self.fx.script("perfect", PERFECT)), "{bundle}"])
self.assertEqual(code, 0, out)
self.assertIn("caught 3 of 3", out)
self.assertIn("class matched 3 of 3", out)
self.assertIn("false alarms 0 of 1", out)
self.assertIn("errors 0 of 4", out)
self.assertIn("verdict: GREEN", out)
# --- fail-closed: anything but a well-formed verdict is an error, never a catch --------------------
def test_crashing_auditor_is_an_error_not_a_catch(self) -> None:
code, out = self.fx.run([PY, str(self.fx.script("crash", CRASH)), "{bundle}"])
self.assertEqual(code, 1)
self.assertIn("caught 0 of 3", out)
self.assertIn("errors 4 of 4", out)
def test_output_that_is_not_json_is_an_error(self) -> None:
_, out = self.fx.run([PY, str(self.fx.script("notjson", NOT_JSON)), "{bundle}"])
self.assertIn("caught 0 of 3", out)
self.assertIn("not valid JSON", out)
def test_exit_status_must_agree_with_findings(self) -> None:
_, out0 = self.fx.run([PY, str(self.fx.script("e0", EXIT0_WITH_FINDINGS)), "{bundle}"])
self.assertIn("caught 0 of 3", out0)
self.assertIn("errors 4 of 4", out0)
_, out1 = self.fx.run([PY, str(self.fx.script("e1", EXIT1_NO_FINDINGS)), "{bundle}"])
self.assertIn("caught 0 of 3", out1)
self.assertIn("errors 4 of 4", out1)
def test_timeout_is_an_error(self) -> None:
_, out = self.fx.run([PY, str(self.fx.script("sleep", SLEEPER)), "{bundle}"], timeout=0.5)
self.assertIn("caught 0 of 3", out)
self.assertIn("timed out", out)
def test_alarm_for_the_wrong_class_is_caught_but_not_class_matched(self) -> None:
code, out = self.fx.run([PY, str(self.fx.script("wrong", WRONG_CLASS)), "{bundle}"])
self.assertEqual(code, 1)
self.assertIn("caught 3 of 3", out)
self.assertIn("class matched 0 of 3", out)
self.assertIn("verdict: RED", out)
# --- reporting -------------------------------------------------------------------------------------
def test_report_lists_what_slipped_through_and_n_per_class(self) -> None:
_, out = self.fx.run([str(self.fx.root / "no-such-auditor"), "{bundle}"])
self.assertIn("slipped through: m01-alpha, m02-beta, m03-beta", out)
self.assertIn("class-alpha: 1", out)
self.assertIn("class-beta: 2", out)
def test_sources_placeholder_is_substituted(self) -> None:
code, out = self.fx.run([PY, str(self.fx.script("src", SOURCES_ECHO)), "{bundle}", "{sources}"])
self.assertIn("false alarms 0 of 1", out)
self.assertNotIn("sources-missing", out)
self.assertEqual(code, 1) # this auditor never catches a mutant
# --- the fixtures themselves are verified before anything runs --------------------------------------
def test_auditor_cannot_alter_the_fixtures(self) -> None:
self.fx.run([PY, str(self.fx.script("vandal", VANDAL)), "{bundle}"])
code, _ = self.fx.run([PY, str(self.fx.script("perfect", PERFECT)), "{bundle}"])
self.assertEqual(code, 0)
def test_changed_case_file_stops_the_gate(self) -> None:
(self.fx.cases / "m01-alpha" / "page.md").write_text("changed\n")
code, out = self.fx.run([PY, str(self.fx.script("perfect", PERFECT)), "{bundle}"])
self.assertEqual(code, 2)
self.assertIn("sha256 mismatch", out)
def test_extra_or_missing_case_file_stops_the_gate(self) -> None:
(self.fx.cases / "m02-beta" / "extra.md").write_text("x\n")
code, out = self.fx.run([PY, str(self.fx.script("perfect", PERFECT)), "{bundle}"])
self.assertEqual(code, 2)
self.assertIn("unexpected file", out)
(self.fx.cases / "m02-beta" / "extra.md").unlink()
(self.fx.cases / "m03-beta" / "page.md").unlink()
code, out = self.fx.run([PY, str(self.fx.script("perfect", PERFECT)), "{bundle}"])
self.assertEqual(code, 2)
self.assertIn("missing file", out)
def test_changed_source_stops_the_gate(self) -> None:
(self.fx.sources / "src" / "a.txt").write_text("beta\n")
code, out = self.fx.run([PY, str(self.fx.script("perfect", PERFECT)), "{bundle}"])
self.assertEqual(code, 2)
self.assertIn("source", out)
def test_manifest_needs_exactly_one_known_positive(self) -> None:
data = json.loads(self.fx.manifest.read_text())
data["cases"] = [c for c in data["cases"] if c["kind"] != "known-positive"]
self.fx.manifest.write_text(json.dumps(data))
code, out = self.fx.run([PY, str(self.fx.script("perfect", PERFECT)), "{bundle}"])
self.assertEqual(code, 2)
self.assertIn("known-positive", out)
def test_missing_command_is_a_usage_error(self) -> None:
out = io.StringIO()
with redirect_stdout(out):
code = mutant_gate.main(["--manifest", str(self.fx.manifest), "--cases", str(self.fx.cases),
"--sources", str(self.fx.sources)])
self.assertEqual(code, 2)
class TrackedManifestTest(unittest.TestCase):
"""The tracked manifest must keep a mutant for every check the L1 row in PLAN.md promises (E1b)."""
LINK_CLASSES = ("broken-link", "orphan-page", "index-coverage", "link-misplaced")
# Wave 1 order 4: criterion 6 under the operator's rule of 06.10.2026 (choice 2), contract K2 in
# docs/bolge1/0-felles.local.md § 4. Every main-topic source has one main page (x-main-for), and every other
# concept page has a link from a main page. The auditor emits none of these four classes yet; Step 7 adds them.
K2_CLASSES = ("main-page-missing", "main-page-duplicate", "main-page-invalid", "no-main-page-link")
# One mutant per case of order 4, point 3; the key is the case, the value the class its mutant must carry.
K2_CASES = {
"main-page-missing: the main page loses its mark for one source": "main-page-missing",
"main-page-duplicate: a non-main page marks the same source": "main-page-duplicate",
"main-page-invalid: the value is not a list": "main-page-invalid",
"main-page-invalid: a value that is not among the page's own sources[].resource": "main-page-invalid",
"main-page-invalid: a release-note path, in K1 but not main_topic": "main-page-invalid",
"no-main-page-link: linked only from the other non-main page": "no-main-page-link",
"no-main-page-link: linked only from index.md": "no-main-page-link",
"no-main-page-link: linked only from log.md": "no-main-page-link",
"no-main-page-link: linked only from itself": "no-main-page-link",
"link-misplaced: the main page's only link to a non-main page sits in code": "link-misplaced",
"link-misplaced: the main page's only link to a non-main page sits in a heading": "link-misplaced",
"link-misplaced: the main page's only link to a non-main page sits in a footnote definition": "link-misplaced",
# K2 point 2, last sentence: the role follows the valid values. The gate counts a mutant matched when its own
# class is among the findings, so each case is one whose own class a wrong reading of the role cannot give.
"main-page-duplicate: an invalid value does not take the main role from a page's valid values": "main-page-duplicate",
"no-main-page-link: a page whose every value is invalid is not a main page": "no-main-page-link",
}
RELEASE = re.compile(r"^kilde/release-v\d+\.\d+\.\d+\.md$")
def path(self) -> Path:
return Path(__file__).resolve().parents[1] / "mutants.manifest.json"
def manifest(self) -> mutant_gate.Manifest:
return mutant_gate.load_manifest(self.path())
def k2(self) -> dict[str, Any]:
k2 = json.loads(self.path().read_text(encoding="utf-8"))["provenance"].get("k2")
self.assertIsInstance(k2, dict, "provenance.k2 is missing")
return k2 # type: ignore[no-any-return]
def test_every_link_class_has_a_mutant(self) -> None:
classes = {m.cls for m in self.manifest().mutants}
for cls in self.LINK_CLASSES + self.K2_CLASSES:
self.assertIn(cls, classes)
def test_every_k2_case_has_its_own_mutant(self) -> None:
manifest = self.manifest()
self.assertEqual(len(manifest.mutants), 37)
self.assertEqual(manifest.known_positive.name, "baseline")
by_name = {m.name: m for m in manifest.mutants}
coverage = self.k2()["coverage"]
named = []
for case, cls in self.K2_CASES.items():
self.assertIn(case, coverage)
name = coverage[case]
self.assertIn(name, by_name, case)
self.assertEqual(by_name[name].cls, cls, case)
named.append(name)
self.assertEqual(len(named), len(set(named)), "two cases share a mutant")
def test_the_inbound_rule_of_04_10_has_no_mutant_left(self) -> None:
# link-rule (an inbound link from any other concept page) goes in Step 7; m24, m28 and m29 now carry K2.
self.assertEqual([m.name for m in self.manifest().mutants if m.cls == "link-rule"], [])
def test_every_mutant_class_is_a_check_the_auditor_emits_or_k2_names(self) -> None:
expected = set(GROUP_OF) | set(self.K2_CLASSES)
for m in self.manifest().mutants:
self.assertIn(m.cls, expected, m.name)
def test_the_sources_hold_the_k1_fixture_and_a_release_note(self) -> None:
k2 = self.k2()
self.assertEqual(k2["sources_root"], "gates-local/mutants-kilder")
self.assertEqual(k2["k1_file"], "source-manifest.json")
self.assertEqual(len(k2["main_topic"]), 3)
self.assertTrue(all(p.startswith("kilde/docs-en-") for p in k2["main_topic"]), k2["main_topic"])
self.assertTrue(any(self.RELEASE.match(p) for p in k2["not_main_topic"]), k2["not_main_topic"])
sources = self.manifest().sources
self.assertEqual(set(sources), {k2["k1_file"], *k2["main_topic"], *k2["not_main_topic"]})
self.assertEqual(k2["k1_sha256"], sources[k2["k1_file"]], "the K1 file the gate checks is the one described")
def test_every_case_documents_its_expected_extras(self) -> None:
cases = json.loads(self.path().read_text(encoding="utf-8"))["cases"]
known = set(GROUP_OF) | set(self.K2_CLASSES)
for c in cases:
extra = c.get("expected_extra")
self.assertIsInstance(extra, list, c["name"])
self.assertLessEqual(set(extra), known - {"link-rule", c["class"]}, c["name"])
def test_the_known_positive_meets_k2(self) -> None:
k2 = self.k2()
kp = k2["known_positive"]
files = self.manifest().known_positive.files
concepts = {f for f in files if f.endswith(".md") and f.rsplit("/", 1)[-1] not in ("index.md", "log.md")}
main = kp["x-main-for"]
self.assertEqual(len(main), 1, "one main page for the three documentation sources")
(values,) = main.values()
self.assertEqual(sorted(values), sorted("/" + p for p in k2["main_topic"]))
non_main = kp["non_main_pages"]
self.assertGreaterEqual(len(non_main), 2)
self.assertIn("bypasspermissions-modus.md", non_main)
self.assertEqual(set(main) | set(non_main), concepts)
self.assertFalse(set(main) & set(non_main))
for page in non_main:
self.assertTrue(kp["links_from_main_pages"][page], page)
self.assertLessEqual(set(kp["links_from_main_pages"][page]), set(main), page)
for page in main:
self.assertEqual(kp["concept_links_into_main_pages"][page], [], "a main page needs no inbound link")
self.assertTrue(kp["cites_release_notes"])
self.assertLessEqual(set(kp["cites_release_notes"]), set(non_main))
if __name__ == "__main__":
unittest.main()