"""Tests for gates/mutant_gate.py, the fail-closed runner of the auditor's mutant gate. Every fixture here is synthetic and neutral text. The real mutants live in gates-local/ (CLAUDE.md rule 3). Run: python3 -m unittest discover -s gates/tests """ from __future__ import annotations import io import json import re import sys import tempfile import textwrap import unittest from contextlib import redirect_stdout from pathlib import Path from typing import Any sys.path.insert(0, str(Path(__file__).resolve().parents[1])) sys.path.insert(0, str(Path(__file__).resolve().parents[2])) import mutant_gate # noqa: E402 from okfwiki.audit import GROUP_OF # noqa: E402 PY = sys.executable # Reads {bundle}/EXPECT: if it names a class, report exactly that class; otherwise report clean. PERFECT = """ import json, sys, pathlib p = pathlib.Path(sys.argv[1]) / "EXPECT" cls = p.read_text().strip() if p.exists() else "" if cls: print(json.dumps({"findings": [{"check": cls}]})); sys.exit(1) print(json.dumps({"findings": []})); sys.exit(0) """ REJECT_ALL = """ import json, sys print(json.dumps({"findings": [{"check": "reject-all"}]})); sys.exit(1) """ CRASH = """ raise RuntimeError("boom") """ WRONG_CLASS = """ import json, sys, pathlib p = pathlib.Path(sys.argv[1]) / "EXPECT" if p.exists(): print(json.dumps({"findings": [{"check": "broken-link"}]})); sys.exit(1) print(json.dumps({"findings": []})); sys.exit(0) """ NOT_JSON = """ import sys print("findings: lots"); sys.exit(1) """ EXIT0_WITH_FINDINGS = """ import json print(json.dumps({"findings": [{"check": "x"}]})) """ EXIT1_NO_FINDINGS = """ import json, sys print(json.dumps({"findings": []})); sys.exit(1) """ SLEEPER = """ import time time.sleep(10) """ VANDAL = """ import json, sys, pathlib for f in pathlib.Path(sys.argv[1]).iterdir(): f.write_text("vandalised") print(json.dumps({"findings": []})); sys.exit(0) """ SOURCES_ECHO = """ import json, sys, pathlib ok = (pathlib.Path(sys.argv[2]) / "src" / "a.txt").read_text() == "alpha\\n" print(json.dumps({"findings": [] if ok else [{"check": "sources-missing"}]})) sys.exit(0 if ok else 1) """ CASES: dict[str, tuple[str, str | None, dict[str, str]]] = { "baseline": ("known-positive", None, {"page.md": "---\ntype: Note\n---\nPlain text.\n"}), "m01-alpha": ("mutant", "class-alpha", {"page.md": "---\ntype: Note\n---\nA.\n", "EXPECT": "class-alpha\n"}), "m02-beta": ("mutant", "class-beta", {"page.md": "---\ntype: Note\n---\nB.\n", "EXPECT": "class-beta\n"}), "m03-beta": ("mutant", "class-beta", {"page.md": "---\ntype: Note\n---\nC.\n", "EXPECT": "class-beta\n"}), } class GateFixture: """A synthetic manifest, case tree and sources tree in a temporary directory.""" def __init__(self, root: Path) -> None: self.root = root self.cases = root / "cases" self.sources = root / "sources" (self.sources / "src").mkdir(parents=True) (self.sources / "src" / "a.txt").write_text("alpha\n") entries = [] for name, (kind, cls, files) in CASES.items(): d = self.cases / name d.mkdir(parents=True) for rel, content in files.items(): (d / rel).write_text(content) entries.append({"name": name, "kind": kind, "class": cls, "mutation": "synthetic", "files": mutant_gate.hash_tree(d)}) self.manifest = root / "manifest.json" self.manifest.write_text(json.dumps({ "schema": 1, "sources": [{"path": "src/a.txt", "sha256": mutant_gate.sha256_file(self.sources / "src" / "a.txt")}], "cases": entries, })) def script(self, name: str, body: str) -> Path: p = self.root / f"{name}.py" p.write_text(textwrap.dedent(body)) return p def run(self, command: list[str], timeout: float = 5.0) -> tuple[int, str]: out = io.StringIO() with redirect_stdout(out): code = mutant_gate.main([ "--manifest", str(self.manifest), "--cases", str(self.cases), "--sources", str(self.sources), "--timeout", str(timeout), "--", *command, ]) return code, out.getvalue() class MutantGateTest(unittest.TestCase): def setUp(self) -> None: self._tmp = tempfile.TemporaryDirectory() self.fx = GateFixture(Path(self._tmp.name)) def tearDown(self) -> None: self._tmp.cleanup() # --- the three outcomes the order names --------------------------------------------------------- def test_missing_auditor_catches_nothing_and_says_so(self) -> None: code, out = self.fx.run([str(self.fx.root / "no-such-auditor"), "{bundle}"]) self.assertEqual(code, 1) self.assertIn("caught 0 of 3", out) self.assertIn("errors 4 of 4", out) self.assertIn("auditor not found", out) self.assertIn("verdict: RED", out) def test_reject_all_catches_everything_but_raises_a_false_alarm(self) -> None: code, out = self.fx.run([PY, str(self.fx.script("reject", REJECT_ALL)), "{bundle}"]) self.assertEqual(code, 1) self.assertIn("caught 3 of 3", out) self.assertIn("false alarms 1 of 1", out) self.assertIn("class matched 0 of 3", out) self.assertIn("verdict: RED", out) def test_perfect_auditor_turns_the_gate_green(self) -> None: code, out = self.fx.run([PY, str(self.fx.script("perfect", PERFECT)), "{bundle}"]) self.assertEqual(code, 0, out) self.assertIn("caught 3 of 3", out) self.assertIn("class matched 3 of 3", out) self.assertIn("false alarms 0 of 1", out) self.assertIn("errors 0 of 4", out) self.assertIn("verdict: GREEN", out) # --- fail-closed: anything but a well-formed verdict is an error, never a catch -------------------- def test_crashing_auditor_is_an_error_not_a_catch(self) -> None: code, out = self.fx.run([PY, str(self.fx.script("crash", CRASH)), "{bundle}"]) self.assertEqual(code, 1) self.assertIn("caught 0 of 3", out) self.assertIn("errors 4 of 4", out) def test_output_that_is_not_json_is_an_error(self) -> None: _, out = self.fx.run([PY, str(self.fx.script("notjson", NOT_JSON)), "{bundle}"]) self.assertIn("caught 0 of 3", out) self.assertIn("not valid JSON", out) def test_exit_status_must_agree_with_findings(self) -> None: _, out0 = self.fx.run([PY, str(self.fx.script("e0", EXIT0_WITH_FINDINGS)), "{bundle}"]) self.assertIn("caught 0 of 3", out0) self.assertIn("errors 4 of 4", out0) _, out1 = self.fx.run([PY, str(self.fx.script("e1", EXIT1_NO_FINDINGS)), "{bundle}"]) self.assertIn("caught 0 of 3", out1) self.assertIn("errors 4 of 4", out1) def test_timeout_is_an_error(self) -> None: _, out = self.fx.run([PY, str(self.fx.script("sleep", SLEEPER)), "{bundle}"], timeout=0.5) self.assertIn("caught 0 of 3", out) self.assertIn("timed out", out) def test_alarm_for_the_wrong_class_is_caught_but_not_class_matched(self) -> None: code, out = self.fx.run([PY, str(self.fx.script("wrong", WRONG_CLASS)), "{bundle}"]) self.assertEqual(code, 1) self.assertIn("caught 3 of 3", out) self.assertIn("class matched 0 of 3", out) self.assertIn("verdict: RED", out) # --- reporting ------------------------------------------------------------------------------------- def test_report_lists_what_slipped_through_and_n_per_class(self) -> None: _, out = self.fx.run([str(self.fx.root / "no-such-auditor"), "{bundle}"]) self.assertIn("slipped through: m01-alpha, m02-beta, m03-beta", out) self.assertIn("class-alpha: 1", out) self.assertIn("class-beta: 2", out) def test_sources_placeholder_is_substituted(self) -> None: code, out = self.fx.run([PY, str(self.fx.script("src", SOURCES_ECHO)), "{bundle}", "{sources}"]) self.assertIn("false alarms 0 of 1", out) self.assertNotIn("sources-missing", out) self.assertEqual(code, 1) # this auditor never catches a mutant # --- the fixtures themselves are verified before anything runs -------------------------------------- def test_auditor_cannot_alter_the_fixtures(self) -> None: self.fx.run([PY, str(self.fx.script("vandal", VANDAL)), "{bundle}"]) code, _ = self.fx.run([PY, str(self.fx.script("perfect", PERFECT)), "{bundle}"]) self.assertEqual(code, 0) def test_changed_case_file_stops_the_gate(self) -> None: (self.fx.cases / "m01-alpha" / "page.md").write_text("changed\n") code, out = self.fx.run([PY, str(self.fx.script("perfect", PERFECT)), "{bundle}"]) self.assertEqual(code, 2) self.assertIn("sha256 mismatch", out) def test_extra_or_missing_case_file_stops_the_gate(self) -> None: (self.fx.cases / "m02-beta" / "extra.md").write_text("x\n") code, out = self.fx.run([PY, str(self.fx.script("perfect", PERFECT)), "{bundle}"]) self.assertEqual(code, 2) self.assertIn("unexpected file", out) (self.fx.cases / "m02-beta" / "extra.md").unlink() (self.fx.cases / "m03-beta" / "page.md").unlink() code, out = self.fx.run([PY, str(self.fx.script("perfect", PERFECT)), "{bundle}"]) self.assertEqual(code, 2) self.assertIn("missing file", out) def test_changed_source_stops_the_gate(self) -> None: (self.fx.sources / "src" / "a.txt").write_text("beta\n") code, out = self.fx.run([PY, str(self.fx.script("perfect", PERFECT)), "{bundle}"]) self.assertEqual(code, 2) self.assertIn("source", out) def test_manifest_needs_exactly_one_known_positive(self) -> None: data = json.loads(self.fx.manifest.read_text()) data["cases"] = [c for c in data["cases"] if c["kind"] != "known-positive"] self.fx.manifest.write_text(json.dumps(data)) code, out = self.fx.run([PY, str(self.fx.script("perfect", PERFECT)), "{bundle}"]) self.assertEqual(code, 2) self.assertIn("known-positive", out) def test_missing_command_is_a_usage_error(self) -> None: out = io.StringIO() with redirect_stdout(out): code = mutant_gate.main(["--manifest", str(self.fx.manifest), "--cases", str(self.fx.cases), "--sources", str(self.fx.sources)]) self.assertEqual(code, 2) class TrackedManifestTest(unittest.TestCase): """The tracked manifest must keep a mutant for every check the L1 row in PLAN.md promises (E1b).""" LINK_CLASSES = ("broken-link", "orphan-page", "index-coverage", "link-misplaced") # Wave 1 order 4: criterion 6 under the operator's rule of 06.10.2026 (choice 2), contract K2 in # docs/bolge1/0-felles.local.md ยง 4. Every main-topic source has one main page (x-main-for), and every other # concept page has a link from a main page. The auditor emits none of these four classes yet; Step 7 adds them. K2_CLASSES = ("main-page-missing", "main-page-duplicate", "main-page-invalid", "no-main-page-link") # One mutant per case of order 4, point 3; the key is the case, the value the class its mutant must carry. K2_CASES = { "main-page-missing: the main page loses its mark for one source": "main-page-missing", "main-page-duplicate: a non-main page marks the same source": "main-page-duplicate", "main-page-invalid: the value is not a list": "main-page-invalid", "main-page-invalid: a value that is not among the page's own sources[].resource": "main-page-invalid", "main-page-invalid: a release-note path, in K1 but not main_topic": "main-page-invalid", "no-main-page-link: linked only from the other non-main page": "no-main-page-link", "no-main-page-link: linked only from index.md": "no-main-page-link", "no-main-page-link: linked only from log.md": "no-main-page-link", "no-main-page-link: linked only from itself": "no-main-page-link", "link-misplaced: the main page's only link to a non-main page sits in code": "link-misplaced", "link-misplaced: the main page's only link to a non-main page sits in a heading": "link-misplaced", "link-misplaced: the main page's only link to a non-main page sits in a footnote definition": "link-misplaced", # K2 point 2, last sentence: the role follows the valid values. The gate counts a mutant matched when its own # class is among the findings, so each case is one whose own class a wrong reading of the role cannot give. "main-page-duplicate: an invalid value does not take the main role from a page's valid values": "main-page-duplicate", "no-main-page-link: a page whose every value is invalid is not a main page": "no-main-page-link", } RELEASE = re.compile(r"^kilde/release-v\d+\.\d+\.\d+\.md$") def path(self) -> Path: return Path(__file__).resolve().parents[1] / "mutants.manifest.json" def manifest(self) -> mutant_gate.Manifest: return mutant_gate.load_manifest(self.path()) def k2(self) -> dict[str, Any]: k2 = json.loads(self.path().read_text(encoding="utf-8"))["provenance"].get("k2") self.assertIsInstance(k2, dict, "provenance.k2 is missing") return k2 # type: ignore[no-any-return] def test_every_link_class_has_a_mutant(self) -> None: classes = {m.cls for m in self.manifest().mutants} for cls in self.LINK_CLASSES + self.K2_CLASSES: self.assertIn(cls, classes) def test_every_k2_case_has_its_own_mutant(self) -> None: manifest = self.manifest() self.assertEqual(len(manifest.mutants), 37) self.assertEqual(manifest.known_positive.name, "baseline") by_name = {m.name: m for m in manifest.mutants} coverage = self.k2()["coverage"] named = [] for case, cls in self.K2_CASES.items(): self.assertIn(case, coverage) name = coverage[case] self.assertIn(name, by_name, case) self.assertEqual(by_name[name].cls, cls, case) named.append(name) self.assertEqual(len(named), len(set(named)), "two cases share a mutant") def test_the_inbound_rule_of_04_10_has_no_mutant_left(self) -> None: # link-rule (an inbound link from any other concept page) goes in Step 7; m24, m28 and m29 now carry K2. self.assertEqual([m.name for m in self.manifest().mutants if m.cls == "link-rule"], []) def test_every_mutant_class_is_a_check_the_auditor_emits_or_k2_names(self) -> None: expected = set(GROUP_OF) | set(self.K2_CLASSES) for m in self.manifest().mutants: self.assertIn(m.cls, expected, m.name) def test_the_sources_hold_the_k1_fixture_and_a_release_note(self) -> None: k2 = self.k2() self.assertEqual(k2["sources_root"], "gates-local/mutants-kilder") self.assertEqual(k2["k1_file"], "source-manifest.json") self.assertEqual(len(k2["main_topic"]), 3) self.assertTrue(all(p.startswith("kilde/docs-en-") for p in k2["main_topic"]), k2["main_topic"]) self.assertTrue(any(self.RELEASE.match(p) for p in k2["not_main_topic"]), k2["not_main_topic"]) sources = self.manifest().sources self.assertEqual(set(sources), {k2["k1_file"], *k2["main_topic"], *k2["not_main_topic"]}) self.assertEqual(k2["k1_sha256"], sources[k2["k1_file"]], "the K1 file the gate checks is the one described") def test_every_case_documents_its_expected_extras(self) -> None: cases = json.loads(self.path().read_text(encoding="utf-8"))["cases"] known = set(GROUP_OF) | set(self.K2_CLASSES) for c in cases: extra = c.get("expected_extra") self.assertIsInstance(extra, list, c["name"]) self.assertLessEqual(set(extra), known - {"link-rule", c["class"]}, c["name"]) def test_the_known_positive_meets_k2(self) -> None: k2 = self.k2() kp = k2["known_positive"] files = self.manifest().known_positive.files concepts = {f for f in files if f.endswith(".md") and f.rsplit("/", 1)[-1] not in ("index.md", "log.md")} main = kp["x-main-for"] self.assertEqual(len(main), 1, "one main page for the three documentation sources") (values,) = main.values() self.assertEqual(sorted(values), sorted("/" + p for p in k2["main_topic"])) non_main = kp["non_main_pages"] self.assertGreaterEqual(len(non_main), 2) self.assertIn("bypasspermissions-modus.md", non_main) self.assertEqual(set(main) | set(non_main), concepts) self.assertFalse(set(main) & set(non_main)) for page in non_main: self.assertTrue(kp["links_from_main_pages"][page], page) self.assertLessEqual(set(kp["links_from_main_pages"][page]), set(main), page) for page in main: self.assertEqual(kp["concept_links_into_main_pages"][page], [], "a main page needs no inbound link") self.assertTrue(kp["cites_release_notes"]) self.assertLessEqual(set(kp["cites_release_notes"]), set(non_main)) if __name__ == "__main__": unittest.main()