"""Mutant gate for the auditor: runs an auditor command on every mutant and on one known-positive bundle. No model call happens here. The gate is fail-closed: an auditor that is missing, crashes, times out or answers in the wrong shape has caught nothing. Auditor contract (provisional until L1 builds the auditor; the command is a template, so L1 may ignore a field): - argv is the command after `--`, with `{bundle}` replaced by a fresh copy of the case bundle and `{sources}` by the raw-source root that `sources[].resource` paths resolve against. - stdout is one JSON object: {"findings": [{"check": "", ...}, ...]}. - exit 0 with no findings means clean; exit 1 with at least one finding means alarm. Anything else is an error. A mutant is *caught* when the auditor raises an alarm on it, and *class matched* when one finding's `check` equals the mutant's class in the manifest. The gate is GREEN only when every mutant is caught and class matched, the known-positive is clean, and nothing errored. Before anything runs, every case file and every listed source file is checked against the manifest's sha256; a mismatch, a missing file or an unexpected file stops the gate (exit 2), because a measurement on other fixtures than the manifest names is no measurement of them. Usage: python3 gates/mutant_gate.py --manifest gates/mutants.manifest.json --cases gates-local/mutants \\ --sources [--timeout 60] -- {bundle} {sources} Exit: 0 GREEN, 1 RED, 2 the gate could not run. """ from __future__ import annotations import argparse import hashlib import json import shutil import subprocess import sys import tempfile from collections import Counter from dataclasses import dataclass, field from pathlib import Path KNOWN_POSITIVE = "known-positive" MUTANT = "mutant" class GateError(Exception): """The gate cannot produce a measurement (bad manifest, drifted fixtures, bad arguments).""" @dataclass(frozen=True) class Case: name: str kind: str cls: str | None files: dict[str, str] @dataclass(frozen=True) class Manifest: sources: dict[str, str] cases: list[Case] @property def known_positive(self) -> Case: return next(c for c in self.cases if c.kind == KNOWN_POSITIVE) @property def mutants(self) -> list[Case]: return [c for c in self.cases if c.kind == MUTANT] @dataclass(frozen=True) class Outcome: """What one auditor run said: 'clean', 'alarm' or 'error'.""" status: str checks: frozenset[str] = field(default_factory=frozenset) reason: str = "" def sha256_file(path: Path) -> str: return hashlib.sha256(path.read_bytes()).hexdigest() def hash_tree(root: Path) -> dict[str, str]: """sha256 of every regular file under root, keyed by its POSIX path relative to root.""" out: dict[str, str] = {} for p in sorted(root.rglob("*")): if p.is_symlink(): raise GateError(f"symlink not allowed in a case: {p}") if p.is_file(): out[p.relative_to(root).as_posix()] = sha256_file(p) return out def load_manifest(path: Path) -> Manifest: try: data = json.loads(path.read_text(encoding="utf-8")) except (OSError, json.JSONDecodeError) as exc: raise GateError(f"cannot read manifest {path}: {exc}") from exc if data.get("schema") != 1: raise GateError("manifest schema must be 1") cases: list[Case] = [] for raw in data.get("cases", []): kind = raw.get("kind") if kind not in (KNOWN_POSITIVE, MUTANT): raise GateError(f"case {raw.get('name')!r}: kind must be {KNOWN_POSITIVE!r} or {MUTANT!r}") cls = raw.get("class") if kind == MUTANT and not cls: raise GateError(f"mutant {raw.get('name')!r} has no class") files = raw.get("files") if not isinstance(files, dict) or not files: raise GateError(f"case {raw.get('name')!r} lists no files") cases.append(Case(name=str(raw["name"]), kind=kind, cls=cls, files={str(k): str(v) for k, v in files.items()})) names = [c.name for c in cases] if len(names) != len(set(names)): raise GateError("case names are not unique") if sum(c.kind == KNOWN_POSITIVE for c in cases) != 1: raise GateError("manifest must have exactly one known-positive case") if not any(c.kind == MUTANT for c in cases): raise GateError("manifest has no mutants") sources = {str(s["path"]): str(s["sha256"]) for s in data.get("sources", [])} return Manifest(sources=sources, cases=cases) def verify_fixtures(manifest: Manifest, cases_root: Path, sources_root: Path) -> None: problems: list[str] = [] for case in manifest.cases: d = cases_root / case.name if not d.is_dir(): problems.append(f"{case.name}: case directory missing: {d}") continue actual = hash_tree(d) for rel in sorted(set(case.files) - set(actual)): problems.append(f"{case.name}: missing file {rel}") for rel in sorted(set(actual) - set(case.files)): problems.append(f"{case.name}: unexpected file {rel}") for rel in sorted(set(actual) & set(case.files)): if actual[rel] != case.files[rel]: problems.append(f"{case.name}: sha256 mismatch in {rel}") for rel, digest in sorted(manifest.sources.items()): p = sources_root / rel if not p.is_file(): problems.append(f"source missing: {p}") elif sha256_file(p) != digest: problems.append(f"source sha256 mismatch: {p}") if problems: raise GateError("fixtures do not match the manifest:\n " + "\n ".join(problems)) def run_auditor(template: list[str], bundle: Path, sources_root: Path, timeout: float) -> Outcome: argv = [a.replace("{bundle}", str(bundle)).replace("{sources}", str(sources_root)) for a in template] try: proc = subprocess.run(argv, capture_output=True, text=True, timeout=timeout, check=False) except FileNotFoundError: return Outcome("error", reason=f"auditor not found: {argv[0]}") except PermissionError: return Outcome("error", reason=f"auditor not executable: {argv[0]}") except subprocess.TimeoutExpired: return Outcome("error", reason=f"auditor timed out after {timeout:g}s") if proc.returncode not in (0, 1): tail = proc.stderr.strip().splitlines()[-1:] or [""] return Outcome("error", reason=f"auditor exited {proc.returncode} {tail[0]}".rstrip()) try: payload = json.loads(proc.stdout) except json.JSONDecodeError: return Outcome("error", reason=f"auditor output is not valid JSON (exit {proc.returncode})") findings = payload.get("findings") if isinstance(payload, dict) else None if not isinstance(findings, list) or not all(isinstance(f, dict) and isinstance(f.get("check"), str) for f in findings): return Outcome("error", reason='auditor output lacks a "findings" list of objects with a string "check"') checks = frozenset(str(f["check"]) for f in findings) if proc.returncode == 0 and not findings: return Outcome("clean") if proc.returncode == 1 and findings: return Outcome("alarm", checks=checks) return Outcome("error", reason=f"exit {proc.returncode} disagrees with {len(findings)} finding(s)") def run_case(case: Case, template: list[str], cases_root: Path, sources_root: Path, timeout: float) -> Outcome: with tempfile.TemporaryDirectory(prefix="mutant-gate-") as tmp: bundle = Path(tmp) / case.name shutil.copytree(cases_root / case.name, bundle) return run_auditor(template, bundle, sources_root, timeout) def report(manifest: Manifest, outcomes: dict[str, Outcome], command: list[str]) -> bool: mutants = manifest.mutants kp = manifest.known_positive n = len(mutants) print(f"mutant gate: {n} mutants, 1 known-positive; auditor: {' '.join(command)}") for case in mutants: o = outcomes[case.name] if o.status == "alarm": tag = "CAUGHT" if case.cls in o.checks else "CAUGHT (wrong class: " + ", ".join(sorted(o.checks)) + ")" elif o.status == "error": tag = f"MISSED (error: {o.reason})" else: tag = "MISSED (auditor said clean)" print(f" {case.name:<40} [{case.cls}] {tag}") o = outcomes[kp.name] kp_tag = {"clean": "CLEAN", "alarm": "FALSE ALARM (" + ", ".join(sorted(o.checks)) + ")"}.get( o.status, f"ERROR ({o.reason})") print(f" {kp.name:<40} [{KNOWN_POSITIVE}] {kp_tag}") print("mutants per class: " + ", ".join(f"{c}: {k}" for c, k in sorted(Counter(m.cls for m in mutants).items()))) caught = [m for m in mutants if outcomes[m.name].status == "alarm"] matched = [m for m in caught if m.cls in outcomes[m.name].checks] false_alarms = int(o.status == "alarm") errors = sum(outcomes[c.name].status == "error" for c in manifest.cases) print(f"caught {len(caught)} of {n} · class matched {len(matched)} of {n} · " f"false alarms {false_alarms} of 1 · errors {errors} of {n + 1}") slipped = [m.name for m in mutants if m not in matched] print("slipped through: " + (", ".join(slipped) if slipped else "none")) green = len(matched) == n and o.status == "clean" and errors == 0 print(f"verdict: {'GREEN' if green else 'RED'}") return green def main(argv: list[str] | None = None) -> int: args_in = sys.argv[1:] if argv is None else argv if "--" in args_in: split = args_in.index("--") own, command = args_in[:split], args_in[split + 1:] else: own, command = args_in, [] parser = argparse.ArgumentParser(description="Run the auditor's mutant gate (fail-closed, no model calls).") parser.add_argument("--manifest", type=Path, required=True) parser.add_argument("--cases", type=Path, required=True, help="directory holding one subdirectory per case") parser.add_argument("--sources", type=Path, required=True, help="raw-source root, substituted for {sources}") parser.add_argument("--timeout", type=float, default=60.0, help="seconds per auditor run") args = parser.parse_args(own) if not command: print("usage error: give the auditor command after --, e.g. -- python3 audit.py {bundle} {sources}") return 2 try: manifest = load_manifest(args.manifest) verify_fixtures(manifest, args.cases, args.sources) except GateError as exc: print(f"GATE CANNOT RUN: {exc}") return 2 outcomes = {c.name: run_case(c, command, args.cases, args.sources, args.timeout) for c in manifest.cases} return 0 if report(manifest, outcomes, command) else 1 if __name__ == "__main__": sys.exit(main())