okf-wiki/gates/mutant_gate.py
Kjell Tore Guttormsen 0fd86a23a3
okf-wiki 1.1.1
First public release, published as one snapshot. What changed between
versions is in CHANGELOG.md.

Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
2026-11-14 21:03:20 +01:00

242 lines
11 KiB
Python

"""Mutant gate for the auditor: runs an auditor command on every mutant and on one known-positive bundle.
No model call happens here. The gate is fail-closed: an auditor that is missing, crashes, times out or answers
in the wrong shape has caught nothing.
Auditor contract (provisional until L1 builds the auditor; the command is a template, so L1 may ignore a field):
- argv is the command after `--`, with `{bundle}` replaced by a fresh copy of the case bundle and `{sources}`
by the raw-source root that `sources[].resource` paths resolve against.
- stdout is one JSON object: {"findings": [{"check": "<class>", ...}, ...]}.
- exit 0 with no findings means clean; exit 1 with at least one finding means alarm. Anything else is an error.
A mutant is *caught* when the auditor raises an alarm on it, and *class matched* when one finding's `check`
equals the mutant's class in the manifest. The gate is GREEN only when every mutant is caught and class
matched, the known-positive is clean, and nothing errored.
Before anything runs, every case file and every listed source file is checked against the manifest's sha256;
a mismatch, a missing file or an unexpected file stops the gate (exit 2), because a measurement on other
fixtures than the manifest names is no measurement of them.
Usage:
python3 gates/mutant_gate.py --manifest gates/mutants.manifest.json --cases gates-local/mutants \\
--sources <raw-source root> [--timeout 60] -- <auditor> {bundle} {sources}
Exit: 0 GREEN, 1 RED, 2 the gate could not run.
"""
from __future__ import annotations
import argparse
import hashlib
import json
import shutil
import subprocess
import sys
import tempfile
from collections import Counter
from dataclasses import dataclass, field
from pathlib import Path
KNOWN_POSITIVE = "known-positive"
MUTANT = "mutant"
class GateError(Exception):
"""The gate cannot produce a measurement (bad manifest, drifted fixtures, bad arguments)."""
@dataclass(frozen=True)
class Case:
name: str
kind: str
cls: str | None
files: dict[str, str]
@dataclass(frozen=True)
class Manifest:
sources: dict[str, str]
cases: list[Case]
@property
def known_positive(self) -> Case:
return next(c for c in self.cases if c.kind == KNOWN_POSITIVE)
@property
def mutants(self) -> list[Case]:
return [c for c in self.cases if c.kind == MUTANT]
@dataclass(frozen=True)
class Outcome:
"""What one auditor run said: 'clean', 'alarm' or 'error'."""
status: str
checks: frozenset[str] = field(default_factory=frozenset)
reason: str = ""
def sha256_file(path: Path) -> str:
return hashlib.sha256(path.read_bytes()).hexdigest()
def hash_tree(root: Path) -> dict[str, str]:
"""sha256 of every regular file under root, keyed by its POSIX path relative to root."""
out: dict[str, str] = {}
for p in sorted(root.rglob("*")):
if p.is_symlink():
raise GateError(f"symlink not allowed in a case: {p}")
if p.is_file():
out[p.relative_to(root).as_posix()] = sha256_file(p)
return out
def load_manifest(path: Path) -> Manifest:
try:
data = json.loads(path.read_text(encoding="utf-8"))
except (OSError, json.JSONDecodeError) as exc:
raise GateError(f"cannot read manifest {path}: {exc}") from exc
if data.get("schema") != 1:
raise GateError("manifest schema must be 1")
cases: list[Case] = []
for raw in data.get("cases", []):
kind = raw.get("kind")
if kind not in (KNOWN_POSITIVE, MUTANT):
raise GateError(f"case {raw.get('name')!r}: kind must be {KNOWN_POSITIVE!r} or {MUTANT!r}")
cls = raw.get("class")
if kind == MUTANT and not cls:
raise GateError(f"mutant {raw.get('name')!r} has no class")
files = raw.get("files")
if not isinstance(files, dict) or not files:
raise GateError(f"case {raw.get('name')!r} lists no files")
cases.append(Case(name=str(raw["name"]), kind=kind, cls=cls, files={str(k): str(v) for k, v in files.items()}))
names = [c.name for c in cases]
if len(names) != len(set(names)):
raise GateError("case names are not unique")
if sum(c.kind == KNOWN_POSITIVE for c in cases) != 1:
raise GateError("manifest must have exactly one known-positive case")
if not any(c.kind == MUTANT for c in cases):
raise GateError("manifest has no mutants")
sources = {str(s["path"]): str(s["sha256"]) for s in data.get("sources", [])}
return Manifest(sources=sources, cases=cases)
def verify_fixtures(manifest: Manifest, cases_root: Path, sources_root: Path) -> None:
problems: list[str] = []
for case in manifest.cases:
d = cases_root / case.name
if not d.is_dir():
problems.append(f"{case.name}: case directory missing: {d}")
continue
actual = hash_tree(d)
for rel in sorted(set(case.files) - set(actual)):
problems.append(f"{case.name}: missing file {rel}")
for rel in sorted(set(actual) - set(case.files)):
problems.append(f"{case.name}: unexpected file {rel}")
for rel in sorted(set(actual) & set(case.files)):
if actual[rel] != case.files[rel]:
problems.append(f"{case.name}: sha256 mismatch in {rel}")
for rel, digest in sorted(manifest.sources.items()):
p = sources_root / rel
if not p.is_file():
problems.append(f"source missing: {p}")
elif sha256_file(p) != digest:
problems.append(f"source sha256 mismatch: {p}")
if problems:
raise GateError("fixtures do not match the manifest:\n " + "\n ".join(problems))
def run_auditor(template: list[str], bundle: Path, sources_root: Path, timeout: float) -> Outcome:
argv = [a.replace("{bundle}", str(bundle)).replace("{sources}", str(sources_root)) for a in template]
try:
proc = subprocess.run(argv, capture_output=True, text=True, timeout=timeout, check=False)
except FileNotFoundError:
return Outcome("error", reason=f"auditor not found: {argv[0]}")
except PermissionError:
return Outcome("error", reason=f"auditor not executable: {argv[0]}")
except subprocess.TimeoutExpired:
return Outcome("error", reason=f"auditor timed out after {timeout:g}s")
if proc.returncode not in (0, 1):
tail = proc.stderr.strip().splitlines()[-1:] or [""]
return Outcome("error", reason=f"auditor exited {proc.returncode} {tail[0]}".rstrip())
try:
payload = json.loads(proc.stdout)
except json.JSONDecodeError:
return Outcome("error", reason=f"auditor output is not valid JSON (exit {proc.returncode})")
findings = payload.get("findings") if isinstance(payload, dict) else None
if not isinstance(findings, list) or not all(isinstance(f, dict) and isinstance(f.get("check"), str) for f in findings):
return Outcome("error", reason='auditor output lacks a "findings" list of objects with a string "check"')
checks = frozenset(str(f["check"]) for f in findings)
if proc.returncode == 0 and not findings:
return Outcome("clean")
if proc.returncode == 1 and findings:
return Outcome("alarm", checks=checks)
return Outcome("error", reason=f"exit {proc.returncode} disagrees with {len(findings)} finding(s)")
def run_case(case: Case, template: list[str], cases_root: Path, sources_root: Path, timeout: float) -> Outcome:
with tempfile.TemporaryDirectory(prefix="mutant-gate-") as tmp:
bundle = Path(tmp) / case.name
shutil.copytree(cases_root / case.name, bundle)
return run_auditor(template, bundle, sources_root, timeout)
def report(manifest: Manifest, outcomes: dict[str, Outcome], command: list[str]) -> bool:
mutants = manifest.mutants
kp = manifest.known_positive
n = len(mutants)
print(f"mutant gate: {n} mutants, 1 known-positive; auditor: {' '.join(command)}")
for case in mutants:
o = outcomes[case.name]
if o.status == "alarm":
tag = "CAUGHT" if case.cls in o.checks else "CAUGHT (wrong class: " + ", ".join(sorted(o.checks)) + ")"
elif o.status == "error":
tag = f"MISSED (error: {o.reason})"
else:
tag = "MISSED (auditor said clean)"
print(f" {case.name:<40} [{case.cls}] {tag}")
o = outcomes[kp.name]
kp_tag = {"clean": "CLEAN", "alarm": "FALSE ALARM (" + ", ".join(sorted(o.checks)) + ")"}.get(
o.status, f"ERROR ({o.reason})")
print(f" {kp.name:<40} [{KNOWN_POSITIVE}] {kp_tag}")
print("mutants per class: " + ", ".join(f"{c}: {k}" for c, k in sorted(Counter(m.cls for m in mutants).items())))
caught = [m for m in mutants if outcomes[m.name].status == "alarm"]
matched = [m for m in caught if m.cls in outcomes[m.name].checks]
false_alarms = int(o.status == "alarm")
errors = sum(outcomes[c.name].status == "error" for c in manifest.cases)
print(f"caught {len(caught)} of {n} · class matched {len(matched)} of {n} · "
f"false alarms {false_alarms} of 1 · errors {errors} of {n + 1}")
slipped = [m.name for m in mutants if m not in matched]
print("slipped through: " + (", ".join(slipped) if slipped else "none"))
green = len(matched) == n and o.status == "clean" and errors == 0
print(f"verdict: {'GREEN' if green else 'RED'}")
return green
def main(argv: list[str] | None = None) -> int:
args_in = sys.argv[1:] if argv is None else argv
if "--" in args_in:
split = args_in.index("--")
own, command = args_in[:split], args_in[split + 1:]
else:
own, command = args_in, []
parser = argparse.ArgumentParser(description="Run the auditor's mutant gate (fail-closed, no model calls).")
parser.add_argument("--manifest", type=Path, required=True)
parser.add_argument("--cases", type=Path, required=True, help="directory holding one subdirectory per case")
parser.add_argument("--sources", type=Path, required=True, help="raw-source root, substituted for {sources}")
parser.add_argument("--timeout", type=float, default=60.0, help="seconds per auditor run")
args = parser.parse_args(own)
if not command:
print("usage error: give the auditor command after --, e.g. -- python3 audit.py {bundle} {sources}")
return 2
try:
manifest = load_manifest(args.manifest)
verify_fixtures(manifest, args.cases, args.sources)
except GateError as exc:
print(f"GATE CANNOT RUN: {exc}")
return 2
outcomes = {c.name: run_case(c, command, args.cases, args.sources, args.timeout) for c in manifest.cases}
return 0 if report(manifest, outcomes, command) else 1
if __name__ == "__main__":
sys.exit(main())