First public release, published as one snapshot. What changed between versions is in CHANGELOG.md. Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
242 lines
11 KiB
Python
242 lines
11 KiB
Python
"""Mutant gate for the auditor: runs an auditor command on every mutant and on one known-positive bundle.
|
|
|
|
No model call happens here. The gate is fail-closed: an auditor that is missing, crashes, times out or answers
|
|
in the wrong shape has caught nothing.
|
|
|
|
Auditor contract (provisional until L1 builds the auditor; the command is a template, so L1 may ignore a field):
|
|
- argv is the command after `--`, with `{bundle}` replaced by a fresh copy of the case bundle and `{sources}`
|
|
by the raw-source root that `sources[].resource` paths resolve against.
|
|
- stdout is one JSON object: {"findings": [{"check": "<class>", ...}, ...]}.
|
|
- exit 0 with no findings means clean; exit 1 with at least one finding means alarm. Anything else is an error.
|
|
|
|
A mutant is *caught* when the auditor raises an alarm on it, and *class matched* when one finding's `check`
|
|
equals the mutant's class in the manifest. The gate is GREEN only when every mutant is caught and class
|
|
matched, the known-positive is clean, and nothing errored.
|
|
|
|
Before anything runs, every case file and every listed source file is checked against the manifest's sha256;
|
|
a mismatch, a missing file or an unexpected file stops the gate (exit 2), because a measurement on other
|
|
fixtures than the manifest names is no measurement of them.
|
|
|
|
Usage:
|
|
python3 gates/mutant_gate.py --manifest gates/mutants.manifest.json --cases gates-local/mutants \\
|
|
--sources <raw-source root> [--timeout 60] -- <auditor> {bundle} {sources}
|
|
Exit: 0 GREEN, 1 RED, 2 the gate could not run.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
from collections import Counter
|
|
from dataclasses import dataclass, field
|
|
from pathlib import Path
|
|
|
|
KNOWN_POSITIVE = "known-positive"
|
|
MUTANT = "mutant"
|
|
|
|
|
|
class GateError(Exception):
|
|
"""The gate cannot produce a measurement (bad manifest, drifted fixtures, bad arguments)."""
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class Case:
|
|
name: str
|
|
kind: str
|
|
cls: str | None
|
|
files: dict[str, str]
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class Manifest:
|
|
sources: dict[str, str]
|
|
cases: list[Case]
|
|
|
|
@property
|
|
def known_positive(self) -> Case:
|
|
return next(c for c in self.cases if c.kind == KNOWN_POSITIVE)
|
|
|
|
@property
|
|
def mutants(self) -> list[Case]:
|
|
return [c for c in self.cases if c.kind == MUTANT]
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class Outcome:
|
|
"""What one auditor run said: 'clean', 'alarm' or 'error'."""
|
|
|
|
status: str
|
|
checks: frozenset[str] = field(default_factory=frozenset)
|
|
reason: str = ""
|
|
|
|
|
|
def sha256_file(path: Path) -> str:
|
|
return hashlib.sha256(path.read_bytes()).hexdigest()
|
|
|
|
|
|
def hash_tree(root: Path) -> dict[str, str]:
|
|
"""sha256 of every regular file under root, keyed by its POSIX path relative to root."""
|
|
out: dict[str, str] = {}
|
|
for p in sorted(root.rglob("*")):
|
|
if p.is_symlink():
|
|
raise GateError(f"symlink not allowed in a case: {p}")
|
|
if p.is_file():
|
|
out[p.relative_to(root).as_posix()] = sha256_file(p)
|
|
return out
|
|
|
|
|
|
def load_manifest(path: Path) -> Manifest:
|
|
try:
|
|
data = json.loads(path.read_text(encoding="utf-8"))
|
|
except (OSError, json.JSONDecodeError) as exc:
|
|
raise GateError(f"cannot read manifest {path}: {exc}") from exc
|
|
if data.get("schema") != 1:
|
|
raise GateError("manifest schema must be 1")
|
|
cases: list[Case] = []
|
|
for raw in data.get("cases", []):
|
|
kind = raw.get("kind")
|
|
if kind not in (KNOWN_POSITIVE, MUTANT):
|
|
raise GateError(f"case {raw.get('name')!r}: kind must be {KNOWN_POSITIVE!r} or {MUTANT!r}")
|
|
cls = raw.get("class")
|
|
if kind == MUTANT and not cls:
|
|
raise GateError(f"mutant {raw.get('name')!r} has no class")
|
|
files = raw.get("files")
|
|
if not isinstance(files, dict) or not files:
|
|
raise GateError(f"case {raw.get('name')!r} lists no files")
|
|
cases.append(Case(name=str(raw["name"]), kind=kind, cls=cls, files={str(k): str(v) for k, v in files.items()}))
|
|
names = [c.name for c in cases]
|
|
if len(names) != len(set(names)):
|
|
raise GateError("case names are not unique")
|
|
if sum(c.kind == KNOWN_POSITIVE for c in cases) != 1:
|
|
raise GateError("manifest must have exactly one known-positive case")
|
|
if not any(c.kind == MUTANT for c in cases):
|
|
raise GateError("manifest has no mutants")
|
|
sources = {str(s["path"]): str(s["sha256"]) for s in data.get("sources", [])}
|
|
return Manifest(sources=sources, cases=cases)
|
|
|
|
|
|
def verify_fixtures(manifest: Manifest, cases_root: Path, sources_root: Path) -> None:
|
|
problems: list[str] = []
|
|
for case in manifest.cases:
|
|
d = cases_root / case.name
|
|
if not d.is_dir():
|
|
problems.append(f"{case.name}: case directory missing: {d}")
|
|
continue
|
|
actual = hash_tree(d)
|
|
for rel in sorted(set(case.files) - set(actual)):
|
|
problems.append(f"{case.name}: missing file {rel}")
|
|
for rel in sorted(set(actual) - set(case.files)):
|
|
problems.append(f"{case.name}: unexpected file {rel}")
|
|
for rel in sorted(set(actual) & set(case.files)):
|
|
if actual[rel] != case.files[rel]:
|
|
problems.append(f"{case.name}: sha256 mismatch in {rel}")
|
|
for rel, digest in sorted(manifest.sources.items()):
|
|
p = sources_root / rel
|
|
if not p.is_file():
|
|
problems.append(f"source missing: {p}")
|
|
elif sha256_file(p) != digest:
|
|
problems.append(f"source sha256 mismatch: {p}")
|
|
if problems:
|
|
raise GateError("fixtures do not match the manifest:\n " + "\n ".join(problems))
|
|
|
|
|
|
def run_auditor(template: list[str], bundle: Path, sources_root: Path, timeout: float) -> Outcome:
|
|
argv = [a.replace("{bundle}", str(bundle)).replace("{sources}", str(sources_root)) for a in template]
|
|
try:
|
|
proc = subprocess.run(argv, capture_output=True, text=True, timeout=timeout, check=False)
|
|
except FileNotFoundError:
|
|
return Outcome("error", reason=f"auditor not found: {argv[0]}")
|
|
except PermissionError:
|
|
return Outcome("error", reason=f"auditor not executable: {argv[0]}")
|
|
except subprocess.TimeoutExpired:
|
|
return Outcome("error", reason=f"auditor timed out after {timeout:g}s")
|
|
if proc.returncode not in (0, 1):
|
|
tail = proc.stderr.strip().splitlines()[-1:] or [""]
|
|
return Outcome("error", reason=f"auditor exited {proc.returncode} {tail[0]}".rstrip())
|
|
try:
|
|
payload = json.loads(proc.stdout)
|
|
except json.JSONDecodeError:
|
|
return Outcome("error", reason=f"auditor output is not valid JSON (exit {proc.returncode})")
|
|
findings = payload.get("findings") if isinstance(payload, dict) else None
|
|
if not isinstance(findings, list) or not all(isinstance(f, dict) and isinstance(f.get("check"), str) for f in findings):
|
|
return Outcome("error", reason='auditor output lacks a "findings" list of objects with a string "check"')
|
|
checks = frozenset(str(f["check"]) for f in findings)
|
|
if proc.returncode == 0 and not findings:
|
|
return Outcome("clean")
|
|
if proc.returncode == 1 and findings:
|
|
return Outcome("alarm", checks=checks)
|
|
return Outcome("error", reason=f"exit {proc.returncode} disagrees with {len(findings)} finding(s)")
|
|
|
|
|
|
def run_case(case: Case, template: list[str], cases_root: Path, sources_root: Path, timeout: float) -> Outcome:
|
|
with tempfile.TemporaryDirectory(prefix="mutant-gate-") as tmp:
|
|
bundle = Path(tmp) / case.name
|
|
shutil.copytree(cases_root / case.name, bundle)
|
|
return run_auditor(template, bundle, sources_root, timeout)
|
|
|
|
|
|
def report(manifest: Manifest, outcomes: dict[str, Outcome], command: list[str]) -> bool:
|
|
mutants = manifest.mutants
|
|
kp = manifest.known_positive
|
|
n = len(mutants)
|
|
print(f"mutant gate: {n} mutants, 1 known-positive; auditor: {' '.join(command)}")
|
|
for case in mutants:
|
|
o = outcomes[case.name]
|
|
if o.status == "alarm":
|
|
tag = "CAUGHT" if case.cls in o.checks else "CAUGHT (wrong class: " + ", ".join(sorted(o.checks)) + ")"
|
|
elif o.status == "error":
|
|
tag = f"MISSED (error: {o.reason})"
|
|
else:
|
|
tag = "MISSED (auditor said clean)"
|
|
print(f" {case.name:<40} [{case.cls}] {tag}")
|
|
o = outcomes[kp.name]
|
|
kp_tag = {"clean": "CLEAN", "alarm": "FALSE ALARM (" + ", ".join(sorted(o.checks)) + ")"}.get(
|
|
o.status, f"ERROR ({o.reason})")
|
|
print(f" {kp.name:<40} [{KNOWN_POSITIVE}] {kp_tag}")
|
|
print("mutants per class: " + ", ".join(f"{c}: {k}" for c, k in sorted(Counter(m.cls for m in mutants).items())))
|
|
caught = [m for m in mutants if outcomes[m.name].status == "alarm"]
|
|
matched = [m for m in caught if m.cls in outcomes[m.name].checks]
|
|
false_alarms = int(o.status == "alarm")
|
|
errors = sum(outcomes[c.name].status == "error" for c in manifest.cases)
|
|
print(f"caught {len(caught)} of {n} · class matched {len(matched)} of {n} · "
|
|
f"false alarms {false_alarms} of 1 · errors {errors} of {n + 1}")
|
|
slipped = [m.name for m in mutants if m not in matched]
|
|
print("slipped through: " + (", ".join(slipped) if slipped else "none"))
|
|
green = len(matched) == n and o.status == "clean" and errors == 0
|
|
print(f"verdict: {'GREEN' if green else 'RED'}")
|
|
return green
|
|
|
|
|
|
def main(argv: list[str] | None = None) -> int:
|
|
args_in = sys.argv[1:] if argv is None else argv
|
|
if "--" in args_in:
|
|
split = args_in.index("--")
|
|
own, command = args_in[:split], args_in[split + 1:]
|
|
else:
|
|
own, command = args_in, []
|
|
parser = argparse.ArgumentParser(description="Run the auditor's mutant gate (fail-closed, no model calls).")
|
|
parser.add_argument("--manifest", type=Path, required=True)
|
|
parser.add_argument("--cases", type=Path, required=True, help="directory holding one subdirectory per case")
|
|
parser.add_argument("--sources", type=Path, required=True, help="raw-source root, substituted for {sources}")
|
|
parser.add_argument("--timeout", type=float, default=60.0, help="seconds per auditor run")
|
|
args = parser.parse_args(own)
|
|
if not command:
|
|
print("usage error: give the auditor command after --, e.g. -- python3 audit.py {bundle} {sources}")
|
|
return 2
|
|
try:
|
|
manifest = load_manifest(args.manifest)
|
|
verify_fixtures(manifest, args.cases, args.sources)
|
|
except GateError as exc:
|
|
print(f"GATE CANNOT RUN: {exc}")
|
|
return 2
|
|
outcomes = {c.name: run_case(c, command, args.cases, args.sources, args.timeout) for c in manifest.cases}
|
|
return 0 if report(manifest, outcomes, command) else 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|