First public release, published as one snapshot. What changed between versions is in CHANGELOG.md. Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
372 lines
18 KiB
Python
372 lines
18 KiB
Python
"""The package boundary (order 5 of wave 1; rev 4 Step 13): no module under okfwiki/ imports the trial or a network
|
|
or model client.
|
|
|
|
The scanner reads every .py file under okfwiki/ with ast and lists each import that reaches a forbidden module, by
|
|
exact module name: the trial (`evals`, `p1`, `direction_probe`) and the modules `docs/bolge1/0-felles` section 2 bars
|
|
(`anthropic`, `openai`, `urllib.request`, `http.client`, `requests`, `socket`, `subprocess`). `urllib.parse` is allowed.
|
|
A name reaches a forbidden module when it is that module or a submodule of it, so `from urllib import request` reaches
|
|
`urllib.request`. A call to `importlib.import_module` or to `__import__` counts as an import, under any name the file
|
|
binds them to, with its name given by position or keyword, relative to its package, and with the names in `fromlist`;
|
|
a call whose names are not string literals cannot be checked and is a hit ("<dynamic import>"). A file that does not
|
|
parse is a hit, never a pass. Not seen: code run from a string (exec, eval) and a loader fetched with getattr. Network
|
|
code lives in tools/, which this test does not scan.
|
|
|
|
Wave 2 order 7 adds the names that start a program as well as `subprocess` does: `os.system`, `os.popen`, `os.exec*`,
|
|
`os.spawn*`, `os.posix_spawn*` and `asyncio.create_subprocess_*` (also under `asyncio.subprocess`, and the os names
|
|
under `posix` and `nt`, where os takes them from). Each is a hit wherever it is reached: an attribute, called or not, of
|
|
a name the file binds to the module by an import or by an assignment (`m = __import__('os')`, `o = os`), or of a
|
|
loader call with a literal name (`__import__('os').system`); a `from ... import` of it; a star import from one of
|
|
these modules; and `getattr(<module>, '<name>')`, where a name that is not a string literal is a dynamic hit. A local
|
|
name that only looks the same (`system = len`) is not one. Not seen: `vars(os)[...]`, `os.__dict__[...]`, and a module
|
|
handed on through a container or a function argument.
|
|
Run: python3 -m unittest discover -s tests -p 'test_package.py'
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import ast
|
|
import re
|
|
import shutil
|
|
import tempfile
|
|
import unittest
|
|
from dataclasses import dataclass
|
|
from pathlib import Path
|
|
from typing import Iterator
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
FORBIDDEN = ("evals", "p1", "direction_probe", "anthropic", "openai", "urllib.request", "http.client", "requests",
|
|
"socket", "subprocess")
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class Hit:
|
|
file: str # relative to the scanned package
|
|
line: int
|
|
module: str # the forbidden module the import reaches, "<dynamic import>" or "<does not parse>"
|
|
|
|
|
|
def reached(name: str) -> str | None:
|
|
"""The forbidden module a dotted name is, or lies under; None for any other name."""
|
|
return next((f for f in FORBIDDEN if name == f or name.startswith(f + ".")), None)
|
|
|
|
|
|
LOADER_MODULES = ("importlib", "builtins")
|
|
LOADERS = ("import_module", "__import__")
|
|
DYNAMIC = "<dynamic import>"
|
|
|
|
|
|
def loader_names(tree: ast.AST) -> tuple[dict[str, str], set[str]]:
|
|
"""(each name this file binds to a loader, mapped to the loader; the names bound to a module that holds one)."""
|
|
funcs = {"__import__": "__import__"}
|
|
modules = set(LOADER_MODULES)
|
|
for node in ast.walk(tree):
|
|
if isinstance(node, ast.ImportFrom) and node.level == 0 and node.module in LOADER_MODULES:
|
|
funcs.update({a.asname or a.name: a.name for a in node.names if a.name in LOADERS})
|
|
elif isinstance(node, ast.Import):
|
|
modules.update(a.asname or a.name for a in node.names if a.name in LOADER_MODULES)
|
|
return funcs, modules
|
|
|
|
|
|
def loader_of(func: ast.expr, funcs: dict[str, str], modules: set[str]) -> str | None:
|
|
"""The loader a call's function is (import_module or __import__), or None."""
|
|
if isinstance(func, ast.Name):
|
|
return funcs.get(func.id)
|
|
if isinstance(func, ast.Attribute) and func.attr == "__import__":
|
|
return "__import__"
|
|
if (isinstance(func, ast.Attribute) and func.attr == "import_module" and isinstance(func.value, ast.Name)
|
|
and func.value.id in modules):
|
|
return "import_module"
|
|
return None
|
|
|
|
|
|
def argument(call: ast.Call, position: int, keyword: str) -> ast.expr | None:
|
|
if len(call.args) > position:
|
|
return call.args[position]
|
|
return next((k.value for k in call.keywords if k.arg == keyword), None)
|
|
|
|
|
|
def literal(node: ast.expr | None) -> str | None:
|
|
return node.value if isinstance(node, ast.Constant) and isinstance(node.value, str) else None
|
|
|
|
|
|
def loaded(call: ast.Call, loader: str) -> list[str] | None:
|
|
"""The dotted names a loader call imports, or None when they are not string literals."""
|
|
name = literal(argument(call, 0, "name"))
|
|
if name is None:
|
|
return None
|
|
if loader == "__import__":
|
|
fromlist = argument(call, 3, "fromlist")
|
|
if fromlist is None:
|
|
return [name]
|
|
items = [literal(e) for e in fromlist.elts] if isinstance(fromlist, (ast.List, ast.Tuple)) else [None]
|
|
return None if None in items else [name] + [f"{name}.{x}" for x in items]
|
|
if name.startswith("."):
|
|
package = literal(argument(call, 1, "package"))
|
|
up = len(name) - len(name.lstrip(".")) - 1
|
|
if package is None or up > package.count("."):
|
|
return None
|
|
base = package.rsplit(".", up)[0] if up else package
|
|
return [f"{base}.{name.lstrip('.')}"]
|
|
return [name]
|
|
|
|
|
|
def imported(tree: ast.AST) -> Iterator[tuple[int, list[str] | None]]:
|
|
"""(line, the dotted names a statement or call can import, or None when they cannot be read) for every import in
|
|
the tree, nested ones too."""
|
|
funcs, modules = loader_names(tree)
|
|
for node in ast.walk(tree):
|
|
if isinstance(node, ast.Import):
|
|
yield node.lineno, [a.name for a in node.names]
|
|
elif isinstance(node, ast.ImportFrom) and node.level == 0: # a relative import stays inside the package
|
|
base = node.module or ""
|
|
yield node.lineno, [base] + [f"{base}.{a.name}" for a in node.names]
|
|
elif isinstance(node, ast.Call):
|
|
loader = loader_of(node.func, funcs, modules)
|
|
if loader is not None:
|
|
yield node.lineno, loaded(node, loader)
|
|
|
|
|
|
OS_NAMES = re.compile(r"system|popen|exec\w*|spawn\w*|posix_spawn\w*")
|
|
SUBPROCESS_NAMES = re.compile(r"create_subprocess_\w+")
|
|
LAUNCHERS = { # a module -> the names under it that start a program (os takes its names from posix or nt)
|
|
"os": OS_NAMES,
|
|
"posix": OS_NAMES,
|
|
"nt": OS_NAMES,
|
|
"asyncio": SUBPROCESS_NAMES,
|
|
"asyncio.subprocess": SUBPROCESS_NAMES,
|
|
}
|
|
|
|
|
|
def launcher(module: str, name: str) -> str | None:
|
|
"""The dotted name when `name` under `module` starts a program, else None."""
|
|
pattern = LAUNCHERS.get(module)
|
|
return f"{module}.{name}" if pattern is not None and pattern.fullmatch(name) else None
|
|
|
|
|
|
class Modules:
|
|
"""Which module of LAUNCHERS an expression is: a name the file binds to one by an import, or by an assignment from
|
|
another such expression (m = __import__('os'), o = os), or a loader call with a literal name."""
|
|
|
|
def __init__(self, tree: ast.AST) -> None:
|
|
self.funcs, self.loader_modules = loader_names(tree)
|
|
self.aliases: dict[str, str] = {}
|
|
for node in ast.walk(tree):
|
|
if isinstance(node, ast.Import):
|
|
for a in node.names:
|
|
top = a.name.split(".")[0]
|
|
if a.asname and a.name in LAUNCHERS:
|
|
self.aliases[a.asname] = a.name
|
|
elif not a.asname and top in LAUNCHERS: # import os.path binds os
|
|
self.aliases[top] = top
|
|
elif isinstance(node, ast.ImportFrom) and node.level == 0 and node.module == "asyncio":
|
|
self.aliases.update({a.asname or a.name: "asyncio.subprocess" for a in node.names if a.name == "subprocess"})
|
|
for _ in range(2): # an assignment may name a module bound by another assignment
|
|
for node in ast.walk(tree):
|
|
if isinstance(node, ast.Assign) and len(node.targets) == 1 and isinstance(node.targets[0], ast.Name):
|
|
module = self.module(node.value)
|
|
if module is not None:
|
|
self.aliases[node.targets[0].id] = module
|
|
|
|
def module(self, node: ast.expr) -> str | None:
|
|
if isinstance(node, ast.Name):
|
|
return self.aliases.get(node.id)
|
|
if not isinstance(node, ast.Call):
|
|
return None
|
|
loader = loader_of(node.func, self.funcs, self.loader_modules)
|
|
names = loaded(node, loader) if loader is not None else None
|
|
if not names:
|
|
return None
|
|
if loader == "__import__" and argument(node, 3, "fromlist") is None:
|
|
return names[0].split(".")[0] if names[0].split(".")[0] in LAUNCHERS else None # returns the top package
|
|
return names[0] if names[0] in LAUNCHERS else None
|
|
|
|
|
|
def dotted(node: ast.expr, modules: Modules) -> str | None:
|
|
"""The dotted name of a chain of attributes on a module of LAUNCHERS (os.path.join -> "os.path.join"), else None."""
|
|
parts: list[str] = []
|
|
while isinstance(node, ast.Attribute):
|
|
parts.append(node.attr)
|
|
node = node.value
|
|
base = modules.module(node)
|
|
return None if base is None else ".".join([base, *reversed(parts)])
|
|
|
|
|
|
def launched(tree: ast.AST) -> Iterator[tuple[int, str]]:
|
|
"""(line, name) for every place the tree reaches a name that starts a program, or DYNAMIC for a getattr on such a
|
|
module whose name is not a string literal."""
|
|
modules = Modules(tree)
|
|
for node in ast.walk(tree):
|
|
if isinstance(node, ast.ImportFrom) and node.level == 0 and node.module in LAUNCHERS:
|
|
for a in node.names:
|
|
hit = f"{node.module}.*" if a.name == "*" else launcher(node.module, a.name)
|
|
if hit is not None:
|
|
yield node.lineno, hit
|
|
elif isinstance(node, ast.Attribute):
|
|
name = dotted(node, modules)
|
|
if name is not None and "." in name:
|
|
module, attr = name.rsplit(".", 1)
|
|
hit = launcher(module, attr)
|
|
if hit is not None:
|
|
yield node.lineno, hit
|
|
elif (isinstance(node, ast.Call) and isinstance(node.func, ast.Name) and node.func.id == "getattr"
|
|
and len(node.args) >= 2):
|
|
target, wanted = dotted(node.args[0], modules), literal(node.args[1])
|
|
if target is not None and target in LAUNCHERS:
|
|
found = DYNAMIC if wanted is None else launcher(target, wanted)
|
|
if found is not None:
|
|
yield node.lineno, found
|
|
|
|
|
|
def scan(package: Path) -> tuple[int, list[Hit]]:
|
|
"""(files read, hits) for every .py file under package."""
|
|
files = sorted(package.rglob("*.py"))
|
|
hits: list[Hit] = []
|
|
for path in files:
|
|
rel = path.relative_to(package).as_posix()
|
|
try:
|
|
tree = ast.parse(path.read_text(encoding="utf-8"), filename=rel)
|
|
except (SyntaxError, UnicodeDecodeError) as exc:
|
|
hits.append(Hit(rel, getattr(exc, "lineno", None) or 1, "<does not parse>"))
|
|
continue
|
|
for line, names in imported(tree):
|
|
if names is None:
|
|
hits.append(Hit(rel, line, DYNAMIC))
|
|
continue
|
|
hits.extend(Hit(rel, line, m) for m in sorted({m for m in map(reached, names) if m is not None}))
|
|
hits.extend(Hit(rel, line, name) for line, name in launched(tree))
|
|
return len(files), sorted(hits, key=lambda h: (h.file, h.line, h.module))
|
|
|
|
|
|
PLANTED = ( # (code, the module it reaches, the line of the import within the code)
|
|
("import evals.p1.kpack", "evals", 1),
|
|
("from p1.split import blocks", "p1", 1),
|
|
("from evals import direction_probe", "evals", 1),
|
|
("import direction_probe", "direction_probe", 1),
|
|
("import anthropic", "anthropic", 1),
|
|
("from openai import OpenAI", "openai", 1),
|
|
("import urllib.request", "urllib.request", 1),
|
|
("from urllib import request", "urllib.request", 1),
|
|
("from urllib import parse, request", "urllib.request", 1),
|
|
("from http import client", "http.client", 1),
|
|
("import http.client as hc", "http.client", 1),
|
|
("import requests", "requests", 1),
|
|
("import socket", "socket", 1),
|
|
("from subprocess import run", "subprocess", 1),
|
|
("import importlib\nm = importlib.import_module('socket')", "socket", 2),
|
|
("m = __import__('subprocess')", "subprocess", 1),
|
|
("def f() -> None:\n import openai", "openai", 2),
|
|
("import importlib\nm = importlib.__import__('subprocess')", "subprocess", 2),
|
|
("import builtins\nm = builtins.__import__('socket')", "socket", 2),
|
|
("import importlib\nm = importlib.import_module(name='socket')", "socket", 2),
|
|
("from importlib import import_module as load\nm = load('socket')", "socket", 2),
|
|
("import importlib as il\nm = il.import_module('requests')", "requests", 2),
|
|
("import importlib\nm = importlib.import_module('.request', 'urllib')", "urllib.request", 2),
|
|
("m = __import__('urllib', fromlist=['request'])", "urllib.request", 1),
|
|
("import importlib\nname = 'sock' + 'et'\nm = importlib.import_module(name)", "<dynamic import>", 3),
|
|
("m = __import__('urllib', fromlist=wanted)", "<dynamic import>", 1),
|
|
# Wave 2 order 7: names that start a program as well as subprocess does (docs/bolge2/0-felles section 2).
|
|
("import os\nos.system('ls')", "os.system", 2),
|
|
("import os\nos.popen('ls')", "os.popen", 2),
|
|
("import os\nos.execv('/bin/ls', ['ls'])", "os.execv", 2),
|
|
("import os\nos.execlp('ls', 'ls')", "os.execlp", 2),
|
|
("import os\nos.spawnv(os.P_WAIT, '/bin/ls', ['ls'])", "os.spawnv", 2),
|
|
("import os\nos.posix_spawn('/bin/ls', ['ls'], {})", "os.posix_spawn", 2),
|
|
("import os\nos.posix_spawnp('ls', ['ls'], {})", "os.posix_spawnp", 2),
|
|
("import os as o\no.system('ls')", "os.system", 2),
|
|
("import os\nrun = os.system", "os.system", 2),
|
|
("def f() -> None:\n import os\n os.system('ls')", "os.system", 3),
|
|
("from os import system\nsystem('ls')", "os.system", 1),
|
|
("from os import popen as p", "os.popen", 1),
|
|
("from os import execvp, path", "os.execvp", 1),
|
|
("from os import *", "os.*", 1),
|
|
("import os\nf = getattr(os, 'system')", "os.system", 2),
|
|
("import os\nname = 'sys' + 'tem'\nf = getattr(os, name)", "<dynamic import>", 3),
|
|
("import asyncio\nasyncio.create_subprocess_exec('ls')", "asyncio.create_subprocess_exec", 2),
|
|
("import asyncio\nasyncio.create_subprocess_shell('ls')", "asyncio.create_subprocess_shell", 2),
|
|
("from asyncio import create_subprocess_exec", "asyncio.create_subprocess_exec", 1),
|
|
("import asyncio.subprocess\nasyncio.subprocess.create_subprocess_exec('ls')",
|
|
"asyncio.subprocess.create_subprocess_exec", 2),
|
|
("from asyncio.subprocess import create_subprocess_shell", "asyncio.subprocess.create_subprocess_shell", 1),
|
|
# After the counter-reading: the module reached through a loader call or a plain assignment, and posix and nt,
|
|
# the platform modules os takes these names from.
|
|
("__import__('os').system('ls')", "os.system", 1),
|
|
("import importlib\nimportlib.import_module('os').system('ls')", "os.system", 2),
|
|
("m = __import__('os')\nm.popen('ls')", "os.popen", 2),
|
|
("import os\no = os\no.system('ls')", "os.system", 3),
|
|
("import posix\nposix.system('ls')", "posix.system", 2),
|
|
("from nt import spawnv", "nt.spawnv", 1),
|
|
)
|
|
HEADER = '"""A planted module."""\n\n' # the planted code starts on file line 3
|
|
|
|
ALLOWED = (
|
|
"from urllib.parse import unquote",
|
|
"import urllib.parse",
|
|
"from urllib import parse",
|
|
"import http",
|
|
"from http import HTTPStatus",
|
|
"import evalsx",
|
|
"import p10",
|
|
"import requests_plain",
|
|
"from okfwiki import common",
|
|
"from . import audit",
|
|
"import importlib\nm = importlib.import_module('json')",
|
|
"import importlib\nm = importlib.import_module('.parse', 'urllib')",
|
|
"m = __import__('urllib', fromlist=['parse'])",
|
|
"import_module = len\nm = import_module('socket')", # a local name, not importlib's
|
|
"import os\nx = os.path.join('a', 'b')",
|
|
"import os\nos.replace('a', 'b')\ny = os.environ.get('HOME')",
|
|
"from os import environ, path, sep",
|
|
"import asyncio\nasyncio.run(main())",
|
|
"system = len\nsystem('ls')", # a local name, not os's
|
|
"import shutil\nshutil.which('ls')",
|
|
"import os\nf = getattr(os, 'sep')",
|
|
"o = len\no.system('ls')", # a local name bound to something else
|
|
"import os\np = os.path\np.join('a', 'b')",
|
|
)
|
|
|
|
|
|
class PackageBoundaryTest(unittest.TestCase):
|
|
def setUp(self) -> None:
|
|
self._tmp = tempfile.TemporaryDirectory()
|
|
self.copy = Path(self._tmp.name) / "okfwiki"
|
|
shutil.copytree(ROOT / "okfwiki", self.copy, ignore=shutil.ignore_patterns("__pycache__"))
|
|
|
|
def tearDown(self) -> None:
|
|
self._tmp.cleanup()
|
|
|
|
def plant(self, code: str) -> list[Hit]:
|
|
path = self.copy / "planted.py"
|
|
path.write_text(HEADER + code + "\n", encoding="utf-8")
|
|
_, hits = scan(self.copy)
|
|
path.unlink()
|
|
return hits
|
|
|
|
def test_no_module_under_okfwiki_imports_a_forbidden_module(self) -> None:
|
|
files, hits = scan(ROOT / "okfwiki")
|
|
self.assertEqual(files, len(list((ROOT / "okfwiki").rglob("*.py"))))
|
|
self.assertGreater(files, 0, "a scan that reads no file cannot pass")
|
|
self.assertEqual(hits, [])
|
|
|
|
def test_a_planted_import_in_a_temporary_copy_is_flagged(self) -> None:
|
|
for code, module, line in PLANTED:
|
|
with self.subTest(code=code):
|
|
self.assertEqual(self.plant(code), [Hit("planted.py", 2 + line, module)])
|
|
|
|
def test_allowed_modules_and_lookalike_names_are_not_flagged(self) -> None:
|
|
for code in ALLOWED:
|
|
with self.subTest(code=code):
|
|
self.assertEqual(self.plant(code), [])
|
|
|
|
def test_a_file_that_does_not_parse_is_a_hit_not_a_pass(self) -> None:
|
|
self.assertEqual(self.plant("def broken(:\n pass"), [Hit("planted.py", 3, "<does not parse>")])
|
|
|
|
def test_files_in_subpackages_are_read(self) -> None:
|
|
(self.copy / "sub").mkdir()
|
|
(self.copy / "sub" / "deep.py").write_text("import subprocess\n", encoding="utf-8")
|
|
files, hits = scan(self.copy)
|
|
self.assertEqual(files, len(list(self.copy.rglob("*.py"))))
|
|
self.assertEqual(hits, [Hit("sub/deep.py", 1, "subprocess")])
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|