okf-wiki 1.1.1
First public release, published as one snapshot. What changed between versions is in CHANGELOG.md. Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
This commit is contained in:
commit
0fd86a23a3
227 changed files with 96077 additions and 0 deletions
372
tests/test_package.py
Normal file
372
tests/test_package.py
Normal file
|
|
@ -0,0 +1,372 @@
|
|||
"""The package boundary (order 5 of wave 1; rev 4 Step 13): no module under okfwiki/ imports the trial or a network
|
||||
or model client.
|
||||
|
||||
The scanner reads every .py file under okfwiki/ with ast and lists each import that reaches a forbidden module, by
|
||||
exact module name: the trial (`evals`, `p1`, `direction_probe`) and the modules `docs/bolge1/0-felles` section 2 bars
|
||||
(`anthropic`, `openai`, `urllib.request`, `http.client`, `requests`, `socket`, `subprocess`). `urllib.parse` is allowed.
|
||||
A name reaches a forbidden module when it is that module or a submodule of it, so `from urllib import request` reaches
|
||||
`urllib.request`. A call to `importlib.import_module` or to `__import__` counts as an import, under any name the file
|
||||
binds them to, with its name given by position or keyword, relative to its package, and with the names in `fromlist`;
|
||||
a call whose names are not string literals cannot be checked and is a hit ("<dynamic import>"). A file that does not
|
||||
parse is a hit, never a pass. Not seen: code run from a string (exec, eval) and a loader fetched with getattr. Network
|
||||
code lives in tools/, which this test does not scan.
|
||||
|
||||
Wave 2 order 7 adds the names that start a program as well as `subprocess` does: `os.system`, `os.popen`, `os.exec*`,
|
||||
`os.spawn*`, `os.posix_spawn*` and `asyncio.create_subprocess_*` (also under `asyncio.subprocess`, and the os names
|
||||
under `posix` and `nt`, where os takes them from). Each is a hit wherever it is reached: an attribute, called or not, of
|
||||
a name the file binds to the module by an import or by an assignment (`m = __import__('os')`, `o = os`), or of a
|
||||
loader call with a literal name (`__import__('os').system`); a `from ... import` of it; a star import from one of
|
||||
these modules; and `getattr(<module>, '<name>')`, where a name that is not a string literal is a dynamic hit. A local
|
||||
name that only looks the same (`system = len`) is not one. Not seen: `vars(os)[...]`, `os.__dict__[...]`, and a module
|
||||
handed on through a container or a function argument.
|
||||
Run: python3 -m unittest discover -s tests -p 'test_package.py'
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import ast
|
||||
import re
|
||||
import shutil
|
||||
import tempfile
|
||||
import unittest
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Iterator
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
FORBIDDEN = ("evals", "p1", "direction_probe", "anthropic", "openai", "urllib.request", "http.client", "requests",
|
||||
"socket", "subprocess")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Hit:
|
||||
file: str # relative to the scanned package
|
||||
line: int
|
||||
module: str # the forbidden module the import reaches, "<dynamic import>" or "<does not parse>"
|
||||
|
||||
|
||||
def reached(name: str) -> str | None:
|
||||
"""The forbidden module a dotted name is, or lies under; None for any other name."""
|
||||
return next((f for f in FORBIDDEN if name == f or name.startswith(f + ".")), None)
|
||||
|
||||
|
||||
LOADER_MODULES = ("importlib", "builtins")
|
||||
LOADERS = ("import_module", "__import__")
|
||||
DYNAMIC = "<dynamic import>"
|
||||
|
||||
|
||||
def loader_names(tree: ast.AST) -> tuple[dict[str, str], set[str]]:
|
||||
"""(each name this file binds to a loader, mapped to the loader; the names bound to a module that holds one)."""
|
||||
funcs = {"__import__": "__import__"}
|
||||
modules = set(LOADER_MODULES)
|
||||
for node in ast.walk(tree):
|
||||
if isinstance(node, ast.ImportFrom) and node.level == 0 and node.module in LOADER_MODULES:
|
||||
funcs.update({a.asname or a.name: a.name for a in node.names if a.name in LOADERS})
|
||||
elif isinstance(node, ast.Import):
|
||||
modules.update(a.asname or a.name for a in node.names if a.name in LOADER_MODULES)
|
||||
return funcs, modules
|
||||
|
||||
|
||||
def loader_of(func: ast.expr, funcs: dict[str, str], modules: set[str]) -> str | None:
|
||||
"""The loader a call's function is (import_module or __import__), or None."""
|
||||
if isinstance(func, ast.Name):
|
||||
return funcs.get(func.id)
|
||||
if isinstance(func, ast.Attribute) and func.attr == "__import__":
|
||||
return "__import__"
|
||||
if (isinstance(func, ast.Attribute) and func.attr == "import_module" and isinstance(func.value, ast.Name)
|
||||
and func.value.id in modules):
|
||||
return "import_module"
|
||||
return None
|
||||
|
||||
|
||||
def argument(call: ast.Call, position: int, keyword: str) -> ast.expr | None:
|
||||
if len(call.args) > position:
|
||||
return call.args[position]
|
||||
return next((k.value for k in call.keywords if k.arg == keyword), None)
|
||||
|
||||
|
||||
def literal(node: ast.expr | None) -> str | None:
|
||||
return node.value if isinstance(node, ast.Constant) and isinstance(node.value, str) else None
|
||||
|
||||
|
||||
def loaded(call: ast.Call, loader: str) -> list[str] | None:
|
||||
"""The dotted names a loader call imports, or None when they are not string literals."""
|
||||
name = literal(argument(call, 0, "name"))
|
||||
if name is None:
|
||||
return None
|
||||
if loader == "__import__":
|
||||
fromlist = argument(call, 3, "fromlist")
|
||||
if fromlist is None:
|
||||
return [name]
|
||||
items = [literal(e) for e in fromlist.elts] if isinstance(fromlist, (ast.List, ast.Tuple)) else [None]
|
||||
return None if None in items else [name] + [f"{name}.{x}" for x in items]
|
||||
if name.startswith("."):
|
||||
package = literal(argument(call, 1, "package"))
|
||||
up = len(name) - len(name.lstrip(".")) - 1
|
||||
if package is None or up > package.count("."):
|
||||
return None
|
||||
base = package.rsplit(".", up)[0] if up else package
|
||||
return [f"{base}.{name.lstrip('.')}"]
|
||||
return [name]
|
||||
|
||||
|
||||
def imported(tree: ast.AST) -> Iterator[tuple[int, list[str] | None]]:
|
||||
"""(line, the dotted names a statement or call can import, or None when they cannot be read) for every import in
|
||||
the tree, nested ones too."""
|
||||
funcs, modules = loader_names(tree)
|
||||
for node in ast.walk(tree):
|
||||
if isinstance(node, ast.Import):
|
||||
yield node.lineno, [a.name for a in node.names]
|
||||
elif isinstance(node, ast.ImportFrom) and node.level == 0: # a relative import stays inside the package
|
||||
base = node.module or ""
|
||||
yield node.lineno, [base] + [f"{base}.{a.name}" for a in node.names]
|
||||
elif isinstance(node, ast.Call):
|
||||
loader = loader_of(node.func, funcs, modules)
|
||||
if loader is not None:
|
||||
yield node.lineno, loaded(node, loader)
|
||||
|
||||
|
||||
OS_NAMES = re.compile(r"system|popen|exec\w*|spawn\w*|posix_spawn\w*")
|
||||
SUBPROCESS_NAMES = re.compile(r"create_subprocess_\w+")
|
||||
LAUNCHERS = { # a module -> the names under it that start a program (os takes its names from posix or nt)
|
||||
"os": OS_NAMES,
|
||||
"posix": OS_NAMES,
|
||||
"nt": OS_NAMES,
|
||||
"asyncio": SUBPROCESS_NAMES,
|
||||
"asyncio.subprocess": SUBPROCESS_NAMES,
|
||||
}
|
||||
|
||||
|
||||
def launcher(module: str, name: str) -> str | None:
|
||||
"""The dotted name when `name` under `module` starts a program, else None."""
|
||||
pattern = LAUNCHERS.get(module)
|
||||
return f"{module}.{name}" if pattern is not None and pattern.fullmatch(name) else None
|
||||
|
||||
|
||||
class Modules:
|
||||
"""Which module of LAUNCHERS an expression is: a name the file binds to one by an import, or by an assignment from
|
||||
another such expression (m = __import__('os'), o = os), or a loader call with a literal name."""
|
||||
|
||||
def __init__(self, tree: ast.AST) -> None:
|
||||
self.funcs, self.loader_modules = loader_names(tree)
|
||||
self.aliases: dict[str, str] = {}
|
||||
for node in ast.walk(tree):
|
||||
if isinstance(node, ast.Import):
|
||||
for a in node.names:
|
||||
top = a.name.split(".")[0]
|
||||
if a.asname and a.name in LAUNCHERS:
|
||||
self.aliases[a.asname] = a.name
|
||||
elif not a.asname and top in LAUNCHERS: # import os.path binds os
|
||||
self.aliases[top] = top
|
||||
elif isinstance(node, ast.ImportFrom) and node.level == 0 and node.module == "asyncio":
|
||||
self.aliases.update({a.asname or a.name: "asyncio.subprocess" for a in node.names if a.name == "subprocess"})
|
||||
for _ in range(2): # an assignment may name a module bound by another assignment
|
||||
for node in ast.walk(tree):
|
||||
if isinstance(node, ast.Assign) and len(node.targets) == 1 and isinstance(node.targets[0], ast.Name):
|
||||
module = self.module(node.value)
|
||||
if module is not None:
|
||||
self.aliases[node.targets[0].id] = module
|
||||
|
||||
def module(self, node: ast.expr) -> str | None:
|
||||
if isinstance(node, ast.Name):
|
||||
return self.aliases.get(node.id)
|
||||
if not isinstance(node, ast.Call):
|
||||
return None
|
||||
loader = loader_of(node.func, self.funcs, self.loader_modules)
|
||||
names = loaded(node, loader) if loader is not None else None
|
||||
if not names:
|
||||
return None
|
||||
if loader == "__import__" and argument(node, 3, "fromlist") is None:
|
||||
return names[0].split(".")[0] if names[0].split(".")[0] in LAUNCHERS else None # returns the top package
|
||||
return names[0] if names[0] in LAUNCHERS else None
|
||||
|
||||
|
||||
def dotted(node: ast.expr, modules: Modules) -> str | None:
|
||||
"""The dotted name of a chain of attributes on a module of LAUNCHERS (os.path.join -> "os.path.join"), else None."""
|
||||
parts: list[str] = []
|
||||
while isinstance(node, ast.Attribute):
|
||||
parts.append(node.attr)
|
||||
node = node.value
|
||||
base = modules.module(node)
|
||||
return None if base is None else ".".join([base, *reversed(parts)])
|
||||
|
||||
|
||||
def launched(tree: ast.AST) -> Iterator[tuple[int, str]]:
|
||||
"""(line, name) for every place the tree reaches a name that starts a program, or DYNAMIC for a getattr on such a
|
||||
module whose name is not a string literal."""
|
||||
modules = Modules(tree)
|
||||
for node in ast.walk(tree):
|
||||
if isinstance(node, ast.ImportFrom) and node.level == 0 and node.module in LAUNCHERS:
|
||||
for a in node.names:
|
||||
hit = f"{node.module}.*" if a.name == "*" else launcher(node.module, a.name)
|
||||
if hit is not None:
|
||||
yield node.lineno, hit
|
||||
elif isinstance(node, ast.Attribute):
|
||||
name = dotted(node, modules)
|
||||
if name is not None and "." in name:
|
||||
module, attr = name.rsplit(".", 1)
|
||||
hit = launcher(module, attr)
|
||||
if hit is not None:
|
||||
yield node.lineno, hit
|
||||
elif (isinstance(node, ast.Call) and isinstance(node.func, ast.Name) and node.func.id == "getattr"
|
||||
and len(node.args) >= 2):
|
||||
target, wanted = dotted(node.args[0], modules), literal(node.args[1])
|
||||
if target is not None and target in LAUNCHERS:
|
||||
found = DYNAMIC if wanted is None else launcher(target, wanted)
|
||||
if found is not None:
|
||||
yield node.lineno, found
|
||||
|
||||
|
||||
def scan(package: Path) -> tuple[int, list[Hit]]:
|
||||
"""(files read, hits) for every .py file under package."""
|
||||
files = sorted(package.rglob("*.py"))
|
||||
hits: list[Hit] = []
|
||||
for path in files:
|
||||
rel = path.relative_to(package).as_posix()
|
||||
try:
|
||||
tree = ast.parse(path.read_text(encoding="utf-8"), filename=rel)
|
||||
except (SyntaxError, UnicodeDecodeError) as exc:
|
||||
hits.append(Hit(rel, getattr(exc, "lineno", None) or 1, "<does not parse>"))
|
||||
continue
|
||||
for line, names in imported(tree):
|
||||
if names is None:
|
||||
hits.append(Hit(rel, line, DYNAMIC))
|
||||
continue
|
||||
hits.extend(Hit(rel, line, m) for m in sorted({m for m in map(reached, names) if m is not None}))
|
||||
hits.extend(Hit(rel, line, name) for line, name in launched(tree))
|
||||
return len(files), sorted(hits, key=lambda h: (h.file, h.line, h.module))
|
||||
|
||||
|
||||
PLANTED = ( # (code, the module it reaches, the line of the import within the code)
|
||||
("import evals.p1.kpack", "evals", 1),
|
||||
("from p1.split import blocks", "p1", 1),
|
||||
("from evals import direction_probe", "evals", 1),
|
||||
("import direction_probe", "direction_probe", 1),
|
||||
("import anthropic", "anthropic", 1),
|
||||
("from openai import OpenAI", "openai", 1),
|
||||
("import urllib.request", "urllib.request", 1),
|
||||
("from urllib import request", "urllib.request", 1),
|
||||
("from urllib import parse, request", "urllib.request", 1),
|
||||
("from http import client", "http.client", 1),
|
||||
("import http.client as hc", "http.client", 1),
|
||||
("import requests", "requests", 1),
|
||||
("import socket", "socket", 1),
|
||||
("from subprocess import run", "subprocess", 1),
|
||||
("import importlib\nm = importlib.import_module('socket')", "socket", 2),
|
||||
("m = __import__('subprocess')", "subprocess", 1),
|
||||
("def f() -> None:\n import openai", "openai", 2),
|
||||
("import importlib\nm = importlib.__import__('subprocess')", "subprocess", 2),
|
||||
("import builtins\nm = builtins.__import__('socket')", "socket", 2),
|
||||
("import importlib\nm = importlib.import_module(name='socket')", "socket", 2),
|
||||
("from importlib import import_module as load\nm = load('socket')", "socket", 2),
|
||||
("import importlib as il\nm = il.import_module('requests')", "requests", 2),
|
||||
("import importlib\nm = importlib.import_module('.request', 'urllib')", "urllib.request", 2),
|
||||
("m = __import__('urllib', fromlist=['request'])", "urllib.request", 1),
|
||||
("import importlib\nname = 'sock' + 'et'\nm = importlib.import_module(name)", "<dynamic import>", 3),
|
||||
("m = __import__('urllib', fromlist=wanted)", "<dynamic import>", 1),
|
||||
# Wave 2 order 7: names that start a program as well as subprocess does (docs/bolge2/0-felles section 2).
|
||||
("import os\nos.system('ls')", "os.system", 2),
|
||||
("import os\nos.popen('ls')", "os.popen", 2),
|
||||
("import os\nos.execv('/bin/ls', ['ls'])", "os.execv", 2),
|
||||
("import os\nos.execlp('ls', 'ls')", "os.execlp", 2),
|
||||
("import os\nos.spawnv(os.P_WAIT, '/bin/ls', ['ls'])", "os.spawnv", 2),
|
||||
("import os\nos.posix_spawn('/bin/ls', ['ls'], {})", "os.posix_spawn", 2),
|
||||
("import os\nos.posix_spawnp('ls', ['ls'], {})", "os.posix_spawnp", 2),
|
||||
("import os as o\no.system('ls')", "os.system", 2),
|
||||
("import os\nrun = os.system", "os.system", 2),
|
||||
("def f() -> None:\n import os\n os.system('ls')", "os.system", 3),
|
||||
("from os import system\nsystem('ls')", "os.system", 1),
|
||||
("from os import popen as p", "os.popen", 1),
|
||||
("from os import execvp, path", "os.execvp", 1),
|
||||
("from os import *", "os.*", 1),
|
||||
("import os\nf = getattr(os, 'system')", "os.system", 2),
|
||||
("import os\nname = 'sys' + 'tem'\nf = getattr(os, name)", "<dynamic import>", 3),
|
||||
("import asyncio\nasyncio.create_subprocess_exec('ls')", "asyncio.create_subprocess_exec", 2),
|
||||
("import asyncio\nasyncio.create_subprocess_shell('ls')", "asyncio.create_subprocess_shell", 2),
|
||||
("from asyncio import create_subprocess_exec", "asyncio.create_subprocess_exec", 1),
|
||||
("import asyncio.subprocess\nasyncio.subprocess.create_subprocess_exec('ls')",
|
||||
"asyncio.subprocess.create_subprocess_exec", 2),
|
||||
("from asyncio.subprocess import create_subprocess_shell", "asyncio.subprocess.create_subprocess_shell", 1),
|
||||
# After the counter-reading: the module reached through a loader call or a plain assignment, and posix and nt,
|
||||
# the platform modules os takes these names from.
|
||||
("__import__('os').system('ls')", "os.system", 1),
|
||||
("import importlib\nimportlib.import_module('os').system('ls')", "os.system", 2),
|
||||
("m = __import__('os')\nm.popen('ls')", "os.popen", 2),
|
||||
("import os\no = os\no.system('ls')", "os.system", 3),
|
||||
("import posix\nposix.system('ls')", "posix.system", 2),
|
||||
("from nt import spawnv", "nt.spawnv", 1),
|
||||
)
|
||||
HEADER = '"""A planted module."""\n\n' # the planted code starts on file line 3
|
||||
|
||||
ALLOWED = (
|
||||
"from urllib.parse import unquote",
|
||||
"import urllib.parse",
|
||||
"from urllib import parse",
|
||||
"import http",
|
||||
"from http import HTTPStatus",
|
||||
"import evalsx",
|
||||
"import p10",
|
||||
"import requests_plain",
|
||||
"from okfwiki import common",
|
||||
"from . import audit",
|
||||
"import importlib\nm = importlib.import_module('json')",
|
||||
"import importlib\nm = importlib.import_module('.parse', 'urllib')",
|
||||
"m = __import__('urllib', fromlist=['parse'])",
|
||||
"import_module = len\nm = import_module('socket')", # a local name, not importlib's
|
||||
"import os\nx = os.path.join('a', 'b')",
|
||||
"import os\nos.replace('a', 'b')\ny = os.environ.get('HOME')",
|
||||
"from os import environ, path, sep",
|
||||
"import asyncio\nasyncio.run(main())",
|
||||
"system = len\nsystem('ls')", # a local name, not os's
|
||||
"import shutil\nshutil.which('ls')",
|
||||
"import os\nf = getattr(os, 'sep')",
|
||||
"o = len\no.system('ls')", # a local name bound to something else
|
||||
"import os\np = os.path\np.join('a', 'b')",
|
||||
)
|
||||
|
||||
|
||||
class PackageBoundaryTest(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
self._tmp = tempfile.TemporaryDirectory()
|
||||
self.copy = Path(self._tmp.name) / "okfwiki"
|
||||
shutil.copytree(ROOT / "okfwiki", self.copy, ignore=shutil.ignore_patterns("__pycache__"))
|
||||
|
||||
def tearDown(self) -> None:
|
||||
self._tmp.cleanup()
|
||||
|
||||
def plant(self, code: str) -> list[Hit]:
|
||||
path = self.copy / "planted.py"
|
||||
path.write_text(HEADER + code + "\n", encoding="utf-8")
|
||||
_, hits = scan(self.copy)
|
||||
path.unlink()
|
||||
return hits
|
||||
|
||||
def test_no_module_under_okfwiki_imports_a_forbidden_module(self) -> None:
|
||||
files, hits = scan(ROOT / "okfwiki")
|
||||
self.assertEqual(files, len(list((ROOT / "okfwiki").rglob("*.py"))))
|
||||
self.assertGreater(files, 0, "a scan that reads no file cannot pass")
|
||||
self.assertEqual(hits, [])
|
||||
|
||||
def test_a_planted_import_in_a_temporary_copy_is_flagged(self) -> None:
|
||||
for code, module, line in PLANTED:
|
||||
with self.subTest(code=code):
|
||||
self.assertEqual(self.plant(code), [Hit("planted.py", 2 + line, module)])
|
||||
|
||||
def test_allowed_modules_and_lookalike_names_are_not_flagged(self) -> None:
|
||||
for code in ALLOWED:
|
||||
with self.subTest(code=code):
|
||||
self.assertEqual(self.plant(code), [])
|
||||
|
||||
def test_a_file_that_does_not_parse_is_a_hit_not_a_pass(self) -> None:
|
||||
self.assertEqual(self.plant("def broken(:\n pass"), [Hit("planted.py", 3, "<does not parse>")])
|
||||
|
||||
def test_files_in_subpackages_are_read(self) -> None:
|
||||
(self.copy / "sub").mkdir()
|
||||
(self.copy / "sub" / "deep.py").write_text("import subprocess\n", encoding="utf-8")
|
||||
files, hits = scan(self.copy)
|
||||
self.assertEqual(files, len(list(self.copy.rglob("*.py"))))
|
||||
self.assertEqual(hits, [Hit("sub/deep.py", 1, "subprocess")])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Loading…
Add table
Add a link
Reference in a new issue