okf-wiki/tests/test_package.py
Kjell Tore Guttormsen 0fd86a23a3
okf-wiki 1.1.1
First public release, published as one snapshot. What changed between
versions is in CHANGELOG.md.

Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
2026-11-14 21:03:20 +01:00

372 lines
18 KiB
Python

"""The package boundary (order 5 of wave 1; rev 4 Step 13): no module under okfwiki/ imports the trial or a network
or model client.
The scanner reads every .py file under okfwiki/ with ast and lists each import that reaches a forbidden module, by
exact module name: the trial (`evals`, `p1`, `direction_probe`) and the modules `docs/bolge1/0-felles` section 2 bars
(`anthropic`, `openai`, `urllib.request`, `http.client`, `requests`, `socket`, `subprocess`). `urllib.parse` is allowed.
A name reaches a forbidden module when it is that module or a submodule of it, so `from urllib import request` reaches
`urllib.request`. A call to `importlib.import_module` or to `__import__` counts as an import, under any name the file
binds them to, with its name given by position or keyword, relative to its package, and with the names in `fromlist`;
a call whose names are not string literals cannot be checked and is a hit ("<dynamic import>"). A file that does not
parse is a hit, never a pass. Not seen: code run from a string (exec, eval) and a loader fetched with getattr. Network
code lives in tools/, which this test does not scan.
Wave 2 order 7 adds the names that start a program as well as `subprocess` does: `os.system`, `os.popen`, `os.exec*`,
`os.spawn*`, `os.posix_spawn*` and `asyncio.create_subprocess_*` (also under `asyncio.subprocess`, and the os names
under `posix` and `nt`, where os takes them from). Each is a hit wherever it is reached: an attribute, called or not, of
a name the file binds to the module by an import or by an assignment (`m = __import__('os')`, `o = os`), or of a
loader call with a literal name (`__import__('os').system`); a `from ... import` of it; a star import from one of
these modules; and `getattr(<module>, '<name>')`, where a name that is not a string literal is a dynamic hit. A local
name that only looks the same (`system = len`) is not one. Not seen: `vars(os)[...]`, `os.__dict__[...]`, and a module
handed on through a container or a function argument.
Run: python3 -m unittest discover -s tests -p 'test_package.py'
"""
from __future__ import annotations
import ast
import re
import shutil
import tempfile
import unittest
from dataclasses import dataclass
from pathlib import Path
from typing import Iterator
ROOT = Path(__file__).resolve().parents[1]
FORBIDDEN = ("evals", "p1", "direction_probe", "anthropic", "openai", "urllib.request", "http.client", "requests",
"socket", "subprocess")
@dataclass(frozen=True)
class Hit:
file: str # relative to the scanned package
line: int
module: str # the forbidden module the import reaches, "<dynamic import>" or "<does not parse>"
def reached(name: str) -> str | None:
"""The forbidden module a dotted name is, or lies under; None for any other name."""
return next((f for f in FORBIDDEN if name == f or name.startswith(f + ".")), None)
LOADER_MODULES = ("importlib", "builtins")
LOADERS = ("import_module", "__import__")
DYNAMIC = "<dynamic import>"
def loader_names(tree: ast.AST) -> tuple[dict[str, str], set[str]]:
"""(each name this file binds to a loader, mapped to the loader; the names bound to a module that holds one)."""
funcs = {"__import__": "__import__"}
modules = set(LOADER_MODULES)
for node in ast.walk(tree):
if isinstance(node, ast.ImportFrom) and node.level == 0 and node.module in LOADER_MODULES:
funcs.update({a.asname or a.name: a.name for a in node.names if a.name in LOADERS})
elif isinstance(node, ast.Import):
modules.update(a.asname or a.name for a in node.names if a.name in LOADER_MODULES)
return funcs, modules
def loader_of(func: ast.expr, funcs: dict[str, str], modules: set[str]) -> str | None:
"""The loader a call's function is (import_module or __import__), or None."""
if isinstance(func, ast.Name):
return funcs.get(func.id)
if isinstance(func, ast.Attribute) and func.attr == "__import__":
return "__import__"
if (isinstance(func, ast.Attribute) and func.attr == "import_module" and isinstance(func.value, ast.Name)
and func.value.id in modules):
return "import_module"
return None
def argument(call: ast.Call, position: int, keyword: str) -> ast.expr | None:
if len(call.args) > position:
return call.args[position]
return next((k.value for k in call.keywords if k.arg == keyword), None)
def literal(node: ast.expr | None) -> str | None:
return node.value if isinstance(node, ast.Constant) and isinstance(node.value, str) else None
def loaded(call: ast.Call, loader: str) -> list[str] | None:
"""The dotted names a loader call imports, or None when they are not string literals."""
name = literal(argument(call, 0, "name"))
if name is None:
return None
if loader == "__import__":
fromlist = argument(call, 3, "fromlist")
if fromlist is None:
return [name]
items = [literal(e) for e in fromlist.elts] if isinstance(fromlist, (ast.List, ast.Tuple)) else [None]
return None if None in items else [name] + [f"{name}.{x}" for x in items]
if name.startswith("."):
package = literal(argument(call, 1, "package"))
up = len(name) - len(name.lstrip(".")) - 1
if package is None or up > package.count("."):
return None
base = package.rsplit(".", up)[0] if up else package
return [f"{base}.{name.lstrip('.')}"]
return [name]
def imported(tree: ast.AST) -> Iterator[tuple[int, list[str] | None]]:
"""(line, the dotted names a statement or call can import, or None when they cannot be read) for every import in
the tree, nested ones too."""
funcs, modules = loader_names(tree)
for node in ast.walk(tree):
if isinstance(node, ast.Import):
yield node.lineno, [a.name for a in node.names]
elif isinstance(node, ast.ImportFrom) and node.level == 0: # a relative import stays inside the package
base = node.module or ""
yield node.lineno, [base] + [f"{base}.{a.name}" for a in node.names]
elif isinstance(node, ast.Call):
loader = loader_of(node.func, funcs, modules)
if loader is not None:
yield node.lineno, loaded(node, loader)
OS_NAMES = re.compile(r"system|popen|exec\w*|spawn\w*|posix_spawn\w*")
SUBPROCESS_NAMES = re.compile(r"create_subprocess_\w+")
LAUNCHERS = { # a module -> the names under it that start a program (os takes its names from posix or nt)
"os": OS_NAMES,
"posix": OS_NAMES,
"nt": OS_NAMES,
"asyncio": SUBPROCESS_NAMES,
"asyncio.subprocess": SUBPROCESS_NAMES,
}
def launcher(module: str, name: str) -> str | None:
"""The dotted name when `name` under `module` starts a program, else None."""
pattern = LAUNCHERS.get(module)
return f"{module}.{name}" if pattern is not None and pattern.fullmatch(name) else None
class Modules:
"""Which module of LAUNCHERS an expression is: a name the file binds to one by an import, or by an assignment from
another such expression (m = __import__('os'), o = os), or a loader call with a literal name."""
def __init__(self, tree: ast.AST) -> None:
self.funcs, self.loader_modules = loader_names(tree)
self.aliases: dict[str, str] = {}
for node in ast.walk(tree):
if isinstance(node, ast.Import):
for a in node.names:
top = a.name.split(".")[0]
if a.asname and a.name in LAUNCHERS:
self.aliases[a.asname] = a.name
elif not a.asname and top in LAUNCHERS: # import os.path binds os
self.aliases[top] = top
elif isinstance(node, ast.ImportFrom) and node.level == 0 and node.module == "asyncio":
self.aliases.update({a.asname or a.name: "asyncio.subprocess" for a in node.names if a.name == "subprocess"})
for _ in range(2): # an assignment may name a module bound by another assignment
for node in ast.walk(tree):
if isinstance(node, ast.Assign) and len(node.targets) == 1 and isinstance(node.targets[0], ast.Name):
module = self.module(node.value)
if module is not None:
self.aliases[node.targets[0].id] = module
def module(self, node: ast.expr) -> str | None:
if isinstance(node, ast.Name):
return self.aliases.get(node.id)
if not isinstance(node, ast.Call):
return None
loader = loader_of(node.func, self.funcs, self.loader_modules)
names = loaded(node, loader) if loader is not None else None
if not names:
return None
if loader == "__import__" and argument(node, 3, "fromlist") is None:
return names[0].split(".")[0] if names[0].split(".")[0] in LAUNCHERS else None # returns the top package
return names[0] if names[0] in LAUNCHERS else None
def dotted(node: ast.expr, modules: Modules) -> str | None:
"""The dotted name of a chain of attributes on a module of LAUNCHERS (os.path.join -> "os.path.join"), else None."""
parts: list[str] = []
while isinstance(node, ast.Attribute):
parts.append(node.attr)
node = node.value
base = modules.module(node)
return None if base is None else ".".join([base, *reversed(parts)])
def launched(tree: ast.AST) -> Iterator[tuple[int, str]]:
"""(line, name) for every place the tree reaches a name that starts a program, or DYNAMIC for a getattr on such a
module whose name is not a string literal."""
modules = Modules(tree)
for node in ast.walk(tree):
if isinstance(node, ast.ImportFrom) and node.level == 0 and node.module in LAUNCHERS:
for a in node.names:
hit = f"{node.module}.*" if a.name == "*" else launcher(node.module, a.name)
if hit is not None:
yield node.lineno, hit
elif isinstance(node, ast.Attribute):
name = dotted(node, modules)
if name is not None and "." in name:
module, attr = name.rsplit(".", 1)
hit = launcher(module, attr)
if hit is not None:
yield node.lineno, hit
elif (isinstance(node, ast.Call) and isinstance(node.func, ast.Name) and node.func.id == "getattr"
and len(node.args) >= 2):
target, wanted = dotted(node.args[0], modules), literal(node.args[1])
if target is not None and target in LAUNCHERS:
found = DYNAMIC if wanted is None else launcher(target, wanted)
if found is not None:
yield node.lineno, found
def scan(package: Path) -> tuple[int, list[Hit]]:
"""(files read, hits) for every .py file under package."""
files = sorted(package.rglob("*.py"))
hits: list[Hit] = []
for path in files:
rel = path.relative_to(package).as_posix()
try:
tree = ast.parse(path.read_text(encoding="utf-8"), filename=rel)
except (SyntaxError, UnicodeDecodeError) as exc:
hits.append(Hit(rel, getattr(exc, "lineno", None) or 1, "<does not parse>"))
continue
for line, names in imported(tree):
if names is None:
hits.append(Hit(rel, line, DYNAMIC))
continue
hits.extend(Hit(rel, line, m) for m in sorted({m for m in map(reached, names) if m is not None}))
hits.extend(Hit(rel, line, name) for line, name in launched(tree))
return len(files), sorted(hits, key=lambda h: (h.file, h.line, h.module))
PLANTED = ( # (code, the module it reaches, the line of the import within the code)
("import evals.p1.kpack", "evals", 1),
("from p1.split import blocks", "p1", 1),
("from evals import direction_probe", "evals", 1),
("import direction_probe", "direction_probe", 1),
("import anthropic", "anthropic", 1),
("from openai import OpenAI", "openai", 1),
("import urllib.request", "urllib.request", 1),
("from urllib import request", "urllib.request", 1),
("from urllib import parse, request", "urllib.request", 1),
("from http import client", "http.client", 1),
("import http.client as hc", "http.client", 1),
("import requests", "requests", 1),
("import socket", "socket", 1),
("from subprocess import run", "subprocess", 1),
("import importlib\nm = importlib.import_module('socket')", "socket", 2),
("m = __import__('subprocess')", "subprocess", 1),
("def f() -> None:\n import openai", "openai", 2),
("import importlib\nm = importlib.__import__('subprocess')", "subprocess", 2),
("import builtins\nm = builtins.__import__('socket')", "socket", 2),
("import importlib\nm = importlib.import_module(name='socket')", "socket", 2),
("from importlib import import_module as load\nm = load('socket')", "socket", 2),
("import importlib as il\nm = il.import_module('requests')", "requests", 2),
("import importlib\nm = importlib.import_module('.request', 'urllib')", "urllib.request", 2),
("m = __import__('urllib', fromlist=['request'])", "urllib.request", 1),
("import importlib\nname = 'sock' + 'et'\nm = importlib.import_module(name)", "<dynamic import>", 3),
("m = __import__('urllib', fromlist=wanted)", "<dynamic import>", 1),
# Wave 2 order 7: names that start a program as well as subprocess does (docs/bolge2/0-felles section 2).
("import os\nos.system('ls')", "os.system", 2),
("import os\nos.popen('ls')", "os.popen", 2),
("import os\nos.execv('/bin/ls', ['ls'])", "os.execv", 2),
("import os\nos.execlp('ls', 'ls')", "os.execlp", 2),
("import os\nos.spawnv(os.P_WAIT, '/bin/ls', ['ls'])", "os.spawnv", 2),
("import os\nos.posix_spawn('/bin/ls', ['ls'], {})", "os.posix_spawn", 2),
("import os\nos.posix_spawnp('ls', ['ls'], {})", "os.posix_spawnp", 2),
("import os as o\no.system('ls')", "os.system", 2),
("import os\nrun = os.system", "os.system", 2),
("def f() -> None:\n import os\n os.system('ls')", "os.system", 3),
("from os import system\nsystem('ls')", "os.system", 1),
("from os import popen as p", "os.popen", 1),
("from os import execvp, path", "os.execvp", 1),
("from os import *", "os.*", 1),
("import os\nf = getattr(os, 'system')", "os.system", 2),
("import os\nname = 'sys' + 'tem'\nf = getattr(os, name)", "<dynamic import>", 3),
("import asyncio\nasyncio.create_subprocess_exec('ls')", "asyncio.create_subprocess_exec", 2),
("import asyncio\nasyncio.create_subprocess_shell('ls')", "asyncio.create_subprocess_shell", 2),
("from asyncio import create_subprocess_exec", "asyncio.create_subprocess_exec", 1),
("import asyncio.subprocess\nasyncio.subprocess.create_subprocess_exec('ls')",
"asyncio.subprocess.create_subprocess_exec", 2),
("from asyncio.subprocess import create_subprocess_shell", "asyncio.subprocess.create_subprocess_shell", 1),
# After the counter-reading: the module reached through a loader call or a plain assignment, and posix and nt,
# the platform modules os takes these names from.
("__import__('os').system('ls')", "os.system", 1),
("import importlib\nimportlib.import_module('os').system('ls')", "os.system", 2),
("m = __import__('os')\nm.popen('ls')", "os.popen", 2),
("import os\no = os\no.system('ls')", "os.system", 3),
("import posix\nposix.system('ls')", "posix.system", 2),
("from nt import spawnv", "nt.spawnv", 1),
)
HEADER = '"""A planted module."""\n\n' # the planted code starts on file line 3
ALLOWED = (
"from urllib.parse import unquote",
"import urllib.parse",
"from urllib import parse",
"import http",
"from http import HTTPStatus",
"import evalsx",
"import p10",
"import requests_plain",
"from okfwiki import common",
"from . import audit",
"import importlib\nm = importlib.import_module('json')",
"import importlib\nm = importlib.import_module('.parse', 'urllib')",
"m = __import__('urllib', fromlist=['parse'])",
"import_module = len\nm = import_module('socket')", # a local name, not importlib's
"import os\nx = os.path.join('a', 'b')",
"import os\nos.replace('a', 'b')\ny = os.environ.get('HOME')",
"from os import environ, path, sep",
"import asyncio\nasyncio.run(main())",
"system = len\nsystem('ls')", # a local name, not os's
"import shutil\nshutil.which('ls')",
"import os\nf = getattr(os, 'sep')",
"o = len\no.system('ls')", # a local name bound to something else
"import os\np = os.path\np.join('a', 'b')",
)
class PackageBoundaryTest(unittest.TestCase):
def setUp(self) -> None:
self._tmp = tempfile.TemporaryDirectory()
self.copy = Path(self._tmp.name) / "okfwiki"
shutil.copytree(ROOT / "okfwiki", self.copy, ignore=shutil.ignore_patterns("__pycache__"))
def tearDown(self) -> None:
self._tmp.cleanup()
def plant(self, code: str) -> list[Hit]:
path = self.copy / "planted.py"
path.write_text(HEADER + code + "\n", encoding="utf-8")
_, hits = scan(self.copy)
path.unlink()
return hits
def test_no_module_under_okfwiki_imports_a_forbidden_module(self) -> None:
files, hits = scan(ROOT / "okfwiki")
self.assertEqual(files, len(list((ROOT / "okfwiki").rglob("*.py"))))
self.assertGreater(files, 0, "a scan that reads no file cannot pass")
self.assertEqual(hits, [])
def test_a_planted_import_in_a_temporary_copy_is_flagged(self) -> None:
for code, module, line in PLANTED:
with self.subTest(code=code):
self.assertEqual(self.plant(code), [Hit("planted.py", 2 + line, module)])
def test_allowed_modules_and_lookalike_names_are_not_flagged(self) -> None:
for code in ALLOWED:
with self.subTest(code=code):
self.assertEqual(self.plant(code), [])
def test_a_file_that_does_not_parse_is_a_hit_not_a_pass(self) -> None:
self.assertEqual(self.plant("def broken(:\n pass"), [Hit("planted.py", 3, "<does not parse>")])
def test_files_in_subpackages_are_read(self) -> None:
(self.copy / "sub").mkdir()
(self.copy / "sub" / "deep.py").write_text("import subprocess\n", encoding="utf-8")
files, hits = scan(self.copy)
self.assertEqual(files, len(list(self.copy.rglob("*.py"))))
self.assertEqual(hits, [Hit("sub/deep.py", 1, "subprocess")])
if __name__ == "__main__":
unittest.main()