test(scripts): term-list hygiene check, term list kept outside the tree

Adds scripts/check-excluded-topics.py: scans every tracked path and text
file case-insensitively against a term pattern and fails on any hit. The
term list is read from <git-common-dir>/info/excluded-topics, which git
never tracks and a clone does not carry; without it the check prints
SKIPPED and exits 3. A self-test proves the pattern can find before a
clean scan is ever reported (exit 2 otherwise).

Red on this commit: exit 1, 180 hits in 20 files, 68 tracked paths.
Without the term list: exit 3, SKIPPED.

Valgt git-katalogens info/ (ikke en gitignored fil i arbeidstreet) fordi
den deles av alle worktrees i klonen, overlever at en worktree slettes og
ikke kan spores ved et uhell. Valgt exit 3 for manglende liste fordi et
søk uten mønster ikke har målt noe og ikke skal lese som grønt.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Kjell Tore Guttormsen 2026-09-23 15:11:39 +02:00
commit 6c4e863608
Signed by: ktg
SSH key fingerprint: SHA256:JakMjO6FTBBzN0Bhfj9saOoEjaFxlSdYuZQQpM/lF9Q

View file

@ -0,0 +1,132 @@
#!/usr/bin/env python3
"""Fail if any tracked text file, or any tracked path, matches a locally kept term list.
This check makes a content rule a gate instead of a promise: it scans every file
`git ls-files` reports under the repository root, skips binary files (a NUL byte), and
matches the term pattern case-insensitively against both the path and the content.
The term list is not part of the tree. It lives in the repository's git directory, at
`<git-common-dir>/info/excluded-topics`, next to `info/exclude`: git never tracks that
directory, every worktree of the clone shares it, and a clone does not carry it. Without
the file the check reports SKIPPED and exits 3, because a scan without a pattern measured
nothing.
File format, one entry per line; blank lines and lines starting with `#` are ignored:
+ <sample> known-positive: the pattern must match it
- <sample> known-negative: the pattern must not match it
<regex> one alternative of the pattern (all alternatives are OR-ed)
Before scanning, the check proves the pattern can find: there must be at least one
known-positive, every known-positive must match and every known-negative must not. If
that self-test fails, the check exits 2 and a clean scan is never reported, because
"found nothing" from a pattern that cannot find is not a measurement.
Exit status: 0 clean, 1 hits found, 2 self-test or git failure, 3 term list absent.
Usage: python3 scripts/check-excluded-topics.py
"""
import re
import subprocess
import sys
from pathlib import Path
TERMS_RELATIVE_TO_GIT_COMMON_DIR = Path("info") / "excluded-topics"
def git(root: Path, *args: str) -> bytes:
return subprocess.run(
["git", *args], cwd=root, check=True, capture_output=True
).stdout
def read_terms(path: Path) -> tuple:
alternatives, positives, negatives = [], [], []
for line in path.read_text(encoding="utf-8").splitlines():
entry = line.strip()
if not entry or entry.startswith("#"):
continue
if entry.startswith("+ "):
positives.append(entry[2:])
elif entry.startswith("- "):
negatives.append(entry[2:])
else:
alternatives.append(entry)
return alternatives, positives, negatives
def self_test(pattern: re.Pattern, positives: list, negatives: list) -> list:
failures = []
if not positives:
failures.append("no known-positive in the term list")
for sample in positives:
if not pattern.search(sample):
failures.append(f"known-positive not matched: {sample!r}")
for sample in negatives:
if pattern.search(sample):
failures.append(f"known-negative matched: {sample!r}")
return failures
def main() -> int:
root = Path(__file__).resolve().parent.parent
try:
common_dir = git(root, "rev-parse", "--git-common-dir").decode("utf-8").strip()
listing = git(root, "ls-files", "-z")
except (OSError, subprocess.CalledProcessError) as error:
print(f"git failed: {error}", file=sys.stderr)
return 2
terms_path = (root / common_dir / TERMS_RELATIVE_TO_GIT_COMMON_DIR).resolve()
if not terms_path.is_file():
print(f"SKIPPED: no term list at {terms_path} -- nothing was measured")
return 3
alternatives, positives, negatives = read_terms(terms_path)
if not alternatives:
print(f"SELF-TEST FAIL: no pattern line in {terms_path}", file=sys.stderr)
return 2
try:
pattern = re.compile("|".join(alternatives), re.IGNORECASE)
except re.error as error:
print(f"SELF-TEST FAIL: pattern does not compile: {error}", file=sys.stderr)
return 2
failures = self_test(pattern, positives, negatives)
if failures:
for failure in failures:
print(f"SELF-TEST FAIL: {failure}", file=sys.stderr)
return 2
paths = [p for p in listing.decode("utf-8").split("\0") if p]
hits = []
text_files = 0
binary_files = 0
for rel in paths:
if pattern.search(rel):
hits.append(f"{rel}: (path)")
data = (root / rel).read_bytes()
if b"\0" in data:
binary_files += 1
continue
text_files += 1
for number, line in enumerate(data.decode("utf-8", errors="replace").splitlines(), 1):
if pattern.search(line):
hits.append(f"{rel}:{number}")
print(
f"scanned {len(paths)} tracked paths: {text_files} text, "
f"{binary_files} binary skipped; self-test passed "
f"({len(positives)} known-positive, {len(negatives)} known-negative)"
)
if hits:
files = sorted({h.split(":", 1)[0] for h in hits})
print(f"FAIL: {len(hits)} hits in {len(files)} files")
for hit in hits:
print(f" {hit}")
return 1
print("PASS: 0 hits")
return 0
if __name__ == "__main__":
sys.exit(main())