portfolio-optimiser-commons/scripts/check-excluded-topics.py

#!/usr/bin/env python3
"""Fail if any tracked text file, or any tracked path, matches a locally kept term list.

This check makes a content rule a gate instead of a promise: it scans every file
`git ls-files` reports under the repository root, skips binary files (a NUL byte), and
matches the term pattern case-insensitively against both the path and the content.

The term list is not part of the tree. It lives in the repository's git directory, at
`<git-common-dir>/info/excluded-topics`, next to `info/exclude`: git never tracks that
directory, every worktree of the clone shares it, and a clone does not carry it. Without
the file the check reports SKIPPED and exits 3, because a scan without a pattern measured
nothing.

File format, one entry per line; blank lines and lines starting with `#` are ignored:

    + <sample>    known-positive: the pattern must match it
    - <sample>    known-negative: the pattern must not match it
    <regex>       one alternative of the pattern (all alternatives are OR-ed)

Before scanning, the check proves the pattern can find: there must be at least one
known-positive, every known-positive must match and every known-negative must not. If
that self-test fails, the check exits 2 and a clean scan is never reported, because
"found nothing" from a pattern that cannot find is not a measurement.

Exit status: 0 clean, 1 hits found, 2 self-test or git failure, 3 term list absent.
Usage: python3 scripts/check-excluded-topics.py
"""

import re
import subprocess
import sys
from pathlib import Path

TERMS_RELATIVE_TO_GIT_COMMON_DIR = Path("info") / "excluded-topics"


def git(root: Path, *args: str) -> bytes:
    return subprocess.run(
        ["git", *args], cwd=root, check=True, capture_output=True
    ).stdout


def read_terms(path: Path) -> tuple:
    alternatives, positives, negatives = [], [], []
    for line in path.read_text(encoding="utf-8").splitlines():
        entry = line.strip()
        if not entry or entry.startswith("#"):
            continue
        if entry.startswith("+ "):
            positives.append(entry[2:])
        elif entry.startswith("- "):
            negatives.append(entry[2:])
        else:
            alternatives.append(entry)
    return alternatives, positives, negatives


def self_test(pattern: re.Pattern, positives: list, negatives: list) -> list:
    failures = []
    if not positives:
        failures.append("no known-positive in the term list")
    for sample in positives:
        if not pattern.search(sample):
            failures.append(f"known-positive not matched: {sample!r}")
    for sample in negatives:
        if pattern.search(sample):
            failures.append(f"known-negative matched: {sample!r}")
    return failures


def main() -> int:
    root = Path(__file__).resolve().parent.parent
    try:
        common_dir = git(root, "rev-parse", "--git-common-dir").decode("utf-8").strip()
        listing = git(root, "ls-files", "-z")
    except (OSError, subprocess.CalledProcessError) as error:
        print(f"git failed: {error}", file=sys.stderr)
        return 2

    terms_path = (root / common_dir / TERMS_RELATIVE_TO_GIT_COMMON_DIR).resolve()
    if not terms_path.is_file():
        print(f"SKIPPED: no term list at {terms_path} -- nothing was measured")
        return 3

    alternatives, positives, negatives = read_terms(terms_path)
    if not alternatives:
        print(f"SELF-TEST FAIL: no pattern line in {terms_path}", file=sys.stderr)
        return 2
    try:
        pattern = re.compile("|".join(alternatives), re.IGNORECASE)
    except re.error as error:
        print(f"SELF-TEST FAIL: pattern does not compile: {error}", file=sys.stderr)
        return 2
    failures = self_test(pattern, positives, negatives)
    if failures:
        for failure in failures:
            print(f"SELF-TEST FAIL: {failure}", file=sys.stderr)
        return 2

    paths = [p for p in listing.decode("utf-8").split("\0") if p]
    hits = []
    text_files = 0
    binary_files = 0
    for rel in paths:
        if pattern.search(rel):
            hits.append(f"{rel}: (path)")
        data = (root / rel).read_bytes()
        if b"\0" in data:
            binary_files += 1
            continue
        text_files += 1
        for number, line in enumerate(data.decode("utf-8", errors="replace").splitlines(), 1):
            if pattern.search(line):
                hits.append(f"{rel}:{number}")

    print(
        f"scanned {len(paths)} tracked paths: {text_files} text, "
        f"{binary_files} binary skipped; self-test passed "
        f"({len(positives)} known-positive, {len(negatives)} known-negative)"
    )
    if hits:
        files = sorted({h.split(":", 1)[0] for h in hits})
        print(f"FAIL: {len(hits)} hits in {len(files)} files")
        for hit in hits:
            print(f"  {hit}")
        return 1
    print("PASS: 0 hits")
    return 0


if __name__ == "__main__":
    sys.exit(main())