test(repo): no tracked file carries a term from the local exclusion list -- red

Reads every file git ls-files reports and fails on any line matching a
pattern from the gitignored tests/excluded-terms.local.md. The list itself
stays local; where it is absent the gate skips and says why.

Red on this tree: 185 tracked files hit (1 failed, 4 passed).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/test_excluded_terms.py

@@ -0,0 +1,161 @@
+"""No tracked file carries a term from the repository's local exclusion list.
+
+The example material was replaced by one generic, fictitious example set (see
+``src/portfolio_optimiser/data/`` and ``contexts/``). This gate keeps the retired material from
+coming back: it reads every file ``git ls-files`` reports and fails on any line matching a pattern
+from ``tests/excluded-terms.local.md``.
+
+**The term list is deliberately NOT tracked.** It lives in a gitignored local file, so the
+published tree carries the mechanism and never the list. Where the file is absent (a fresh clone,
+the handover archive) the gate SKIPS and says why; it is measured red -> green with the file in
+place. ``test_the_term_list_is_never_tracked`` holds the other half: the list stays local.
+
+The same file carries each pattern's known-positive and a set of known-negatives, so a pattern
+that cannot fire, or one that fires on ordinary prose, is red rather than silently useless.
+
+**One named, tested exception:** ``shared/``, the ``git subtree`` of
+``portfolio-optimiser-commons``. Its sync is pull-only: content changes there are committed in
+commons and pulled here, never edited here. The exception is itself gated — once no file under
+the prefix hits, the exception is dead and a test says to remove it.
+
+Every scan carries its denominator: a gate that read zero files would be green and prove nothing.
+"""
+
+from __future__ import annotations
+
+import re
+import subprocess
+from pathlib import Path
+
+import pytest
+
+_REPO_ROOT = Path(__file__).resolve().parents[1]
+TERMS_FILE = _REPO_ROOT / "tests" / "excluded-terms.local.md"
+
+#: The subtree prefix whose content is owned by another repository (see the module docstring).
+SUBTREE_PREFIX = "shared/"
+
+
+def load_terms(
+    path: Path = TERMS_FILE,
+) -> tuple[list[tuple[str, re.Pattern[str]]], list[tuple[str, str]], list[str]]:
+    """(patterns, known-positives, known-negatives) from the local term file."""
+    patterns: list[tuple[str, re.Pattern[str]]] = []
+    positives: list[tuple[str, str]] = []
+    negatives: list[str] = []
+    for raw in path.read_text(encoding="utf-8").splitlines():
+        if not raw.strip() or raw.startswith("#"):
+            continue
+        kind, _, rest = raw.partition("\t")
+        if kind == "P":
+            label, _, regex = rest.partition("\t")
+            patterns.append((label, re.compile(regex, re.IGNORECASE)))
+        elif kind == "K":
+            label, _, sample = rest.partition("\t")
+            positives.append((label, sample))
+        elif kind == "N":
+            negatives.append(rest)
+        else:
+            raise ValueError(f"unknown entry kind {kind!r} in {path.name}: {raw!r}")
+    return patterns, positives, negatives
+
+
+@pytest.fixture(scope="module")
+def terms() -> tuple[list[tuple[str, re.Pattern[str]]], list[tuple[str, str]], list[str]]:
+    if not TERMS_FILE.is_file():
+        pytest.skip(f"local term list {TERMS_FILE.name} is not present (local-only by design)")
+    loaded = load_terms()
+    assert loaded[0], f"{TERMS_FILE.name} declares no patterns, so nothing would be checked"
+    return loaded
+
+
+def tracked_files() -> list[str]:
+    done = subprocess.run(
+        ["git", "ls-files", "-z"], cwd=_REPO_ROOT, capture_output=True, check=True
+    )
+    return [name for name in done.stdout.decode("utf-8").split("\0") if name]
+
+
+def hits_in(text: str, patterns: list[tuple[str, re.Pattern[str]]]) -> list[tuple[str, int, str]]:
+    """(pattern label, 1-based line number, line) for every matching line."""
+    found = []
+    for number, line in enumerate(text.splitlines(), start=1):
+        for label, pattern in patterns:
+            if pattern.search(line):
+                found.append((label, number, line.strip()[:160]))
+    return found
+
+
+def scan(
+    names: list[str], patterns: list[tuple[str, re.Pattern[str]]]
+) -> dict[str, list[tuple[str, int, str]]]:
+    """Every file with at least one hit. Binary files are read too: a string inside a fixture
+    database is as tracked as a line of prose, so bytes are decoded leniently, never skipped."""
+    out = {}
+    for name in names:
+        path = _REPO_ROOT / name
+        if not path.is_file():  # a deletion staged but not yet committed
+            continue
+        found = hits_in(path.read_bytes().decode("utf-8", errors="replace"), patterns)
+        if found:
+            out[name] = found
+    return out
+
+
+def test_no_tracked_file_carries_an_excluded_term(terms) -> None:
+    patterns = terms[0]
+    names = tracked_files()
+    assert len(names) > 100, f"git ls-files listed {len(names)} files -- too few to be this repo"
+    offending = {
+        name: hits
+        for name, hits in scan(names, patterns).items()
+        if not name.startswith(SUBTREE_PREFIX)
+    }
+    report = "\n".join(
+        f"{name}:{number}: [{label}] {line}"
+        for name, hits in sorted(offending.items())
+        for label, number, line in hits[:5]
+    )
+    assert not offending, f"{len(offending)} tracked file(s) carry excluded terms:\n{report}"
+
+
+def test_every_pattern_fires_on_its_known_positive(terms) -> None:
+    patterns, positives, _ = terms
+    labels = {label for label, _ in patterns}
+    assert labels <= {label for label, _ in positives}, "every pattern needs a known-positive"
+    for label, sample in positives:
+        assert label in {hit[0] for hit in hits_in(sample, patterns)}, (
+            f"pattern {label!r} missed its known-positive {sample!r}"
+        )
+
+
+def test_ordinary_prose_does_not_fire(terms) -> None:
+    patterns, _, negatives = terms
+    assert negatives, "the term list carries no known-negatives, so over-matching is unchecked"
+    for sample in negatives:
+        assert hits_in(sample, patterns) == [], f"the gate fired on ordinary prose: {sample!r}"
+
+
+def test_the_term_list_is_never_tracked() -> None:
+    """The list must stay local: tracked, it would publish exactly what the gate keeps out."""
+    rel = TERMS_FILE.relative_to(_REPO_ROOT).as_posix()
+    assert rel not in tracked_files(), f"{rel} is tracked -- it must stay local-only"
+    ignored = subprocess.run(["git", "check-ignore", "-q", rel], cwd=_REPO_ROOT)
+    assert ignored.returncode == 0, f"{rel} is not covered by .gitignore"
+
+
+def test_the_subtree_exception_is_still_needed_and_is_the_subtree(terms) -> None:
+    """The ``shared/`` exception is alive only while the subtree still carries excluded terms.
+
+    Both halves are the claim: the prefix is the subtree (its README says so), and at least one
+    file under it still hits. When the cleanup has been pulled from commons, the second half goes
+    red and the exception must be deleted here rather than linger as a hole nobody remembers.
+    """
+    readme = (_REPO_ROOT / SUBTREE_PREFIX / "README.md").read_text(encoding="utf-8")
+    assert "git subtree" in readme and "shared/" in readme
+    under = [name for name in tracked_files() if name.startswith(SUBTREE_PREFIX)]
+    assert under, "no tracked file under the subtree prefix -- the exception names nothing"
+    assert scan(under, terms[0]), (
+        "no file under shared/ carries an excluded term any more: remove SUBTREE_PREFIX from "
+        "this file"
+    )