portfolio-optimiser/tests/test_excluded_terms.py

"""No tracked file carries a term from the repository's local exclusion list.

The example material was replaced by one generic, fictitious example set (see
``src/portfolio_optimiser/data/`` and ``contexts/``). This gate keeps the retired material from
coming back: it reads every file ``git ls-files`` reports and fails on any line matching a pattern
from ``tests/excluded-terms.local.md``.

**The term list is deliberately NOT tracked.** It lives in a gitignored local file, so the
published tree carries the mechanism and never the list. Where the file is absent (a fresh clone,
the handover archive) the gate SKIPS and says why; it is measured red -> green with the file in
place. ``test_the_term_list_is_never_tracked`` holds the other half: the list stays local.

The same file carries each pattern's known-positive and a set of known-negatives, so a pattern
that cannot fire, or one that fires on ordinary prose, is red rather than silently useless.

``shared/``, the ``git subtree`` of ``portfolio-optimiser-commons``, is scanned like every other
prefix: its content is cleaned in commons and pulled here, so a hit there is fixed in commons.

Every scan carries its denominator: a gate that read zero files would be green and prove nothing.
"""

from __future__ import annotations

import re
import subprocess
from pathlib import Path

import pytest

_REPO_ROOT = Path(__file__).resolve().parents[1]
TERMS_FILE = _REPO_ROOT / "tests" / "excluded-terms.local.md"


def load_terms(
    path: Path = TERMS_FILE,
) -> tuple[list[tuple[str, re.Pattern[str]]], list[tuple[str, str]], list[str]]:
    """(patterns, known-positives, known-negatives) from the local term file."""
    patterns: list[tuple[str, re.Pattern[str]]] = []
    positives: list[tuple[str, str]] = []
    negatives: list[str] = []
    for raw in path.read_text(encoding="utf-8").splitlines():
        if not raw.strip() or raw.startswith("#"):
            continue
        kind, _, rest = raw.partition("\t")
        if kind == "P":
            label, _, regex = rest.partition("\t")
            patterns.append((label, re.compile(regex, re.IGNORECASE)))
        elif kind == "K":
            label, _, sample = rest.partition("\t")
            positives.append((label, sample))
        elif kind == "N":
            negatives.append(rest)
        else:
            raise ValueError(f"unknown entry kind {kind!r} in {path.name}: {raw!r}")
    return patterns, positives, negatives


@pytest.fixture(scope="module")
def terms() -> tuple[list[tuple[str, re.Pattern[str]]], list[tuple[str, str]], list[str]]:
    if not TERMS_FILE.is_file():
        pytest.skip(f"local term list {TERMS_FILE.name} is not present (local-only by design)")
    loaded = load_terms()
    assert loaded[0], f"{TERMS_FILE.name} declares no patterns, so nothing would be checked"
    return loaded


def tracked_files() -> list[str]:
    done = subprocess.run(
        ["git", "ls-files", "-z"], cwd=_REPO_ROOT, capture_output=True, check=True
    )
    return [name for name in done.stdout.decode("utf-8").split("\0") if name]


def hits_in(text: str, patterns: list[tuple[str, re.Pattern[str]]]) -> list[tuple[str, int, str]]:
    """(pattern label, 1-based line number, line) for every matching line."""
    found = []
    for number, line in enumerate(text.splitlines(), start=1):
        for label, pattern in patterns:
            if pattern.search(line):
                found.append((label, number, line.strip()[:160]))
    return found


def scan(
    names: list[str], patterns: list[tuple[str, re.Pattern[str]]]
) -> dict[str, list[tuple[str, int, str]]]:
    """Every file with at least one hit. Binary files are read too: a string inside a fixture
    database is as tracked as a line of prose, so bytes are decoded leniently, never skipped."""
    out = {}
    for name in names:
        path = _REPO_ROOT / name
        if not path.is_file():  # a deletion staged but not yet committed
            continue
        found = hits_in(path.read_bytes().decode("utf-8", errors="replace"), patterns)
        if found:
            out[name] = found
    return out


def test_no_tracked_file_carries_an_excluded_term(terms) -> None:
    patterns = terms[0]
    names = tracked_files()
    assert len(names) > 100, f"git ls-files listed {len(names)} files -- too few to be this repo"
    offending = scan(names, patterns)
    report = "\n".join(
        f"{name}:{number}: [{label}] {line}"
        for name, hits in sorted(offending.items())
        for label, number, line in hits[:5]
    )
    assert not offending, f"{len(offending)} tracked file(s) carry excluded terms:\n{report}"


def test_every_pattern_fires_on_its_known_positive(terms) -> None:
    patterns, positives, _ = terms
    labels = {label for label, _ in patterns}
    assert labels <= {label for label, _ in positives}, "every pattern needs a known-positive"
    for label, sample in positives:
        assert label in {hit[0] for hit in hits_in(sample, patterns)}, (
            f"pattern {label!r} missed its known-positive {sample!r}"
        )


def test_ordinary_prose_does_not_fire(terms) -> None:
    patterns, _, negatives = terms
    assert negatives, "the term list carries no known-negatives, so over-matching is unchecked"
    for sample in negatives:
        assert hits_in(sample, patterns) == [], f"the gate fired on ordinary prose: {sample!r}"


def test_the_term_list_is_never_tracked() -> None:
    """The list must stay local: tracked, it would publish exactly what the gate keeps out."""
    rel = TERMS_FILE.relative_to(_REPO_ROOT).as_posix()
    assert rel not in tracked_files(), f"{rel} is tracked -- it must stay local-only"
    ignored = subprocess.run(["git", "check-ignore", "-q", rel], cwd=_REPO_ROOT)
    assert ignored.returncode == 0, f"{rel} is not covered by .gitignore"