portfolio-optimiser/tests/test_excluded_terms.py

"""No tracked file carries a term from the repository's local exclusion list.

The example material was replaced by one generic, fictitious example set (see
``src/portfolio_optimiser/data/`` and ``contexts/``). This gate keeps the retired material from
coming back: it reads every file ``git ls-files`` reports and fails on any line matching a pattern
from ``tests/excluded-terms.local.md``.

**The term list is deliberately NOT tracked.** It lives in a gitignored local file, so the
published tree carries the mechanism and never the list. Where the file is absent (a fresh clone,
the handover archive) the gate SKIPS and says why; it is measured red -> green with the file in
place. ``test_the_term_list_is_never_tracked`` holds the other half: the list stays local.

The same file carries each pattern's known-positive and a set of known-negatives, so a pattern
that cannot fire, or one that fires on ordinary prose, is red rather than silently useless.

**One named, tested exception:** ``shared/``, the ``git subtree`` of
``portfolio-optimiser-commons``. Its sync is pull-only: content changes there are committed in
commons and pulled here, never edited here. The exception is itself gated — once no file under
the prefix hits, the exception is dead and a test says to remove it.

Every scan carries its denominator: a gate that read zero files would be green and prove nothing.
"""

from __future__ import annotations

import re
import subprocess
from pathlib import Path

import pytest

_REPO_ROOT = Path(__file__).resolve().parents[1]
TERMS_FILE = _REPO_ROOT / "tests" / "excluded-terms.local.md"

#: The subtree prefix whose content is owned by another repository (see the module docstring).
SUBTREE_PREFIX = "shared/"


def load_terms(
    path: Path = TERMS_FILE,
) -> tuple[list[tuple[str, re.Pattern[str]]], list[tuple[str, str]], list[str]]:
    """(patterns, known-positives, known-negatives) from the local term file."""
    patterns: list[tuple[str, re.Pattern[str]]] = []
    positives: list[tuple[str, str]] = []
    negatives: list[str] = []
    for raw in path.read_text(encoding="utf-8").splitlines():
        if not raw.strip() or raw.startswith("#"):
            continue
        kind, _, rest = raw.partition("\t")
        if kind == "P":
            label, _, regex = rest.partition("\t")
            patterns.append((label, re.compile(regex, re.IGNORECASE)))
        elif kind == "K":
            label, _, sample = rest.partition("\t")
            positives.append((label, sample))
        elif kind == "N":
            negatives.append(rest)
        else:
            raise ValueError(f"unknown entry kind {kind!r} in {path.name}: {raw!r}")
    return patterns, positives, negatives


@pytest.fixture(scope="module")
def terms() -> tuple[list[tuple[str, re.Pattern[str]]], list[tuple[str, str]], list[str]]:
    if not TERMS_FILE.is_file():
        pytest.skip(f"local term list {TERMS_FILE.name} is not present (local-only by design)")
    loaded = load_terms()
    assert loaded[0], f"{TERMS_FILE.name} declares no patterns, so nothing would be checked"
    return loaded


def tracked_files() -> list[str]:
    done = subprocess.run(
        ["git", "ls-files", "-z"], cwd=_REPO_ROOT, capture_output=True, check=True
    )
    return [name for name in done.stdout.decode("utf-8").split("\0") if name]


def hits_in(text: str, patterns: list[tuple[str, re.Pattern[str]]]) -> list[tuple[str, int, str]]:
    """(pattern label, 1-based line number, line) for every matching line."""
    found = []
    for number, line in enumerate(text.splitlines(), start=1):
        for label, pattern in patterns:
            if pattern.search(line):
                found.append((label, number, line.strip()[:160]))
    return found


def scan(
    names: list[str], patterns: list[tuple[str, re.Pattern[str]]]
) -> dict[str, list[tuple[str, int, str]]]:
    """Every file with at least one hit. Binary files are read too: a string inside a fixture
    database is as tracked as a line of prose, so bytes are decoded leniently, never skipped."""
    out = {}
    for name in names:
        path = _REPO_ROOT / name
        if not path.is_file():  # a deletion staged but not yet committed
            continue
        found = hits_in(path.read_bytes().decode("utf-8", errors="replace"), patterns)
        if found:
            out[name] = found
    return out


def test_no_tracked_file_carries_an_excluded_term(terms) -> None:
    patterns = terms[0]
    names = tracked_files()
    assert len(names) > 100, f"git ls-files listed {len(names)} files -- too few to be this repo"
    offending = {
        name: hits
        for name, hits in scan(names, patterns).items()
        if not name.startswith(SUBTREE_PREFIX)
    }
    report = "\n".join(
        f"{name}:{number}: [{label}] {line}"
        for name, hits in sorted(offending.items())
        for label, number, line in hits[:5]
    )
    assert not offending, f"{len(offending)} tracked file(s) carry excluded terms:\n{report}"


def test_every_pattern_fires_on_its_known_positive(terms) -> None:
    patterns, positives, _ = terms
    labels = {label for label, _ in patterns}
    assert labels <= {label for label, _ in positives}, "every pattern needs a known-positive"
    for label, sample in positives:
        assert label in {hit[0] for hit in hits_in(sample, patterns)}, (
            f"pattern {label!r} missed its known-positive {sample!r}"
        )


def test_ordinary_prose_does_not_fire(terms) -> None:
    patterns, _, negatives = terms
    assert negatives, "the term list carries no known-negatives, so over-matching is unchecked"
    for sample in negatives:
        assert hits_in(sample, patterns) == [], f"the gate fired on ordinary prose: {sample!r}"


def test_the_term_list_is_never_tracked() -> None:
    """The list must stay local: tracked, it would publish exactly what the gate keeps out."""
    rel = TERMS_FILE.relative_to(_REPO_ROOT).as_posix()
    assert rel not in tracked_files(), f"{rel} is tracked -- it must stay local-only"
    ignored = subprocess.run(["git", "check-ignore", "-q", rel], cwd=_REPO_ROOT)
    assert ignored.returncode == 0, f"{rel} is not covered by .gitignore"


def test_the_subtree_exception_is_still_needed_and_is_the_subtree(terms) -> None:
    """The ``shared/`` exception is alive only while the subtree still carries excluded terms.

    Both halves are the claim: the prefix is the subtree (its README says so), and at least one
    file under it still hits. When the cleanup has been pulled from commons, the second half goes
    red and the exception must be deleted here rather than linger as a hole nobody remembers.
    """
    readme = (_REPO_ROOT / SUBTREE_PREFIX / "README.md").read_text(encoding="utf-8")
    assert "git subtree" in readme and "shared/" in readme
    under = [name for name in tracked_files() if name.startswith(SUBTREE_PREFIX)]
    assert under, "no tracked file under the subtree prefix -- the exception names nothing"
    assert scan(under, terms[0]), (
        "no file under shared/ carries an excluded term any more: remove SUBTREE_PREFIX from "
        "this file"
    )