llm-ingestion-pipeline-security/tests/test_catalog_numbers.py

"""Every OWASP LLM Top 10 number carries the newest edition's year — row 2 of the 2026 done-criterion.

OWASP's 2026 edition renumbered the list (`docs/PLAN.md` § T.5): only LLM01 and
LLM02 kept their meaning, so a bare ``LLM05`` written against 2025 now reads as
Data and Model Poisoning. Row 2 (`docs/PLAN.md` § T.2) is the count of places a
number is used with an older edition's meaning, or with a meaning no edition gave
it; it was 95 at ``6702d7a`` and this file holds it at 0.

Two guards, because the number reaches a consumer two ways:

* TEXT — ``docs/catalog-census.py`` over the tracked files. No site may be
  ``old`` or ``wrong``, and none may lack the year (``same``) except the vendored
  lexicon, whose data stays byte-equal with commons and gets its year where the
  finding is made (``lexicon.load_lexicon``).
* FINDINGS — every finding the coverage matrix drives out of the real guard is
  recorded as it is constructed, and each ``owasp`` is ``None`` or ``LLMnn:<newest>``.

"Newest" is read from the catalog register (`docs/threat-2026/catalogs.json`), not
written here, so the next edition moves one place.
"""
from __future__ import annotations

import importlib.util
import json
import re
import subprocess
from pathlib import Path

import pytest

from llm_ingestion_guard import report as report_module
from llm_ingestion_guard.coverage import CORE_CASES, run_matrix
from llm_ingestion_guard.lexicon import load_lexicon

ROOT = Path(__file__).resolve().parent.parent
REGISTER = ROOT / "docs" / "threat-2026" / "catalogs.json"
VENDORED_LEXICON = "src/llm_ingestion_guard/injection_lexicon.json"


def _newest_year() -> str:
    entries = json.loads(REGISTER.read_text(encoding="utf-8"))["entries"]
    (owasp,) = [e for e in entries if e["id"] == "owasp-llm-top10"]
    return owasp["latest"]


def _census():
    spec = importlib.util.spec_from_file_location("catalog_census", ROOT / "docs" / "catalog-census.py")
    module = importlib.util.module_from_spec(spec)
    spec.loader.exec_module(module)
    return module


def _in_git_checkout() -> bool:
    try:
        subprocess.run(("git", "-C", str(ROOT), "rev-parse"), check=True, capture_output=True)
    except (OSError, subprocess.CalledProcessError):
        return False
    return True


needs_git = pytest.mark.skipif(not _in_git_checkout(), reason="the census reads tracked files")


@needs_git
def test_census_counts_the_register_edition_as_current():
    # The census and the register must agree on which edition is newest; if they
    # drift, "current" means a year the register no longer calls latest.
    assert _census().CURRENT_YEAR == _newest_year()


@needs_git
def test_no_catalog_number_with_an_older_or_no_meaning():
    rows = _census().census(None)
    counted = [r for r in rows if r[0] in ("old", "wrong")]
    assert not counted, "catalog numbers with an older edition's meaning, or none:\n" + "\n".join(
        f"  {r[1]}:{r[2]} {r[3]} [{r[0]}] -> {r[6]}" for r in counted)


@needs_git
def test_no_catalog_number_without_the_year():
    rows = _census().census(None)
    bare = [r for r in rows if r[0] == "same" and r[1] != VENDORED_LEXICON]
    assert not bare, "catalog numbers without the year:\n" + "\n".join(
        f"  {r[1]}:{r[2]} {r[3]} -> {r[6]}" for r in bare)


def test_vendored_lexicon_gets_the_year_where_the_finding_is_made():
    year = _newest_year()
    assert {p.owasp for p in load_lexicon()} == {f"LLM01:{year}"}


def test_a_moved_number_from_commons_is_refused_not_guessed():
    # A bare number whose meaning moved between editions cannot be labelled
    # without knowing which edition wrote it; mislabelling it is the defect this
    # file exists for, so it raises instead.
    from llm_ingestion_guard import _owasp

    assert _owasp.from_commons("LLM02") == f"LLM02:{_newest_year()}"
    assert _owasp.from_commons(f"LLM10:{_newest_year()}") == f"LLM10:{_newest_year()}"
    with pytest.raises(ValueError):
        _owasp.from_commons("LLM05")
    with pytest.raises(ValueError):
        _owasp.from_commons("LLM01:2025")


def test_every_finding_the_matrix_drives_carries_the_newest_year(monkeypatch):
    seen: list[tuple[str, object]] = []
    original = report_module.Finding.__init__

    def recording_init(self, *args, **kwargs):
        original(self, *args, **kwargs)
        seen.append((self.label, self.owasp))

    monkeypatch.setattr(report_module.Finding, "__init__", recording_init)
    run_matrix()
    assert len(seen) > 100, f"recorder saw only {len(seen)} findings"
    shape = re.compile(rf"LLM(0[1-9]|10):{_newest_year()}")
    bad = sorted({(label, owasp) for label, owasp in seen
                  if owasp is not None and not shape.fullmatch(owasp)})
    assert not bad, f"findings with an un-yeared or older-edition owasp: {bad}"


def test_matrix_rows_carry_the_newest_year_or_no_number():
    shape = re.compile(rf"LLM(0[1-9]|10):{_newest_year()}")
    bad = [(c.group, c.klasse, c.owasp) for c in CORE_CASES
           if c.owasp is not None and not shape.fullmatch(c.owasp)]
    assert not bad, bad


# The 13 sites the census classed `wrong` at 6702d7a, re-judged in session 3.
# Each class below either has a 2026 entry that describes it, or no entry does,
# and then it carries no number rather than a doubtful one.
_NO_CATALOG_ENTRY = (
    "un-scannable artifact fails closed",       # the guard's own failure semantics
    "T2 frontmatter nested mapping",            # parser-surface reduction
    "T2 frontmatter block scalar",
    "T2 frontmatter flow sequence, quoted element",
    "T2 mapping key off the allowlist",
    "T3 resource non-https (http)",             # received metadata, not model output
    "T3 resource data: scheme",
    "T3 resource javascript: scheme",
    "T5a cross-link dangerous scheme",
    "T4 path traversal",                        # bundle file-system integrity
    "T4 absolute path",
    "T4 reserved filename shadow",
    "T4 non-.md concept",
    "T5a cross-link bundle escape",
)


@pytest.mark.parametrize("klasse", _NO_CATALOG_ENTRY)
def test_rows_no_edition_describes_carry_no_number(klasse):
    (case,) = [c for c in CORE_CASES if c.klasse == klasse]
    assert case.owasp is None


def test_factual_poisoning_gap_is_data_and_model_poisoning():
    # LLM05:2026 names "RAG Knowledge Base Poisoning" (PoisonedRAG) as its example 6;
    # the row used to say LLM09, which in 2025 meant Misinformation.
    (case,) = [c for c in CORE_CASES if c.klasse == "semantic / factual poisoning"]
    assert case.owasp == f"LLM05:{_newest_year()}"