"""Every OWASP LLM Top 10 number carries the newest edition's year — row 2 of the 2026 done-criterion. OWASP's 2026 edition renumbered the list (`docs/PLAN.md` § T.5): only LLM01 and LLM02 kept their meaning, so a bare ``LLM05`` written against 2025 now reads as Data and Model Poisoning. Row 2 (`docs/PLAN.md` § T.2) is the count of places a number is used with an older edition's meaning, or with a meaning no edition gave it; it was 95 at ``6702d7a`` and this file holds it at 0. Two guards, because the number reaches a consumer two ways: * TEXT — ``docs/catalog-census.py`` over the tracked files. No site may be ``old`` or ``wrong``, and none may lack the year (``same``) except the vendored lexicon, whose data stays byte-equal with commons and gets its year where the finding is made (``lexicon.load_lexicon``). * FINDINGS — every finding the coverage matrix drives out of the real guard is recorded as it is constructed, and each ``owasp`` is ``None`` or ``LLMnn:``. "Newest" is read from the catalog register (`docs/threat-2026/catalogs.json`), not written here, so the next edition moves one place. """ from __future__ import annotations import importlib.util import json import re import subprocess from pathlib import Path import pytest from llm_ingestion_guard import report as report_module from llm_ingestion_guard.coverage import CORE_CASES, run_matrix from llm_ingestion_guard.lexicon import load_lexicon ROOT = Path(__file__).resolve().parent.parent REGISTER = ROOT / "docs" / "threat-2026" / "catalogs.json" VENDORED_LEXICON = "src/llm_ingestion_guard/injection_lexicon.json" def _newest_year() -> str: entries = json.loads(REGISTER.read_text(encoding="utf-8"))["entries"] (owasp,) = [e for e in entries if e["id"] == "owasp-llm-top10"] return owasp["latest"] def _census(): spec = importlib.util.spec_from_file_location("catalog_census", ROOT / "docs" / "catalog-census.py") module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) return module def _in_git_checkout() -> bool: try: subprocess.run(("git", "-C", str(ROOT), "rev-parse"), check=True, capture_output=True) except (OSError, subprocess.CalledProcessError): return False return True needs_git = pytest.mark.skipif(not _in_git_checkout(), reason="the census reads tracked files") @needs_git def test_census_counts_the_register_edition_as_current(): # The census and the register must agree on which edition is newest; if they # drift, "current" means a year the register no longer calls latest. assert _census().CURRENT_YEAR == _newest_year() @needs_git def test_no_catalog_number_with_an_older_or_no_meaning(): rows = _census().census(None) counted = [r for r in rows if r[0] in ("old", "wrong")] assert not counted, "catalog numbers with an older edition's meaning, or none:\n" + "\n".join( f" {r[1]}:{r[2]} {r[3]} [{r[0]}] -> {r[6]}" for r in counted) @needs_git def test_no_catalog_number_without_the_year(): rows = _census().census(None) bare = [r for r in rows if r[0] == "same" and r[1] != VENDORED_LEXICON] assert not bare, "catalog numbers without the year:\n" + "\n".join( f" {r[1]}:{r[2]} {r[3]} -> {r[6]}" for r in bare) def test_vendored_lexicon_gets_the_year_where_the_finding_is_made(): year = _newest_year() assert {p.owasp for p in load_lexicon()} == {f"LLM01:{year}"} def test_a_moved_number_from_commons_is_refused_not_guessed(): # A bare number whose meaning moved between editions cannot be labelled # without knowing which edition wrote it; mislabelling it is the defect this # file exists for, so it raises instead. from llm_ingestion_guard import _owasp assert _owasp.from_commons("LLM02") == f"LLM02:{_newest_year()}" assert _owasp.from_commons(f"LLM10:{_newest_year()}") == f"LLM10:{_newest_year()}" with pytest.raises(ValueError): _owasp.from_commons("LLM05") with pytest.raises(ValueError): _owasp.from_commons("LLM01:2025") def test_every_finding_the_matrix_drives_carries_the_newest_year(monkeypatch): seen: list[tuple[str, object]] = [] original = report_module.Finding.__init__ def recording_init(self, *args, **kwargs): original(self, *args, **kwargs) seen.append((self.label, self.owasp)) monkeypatch.setattr(report_module.Finding, "__init__", recording_init) run_matrix() assert len(seen) > 100, f"recorder saw only {len(seen)} findings" shape = re.compile(rf"LLM(0[1-9]|10):{_newest_year()}") bad = sorted({(label, owasp) for label, owasp in seen if owasp is not None and not shape.fullmatch(owasp)}) assert not bad, f"findings with an un-yeared or older-edition owasp: {bad}" def test_matrix_rows_carry_the_newest_year_or_no_number(): shape = re.compile(rf"LLM(0[1-9]|10):{_newest_year()}") bad = [(c.group, c.klasse, c.owasp) for c in CORE_CASES if c.owasp is not None and not shape.fullmatch(c.owasp)] assert not bad, bad # The 13 sites the census classed `wrong` at 6702d7a, re-judged in session 3. # Each class below either has a 2026 entry that describes it, or no entry does, # and then it carries no number rather than a doubtful one. _NO_CATALOG_ENTRY = ( "un-scannable artifact fails closed", # the guard's own failure semantics "T2 frontmatter nested mapping", # parser-surface reduction "T2 frontmatter block scalar", "T2 frontmatter flow sequence, quoted element", "T2 mapping key off the allowlist", "T3 resource non-https (http)", # received metadata, not model output "T3 resource data: scheme", "T3 resource javascript: scheme", "T5a cross-link dangerous scheme", "T4 path traversal", # bundle file-system integrity "T4 absolute path", "T4 reserved filename shadow", "T4 non-.md concept", "T5a cross-link bundle escape", ) @pytest.mark.parametrize("klasse", _NO_CATALOG_ENTRY) def test_rows_no_edition_describes_carry_no_number(klasse): (case,) = [c for c in CORE_CASES if c.klasse == klasse] assert case.owasp is None def test_factual_poisoning_gap_is_data_and_model_poisoning(): # LLM05:2026 names "RAG Knowledge Base Poisoning" (PoisonedRAG) as its example 6; # the row used to say LLM09, which in 2025 meant Misinformation. (case,) = [c for c in CORE_CASES if c.klasse == "semantic / factual poisoning"] assert case.owasp == f"LLM05:{_newest_year()}"