Kjell Tore Guttormsen
2e3d33efb4
docs/catalog-census.py classifies every OWASP LLM number in tracked files:
old (2025 meaning, maps mechanically), wrong (a meaning no edition gave it,
each with its reason), same, current, history, migration-plan, self.
At 6702d7a the denominator is 95 sites in 23 files (82 old, 13 wrong);
the listing is frozen with its sha256 and a test reproduces it from git.
227 lines
10 KiB
Python
227 lines
10 KiB
Python
"""catalog-number census — every place an OWASP LLM Top 10 number is used, and what it means.
|
|
|
|
OWASP published the 2026 edition of the Top 10 for LLM Applications with a new
|
|
order: only LLM01 and LLM02 kept their meaning (`docs/PLAN.md` § T.5). A bare
|
|
`LLM05` written against 2025 now reads as Data and Model Poisoning, and the
|
|
`owasp` field carries that misreading to every consumer of a finding. Row 2 of the
|
|
2026 done-criterion (`docs/PLAN.md` § T.2) counts those places; this script is the
|
|
count, so the number is reproducible instead of a grep someone remembers.
|
|
|
|
WHAT IT CLASSIFIES. Every `LLMnn` token in a tracked file, with slash lists
|
|
expanded (`LLM01/02/04` is three numbers), lands in exactly one class:
|
|
|
|
- ``old`` — used with its 2025 meaning and no year. Maps mechanically (§ T.5).
|
|
- ``wrong`` — used with a meaning no edition gave the number, or one too doubtful
|
|
to map mechanically. Needs a decision; each rule below carries its reason.
|
|
- ``same`` — LLM01 / LLM02 used with the meaning both editions share. Listed,
|
|
not counted: the number is right, only the year is missing (session 3).
|
|
- ``current`` — carries `:2026`, the newest edition. Listed, not counted.
|
|
- ``history`` — CHANGELOG, the v1 plan and published reviews. Snapshots of what
|
|
was true when written; they are never corrected (STATE conventions), so they
|
|
sit outside the denominator by definition (§ T.2 row 2).
|
|
- ``migration-plan`` — the 2026 section of `docs/PLAN.md`. It names both editions
|
|
by construction (its mapping table IS the old numbers); counting it would make
|
|
row 2 unreachable by the very text that plans the move.
|
|
- ``self`` — this script and the session-1 artefacts in `docs/threat-2026/` (the
|
|
catalog register and the frozen lists), which name both editions by construction.
|
|
|
|
The DENOMINATOR is ``old + wrong``.
|
|
|
|
WHY THE `wrong` RULES MATCH ON CONTEXT, NOT LINE NUMBERS. A rule names a file, a
|
|
substring of the token's line or of up to three lines before it (stopping at a
|
|
line that carries another token, so a rule cannot reach a neighbour), the number, and how
|
|
many sites it must match. A rule that matches a different count than it states
|
|
raises: an edit that moves or removes a judged site must re-judge it, not let
|
|
the rule go stale and the site silently fall back to ``old``.
|
|
|
|
THE FROZEN DENOMINATOR is measured at a fixed revision, read from git objects,
|
|
not the working tree, so it is reproducible after every later edit:
|
|
|
|
python3 docs/catalog-census.py --rev 6702d7a --tsv docs/threat-2026/census-6702d7a.tsv
|
|
|
|
The sha256 printed last is over the listing (every class, every site); the
|
|
listing and its hash are frozen in `docs/threat-2026/` and `docs/PLAN.md`.
|
|
Without ``--rev`` it measures the tracked files of the working tree.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import hashlib
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parent.parent
|
|
|
|
TITLES_2025 = {
|
|
"01": "Prompt Injection",
|
|
"02": "Sensitive Information Disclosure",
|
|
"03": "Supply Chain",
|
|
"04": "Data and Model Poisoning",
|
|
"05": "Improper Output Handling",
|
|
"06": "Excessive Agency",
|
|
"07": "System Prompt Leakage",
|
|
"08": "Vector and Embedding Weaknesses",
|
|
"09": "Misinformation",
|
|
"10": "Unbounded Consumption",
|
|
}
|
|
|
|
# 2025 number -> 2026 number, same risk. Verified against the publisher in
|
|
# session 1; the source and check date live in docs/threat-2026/catalogs.json.
|
|
MAP_2025_TO_2026 = {
|
|
"01": "01", "02": "02", "03": "04", "04": "05", "05": "10",
|
|
"06": "03", "07": "08", "08": "09", "09": "07", "10": "06",
|
|
}
|
|
|
|
HISTORY_FILES = {
|
|
"CHANGELOG.md": "released entries are snapshots",
|
|
"docs/PLAN-v1.md": "the v1 plan",
|
|
"docs/review-2026-07.md": "published review",
|
|
"docs/GATE-G-v1.md": "published gate record",
|
|
"docs/2026-09-07-limitations-44-maaling.md": "published measurement",
|
|
}
|
|
|
|
SELF_PREFIXES = ("docs/catalog-census.py", "docs/threat-2026/")
|
|
|
|
# `docs/PLAN.md` is split by its top-level headings.
|
|
PLAN = "docs/PLAN.md"
|
|
PLAN_SECTIONS = (
|
|
("# Trusselbildet 2026", "migration-plan"),
|
|
("# Implementation Plan", "history"),
|
|
)
|
|
|
|
_FAIL_SECURE = ("Fail-secure disposition is not Unbounded Consumption in any edition; "
|
|
"it is the guard's own failure semantics")
|
|
_PARSER_SURFACE = ("A frontmatter shape the parser refuses to interpret is parser-surface "
|
|
"reduction, not resource consumption; no edition defines it")
|
|
_NOT_MODEL_OUTPUT = ("Validates received bundle metadata, not model output; "
|
|
"Improper Output Handling is about LLM output passed downstream")
|
|
_FACTUAL_POISONING = ("Factual poisoning of the ingested store (PoisonedRAG) is Data and "
|
|
"Model Poisoning or Vector and Embedding Weaknesses, not Misinformation")
|
|
|
|
# (path, context substring, number, expected sites, reason)
|
|
WRONG_RULES = (
|
|
("README.md", "LLM10 · fail-secure", "10", 1, _FAIL_SECURE),
|
|
("src/llm_ingestion_guard/coverage.py", "_fail_closed_probe, owasp=", "10", 1,
|
|
_FAIL_SECURE),
|
|
("src/llm_ingestion_guard/coverage.py", "nested: x", "10", 1, _PARSER_SURFACE),
|
|
("src/llm_ingestion_guard/coverage.py", "desc: |", "10", 1, _PARSER_SURFACE),
|
|
("src/llm_ingestion_guard/coverage.py", "flow sequence, quoted element", "10", 1,
|
|
_PARSER_SURFACE),
|
|
("src/llm_ingestion_guard/coverage.py", "mapping key off the allowlist", "10", 1,
|
|
_PARSER_SURFACE),
|
|
("src/llm_ingestion_guard/coverage.py", "validate_resource_url(", "05", 3,
|
|
_NOT_MODEL_OUTPUT),
|
|
("src/llm_ingestion_guard/coverage.py", "T5a cross-link dangerous scheme", "05", 1,
|
|
_NOT_MODEL_OUTPUT),
|
|
("src/llm_ingestion_guard/coverage.py", "_semantic_poisoning_gap", "09", 1,
|
|
_FACTUAL_POISONING),
|
|
("tests/test_coverage_matrix.py", '"LLM06", "LLM09"', "09", 1,
|
|
"Asserts the matrix carries the semantic-poisoning row's LLM09; follows that row"),
|
|
("src/llm_ingestion_guard/coverage.py", "def _raise_case(", "01", 1,
|
|
"Default tags the OKF path-traversal, absolute-path, reserved-name, non-.md and "
|
|
"bundle-escape rows as Prompt Injection; none of them is"),
|
|
)
|
|
|
|
_TOKEN = re.compile(r"\bLLM(\d\d)((?:/(?:LLM)?\d\d)*)(?::(\d{4}))?(?!\d)")
|
|
_VALID = set(TITLES_2025)
|
|
|
|
|
|
def _git(*args: str) -> str:
|
|
return subprocess.run(("git", "-C", str(ROOT)) + args, check=True,
|
|
capture_output=True, text=True).stdout
|
|
|
|
|
|
def _files(rev: str | None) -> list[tuple[str, str]]:
|
|
names = (_git("ls-tree", "-r", "--name-only", rev) if rev else _git("ls-files")).split("\n")
|
|
out = []
|
|
for name in sorted(n for n in names if n):
|
|
if name.startswith("dist/") or ".local." in name:
|
|
continue
|
|
try:
|
|
text = (_git("show", f"{rev}:{name}") if rev
|
|
else (ROOT / name).read_text(encoding="utf-8"))
|
|
except (UnicodeDecodeError, subprocess.CalledProcessError, FileNotFoundError):
|
|
continue
|
|
out.append((name, text))
|
|
return out
|
|
|
|
|
|
def _plan_class(lines: list[str]) -> list[str | None]:
|
|
"""Per-line section class for docs/PLAN.md (None = live text)."""
|
|
current: str | None = None
|
|
out = []
|
|
for line in lines:
|
|
if line.startswith("# "):
|
|
current = next((cls for head, cls in PLAN_SECTIONS if line.startswith(head)), None)
|
|
out.append(current)
|
|
return out
|
|
|
|
|
|
def census(rev: str | None) -> list[tuple[str, ...]]:
|
|
rows = []
|
|
rule_hits = [0] * len(WRONG_RULES)
|
|
for path, text in _files(rev):
|
|
lines = text.split("\n")
|
|
sections = _plan_class(lines) if path == PLAN else [None] * len(lines)
|
|
for i, line in enumerate(lines):
|
|
for m in _TOKEN.finditer(line):
|
|
numbers = [m.group(1)] + re.findall(r"\d\d", m.group(2))
|
|
year = m.group(3) or ""
|
|
for nn in numbers:
|
|
if nn not in _VALID:
|
|
continue
|
|
reason = ""
|
|
if path.startswith(SELF_PREFIXES):
|
|
cls = "self"
|
|
elif path in HISTORY_FILES:
|
|
cls, reason = "history", HISTORY_FILES[path]
|
|
elif sections[i]:
|
|
cls = sections[i]
|
|
elif year == "2026":
|
|
cls = "current"
|
|
else:
|
|
cls = "same" if nn in ("01", "02") else "old"
|
|
start = i
|
|
while start > max(0, i - 3) and not _TOKEN.search(lines[start - 1]):
|
|
start -= 1
|
|
context = "\n".join(lines[start:i + 1])
|
|
for k, (rpath, sub, rnn, _, rreason) in enumerate(WRONG_RULES):
|
|
if rpath == path and rnn == nn and sub in context:
|
|
cls, reason = "wrong", rreason
|
|
rule_hits[k] += 1
|
|
break
|
|
rows.append((cls, path, str(i + 1), f"LLM{nn}", year,
|
|
TITLES_2025[nn], f"LLM{MAP_2025_TO_2026[nn]}:2026", reason))
|
|
stale = [(r[0], r[1], r[3], hits) for r, hits in zip(WRONG_RULES, rule_hits) if hits != r[3]]
|
|
if stale:
|
|
raise SystemExit(f"stale wrong-rules (path, substring, expected, matched): {stale}")
|
|
return rows
|
|
|
|
|
|
def main(argv: list[str] | None = None) -> int:
|
|
ap = argparse.ArgumentParser(description=__doc__.split("\n")[0])
|
|
ap.add_argument("--rev", help="measure this git revision instead of the working tree")
|
|
ap.add_argument("--tsv", help="also write the full listing to this path")
|
|
args = ap.parse_args(argv)
|
|
rows = census(args.rev)
|
|
header = "class\tpath\tline\tnumber\tyear\tmeaning_2025\tsame_risk_2026\treason"
|
|
body = "\n".join([header] + ["\t".join(r) for r in rows]) + "\n"
|
|
if args.tsv:
|
|
Path(args.tsv).write_text(body, encoding="utf-8")
|
|
counts: dict[str, int] = {}
|
|
for r in rows:
|
|
counts[r[0]] = counts.get(r[0], 0) + 1
|
|
files = sorted({r[1] for r in rows if r[0] in ("old", "wrong")})
|
|
print(f"revision: {args.rev or 'working tree'}")
|
|
for cls in ("old", "wrong", "same", "current", "history", "migration-plan", "self"):
|
|
print(f" {cls:<15} {counts.get(cls, 0)}")
|
|
print(f"denominator (old + wrong): {counts.get('old', 0) + counts.get('wrong', 0)}"
|
|
f" in {len(files)} files")
|
|
print(f"sha256: {hashlib.sha256(body.encode('utf-8')).hexdigest()}")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|