test(threat-2026): freeze the catalog-number census at v1.4.1
docs/catalog-census.py classifies every OWASP LLM number in tracked files:
old (2025 meaning, maps mechanically), wrong (a meaning no edition gave it,
each with its reason), same, current, history, migration-plan, self.
At 6702d7a the denominator is 95 sites in 23 files (82 old, 13 wrong);
the listing is frozen with its sha256 and a test reproduces it from git.
This commit is contained in:
parent
61475686bf
commit
2e3d33efb4
SSH key fingerprint:
SHA256:JakMjO6FTBBzN0Bhfj9saOoEjaFxlSdYuZQQpM/lF9Q
3 changed files with 564 additions and 0 deletions
227
docs/catalog-census.py
Normal file
227
docs/catalog-census.py
Normal file
|
|
@ -0,0 +1,227 @@
|
|||
"""catalog-number census — every place an OWASP LLM Top 10 number is used, and what it means.
|
||||
|
||||
OWASP published the 2026 edition of the Top 10 for LLM Applications with a new
|
||||
order: only LLM01 and LLM02 kept their meaning (`docs/PLAN.md` § T.5). A bare
|
||||
`LLM05` written against 2025 now reads as Data and Model Poisoning, and the
|
||||
`owasp` field carries that misreading to every consumer of a finding. Row 2 of the
|
||||
2026 done-criterion (`docs/PLAN.md` § T.2) counts those places; this script is the
|
||||
count, so the number is reproducible instead of a grep someone remembers.
|
||||
|
||||
WHAT IT CLASSIFIES. Every `LLMnn` token in a tracked file, with slash lists
|
||||
expanded (`LLM01/02/04` is three numbers), lands in exactly one class:
|
||||
|
||||
- ``old`` — used with its 2025 meaning and no year. Maps mechanically (§ T.5).
|
||||
- ``wrong`` — used with a meaning no edition gave the number, or one too doubtful
|
||||
to map mechanically. Needs a decision; each rule below carries its reason.
|
||||
- ``same`` — LLM01 / LLM02 used with the meaning both editions share. Listed,
|
||||
not counted: the number is right, only the year is missing (session 3).
|
||||
- ``current`` — carries `:2026`, the newest edition. Listed, not counted.
|
||||
- ``history`` — CHANGELOG, the v1 plan and published reviews. Snapshots of what
|
||||
was true when written; they are never corrected (STATE conventions), so they
|
||||
sit outside the denominator by definition (§ T.2 row 2).
|
||||
- ``migration-plan`` — the 2026 section of `docs/PLAN.md`. It names both editions
|
||||
by construction (its mapping table IS the old numbers); counting it would make
|
||||
row 2 unreachable by the very text that plans the move.
|
||||
- ``self`` — this script and the session-1 artefacts in `docs/threat-2026/` (the
|
||||
catalog register and the frozen lists), which name both editions by construction.
|
||||
|
||||
The DENOMINATOR is ``old + wrong``.
|
||||
|
||||
WHY THE `wrong` RULES MATCH ON CONTEXT, NOT LINE NUMBERS. A rule names a file, a
|
||||
substring of the token's line or of up to three lines before it (stopping at a
|
||||
line that carries another token, so a rule cannot reach a neighbour), the number, and how
|
||||
many sites it must match. A rule that matches a different count than it states
|
||||
raises: an edit that moves or removes a judged site must re-judge it, not let
|
||||
the rule go stale and the site silently fall back to ``old``.
|
||||
|
||||
THE FROZEN DENOMINATOR is measured at a fixed revision, read from git objects,
|
||||
not the working tree, so it is reproducible after every later edit:
|
||||
|
||||
python3 docs/catalog-census.py --rev 6702d7a --tsv docs/threat-2026/census-6702d7a.tsv
|
||||
|
||||
The sha256 printed last is over the listing (every class, every site); the
|
||||
listing and its hash are frozen in `docs/threat-2026/` and `docs/PLAN.md`.
|
||||
Without ``--rev`` it measures the tracked files of the working tree.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
|
||||
TITLES_2025 = {
|
||||
"01": "Prompt Injection",
|
||||
"02": "Sensitive Information Disclosure",
|
||||
"03": "Supply Chain",
|
||||
"04": "Data and Model Poisoning",
|
||||
"05": "Improper Output Handling",
|
||||
"06": "Excessive Agency",
|
||||
"07": "System Prompt Leakage",
|
||||
"08": "Vector and Embedding Weaknesses",
|
||||
"09": "Misinformation",
|
||||
"10": "Unbounded Consumption",
|
||||
}
|
||||
|
||||
# 2025 number -> 2026 number, same risk. Verified against the publisher in
|
||||
# session 1; the source and check date live in docs/threat-2026/catalogs.json.
|
||||
MAP_2025_TO_2026 = {
|
||||
"01": "01", "02": "02", "03": "04", "04": "05", "05": "10",
|
||||
"06": "03", "07": "08", "08": "09", "09": "07", "10": "06",
|
||||
}
|
||||
|
||||
HISTORY_FILES = {
|
||||
"CHANGELOG.md": "released entries are snapshots",
|
||||
"docs/PLAN-v1.md": "the v1 plan",
|
||||
"docs/review-2026-07.md": "published review",
|
||||
"docs/GATE-G-v1.md": "published gate record",
|
||||
"docs/2026-09-07-limitations-44-maaling.md": "published measurement",
|
||||
}
|
||||
|
||||
SELF_PREFIXES = ("docs/catalog-census.py", "docs/threat-2026/")
|
||||
|
||||
# `docs/PLAN.md` is split by its top-level headings.
|
||||
PLAN = "docs/PLAN.md"
|
||||
PLAN_SECTIONS = (
|
||||
("# Trusselbildet 2026", "migration-plan"),
|
||||
("# Implementation Plan", "history"),
|
||||
)
|
||||
|
||||
_FAIL_SECURE = ("Fail-secure disposition is not Unbounded Consumption in any edition; "
|
||||
"it is the guard's own failure semantics")
|
||||
_PARSER_SURFACE = ("A frontmatter shape the parser refuses to interpret is parser-surface "
|
||||
"reduction, not resource consumption; no edition defines it")
|
||||
_NOT_MODEL_OUTPUT = ("Validates received bundle metadata, not model output; "
|
||||
"Improper Output Handling is about LLM output passed downstream")
|
||||
_FACTUAL_POISONING = ("Factual poisoning of the ingested store (PoisonedRAG) is Data and "
|
||||
"Model Poisoning or Vector and Embedding Weaknesses, not Misinformation")
|
||||
|
||||
# (path, context substring, number, expected sites, reason)
|
||||
WRONG_RULES = (
|
||||
("README.md", "LLM10 · fail-secure", "10", 1, _FAIL_SECURE),
|
||||
("src/llm_ingestion_guard/coverage.py", "_fail_closed_probe, owasp=", "10", 1,
|
||||
_FAIL_SECURE),
|
||||
("src/llm_ingestion_guard/coverage.py", "nested: x", "10", 1, _PARSER_SURFACE),
|
||||
("src/llm_ingestion_guard/coverage.py", "desc: |", "10", 1, _PARSER_SURFACE),
|
||||
("src/llm_ingestion_guard/coverage.py", "flow sequence, quoted element", "10", 1,
|
||||
_PARSER_SURFACE),
|
||||
("src/llm_ingestion_guard/coverage.py", "mapping key off the allowlist", "10", 1,
|
||||
_PARSER_SURFACE),
|
||||
("src/llm_ingestion_guard/coverage.py", "validate_resource_url(", "05", 3,
|
||||
_NOT_MODEL_OUTPUT),
|
||||
("src/llm_ingestion_guard/coverage.py", "T5a cross-link dangerous scheme", "05", 1,
|
||||
_NOT_MODEL_OUTPUT),
|
||||
("src/llm_ingestion_guard/coverage.py", "_semantic_poisoning_gap", "09", 1,
|
||||
_FACTUAL_POISONING),
|
||||
("tests/test_coverage_matrix.py", '"LLM06", "LLM09"', "09", 1,
|
||||
"Asserts the matrix carries the semantic-poisoning row's LLM09; follows that row"),
|
||||
("src/llm_ingestion_guard/coverage.py", "def _raise_case(", "01", 1,
|
||||
"Default tags the OKF path-traversal, absolute-path, reserved-name, non-.md and "
|
||||
"bundle-escape rows as Prompt Injection; none of them is"),
|
||||
)
|
||||
|
||||
_TOKEN = re.compile(r"\bLLM(\d\d)((?:/(?:LLM)?\d\d)*)(?::(\d{4}))?(?!\d)")
|
||||
_VALID = set(TITLES_2025)
|
||||
|
||||
|
||||
def _git(*args: str) -> str:
|
||||
return subprocess.run(("git", "-C", str(ROOT)) + args, check=True,
|
||||
capture_output=True, text=True).stdout
|
||||
|
||||
|
||||
def _files(rev: str | None) -> list[tuple[str, str]]:
|
||||
names = (_git("ls-tree", "-r", "--name-only", rev) if rev else _git("ls-files")).split("\n")
|
||||
out = []
|
||||
for name in sorted(n for n in names if n):
|
||||
if name.startswith("dist/") or ".local." in name:
|
||||
continue
|
||||
try:
|
||||
text = (_git("show", f"{rev}:{name}") if rev
|
||||
else (ROOT / name).read_text(encoding="utf-8"))
|
||||
except (UnicodeDecodeError, subprocess.CalledProcessError, FileNotFoundError):
|
||||
continue
|
||||
out.append((name, text))
|
||||
return out
|
||||
|
||||
|
||||
def _plan_class(lines: list[str]) -> list[str | None]:
|
||||
"""Per-line section class for docs/PLAN.md (None = live text)."""
|
||||
current: str | None = None
|
||||
out = []
|
||||
for line in lines:
|
||||
if line.startswith("# "):
|
||||
current = next((cls for head, cls in PLAN_SECTIONS if line.startswith(head)), None)
|
||||
out.append(current)
|
||||
return out
|
||||
|
||||
|
||||
def census(rev: str | None) -> list[tuple[str, ...]]:
|
||||
rows = []
|
||||
rule_hits = [0] * len(WRONG_RULES)
|
||||
for path, text in _files(rev):
|
||||
lines = text.split("\n")
|
||||
sections = _plan_class(lines) if path == PLAN else [None] * len(lines)
|
||||
for i, line in enumerate(lines):
|
||||
for m in _TOKEN.finditer(line):
|
||||
numbers = [m.group(1)] + re.findall(r"\d\d", m.group(2))
|
||||
year = m.group(3) or ""
|
||||
for nn in numbers:
|
||||
if nn not in _VALID:
|
||||
continue
|
||||
reason = ""
|
||||
if path.startswith(SELF_PREFIXES):
|
||||
cls = "self"
|
||||
elif path in HISTORY_FILES:
|
||||
cls, reason = "history", HISTORY_FILES[path]
|
||||
elif sections[i]:
|
||||
cls = sections[i]
|
||||
elif year == "2026":
|
||||
cls = "current"
|
||||
else:
|
||||
cls = "same" if nn in ("01", "02") else "old"
|
||||
start = i
|
||||
while start > max(0, i - 3) and not _TOKEN.search(lines[start - 1]):
|
||||
start -= 1
|
||||
context = "\n".join(lines[start:i + 1])
|
||||
for k, (rpath, sub, rnn, _, rreason) in enumerate(WRONG_RULES):
|
||||
if rpath == path and rnn == nn and sub in context:
|
||||
cls, reason = "wrong", rreason
|
||||
rule_hits[k] += 1
|
||||
break
|
||||
rows.append((cls, path, str(i + 1), f"LLM{nn}", year,
|
||||
TITLES_2025[nn], f"LLM{MAP_2025_TO_2026[nn]}:2026", reason))
|
||||
stale = [(r[0], r[1], r[3], hits) for r, hits in zip(WRONG_RULES, rule_hits) if hits != r[3]]
|
||||
if stale:
|
||||
raise SystemExit(f"stale wrong-rules (path, substring, expected, matched): {stale}")
|
||||
return rows
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
ap = argparse.ArgumentParser(description=__doc__.split("\n")[0])
|
||||
ap.add_argument("--rev", help="measure this git revision instead of the working tree")
|
||||
ap.add_argument("--tsv", help="also write the full listing to this path")
|
||||
args = ap.parse_args(argv)
|
||||
rows = census(args.rev)
|
||||
header = "class\tpath\tline\tnumber\tyear\tmeaning_2025\tsame_risk_2026\treason"
|
||||
body = "\n".join([header] + ["\t".join(r) for r in rows]) + "\n"
|
||||
if args.tsv:
|
||||
Path(args.tsv).write_text(body, encoding="utf-8")
|
||||
counts: dict[str, int] = {}
|
||||
for r in rows:
|
||||
counts[r[0]] = counts.get(r[0], 0) + 1
|
||||
files = sorted({r[1] for r in rows if r[0] in ("old", "wrong")})
|
||||
print(f"revision: {args.rev or 'working tree'}")
|
||||
for cls in ("old", "wrong", "same", "current", "history", "migration-plan", "self"):
|
||||
print(f" {cls:<15} {counts.get(cls, 0)}")
|
||||
print(f"denominator (old + wrong): {counts.get('old', 0) + counts.get('wrong', 0)}"
|
||||
f" in {len(files)} files")
|
||||
print(f"sha256: {hashlib.sha256(body.encode('utf-8')).hexdigest()}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
274
docs/threat-2026/census-6702d7a.tsv
Normal file
274
docs/threat-2026/census-6702d7a.tsv
Normal file
|
|
@ -0,0 +1,274 @@
|
|||
class path line number year meaning_2025 same_risk_2026 reason
|
||||
history CHANGELOG.md 512 LLM10 Unbounded Consumption LLM06:2026 released entries are snapshots
|
||||
history CHANGELOG.md 689 LLM10 Unbounded Consumption LLM06:2026 released entries are snapshots
|
||||
history CHANGELOG.md 722 LLM10 Unbounded Consumption LLM06:2026 released entries are snapshots
|
||||
history CHANGELOG.md 896 LLM10 Unbounded Consumption LLM06:2026 released entries are snapshots
|
||||
history CHANGELOG.md 1043 LLM05 Improper Output Handling LLM10:2026 released entries are snapshots
|
||||
history CHANGELOG.md 1051 LLM02 Sensitive Information Disclosure LLM02:2026 released entries are snapshots
|
||||
history CHANGELOG.md 1068 LLM02 Sensitive Information Disclosure LLM02:2026 released entries are snapshots
|
||||
history CHANGELOG.md 1146 LLM02 Sensitive Information Disclosure LLM02:2026 released entries are snapshots
|
||||
same CLAUDE.md 54 LLM02 Sensitive Information Disclosure LLM02:2026
|
||||
same README.md 216 LLM01 Prompt Injection LLM01:2026
|
||||
same README.md 217 LLM02 Sensitive Information Disclosure LLM02:2026
|
||||
old README.md 218 LLM05 Improper Output Handling LLM10:2026
|
||||
old README.md 219 LLM06 Excessive Agency LLM03:2026
|
||||
wrong README.md 220 LLM10 Unbounded Consumption LLM06:2026 Fail-secure disposition is not Unbounded Consumption in any edition; it is the guard's own failure semantics
|
||||
old README.md 299 LLM08 Vector and Embedding Weaknesses LLM09:2026
|
||||
same README.md 315 LLM01 Prompt Injection LLM01:2026
|
||||
same README.md 315 LLM02 Sensitive Information Disclosure LLM02:2026
|
||||
old README.md 315 LLM04 Data and Model Poisoning LLM05:2026
|
||||
old README.md 315 LLM05 Improper Output Handling LLM10:2026
|
||||
old README.md 315 LLM06 Excessive Agency LLM03:2026
|
||||
old README.md 315 LLM08 Vector and Embedding Weaknesses LLM09:2026
|
||||
old README.md 315 LLM09 Misinformation LLM07:2026
|
||||
old README.md 315 LLM10 Unbounded Consumption LLM06:2026
|
||||
same docs/ADOPTION-BRIEF.md 149 LLM02 Sensitive Information Disclosure LLM02:2026
|
||||
same docs/ADOPTION-BRIEF.md 236 LLM01 Prompt Injection LLM01:2026
|
||||
same docs/ADOPTION-BRIEF.md 236 LLM02 Sensitive Information Disclosure LLM02:2026
|
||||
old docs/ADOPTION-BRIEF.md 236 LLM04 Data and Model Poisoning LLM05:2026
|
||||
old docs/ADOPTION-BRIEF.md 236 LLM05 Improper Output Handling LLM10:2026
|
||||
old docs/ADOPTION-BRIEF.md 236 LLM06 Excessive Agency LLM03:2026
|
||||
old docs/ADOPTION-BRIEF.md 236 LLM08 Vector and Embedding Weaknesses LLM09:2026
|
||||
same docs/BRIEF.md 209 LLM01 Prompt Injection LLM01:2026
|
||||
same docs/LIMITATIONS.md 221 LLM02 Sensitive Information Disclosure LLM02:2026
|
||||
old docs/LIMITATIONS.md 710 LLM10 Unbounded Consumption LLM06:2026
|
||||
old docs/LIMITATIONS.md 810 LLM08 Vector and Embedding Weaknesses LLM09:2026
|
||||
history docs/PLAN-v1.md 57 LLM05 Improper Output Handling LLM10:2026 the v1 plan
|
||||
history docs/PLAN-v1.md 121 LLM02 Sensitive Information Disclosure LLM02:2026 the v1 plan
|
||||
history docs/PLAN.md 34 LLM08 2025 Vector and Embedding Weaknesses LLM09:2026
|
||||
history docs/PLAN.md 65 LLM02 Sensitive Information Disclosure LLM02:2026
|
||||
history docs/PLAN.md 69 LLM10 Unbounded Consumption LLM06:2026
|
||||
history docs/PLAN.md 115 LLM02 Sensitive Information Disclosure LLM02:2026
|
||||
history docs/PLAN.md 170 LLM08 Vector and Embedding Weaknesses LLM09:2026
|
||||
history docs/PLAN.md 186 LLM01 Prompt Injection LLM01:2026
|
||||
history docs/PLAN.md 186 LLM02 Sensitive Information Disclosure LLM02:2026
|
||||
history docs/PLAN.md 186 LLM04 Data and Model Poisoning LLM05:2026
|
||||
history docs/PLAN.md 186 LLM05 Improper Output Handling LLM10:2026
|
||||
history docs/PLAN.md 186 LLM06 Excessive Agency LLM03:2026
|
||||
history docs/PLAN.md 186 LLM08 Vector and Embedding Weaknesses LLM09:2026
|
||||
history docs/PLAN.md 187 LLM09 Misinformation LLM07:2026
|
||||
history docs/PLAN.md 187 LLM10 Unbounded Consumption LLM06:2026
|
||||
history docs/PLAN.md 312 LLM10 Unbounded Consumption LLM06:2026
|
||||
history docs/review-2026-07.md 77 LLM05 Improper Output Handling LLM10:2026 published review
|
||||
history docs/review-2026-07.md 156 LLM02 Sensitive Information Disclosure LLM02:2026 published review
|
||||
history docs/review-2026-07.md 164 LLM02 Sensitive Information Disclosure LLM02:2026 published review
|
||||
history docs/review-2026-07.md 304 LLM09 Misinformation LLM07:2026 published review
|
||||
history docs/review-2026-07.md 338 LLM01 Prompt Injection LLM01:2026 published review
|
||||
history docs/review-2026-07.md 338 LLM02 Sensitive Information Disclosure LLM02:2026 published review
|
||||
history docs/review-2026-07.md 338 LLM04 Data and Model Poisoning LLM05:2026 published review
|
||||
history docs/review-2026-07.md 338 LLM05 Improper Output Handling LLM10:2026 published review
|
||||
history docs/review-2026-07.md 338 LLM06 Excessive Agency LLM03:2026 published review
|
||||
history docs/review-2026-07.md 338 LLM08 Vector and Embedding Weaknesses LLM09:2026 published review
|
||||
history docs/review-2026-07.md 338 LLM09 Misinformation LLM07:2026 published review
|
||||
history docs/review-2026-07.md 338 LLM10 Unbounded Consumption LLM06:2026 published review
|
||||
history docs/review-2026-07.md 343 LLM10 Unbounded Consumption LLM06:2026 published review
|
||||
old src/llm_ingestion_guard/active_content.py 9 LLM05 Improper Output Handling LLM10:2026
|
||||
old src/llm_ingestion_guard/active_content.py 156 LLM10 Unbounded Consumption LLM06:2026
|
||||
old src/llm_ingestion_guard/active_content.py 199 LLM10 Unbounded Consumption LLM06:2026
|
||||
old src/llm_ingestion_guard/active_content.py 277 LLM10 Unbounded Consumption LLM06:2026
|
||||
old src/llm_ingestion_guard/active_content.py 449 LLM10 Unbounded Consumption LLM06:2026
|
||||
old src/llm_ingestion_guard/active_content.py 462 LLM10 Unbounded Consumption LLM06:2026
|
||||
old src/llm_ingestion_guard/active_content.py 479 LLM05 Improper Output Handling LLM10:2026
|
||||
old src/llm_ingestion_guard/calibration.py 42 LLM10 Unbounded Consumption LLM06:2026
|
||||
same src/llm_ingestion_guard/coverage.py 25 LLM02 Sensitive Information Disclosure LLM02:2026
|
||||
same src/llm_ingestion_guard/coverage.py 29 LLM02 Sensitive Information Disclosure LLM02:2026
|
||||
same src/llm_ingestion_guard/coverage.py 113 LLM01 Prompt Injection LLM01:2026
|
||||
wrong src/llm_ingestion_guard/coverage.py 126 LLM01 Prompt Injection LLM01:2026 Default tags the OKF path-traversal, absolute-path, reserved-name, non-.md and bundle-escape rows as Prompt Injection; none of them is
|
||||
same src/llm_ingestion_guard/coverage.py 146 LLM01 Prompt Injection LLM01:2026
|
||||
same src/llm_ingestion_guard/coverage.py 467 LLM02 Sensitive Information Disclosure LLM02:2026
|
||||
same src/llm_ingestion_guard/coverage.py 468 LLM02 Sensitive Information Disclosure LLM02:2026
|
||||
old src/llm_ingestion_guard/coverage.py 471 LLM05 Improper Output Handling LLM10:2026
|
||||
old src/llm_ingestion_guard/coverage.py 477 LLM05 Improper Output Handling LLM10:2026
|
||||
old src/llm_ingestion_guard/coverage.py 479 LLM05 Improper Output Handling LLM10:2026
|
||||
old src/llm_ingestion_guard/coverage.py 481 LLM05 Improper Output Handling LLM10:2026
|
||||
old src/llm_ingestion_guard/coverage.py 483 LLM05 Improper Output Handling LLM10:2026
|
||||
old src/llm_ingestion_guard/coverage.py 485 LLM05 Improper Output Handling LLM10:2026
|
||||
old src/llm_ingestion_guard/coverage.py 487 LLM05 Improper Output Handling LLM10:2026
|
||||
old src/llm_ingestion_guard/coverage.py 493 LLM05 Improper Output Handling LLM10:2026
|
||||
old src/llm_ingestion_guard/coverage.py 495 LLM05 Improper Output Handling LLM10:2026
|
||||
old src/llm_ingestion_guard/coverage.py 497 LLM05 Improper Output Handling LLM10:2026
|
||||
same src/llm_ingestion_guard/coverage.py 504 LLM02 Sensitive Information Disclosure LLM02:2026
|
||||
old src/llm_ingestion_guard/coverage.py 511 LLM06 Excessive Agency LLM03:2026
|
||||
same src/llm_ingestion_guard/coverage.py 515 LLM02 Sensitive Information Disclosure LLM02:2026
|
||||
old src/llm_ingestion_guard/coverage.py 517 LLM06 Excessive Agency LLM03:2026
|
||||
wrong src/llm_ingestion_guard/coverage.py 527 LLM10 Unbounded Consumption LLM06:2026 Fail-secure disposition is not Unbounded Consumption in any edition; it is the guard's own failure semantics
|
||||
old src/llm_ingestion_guard/coverage.py 538 LLM10 Unbounded Consumption LLM06:2026
|
||||
old src/llm_ingestion_guard/coverage.py 540 LLM10 Unbounded Consumption LLM06:2026
|
||||
wrong src/llm_ingestion_guard/coverage.py 542 LLM10 Unbounded Consumption LLM06:2026 A frontmatter shape the parser refuses to interpret is parser-surface reduction, not resource consumption; no edition defines it
|
||||
wrong src/llm_ingestion_guard/coverage.py 544 LLM10 Unbounded Consumption LLM06:2026 A frontmatter shape the parser refuses to interpret is parser-surface reduction, not resource consumption; no edition defines it
|
||||
wrong src/llm_ingestion_guard/coverage.py 552 LLM10 Unbounded Consumption LLM06:2026 A frontmatter shape the parser refuses to interpret is parser-surface reduction, not resource consumption; no edition defines it
|
||||
wrong src/llm_ingestion_guard/coverage.py 555 LLM10 Unbounded Consumption LLM06:2026 A frontmatter shape the parser refuses to interpret is parser-surface reduction, not resource consumption; no edition defines it
|
||||
wrong src/llm_ingestion_guard/coverage.py 557 LLM05 Improper Output Handling LLM10:2026 Validates received bundle metadata, not model output; Improper Output Handling is about LLM output passed downstream
|
||||
wrong src/llm_ingestion_guard/coverage.py 559 LLM05 Improper Output Handling LLM10:2026 Validates received bundle metadata, not model output; Improper Output Handling is about LLM output passed downstream
|
||||
wrong src/llm_ingestion_guard/coverage.py 561 LLM05 Improper Output Handling LLM10:2026 Validates received bundle metadata, not model output; Improper Output Handling is about LLM output passed downstream
|
||||
wrong src/llm_ingestion_guard/coverage.py 571 LLM05 Improper Output Handling LLM10:2026 Validates received bundle metadata, not model output; Improper Output Handling is about LLM output passed downstream
|
||||
same src/llm_ingestion_guard/coverage.py 583 LLM02 Sensitive Information Disclosure LLM02:2026
|
||||
wrong src/llm_ingestion_guard/coverage.py 586 LLM09 Misinformation LLM07:2026 Factual poisoning of the ingested store (PoisonedRAG) is Data and Model Poisoning or Vector and Embedding Weaknesses, not Misinformation
|
||||
old src/llm_ingestion_guard/coverage.py 595 LLM05 Improper Output Handling LLM10:2026
|
||||
old src/llm_ingestion_guard/coverage.py 598 LLM05 Improper Output Handling LLM10:2026
|
||||
same src/llm_ingestion_guard/coverage.py 605 LLM01 Prompt Injection LLM01:2026
|
||||
same src/llm_ingestion_guard/coverage.py 624 LLM01 Prompt Injection LLM01:2026
|
||||
same src/llm_ingestion_guard/coverage.py 625 LLM01 Prompt Injection LLM01:2026
|
||||
same src/llm_ingestion_guard/coverage.py 626 LLM01 Prompt Injection LLM01:2026
|
||||
same src/llm_ingestion_guard/coverage.py 627 LLM01 Prompt Injection LLM01:2026
|
||||
same src/llm_ingestion_guard/coverage.py 627 LLM02 Sensitive Information Disclosure LLM02:2026
|
||||
old src/llm_ingestion_guard/coverage.py 628 LLM05 Improper Output Handling LLM10:2026
|
||||
same src/llm_ingestion_guard/coverage.py 629 LLM02 Sensitive Information Disclosure LLM02:2026
|
||||
same src/llm_ingestion_guard/coverage.py 630 LLM02 Sensitive Information Disclosure LLM02:2026
|
||||
old src/llm_ingestion_guard/coverage.py 630 LLM06 Excessive Agency LLM03:2026
|
||||
old src/llm_ingestion_guard/entropy.py 31 LLM10 Unbounded Consumption LLM06:2026
|
||||
same src/llm_ingestion_guard/entropy.py 244 LLM01 Prompt Injection LLM01:2026
|
||||
same src/llm_ingestion_guard/fence.py 99 LLM01 Prompt Injection LLM01:2026
|
||||
same src/llm_ingestion_guard/injection_lexicon.json 10 LLM01 Prompt Injection LLM01:2026
|
||||
same src/llm_ingestion_guard/injection_lexicon.json 18 LLM01 Prompt Injection LLM01:2026
|
||||
same src/llm_ingestion_guard/injection_lexicon.json 26 LLM01 Prompt Injection LLM01:2026
|
||||
same src/llm_ingestion_guard/injection_lexicon.json 34 LLM01 Prompt Injection LLM01:2026
|
||||
same src/llm_ingestion_guard/injection_lexicon.json 42 LLM01 Prompt Injection LLM01:2026
|
||||
same src/llm_ingestion_guard/injection_lexicon.json 50 LLM01 Prompt Injection LLM01:2026
|
||||
same src/llm_ingestion_guard/injection_lexicon.json 58 LLM01 Prompt Injection LLM01:2026
|
||||
same src/llm_ingestion_guard/injection_lexicon.json 66 LLM01 Prompt Injection LLM01:2026
|
||||
same src/llm_ingestion_guard/injection_lexicon.json 74 LLM01 Prompt Injection LLM01:2026
|
||||
same src/llm_ingestion_guard/injection_lexicon.json 82 LLM01 Prompt Injection LLM01:2026
|
||||
same src/llm_ingestion_guard/injection_lexicon.json 90 LLM01 Prompt Injection LLM01:2026
|
||||
same src/llm_ingestion_guard/injection_lexicon.json 98 LLM01 Prompt Injection LLM01:2026
|
||||
same src/llm_ingestion_guard/injection_lexicon.json 106 LLM01 Prompt Injection LLM01:2026
|
||||
same src/llm_ingestion_guard/injection_lexicon.json 114 LLM01 Prompt Injection LLM01:2026
|
||||
same src/llm_ingestion_guard/injection_lexicon.json 122 LLM01 Prompt Injection LLM01:2026
|
||||
same src/llm_ingestion_guard/injection_lexicon.json 130 LLM01 Prompt Injection LLM01:2026
|
||||
same src/llm_ingestion_guard/injection_lexicon.json 138 LLM01 Prompt Injection LLM01:2026
|
||||
same src/llm_ingestion_guard/injection_lexicon.json 146 LLM01 Prompt Injection LLM01:2026
|
||||
same src/llm_ingestion_guard/injection_lexicon.json 154 LLM01 Prompt Injection LLM01:2026
|
||||
same src/llm_ingestion_guard/injection_lexicon.json 162 LLM01 Prompt Injection LLM01:2026
|
||||
same src/llm_ingestion_guard/injection_lexicon.json 170 LLM01 Prompt Injection LLM01:2026
|
||||
same src/llm_ingestion_guard/injection_lexicon.json 178 LLM01 Prompt Injection LLM01:2026
|
||||
same src/llm_ingestion_guard/injection_lexicon.json 186 LLM01 Prompt Injection LLM01:2026
|
||||
same src/llm_ingestion_guard/injection_lexicon.json 194 LLM01 Prompt Injection LLM01:2026
|
||||
same src/llm_ingestion_guard/injection_lexicon.json 202 LLM01 Prompt Injection LLM01:2026
|
||||
same src/llm_ingestion_guard/injection_lexicon.json 210 LLM01 Prompt Injection LLM01:2026
|
||||
same src/llm_ingestion_guard/injection_lexicon.json 218 LLM01 Prompt Injection LLM01:2026
|
||||
same src/llm_ingestion_guard/injection_lexicon.json 226 LLM01 Prompt Injection LLM01:2026
|
||||
same src/llm_ingestion_guard/injection_lexicon.json 234 LLM01 Prompt Injection LLM01:2026
|
||||
same src/llm_ingestion_guard/injection_lexicon.json 242 LLM01 Prompt Injection LLM01:2026
|
||||
same src/llm_ingestion_guard/injection_lexicon.json 250 LLM01 Prompt Injection LLM01:2026
|
||||
same src/llm_ingestion_guard/injection_lexicon.json 258 LLM01 Prompt Injection LLM01:2026
|
||||
same src/llm_ingestion_guard/injection_lexicon.json 266 LLM01 Prompt Injection LLM01:2026
|
||||
same src/llm_ingestion_guard/injection_lexicon.json 274 LLM01 Prompt Injection LLM01:2026
|
||||
same src/llm_ingestion_guard/injection_lexicon.json 282 LLM01 Prompt Injection LLM01:2026
|
||||
same src/llm_ingestion_guard/injection_lexicon.json 290 LLM01 Prompt Injection LLM01:2026
|
||||
same src/llm_ingestion_guard/injection_lexicon.json 298 LLM01 Prompt Injection LLM01:2026
|
||||
same src/llm_ingestion_guard/injection_lexicon.json 306 LLM01 Prompt Injection LLM01:2026
|
||||
same src/llm_ingestion_guard/injection_lexicon.json 314 LLM01 Prompt Injection LLM01:2026
|
||||
same src/llm_ingestion_guard/injection_lexicon.json 322 LLM01 Prompt Injection LLM01:2026
|
||||
same src/llm_ingestion_guard/injection_lexicon.json 330 LLM01 Prompt Injection LLM01:2026
|
||||
same src/llm_ingestion_guard/injection_lexicon.json 338 LLM01 Prompt Injection LLM01:2026
|
||||
same src/llm_ingestion_guard/injection_lexicon.json 346 LLM01 Prompt Injection LLM01:2026
|
||||
same src/llm_ingestion_guard/injection_lexicon.json 354 LLM01 Prompt Injection LLM01:2026
|
||||
same src/llm_ingestion_guard/injection_lexicon.json 362 LLM01 Prompt Injection LLM01:2026
|
||||
same src/llm_ingestion_guard/injection_lexicon.json 370 LLM01 Prompt Injection LLM01:2026
|
||||
same src/llm_ingestion_guard/injection_lexicon.json 378 LLM01 Prompt Injection LLM01:2026
|
||||
same src/llm_ingestion_guard/injection_lexicon.json 386 LLM01 Prompt Injection LLM01:2026
|
||||
same src/llm_ingestion_guard/injection_lexicon.json 394 LLM01 Prompt Injection LLM01:2026
|
||||
same src/llm_ingestion_guard/injection_lexicon.json 402 LLM01 Prompt Injection LLM01:2026
|
||||
same src/llm_ingestion_guard/injection_lexicon.json 410 LLM01 Prompt Injection LLM01:2026
|
||||
same src/llm_ingestion_guard/injection_lexicon.json 418 LLM01 Prompt Injection LLM01:2026
|
||||
same src/llm_ingestion_guard/injection_lexicon.json 426 LLM01 Prompt Injection LLM01:2026
|
||||
same src/llm_ingestion_guard/injection_lexicon.json 434 LLM01 Prompt Injection LLM01:2026
|
||||
same src/llm_ingestion_guard/injection_lexicon.json 442 LLM01 Prompt Injection LLM01:2026
|
||||
same src/llm_ingestion_guard/injection_lexicon.json 450 LLM01 Prompt Injection LLM01:2026
|
||||
same src/llm_ingestion_guard/injection_lexicon.json 458 LLM01 Prompt Injection LLM01:2026
|
||||
same src/llm_ingestion_guard/injection_lexicon.json 466 LLM01 Prompt Injection LLM01:2026
|
||||
same src/llm_ingestion_guard/injection_lexicon.json 474 LLM01 Prompt Injection LLM01:2026
|
||||
same src/llm_ingestion_guard/injection_lexicon.json 482 LLM01 Prompt Injection LLM01:2026
|
||||
same src/llm_ingestion_guard/injection_lexicon.json 490 LLM01 Prompt Injection LLM01:2026
|
||||
same src/llm_ingestion_guard/injection_lexicon.json 498 LLM01 Prompt Injection LLM01:2026
|
||||
same src/llm_ingestion_guard/injection_lexicon.json 506 LLM01 Prompt Injection LLM01:2026
|
||||
same src/llm_ingestion_guard/injection_lexicon.json 514 LLM01 Prompt Injection LLM01:2026
|
||||
same src/llm_ingestion_guard/injection_lexicon.json 522 LLM01 Prompt Injection LLM01:2026
|
||||
same src/llm_ingestion_guard/injection_lexicon.json 530 LLM01 Prompt Injection LLM01:2026
|
||||
same src/llm_ingestion_guard/injection_lexicon.json 538 LLM01 Prompt Injection LLM01:2026
|
||||
same src/llm_ingestion_guard/injection_lexicon.json 546 LLM01 Prompt Injection LLM01:2026
|
||||
same src/llm_ingestion_guard/injection_lexicon.json 554 LLM01 Prompt Injection LLM01:2026
|
||||
same src/llm_ingestion_guard/injection_lexicon.json 562 LLM01 Prompt Injection LLM01:2026
|
||||
same src/llm_ingestion_guard/injection_lexicon.json 570 LLM01 Prompt Injection LLM01:2026
|
||||
same src/llm_ingestion_guard/injection_lexicon.json 578 LLM01 Prompt Injection LLM01:2026
|
||||
same src/llm_ingestion_guard/injection_lexicon.json 586 LLM01 Prompt Injection LLM01:2026
|
||||
same src/llm_ingestion_guard/injection_lexicon.json 594 LLM01 Prompt Injection LLM01:2026
|
||||
same src/llm_ingestion_guard/injection_lexicon.json 602 LLM01 Prompt Injection LLM01:2026
|
||||
same src/llm_ingestion_guard/injection_lexicon.json 610 LLM01 Prompt Injection LLM01:2026
|
||||
same src/llm_ingestion_guard/injection_lexicon.json 618 LLM01 Prompt Injection LLM01:2026
|
||||
same src/llm_ingestion_guard/injection_lexicon.json 626 LLM01 Prompt Injection LLM01:2026
|
||||
same src/llm_ingestion_guard/injection_lexicon.json 634 LLM01 Prompt Injection LLM01:2026
|
||||
same src/llm_ingestion_guard/injection_lexicon.json 642 LLM01 Prompt Injection LLM01:2026
|
||||
same src/llm_ingestion_guard/injection_lexicon.json 650 LLM01 Prompt Injection LLM01:2026
|
||||
same src/llm_ingestion_guard/injection_lexicon.json 658 LLM01 Prompt Injection LLM01:2026
|
||||
same src/llm_ingestion_guard/injection_lexicon.json 666 LLM01 Prompt Injection LLM01:2026
|
||||
old src/llm_ingestion_guard/lexicon.py 23 LLM10 Unbounded Consumption LLM06:2026
|
||||
old src/llm_ingestion_guard/lexicon.py 69 LLM10 Unbounded Consumption LLM06:2026
|
||||
same src/llm_ingestion_guard/lexicon.py 108 LLM01 Prompt Injection LLM01:2026
|
||||
old src/llm_ingestion_guard/lexicon.py 341 LLM10 Unbounded Consumption LLM06:2026
|
||||
same src/llm_ingestion_guard/lexicon.py 381 LLM01 Prompt Injection LLM01:2026
|
||||
same src/llm_ingestion_guard/lexicon.py 394 LLM01 Prompt Injection LLM01:2026
|
||||
old src/llm_ingestion_guard/neutralize.py 10 LLM05 Improper Output Handling LLM10:2026
|
||||
old src/llm_ingestion_guard/neutralize.py 92 LLM05 Improper Output Handling LLM10:2026
|
||||
old src/llm_ingestion_guard/okf.py 491 LLM10 Unbounded Consumption LLM06:2026
|
||||
old src/llm_ingestion_guard/okf.py 514 LLM10 Unbounded Consumption LLM06:2026
|
||||
old src/llm_ingestion_guard/okf.py 568 LLM10 Unbounded Consumption LLM06:2026
|
||||
same src/llm_ingestion_guard/output.py 25 LLM02 Sensitive Information Disclosure LLM02:2026
|
||||
old src/llm_ingestion_guard/output.py 38 LLM05 Improper Output Handling LLM10:2026
|
||||
old src/llm_ingestion_guard/output.py 47 LLM10 Unbounded Consumption LLM06:2026
|
||||
same src/llm_ingestion_guard/output.py 76 LLM02 Sensitive Information Disclosure LLM02:2026
|
||||
same src/llm_ingestion_guard/output.py 218 LLM02 Sensitive Information Disclosure LLM02:2026
|
||||
same src/llm_ingestion_guard/output.py 235 LLM02 Sensitive Information Disclosure LLM02:2026
|
||||
same src/llm_ingestion_guard/output.py 269 LLM01 Prompt Injection LLM01:2026
|
||||
same src/llm_ingestion_guard/output.py 275 LLM01 Prompt Injection LLM01:2026
|
||||
old src/llm_ingestion_guard/output.py 303 LLM10 Unbounded Consumption LLM06:2026
|
||||
same src/llm_ingestion_guard/output.py 336 LLM02 Sensitive Information Disclosure LLM02:2026
|
||||
old src/llm_ingestion_guard/output.py 345 LLM05 Improper Output Handling LLM10:2026
|
||||
old src/llm_ingestion_guard/sanitize.py 80 LLM10 Unbounded Consumption LLM06:2026
|
||||
same src/llm_ingestion_guard/sanitize.py 193 LLM01 Prompt Injection LLM01:2026
|
||||
same src/llm_ingestion_guard/sanitize.py 196 LLM01 Prompt Injection LLM01:2026
|
||||
same src/llm_ingestion_guard/sanitize.py 200 LLM01 Prompt Injection LLM01:2026
|
||||
same src/llm_ingestion_guard/sanitize.py 203 LLM01 Prompt Injection LLM01:2026
|
||||
same src/llm_ingestion_guard/sanitize.py 206 LLM01 Prompt Injection LLM01:2026
|
||||
old tests/inbox_frontend.py 52 LLM10 Unbounded Consumption LLM06:2026
|
||||
old tests/test_active_content.py 8 LLM05 Improper Output Handling LLM10:2026
|
||||
old tests/test_active_content.py 44 LLM05 Improper Output Handling LLM10:2026
|
||||
old tests/test_active_content.py 436 LLM10 Unbounded Consumption LLM06:2026
|
||||
old tests/test_active_content.py 486 LLM10 Unbounded Consumption LLM06:2026
|
||||
old tests/test_active_content.py 500 LLM10 Unbounded Consumption LLM06:2026
|
||||
same tests/test_coverage_matrix.py 11 LLM02 Sensitive Information Disclosure LLM02:2026
|
||||
same tests/test_coverage_matrix.py 77 LLM01 Prompt Injection LLM01:2026
|
||||
same tests/test_coverage_matrix.py 77 LLM02 Sensitive Information Disclosure LLM02:2026
|
||||
old tests/test_coverage_matrix.py 77 LLM05 Improper Output Handling LLM10:2026
|
||||
old tests/test_coverage_matrix.py 77 LLM06 Excessive Agency LLM03:2026
|
||||
wrong tests/test_coverage_matrix.py 77 LLM09 Misinformation LLM07:2026 Asserts the matrix carries the semantic-poisoning row's LLM09; follows that row
|
||||
old tests/test_coverage_matrix.py 77 LLM10 Unbounded Consumption LLM06:2026
|
||||
same tests/test_coverage_matrix.py 81 LLM02 Sensitive Information Disclosure LLM02:2026
|
||||
same tests/test_coverage_matrix.py 120 LLM02 Sensitive Information Disclosure LLM02:2026
|
||||
same tests/test_entropy.py 83 LLM01 Prompt Injection LLM01:2026
|
||||
old tests/test_input_cap.py 1 LLM10 Unbounded Consumption LLM06:2026
|
||||
old tests/test_lexicon.py 6 LLM10 Unbounded Consumption LLM06:2026
|
||||
same tests/test_lexicon.py 95 LLM01 Prompt Injection LLM01:2026
|
||||
old tests/test_lexicon.py 220 LLM10 Unbounded Consumption LLM06:2026
|
||||
old tests/test_lexicon.py 227 LLM10 Unbounded Consumption LLM06:2026
|
||||
old tests/test_lexicon.py 263 LLM10 Unbounded Consumption LLM06:2026
|
||||
old tests/test_neutralize.py 45 LLM05 Improper Output Handling LLM10:2026
|
||||
old tests/test_neutralize.py 175 LLM10 Unbounded Consumption LLM06:2026
|
||||
old tests/test_okf.py 503 LLM10 Unbounded Consumption LLM06:2026
|
||||
old tests/test_okf.py 754 LLM10 Unbounded Consumption LLM06:2026
|
||||
old tests/test_okf.py 1217 LLM10 Unbounded Consumption LLM06:2026
|
||||
old tests/test_okf_showcase.py 81 LLM05 Improper Output Handling LLM10:2026
|
||||
old tests/test_okf_showcase.py 134 LLM05 Improper Output Handling LLM10:2026
|
||||
same tests/test_output.py 12 LLM02 Sensitive Information Disclosure LLM02:2026
|
||||
same tests/test_output.py 132 LLM02 Sensitive Information Disclosure LLM02:2026
|
||||
same tests/test_output.py 205 LLM02 Sensitive Information Disclosure LLM02:2026
|
||||
same tests/test_output.py 212 LLM02 Sensitive Information Disclosure LLM02:2026
|
||||
old tests/test_output.py 318 LLM10 Unbounded Consumption LLM06:2026
|
||||
old tests/test_output.py 325 LLM10 Unbounded Consumption LLM06:2026
|
||||
old tests/test_output.py 381 LLM10 Unbounded Consumption LLM06:2026
|
||||
old tests/test_sanitize.py 81 LLM10 Unbounded Consumption LLM06:2026
|
||||
same tests/test_showcase.py 67 LLM02 Sensitive Information Disclosure LLM02:2026
|
||||
old tests/test_showcase.py 69 LLM05 Improper Output Handling LLM10:2026
|
||||
same tests/test_showcase.py 117 LLM02 Sensitive Information Disclosure LLM02:2026
|
||||
old tests/test_showcase.py 119 LLM05 Improper Output Handling LLM10:2026
|
||||
same tests/test_showcase.py 138 LLM02 Sensitive Information Disclosure LLM02:2026
|
||||
|
63
tests/test_catalog_census.py
Normal file
63
tests/test_catalog_census.py
Normal file
|
|
@ -0,0 +1,63 @@
|
|||
"""The frozen row-2 denominator stays reproducible (`docs/catalog-census.py`).
|
||||
|
||||
Session 1 froze the count of OWASP LLM numbers used with a 2025 or a wrong meaning
|
||||
at v1.4.1 (`6702d7a`): 95 sites in 23 files, listing hash in `docs/PLAN.md` § T.2.
|
||||
The census reads that revision from git objects, so the number may not move when
|
||||
the working tree does. It DOES move if someone edits a classification rule, and
|
||||
that is the point: a re-judged site must re-freeze the listing deliberately, not
|
||||
drift the denominator under a later measurement.
|
||||
|
||||
Skipped outside a git checkout that holds the revision (an sdist has no history).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import importlib.util
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
_ROOT = Path(__file__).resolve().parent.parent
|
||||
_REV = "6702d7a"
|
||||
_FROZEN = _ROOT / "docs" / "threat-2026" / "census-6702d7a.tsv"
|
||||
_FROZEN_SHA256 = "da1aa0fab63636e7e56c520f4856504c156fe4aa223f30b71edd7729fe1da16f"
|
||||
|
||||
|
||||
def _load():
|
||||
spec = importlib.util.spec_from_file_location("_docs_catalog_census",
|
||||
_ROOT / "docs" / "catalog-census.py")
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
sys.modules[spec.name] = module
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
def _have_rev() -> bool:
|
||||
try:
|
||||
subprocess.run(["git", "-C", str(_ROOT), "cat-file", "-e", f"{_REV}^{{commit}}"],
|
||||
check=True, capture_output=True)
|
||||
return True
|
||||
except (OSError, subprocess.CalledProcessError):
|
||||
return False
|
||||
|
||||
|
||||
def test_frozen_listing_matches_its_pinned_hash():
|
||||
assert hashlib.sha256(_FROZEN.read_bytes()).hexdigest() == _FROZEN_SHA256
|
||||
|
||||
|
||||
@pytest.mark.skipif(not _have_rev(), reason=f"no git history holding {_REV}")
|
||||
def test_census_at_the_frozen_revision_reproduces_the_listing(tmp_path):
|
||||
census = _load()
|
||||
out = tmp_path / "census.tsv"
|
||||
assert census.main(["--rev", _REV, "--tsv", str(out)]) == 0
|
||||
assert out.read_bytes() == _FROZEN.read_bytes()
|
||||
|
||||
|
||||
def test_frozen_denominator_is_95_in_23_files():
|
||||
rows = [line.split("\t") for line in _FROZEN.read_text(encoding="utf-8").splitlines()[1:]]
|
||||
counted = [r for r in rows if r[0] in ("old", "wrong")]
|
||||
assert len(counted) == 95
|
||||
assert len({r[1] for r in counted}) == 23
|
||||
assert sum(r[0] == "wrong" for r in counted) == 13
|
||||
Loading…
Add table
Add a link
Reference in a new issue