test(threat-2026): freeze the catalog-number census at v1.4.1
docs/catalog-census.py classifies every OWASP LLM number in tracked files:
old (2025 meaning, maps mechanically), wrong (a meaning no edition gave it,
each with its reason), same, current, history, migration-plan, self.
At 6702d7a the denominator is 95 sites in 23 files (82 old, 13 wrong);
the listing is frozen with its sha256 and a test reproduces it from git.
This commit is contained in:
parent
61475686bf
commit
2e3d33efb4
3 changed files with 564 additions and 0 deletions
63
tests/test_catalog_census.py
Normal file
63
tests/test_catalog_census.py
Normal file
|
|
@ -0,0 +1,63 @@
|
|||
"""The frozen row-2 denominator stays reproducible (`docs/catalog-census.py`).
|
||||
|
||||
Session 1 froze the count of OWASP LLM numbers used with a 2025 or a wrong meaning
|
||||
at v1.4.1 (`6702d7a`): 95 sites in 23 files, listing hash in `docs/PLAN.md` § T.2.
|
||||
The census reads that revision from git objects, so the number may not move when
|
||||
the working tree does. It DOES move if someone edits a classification rule, and
|
||||
that is the point: a re-judged site must re-freeze the listing deliberately, not
|
||||
drift the denominator under a later measurement.
|
||||
|
||||
Skipped outside a git checkout that holds the revision (an sdist has no history).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import importlib.util
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
_ROOT = Path(__file__).resolve().parent.parent
|
||||
_REV = "6702d7a"
|
||||
_FROZEN = _ROOT / "docs" / "threat-2026" / "census-6702d7a.tsv"
|
||||
_FROZEN_SHA256 = "da1aa0fab63636e7e56c520f4856504c156fe4aa223f30b71edd7729fe1da16f"
|
||||
|
||||
|
||||
def _load():
|
||||
spec = importlib.util.spec_from_file_location("_docs_catalog_census",
|
||||
_ROOT / "docs" / "catalog-census.py")
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
sys.modules[spec.name] = module
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
def _have_rev() -> bool:
|
||||
try:
|
||||
subprocess.run(["git", "-C", str(_ROOT), "cat-file", "-e", f"{_REV}^{{commit}}"],
|
||||
check=True, capture_output=True)
|
||||
return True
|
||||
except (OSError, subprocess.CalledProcessError):
|
||||
return False
|
||||
|
||||
|
||||
def test_frozen_listing_matches_its_pinned_hash():
|
||||
assert hashlib.sha256(_FROZEN.read_bytes()).hexdigest() == _FROZEN_SHA256
|
||||
|
||||
|
||||
@pytest.mark.skipif(not _have_rev(), reason=f"no git history holding {_REV}")
|
||||
def test_census_at_the_frozen_revision_reproduces_the_listing(tmp_path):
|
||||
census = _load()
|
||||
out = tmp_path / "census.tsv"
|
||||
assert census.main(["--rev", _REV, "--tsv", str(out)]) == 0
|
||||
assert out.read_bytes() == _FROZEN.read_bytes()
|
||||
|
||||
|
||||
def test_frozen_denominator_is_95_in_23_files():
|
||||
rows = [line.split("\t") for line in _FROZEN.read_text(encoding="utf-8").splitlines()[1:]]
|
||||
counted = [r for r in rows if r[0] in ("old", "wrong")]
|
||||
assert len(counted) == 95
|
||||
assert len({r[1] for r in counted}) == 23
|
||||
assert sum(r[0] == "wrong" for r in counted) == 13
|
||||
Loading…
Add table
Add a link
Reference in a new issue