Kjell Tore Guttormsen
2e3d33efb4
docs/catalog-census.py classifies every OWASP LLM number in tracked files:
old (2025 meaning, maps mechanically), wrong (a meaning no edition gave it,
each with its reason), same, current, history, migration-plan, self.
At 6702d7a the denominator is 95 sites in 23 files (82 old, 13 wrong);
the listing is frozen with its sha256 and a test reproduces it from git.
63 lines
2.3 KiB
Python
63 lines
2.3 KiB
Python
"""The frozen row-2 denominator stays reproducible (`docs/catalog-census.py`).
|
|
|
|
Session 1 froze the count of OWASP LLM numbers used with a 2025 or a wrong meaning
|
|
at v1.4.1 (`6702d7a`): 95 sites in 23 files, listing hash in `docs/PLAN.md` § T.2.
|
|
The census reads that revision from git objects, so the number may not move when
|
|
the working tree does. It DOES move if someone edits a classification rule, and
|
|
that is the point: a re-judged site must re-freeze the listing deliberately, not
|
|
drift the denominator under a later measurement.
|
|
|
|
Skipped outside a git checkout that holds the revision (an sdist has no history).
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import importlib.util
|
|
import subprocess
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
_ROOT = Path(__file__).resolve().parent.parent
|
|
_REV = "6702d7a"
|
|
_FROZEN = _ROOT / "docs" / "threat-2026" / "census-6702d7a.tsv"
|
|
_FROZEN_SHA256 = "da1aa0fab63636e7e56c520f4856504c156fe4aa223f30b71edd7729fe1da16f"
|
|
|
|
|
|
def _load():
|
|
spec = importlib.util.spec_from_file_location("_docs_catalog_census",
|
|
_ROOT / "docs" / "catalog-census.py")
|
|
module = importlib.util.module_from_spec(spec)
|
|
sys.modules[spec.name] = module
|
|
spec.loader.exec_module(module)
|
|
return module
|
|
|
|
|
|
def _have_rev() -> bool:
|
|
try:
|
|
subprocess.run(["git", "-C", str(_ROOT), "cat-file", "-e", f"{_REV}^{{commit}}"],
|
|
check=True, capture_output=True)
|
|
return True
|
|
except (OSError, subprocess.CalledProcessError):
|
|
return False
|
|
|
|
|
|
def test_frozen_listing_matches_its_pinned_hash():
|
|
assert hashlib.sha256(_FROZEN.read_bytes()).hexdigest() == _FROZEN_SHA256
|
|
|
|
|
|
@pytest.mark.skipif(not _have_rev(), reason=f"no git history holding {_REV}")
|
|
def test_census_at_the_frozen_revision_reproduces_the_listing(tmp_path):
|
|
census = _load()
|
|
out = tmp_path / "census.tsv"
|
|
assert census.main(["--rev", _REV, "--tsv", str(out)]) == 0
|
|
assert out.read_bytes() == _FROZEN.read_bytes()
|
|
|
|
|
|
def test_frozen_denominator_is_95_in_23_files():
|
|
rows = [line.split("\t") for line in _FROZEN.read_text(encoding="utf-8").splitlines()[1:]]
|
|
counted = [r for r in rows if r[0] in ("old", "wrong")]
|
|
assert len(counted) == 95
|
|
assert len({r[1] for r in counted}) == 23
|
|
assert sum(r[0] == "wrong" for r in counted) == 13
|