"""The release rule: no release with a catalog check older than 30 days, row 1 of the 2026-11 done-criterion. Row 1 (`docs/PLAN.md` § V.2) asks that each of the 16 catalogs in `docs/threat-2026/catalogs.json` is followed by llm-security's edition watch, or stands as manual with a reason, and that a test stops a release whose manual check is more than 30 days older than the release date. The source list `docs/threat-2026/watch-sources.json` is the register in the watch's own input format: 12 entries carry a probe that makes a request, and 4 carry class S (no request) with the reason no probe can answer. The newest watch output for that list is committed as `docs/threat-2026/watch-last.json`. The rule, read against the release date (the date in the newest CHANGELOG heading, so it binds on the release commit that writes it and on nothing else): - every manual entry's ``known_at`` is at most 30 days before the release date; - the watch output is a run of this list (the same ids) and its ``lastRun`` is at most 30 days before the release date, since that run is the check date of the 12 probed entries; - the output has no ``behind`` row, no ``notVerified`` row and no ``watch`` row. A ``watch`` row (class L: a living document or a CVE record moved) never counts as behind in the watch's own exit code, so the rule reads it separately: it is cleared by a check by hand that updates the entry's ``known_value``, then a new run. To release: run the watch on the list (command in `docs/PLAN.md` § V.7) and commit its output here. The positive control is a frozen copy of the catalogs as v1.4.1 (tagged 2026-09-23) used them, `watch-sources-v1.4.1.json`, and its live output `watch-v1.4.1-run.json`: the watch reports it behind, and this rule stops it. """ from __future__ import annotations import datetime import hashlib import json import re from pathlib import Path ROOT = Path(__file__).resolve().parent.parent THREAT = ROOT / "docs" / "threat-2026" REGISTER = THREAT / "catalogs.json" SOURCES = THREAT / "watch-sources.json" LAST_RUN = THREAT / "watch-last.json" SOURCES_V141 = THREAT / "watch-sources-v1.4.1.json" RUN_V141 = THREAT / "watch-v1.4.1-run.json" CHANGELOG = ROOT / "CHANGELOG.md" MAX_AGE_DAYS = 30 SOURCES_V141_SHA256 = "49551e7c142488b3a86da099583828471ed3d1bf8ed8875f7439865b00dd10da" MANUAL_IDS = frozenset({"unicode", "rfc-3986", "llm-security-commons", "willison-dual-llm"}) # Probe classes that make no request (the watch's own NO_REQUEST_CLASSES). NO_REQUEST = frozenset({"S", "R-age"}) _HEADING = re.compile(r"^## \[(\d+\.\d+\.\d+)\] \S+ (\d{4}-\d{2}-\d{2})\s*$", re.M) def _json(path: Path): return json.loads(path.read_text(encoding="utf-8")) def release_date() -> datetime.date: """The date of the newest CHANGELOG heading: the release the tree describes.""" m = _HEADING.search(CHANGELOG.read_text(encoding="utf-8")) assert m, "no release heading in CHANGELOG.md" return datetime.date.fromisoformat(m.group(2)) def _run_ids(run: dict) -> list[str]: ids = [r["id"] for key in ("behind", "notVerified", "watch") for r in run[key]] return ids + list(run["current"]) def problems(sources: list[dict], run: dict, released: datetime.date) -> list[str]: """Why a release on ``released`` must stop; empty when it may go.""" out = [] oldest = released - datetime.timedelta(days=MAX_AGE_DAYS) for e in sources: if e["probe"]["class"] in NO_REQUEST: checked = datetime.date.fromisoformat(e["probe"]["known_at"]) if checked < oldest: out.append(f"manual check of {e['id']} on {checked} is more than {MAX_AGE_DAYS} days before {released}") if sorted(_run_ids(run)) != sorted(e["id"] for e in sources): out.append("the watch output is not a run of this source list") ran = datetime.datetime.fromisoformat(run["lastRun"].replace("Z", "+00:00")).date() if ran < oldest: out.append(f"the watch last ran on {ran}, more than {MAX_AGE_DAYS} days before {released}") for key in ("behind", "notVerified", "watch"): for r in run[key]: out.append(f"{key}: {r['id']}") return out # --- the rule on the tree as it stands ----------------------------------------- def test_this_tree_may_be_released(): assert problems(_json(SOURCES), _json(LAST_RUN), release_date()) == [] # --- the source list is the register in the watch's format ----------------------- def test_source_list_covers_every_register_entry(): register = {e["id"]: e for e in _json(REGISTER)["entries"]} sources = _json(SOURCES) assert [e["id"] for e in sources] == list(register) for e in sources: r = register[e["id"]] assert e["used_edition"] == r["cited"], e["id"] assert e["newest_edition"] == r["latest"], e["id"] assert e["newest_source_url"] == r["source"], e["id"] assert e["checked"] == r["checked"], e["id"] def test_twelve_probed_and_four_manual_with_a_reason(): sources = _json(SOURCES) manual = {e["id"] for e in sources if e["probe"]["class"] in NO_REQUEST} assert manual == MANUAL_IDS assert len(sources) - len(manual) == 12 register = {e["id"]: e for e in _json(REGISTER)["entries"]} for e in sources: p = e["probe"] datetime.date.fromisoformat(p["known_at"]) if e["id"] in MANUAL_IDS: assert p["class"] == "S" and p["known_value"] is None assert p["reason"].startswith("manual: "), e["id"] # The manual check date is the register's check date: one truth. assert p["known_at"] == register[e["id"]]["checked"], e["id"] else: assert p["known_value"], e["id"] def test_arxiv_entries_stand_one_per_paper(): # Seven ids in one entry exceed the watch's 80-character VERSION_RE (PLAN § V.6). for e in _json(SOURCES): if e["probe"]["class"] == "O": assert e["probe"]["known_value"] == [e["id"].removeprefix("arxiv-") + e["used_edition"]] # --- the positive control: the catalogs as v1.4.1 used them --------------------- def test_v141_copy_is_frozen(): assert hashlib.sha256(SOURCES_V141.read_bytes()).hexdigest() == SOURCES_V141_SHA256 def test_v141_copy_holds_only_what_v141_cited(): register = {e["id"]: e for e in _json(REGISTER)["entries"]} copy = {e["id"]: e for e in _json(SOURCES_V141)} assert set(copy) == {i for i, r in register.items() if r["cited_at_v1_4_1"] != "not cited"} assert copy["owasp-llm-top10"]["used_edition"] == "2025" assert copy["unicode"]["used_edition"] == "17.0" def test_watch_reports_v141_behind_and_the_rule_stops_it(): sources, run = _json(SOURCES_V141), _json(RUN_V141) assert [r["id"] for r in run["behind"]] == ["owasp-llm-top10"] assert [(r["id"], r["observed"][:8]) for r in run["watch"]] == [("owasp-rag-cheat-sheet", "510ea484")] # The watch says "current" for unicode at 17.0 (class S makes no request); # only the 30-day clause catches it, together with the other three manual # entries whose last check was the v1.4.1 tag date. assert "unicode" in run["current"] found = problems(sources, run, datetime.date(2026, 11, 1)) assert found == [ f"manual check of {i} on 2026-09-23 is more than 30 days before 2026-11-01" for i in ("unicode", "willison-dual-llm", "rfc-3986", "llm-security-commons") ] + ["behind: owasp-llm-top10", "watch: owasp-rag-cheat-sheet"] # --- each clause of the rule goes red on its own ------------------------------ def _clean() -> tuple[list[dict], dict, datetime.date]: return _json(SOURCES), _json(LAST_RUN), datetime.date(2026, 11, 1) def test_manual_check_31_days_old_stops_and_30_does_not(): sources, run, _ = _clean() checked = next(e for e in sources if e["id"] == "unicode")["probe"]["known_at"] day = datetime.date.fromisoformat(checked) ran = datetime.datetime.fromisoformat(run["lastRun"].replace("Z", "+00:00")).date() assert ran >= day # the run is not the older of the two assert problems(sources, run, day + datetime.timedelta(days=30)) == [] assert problems(sources, run, day + datetime.timedelta(days=31)) == [ f"manual check of {e['id']} on {e['probe']['known_at']} is more than 30 days before " f"{day + datetime.timedelta(days=31)}" for e in sources if e["id"] in MANUAL_IDS ] def test_old_watch_run_stops(): sources, run, released = _clean() run["lastRun"] = (released - datetime.timedelta(days=31)).isoformat() + "T09:15:00.000Z" assert problems(sources, run, released) == [ f"the watch last ran on {released - datetime.timedelta(days=31)}, more than 30 days before {released}" ] def test_watch_row_stops_though_the_watch_does_not_count_it_behind(): sources, run, released = _clean() run["current"].remove("cve-2025-32711") run["watch"].append({"id": "cve-2025-32711", "signal": "watch-only", "observed": "2026-12-01T00:00:00.000Z"}) assert problems(sources, run, released) == ["watch: cve-2025-32711"] def test_not_verified_stops(): sources, run, released = _clean() run["current"].remove("mitre-atlas") run["notVerified"].append({"id": "mitre-atlas", "probe_status": "unreachable", "reason": "deadline reached"}) assert problems(sources, run, released) == ["notVerified: mitre-atlas"] def test_output_of_another_list_stops(): sources, run, released = _clean() run["current"].remove("rfc-3986") assert problems(sources, run, released) == ["the watch output is not a run of this source list"] def test_release_date_belongs_to_the_version_being_released(): # The rule must read the heading of the version pyproject declares, or a # release commit that forgets its CHANGELOG heading would be judged on the # previous release's date. m = _HEADING.search(CHANGELOG.read_text(encoding="utf-8")) version = re.search(r'^version = "([^"]+)"', (ROOT / "pyproject.toml").read_text(encoding="utf-8"), re.M) assert m and version and m.group(1) == version.group(1)