feat(watch): source list for the 16 catalogs and the 30-day release rule

docs/threat-2026/watch-sources.json is the catalog register in the edition
watch's input format: 12 entries probed (A, B, F, L x2, O x7), 4 manual as
class S with a reason. Run live on 2026-11-01: 16 current, 12 probes ok.

A frozen copy of the catalogs as v1.4.1 used them is reported behind
(OWASP LLM 2025) and on watch (RAG cheat sheet); the watch calls its Unicode
17.0 current, which only the 30-day clause catches.

tests/test_release_rule.py stops a release whose manual check or watch run
is more than 30 days older than the newest CHANGELOG date, or whose watch
output has a behind, notVerified or watch row. Each clause mutated away
turns a test red (5 of 5). No release: the package is unchanged.

tests/test_catalog_register.py

@@ -6,9 +6,10 @@ is `docs/threat-2026/catalogs.json`; its denominator was frozen at 15 entries in
 session 1 and may grow, never shrink, without the operator's yes.
 
 This file fails when an entry lacks a date or a link, or cites an edition other
-than the one recorded as latest. It deliberately does **not** fail on age: an
-automatic "this check is stale" alarm is outside the approved direction, so a date
-from last year passes as long as it is a date.
+than the one recorded as latest. It does not fail on age: that is the release
+rule in `tests/test_release_rule.py` (direction of 2026-11-01), which judges the
+check dates against the release date, so the suite does not turn red merely
+because a day passed between releases.
 
 The README may only name an OWASP Agentic or MITRE ATLAS entry the register
 records as read at the publisher, and must name the ATLAS release the register

tests/test_release_rule.py

@@ -0,0 +1,226 @@
+"""The release rule: no release with a catalog check older than 30 days, row 1 of the 2026-11 done-criterion.
+
+Row 1 (`docs/PLAN.md` § V.2) asks that each of the 16 catalogs in
+`docs/threat-2026/catalogs.json` is followed by llm-security's edition watch, or
+stands as manual with a reason, and that a test stops a release whose manual check
+is more than 30 days older than the release date.
+
+The source list `docs/threat-2026/watch-sources.json` is the register in the
+watch's own input format: 12 entries carry a probe that makes a request, and 4
+carry class S (no request) with the reason no probe can answer. The newest watch
+output for that list is committed as `docs/threat-2026/watch-last.json`.
+
+The rule, read against the release date (the date in the newest CHANGELOG
+heading, so it binds on the release commit that writes it and on nothing else):
+
+- every manual entry's ``known_at`` is at most 30 days before the release date;
+- the watch output is a run of this list (the same ids) and its ``lastRun`` is at
+  most 30 days before the release date, since that run is the check date of the
+  12 probed entries;
+- the output has no ``behind`` row, no ``notVerified`` row and no ``watch`` row. A
+  ``watch`` row (class L: a living document or a CVE record moved) never counts
+  as behind in the watch's own exit code, so the rule reads it separately: it is
+  cleared by a check by hand that updates the entry's ``known_value``, then a new
+  run.
+
+To release: run the watch on the list (command in `docs/PLAN.md` § V.7) and commit
+its output here. The positive control is a frozen copy of the catalogs as v1.4.1
+(tagged 2026-09-23) used them, `watch-sources-v1.4.1.json`, and its live output
+`watch-v1.4.1-run.json`: the watch reports it behind, and this rule stops it.
+"""
+from __future__ import annotations
+
+import datetime
+import hashlib
+import json
+import re
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parent.parent
+THREAT = ROOT / "docs" / "threat-2026"
+REGISTER = THREAT / "catalogs.json"
+SOURCES = THREAT / "watch-sources.json"
+LAST_RUN = THREAT / "watch-last.json"
+SOURCES_V141 = THREAT / "watch-sources-v1.4.1.json"
+RUN_V141 = THREAT / "watch-v1.4.1-run.json"
+CHANGELOG = ROOT / "CHANGELOG.md"
+
+MAX_AGE_DAYS = 30
+SOURCES_V141_SHA256 = "49551e7c142488b3a86da099583828471ed3d1bf8ed8875f7439865b00dd10da"
+MANUAL_IDS = frozenset({"unicode", "rfc-3986", "llm-security-commons", "willison-dual-llm"})
+# Probe classes that make no request (the watch's own NO_REQUEST_CLASSES).
+NO_REQUEST = frozenset({"S", "R-age"})
+
+_HEADING = re.compile(r"^## \[(\d+\.\d+\.\d+)\] \S+ (\d{4}-\d{2}-\d{2})\s*$", re.M)
+
+
+def _json(path: Path):
+    return json.loads(path.read_text(encoding="utf-8"))
+
+
+def release_date() -> datetime.date:
+    """The date of the newest CHANGELOG heading: the release the tree describes."""
+    m = _HEADING.search(CHANGELOG.read_text(encoding="utf-8"))
+    assert m, "no release heading in CHANGELOG.md"
+    return datetime.date.fromisoformat(m.group(2))
+
+
+def _run_ids(run: dict) -> list[str]:
+    ids = [r["id"] for key in ("behind", "notVerified", "watch") for r in run[key]]
+    return ids + list(run["current"])
+
+
+def problems(sources: list[dict], run: dict, released: datetime.date) -> list[str]:
+    """Why a release on ``released`` must stop; empty when it may go."""
+    out = []
+    oldest = released - datetime.timedelta(days=MAX_AGE_DAYS)
+    for e in sources:
+        if e["probe"]["class"] in NO_REQUEST:
+            checked = datetime.date.fromisoformat(e["probe"]["known_at"])
+            if checked < oldest:
+                out.append(f"manual check of {e['id']} on {checked} is more than {MAX_AGE_DAYS} days before {released}")
+    if sorted(_run_ids(run)) != sorted(e["id"] for e in sources):
+        out.append("the watch output is not a run of this source list")
+    ran = datetime.datetime.fromisoformat(run["lastRun"].replace("Z", "+00:00")).date()
+    if ran < oldest:
+        out.append(f"the watch last ran on {ran}, more than {MAX_AGE_DAYS} days before {released}")
+    for key in ("behind", "notVerified", "watch"):
+        for r in run[key]:
+            out.append(f"{key}: {r['id']}")
+    return out
+
+
+# --- the rule on the tree as it stands -----------------------------------------
+
+
+def test_this_tree_may_be_released():
+    assert problems(_json(SOURCES), _json(LAST_RUN), release_date()) == []
+
+
+# --- the source list is the register in the watch's format -----------------------
+
+
+def test_source_list_covers_every_register_entry():
+    register = {e["id"]: e for e in _json(REGISTER)["entries"]}
+    sources = _json(SOURCES)
+    assert [e["id"] for e in sources] == list(register)
+    for e in sources:
+        r = register[e["id"]]
+        assert e["used_edition"] == r["cited"], e["id"]
+        assert e["newest_edition"] == r["latest"], e["id"]
+        assert e["newest_source_url"] == r["source"], e["id"]
+        assert e["checked"] == r["checked"], e["id"]
+
+
+def test_twelve_probed_and_four_manual_with_a_reason():
+    sources = _json(SOURCES)
+    manual = {e["id"] for e in sources if e["probe"]["class"] in NO_REQUEST}
+    assert manual == MANUAL_IDS
+    assert len(sources) - len(manual) == 12
+    register = {e["id"]: e for e in _json(REGISTER)["entries"]}
+    for e in sources:
+        p = e["probe"]
+        datetime.date.fromisoformat(p["known_at"])
+        if e["id"] in MANUAL_IDS:
+            assert p["class"] == "S" and p["known_value"] is None
+            assert p["reason"].startswith("manual: "), e["id"]
+            # The manual check date is the register's check date: one truth.
+            assert p["known_at"] == register[e["id"]]["checked"], e["id"]
+        else:
+            assert p["known_value"], e["id"]
+
+
+def test_arxiv_entries_stand_one_per_paper():
+    # Seven ids in one entry exceed the watch's 80-character VERSION_RE (PLAN § V.6).
+    for e in _json(SOURCES):
+        if e["probe"]["class"] == "O":
+            assert e["probe"]["known_value"] == [e["id"].removeprefix("arxiv-") + e["used_edition"]]
+
+
+# --- the positive control: the catalogs as v1.4.1 used them ---------------------
+
+
+def test_v141_copy_is_frozen():
+    assert hashlib.sha256(SOURCES_V141.read_bytes()).hexdigest() == SOURCES_V141_SHA256
+
+
+def test_v141_copy_holds_only_what_v141_cited():
+    register = {e["id"]: e for e in _json(REGISTER)["entries"]}
+    copy = {e["id"]: e for e in _json(SOURCES_V141)}
+    assert set(copy) == {i for i, r in register.items() if r["cited_at_v1_4_1"] != "not cited"}
+    assert copy["owasp-llm-top10"]["used_edition"] == "2025"
+    assert copy["unicode"]["used_edition"] == "17.0"
+
+
+def test_watch_reports_v141_behind_and_the_rule_stops_it():
+    sources, run = _json(SOURCES_V141), _json(RUN_V141)
+    assert [r["id"] for r in run["behind"]] == ["owasp-llm-top10"]
+    assert [(r["id"], r["observed"][:8]) for r in run["watch"]] == [("owasp-rag-cheat-sheet", "510ea484")]
+    # The watch says "current" for unicode at 17.0 (class S makes no request);
+    # only the 30-day clause catches it, together with the other three manual
+    # entries whose last check was the v1.4.1 tag date.
+    assert "unicode" in run["current"]
+    found = problems(sources, run, datetime.date(2026, 11, 1))
+    assert found == [
+        f"manual check of {i} on 2026-09-23 is more than 30 days before 2026-11-01"
+        for i in ("unicode", "willison-dual-llm", "rfc-3986", "llm-security-commons")
+    ] + ["behind: owasp-llm-top10", "watch: owasp-rag-cheat-sheet"]
+
+
+# --- each clause of the rule goes red on its own ------------------------------
+
+
+def _clean() -> tuple[list[dict], dict, datetime.date]:
+    return _json(SOURCES), _json(LAST_RUN), datetime.date(2026, 11, 1)
+
+
+def test_manual_check_31_days_old_stops_and_30_does_not():
+    sources, run, _ = _clean()
+    checked = next(e for e in sources if e["id"] == "unicode")["probe"]["known_at"]
+    day = datetime.date.fromisoformat(checked)
+    ran = datetime.datetime.fromisoformat(run["lastRun"].replace("Z", "+00:00")).date()
+    assert ran >= day  # the run is not the older of the two
+    assert problems(sources, run, day + datetime.timedelta(days=30)) == []
+    assert problems(sources, run, day + datetime.timedelta(days=31)) == [
+        f"manual check of {e['id']} on {e['probe']['known_at']} is more than 30 days before "
+        f"{day + datetime.timedelta(days=31)}"
+        for e in sources
+        if e["id"] in MANUAL_IDS
+    ]
+
+
+def test_old_watch_run_stops():
+    sources, run, released = _clean()
+    run["lastRun"] = (released - datetime.timedelta(days=31)).isoformat() + "T09:15:00.000Z"
+    assert problems(sources, run, released) == [
+        f"the watch last ran on {released - datetime.timedelta(days=31)}, more than 30 days before {released}"
+    ]
+
+
+def test_watch_row_stops_though_the_watch_does_not_count_it_behind():
+    sources, run, released = _clean()
+    run["current"].remove("cve-2025-32711")
+    run["watch"].append({"id": "cve-2025-32711", "signal": "watch-only", "observed": "2026-12-01T00:00:00.000Z"})
+    assert problems(sources, run, released) == ["watch: cve-2025-32711"]
+
+
+def test_not_verified_stops():
+    sources, run, released = _clean()
+    run["current"].remove("mitre-atlas")
+    run["notVerified"].append({"id": "mitre-atlas", "probe_status": "unreachable", "reason": "deadline reached"})
+    assert problems(sources, run, released) == ["notVerified: mitre-atlas"]
+
+
+def test_output_of_another_list_stops():
+    sources, run, released = _clean()
+    run["current"].remove("rfc-3986")
+    assert problems(sources, run, released) == ["the watch output is not a run of this source list"]
+
+
+def test_release_date_belongs_to_the_version_being_released():
+    # The rule must read the heading of the version pyproject declares, or a
+    # release commit that forgets its CHANGELOG heading would be judged on the
+    # previous release's date.
+    m = _HEADING.search(CHANGELOG.read_text(encoding="utf-8"))
+    version = re.search(r'^version = "([^"]+)"', (ROOT / "pyproject.toml").read_text(encoding="utf-8"), re.M)
+    assert m and version and m.group(1) == version.group(1)