llm-ingestion-pipeline-security/tests/test_release_rule.py

"""The release rule: no release with a catalog check older than 30 days, row 1 of the 2026-11 done-criterion.

Row 1 (`docs/PLAN.md` § V.2) asks that each of the 16 catalogs in
`docs/threat-2026/catalogs.json` is followed by llm-security's edition watch, or
stands as manual with a reason, and that a test stops a release whose manual check
is more than 30 days older than the release date.

The source list `docs/threat-2026/watch-sources.json` is the register in the
watch's own input format: 12 entries carry a probe that makes a request, and 4
carry class S (no request) with the reason no probe can answer. The newest watch
output for that list is committed as `docs/threat-2026/watch-last.json`.

The rule, read against the release date (the date in the newest CHANGELOG
heading, so it binds on the release commit that writes it and on nothing else):

- every manual entry's ``known_at`` is at most 30 days before the release date;
- the watch output is a run of this list (the same ids) and its ``lastRun`` is at
  most 30 days before the release date, since that run is the check date of the
  12 probed entries;
- the output has no ``behind`` row, no ``notVerified`` row and no ``watch`` row. A
  ``watch`` row (class L: a living document or a CVE record moved) never counts
  as behind in the watch's own exit code, so the rule reads it separately: it is
  cleared by a check by hand that updates the entry's ``known_value``, then a new
  run.

To release: run the watch on the list (command in `docs/PLAN.md` § V.7) and commit
its output here. The positive control is a frozen copy of the catalogs as v1.4.1
(tagged 2026-09-23) used them, `watch-sources-v1.4.1.json`, and its live output
`watch-v1.4.1-run.json`: the watch reports it behind, and this rule stops it.
"""
from __future__ import annotations

import datetime
import hashlib
import json
import re
from pathlib import Path

ROOT = Path(__file__).resolve().parent.parent
THREAT = ROOT / "docs" / "threat-2026"
REGISTER = THREAT / "catalogs.json"
SOURCES = THREAT / "watch-sources.json"
LAST_RUN = THREAT / "watch-last.json"
SOURCES_V141 = THREAT / "watch-sources-v1.4.1.json"
RUN_V141 = THREAT / "watch-v1.4.1-run.json"
CHANGELOG = ROOT / "CHANGELOG.md"

MAX_AGE_DAYS = 30
SOURCES_V141_SHA256 = "49551e7c142488b3a86da099583828471ed3d1bf8ed8875f7439865b00dd10da"
MANUAL_IDS = frozenset({"unicode", "rfc-3986", "llm-security-commons", "willison-dual-llm"})
# Probe classes that make no request (the watch's own NO_REQUEST_CLASSES).
NO_REQUEST = frozenset({"S", "R-age"})

_HEADING = re.compile(r"^## \[(\d+\.\d+\.\d+)\] \S+ (\d{4}-\d{2}-\d{2})\s*$", re.M)


def _json(path: Path):
    return json.loads(path.read_text(encoding="utf-8"))


def release_date() -> datetime.date:
    """The date of the newest CHANGELOG heading: the release the tree describes."""
    m = _HEADING.search(CHANGELOG.read_text(encoding="utf-8"))
    assert m, "no release heading in CHANGELOG.md"
    return datetime.date.fromisoformat(m.group(2))


def _run_ids(run: dict) -> list[str]:
    ids = [r["id"] for key in ("behind", "notVerified", "watch") for r in run[key]]
    return ids + list(run["current"])


def problems(sources: list[dict], run: dict, released: datetime.date) -> list[str]:
    """Why a release on ``released`` must stop; empty when it may go."""
    out = []
    oldest = released - datetime.timedelta(days=MAX_AGE_DAYS)
    for e in sources:
        if e["probe"]["class"] in NO_REQUEST:
            checked = datetime.date.fromisoformat(e["probe"]["known_at"])
            if checked < oldest:
                out.append(f"manual check of {e['id']} on {checked} is more than {MAX_AGE_DAYS} days before {released}")
    if sorted(_run_ids(run)) != sorted(e["id"] for e in sources):
        out.append("the watch output is not a run of this source list")
    ran = datetime.datetime.fromisoformat(run["lastRun"].replace("Z", "+00:00")).date()
    if ran < oldest:
        out.append(f"the watch last ran on {ran}, more than {MAX_AGE_DAYS} days before {released}")
    for key in ("behind", "notVerified", "watch"):
        for r in run[key]:
            out.append(f"{key}: {r['id']}")
    return out


# --- the rule on the tree as it stands -----------------------------------------


def test_this_tree_may_be_released():
    assert problems(_json(SOURCES), _json(LAST_RUN), release_date()) == []


# --- the source list is the register in the watch's format -----------------------


def test_source_list_covers_every_register_entry():
    register = {e["id"]: e for e in _json(REGISTER)["entries"]}
    sources = _json(SOURCES)
    assert [e["id"] for e in sources] == list(register)
    for e in sources:
        r = register[e["id"]]
        assert e["used_edition"] == r["cited"], e["id"]
        assert e["newest_edition"] == r["latest"], e["id"]
        assert e["newest_source_url"] == r["source"], e["id"]
        assert e["checked"] == r["checked"], e["id"]


def test_twelve_probed_and_four_manual_with_a_reason():
    sources = _json(SOURCES)
    manual = {e["id"] for e in sources if e["probe"]["class"] in NO_REQUEST}
    assert manual == MANUAL_IDS
    assert len(sources) - len(manual) == 12
    register = {e["id"]: e for e in _json(REGISTER)["entries"]}
    for e in sources:
        p = e["probe"]
        datetime.date.fromisoformat(p["known_at"])
        if e["id"] in MANUAL_IDS:
            assert p["class"] == "S" and p["known_value"] is None
            assert p["reason"].startswith("manual: "), e["id"]
            # The manual check date is the register's check date: one truth.
            assert p["known_at"] == register[e["id"]]["checked"], e["id"]
        else:
            assert p["known_value"], e["id"]


def test_arxiv_entries_stand_one_per_paper():
    # Seven ids in one entry exceed the watch's 80-character VERSION_RE (PLAN § V.6).
    for e in _json(SOURCES):
        if e["probe"]["class"] == "O":
            assert e["probe"]["known_value"] == [e["id"].removeprefix("arxiv-") + e["used_edition"]]


# --- the positive control: the catalogs as v1.4.1 used them ---------------------


def test_v141_copy_is_frozen():
    assert hashlib.sha256(SOURCES_V141.read_bytes()).hexdigest() == SOURCES_V141_SHA256


def test_v141_copy_holds_only_what_v141_cited():
    register = {e["id"]: e for e in _json(REGISTER)["entries"]}
    copy = {e["id"]: e for e in _json(SOURCES_V141)}
    assert set(copy) == {i for i, r in register.items() if r["cited_at_v1_4_1"] != "not cited"}
    assert copy["owasp-llm-top10"]["used_edition"] == "2025"
    assert copy["unicode"]["used_edition"] == "17.0"


def test_watch_reports_v141_behind_and_the_rule_stops_it():
    sources, run = _json(SOURCES_V141), _json(RUN_V141)
    assert [r["id"] for r in run["behind"]] == ["owasp-llm-top10"]
    assert [(r["id"], r["observed"][:8]) for r in run["watch"]] == [("owasp-rag-cheat-sheet", "510ea484")]
    # The watch says "current" for unicode at 17.0 (class S makes no request);
    # only the 30-day clause catches it, together with the other three manual
    # entries whose last check was the v1.4.1 tag date.
    assert "unicode" in run["current"]
    found = problems(sources, run, datetime.date(2026, 11, 1))
    assert found == [
        f"manual check of {i} on 2026-09-23 is more than 30 days before 2026-11-01"
        for i in ("unicode", "willison-dual-llm", "rfc-3986", "llm-security-commons")
    ] + ["behind: owasp-llm-top10", "watch: owasp-rag-cheat-sheet"]


# --- each clause of the rule goes red on its own ------------------------------


def _clean() -> tuple[list[dict], dict, datetime.date]:
    return _json(SOURCES), _json(LAST_RUN), datetime.date(2026, 11, 1)


def test_manual_check_31_days_old_stops_and_30_does_not():
    sources, run, _ = _clean()
    checked = next(e for e in sources if e["id"] == "unicode")["probe"]["known_at"]
    day = datetime.date.fromisoformat(checked)
    ran = datetime.datetime.fromisoformat(run["lastRun"].replace("Z", "+00:00")).date()
    assert ran >= day  # the run is not the older of the two
    assert problems(sources, run, day + datetime.timedelta(days=30)) == []
    assert problems(sources, run, day + datetime.timedelta(days=31)) == [
        f"manual check of {e['id']} on {e['probe']['known_at']} is more than 30 days before "
        f"{day + datetime.timedelta(days=31)}"
        for e in sources
        if e["id"] in MANUAL_IDS
    ]


def test_old_watch_run_stops():
    sources, run, released = _clean()
    run["lastRun"] = (released - datetime.timedelta(days=31)).isoformat() + "T09:15:00.000Z"
    assert problems(sources, run, released) == [
        f"the watch last ran on {released - datetime.timedelta(days=31)}, more than 30 days before {released}"
    ]


def test_watch_row_stops_though_the_watch_does_not_count_it_behind():
    sources, run, released = _clean()
    run["current"].remove("cve-2025-32711")
    run["watch"].append({"id": "cve-2025-32711", "signal": "watch-only", "observed": "2026-12-01T00:00:00.000Z"})
    assert problems(sources, run, released) == ["watch: cve-2025-32711"]


def test_not_verified_stops():
    sources, run, released = _clean()
    run["current"].remove("mitre-atlas")
    run["notVerified"].append({"id": "mitre-atlas", "probe_status": "unreachable", "reason": "deadline reached"})
    assert problems(sources, run, released) == ["notVerified: mitre-atlas"]


def test_output_of_another_list_stops():
    sources, run, released = _clean()
    run["current"].remove("rfc-3986")
    assert problems(sources, run, released) == ["the watch output is not a run of this source list"]


def test_release_date_belongs_to_the_version_being_released():
    # The rule must read the heading of the version pyproject declares, or a
    # release commit that forgets its CHANGELOG heading would be judged on the
    # previous release's date.
    m = _HEADING.search(CHANGELOG.read_text(encoding="utf-8"))
    version = re.search(r'^version = "([^"]+)"', (ROOT / "pyproject.toml").read_text(encoding="utf-8"), re.M)
    assert m and version and m.group(1) == version.group(1)