open/llm-ingestion-pipeline-security
1
0
Fork 0
Code Issues Releases 6 Activity

docs(threat-2026): register the threat catalogs at their newest editions

Browse source 
Fifteen catalogs and sources, each with the edition cited at v1.4.1, the
newest edition read from its publisher, the check date and the link.
Behind: OWASP LLM Top 10 (2025 -> 2026) and Unicode (17.0 -> 18.0).

The invisible-character denominator for the next session is Unicode
18.0's own: Default_Ignorable_Code_Point, format (Cf) and private use
(Co), 141 674 code points, built from sha256-pinned UCD files.
This commit is contained in:
Kjell Tore Guttormsen 2026-10-28 21:27:32 +01:00
commit e702505312
Signed by: ktg
SSH key fingerprint: SHA256:JakMjO6FTBBzN0Bhfj9saOoEjaFxlSdYuZQQpM/lF9Q
3 changed files with 491 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

94
docs/threat-2026/build-codepoints.py Normal file
View file

@ -0,0 +1,94 @@
"""Build `codepoints.json` — the invisible-character denominator for session 2.
The denominator is Unicode's own, not a hand-picked list: every code point that is
Default_Ignorable_Code_Point (DerivedCoreProperties.txt), General_Category Cf
(format) or Co (private use), in the Unicode version pinned in `catalogs.json`.
Cf was added on top of the plan's DICP + private use by the operator on
2026-10-28: U+FFF9, measured red on 2026-10-28, is Cf but not DICP, as are 31
others.
The two UCD files are downloaded by hand into a directory of their own and passed
in; the script checks their sha256 against the pins before reading a byte, so a
different file can never produce the frozen output silently:
python3 -I docs/threat-2026/build-codepoints.py <dir with the two UCD files>
"""
from __future__ import annotations
import hashlib
import json
import sys
from pathlib import Path
HERE = Path(__file__).resolve().parent
UNICODE = "18.0.0"
PINS = {
"DerivedCoreProperties.txt": "09c928886a178fcafd93c29e4bd59073a058e5a100b716d425cb563ab50f68c9",
"UnicodeData.txt": "0736451de439ae7baf1425136617da495e09ee5afbe6e394374db7009ea08950",
}
def _ranges(points: set[int]) -> list[list[str]]:
out, xs = [], sorted(points)
start = prev = xs[0]
for x in xs[1:]:
if x != prev + 1:
out.append([f"{start:04X}", f"{prev:04X}"])
start = x
prev = x
out.append([f"{start:04X}", f"{prev:04X}"])
return out
def main(src: Path) -> int:
for name, pin in PINS.items():
digest = hashlib.sha256((src / name).read_bytes()).hexdigest()
if digest != pin:
raise SystemExit(f"{name}: sha256 {digest} is not the pinned {pin}")
dicp: set[int] = set()
for line in (src / "DerivedCoreProperties.txt").read_text(encoding="utf-8").splitlines():
line = line.split("#")[0].strip()
if not line:
continue
rng, prop = (x.strip() for x in line.split(";")[:2])
if prop == "Default_Ignorable_Code_Point":
lo, _, hi = rng.partition("..")
dicp.update(range(int(lo, 16), int(hi or lo, 16) + 1))
cat: dict[str, set[int]] = {"Cf": set(), "Co": set()}
first = None
for line in (src / "UnicodeData.txt").read_text(encoding="utf-8").splitlines():
fields = line.split(";")
cp, name, gc = int(fields[0], 16), fields[1], fields[2]
if name.endswith(", First>"):
first = cp
continue
span = range(first, cp + 1) if name.endswith(", Last>") else (cp,)
if gc in cat:
cat[gc].update(span)
union = dicp | cat["Cf"] | cat["Co"]
doc = {
"id": "invisible-codepoint-denominator",
"unicode": UNICODE,
"sources": {f"https://www.unicode.org/Public/{UNICODE}/ucd/{n}": f"sha256:{p}"
for n, p in PINS.items()},
"rule": "Default_Ignorable_Code_Point | General_Category=Cf | General_Category=Co",
"decided": "operator 2026-10-28: Cf added to the plan's DICP + private use",
"counts": {
"default_ignorable": len(dicp),
"format_cf": len(cat["Cf"]),
"private_use_co": len(cat["Co"]),
"cf_not_default_ignorable": len(cat["Cf"] - dicp),
"total": len(union),
},
"cf_not_default_ignorable": [f"{c:04X}" for c in sorted(cat["Cf"] - dicp)],
"ranges": _ranges(union),
}
(HERE / "codepoints.json").write_text(json.dumps(doc, indent=1) + "\n", encoding="utf-8")
print(json.dumps(doc["counts"]))
return 0
if __name__ == "__main__":
if len(sys.argv) != 2:
raise SystemExit(__doc__)
sys.exit(main(Path(sys.argv[1])))

232
docs/threat-2026/catalogs.json Normal file
View file

@ -0,0 +1,232 @@
{
"id": "threat-catalog-register",
"version": "1.0.0",
"description": "Every threat catalog, standard and source the guard cites: the edition cited at v1.4.1, the newest edition, when it was checked and where. Row 1 of the 2026 done-criterion (docs/PLAN.md § T.2) is N of N entries with cited == latest, a check date and a source link.",
"denominator_rule": "Frozen 2026-10-28 in session 1. Entries may be added; none is removed without the operator's yes.",
"verification_rule": "Each latest edition was read from the PUBLISHER (its own site, its official repository or API). A copy verified elsewhere is a premise, not a measurement.",
"checked": "2026-10-28",
"entries": [
{
"id": "owasp-llm-top10",
"kind": "catalog",
"name": "OWASP Top 10 for LLM Applications",
"publisher": "OWASP GenAI Security Project",
"cited_at_v1_4_1": "2025 (numbers written without a year)",
"latest": "2026",
"latest_published": "2026-08-04",
"checked": "2026-10-28",
"source": "https://genai.owasp.org/resource/owasp-genai-llm-top-10-2026/",
"source_data": "https://github.com/GenAI-Security-Project/GenAI-LLM-Top10/tree/main/2026/final",
"evidence": "Project README: 'Current release: OWASP GenAI LLM Top 10 2026 — published August 4, 2026', 2025 marked 'archived release'. The page's datePublished is 2026-08-04T05:50:36Z; its visible date reads August 3, 2026. Headings in 2026/final/*.md: '## LLM01:2026 Prompt Injection' … '## LLM10:2026 Improper Output Handling'. The landing page genai.owasp.org/llm-top-10/ still shows the 2025 list.",
"citation_style": "LLMnn:2026",
"map_2025_to_2026": {
"LLM01": "LLM01", "LLM02": "LLM02", "LLM03": "LLM04", "LLM04": "LLM05",
"LLM05": "LLM10", "LLM06": "LLM03", "LLM07": "LLM08", "LLM08": "LLM09",
"LLM09": "LLM07", "LLM10": "LLM06"
},
"map_evidence": "Each 2026 ID and title read against the 2025 ID and title; LLM00_Preface.md confirms the moves ('Excessive Agency climbed to third', 'Improper Output Handling fell the furthest, from fifth to tenth', 'What used to be System Prompt Leakage is now Hidden Context Exposure'). No publisher-side 2025-to-2026 table was found.",
"titles_2026": {
"LLM01": "Prompt Injection", "LLM02": "Sensitive Information Disclosure",
"LLM03": "Excessive Agency", "LLM04": "Supply Chain",
"LLM05": "Data and Model Poisoning", "LLM06": "Unbounded Consumption",
"LLM07": "Misinformation", "LLM08": "Hidden Context Exposure",
"LLM09": "Vector and Embedding Weaknesses", "LLM10": "Improper Output Handling"
},
"used_in": "Finding.owasp on every finding; README, LIMITATIONS, coverage matrix"
},
{
"id": "owasp-agentic-top10",
"kind": "catalog",
"name": "OWASP Top 10 for Agentic Applications",
"publisher": "OWASP GenAI Security Project",
"cited_at_v1_4_1": "not cited",
"latest": "for 2026",
"latest_published": "2025-12-09",
"checked": "2026-10-28",
"source": "https://genai.owasp.org/resource/owasp-top-10-for-agentic-applications-for-2026/",
"evidence": "Visible date 'December 9, 2025' (datePublished 2025-12-10T07:55:54Z). IDs and titles from the publisher's 2026/final/mappings/asi-2026.json; no year-suffixed form such as 'ASI06:2026' was seen.",
"relevant_entries": {
"ASI06": "Memory & Context Poisoning",
"ASI01": "Agent Goal Hijack"
},
"used_in": "not yet (session 4 links ASI06)"
},
{
"id": "mitre-atlas",
"kind": "catalog",
"name": "MITRE ATLAS",
"publisher": "MITRE",
"cited_at_v1_4_1": "not cited",
"latest": "v2026.09",
"latest_published": "2026-09-15",
"checked": "2026-10-28",
"source": "https://github.com/mitre-atlas/atlas-data/releases/tag/v2026.09",
"evidence": "GitHub releases/latest: v2026.09, 2026-09-15T15:03:27Z; asset ATLAS-2026.09.yaml with version '2026.09', format-version 6.0.0; '1 matrix, 16 tactics, 120 techniques, 88 sub-techniques'. Versioning moved from 5.x (last v5.6.0) to calendar versions at v2026.05; dist/ATLAS.yaml is marked deprecated.",
"relevant_entries": {
"AML.T0051.001": "LLM Prompt Injection: Indirect",
"AML.T0068": "LLM Prompt Obfuscation",
"AML.T0070": "RAG Poisoning",
"AML.T0071": "False RAG Entry Injection",
"AML.T0066": "Retrieval Content Crafting",
"AML.T0057": "LLM Data Leakage",
"AML.T0077": "LLM Response Rendering",
"AML.T0080.000": "AI Agent Context Poisoning: Memory"
},
"relevant_entries_note": "Read from the v2026.09 data file only; not cross-checked on atlas.mitre.org. Session 4 re-reads each before linking it.",
"used_in": "not yet (session 4)"
},
{
"id": "owasp-rag-cheat-sheet",
"kind": "guidance",
"name": "Retrieval-Augmented Generation (RAG) Security Cheat Sheet",
"publisher": "OWASP Cheat Sheet Series",
"cited_at_v1_4_1": "by name only, no date or link (docs/PLAN.md v1 positioning)",
"latest": "living document, last modified 2026-10-04 (commit 510ea484)",
"latest_published": "2026-10-04",
"checked": "2026-10-28",
"source": "https://cheatsheetseries.owasp.org/cheatsheets/RAG_Security_Cheat_Sheet.html",
"evidence": "OWASP/CheatSheetSeries history: created 2026-05-06 (41f67299, 'New Cheat Sheet: RAG Security (#2131)'), last modified 2026-10-04T18:12:29Z (510ea484). The repo had cited it under a shorter name.",
"used_in": "positioning text"
},
{
"id": "unicode",
"kind": "standard",
"name": "The Unicode Standard (UCD)",
"publisher": "Unicode Consortium",
"cited_at_v1_4_1": "17.0 (sanitize.py block comment, LIMITATIONS ZWJ measurement)",
"latest": "18.0.0",
"latest_published": "2026-09-16",
"checked": "2026-10-28",
"source": "https://www.unicode.org/versions/Unicode18.0.0/",
"evidence": "unicode.org/history/publicationdates.html: 'Version 18.0 | 2026 | September 16'. /Public/UCD/latest/ReadMe.txt: 'final data files and charts for version 18.0.0'.",
"pinned_files": {
"https://www.unicode.org/Public/18.0.0/ucd/DerivedCoreProperties.txt": "sha256:09c928886a178fcafd93c29e4bd59073a058e5a100b716d425cb563ab50f68c9",
"https://www.unicode.org/Public/18.0.0/ucd/UnicodeData.txt": "sha256:0736451de439ae7baf1425136617da495e09ee5afbe6e394374db7009ea08950"
},
"used_in": "invisible-character tables (sanitize, output); the character denominator in docs/threat-2026/codepoints.json"
},
{
"id": "cve-2025-32711",
"kind": "advisory",
"name": "CVE-2025-32711 (EchoLeak), M365 Copilot Information Disclosure",
"publisher": "MITRE CVE / NVD; Microsoft MSRC",
"cited_at_v1_4_1": "CVE-2025-32711",
"latest": "MSRC revision 2026-02-20; NVD last modified 2026-06-17",
"latest_published": "2025-06-11",
"checked": "2026-10-28",
"source": "https://nvd.nist.gov/vuln/detail/CVE-2025-32711",
"evidence": "NVD: 'Ai command injection in M365 Copilot allows an unauthorized attacker to disclose information over a network.' CWE-74. MSRC title 'M365 Copilot Information Disclosure Vulnerability', Critical. NVD's references include the researcher's EchoLeak page; neither record says 'EchoLeak' in prose.",
"used_in": "active-content detector docs, coverage matrix, README"
},
{
"id": "arxiv-2402.07867",
"kind": "paper",
"name": "PoisonedRAG: Knowledge Corruption Attacks to Retrieval-Augmented Generation of Large Language Models",
"publisher": "arXiv",
"cited_at_v1_4_1": "2402.07867 (no version)",
"latest": "v3",
"latest_published": "2024-08-13",
"checked": "2026-10-28",
"source": "https://arxiv.org/abs/2402.07867",
"used_in": "claims we will not make; semantic-poisoning gap"
},
{
"id": "arxiv-2504.11168",
"kind": "paper",
"name": "Bypassing LLM Guardrails: An Empirical Analysis of Evasion Attacks against Prompt Injection and Jailbreak Detection Systems",
"publisher": "arXiv",
"cited_at_v1_4_1": "2504.11168 (no version)",
"latest": "v3",
"latest_published": "2025-07-14",
"checked": "2026-10-28",
"source": "https://arxiv.org/abs/2504.11168",
"used_in": "README and BRIEF, detection-is-evadable claim"
},
{
"id": "arxiv-2509.14285",
"kind": "paper",
"name": "A Multi-Agent LLM Defense Pipeline Against Prompt Injection Attacks",
"publisher": "arXiv",
"cited_at_v1_4_1": "2509.14285 (no version)",
"latest": "v4",
"latest_published": "2025-12-17",
"checked": "2026-10-28",
"source": "https://arxiv.org/abs/2509.14285",
"used_in": "docs/BRIEF.md"
},
{
"id": "arxiv-2505.03574",
"kind": "paper",
"name": "LlamaFirewall: An open source guardrail system for building secure AI agents",
"publisher": "arXiv",
"cited_at_v1_4_1": "2505.03574 (no version)",
"latest": "v1",
"latest_published": "2025-05-06",
"checked": "2026-10-28",
"source": "https://arxiv.org/abs/2505.03574",
"used_in": "docs/BRIEF.md"
},
{
"id": "arxiv-2503.18813",
"kind": "paper",
"name": "Defeating Prompt Injections by Design (CaMeL)",
"publisher": "arXiv",
"cited_at_v1_4_1": "2503.18813 (no version)",
"latest": "v2",
"latest_published": "2025-06-24",
"checked": "2026-10-28",
"source": "https://arxiv.org/abs/2503.18813",
"used_in": "architecture lineage"
},
{
"id": "arxiv-2604.00387",
"kind": "paper",
"name": "RAGShield: Detecting Numerical Claim Manipulation in Government RAG Systems",
"publisher": "arXiv",
"cited_at_v1_4_1": "2604.00387 (no version), labelled 'RAGShield'",
"latest": "v2",
"latest_published": "2026-04-04",
"checked": "2026-10-28",
"source": "https://arxiv.org/abs/2604.00387",
"note": "Narrower than the label suggests: numerical claim manipulation in government RAG systems.",
"used_in": "docs/PLAN.md v1 anchors (history)"
},
{
"id": "willison-dual-llm",
"kind": "article",
"name": "The Dual LLM pattern for building AI assistants that can resist prompt injection",
"publisher": "Simon Willison (author's own blog)",
"cited_at_v1_4_1": "by name, 2023",
"latest": "2023-04-25 (single post)",
"latest_published": "2023-04-25",
"checked": "2026-10-28",
"source": "https://simonwillison.net/2023/Apr/25/dual-llm-pattern/",
"used_in": "architecture lineage"
},
{
"id": "rfc-3986",
"kind": "standard",
"name": "RFC 3986, Uniform Resource Identifier (URI): Generic Syntax (STD 66)",
"publisher": "RFC Editor",
"cited_at_v1_4_1": "RFC 3986",
"latest": "RFC 3986, INTERNET STANDARD, not obsoleted (updated by RFC 7320, RFC 8820)",
"latest_published": "2005-01",
"checked": "2026-10-28",
"source": "https://www.rfc-editor.org/rfc/rfc3986.json",
"used_in": "active_content.py scheme grammar"
},
{
"id": "llm-security-commons",
"kind": "data-source",
"name": "llm-security-commons (injection lexicon, carrier code points)",
"publisher": "the commons repository on the same Forgejo instance",
"cited_at_v1_4_1": "lexicon 0.8.0 (83 patterns, byte-equal copy)",
"latest": "v0.9.0 (lexicon/injection-lexicon.json 0.8.0, codepoints/carriers.json 0.2.0)",
"latest_published": "2026-08-21",
"checked": "2026-10-28",
"source": "ssh://git@git.fromaitochitta.com/ktg/llm-security-commons.git",
"evidence": "Fetched origin 2026-10-28: newest tag v0.9.0, origin/main c362717 (2026-08-21). Its mapping/owasp-map.json carries 2025-edition numbers; that is input for commons, not a change here.",
"used_in": "src/llm_ingestion_guard/injection_lexicon.json"
}
]
}

165
docs/threat-2026/codepoints.json Normal file
View file

@ -0,0 +1,165 @@
{
"id": "invisible-codepoint-denominator",
"unicode": "18.0.0",
"sources": {
"https://www.unicode.org/Public/18.0.0/ucd/DerivedCoreProperties.txt": "sha256:09c928886a178fcafd93c29e4bd59073a058e5a100b716d425cb563ab50f68c9",
"https://www.unicode.org/Public/18.0.0/ucd/UnicodeData.txt": "sha256:0736451de439ae7baf1425136617da495e09ee5afbe6e394374db7009ea08950"
},
"rule": "Default_Ignorable_Code_Point | General_Category=Cf | General_Category=Co",
"decided": "operator 2026-10-28: Cf added to the plan's DICP + private use",
"counts": {
"default_ignorable": 4174,
"format_cf": 170,
"private_use_co": 137468,
"cf_not_default_ignorable": 32,
"total": 141674
},
"cf_not_default_ignorable": [
"0600",
"0601",
"0602",
"0603",
"0604",
"0605",
"06DD",
"070F",
"0890",
"0891",
"08E2",
"FFF9",
"FFFA",
"FFFB",
"110BD",
"110CD",
"13430",
"13431",
"13432",
"13433",
"13434",
"13435",
"13436",
"13437",
"13438",
"13439",
"1343A",
"1343B",
"1343C",
"1343D",
"1343E",
"1343F"
],
"ranges": [
[
"00AD",
"00AD"
],
[
"034F",
"034F"
],
[
"0600",
"0605"
],
[
"061C",
"061C"
],
[
"06DD",
"06DD"
],
[
"070F",
"070F"
],
[
"0890",
"0891"
],
[
"08E2",
"08E2"
],
[
"115F",
"1160"
],
[
"17B4",
"17B5"
],
[
"180B",
"180F"
],
[
"200B",
"200F"
],
[
"202A",
"202E"
],
[
"2060",
"206F"
],
[
"3164",
"3164"
],
[
"E000",
"F8FF"
],
[
"FE00",
"FE0F"
],
[
"FEFF",
"FEFF"
],
[
"FFA0",
"FFA0"
],
[
"FFF0",
"FFFB"
],
[
"110BD",
"110BD"
],
[
"110CD",
"110CD"
],
[
"13430",
"1343F"
],
[
"1BCA0",
"1BCA3"
],
[
"1D173",
"1D17A"
],
[
"E0000",
"E0FFF"
],
[
"F0000",
"FFFFD"
],
[
"100000",
"10FFFD"
]
]
}