Kjell Tore Guttormsen
a640f43d73
git-subtree-dir: scanners/commons git-subtree-split: 0ffee85a4b83b3661185488c06ed9a9994c11412
506 lines
16 KiB
JSON
506 lines
16 KiB
JSON
{
|
||
"version": "0.1.0",
|
||
"id": "carriers",
|
||
"description": "Invisible and deceptive code-point carriers: characters and ranges that let text carry content a reader cannot see, or that let one script impersonate another. Six independent tables. They overlap but are NOT interchangeable, and this file deliberately does not merge them.",
|
||
"owasp": "LLM01",
|
||
"$comment": "Extracted without behaviour change from llm-security/scanners/unicode-scanner.mjs (the charset constants) and llm-security/scanners/lib/string-utils.mjs (HOMOGLYPH_MAP), delivered as operator dump 2/2 through the local coord mailbox on 2026-08-09. The fold algorithm itself (NFKC normalise, then map lookup) is ENGINE code and stays in the consumer; only the table moves here. Character names are resolved from the Unicode character database via Python's unicodedata, not written from recollection.",
|
||
"provenance": {
|
||
"source_repo": "llm-security",
|
||
"source_files": [
|
||
"scanners/unicode-scanner.mjs",
|
||
"scanners/lib/string-utils.mjs"
|
||
],
|
||
"source_exports": [
|
||
"ZERO_WIDTH_CHARS",
|
||
"UNICODE_TAG_START",
|
||
"UNICODE_TAG_END",
|
||
"BIDI_CHARS",
|
||
"CYRILLIC_CONFUSABLES",
|
||
"HOMOGLYPH_MAP"
|
||
],
|
||
"source_delivery": "operator dump 2/2, coord message from llm-security, 2026-08-09",
|
||
"source_commit": "unknown - not supplied with the dump",
|
||
"verified": "differentially, against the dump - except private_use, see that table's own verified field",
|
||
"evidence_limits": [
|
||
"The dump is a transcription of the source modules, not the module files themselves. The checks recorded for this file prove that this JSON agrees with the DUMP. That the dump agrees with the modules is llm-security's assertion, not a result reproduced here.",
|
||
"Five of the six tables were delivered as executable constants and were imported and compared value by value. The private-use ranges were delivered as a source COMMENT only, with no constant behind them; they are marked verified: false and must not be treated as equal evidence.",
|
||
"The dump's own comment describes HOMOGLYPH_MAP as having '~25 entries'. Counted mechanically it holds 28. The count here is the counted one."
|
||
]
|
||
},
|
||
"tables": {
|
||
"zero_width": {
|
||
"$comment": "Characters that occupy no visual width, so text containing them reads identically to text without them. Note that this set includes U+00AD SOFT HYPHEN, which is conditionally visible rather than strictly zero-width, and that it is a DIFFERENT set from the four-member class inside the injection lexicon's zero-width pattern. Neither is a superset of the other by accident: see cross_table_notes.",
|
||
"verified": true,
|
||
"codepoints": [
|
||
{
|
||
"codepoint": "U+200B",
|
||
"name": "ZERO WIDTH SPACE"
|
||
},
|
||
{
|
||
"codepoint": "U+200C",
|
||
"name": "ZERO WIDTH NON-JOINER"
|
||
},
|
||
{
|
||
"codepoint": "U+200D",
|
||
"name": "ZERO WIDTH JOINER"
|
||
},
|
||
{
|
||
"codepoint": "U+FEFF",
|
||
"name": "ZERO WIDTH NO-BREAK SPACE"
|
||
},
|
||
{
|
||
"codepoint": "U+00AD",
|
||
"name": "SOFT HYPHEN"
|
||
}
|
||
],
|
||
"count": 5
|
||
},
|
||
"unicode_tags": {
|
||
"$comment": "The Unicode Tags block, used for steganography: each tag character mirrors an ASCII character and is invisible when rendered, so a whole instruction can be smuggled inside otherwise innocent text.",
|
||
"verified": true,
|
||
"range": {
|
||
"start": "U+E0001",
|
||
"end": "U+E007F",
|
||
"count": 127
|
||
},
|
||
"decode": {
|
||
"rule": "ascii_codepoint = tag_codepoint - 0xE0000",
|
||
"offset": "U+E0000",
|
||
"$comment": "Stated as a rule rather than a table because it is a subtraction, not a mapping. A consumer that decodes differently will disagree with the seed runtime on identical input."
|
||
}
|
||
},
|
||
"private_use": {
|
||
"$comment": "The two Supplementary Private Use Areas. They carry no assigned meaning and no ASCII mapping, so their presence in text is itself the signal - there is nothing to decode. The Basic Multilingual Plane's private use area (U+E000-U+F8FF) is NOT part of this table; the seed implementation does not include it, and adding it here would change behaviour.",
|
||
"verified": false,
|
||
"verification_note": "Delivered as a source comment, with no constant behind it in the dump. Unlike the other five tables there was nothing to import and diff, so this table is transcription only. Treat it as the weakest evidence in this file until the constant is supplied.",
|
||
"ranges": [
|
||
{
|
||
"id": "PUA-A",
|
||
"start": "U+F0000",
|
||
"end": "U+FFFFD"
|
||
},
|
||
{
|
||
"id": "PUA-B",
|
||
"start": "U+100000",
|
||
"end": "U+10FFFD"
|
||
}
|
||
]
|
||
},
|
||
"bidi": {
|
||
"$comment": "Bidirectional formatting controls. Reordering rendered text away from its logical byte order is the Trojan Source class (CVE-2021-42574): source code or a prompt that reads one way to a human and another to a parser.",
|
||
"verified": true,
|
||
"cve": "CVE-2021-42574",
|
||
"codepoints": [
|
||
{
|
||
"codepoint": "U+202A",
|
||
"name": "LEFT-TO-RIGHT EMBEDDING"
|
||
},
|
||
{
|
||
"codepoint": "U+202B",
|
||
"name": "RIGHT-TO-LEFT EMBEDDING"
|
||
},
|
||
{
|
||
"codepoint": "U+202C",
|
||
"name": "POP DIRECTIONAL FORMATTING"
|
||
},
|
||
{
|
||
"codepoint": "U+202D",
|
||
"name": "LEFT-TO-RIGHT OVERRIDE"
|
||
},
|
||
{
|
||
"codepoint": "U+202E",
|
||
"name": "RIGHT-TO-LEFT OVERRIDE"
|
||
},
|
||
{
|
||
"codepoint": "U+2066",
|
||
"name": "LEFT-TO-RIGHT ISOLATE"
|
||
},
|
||
{
|
||
"codepoint": "U+2067",
|
||
"name": "RIGHT-TO-LEFT ISOLATE"
|
||
},
|
||
{
|
||
"codepoint": "U+2068",
|
||
"name": "FIRST STRONG ISOLATE"
|
||
},
|
||
{
|
||
"codepoint": "U+2069",
|
||
"name": "POP DIRECTIONAL ISOLATE"
|
||
}
|
||
],
|
||
"count": 9
|
||
},
|
||
"cyrillic_confusables": {
|
||
"$comment": "Cyrillic characters treated as a signal by BARE PRESENCE when adjacent to Latin, rather than by folding. This is a detection set, not a translation set: it answers 'is a script being mixed here', and it is DISTINCT from homoglyph_map below. The dump states the distinction is deliberate.",
|
||
"verified": true,
|
||
"usage": "adjacency to Latin characters; consumed by the injection lexicon's homoglyph pattern",
|
||
"codepoints": [
|
||
{
|
||
"codepoint": "U+0430",
|
||
"char": "а",
|
||
"name": "CYRILLIC SMALL LETTER A"
|
||
},
|
||
{
|
||
"codepoint": "U+0435",
|
||
"char": "е",
|
||
"name": "CYRILLIC SMALL LETTER IE"
|
||
},
|
||
{
|
||
"codepoint": "U+043E",
|
||
"char": "о",
|
||
"name": "CYRILLIC SMALL LETTER O"
|
||
},
|
||
{
|
||
"codepoint": "U+0441",
|
||
"char": "с",
|
||
"name": "CYRILLIC SMALL LETTER ES"
|
||
},
|
||
{
|
||
"codepoint": "U+0440",
|
||
"char": "р",
|
||
"name": "CYRILLIC SMALL LETTER ER"
|
||
},
|
||
{
|
||
"codepoint": "U+0443",
|
||
"char": "у",
|
||
"name": "CYRILLIC SMALL LETTER U"
|
||
},
|
||
{
|
||
"codepoint": "U+0445",
|
||
"char": "х",
|
||
"name": "CYRILLIC SMALL LETTER HA"
|
||
},
|
||
{
|
||
"codepoint": "U+0410",
|
||
"char": "А",
|
||
"name": "CYRILLIC CAPITAL LETTER A"
|
||
},
|
||
{
|
||
"codepoint": "U+0415",
|
||
"char": "Е",
|
||
"name": "CYRILLIC CAPITAL LETTER IE"
|
||
},
|
||
{
|
||
"codepoint": "U+041E",
|
||
"char": "О",
|
||
"name": "CYRILLIC CAPITAL LETTER O"
|
||
},
|
||
{
|
||
"codepoint": "U+0421",
|
||
"char": "С",
|
||
"name": "CYRILLIC CAPITAL LETTER ES"
|
||
},
|
||
{
|
||
"codepoint": "U+0420",
|
||
"char": "Р",
|
||
"name": "CYRILLIC CAPITAL LETTER ER"
|
||
},
|
||
{
|
||
"codepoint": "U+0425",
|
||
"char": "Х",
|
||
"name": "CYRILLIC CAPITAL LETTER HA"
|
||
}
|
||
],
|
||
"count": 13
|
||
},
|
||
"homoglyph_map": {
|
||
"$comment": "The fold-to-Latin table: what a confusable character becomes before a pattern is matched against the folded text. Deliberately small. The source comment records the exclusion rationale: Latin Extended characters used by ordinary Norwegian, German and similar orthography are NOT included, because folding them would corrupt legitimate text; only letters that appear in injection vocabulary are mapped. Absence from this table is therefore not evidence that a character is safe.",
|
||
"verified": true,
|
||
"algorithm_note": "The fold algorithm (NFKC normalise, then look up each character in this map) is ENGINE code and stays in the consumer. This file publishes only the table. Two runtimes that normalise differently before the lookup will disagree on identical input even with an identical table.",
|
||
"entries": [
|
||
{
|
||
"from": "U+0430",
|
||
"from_char": "а",
|
||
"to": "a",
|
||
"script": "Cyrillic",
|
||
"name": "CYRILLIC SMALL LETTER A"
|
||
},
|
||
{
|
||
"from": "U+0435",
|
||
"from_char": "е",
|
||
"to": "e",
|
||
"script": "Cyrillic",
|
||
"name": "CYRILLIC SMALL LETTER IE"
|
||
},
|
||
{
|
||
"from": "U+043E",
|
||
"from_char": "о",
|
||
"to": "o",
|
||
"script": "Cyrillic",
|
||
"name": "CYRILLIC SMALL LETTER O"
|
||
},
|
||
{
|
||
"from": "U+0441",
|
||
"from_char": "с",
|
||
"to": "c",
|
||
"script": "Cyrillic",
|
||
"name": "CYRILLIC SMALL LETTER ES"
|
||
},
|
||
{
|
||
"from": "U+0440",
|
||
"from_char": "р",
|
||
"to": "p",
|
||
"script": "Cyrillic",
|
||
"name": "CYRILLIC SMALL LETTER ER"
|
||
},
|
||
{
|
||
"from": "U+0445",
|
||
"from_char": "х",
|
||
"to": "x",
|
||
"script": "Cyrillic",
|
||
"name": "CYRILLIC SMALL LETTER HA"
|
||
},
|
||
{
|
||
"from": "U+0443",
|
||
"from_char": "у",
|
||
"to": "y",
|
||
"script": "Cyrillic",
|
||
"name": "CYRILLIC SMALL LETTER U"
|
||
},
|
||
{
|
||
"from": "U+0456",
|
||
"from_char": "і",
|
||
"to": "i",
|
||
"script": "Cyrillic",
|
||
"name": "CYRILLIC SMALL LETTER BYELORUSSIAN-UKRAINIAN I"
|
||
},
|
||
{
|
||
"from": "U+0458",
|
||
"from_char": "ј",
|
||
"to": "j",
|
||
"script": "Cyrillic",
|
||
"name": "CYRILLIC SMALL LETTER JE"
|
||
},
|
||
{
|
||
"from": "U+0455",
|
||
"from_char": "ѕ",
|
||
"to": "s",
|
||
"script": "Cyrillic",
|
||
"name": "CYRILLIC SMALL LETTER DZE"
|
||
},
|
||
{
|
||
"from": "U+04CF",
|
||
"from_char": "ӏ",
|
||
"to": "l",
|
||
"script": "Cyrillic",
|
||
"name": "CYRILLIC SMALL LETTER PALOCHKA"
|
||
},
|
||
{
|
||
"from": "U+0410",
|
||
"from_char": "А",
|
||
"to": "A",
|
||
"script": "Cyrillic",
|
||
"name": "CYRILLIC CAPITAL LETTER A"
|
||
},
|
||
{
|
||
"from": "U+0415",
|
||
"from_char": "Е",
|
||
"to": "E",
|
||
"script": "Cyrillic",
|
||
"name": "CYRILLIC CAPITAL LETTER IE"
|
||
},
|
||
{
|
||
"from": "U+041E",
|
||
"from_char": "О",
|
||
"to": "O",
|
||
"script": "Cyrillic",
|
||
"name": "CYRILLIC CAPITAL LETTER O"
|
||
},
|
||
{
|
||
"from": "U+0421",
|
||
"from_char": "С",
|
||
"to": "C",
|
||
"script": "Cyrillic",
|
||
"name": "CYRILLIC CAPITAL LETTER ES"
|
||
},
|
||
{
|
||
"from": "U+0420",
|
||
"from_char": "Р",
|
||
"to": "P",
|
||
"script": "Cyrillic",
|
||
"name": "CYRILLIC CAPITAL LETTER ER"
|
||
},
|
||
{
|
||
"from": "U+0425",
|
||
"from_char": "Х",
|
||
"to": "X",
|
||
"script": "Cyrillic",
|
||
"name": "CYRILLIC CAPITAL LETTER HA"
|
||
},
|
||
{
|
||
"from": "U+0423",
|
||
"from_char": "У",
|
||
"to": "Y",
|
||
"script": "Cyrillic",
|
||
"name": "CYRILLIC CAPITAL LETTER U"
|
||
},
|
||
{
|
||
"from": "U+03B1",
|
||
"from_char": "α",
|
||
"to": "a",
|
||
"script": "Greek",
|
||
"name": "GREEK SMALL LETTER ALPHA"
|
||
},
|
||
{
|
||
"from": "U+03BF",
|
||
"from_char": "ο",
|
||
"to": "o",
|
||
"script": "Greek",
|
||
"name": "GREEK SMALL LETTER OMICRON"
|
||
},
|
||
{
|
||
"from": "U+03C1",
|
||
"from_char": "ρ",
|
||
"to": "p",
|
||
"script": "Greek",
|
||
"name": "GREEK SMALL LETTER RHO"
|
||
},
|
||
{
|
||
"from": "U+03B9",
|
||
"from_char": "ι",
|
||
"to": "i",
|
||
"script": "Greek",
|
||
"name": "GREEK SMALL LETTER IOTA"
|
||
},
|
||
{
|
||
"from": "U+03BD",
|
||
"from_char": "ν",
|
||
"to": "v",
|
||
"script": "Greek",
|
||
"name": "GREEK SMALL LETTER NU"
|
||
},
|
||
{
|
||
"from": "U+03C4",
|
||
"from_char": "τ",
|
||
"to": "t",
|
||
"script": "Greek",
|
||
"name": "GREEK SMALL LETTER TAU"
|
||
},
|
||
{
|
||
"from": "U+0391",
|
||
"from_char": "Α",
|
||
"to": "A",
|
||
"script": "Greek",
|
||
"name": "GREEK CAPITAL LETTER ALPHA"
|
||
},
|
||
{
|
||
"from": "U+039F",
|
||
"from_char": "Ο",
|
||
"to": "O",
|
||
"script": "Greek",
|
||
"name": "GREEK CAPITAL LETTER OMICRON"
|
||
},
|
||
{
|
||
"from": "U+03A1",
|
||
"from_char": "Ρ",
|
||
"to": "P",
|
||
"script": "Greek",
|
||
"name": "GREEK CAPITAL LETTER RHO"
|
||
},
|
||
{
|
||
"from": "U+03A4",
|
||
"from_char": "Τ",
|
||
"to": "T",
|
||
"script": "Greek",
|
||
"name": "GREEK CAPITAL LETTER TAU"
|
||
}
|
||
],
|
||
"count": 28,
|
||
"counts_by_script": {
|
||
"Cyrillic": 18,
|
||
"Greek": 10
|
||
}
|
||
}
|
||
},
|
||
"cross_table_notes": {
|
||
"$comment": "Three tables in this repository describe overlapping Cyrillic sets and two describe overlapping zero-width sets, and none of them agree exactly. The divergences are recorded here as fact. They are NOT reconciled: this repository preserves the seed implementation's behaviour, and a disagreement is reported to the implementation and decided there, where it is tested.",
|
||
"zero_width": {
|
||
"this_file": [
|
||
"U+00AD",
|
||
"U+200B",
|
||
"U+200C",
|
||
"U+200D",
|
||
"U+FEFF"
|
||
],
|
||
"injection_lexicon_pattern_class": [
|
||
"U+200B",
|
||
"U+200C",
|
||
"U+200D",
|
||
"U+FEFF"
|
||
],
|
||
"only_here": [
|
||
"U+00AD"
|
||
],
|
||
"only_in_lexicon": []
|
||
},
|
||
"cyrillic": {
|
||
"confusables_presence_set": [
|
||
"U+0410",
|
||
"U+0415",
|
||
"U+041E",
|
||
"U+0420",
|
||
"U+0421",
|
||
"U+0425",
|
||
"U+0430",
|
||
"U+0435",
|
||
"U+043E",
|
||
"U+0440",
|
||
"U+0441",
|
||
"U+0443",
|
||
"U+0445"
|
||
],
|
||
"injection_lexicon_pattern_class": [
|
||
"U+0430",
|
||
"U+0435",
|
||
"U+043E",
|
||
"U+0440",
|
||
"U+0441",
|
||
"U+0443",
|
||
"U+0456"
|
||
],
|
||
"homoglyph_fold_map": [
|
||
"U+0391",
|
||
"U+039F",
|
||
"U+03A1",
|
||
"U+03A4",
|
||
"U+03B1",
|
||
"U+03B9",
|
||
"U+03BD",
|
||
"U+03BF",
|
||
"U+03C1",
|
||
"U+03C4",
|
||
"U+0410",
|
||
"U+0415",
|
||
"U+041E",
|
||
"U+0420",
|
||
"U+0421",
|
||
"U+0423",
|
||
"U+0425",
|
||
"U+0430",
|
||
"U+0435",
|
||
"U+043E",
|
||
"U+0440",
|
||
"U+0441",
|
||
"U+0443",
|
||
"U+0445",
|
||
"U+0455",
|
||
"U+0456",
|
||
"U+0458",
|
||
"U+04CF"
|
||
],
|
||
"in_lexicon_class_but_not_in_confusables": [
|
||
"U+0456"
|
||
],
|
||
"in_confusables_but_not_in_lexicon_class": [
|
||
"U+0410",
|
||
"U+0415",
|
||
"U+041E",
|
||
"U+0420",
|
||
"U+0421",
|
||
"U+0425",
|
||
"U+0445"
|
||
],
|
||
"in_confusables_but_not_foldable": []
|
||
}
|
||
}
|
||
}
|