Kjell Tore Guttormsen
e702505312
Fifteen catalogs and sources, each with the edition cited at v1.4.1, the newest edition read from its publisher, the check date and the link. Behind: OWASP LLM Top 10 (2025 -> 2026) and Unicode (17.0 -> 18.0). The invisible-character denominator for the next session is Unicode 18.0's own: Default_Ignorable_Code_Point, format (Cf) and private use (Co), 141 674 code points, built from sha256-pinned UCD files.
94 lines
3.6 KiB
Python
94 lines
3.6 KiB
Python
"""Build `codepoints.json` — the invisible-character denominator for session 2.
|
|
|
|
The denominator is Unicode's own, not a hand-picked list: every code point that is
|
|
Default_Ignorable_Code_Point (DerivedCoreProperties.txt), General_Category Cf
|
|
(format) or Co (private use), in the Unicode version pinned in `catalogs.json`.
|
|
Cf was added on top of the plan's DICP + private use by the operator on
|
|
2026-10-28: U+FFF9, measured red on 2026-10-28, is Cf but not DICP, as are 31
|
|
others.
|
|
|
|
The two UCD files are downloaded by hand into a directory of their own and passed
|
|
in; the script checks their sha256 against the pins before reading a byte, so a
|
|
different file can never produce the frozen output silently:
|
|
|
|
python3 -I docs/threat-2026/build-codepoints.py <dir with the two UCD files>
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import json
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
HERE = Path(__file__).resolve().parent
|
|
UNICODE = "18.0.0"
|
|
PINS = {
|
|
"DerivedCoreProperties.txt": "09c928886a178fcafd93c29e4bd59073a058e5a100b716d425cb563ab50f68c9",
|
|
"UnicodeData.txt": "0736451de439ae7baf1425136617da495e09ee5afbe6e394374db7009ea08950",
|
|
}
|
|
|
|
|
|
def _ranges(points: set[int]) -> list[list[str]]:
|
|
out, xs = [], sorted(points)
|
|
start = prev = xs[0]
|
|
for x in xs[1:]:
|
|
if x != prev + 1:
|
|
out.append([f"{start:04X}", f"{prev:04X}"])
|
|
start = x
|
|
prev = x
|
|
out.append([f"{start:04X}", f"{prev:04X}"])
|
|
return out
|
|
|
|
|
|
def main(src: Path) -> int:
|
|
for name, pin in PINS.items():
|
|
digest = hashlib.sha256((src / name).read_bytes()).hexdigest()
|
|
if digest != pin:
|
|
raise SystemExit(f"{name}: sha256 {digest} is not the pinned {pin}")
|
|
dicp: set[int] = set()
|
|
for line in (src / "DerivedCoreProperties.txt").read_text(encoding="utf-8").splitlines():
|
|
line = line.split("#")[0].strip()
|
|
if not line:
|
|
continue
|
|
rng, prop = (x.strip() for x in line.split(";")[:2])
|
|
if prop == "Default_Ignorable_Code_Point":
|
|
lo, _, hi = rng.partition("..")
|
|
dicp.update(range(int(lo, 16), int(hi or lo, 16) + 1))
|
|
cat: dict[str, set[int]] = {"Cf": set(), "Co": set()}
|
|
first = None
|
|
for line in (src / "UnicodeData.txt").read_text(encoding="utf-8").splitlines():
|
|
fields = line.split(";")
|
|
cp, name, gc = int(fields[0], 16), fields[1], fields[2]
|
|
if name.endswith(", First>"):
|
|
first = cp
|
|
continue
|
|
span = range(first, cp + 1) if name.endswith(", Last>") else (cp,)
|
|
if gc in cat:
|
|
cat[gc].update(span)
|
|
union = dicp | cat["Cf"] | cat["Co"]
|
|
doc = {
|
|
"id": "invisible-codepoint-denominator",
|
|
"unicode": UNICODE,
|
|
"sources": {f"https://www.unicode.org/Public/{UNICODE}/ucd/{n}": f"sha256:{p}"
|
|
for n, p in PINS.items()},
|
|
"rule": "Default_Ignorable_Code_Point | General_Category=Cf | General_Category=Co",
|
|
"decided": "operator 2026-10-28: Cf added to the plan's DICP + private use",
|
|
"counts": {
|
|
"default_ignorable": len(dicp),
|
|
"format_cf": len(cat["Cf"]),
|
|
"private_use_co": len(cat["Co"]),
|
|
"cf_not_default_ignorable": len(cat["Cf"] - dicp),
|
|
"total": len(union),
|
|
},
|
|
"cf_not_default_ignorable": [f"{c:04X}" for c in sorted(cat["Cf"] - dicp)],
|
|
"ranges": _ranges(union),
|
|
}
|
|
(HERE / "codepoints.json").write_text(json.dumps(doc, indent=1) + "\n", encoding="utf-8")
|
|
print(json.dumps(doc["counts"]))
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
if len(sys.argv) != 2:
|
|
raise SystemExit(__doc__)
|
|
sys.exit(main(Path(sys.argv[1])))
|