"""Build `codepoints.json` — the invisible-character denominator for session 2.

The denominator is Unicode's own, not a hand-picked list: every code point that is
Default_Ignorable_Code_Point (DerivedCoreProperties.txt), General_Category Cf
(format) or Co (private use), in the Unicode version pinned in `catalogs.json`.
Cf was added on top of the plan's DICP + private use by the operator on
2026-10-28: U+FFF9, measured red on 2026-10-28, is Cf but not DICP, as are 31
others.

The two UCD files are downloaded by hand into a directory of their own and passed
in; the script checks their sha256 against the pins before reading a byte, so a
different file can never produce the frozen output silently:

    python3 -I docs/threat-2026/build-codepoints.py <dir with the two UCD files>
"""
from __future__ import annotations

import hashlib
import json
import sys
from pathlib import Path

HERE = Path(__file__).resolve().parent
UNICODE = "18.0.0"
PINS = {
    "DerivedCoreProperties.txt": "09c928886a178fcafd93c29e4bd59073a058e5a100b716d425cb563ab50f68c9",
    "UnicodeData.txt": "0736451de439ae7baf1425136617da495e09ee5afbe6e394374db7009ea08950",
}


def _ranges(points: set[int]) -> list[list[str]]:
    out, xs = [], sorted(points)
    start = prev = xs[0]
    for x in xs[1:]:
        if x != prev + 1:
            out.append([f"{start:04X}", f"{prev:04X}"])
            start = x
        prev = x
    out.append([f"{start:04X}", f"{prev:04X}"])
    return out


def main(src: Path) -> int:
    for name, pin in PINS.items():
        digest = hashlib.sha256((src / name).read_bytes()).hexdigest()
        if digest != pin:
            raise SystemExit(f"{name}: sha256 {digest} is not the pinned {pin}")
    dicp: set[int] = set()
    for line in (src / "DerivedCoreProperties.txt").read_text(encoding="utf-8").splitlines():
        line = line.split("#")[0].strip()
        if not line:
            continue
        rng, prop = (x.strip() for x in line.split(";")[:2])
        if prop == "Default_Ignorable_Code_Point":
            lo, _, hi = rng.partition("..")
            dicp.update(range(int(lo, 16), int(hi or lo, 16) + 1))
    cat: dict[str, set[int]] = {"Cf": set(), "Co": set()}
    first = None
    for line in (src / "UnicodeData.txt").read_text(encoding="utf-8").splitlines():
        fields = line.split(";")
        cp, name, gc = int(fields[0], 16), fields[1], fields[2]
        if name.endswith(", First>"):
            first = cp
            continue
        span = range(first, cp + 1) if name.endswith(", Last>") else (cp,)
        if gc in cat:
            cat[gc].update(span)
    union = dicp | cat["Cf"] | cat["Co"]
    doc = {
        "id": "invisible-codepoint-denominator",
        "unicode": UNICODE,
        "sources": {f"https://www.unicode.org/Public/{UNICODE}/ucd/{n}": f"sha256:{p}"
                    for n, p in PINS.items()},
        "rule": "Default_Ignorable_Code_Point | General_Category=Cf | General_Category=Co",
        "decided": "operator 2026-10-28: Cf added to the plan's DICP + private use",
        "counts": {
            "default_ignorable": len(dicp),
            "format_cf": len(cat["Cf"]),
            "private_use_co": len(cat["Co"]),
            "cf_not_default_ignorable": len(cat["Cf"] - dicp),
            "total": len(union),
        },
        "cf_not_default_ignorable": [f"{c:04X}" for c in sorted(cat["Cf"] - dicp)],
        "ranges": _ranges(union),
    }
    (HERE / "codepoints.json").write_text(json.dumps(doc, indent=1) + "\n", encoding="utf-8")
    print(json.dumps(doc["counts"]))
    return 0


if __name__ == "__main__":
    if len(sys.argv) != 2:
        raise SystemExit(__doc__)
    sys.exit(main(Path(sys.argv[1])))