"""Fetch the Claude Code documentation tree into a staging folder that `okfwiki.sources freeze` reads. No model. Usage: python3 tools/fetch_docs.py --staging DIR --meta FILE --report FILE --preamble FILE [--base URL] [--index URL] [--changelog URL] [--changelog-page URL] [--pause SECONDS] [--cafile FILE] [--collection NAME] The index (llms.txt) lists the pages. Every unique address under --base is fetched as .md, one at a time with a pause; "\\r\\n" becomes "\\n". Addresses outside --base (the indexes of other languages) are listed, not fetched. Preamble Each page starts with a block of "> " lines whose first is a level-2 heading, then one blank line. It is removed when its number of lines and the sha256 of its lines ("\\n"-joined) equal the --preamble file ({"lines": N, "sha256": ""}, kept outside git). Any other start is an error, never a guess. Moved out A page whose final address after redirects lies outside --base is no longer a page of this tree: it is not a source, and it is listed in the report (the operator's decision of 08.10.2026 on such a page). History The weekly digest pages (whats-new/YYYY-wNN.md under --base) are docs sources with main_topic false: they add history to the concepts and get no main page of their own (the operator's decision of 08.10.2026). The overview whats-new/index.md is an ordinary docs page. Changelog The release-note page is not a docs source. After its preamble it is split into one file per release, the text untouched, named release-vX.Y.Z.md, with the url --changelog-page#X-Y-Z; every anchor is checked on that page. Preamble + releases + the rest = the page. Two forms, never mixed on one page: component one ... block per release (the copy of 08.10); bold a line that is only "**X.Y.Z**", a blank line and the date ("May 1, 2026"); the release runs to the next such line or the end of the page (the page's form after 20.10, 437 of 437 on 07.11). Complete docs sources = unique index addresses under --base - the changelog - the pages moved out, and release-note sources = release_count(page), counted apart: the "]*>$') CLOSE = "" VERSION = re.compile(r"^\d+\.\d+\.\d+$") BOLD = re.compile(r"^\*\*(\d+\.\d+\.\d+)\*\*$") BOLD_LIKE = re.compile(r"^\*\*v?\d+\.\d[^*]*\*\*\s*$") DATE = re.compile(r"^[A-Z][a-z]+ \d{1,2}, \d{4}$") HISTORY = re.compile(r"^whats-new/\d{4}-w\d{2}\.md$") HEX64 = re.compile(r"^[0-9a-f]{64}$") class FetchError(Exception): """A page or the index breaks a rule of the fetch.""" @dataclass(frozen=True) class Fetched: status: int url: str # the final address after redirects content_type: str body: bytes Fetcher = Callable[[str], Fetched] @dataclass(frozen=True) class Preamble: lines: int sha256: str @dataclass(frozen=True) class IndexPage: url: str title: str sections: tuple[str, ...] @dataclass(frozen=True) class Index: items: int pages: list[IndexPage] other_links: list[str] @dataclass(frozen=True) class Block: version: str text: str @dataclass(frozen=True) class ChangelogSplit: blocks: list[Block] preamble_bytes: int block_bytes: int rest_bytes: int @dataclass class Collected: files: dict[str, bytes] = field(default_factory=dict) meta: dict[str, Any] = field(default_factory=dict) report: dict[str, Any] = field(default_factory=dict) problems: list[str] = field(default_factory=list) def parse_index(text: str, base: str) -> Index: """The list items of the index: unique addresses under base in order, each with its ##/###/#### headings.""" items = 0 stack: list[str] = [] pages: dict[str, IndexPage] = {} other: list[str] = [] for line in text.split("\n"): h = HEADING.match(line) if h: stack = stack[: len(h.group(1)) - 2] + [h.group(2)] continue m = LINK.match(line) if not m: continue items += 1 title, url = m.group(1), m.group(2) if url.startswith(base): pages.setdefault(url, IndexPage(url, title, tuple(stack))) elif url not in other: other.append(url) return Index(items, list(pages.values()), other) def source_name(url: str, base: str) -> str: """https://…/docs/en/a/b.md -> docs-en-a-b.md (the pattern of the copy of 21.09).""" if not url.startswith(base) or not url.endswith(".md"): raise FetchError(f"{url} is not a .md address under {base}") rel = url[len(base): -len(".md")] name = f"docs-{base.rstrip('/').rsplit('/', 1)[-1]}-{rel.replace('/', '-')}.md" if not rel or not NAME.match(name): raise FetchError(f"{url} gives the name {name!r}, not [a-z0-9._-]") return name def is_history(rel: str) -> bool: """A weekly digest page, by its path under the docs base: history, not a main topic (operator 08.10.2026).""" return bool(HISTORY.match(rel)) def load_preamble(path: Path) -> Preamble: try: doc = json.loads(path.read_text(encoding="utf-8")) except (OSError, json.JSONDecodeError) as exc: raise FetchError(f"the preamble file {path} cannot be read: {exc}") from None lines, digest = (doc.get("lines"), doc.get("sha256")) if isinstance(doc, dict) else (None, None) if not (isinstance(lines, int) and not isinstance(lines, bool) and lines >= 1 and isinstance(digest, str) and HEX64.match(digest)): raise FetchError(f"the preamble file {path} is not {{\"lines\": N, \"sha256\": \"\"}}") return Preamble(lines, digest) def strip_preamble(text: str, spec: Preamble) -> tuple[str, int]: """The page without its preamble and the blank line after it, and the number of bytes removed.""" lines = text.split("\n") n = 0 while n < len(lines) and lines[n].startswith("> "): n += 1 pre = lines[:n] if n != spec.lines or not pre[0].startswith("> ## ") or n >= len(lines) or lines[n] != "": raise FetchError(f"the page does not start with {spec.lines} '> ' lines, a '> ## ' heading first, " f"and a blank line (found {n} '> ' lines)") if hashlib.sha256("\n".join(pre).encode("utf-8")).hexdigest() != spec.sha256: raise FetchError("the preamble has the known shape but not the known sha256") rest = "\n".join(lines[n + 1:]) return rest, len(text.encode("utf-8")) - len(rest.encode("utf-8")) def release_count(text: str) -> int: """The releases on the page, counted apart from the split: its " ChangelogSplit: """One block per "**X.Y.Z**" line with a blank line and a date after it, running to the next one.""" lines = [line + "\n" for line in body.split("\n")] lines[-1] = lines[-1][:-1] bare = [line.rstrip("\n") for line in lines] starts: list[tuple[int, str]] = [] for i, line in enumerate(bare): if not BOLD_LIKE.match(line): continue m = BOLD.match(line) if not m: raise FetchError(f"line {i + 1}: {line!r} is not **X.Y.Z**") if not (i + 2 < len(bare) and bare[i + 1] == "" and DATE.match(bare[i + 2])): raise FetchError(f"line {i + 1}: version {m.group(1)} is not followed by a blank line and a date") if any(v == m.group(1) for _, v in starts): raise FetchError(f"line {i + 1}: version {m.group(1)} appears twice") starts.append((i, m.group(1))) if not starts: raise FetchError("the page has no block and no **X.Y.Z** release line") ends = [i for i, _ in starts[1:]] + [len(lines)] blocks = [Block(v, "".join(lines[i:end])) for (i, v), end in zip(starts, ends)] rest_bytes = sum(len(line.encode("utf-8")) for line in lines[:starts[0][0]]) block_bytes = sum(len(b.text.encode("utf-8")) for b in blocks) if pre_bytes + block_bytes + rest_bytes != total: raise FetchError("preamble + blocks + rest is not the length of the page") return ChangelogSplit(blocks, pre_bytes, block_bytes, rest_bytes) def split_changelog(text: str, spec: Preamble) -> ChangelogSplit: """One block per release, the text untouched, in the component or the bold form; every byte is counted.""" body, pre_bytes = strip_preamble(text, spec) if " blocks and **X.Y.Z** release lines") lines = [line + "\n" for line in body.split("\n")] lines[-1] = lines[-1][:-1] blocks: list[Block] = [] rest_bytes = 0 current: list[str] | None = None version = "" for n, line in enumerate(lines, 1): bare = line.rstrip("\n") opened = OPEN.match(bare) if current is None: if opened: version = opened.group(1) if not VERSION.match(version): raise FetchError(f"line {n}: the label {version!r} is not X.Y.Z") if any(b.version == version for b in blocks): raise FetchError(f"line {n}: version {version} appears twice") current = [line] elif bare == CLOSE: raise FetchError(f"line {n}: {CLOSE} without an open block") else: rest_bytes += len(line.encode("utf-8")) else: if opened: raise FetchError(f"line {n}: a block opens inside version {version}") current.append(line) if bare == CLOSE: blocks.append(Block(version, "".join(current))) current = None if current is not None: raise FetchError(f"version {version} is never closed") if not blocks: raise FetchError("the page has no block") if body.count(" list[str]: return [v for v in versions if f'id="{v.replace(".", "-")}"' not in html] def collect(fetcher: Fetcher, preamble: Preamble, *, base: str = BASE, index_url: str = INDEX_URL, changelog: str = CHANGELOG, changelog_page: str = CHANGELOG_PAGE, pause: float = 0.5, collection: str = "claude-code") -> Collected: """Fetch and split everything in memory; nothing is written here.""" got = Collected() problems = got.problems def fetch(url: str) -> Fetched: f = fetcher(url) if pause: time.sleep(pause) return f raw_index = fetch(index_url) if raw_index.status != 200: raise RunError(f"the index {index_url} gave status {raw_index.status}") index_text = raw_index.body.decode("utf-8").replace("\r\n", "\n") idx = parse_index(index_text, base) urls = [p.url for p in idx.pages] if changelog not in urls: problems.append(f"the changelog {changelog} is not in the index") sections = {p.url: p.sections for p in idx.pages} names: dict[str, str] = {} by_name: dict[str, str] = {} for url in urls: try: name = source_name(url, base) except FetchError as exc: problems.append(str(exc)) continue if name in by_name: problems.append(f"{name}: same path for {by_name[name]} and {url}") continue by_name[name] = url names[url] = name sources: dict[str, dict[str, Any]] = {} moved_out: list[dict[str, str]] = [] crlf: list[str] = [] preamble_pages = preamble_bytes = raw_updates = 0 changelog_bytes: dict[str, int] = {} docs_sections: dict[str, list[str]] = {} versions: list[str] = [] history_pages: list[str] = [] for url in urls: if url not in names: continue name = names[url] f = fetch(url) if f.status != 200: problems.append(f"{url}: status {f.status}") continue if not f.url.startswith(base): moved_out.append({"content_type": f.content_type, "final_url": f.url, "url": url}) continue if not f.content_type.startswith("text/markdown"): problems.append(f"{url} ({name}): {f.content_type or 'no content type'}, not markdown") continue try: text = f.body.decode("utf-8") except UnicodeDecodeError: problems.append(f"{url} ({name}): not UTF-8") continue if "\r\n" in text: crlf.append(name) text = text.replace("\r\n", "\n") try: if url == changelog: raw_updates = release_count(text) split = split_changelog(text, preamble) changelog_bytes = {"page": len(text.encode("utf-8")), "preamble": split.preamble_bytes, "blocks": split.block_bytes, "rest": split.rest_bytes} preamble_pages += 1 preamble_bytes += split.preamble_bytes for b in split.blocks: rn = f"release-v{b.version}.md" got.files[rn] = b.text.encode("utf-8") sources[rn] = {"kind": "release-notes", "main_topic": False, "url": anchor_url(changelog_page, b.version), "version": b.version} versions.append(b.version) else: body, removed = strip_preamble(text, preamble) preamble_pages += 1 preamble_bytes += removed got.files[name] = body.encode("utf-8") history = is_history(url[len(base):]) sources[name] = {"kind": "docs", "main_topic": not history, "url": url} if history: history_pages.append(name) docs_sections[name] = list(sections[url]) except FetchError as exc: problems.append(f"{url} ({name}): {exc}") if versions: page = fetch(changelog_page) if page.status != 200: problems.append(f"{changelog_page}: status {page.status}; the version anchors cannot be checked") else: for v in missing_anchors(page.body.decode("utf-8", errors="replace"), versions): problems.append(f"{v}: no anchor id=\"{v.replace('.', '-')}\" on {changelog_page}") docs = sorted(n for n, s in sources.items() if s["kind"] == "docs") notes = sorted(n for n, s in sources.items() if s["kind"] == "release-notes") has_changelog = int(changelog in urls) complete = {"index_pages": len(urls), "changelog": has_changelog, "moved_out": len(moved_out), "docs_expected": len(urls) - has_changelog - len(moved_out), "docs": len(docs), "release_notes_expected": raw_updates, "release_notes": len(notes)} complete["pass"] = (complete["docs"] == complete["docs_expected"] and complete["release_notes"] == complete["release_notes_expected"] and raw_updates > 0) if not complete["pass"]: problems.append(f"completeness FAIL: docs {complete['docs']} of {complete['docs_expected']} expected, " f"release notes {complete['release_notes']} of {complete['release_notes_expected']}") got.meta = {"collection": collection, "schema": META_SCHEMA, "index": {"pages": len(urls), "sha256": hashlib.sha256(raw_index.body).hexdigest(), "url": index_url}, "sources": dict(sorted(sources.items()))} got.report = { "index": {"url": index_url, "sha256": got.meta["index"]["sha256"], "bytes": len(raw_index.body), "items": idx.items, "pages": len(urls)}, "other_links": idx.other_links, "moved_out": moved_out, "completeness": complete, "crlf_pages": crlf, "history_pages": sorted(history_pages), "preamble": {"pages": preamble_pages, "bytes": preamble_bytes}, "changelog_bytes": changelog_bytes, "docs": {"count": len(docs), "bytes": sum(len(got.files[n]) for n in docs)}, "release_notes": {"count": len(notes), "bytes": sum(len(got.files[n]) for n in notes)}, "sections": docs_sections, "problems": problems} return got def net_fetcher(cafile: str | None, timeout: float = 60.0) -> Fetcher: """urllib with certificate checks on. The python.org build here has no CA store, so the system's file is used.""" if cafile is None and not os.environ.get("SSL_CERT_FILE") and os.path.isfile(SYSTEM_CA): cafile = SYSTEM_CA context = ssl.create_default_context(cafile=cafile) def fetch(url: str) -> Fetched: req = urllib.request.Request(url, headers={"User-Agent": USER_AGENT}) try: with urllib.request.urlopen(req, timeout=timeout, context=context) as r: return Fetched(r.status, r.geturl(), r.headers.get("Content-Type", ""), r.read()) except urllib.error.HTTPError as exc: return Fetched(exc.code, exc.geturl() or url, exc.headers.get("Content-Type", "") if exc.headers else "", b"") return fetch def build_parser() -> argparse.ArgumentParser: p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) p.add_argument("--staging", required=True) p.add_argument("--meta", required=True) p.add_argument("--report", required=True) p.add_argument("--preamble", required=True) p.add_argument("--base", default=BASE) p.add_argument("--index", default=INDEX_URL) p.add_argument("--changelog", default=CHANGELOG) p.add_argument("--changelog-page", default=CHANGELOG_PAGE) p.add_argument("--pause", type=float, default=0.5) p.add_argument("--cafile") p.add_argument("--collection", default="claude-code") return p def main(argv: list[str] | None = None, fetcher: Fetcher | None = None) -> int: args = build_parser().parse_args(argv) staging, meta, report = Path(args.staging), Path(args.meta), Path(args.report) try: for p in (staging, meta, report): if p.exists(): raise RunError(f"{p} exists; refusing to overwrite") got = collect(fetcher or net_fetcher(args.cafile), load_preamble(Path(args.preamble)), base=args.base, index_url=args.index, changelog=args.changelog, changelog_page=args.changelog_page, pause=args.pause, collection=args.collection) write_new(report, dump(got.report)) c = got.report["completeness"] summary = (f"fetch: {c['docs']} docs of {c['docs_expected']} expected, {c['release_notes']} release notes of " f"{c['release_notes_expected']}, {c['moved_out']} moved out, {len(got.report['other_links'])} " f"other links not fetched") if got.problems: print(f"{summary}; FAIL with {len(got.problems)} problems, nothing staged (report: {report})", file=sys.stderr) for problem in got.problems: print(f" {problem}", file=sys.stderr) return 1 staging.mkdir(parents=True) for name, data in got.files.items(): (staging / name).write_bytes(data) write_new(meta, dump(got.meta)) print(f"{summary}; completeness pass -> {staging}") for m in got.report["moved_out"]: print(f" moved out: {m['url']} -> {m['final_url']}") return 0 except (RunError, FetchError, OSError, UnicodeDecodeError) as exc: print(f"fetch cannot run: {exc}", file=sys.stderr) return 2 if __name__ == "__main__": sys.exit(main())