okf-wiki/tools/fetch_docs.py
Kjell Tore Guttormsen 0fd86a23a3
okf-wiki 1.1.1
First public release, published as one snapshot. What changed between
versions is in CHANGELOG.md.

Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
2026-11-14 21:03:20 +01:00

482 lines
21 KiB
Python

"""Fetch the Claude Code documentation tree into a staging folder that `okfwiki.sources freeze` reads. No model.
Usage:
python3 tools/fetch_docs.py --staging DIR --meta FILE --report FILE --preamble FILE
[--base URL] [--index URL] [--changelog URL] [--changelog-page URL]
[--pause SECONDS] [--cafile FILE] [--collection NAME]
The index (llms.txt) lists the pages. Every unique address under --base is fetched as .md, one at a time with a pause;
"\\r\\n" becomes "\\n". Addresses outside --base (the indexes of other languages) are listed, not fetched.
Preamble Each page starts with a block of "> " lines whose first is a level-2 heading, then one blank line. It is
removed when its number of lines and the sha256 of its lines ("\\n"-joined) equal the --preamble file
({"lines": N, "sha256": "<hex>"}, kept outside git). Any other start is an error, never a guess.
Moved out A page whose final address after redirects lies outside --base is no longer a page of this tree: it is
not a source, and it is listed in the report (the operator's decision of 08.10.2026 on such a page).
History The weekly digest pages (whats-new/YYYY-wNN.md under --base) are docs sources with main_topic false: they
add history to the concepts and get no main page of their own (the operator's decision of 08.10.2026). The
overview whats-new/index.md is an ordinary docs page.
Changelog The release-note page is not a docs source. After its preamble it is split into one file per release,
the text untouched, named release-vX.Y.Z.md, with the url --changelog-page#X-Y-Z; every anchor is checked
on that page. Preamble + releases + the rest = the page. Two forms, never mixed on one page:
component one <Update label="X.Y.Z"> ... </Update> block per release (the copy of 08.10);
bold a line that is only "**X.Y.Z**", a blank line and the date ("May 1, 2026"); the release runs to
the next such line or the end of the page (the page's form after 20.10, 437 of 437 on 07.11).
Complete docs sources = unique index addresses under --base - the changelog - the pages moved out, and release-note
sources = release_count(page), counted apart: the "<Update label=" on the page, or, in the bold form, every
bold line that starts with a version. A difference is a FAIL.
Exit 0: staging folder and meta written. Exit 1: a page broke a rule or the completeness check failed; the report
lists why and nothing else is written. Exit 2: cannot run (staging, meta or report exists, the index cannot be
fetched, the network fails).
"""
from __future__ import annotations
import argparse
import hashlib
import json
import os
import re
import ssl
import sys
import time
import urllib.error
import urllib.request
from collections.abc import Callable
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from okfwiki.common import RunError, write_new # noqa: E402
from okfwiki.sources import META_SCHEMA, NAME, dump # noqa: E402
BASE = "https://code.claude.com/docs/en/"
INDEX_URL = "https://code.claude.com/docs/llms.txt"
CHANGELOG = BASE + "changelog.md"
CHANGELOG_PAGE = BASE + "changelog"
SYSTEM_CA = "/etc/ssl/cert.pem"
USER_AGENT = "okf-wiki-fetch-docs/1"
LINK = re.compile(r"^\s*[-*]\s+\[([^\]]*)\]\((\S+?)\)")
HEADING = re.compile(r"^(#{2,4})\s+(.*?)\s*$")
OPEN = re.compile(r'^<Update label="([^"]*)"[^>]*>$')
CLOSE = "</Update>"
VERSION = re.compile(r"^\d+\.\d+\.\d+$")
BOLD = re.compile(r"^\*\*(\d+\.\d+\.\d+)\*\*$")
BOLD_LIKE = re.compile(r"^\*\*v?\d+\.\d[^*]*\*\*\s*$")
DATE = re.compile(r"^[A-Z][a-z]+ \d{1,2}, \d{4}$")
HISTORY = re.compile(r"^whats-new/\d{4}-w\d{2}\.md$")
HEX64 = re.compile(r"^[0-9a-f]{64}$")
class FetchError(Exception):
"""A page or the index breaks a rule of the fetch."""
@dataclass(frozen=True)
class Fetched:
status: int
url: str # the final address after redirects
content_type: str
body: bytes
Fetcher = Callable[[str], Fetched]
@dataclass(frozen=True)
class Preamble:
lines: int
sha256: str
@dataclass(frozen=True)
class IndexPage:
url: str
title: str
sections: tuple[str, ...]
@dataclass(frozen=True)
class Index:
items: int
pages: list[IndexPage]
other_links: list[str]
@dataclass(frozen=True)
class Block:
version: str
text: str
@dataclass(frozen=True)
class ChangelogSplit:
blocks: list[Block]
preamble_bytes: int
block_bytes: int
rest_bytes: int
@dataclass
class Collected:
files: dict[str, bytes] = field(default_factory=dict)
meta: dict[str, Any] = field(default_factory=dict)
report: dict[str, Any] = field(default_factory=dict)
problems: list[str] = field(default_factory=list)
def parse_index(text: str, base: str) -> Index:
"""The list items of the index: unique addresses under base in order, each with its ##/###/#### headings."""
items = 0
stack: list[str] = []
pages: dict[str, IndexPage] = {}
other: list[str] = []
for line in text.split("\n"):
h = HEADING.match(line)
if h:
stack = stack[: len(h.group(1)) - 2] + [h.group(2)]
continue
m = LINK.match(line)
if not m:
continue
items += 1
title, url = m.group(1), m.group(2)
if url.startswith(base):
pages.setdefault(url, IndexPage(url, title, tuple(stack)))
elif url not in other:
other.append(url)
return Index(items, list(pages.values()), other)
def source_name(url: str, base: str) -> str:
"""https://…/docs/en/a/b.md -> docs-en-a-b.md (the pattern of the copy of 21.09)."""
if not url.startswith(base) or not url.endswith(".md"):
raise FetchError(f"{url} is not a .md address under {base}")
rel = url[len(base): -len(".md")]
name = f"docs-{base.rstrip('/').rsplit('/', 1)[-1]}-{rel.replace('/', '-')}.md"
if not rel or not NAME.match(name):
raise FetchError(f"{url} gives the name {name!r}, not [a-z0-9._-]")
return name
def is_history(rel: str) -> bool:
"""A weekly digest page, by its path under the docs base: history, not a main topic (operator 08.10.2026)."""
return bool(HISTORY.match(rel))
def load_preamble(path: Path) -> Preamble:
try:
doc = json.loads(path.read_text(encoding="utf-8"))
except (OSError, json.JSONDecodeError) as exc:
raise FetchError(f"the preamble file {path} cannot be read: {exc}") from None
lines, digest = (doc.get("lines"), doc.get("sha256")) if isinstance(doc, dict) else (None, None)
if not (isinstance(lines, int) and not isinstance(lines, bool) and lines >= 1
and isinstance(digest, str) and HEX64.match(digest)):
raise FetchError(f"the preamble file {path} is not {{\"lines\": N, \"sha256\": \"<hex>\"}}")
return Preamble(lines, digest)
def strip_preamble(text: str, spec: Preamble) -> tuple[str, int]:
"""The page without its preamble and the blank line after it, and the number of bytes removed."""
lines = text.split("\n")
n = 0
while n < len(lines) and lines[n].startswith("> "):
n += 1
pre = lines[:n]
if n != spec.lines or not pre[0].startswith("> ## ") or n >= len(lines) or lines[n] != "":
raise FetchError(f"the page does not start with {spec.lines} '> ' lines, a '> ## ' heading first, "
f"and a blank line (found {n} '> ' lines)")
if hashlib.sha256("\n".join(pre).encode("utf-8")).hexdigest() != spec.sha256:
raise FetchError("the preamble has the known shape but not the known sha256")
rest = "\n".join(lines[n + 1:])
return rest, len(text.encode("utf-8")) - len(rest.encode("utf-8"))
def release_count(text: str) -> int:
"""The releases on the page, counted apart from the split: its "<Update label=", or its bold version-like lines."""
if "<Update label=" in text:
return text.count("<Update label=")
return sum(1 for line in text.split("\n") if BOLD_LIKE.match(line))
def split_bold(body: str, pre_bytes: int, total: int) -> ChangelogSplit:
"""One block per "**X.Y.Z**" line with a blank line and a date after it, running to the next one."""
lines = [line + "\n" for line in body.split("\n")]
lines[-1] = lines[-1][:-1]
bare = [line.rstrip("\n") for line in lines]
starts: list[tuple[int, str]] = []
for i, line in enumerate(bare):
if not BOLD_LIKE.match(line):
continue
m = BOLD.match(line)
if not m:
raise FetchError(f"line {i + 1}: {line!r} is not **X.Y.Z**")
if not (i + 2 < len(bare) and bare[i + 1] == "" and DATE.match(bare[i + 2])):
raise FetchError(f"line {i + 1}: version {m.group(1)} is not followed by a blank line and a date")
if any(v == m.group(1) for _, v in starts):
raise FetchError(f"line {i + 1}: version {m.group(1)} appears twice")
starts.append((i, m.group(1)))
if not starts:
raise FetchError("the page has no <Update label=…> block and no **X.Y.Z** release line")
ends = [i for i, _ in starts[1:]] + [len(lines)]
blocks = [Block(v, "".join(lines[i:end])) for (i, v), end in zip(starts, ends)]
rest_bytes = sum(len(line.encode("utf-8")) for line in lines[:starts[0][0]])
block_bytes = sum(len(b.text.encode("utf-8")) for b in blocks)
if pre_bytes + block_bytes + rest_bytes != total:
raise FetchError("preamble + blocks + rest is not the length of the page")
return ChangelogSplit(blocks, pre_bytes, block_bytes, rest_bytes)
def split_changelog(text: str, spec: Preamble) -> ChangelogSplit:
"""One block per release, the text untouched, in the component or the bold form; every byte is counted."""
body, pre_bytes = strip_preamble(text, spec)
if "<Update label=" not in body:
return split_bold(body, pre_bytes, len(text.encode("utf-8")))
if any(BOLD_LIKE.match(line) for line in body.split("\n")):
raise FetchError("the page mixes <Update label=…> blocks and **X.Y.Z** release lines")
lines = [line + "\n" for line in body.split("\n")]
lines[-1] = lines[-1][:-1]
blocks: list[Block] = []
rest_bytes = 0
current: list[str] | None = None
version = ""
for n, line in enumerate(lines, 1):
bare = line.rstrip("\n")
opened = OPEN.match(bare)
if current is None:
if opened:
version = opened.group(1)
if not VERSION.match(version):
raise FetchError(f"line {n}: the label {version!r} is not X.Y.Z")
if any(b.version == version for b in blocks):
raise FetchError(f"line {n}: version {version} appears twice")
current = [line]
elif bare == CLOSE:
raise FetchError(f"line {n}: {CLOSE} without an open block")
else:
rest_bytes += len(line.encode("utf-8"))
else:
if opened:
raise FetchError(f"line {n}: a block opens inside version {version}")
current.append(line)
if bare == CLOSE:
blocks.append(Block(version, "".join(current)))
current = None
if current is not None:
raise FetchError(f"version {version} is never closed")
if not blocks:
raise FetchError("the page has no <Update label=…> block")
if body.count("<Update label=") != len(blocks):
raise FetchError(f"the page has {body.count('<Update label=')} '<Update label=', but {len(blocks)} blocks "
"start a line")
block_bytes = sum(len(b.text.encode("utf-8")) for b in blocks)
if pre_bytes + block_bytes + rest_bytes != len(text.encode("utf-8")):
raise FetchError("preamble + blocks + rest is not the length of the page")
return ChangelogSplit(blocks, pre_bytes, block_bytes, rest_bytes)
def anchor_url(page: str, version: str) -> str:
return f"{page}#{version.replace('.', '-')}"
def missing_anchors(html: str, versions: list[str]) -> list[str]:
return [v for v in versions if f'id="{v.replace(".", "-")}"' not in html]
def collect(fetcher: Fetcher, preamble: Preamble, *, base: str = BASE, index_url: str = INDEX_URL,
changelog: str = CHANGELOG, changelog_page: str = CHANGELOG_PAGE, pause: float = 0.5,
collection: str = "claude-code") -> Collected:
"""Fetch and split everything in memory; nothing is written here."""
got = Collected()
problems = got.problems
def fetch(url: str) -> Fetched:
f = fetcher(url)
if pause:
time.sleep(pause)
return f
raw_index = fetch(index_url)
if raw_index.status != 200:
raise RunError(f"the index {index_url} gave status {raw_index.status}")
index_text = raw_index.body.decode("utf-8").replace("\r\n", "\n")
idx = parse_index(index_text, base)
urls = [p.url for p in idx.pages]
if changelog not in urls:
problems.append(f"the changelog {changelog} is not in the index")
sections = {p.url: p.sections for p in idx.pages}
names: dict[str, str] = {}
by_name: dict[str, str] = {}
for url in urls:
try:
name = source_name(url, base)
except FetchError as exc:
problems.append(str(exc))
continue
if name in by_name:
problems.append(f"{name}: same path for {by_name[name]} and {url}")
continue
by_name[name] = url
names[url] = name
sources: dict[str, dict[str, Any]] = {}
moved_out: list[dict[str, str]] = []
crlf: list[str] = []
preamble_pages = preamble_bytes = raw_updates = 0
changelog_bytes: dict[str, int] = {}
docs_sections: dict[str, list[str]] = {}
versions: list[str] = []
history_pages: list[str] = []
for url in urls:
if url not in names:
continue
name = names[url]
f = fetch(url)
if f.status != 200:
problems.append(f"{url}: status {f.status}")
continue
if not f.url.startswith(base):
moved_out.append({"content_type": f.content_type, "final_url": f.url, "url": url})
continue
if not f.content_type.startswith("text/markdown"):
problems.append(f"{url} ({name}): {f.content_type or 'no content type'}, not markdown")
continue
try:
text = f.body.decode("utf-8")
except UnicodeDecodeError:
problems.append(f"{url} ({name}): not UTF-8")
continue
if "\r\n" in text:
crlf.append(name)
text = text.replace("\r\n", "\n")
try:
if url == changelog:
raw_updates = release_count(text)
split = split_changelog(text, preamble)
changelog_bytes = {"page": len(text.encode("utf-8")), "preamble": split.preamble_bytes,
"blocks": split.block_bytes, "rest": split.rest_bytes}
preamble_pages += 1
preamble_bytes += split.preamble_bytes
for b in split.blocks:
rn = f"release-v{b.version}.md"
got.files[rn] = b.text.encode("utf-8")
sources[rn] = {"kind": "release-notes", "main_topic": False,
"url": anchor_url(changelog_page, b.version), "version": b.version}
versions.append(b.version)
else:
body, removed = strip_preamble(text, preamble)
preamble_pages += 1
preamble_bytes += removed
got.files[name] = body.encode("utf-8")
history = is_history(url[len(base):])
sources[name] = {"kind": "docs", "main_topic": not history, "url": url}
if history:
history_pages.append(name)
docs_sections[name] = list(sections[url])
except FetchError as exc:
problems.append(f"{url} ({name}): {exc}")
if versions:
page = fetch(changelog_page)
if page.status != 200:
problems.append(f"{changelog_page}: status {page.status}; the version anchors cannot be checked")
else:
for v in missing_anchors(page.body.decode("utf-8", errors="replace"), versions):
problems.append(f"{v}: no anchor id=\"{v.replace('.', '-')}\" on {changelog_page}")
docs = sorted(n for n, s in sources.items() if s["kind"] == "docs")
notes = sorted(n for n, s in sources.items() if s["kind"] == "release-notes")
has_changelog = int(changelog in urls)
complete = {"index_pages": len(urls), "changelog": has_changelog, "moved_out": len(moved_out),
"docs_expected": len(urls) - has_changelog - len(moved_out), "docs": len(docs),
"release_notes_expected": raw_updates, "release_notes": len(notes)}
complete["pass"] = (complete["docs"] == complete["docs_expected"]
and complete["release_notes"] == complete["release_notes_expected"] and raw_updates > 0)
if not complete["pass"]:
problems.append(f"completeness FAIL: docs {complete['docs']} of {complete['docs_expected']} expected, "
f"release notes {complete['release_notes']} of {complete['release_notes_expected']}")
got.meta = {"collection": collection, "schema": META_SCHEMA,
"index": {"pages": len(urls), "sha256": hashlib.sha256(raw_index.body).hexdigest(), "url": index_url},
"sources": dict(sorted(sources.items()))}
got.report = {
"index": {"url": index_url, "sha256": got.meta["index"]["sha256"], "bytes": len(raw_index.body),
"items": idx.items, "pages": len(urls)},
"other_links": idx.other_links, "moved_out": moved_out, "completeness": complete, "crlf_pages": crlf,
"history_pages": sorted(history_pages),
"preamble": {"pages": preamble_pages, "bytes": preamble_bytes}, "changelog_bytes": changelog_bytes,
"docs": {"count": len(docs), "bytes": sum(len(got.files[n]) for n in docs)},
"release_notes": {"count": len(notes), "bytes": sum(len(got.files[n]) for n in notes)},
"sections": docs_sections, "problems": problems}
return got
def net_fetcher(cafile: str | None, timeout: float = 60.0) -> Fetcher:
"""urllib with certificate checks on. The python.org build here has no CA store, so the system's file is used."""
if cafile is None and not os.environ.get("SSL_CERT_FILE") and os.path.isfile(SYSTEM_CA):
cafile = SYSTEM_CA
context = ssl.create_default_context(cafile=cafile)
def fetch(url: str) -> Fetched:
req = urllib.request.Request(url, headers={"User-Agent": USER_AGENT})
try:
with urllib.request.urlopen(req, timeout=timeout, context=context) as r:
return Fetched(r.status, r.geturl(), r.headers.get("Content-Type", ""), r.read())
except urllib.error.HTTPError as exc:
return Fetched(exc.code, exc.geturl() or url, exc.headers.get("Content-Type", "") if exc.headers else "",
b"")
return fetch
def build_parser() -> argparse.ArgumentParser:
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
p.add_argument("--staging", required=True)
p.add_argument("--meta", required=True)
p.add_argument("--report", required=True)
p.add_argument("--preamble", required=True)
p.add_argument("--base", default=BASE)
p.add_argument("--index", default=INDEX_URL)
p.add_argument("--changelog", default=CHANGELOG)
p.add_argument("--changelog-page", default=CHANGELOG_PAGE)
p.add_argument("--pause", type=float, default=0.5)
p.add_argument("--cafile")
p.add_argument("--collection", default="claude-code")
return p
def main(argv: list[str] | None = None, fetcher: Fetcher | None = None) -> int:
args = build_parser().parse_args(argv)
staging, meta, report = Path(args.staging), Path(args.meta), Path(args.report)
try:
for p in (staging, meta, report):
if p.exists():
raise RunError(f"{p} exists; refusing to overwrite")
got = collect(fetcher or net_fetcher(args.cafile), load_preamble(Path(args.preamble)), base=args.base,
index_url=args.index, changelog=args.changelog, changelog_page=args.changelog_page,
pause=args.pause, collection=args.collection)
write_new(report, dump(got.report))
c = got.report["completeness"]
summary = (f"fetch: {c['docs']} docs of {c['docs_expected']} expected, {c['release_notes']} release notes of "
f"{c['release_notes_expected']}, {c['moved_out']} moved out, {len(got.report['other_links'])} "
f"other links not fetched")
if got.problems:
print(f"{summary}; FAIL with {len(got.problems)} problems, nothing staged (report: {report})",
file=sys.stderr)
for problem in got.problems:
print(f" {problem}", file=sys.stderr)
return 1
staging.mkdir(parents=True)
for name, data in got.files.items():
(staging / name).write_bytes(data)
write_new(meta, dump(got.meta))
print(f"{summary}; completeness pass -> {staging}")
for m in got.report["moved_out"]:
print(f" moved out: {m['url']} -> {m['final_url']}")
return 0
except (RunError, FetchError, OSError, UnicodeDecodeError) as exc:
print(f"fetch cannot run: {exc}", file=sys.stderr)
return 2
if __name__ == "__main__":
sys.exit(main())