okf-wiki 1.1.1
First public release, published as one snapshot. What changed between versions is in CHANGELOG.md. Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
This commit is contained in:
commit
0fd86a23a3
227 changed files with 96077 additions and 0 deletions
482
tools/fetch_docs.py
Normal file
482
tools/fetch_docs.py
Normal file
|
|
@ -0,0 +1,482 @@
|
|||
"""Fetch the Claude Code documentation tree into a staging folder that `okfwiki.sources freeze` reads. No model.
|
||||
|
||||
Usage:
|
||||
python3 tools/fetch_docs.py --staging DIR --meta FILE --report FILE --preamble FILE
|
||||
[--base URL] [--index URL] [--changelog URL] [--changelog-page URL]
|
||||
[--pause SECONDS] [--cafile FILE] [--collection NAME]
|
||||
|
||||
The index (llms.txt) lists the pages. Every unique address under --base is fetched as .md, one at a time with a pause;
|
||||
"\\r\\n" becomes "\\n". Addresses outside --base (the indexes of other languages) are listed, not fetched.
|
||||
|
||||
Preamble Each page starts with a block of "> " lines whose first is a level-2 heading, then one blank line. It is
|
||||
removed when its number of lines and the sha256 of its lines ("\\n"-joined) equal the --preamble file
|
||||
({"lines": N, "sha256": "<hex>"}, kept outside git). Any other start is an error, never a guess.
|
||||
Moved out A page whose final address after redirects lies outside --base is no longer a page of this tree: it is
|
||||
not a source, and it is listed in the report (the operator's decision of 08.10.2026 on such a page).
|
||||
History The weekly digest pages (whats-new/YYYY-wNN.md under --base) are docs sources with main_topic false: they
|
||||
add history to the concepts and get no main page of their own (the operator's decision of 08.10.2026). The
|
||||
overview whats-new/index.md is an ordinary docs page.
|
||||
Changelog The release-note page is not a docs source. After its preamble it is split into one file per release,
|
||||
the text untouched, named release-vX.Y.Z.md, with the url --changelog-page#X-Y-Z; every anchor is checked
|
||||
on that page. Preamble + releases + the rest = the page. Two forms, never mixed on one page:
|
||||
component one <Update label="X.Y.Z"> ... </Update> block per release (the copy of 08.10);
|
||||
bold a line that is only "**X.Y.Z**", a blank line and the date ("May 1, 2026"); the release runs to
|
||||
the next such line or the end of the page (the page's form after 20.10, 437 of 437 on 07.11).
|
||||
Complete docs sources = unique index addresses under --base - the changelog - the pages moved out, and release-note
|
||||
sources = release_count(page), counted apart: the "<Update label=" on the page, or, in the bold form, every
|
||||
bold line that starts with a version. A difference is a FAIL.
|
||||
|
||||
Exit 0: staging folder and meta written. Exit 1: a page broke a rule or the completeness check failed; the report
|
||||
lists why and nothing else is written. Exit 2: cannot run (staging, meta or report exists, the index cannot be
|
||||
fetched, the network fails).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import ssl
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
|
||||
from okfwiki.common import RunError, write_new # noqa: E402
|
||||
from okfwiki.sources import META_SCHEMA, NAME, dump # noqa: E402
|
||||
|
||||
BASE = "https://code.claude.com/docs/en/"
|
||||
INDEX_URL = "https://code.claude.com/docs/llms.txt"
|
||||
CHANGELOG = BASE + "changelog.md"
|
||||
CHANGELOG_PAGE = BASE + "changelog"
|
||||
SYSTEM_CA = "/etc/ssl/cert.pem"
|
||||
USER_AGENT = "okf-wiki-fetch-docs/1"
|
||||
LINK = re.compile(r"^\s*[-*]\s+\[([^\]]*)\]\((\S+?)\)")
|
||||
HEADING = re.compile(r"^(#{2,4})\s+(.*?)\s*$")
|
||||
OPEN = re.compile(r'^<Update label="([^"]*)"[^>]*>$')
|
||||
CLOSE = "</Update>"
|
||||
VERSION = re.compile(r"^\d+\.\d+\.\d+$")
|
||||
BOLD = re.compile(r"^\*\*(\d+\.\d+\.\d+)\*\*$")
|
||||
BOLD_LIKE = re.compile(r"^\*\*v?\d+\.\d[^*]*\*\*\s*$")
|
||||
DATE = re.compile(r"^[A-Z][a-z]+ \d{1,2}, \d{4}$")
|
||||
HISTORY = re.compile(r"^whats-new/\d{4}-w\d{2}\.md$")
|
||||
HEX64 = re.compile(r"^[0-9a-f]{64}$")
|
||||
|
||||
|
||||
class FetchError(Exception):
|
||||
"""A page or the index breaks a rule of the fetch."""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Fetched:
|
||||
status: int
|
||||
url: str # the final address after redirects
|
||||
content_type: str
|
||||
body: bytes
|
||||
|
||||
|
||||
Fetcher = Callable[[str], Fetched]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Preamble:
|
||||
lines: int
|
||||
sha256: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class IndexPage:
|
||||
url: str
|
||||
title: str
|
||||
sections: tuple[str, ...]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Index:
|
||||
items: int
|
||||
pages: list[IndexPage]
|
||||
other_links: list[str]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Block:
|
||||
version: str
|
||||
text: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ChangelogSplit:
|
||||
blocks: list[Block]
|
||||
preamble_bytes: int
|
||||
block_bytes: int
|
||||
rest_bytes: int
|
||||
|
||||
|
||||
@dataclass
|
||||
class Collected:
|
||||
files: dict[str, bytes] = field(default_factory=dict)
|
||||
meta: dict[str, Any] = field(default_factory=dict)
|
||||
report: dict[str, Any] = field(default_factory=dict)
|
||||
problems: list[str] = field(default_factory=list)
|
||||
|
||||
|
||||
def parse_index(text: str, base: str) -> Index:
|
||||
"""The list items of the index: unique addresses under base in order, each with its ##/###/#### headings."""
|
||||
items = 0
|
||||
stack: list[str] = []
|
||||
pages: dict[str, IndexPage] = {}
|
||||
other: list[str] = []
|
||||
for line in text.split("\n"):
|
||||
h = HEADING.match(line)
|
||||
if h:
|
||||
stack = stack[: len(h.group(1)) - 2] + [h.group(2)]
|
||||
continue
|
||||
m = LINK.match(line)
|
||||
if not m:
|
||||
continue
|
||||
items += 1
|
||||
title, url = m.group(1), m.group(2)
|
||||
if url.startswith(base):
|
||||
pages.setdefault(url, IndexPage(url, title, tuple(stack)))
|
||||
elif url not in other:
|
||||
other.append(url)
|
||||
return Index(items, list(pages.values()), other)
|
||||
|
||||
|
||||
def source_name(url: str, base: str) -> str:
|
||||
"""https://…/docs/en/a/b.md -> docs-en-a-b.md (the pattern of the copy of 21.09)."""
|
||||
if not url.startswith(base) or not url.endswith(".md"):
|
||||
raise FetchError(f"{url} is not a .md address under {base}")
|
||||
rel = url[len(base): -len(".md")]
|
||||
name = f"docs-{base.rstrip('/').rsplit('/', 1)[-1]}-{rel.replace('/', '-')}.md"
|
||||
if not rel or not NAME.match(name):
|
||||
raise FetchError(f"{url} gives the name {name!r}, not [a-z0-9._-]")
|
||||
return name
|
||||
|
||||
|
||||
def is_history(rel: str) -> bool:
|
||||
"""A weekly digest page, by its path under the docs base: history, not a main topic (operator 08.10.2026)."""
|
||||
return bool(HISTORY.match(rel))
|
||||
|
||||
|
||||
def load_preamble(path: Path) -> Preamble:
|
||||
try:
|
||||
doc = json.loads(path.read_text(encoding="utf-8"))
|
||||
except (OSError, json.JSONDecodeError) as exc:
|
||||
raise FetchError(f"the preamble file {path} cannot be read: {exc}") from None
|
||||
lines, digest = (doc.get("lines"), doc.get("sha256")) if isinstance(doc, dict) else (None, None)
|
||||
if not (isinstance(lines, int) and not isinstance(lines, bool) and lines >= 1
|
||||
and isinstance(digest, str) and HEX64.match(digest)):
|
||||
raise FetchError(f"the preamble file {path} is not {{\"lines\": N, \"sha256\": \"<hex>\"}}")
|
||||
return Preamble(lines, digest)
|
||||
|
||||
|
||||
def strip_preamble(text: str, spec: Preamble) -> tuple[str, int]:
|
||||
"""The page without its preamble and the blank line after it, and the number of bytes removed."""
|
||||
lines = text.split("\n")
|
||||
n = 0
|
||||
while n < len(lines) and lines[n].startswith("> "):
|
||||
n += 1
|
||||
pre = lines[:n]
|
||||
if n != spec.lines or not pre[0].startswith("> ## ") or n >= len(lines) or lines[n] != "":
|
||||
raise FetchError(f"the page does not start with {spec.lines} '> ' lines, a '> ## ' heading first, "
|
||||
f"and a blank line (found {n} '> ' lines)")
|
||||
if hashlib.sha256("\n".join(pre).encode("utf-8")).hexdigest() != spec.sha256:
|
||||
raise FetchError("the preamble has the known shape but not the known sha256")
|
||||
rest = "\n".join(lines[n + 1:])
|
||||
return rest, len(text.encode("utf-8")) - len(rest.encode("utf-8"))
|
||||
|
||||
|
||||
def release_count(text: str) -> int:
|
||||
"""The releases on the page, counted apart from the split: its "<Update label=", or its bold version-like lines."""
|
||||
if "<Update label=" in text:
|
||||
return text.count("<Update label=")
|
||||
return sum(1 for line in text.split("\n") if BOLD_LIKE.match(line))
|
||||
|
||||
|
||||
def split_bold(body: str, pre_bytes: int, total: int) -> ChangelogSplit:
|
||||
"""One block per "**X.Y.Z**" line with a blank line and a date after it, running to the next one."""
|
||||
lines = [line + "\n" for line in body.split("\n")]
|
||||
lines[-1] = lines[-1][:-1]
|
||||
bare = [line.rstrip("\n") for line in lines]
|
||||
starts: list[tuple[int, str]] = []
|
||||
for i, line in enumerate(bare):
|
||||
if not BOLD_LIKE.match(line):
|
||||
continue
|
||||
m = BOLD.match(line)
|
||||
if not m:
|
||||
raise FetchError(f"line {i + 1}: {line!r} is not **X.Y.Z**")
|
||||
if not (i + 2 < len(bare) and bare[i + 1] == "" and DATE.match(bare[i + 2])):
|
||||
raise FetchError(f"line {i + 1}: version {m.group(1)} is not followed by a blank line and a date")
|
||||
if any(v == m.group(1) for _, v in starts):
|
||||
raise FetchError(f"line {i + 1}: version {m.group(1)} appears twice")
|
||||
starts.append((i, m.group(1)))
|
||||
if not starts:
|
||||
raise FetchError("the page has no <Update label=…> block and no **X.Y.Z** release line")
|
||||
ends = [i for i, _ in starts[1:]] + [len(lines)]
|
||||
blocks = [Block(v, "".join(lines[i:end])) for (i, v), end in zip(starts, ends)]
|
||||
rest_bytes = sum(len(line.encode("utf-8")) for line in lines[:starts[0][0]])
|
||||
block_bytes = sum(len(b.text.encode("utf-8")) for b in blocks)
|
||||
if pre_bytes + block_bytes + rest_bytes != total:
|
||||
raise FetchError("preamble + blocks + rest is not the length of the page")
|
||||
return ChangelogSplit(blocks, pre_bytes, block_bytes, rest_bytes)
|
||||
|
||||
|
||||
def split_changelog(text: str, spec: Preamble) -> ChangelogSplit:
|
||||
"""One block per release, the text untouched, in the component or the bold form; every byte is counted."""
|
||||
body, pre_bytes = strip_preamble(text, spec)
|
||||
if "<Update label=" not in body:
|
||||
return split_bold(body, pre_bytes, len(text.encode("utf-8")))
|
||||
if any(BOLD_LIKE.match(line) for line in body.split("\n")):
|
||||
raise FetchError("the page mixes <Update label=…> blocks and **X.Y.Z** release lines")
|
||||
lines = [line + "\n" for line in body.split("\n")]
|
||||
lines[-1] = lines[-1][:-1]
|
||||
blocks: list[Block] = []
|
||||
rest_bytes = 0
|
||||
current: list[str] | None = None
|
||||
version = ""
|
||||
for n, line in enumerate(lines, 1):
|
||||
bare = line.rstrip("\n")
|
||||
opened = OPEN.match(bare)
|
||||
if current is None:
|
||||
if opened:
|
||||
version = opened.group(1)
|
||||
if not VERSION.match(version):
|
||||
raise FetchError(f"line {n}: the label {version!r} is not X.Y.Z")
|
||||
if any(b.version == version for b in blocks):
|
||||
raise FetchError(f"line {n}: version {version} appears twice")
|
||||
current = [line]
|
||||
elif bare == CLOSE:
|
||||
raise FetchError(f"line {n}: {CLOSE} without an open block")
|
||||
else:
|
||||
rest_bytes += len(line.encode("utf-8"))
|
||||
else:
|
||||
if opened:
|
||||
raise FetchError(f"line {n}: a block opens inside version {version}")
|
||||
current.append(line)
|
||||
if bare == CLOSE:
|
||||
blocks.append(Block(version, "".join(current)))
|
||||
current = None
|
||||
if current is not None:
|
||||
raise FetchError(f"version {version} is never closed")
|
||||
if not blocks:
|
||||
raise FetchError("the page has no <Update label=…> block")
|
||||
if body.count("<Update label=") != len(blocks):
|
||||
raise FetchError(f"the page has {body.count('<Update label=')} '<Update label=', but {len(blocks)} blocks "
|
||||
"start a line")
|
||||
block_bytes = sum(len(b.text.encode("utf-8")) for b in blocks)
|
||||
if pre_bytes + block_bytes + rest_bytes != len(text.encode("utf-8")):
|
||||
raise FetchError("preamble + blocks + rest is not the length of the page")
|
||||
return ChangelogSplit(blocks, pre_bytes, block_bytes, rest_bytes)
|
||||
|
||||
|
||||
def anchor_url(page: str, version: str) -> str:
|
||||
return f"{page}#{version.replace('.', '-')}"
|
||||
|
||||
|
||||
def missing_anchors(html: str, versions: list[str]) -> list[str]:
|
||||
return [v for v in versions if f'id="{v.replace(".", "-")}"' not in html]
|
||||
|
||||
|
||||
def collect(fetcher: Fetcher, preamble: Preamble, *, base: str = BASE, index_url: str = INDEX_URL,
|
||||
changelog: str = CHANGELOG, changelog_page: str = CHANGELOG_PAGE, pause: float = 0.5,
|
||||
collection: str = "claude-code") -> Collected:
|
||||
"""Fetch and split everything in memory; nothing is written here."""
|
||||
got = Collected()
|
||||
problems = got.problems
|
||||
|
||||
def fetch(url: str) -> Fetched:
|
||||
f = fetcher(url)
|
||||
if pause:
|
||||
time.sleep(pause)
|
||||
return f
|
||||
|
||||
raw_index = fetch(index_url)
|
||||
if raw_index.status != 200:
|
||||
raise RunError(f"the index {index_url} gave status {raw_index.status}")
|
||||
index_text = raw_index.body.decode("utf-8").replace("\r\n", "\n")
|
||||
idx = parse_index(index_text, base)
|
||||
urls = [p.url for p in idx.pages]
|
||||
if changelog not in urls:
|
||||
problems.append(f"the changelog {changelog} is not in the index")
|
||||
sections = {p.url: p.sections for p in idx.pages}
|
||||
names: dict[str, str] = {}
|
||||
by_name: dict[str, str] = {}
|
||||
for url in urls:
|
||||
try:
|
||||
name = source_name(url, base)
|
||||
except FetchError as exc:
|
||||
problems.append(str(exc))
|
||||
continue
|
||||
if name in by_name:
|
||||
problems.append(f"{name}: same path for {by_name[name]} and {url}")
|
||||
continue
|
||||
by_name[name] = url
|
||||
names[url] = name
|
||||
sources: dict[str, dict[str, Any]] = {}
|
||||
moved_out: list[dict[str, str]] = []
|
||||
crlf: list[str] = []
|
||||
preamble_pages = preamble_bytes = raw_updates = 0
|
||||
changelog_bytes: dict[str, int] = {}
|
||||
docs_sections: dict[str, list[str]] = {}
|
||||
versions: list[str] = []
|
||||
history_pages: list[str] = []
|
||||
for url in urls:
|
||||
if url not in names:
|
||||
continue
|
||||
name = names[url]
|
||||
f = fetch(url)
|
||||
if f.status != 200:
|
||||
problems.append(f"{url}: status {f.status}")
|
||||
continue
|
||||
if not f.url.startswith(base):
|
||||
moved_out.append({"content_type": f.content_type, "final_url": f.url, "url": url})
|
||||
continue
|
||||
if not f.content_type.startswith("text/markdown"):
|
||||
problems.append(f"{url} ({name}): {f.content_type or 'no content type'}, not markdown")
|
||||
continue
|
||||
try:
|
||||
text = f.body.decode("utf-8")
|
||||
except UnicodeDecodeError:
|
||||
problems.append(f"{url} ({name}): not UTF-8")
|
||||
continue
|
||||
if "\r\n" in text:
|
||||
crlf.append(name)
|
||||
text = text.replace("\r\n", "\n")
|
||||
try:
|
||||
if url == changelog:
|
||||
raw_updates = release_count(text)
|
||||
split = split_changelog(text, preamble)
|
||||
changelog_bytes = {"page": len(text.encode("utf-8")), "preamble": split.preamble_bytes,
|
||||
"blocks": split.block_bytes, "rest": split.rest_bytes}
|
||||
preamble_pages += 1
|
||||
preamble_bytes += split.preamble_bytes
|
||||
for b in split.blocks:
|
||||
rn = f"release-v{b.version}.md"
|
||||
got.files[rn] = b.text.encode("utf-8")
|
||||
sources[rn] = {"kind": "release-notes", "main_topic": False,
|
||||
"url": anchor_url(changelog_page, b.version), "version": b.version}
|
||||
versions.append(b.version)
|
||||
else:
|
||||
body, removed = strip_preamble(text, preamble)
|
||||
preamble_pages += 1
|
||||
preamble_bytes += removed
|
||||
got.files[name] = body.encode("utf-8")
|
||||
history = is_history(url[len(base):])
|
||||
sources[name] = {"kind": "docs", "main_topic": not history, "url": url}
|
||||
if history:
|
||||
history_pages.append(name)
|
||||
docs_sections[name] = list(sections[url])
|
||||
except FetchError as exc:
|
||||
problems.append(f"{url} ({name}): {exc}")
|
||||
if versions:
|
||||
page = fetch(changelog_page)
|
||||
if page.status != 200:
|
||||
problems.append(f"{changelog_page}: status {page.status}; the version anchors cannot be checked")
|
||||
else:
|
||||
for v in missing_anchors(page.body.decode("utf-8", errors="replace"), versions):
|
||||
problems.append(f"{v}: no anchor id=\"{v.replace('.', '-')}\" on {changelog_page}")
|
||||
docs = sorted(n for n, s in sources.items() if s["kind"] == "docs")
|
||||
notes = sorted(n for n, s in sources.items() if s["kind"] == "release-notes")
|
||||
has_changelog = int(changelog in urls)
|
||||
complete = {"index_pages": len(urls), "changelog": has_changelog, "moved_out": len(moved_out),
|
||||
"docs_expected": len(urls) - has_changelog - len(moved_out), "docs": len(docs),
|
||||
"release_notes_expected": raw_updates, "release_notes": len(notes)}
|
||||
complete["pass"] = (complete["docs"] == complete["docs_expected"]
|
||||
and complete["release_notes"] == complete["release_notes_expected"] and raw_updates > 0)
|
||||
if not complete["pass"]:
|
||||
problems.append(f"completeness FAIL: docs {complete['docs']} of {complete['docs_expected']} expected, "
|
||||
f"release notes {complete['release_notes']} of {complete['release_notes_expected']}")
|
||||
got.meta = {"collection": collection, "schema": META_SCHEMA,
|
||||
"index": {"pages": len(urls), "sha256": hashlib.sha256(raw_index.body).hexdigest(), "url": index_url},
|
||||
"sources": dict(sorted(sources.items()))}
|
||||
got.report = {
|
||||
"index": {"url": index_url, "sha256": got.meta["index"]["sha256"], "bytes": len(raw_index.body),
|
||||
"items": idx.items, "pages": len(urls)},
|
||||
"other_links": idx.other_links, "moved_out": moved_out, "completeness": complete, "crlf_pages": crlf,
|
||||
"history_pages": sorted(history_pages),
|
||||
"preamble": {"pages": preamble_pages, "bytes": preamble_bytes}, "changelog_bytes": changelog_bytes,
|
||||
"docs": {"count": len(docs), "bytes": sum(len(got.files[n]) for n in docs)},
|
||||
"release_notes": {"count": len(notes), "bytes": sum(len(got.files[n]) for n in notes)},
|
||||
"sections": docs_sections, "problems": problems}
|
||||
return got
|
||||
|
||||
|
||||
def net_fetcher(cafile: str | None, timeout: float = 60.0) -> Fetcher:
|
||||
"""urllib with certificate checks on. The python.org build here has no CA store, so the system's file is used."""
|
||||
if cafile is None and not os.environ.get("SSL_CERT_FILE") and os.path.isfile(SYSTEM_CA):
|
||||
cafile = SYSTEM_CA
|
||||
context = ssl.create_default_context(cafile=cafile)
|
||||
|
||||
def fetch(url: str) -> Fetched:
|
||||
req = urllib.request.Request(url, headers={"User-Agent": USER_AGENT})
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=timeout, context=context) as r:
|
||||
return Fetched(r.status, r.geturl(), r.headers.get("Content-Type", ""), r.read())
|
||||
except urllib.error.HTTPError as exc:
|
||||
return Fetched(exc.code, exc.geturl() or url, exc.headers.get("Content-Type", "") if exc.headers else "",
|
||||
b"")
|
||||
|
||||
return fetch
|
||||
|
||||
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
p.add_argument("--staging", required=True)
|
||||
p.add_argument("--meta", required=True)
|
||||
p.add_argument("--report", required=True)
|
||||
p.add_argument("--preamble", required=True)
|
||||
p.add_argument("--base", default=BASE)
|
||||
p.add_argument("--index", default=INDEX_URL)
|
||||
p.add_argument("--changelog", default=CHANGELOG)
|
||||
p.add_argument("--changelog-page", default=CHANGELOG_PAGE)
|
||||
p.add_argument("--pause", type=float, default=0.5)
|
||||
p.add_argument("--cafile")
|
||||
p.add_argument("--collection", default="claude-code")
|
||||
return p
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None, fetcher: Fetcher | None = None) -> int:
|
||||
args = build_parser().parse_args(argv)
|
||||
staging, meta, report = Path(args.staging), Path(args.meta), Path(args.report)
|
||||
try:
|
||||
for p in (staging, meta, report):
|
||||
if p.exists():
|
||||
raise RunError(f"{p} exists; refusing to overwrite")
|
||||
got = collect(fetcher or net_fetcher(args.cafile), load_preamble(Path(args.preamble)), base=args.base,
|
||||
index_url=args.index, changelog=args.changelog, changelog_page=args.changelog_page,
|
||||
pause=args.pause, collection=args.collection)
|
||||
write_new(report, dump(got.report))
|
||||
c = got.report["completeness"]
|
||||
summary = (f"fetch: {c['docs']} docs of {c['docs_expected']} expected, {c['release_notes']} release notes of "
|
||||
f"{c['release_notes_expected']}, {c['moved_out']} moved out, {len(got.report['other_links'])} "
|
||||
f"other links not fetched")
|
||||
if got.problems:
|
||||
print(f"{summary}; FAIL with {len(got.problems)} problems, nothing staged (report: {report})",
|
||||
file=sys.stderr)
|
||||
for problem in got.problems:
|
||||
print(f" {problem}", file=sys.stderr)
|
||||
return 1
|
||||
staging.mkdir(parents=True)
|
||||
for name, data in got.files.items():
|
||||
(staging / name).write_bytes(data)
|
||||
write_new(meta, dump(got.meta))
|
||||
print(f"{summary}; completeness pass -> {staging}")
|
||||
for m in got.report["moved_out"]:
|
||||
print(f" moved out: {m['url']} -> {m['final_url']}")
|
||||
return 0
|
||||
except (RunError, FetchError, OSError, UnicodeDecodeError) as exc:
|
||||
print(f"fetch cannot run: {exc}", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Loading…
Add table
Add a link
Reference in a new issue