okf-wiki/tests/test_fetch_docs.py
Kjell Tore Guttormsen 0fd86a23a3
okf-wiki 1.1.1
First public release, published as one snapshot. What changed between
versions is in CHANGELOG.md.

Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
2026-11-14 21:03:20 +01:00

394 lines
22 KiB
Python

"""Tests for tools/fetch_docs.py: the index, the preamble rule, the release-note split and the completeness check.
The network is replaced by a dictionary of synthetic pages on example.org (a garden site); no text from any
documentation. Run: python3 -m unittest discover -s tests -p 'test_fetch_docs.py'
"""
from __future__ import annotations
import hashlib
import importlib.util
import io
import json
import sys
import tempfile
import unittest
from contextlib import redirect_stderr, redirect_stdout
from pathlib import Path
from typing import Any
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT))
from okfwiki import sources # noqa: E402
# Loaded from its file, not imported by name, so `mypy --strict okfwiki gates tests` (tests/v1_repo.sh, frozen in
# wave 1) stays clean; the tool itself is type-checked when it is named: `mypy --strict ... tools/fetch_docs.py`.
_SPEC = importlib.util.spec_from_file_location("fetch_docs", ROOT / "tools" / "fetch_docs.py")
assert _SPEC is not None and _SPEC.loader is not None
fetch_docs: Any = importlib.util.module_from_spec(_SPEC)
sys.modules["fetch_docs"] = fetch_docs
_SPEC.loader.exec_module(fetch_docs)
SITE = "https://example.org/docs/"
BASE = SITE + "en/"
INDEX_URL = SITE + "llms.txt"
CHANGELOG = BASE + "changelog.md"
CHANGELOG_PAGE = BASE + "changelog"
PREAMBLE = ("> ## Garden Index\n> Fetch the garden index at: https://example.org/docs/llms.txt\n"
"> Use this file to find pages.\n\n")
PREAMBLE_SHA = hashlib.sha256(PREAMBLE.rstrip("\n").encode()).hexdigest()
PREAMBLE_SPEC = {"lines": 3, "sha256": PREAMBLE_SHA}
INDEX = """# Garden Docs
> Docs for the garden.
## Getting started
### Basics
- [Soil](https://example.org/docs/en/soil.md): About soil.
- [Changelog](https://example.org/docs/en/changelog.md): Releases by version.
### Tools
- [Spade](https://example.org/docs/en/tools/spade.md): Digging.
- [Soil, listed again](https://example.org/docs/en/soil.md): The same page.
## Reference
#### Deep
- [Moved away](https://example.org/docs/en/moved.md): Lives elsewhere now.
## Indexes
- [French](https://example.org/docs/_llms/fr.md): The garden in French.
"""
SOIL = "# Soil\n\nCompost feeds the soil.\n"
SPADE = "# Spade\n\nDig before the frost.\n"
BLOCK_A = '<Update label="1.1.0" description="June 2, 2026">\n * Added `rake`\n</Update>\n'
BLOCK_B = '<Update label="1.0.0" description="May 1, 2026">\n * First tools\n * Seeds\n</Update>\n'
CHANGELOG_BODY = "# Changelog\n\nGenerated from the shed log.\n\n" + BLOCK_A + "\n" + BLOCK_B
BOLD_A = "**1.1.0**\n\nJune 2, 2026\n\n**Added**\n\n* Added `rake`\n\n"
BOLD_B = "**1.0.0**\n\nMay 1, 2026\n\n* First tools\n* Seeds\n"
BOLD_BODY = "# Changelog\n\nGenerated from the shed log.\n\n" + BOLD_A + BOLD_B
CHANGELOG_HTML = '<html><h2 id="1-1-0">1.1.0</h2><h2 id="1-0-0">1.0.0</h2></html>'
MD = "text/markdown; charset=utf-8"
def page(body: str) -> bytes:
return (PREAMBLE + body).encode("utf-8")
def site(**changes: Any) -> dict[str, Any]:
pages: dict[str, Any] = {
INDEX_URL: fetch_docs.Fetched(200, INDEX_URL, "text/plain; charset=utf-8", INDEX.encode()),
BASE + "soil.md": fetch_docs.Fetched(200, BASE + "soil.md", MD, page(SOIL)),
BASE + "tools/spade.md": fetch_docs.Fetched(200, BASE + "tools/spade.md", MD, page(SPADE)),
CHANGELOG: fetch_docs.Fetched(200, CHANGELOG, MD, page(CHANGELOG_BODY)),
BASE + "moved.md": fetch_docs.Fetched(200, "https://other.example.net/garden/moved", "text/html",
b"<html>elsewhere</html>"),
CHANGELOG_PAGE: fetch_docs.Fetched(200, CHANGELOG_PAGE, "text/html", CHANGELOG_HTML.encode()),
}
pages.update(changes)
return {k: v for k, v in pages.items() if v is not None}
class FakeNet:
def __init__(self, pages: dict[str, Any]) -> None:
self.pages = pages
self.asked: list[str] = []
def __call__(self, url: str) -> Any:
self.asked.append(url)
return self.pages.get(url, fetch_docs.Fetched(404, url, "text/html", b"not found"))
def preamble() -> Any:
return fetch_docs.Preamble(lines=3, sha256=PREAMBLE_SHA)
def collect(pages: dict[str, Any]) -> Any:
return fetch_docs.collect(FakeNet(pages), preamble(), base=BASE, index_url=INDEX_URL, changelog=CHANGELOG,
changelog_page=CHANGELOG_PAGE, pause=0.0)
class IndexTest(unittest.TestCase):
def test_the_index_gives_the_unique_pages_under_the_base_with_their_sections(self) -> None:
idx = fetch_docs.parse_index(INDEX, BASE)
self.assertEqual(idx.items, 6)
self.assertEqual([p.url for p in idx.pages],
[BASE + "soil.md", CHANGELOG, BASE + "tools/spade.md", BASE + "moved.md"])
self.assertEqual(idx.pages[0].sections, ("Getting started", "Basics"))
self.assertEqual(idx.pages[2].sections, ("Getting started", "Tools"))
self.assertEqual(idx.pages[3].sections, ("Reference", "Deep"))
self.assertEqual(idx.pages[0].title, "Soil")
self.assertEqual(idx.other_links, [SITE + "_llms/fr.md"])
def test_the_source_name_joins_the_path_with_dashes(self) -> None:
self.assertEqual(fetch_docs.source_name(BASE + "soil.md", BASE), "docs-en-soil.md")
self.assertEqual(fetch_docs.source_name(BASE + "tools/hand/spade.md", BASE), "docs-en-tools-hand-spade.md")
for bad in (BASE + "Soil.md", BASE + "soil", BASE + "soil.md#top", SITE + "fr/soil.md", BASE + "a b.md"):
with self.subTest(bad=bad):
with self.assertRaises(fetch_docs.FetchError):
fetch_docs.source_name(bad, BASE)
def test_two_addresses_with_the_same_path_are_refused(self) -> None:
index = INDEX.replace("- [Spade](https://example.org/docs/en/tools/spade.md)",
"- [Spade](https://example.org/docs/en/tools/spade.md)\n"
"- [Spade 2](https://example.org/docs/en/tools-spade.md)")
pages = site(**{INDEX_URL: fetch_docs.Fetched(200, INDEX_URL, "text/plain", index.encode()),
BASE + "tools-spade.md": fetch_docs.Fetched(200, BASE + "tools-spade.md", MD, page(SPADE))})
got = collect(pages)
self.assertTrue(any("docs-en-tools-spade.md" in p and "same path" in p for p in got.problems), got.problems)
class PreambleTest(unittest.TestCase):
def test_the_known_preamble_and_its_blank_line_are_removed(self) -> None:
text, removed = fetch_docs.strip_preamble(PREAMBLE + SOIL, preamble())
self.assertEqual(text, SOIL)
self.assertEqual(removed, len(PREAMBLE.encode()))
def test_another_preamble_is_an_error_never_a_guess(self) -> None:
for bad in (PREAMBLE.replace("garden index", "orchard index") + SOIL, # same shape, other text
"> ## Garden Index\n> Use this file to find pages.\n\n" + SOIL, # two lines
"> # Garden Index\n> a\n> b\n\n" + SOIL, # not a level-2 heading
PREAMBLE.rstrip("\n") + "\n" + SOIL, # no blank line after
SOIL): # none at all
with self.subTest(bad=bad[:30]):
with self.assertRaises(fetch_docs.FetchError):
fetch_docs.strip_preamble(bad, preamble())
def test_the_spec_is_read_from_a_file(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
p = Path(tmp) / "preamble.json"
p.write_text(json.dumps(PREAMBLE_SPEC), encoding="utf-8")
self.assertEqual(fetch_docs.load_preamble(p), preamble())
p.write_text(json.dumps({"lines": 3}), encoding="utf-8")
with self.assertRaises(fetch_docs.FetchError):
fetch_docs.load_preamble(p)
class ChangelogTest(unittest.TestCase):
def test_one_file_per_block_with_every_byte_counted(self) -> None:
text = PREAMBLE + CHANGELOG_BODY
split = fetch_docs.split_changelog(text, preamble())
self.assertEqual([(b.version, b.text) for b in split.blocks], [("1.1.0", BLOCK_A), ("1.0.0", BLOCK_B)])
self.assertEqual(split.preamble_bytes, len(PREAMBLE.encode()))
self.assertEqual(split.block_bytes, len((BLOCK_A + BLOCK_B).encode()))
self.assertEqual(split.rest_bytes, len("# Changelog\n\nGenerated from the shed log.\n\n\n".encode()))
self.assertEqual(split.preamble_bytes + split.block_bytes + split.rest_bytes, len(text.encode()))
def test_a_last_block_without_a_final_newline_is_kept_as_it_is(self) -> None:
text = PREAMBLE + CHANGELOG_BODY.rstrip("\n")
split = fetch_docs.split_changelog(text, preamble())
self.assertEqual(split.blocks[-1].text, BLOCK_B.rstrip("\n"))
self.assertEqual(split.preamble_bytes + split.block_bytes + split.rest_bytes, len(text.encode()))
def test_broken_blocks_are_refused(self) -> None:
cases = {
"duplicate": CHANGELOG_BODY.replace('label="1.0.0"', 'label="1.1.0"'),
"unclosed": CHANGELOG_BODY.replace(BLOCK_B, BLOCK_B.replace("</Update>\n", "")),
"nested": CHANGELOG_BODY.replace(" * Added `rake`\n", " * Added `rake`\n" + BLOCK_B),
"stray close": CHANGELOG_BODY + "</Update>\n",
"not a version": CHANGELOG_BODY.replace('label="1.0.0"', 'label="first"'),
"mid-line": CHANGELOG_BODY.replace("Generated from", 'Generated <Update label="0.9.0"> from'),
"no blocks": "# Changelog\n\nNothing yet.\n",
}
for name, body in cases.items():
with self.subTest(name=name):
with self.assertRaises(fetch_docs.FetchError):
fetch_docs.split_changelog(PREAMBLE + body, preamble())
def test_the_bold_form_gives_one_file_per_release_with_every_byte_counted(self) -> None:
# the page's shape after 20.10: "**X.Y.Z**", a blank line, the date; a release runs to the next one
text = PREAMBLE + BOLD_BODY
split = fetch_docs.split_changelog(text, preamble())
self.assertEqual([(b.version, b.text) for b in split.blocks], [("1.1.0", BOLD_A), ("1.0.0", BOLD_B)])
self.assertEqual(split.rest_bytes, len("# Changelog\n\nGenerated from the shed log.\n\n".encode()))
self.assertEqual(split.preamble_bytes + split.block_bytes + split.rest_bytes, len(text.encode()))
self.assertEqual(fetch_docs.release_count(text), 2)
def test_broken_bold_releases_are_refused(self) -> None:
cases = {
"duplicate": BOLD_BODY.replace("**1.0.0**", "**1.1.0**"),
"no date": BOLD_BODY.replace("\nMay 1, 2026\n", "\nSoon\n"),
"no blank line": BOLD_BODY.replace("**1.0.0**\n\n", "**1.0.0**\n"),
"not X.Y.Z": BOLD_BODY.replace("**1.0.0**", "**1.0**"),
"both forms": BOLD_BODY + BLOCK_B,
}
for name, body in cases.items():
with self.subTest(name=name):
with self.assertRaises(fetch_docs.FetchError):
fetch_docs.split_changelog(PREAMBLE + body, preamble())
def test_the_anchor_url_is_checked_against_the_page(self) -> None:
self.assertEqual(fetch_docs.anchor_url(CHANGELOG_PAGE, "1.1.0"), CHANGELOG_PAGE + "#1-1-0")
self.assertEqual(fetch_docs.missing_anchors(CHANGELOG_HTML, ["1.1.0", "1.0.0"]), [])
self.assertEqual(fetch_docs.missing_anchors(CHANGELOG_HTML, ["1.1.0", "0.9.0"]), ["0.9.0"])
class CollectTest(unittest.TestCase):
def test_a_whole_run_gives_the_sources_and_the_meta_freeze_reads(self) -> None:
got = collect(site())
self.assertEqual(got.problems, [])
self.assertEqual(got.files, {"docs-en-soil.md": SOIL.encode(), "docs-en-tools-spade.md": SPADE.encode(),
"release-v1.0.0.md": BLOCK_B.encode(), "release-v1.1.0.md": BLOCK_A.encode()})
self.assertEqual(got.meta, {
"collection": "claude-code", "schema": "okf-wiki/source-meta/1",
"index": {"pages": 4, "sha256": hashlib.sha256(INDEX.encode()).hexdigest(), "url": INDEX_URL},
"sources": {
"docs-en-soil.md": {"kind": "docs", "main_topic": True, "url": BASE + "soil.md"},
"docs-en-tools-spade.md": {"kind": "docs", "main_topic": True, "url": BASE + "tools/spade.md"},
"release-v1.0.0.md": {"kind": "release-notes", "main_topic": False,
"url": CHANGELOG_PAGE + "#1-0-0", "version": "1.0.0"},
"release-v1.1.0.md": {"kind": "release-notes", "main_topic": False,
"url": CHANGELOG_PAGE + "#1-1-0", "version": "1.1.0"}}})
r = got.report
self.assertEqual(r["moved_out"], [{"url": BASE + "moved.md", "final_url": "https://other.example.net/garden/moved",
"content_type": "text/html"}])
self.assertEqual(r["completeness"], {"index_pages": 4, "changelog": 1, "moved_out": 1, "docs_expected": 2,
"docs": 2, "release_notes_expected": 2, "release_notes": 2, "pass": True})
self.assertEqual(r["other_links"], [SITE + "_llms/fr.md"])
self.assertEqual(r["sections"]["docs-en-tools-spade.md"], ["Getting started", "Tools"])
def test_a_missing_page_fails_the_completeness_check(self) -> None:
got = collect(site(**{BASE + "tools/spade.md": None}))
self.assertFalse(got.report["completeness"]["pass"])
self.assertTrue(any("tools/spade.md" in p and "404" in p for p in got.problems), got.problems)
self.assertTrue(any("completeness" in p for p in got.problems), got.problems)
def test_a_missing_version_fails_the_completeness_check(self) -> None:
body = CHANGELOG_BODY.replace("Generated from the shed log.", 'Generated <Update label="0.9.0"> log.')
got = collect(site(**{CHANGELOG: fetch_docs.Fetched(200, CHANGELOG, MD, page(body))}))
self.assertTrue(any("0.9.0" in p or "Update" in p for p in got.problems), got.problems)
self.assertFalse(got.report["completeness"]["pass"])
self.assertFalse(any(name.startswith("release-") for name in got.files))
def test_a_bold_form_changelog_passes_the_completeness_check(self) -> None:
got = collect(site(**{CHANGELOG: fetch_docs.Fetched(200, CHANGELOG, MD, page(BOLD_BODY))}))
self.assertEqual(got.problems, [])
self.assertEqual(got.files["release-v1.1.0.md"], BOLD_A.encode())
self.assertEqual(got.report["completeness"]["release_notes_expected"], 2)
self.assertTrue(got.report["completeness"]["pass"])
def test_a_bold_version_split_would_miss_fails_the_completeness_check(self) -> None:
body = BOLD_BODY.replace("Generated from the shed log.", "**0.9.0-beta**")
got = collect(site(**{CHANGELOG: fetch_docs.Fetched(200, CHANGELOG, MD, page(body))}))
self.assertFalse(got.report["completeness"]["pass"])
self.assertFalse(any(name.startswith("release-") for name in got.files))
def test_html_on_the_docs_site_is_refused_not_moved_out(self) -> None:
got = collect(site(**{BASE + "soil.md": fetch_docs.Fetched(200, BASE + "soil.md", "text/html; charset=utf-8",
b"<!doctype html><p>Soil</p>")}))
self.assertTrue(any("docs-en-soil.md" in p or "soil.md" in p for p in got.problems), got.problems)
self.assertFalse(got.report["completeness"]["pass"])
def test_a_redirect_inside_the_docs_site_is_still_a_docs_page(self) -> None:
got = collect(site(**{BASE + "soil.md": fetch_docs.Fetched(200, BASE + "garden/soil.md", MD, page(SOIL))}))
self.assertEqual(got.problems, [])
self.assertEqual(got.meta["sources"]["docs-en-soil.md"]["url"], BASE + "soil.md")
def test_crlf_becomes_lf_and_is_counted(self) -> None:
crlf = page(SOIL).replace(b"\n", b"\r\n")
got = collect(site(**{BASE + "soil.md": fetch_docs.Fetched(200, BASE + "soil.md", MD, crlf)}))
self.assertEqual(got.problems, [])
self.assertEqual(got.files["docs-en-soil.md"], SOIL.encode())
self.assertEqual(got.report["crlf_pages"], ["docs-en-soil.md"])
def test_a_missing_anchor_is_refused(self) -> None:
html = CHANGELOG_HTML.replace('id="1-0-0"', 'id="v1"')
got = collect(site(**{CHANGELOG_PAGE: fetch_docs.Fetched(200, CHANGELOG_PAGE, "text/html", html.encode())}))
self.assertTrue(any("1.0.0" in p and "anchor" in p for p in got.problems), got.problems)
def test_a_changelog_missing_from_the_index_is_refused(self) -> None:
index = INDEX.replace("- [Changelog](https://example.org/docs/en/changelog.md): Releases by version.\n", "")
got = collect(site(**{INDEX_URL: fetch_docs.Fetched(200, INDEX_URL, "text/plain", index.encode())}))
self.assertTrue(any("changelog" in p for p in got.problems), got.problems)
def test_only_the_weekly_digest_pages_are_history(self) -> None:
for rel, history in (("whats-new/2026-w37.md", True), ("whats-new/2025-w01.md", True),
("whats-new/index.md", False), ("whats-new-2026-w37.md", False),
("guide/whats-new/2026-w37.md", False), ("whats-new/2026-w7.md", False)):
with self.subTest(rel=rel):
self.assertEqual(fetch_docs.is_history(rel), history)
def test_weekly_digest_pages_are_docs_without_a_main_topic(self) -> None:
"""The operator 08.10.2026: the weekly pages add history to the concepts and get no main page."""
week, overview = BASE + "whats-new/2026-w01.md", BASE + "whats-new/index.md"
index = INDEX.replace("## Indexes", f"## News\n\n- [News]({overview}): Every week.\n"
f"- [Week 1]({week}): Rakes arrived.\n\n## Indexes")
got = collect(site(**{INDEX_URL: fetch_docs.Fetched(200, INDEX_URL, "text/plain", index.encode()),
overview: fetch_docs.Fetched(200, overview, MD, page("# News\n\nEvery week.\n")),
week: fetch_docs.Fetched(200, week, MD, page("# Week 1\n\nRakes arrived.\n"))}))
self.assertEqual(got.problems, [])
self.assertEqual(got.meta["sources"]["docs-en-whats-new-2026-w01.md"],
{"kind": "docs", "main_topic": False, "url": week})
self.assertEqual(got.meta["sources"]["docs-en-whats-new-index.md"],
{"kind": "docs", "main_topic": True, "url": overview})
self.assertEqual(got.report["history_pages"], ["docs-en-whats-new-2026-w01.md"])
self.assertEqual(got.report["completeness"]["docs"], 4)
self.assertTrue(got.report["completeness"]["pass"])
def test_the_counts_and_byte_shares_are_reported(self) -> None:
r = collect(site()).report
self.assertEqual(r["docs"], {"count": 2, "bytes": len((SOIL + SPADE).encode())})
self.assertEqual(r["release_notes"], {"count": 2, "bytes": len((BLOCK_A + BLOCK_B).encode())})
self.assertEqual(r["changelog_bytes"]["page"], len(page(CHANGELOG_BODY)))
self.assertEqual(r["preamble"], {"pages": 3, "bytes": 3 * len(PREAMBLE.encode())})
class RunTest(unittest.TestCase):
def setUp(self) -> None:
self._tmp = tempfile.TemporaryDirectory()
self.tmp = Path(self._tmp.name)
self.spec = self.tmp / "preamble.json"
self.spec.write_text(json.dumps(PREAMBLE_SPEC), encoding="utf-8")
def tearDown(self) -> None:
self._tmp.cleanup()
def run_tool(self, pages: dict[str, Any], staging: Path) -> tuple[int, str]:
argv = ["--staging", str(staging), "--meta", str(self.tmp / "meta.json"), "--report",
str(self.tmp / "report.json"), "--preamble", str(self.spec), "--base", BASE, "--index", INDEX_URL,
"--changelog", CHANGELOG, "--changelog-page", CHANGELOG_PAGE, "--pause", "0"]
out, err = io.StringIO(), io.StringIO()
with redirect_stdout(out), redirect_stderr(err):
code = fetch_docs.main(argv, fetcher=FakeNet(pages))
return code, out.getvalue() + err.getvalue()
def test_a_good_run_writes_a_staging_folder_that_freezes_and_checks_clean(self) -> None:
staging = self.tmp / "staging"
code, text = self.run_tool(site(), staging)
self.assertEqual(code, 0, text)
root = self.tmp / "frozen"
with redirect_stdout(io.StringIO()):
frozen = sources.main(["freeze", str(staging), "--meta", str(self.tmp / "meta.json"),
"--out", str(root), "--frozen-at", "2026-10-08T12:00:00Z"])
self.assertEqual(frozen, 0)
self.assertEqual(sources.check(root), [])
self.assertTrue(json.loads((self.tmp / "report.json").read_text(encoding="utf-8"))["completeness"]["pass"])
def test_a_failed_run_writes_the_report_but_no_staging(self) -> None:
staging = self.tmp / "staging"
code, text = self.run_tool(site(**{BASE + "tools/spade.md": None}), staging)
self.assertEqual(code, 1)
self.assertIn("completeness", text)
self.assertFalse(staging.exists())
self.assertFalse((self.tmp / "meta.json").exists())
self.assertFalse(json.loads((self.tmp / "report.json").read_text(encoding="utf-8"))["completeness"]["pass"])
def test_an_existing_staging_folder_cannot_run_and_nothing_is_fetched(self) -> None:
staging = self.tmp / "staging"
staging.mkdir()
net = FakeNet(site())
argv = ["--staging", str(staging), "--meta", str(self.tmp / "meta.json"), "--report",
str(self.tmp / "report.json"), "--preamble", str(self.spec), "--pause", "0"]
with redirect_stdout(io.StringIO()), redirect_stderr(io.StringIO()):
self.assertEqual(fetch_docs.main(argv, fetcher=net), 2)
self.assertEqual(net.asked, [])
if __name__ == "__main__":
unittest.main()