"""Tests for tools/fetch_docs.py: the index, the preamble rule, the release-note split and the completeness check. The network is replaced by a dictionary of synthetic pages on example.org (a garden site); no text from any documentation. Run: python3 -m unittest discover -s tests -p 'test_fetch_docs.py' """ from __future__ import annotations import hashlib import importlib.util import io import json import sys import tempfile import unittest from contextlib import redirect_stderr, redirect_stdout from pathlib import Path from typing import Any ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT)) from okfwiki import sources # noqa: E402 # Loaded from its file, not imported by name, so `mypy --strict okfwiki gates tests` (tests/v1_repo.sh, frozen in # wave 1) stays clean; the tool itself is type-checked when it is named: `mypy --strict ... tools/fetch_docs.py`. _SPEC = importlib.util.spec_from_file_location("fetch_docs", ROOT / "tools" / "fetch_docs.py") assert _SPEC is not None and _SPEC.loader is not None fetch_docs: Any = importlib.util.module_from_spec(_SPEC) sys.modules["fetch_docs"] = fetch_docs _SPEC.loader.exec_module(fetch_docs) SITE = "https://example.org/docs/" BASE = SITE + "en/" INDEX_URL = SITE + "llms.txt" CHANGELOG = BASE + "changelog.md" CHANGELOG_PAGE = BASE + "changelog" PREAMBLE = ("> ## Garden Index\n> Fetch the garden index at: https://example.org/docs/llms.txt\n" "> Use this file to find pages.\n\n") PREAMBLE_SHA = hashlib.sha256(PREAMBLE.rstrip("\n").encode()).hexdigest() PREAMBLE_SPEC = {"lines": 3, "sha256": PREAMBLE_SHA} INDEX = """# Garden Docs > Docs for the garden. ## Getting started ### Basics - [Soil](https://example.org/docs/en/soil.md): About soil. - [Changelog](https://example.org/docs/en/changelog.md): Releases by version. ### Tools - [Spade](https://example.org/docs/en/tools/spade.md): Digging. - [Soil, listed again](https://example.org/docs/en/soil.md): The same page. ## Reference #### Deep - [Moved away](https://example.org/docs/en/moved.md): Lives elsewhere now. ## Indexes - [French](https://example.org/docs/_llms/fr.md): The garden in French. """ SOIL = "# Soil\n\nCompost feeds the soil.\n" SPADE = "# Spade\n\nDig before the frost.\n" BLOCK_A = '\n * Added `rake`\n\n' BLOCK_B = '\n * First tools\n * Seeds\n\n' CHANGELOG_BODY = "# Changelog\n\nGenerated from the shed log.\n\n" + BLOCK_A + "\n" + BLOCK_B BOLD_A = "**1.1.0**\n\nJune 2, 2026\n\n**Added**\n\n* Added `rake`\n\n" BOLD_B = "**1.0.0**\n\nMay 1, 2026\n\n* First tools\n* Seeds\n" BOLD_BODY = "# Changelog\n\nGenerated from the shed log.\n\n" + BOLD_A + BOLD_B CHANGELOG_HTML = '

1.1.0

1.0.0

' MD = "text/markdown; charset=utf-8" def page(body: str) -> bytes: return (PREAMBLE + body).encode("utf-8") def site(**changes: Any) -> dict[str, Any]: pages: dict[str, Any] = { INDEX_URL: fetch_docs.Fetched(200, INDEX_URL, "text/plain; charset=utf-8", INDEX.encode()), BASE + "soil.md": fetch_docs.Fetched(200, BASE + "soil.md", MD, page(SOIL)), BASE + "tools/spade.md": fetch_docs.Fetched(200, BASE + "tools/spade.md", MD, page(SPADE)), CHANGELOG: fetch_docs.Fetched(200, CHANGELOG, MD, page(CHANGELOG_BODY)), BASE + "moved.md": fetch_docs.Fetched(200, "https://other.example.net/garden/moved", "text/html", b"elsewhere"), CHANGELOG_PAGE: fetch_docs.Fetched(200, CHANGELOG_PAGE, "text/html", CHANGELOG_HTML.encode()), } pages.update(changes) return {k: v for k, v in pages.items() if v is not None} class FakeNet: def __init__(self, pages: dict[str, Any]) -> None: self.pages = pages self.asked: list[str] = [] def __call__(self, url: str) -> Any: self.asked.append(url) return self.pages.get(url, fetch_docs.Fetched(404, url, "text/html", b"not found")) def preamble() -> Any: return fetch_docs.Preamble(lines=3, sha256=PREAMBLE_SHA) def collect(pages: dict[str, Any]) -> Any: return fetch_docs.collect(FakeNet(pages), preamble(), base=BASE, index_url=INDEX_URL, changelog=CHANGELOG, changelog_page=CHANGELOG_PAGE, pause=0.0) class IndexTest(unittest.TestCase): def test_the_index_gives_the_unique_pages_under_the_base_with_their_sections(self) -> None: idx = fetch_docs.parse_index(INDEX, BASE) self.assertEqual(idx.items, 6) self.assertEqual([p.url for p in idx.pages], [BASE + "soil.md", CHANGELOG, BASE + "tools/spade.md", BASE + "moved.md"]) self.assertEqual(idx.pages[0].sections, ("Getting started", "Basics")) self.assertEqual(idx.pages[2].sections, ("Getting started", "Tools")) self.assertEqual(idx.pages[3].sections, ("Reference", "Deep")) self.assertEqual(idx.pages[0].title, "Soil") self.assertEqual(idx.other_links, [SITE + "_llms/fr.md"]) def test_the_source_name_joins_the_path_with_dashes(self) -> None: self.assertEqual(fetch_docs.source_name(BASE + "soil.md", BASE), "docs-en-soil.md") self.assertEqual(fetch_docs.source_name(BASE + "tools/hand/spade.md", BASE), "docs-en-tools-hand-spade.md") for bad in (BASE + "Soil.md", BASE + "soil", BASE + "soil.md#top", SITE + "fr/soil.md", BASE + "a b.md"): with self.subTest(bad=bad): with self.assertRaises(fetch_docs.FetchError): fetch_docs.source_name(bad, BASE) def test_two_addresses_with_the_same_path_are_refused(self) -> None: index = INDEX.replace("- [Spade](https://example.org/docs/en/tools/spade.md)", "- [Spade](https://example.org/docs/en/tools/spade.md)\n" "- [Spade 2](https://example.org/docs/en/tools-spade.md)") pages = site(**{INDEX_URL: fetch_docs.Fetched(200, INDEX_URL, "text/plain", index.encode()), BASE + "tools-spade.md": fetch_docs.Fetched(200, BASE + "tools-spade.md", MD, page(SPADE))}) got = collect(pages) self.assertTrue(any("docs-en-tools-spade.md" in p and "same path" in p for p in got.problems), got.problems) class PreambleTest(unittest.TestCase): def test_the_known_preamble_and_its_blank_line_are_removed(self) -> None: text, removed = fetch_docs.strip_preamble(PREAMBLE + SOIL, preamble()) self.assertEqual(text, SOIL) self.assertEqual(removed, len(PREAMBLE.encode())) def test_another_preamble_is_an_error_never_a_guess(self) -> None: for bad in (PREAMBLE.replace("garden index", "orchard index") + SOIL, # same shape, other text "> ## Garden Index\n> Use this file to find pages.\n\n" + SOIL, # two lines "> # Garden Index\n> a\n> b\n\n" + SOIL, # not a level-2 heading PREAMBLE.rstrip("\n") + "\n" + SOIL, # no blank line after SOIL): # none at all with self.subTest(bad=bad[:30]): with self.assertRaises(fetch_docs.FetchError): fetch_docs.strip_preamble(bad, preamble()) def test_the_spec_is_read_from_a_file(self) -> None: with tempfile.TemporaryDirectory() as tmp: p = Path(tmp) / "preamble.json" p.write_text(json.dumps(PREAMBLE_SPEC), encoding="utf-8") self.assertEqual(fetch_docs.load_preamble(p), preamble()) p.write_text(json.dumps({"lines": 3}), encoding="utf-8") with self.assertRaises(fetch_docs.FetchError): fetch_docs.load_preamble(p) class ChangelogTest(unittest.TestCase): def test_one_file_per_block_with_every_byte_counted(self) -> None: text = PREAMBLE + CHANGELOG_BODY split = fetch_docs.split_changelog(text, preamble()) self.assertEqual([(b.version, b.text) for b in split.blocks], [("1.1.0", BLOCK_A), ("1.0.0", BLOCK_B)]) self.assertEqual(split.preamble_bytes, len(PREAMBLE.encode())) self.assertEqual(split.block_bytes, len((BLOCK_A + BLOCK_B).encode())) self.assertEqual(split.rest_bytes, len("# Changelog\n\nGenerated from the shed log.\n\n\n".encode())) self.assertEqual(split.preamble_bytes + split.block_bytes + split.rest_bytes, len(text.encode())) def test_a_last_block_without_a_final_newline_is_kept_as_it_is(self) -> None: text = PREAMBLE + CHANGELOG_BODY.rstrip("\n") split = fetch_docs.split_changelog(text, preamble()) self.assertEqual(split.blocks[-1].text, BLOCK_B.rstrip("\n")) self.assertEqual(split.preamble_bytes + split.block_bytes + split.rest_bytes, len(text.encode())) def test_broken_blocks_are_refused(self) -> None: cases = { "duplicate": CHANGELOG_BODY.replace('label="1.0.0"', 'label="1.1.0"'), "unclosed": CHANGELOG_BODY.replace(BLOCK_B, BLOCK_B.replace("\n", "")), "nested": CHANGELOG_BODY.replace(" * Added `rake`\n", " * Added `rake`\n" + BLOCK_B), "stray close": CHANGELOG_BODY + "\n", "not a version": CHANGELOG_BODY.replace('label="1.0.0"', 'label="first"'), "mid-line": CHANGELOG_BODY.replace("Generated from", 'Generated from'), "no blocks": "# Changelog\n\nNothing yet.\n", } for name, body in cases.items(): with self.subTest(name=name): with self.assertRaises(fetch_docs.FetchError): fetch_docs.split_changelog(PREAMBLE + body, preamble()) def test_the_bold_form_gives_one_file_per_release_with_every_byte_counted(self) -> None: # the page's shape after 20.10: "**X.Y.Z**", a blank line, the date; a release runs to the next one text = PREAMBLE + BOLD_BODY split = fetch_docs.split_changelog(text, preamble()) self.assertEqual([(b.version, b.text) for b in split.blocks], [("1.1.0", BOLD_A), ("1.0.0", BOLD_B)]) self.assertEqual(split.rest_bytes, len("# Changelog\n\nGenerated from the shed log.\n\n".encode())) self.assertEqual(split.preamble_bytes + split.block_bytes + split.rest_bytes, len(text.encode())) self.assertEqual(fetch_docs.release_count(text), 2) def test_broken_bold_releases_are_refused(self) -> None: cases = { "duplicate": BOLD_BODY.replace("**1.0.0**", "**1.1.0**"), "no date": BOLD_BODY.replace("\nMay 1, 2026\n", "\nSoon\n"), "no blank line": BOLD_BODY.replace("**1.0.0**\n\n", "**1.0.0**\n"), "not X.Y.Z": BOLD_BODY.replace("**1.0.0**", "**1.0**"), "both forms": BOLD_BODY + BLOCK_B, } for name, body in cases.items(): with self.subTest(name=name): with self.assertRaises(fetch_docs.FetchError): fetch_docs.split_changelog(PREAMBLE + body, preamble()) def test_the_anchor_url_is_checked_against_the_page(self) -> None: self.assertEqual(fetch_docs.anchor_url(CHANGELOG_PAGE, "1.1.0"), CHANGELOG_PAGE + "#1-1-0") self.assertEqual(fetch_docs.missing_anchors(CHANGELOG_HTML, ["1.1.0", "1.0.0"]), []) self.assertEqual(fetch_docs.missing_anchors(CHANGELOG_HTML, ["1.1.0", "0.9.0"]), ["0.9.0"]) class CollectTest(unittest.TestCase): def test_a_whole_run_gives_the_sources_and_the_meta_freeze_reads(self) -> None: got = collect(site()) self.assertEqual(got.problems, []) self.assertEqual(got.files, {"docs-en-soil.md": SOIL.encode(), "docs-en-tools-spade.md": SPADE.encode(), "release-v1.0.0.md": BLOCK_B.encode(), "release-v1.1.0.md": BLOCK_A.encode()}) self.assertEqual(got.meta, { "collection": "claude-code", "schema": "okf-wiki/source-meta/1", "index": {"pages": 4, "sha256": hashlib.sha256(INDEX.encode()).hexdigest(), "url": INDEX_URL}, "sources": { "docs-en-soil.md": {"kind": "docs", "main_topic": True, "url": BASE + "soil.md"}, "docs-en-tools-spade.md": {"kind": "docs", "main_topic": True, "url": BASE + "tools/spade.md"}, "release-v1.0.0.md": {"kind": "release-notes", "main_topic": False, "url": CHANGELOG_PAGE + "#1-0-0", "version": "1.0.0"}, "release-v1.1.0.md": {"kind": "release-notes", "main_topic": False, "url": CHANGELOG_PAGE + "#1-1-0", "version": "1.1.0"}}}) r = got.report self.assertEqual(r["moved_out"], [{"url": BASE + "moved.md", "final_url": "https://other.example.net/garden/moved", "content_type": "text/html"}]) self.assertEqual(r["completeness"], {"index_pages": 4, "changelog": 1, "moved_out": 1, "docs_expected": 2, "docs": 2, "release_notes_expected": 2, "release_notes": 2, "pass": True}) self.assertEqual(r["other_links"], [SITE + "_llms/fr.md"]) self.assertEqual(r["sections"]["docs-en-tools-spade.md"], ["Getting started", "Tools"]) def test_a_missing_page_fails_the_completeness_check(self) -> None: got = collect(site(**{BASE + "tools/spade.md": None})) self.assertFalse(got.report["completeness"]["pass"]) self.assertTrue(any("tools/spade.md" in p and "404" in p for p in got.problems), got.problems) self.assertTrue(any("completeness" in p for p in got.problems), got.problems) def test_a_missing_version_fails_the_completeness_check(self) -> None: body = CHANGELOG_BODY.replace("Generated from the shed log.", 'Generated log.') got = collect(site(**{CHANGELOG: fetch_docs.Fetched(200, CHANGELOG, MD, page(body))})) self.assertTrue(any("0.9.0" in p or "Update" in p for p in got.problems), got.problems) self.assertFalse(got.report["completeness"]["pass"]) self.assertFalse(any(name.startswith("release-") for name in got.files)) def test_a_bold_form_changelog_passes_the_completeness_check(self) -> None: got = collect(site(**{CHANGELOG: fetch_docs.Fetched(200, CHANGELOG, MD, page(BOLD_BODY))})) self.assertEqual(got.problems, []) self.assertEqual(got.files["release-v1.1.0.md"], BOLD_A.encode()) self.assertEqual(got.report["completeness"]["release_notes_expected"], 2) self.assertTrue(got.report["completeness"]["pass"]) def test_a_bold_version_split_would_miss_fails_the_completeness_check(self) -> None: body = BOLD_BODY.replace("Generated from the shed log.", "**0.9.0-beta**") got = collect(site(**{CHANGELOG: fetch_docs.Fetched(200, CHANGELOG, MD, page(body))})) self.assertFalse(got.report["completeness"]["pass"]) self.assertFalse(any(name.startswith("release-") for name in got.files)) def test_html_on_the_docs_site_is_refused_not_moved_out(self) -> None: got = collect(site(**{BASE + "soil.md": fetch_docs.Fetched(200, BASE + "soil.md", "text/html; charset=utf-8", b"

Soil

")})) self.assertTrue(any("docs-en-soil.md" in p or "soil.md" in p for p in got.problems), got.problems) self.assertFalse(got.report["completeness"]["pass"]) def test_a_redirect_inside_the_docs_site_is_still_a_docs_page(self) -> None: got = collect(site(**{BASE + "soil.md": fetch_docs.Fetched(200, BASE + "garden/soil.md", MD, page(SOIL))})) self.assertEqual(got.problems, []) self.assertEqual(got.meta["sources"]["docs-en-soil.md"]["url"], BASE + "soil.md") def test_crlf_becomes_lf_and_is_counted(self) -> None: crlf = page(SOIL).replace(b"\n", b"\r\n") got = collect(site(**{BASE + "soil.md": fetch_docs.Fetched(200, BASE + "soil.md", MD, crlf)})) self.assertEqual(got.problems, []) self.assertEqual(got.files["docs-en-soil.md"], SOIL.encode()) self.assertEqual(got.report["crlf_pages"], ["docs-en-soil.md"]) def test_a_missing_anchor_is_refused(self) -> None: html = CHANGELOG_HTML.replace('id="1-0-0"', 'id="v1"') got = collect(site(**{CHANGELOG_PAGE: fetch_docs.Fetched(200, CHANGELOG_PAGE, "text/html", html.encode())})) self.assertTrue(any("1.0.0" in p and "anchor" in p for p in got.problems), got.problems) def test_a_changelog_missing_from_the_index_is_refused(self) -> None: index = INDEX.replace("- [Changelog](https://example.org/docs/en/changelog.md): Releases by version.\n", "") got = collect(site(**{INDEX_URL: fetch_docs.Fetched(200, INDEX_URL, "text/plain", index.encode())})) self.assertTrue(any("changelog" in p for p in got.problems), got.problems) def test_only_the_weekly_digest_pages_are_history(self) -> None: for rel, history in (("whats-new/2026-w37.md", True), ("whats-new/2025-w01.md", True), ("whats-new/index.md", False), ("whats-new-2026-w37.md", False), ("guide/whats-new/2026-w37.md", False), ("whats-new/2026-w7.md", False)): with self.subTest(rel=rel): self.assertEqual(fetch_docs.is_history(rel), history) def test_weekly_digest_pages_are_docs_without_a_main_topic(self) -> None: """The operator 08.10.2026: the weekly pages add history to the concepts and get no main page.""" week, overview = BASE + "whats-new/2026-w01.md", BASE + "whats-new/index.md" index = INDEX.replace("## Indexes", f"## News\n\n- [News]({overview}): Every week.\n" f"- [Week 1]({week}): Rakes arrived.\n\n## Indexes") got = collect(site(**{INDEX_URL: fetch_docs.Fetched(200, INDEX_URL, "text/plain", index.encode()), overview: fetch_docs.Fetched(200, overview, MD, page("# News\n\nEvery week.\n")), week: fetch_docs.Fetched(200, week, MD, page("# Week 1\n\nRakes arrived.\n"))})) self.assertEqual(got.problems, []) self.assertEqual(got.meta["sources"]["docs-en-whats-new-2026-w01.md"], {"kind": "docs", "main_topic": False, "url": week}) self.assertEqual(got.meta["sources"]["docs-en-whats-new-index.md"], {"kind": "docs", "main_topic": True, "url": overview}) self.assertEqual(got.report["history_pages"], ["docs-en-whats-new-2026-w01.md"]) self.assertEqual(got.report["completeness"]["docs"], 4) self.assertTrue(got.report["completeness"]["pass"]) def test_the_counts_and_byte_shares_are_reported(self) -> None: r = collect(site()).report self.assertEqual(r["docs"], {"count": 2, "bytes": len((SOIL + SPADE).encode())}) self.assertEqual(r["release_notes"], {"count": 2, "bytes": len((BLOCK_A + BLOCK_B).encode())}) self.assertEqual(r["changelog_bytes"]["page"], len(page(CHANGELOG_BODY))) self.assertEqual(r["preamble"], {"pages": 3, "bytes": 3 * len(PREAMBLE.encode())}) class RunTest(unittest.TestCase): def setUp(self) -> None: self._tmp = tempfile.TemporaryDirectory() self.tmp = Path(self._tmp.name) self.spec = self.tmp / "preamble.json" self.spec.write_text(json.dumps(PREAMBLE_SPEC), encoding="utf-8") def tearDown(self) -> None: self._tmp.cleanup() def run_tool(self, pages: dict[str, Any], staging: Path) -> tuple[int, str]: argv = ["--staging", str(staging), "--meta", str(self.tmp / "meta.json"), "--report", str(self.tmp / "report.json"), "--preamble", str(self.spec), "--base", BASE, "--index", INDEX_URL, "--changelog", CHANGELOG, "--changelog-page", CHANGELOG_PAGE, "--pause", "0"] out, err = io.StringIO(), io.StringIO() with redirect_stdout(out), redirect_stderr(err): code = fetch_docs.main(argv, fetcher=FakeNet(pages)) return code, out.getvalue() + err.getvalue() def test_a_good_run_writes_a_staging_folder_that_freezes_and_checks_clean(self) -> None: staging = self.tmp / "staging" code, text = self.run_tool(site(), staging) self.assertEqual(code, 0, text) root = self.tmp / "frozen" with redirect_stdout(io.StringIO()): frozen = sources.main(["freeze", str(staging), "--meta", str(self.tmp / "meta.json"), "--out", str(root), "--frozen-at", "2026-10-08T12:00:00Z"]) self.assertEqual(frozen, 0) self.assertEqual(sources.check(root), []) self.assertTrue(json.loads((self.tmp / "report.json").read_text(encoding="utf-8"))["completeness"]["pass"]) def test_a_failed_run_writes_the_report_but_no_staging(self) -> None: staging = self.tmp / "staging" code, text = self.run_tool(site(**{BASE + "tools/spade.md": None}), staging) self.assertEqual(code, 1) self.assertIn("completeness", text) self.assertFalse(staging.exists()) self.assertFalse((self.tmp / "meta.json").exists()) self.assertFalse(json.loads((self.tmp / "report.json").read_text(encoding="utf-8"))["completeness"]["pass"]) def test_an_existing_staging_folder_cannot_run_and_nothing_is_fetched(self) -> None: staging = self.tmp / "staging" staging.mkdir() net = FakeNet(site()) argv = ["--staging", str(staging), "--meta", str(self.tmp / "meta.json"), "--report", str(self.tmp / "report.json"), "--preamble", str(self.spec), "--pause", "0"] with redirect_stdout(io.StringIO()), redirect_stderr(io.StringIO()): self.assertEqual(fetch_docs.main(argv, fetcher=net), 2) self.assertEqual(net.asked, []) if __name__ == "__main__": unittest.main()