"""Tests for okfwiki/sources.py: the source manifest (contract K1) with check, freeze and subset. Every source here is synthetic (bread and gardening); no text from any documentation. Run: python3 -m unittest discover -s tests -p 'test_sources.py' """ from __future__ import annotations import hashlib import io import json import os import stat import sys import tempfile import unittest from contextlib import redirect_stderr, redirect_stdout from pathlib import Path from typing import Any from unittest import mock ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT)) from okfwiki import kblocks, readers, sources # noqa: E402 FROZEN = "2026-10-08T12:00:00Z" LATER = "2026-10-09T06:30:00Z" FILES = { "docs-en-bread.md": "# Bread\n\nFlour, water and salt.\nKnead for ten minutes.\n", "docs-en-garden-soil.md": "# Soil\n\nCompost feeds the soil.\n", "release-v1.0.0.md": '\n * Added rye\n\n', } META_SOURCES: dict[str, dict[str, Any]] = { "docs-en-bread.md": {"kind": "docs", "main_topic": True, "url": "https://example.org/docs/en/bread.md"}, "docs-en-garden-soil.md": {"kind": "docs", "main_topic": True, "url": "https://example.org/docs/en/garden/soil.md"}, "release-v1.0.0.md": {"kind": "release-notes", "main_topic": False, "url": "https://example.org/docs/en/changelog#1-0-0", "version": "1.0.0"}, } INDEX = {"pages": 3, "sha256": "a" * 64, "url": "https://example.org/docs/llms.txt"} def sha(data: bytes) -> str: return hashlib.sha256(data).hexdigest() def run_main(argv: list[str]) -> tuple[int, str, str]: out, err = io.StringIO(), io.StringIO() with redirect_stdout(out), redirect_stderr(err): code = sources.main(argv) return code, out.getvalue(), err.getvalue() def entry(name: str, text: str, **meta: Any) -> dict[str, Any]: data = text.encode("utf-8") return {"bytes": len(data), "lines": len(text.splitlines()), "path": f"kilde/{name}", "sha256": sha(data), **meta} def write_root(root: Path, files: dict[str, str], entries: list[dict[str, Any]] | None = None, **top: Any) -> Path: """A K1 root written by hand, so check is tested apart from freeze.""" (root / "kilde").mkdir(parents=True) for name, text in files.items(): (root / "kilde" / name).write_bytes(text.encode("utf-8")) if entries is None: entries = [entry(n, t, **META_SOURCES[n]) for n, t in sorted(files.items())] doc = {"collection": "garden", "frozen_at": FROZEN, "index": INDEX, "schema": "okf-wiki/source-manifest/1", "sources": entries, "subset_of": None, **top} (root / "source-manifest.json").write_text(sources.dump(doc), encoding="utf-8") return root def manifest(root: Path) -> dict[str, Any]: doc: dict[str, Any] = json.loads((root / "source-manifest.json").read_text(encoding="utf-8")) return doc class Base(unittest.TestCase): def setUp(self) -> None: self._tmp = tempfile.TemporaryDirectory() self.tmp = Path(self._tmp.name) def tearDown(self) -> None: self._tmp.cleanup() class DumpTest(unittest.TestCase): def test_the_written_form_is_indent_1_sorted_keys_utf8_and_a_final_newline(self) -> None: self.assertEqual(sources.dump({"b": "ø", "a": [1]}), '{\n "a": [\n 1\n ],\n "b": "ø"\n}\n') class LinesTest(unittest.TestCase): def test_plain_newlines_are_not_ambiguous(self) -> None: for text in ("", "a", "a\n", "a\n\nb\n", "\n\n"): self.assertFalse(sources.ambiguous_lines(text), repr(text)) def test_any_other_line_break_is_ambiguous(self) -> None: for text in ("a\r\nb\n", "a\rb", "a\x0cb\n", "a\x0bb", "a\x1cb", "a\x85b", "a\u2028b", "a\u2029b"): self.assertTrue(sources.ambiguous_lines(text), repr(text)) class CheckTest(Base): def check(self, root: Path) -> tuple[int, str]: code, out, err = run_main(["check", str(root)]) return code, out + err def test_a_clean_root_is_0(self) -> None: root = write_root(self.tmp / "r", FILES) self.assertEqual(sources.check(root), []) code, text = self.check(root) self.assertEqual(code, 0, text) self.assertIn("3 sources", text) def test_a_missing_file_is_1_and_named(self) -> None: root = write_root(self.tmp / "r", FILES) (root / "kilde" / "docs-en-bread.md").unlink() code, text = self.check(root) self.assertEqual(code, 1) self.assertIn("kilde/docs-en-bread.md", text) self.assertIn("missing", text) def test_one_changed_byte_is_1(self) -> None: root = write_root(self.tmp / "r", FILES) p = root / "kilde" / "docs-en-bread.md" p.write_bytes(p.read_bytes().replace(b"salt", b"sale")) problems = sources.check(root) self.assertEqual(len(problems), 1, problems) self.assertIn("kilde/docs-en-bread.md", problems[0]) self.assertIn("sha256", problems[0]) def test_a_wrong_byte_or_line_count_is_1(self) -> None: good = entry("docs-en-bread.md", FILES["docs-en-bread.md"], **META_SOURCES["docs-en-bread.md"]) for key in ("bytes", "lines"): with self.subTest(key=key): root = write_root(self.tmp / key, {"docs-en-bread.md": FILES["docs-en-bread.md"]}, [{**good, key: good[key] + 1}]) problems = sources.check(root) self.assertEqual(len(problems), 1, problems) self.assertIn(key, problems[0]) def test_an_unlisted_file_is_1_also_in_a_subfolder(self) -> None: root = write_root(self.tmp / "r", FILES) (root / "kilde" / "stray.md").write_text("Tomatoes.\n", encoding="utf-8") (root / "kilde" / "sub").mkdir() (root / "kilde" / "sub" / "deep.md").write_text("Beans.\n", encoding="utf-8") problems = sources.check(root) self.assertEqual(len(problems), 2, problems) self.assertTrue(any("kilde/stray.md" in p and "not listed" in p for p in problems)) self.assertTrue(any("kilde/sub/deep.md" in p and "not listed" in p for p in problems)) def test_a_double_path_is_1(self) -> None: e = entry("docs-en-bread.md", FILES["docs-en-bread.md"], **META_SOURCES["docs-en-bread.md"]) root = write_root(self.tmp / "r", {"docs-en-bread.md": FILES["docs-en-bread.md"]}, [e, dict(e)]) problems = sources.check(root) self.assertTrue(any("kilde/docs-en-bread.md" in p and "more than once" in p for p in problems), problems) def test_an_invalid_main_topic_is_1(self) -> None: for bad in ("true", 1, None): with self.subTest(bad=bad): e = entry("docs-en-bread.md", FILES["docs-en-bread.md"], **{**META_SOURCES["docs-en-bread.md"], "main_topic": bad}) root = write_root(self.tmp / repr(bad), {"docs-en-bread.md": FILES["docs-en-bread.md"]}, [e]) problems = sources.check(root) self.assertEqual(len(problems), 1, problems) self.assertIn("main_topic", problems[0]) def test_a_missing_main_topic_is_1(self) -> None: meta = {k: v for k, v in META_SOURCES["docs-en-bread.md"].items() if k != "main_topic"} root = write_root(self.tmp / "r", {"docs-en-bread.md": FILES["docs-en-bread.md"]}, [entry("docs-en-bread.md", FILES["docs-en-bread.md"], **meta)]) self.assertTrue(any("main_topic" in p for p in sources.check(root))) def test_release_notes_need_a_version(self) -> None: meta = {k: v for k, v in META_SOURCES["release-v1.0.0.md"].items() if k != "version"} text = FILES["release-v1.0.0.md"] root = write_root(self.tmp / "r", {"release-v1.0.0.md": text}, [entry("release-v1.0.0.md", text, **meta)]) problems = sources.check(root) self.assertEqual(len(problems), 1, problems) self.assertIn("version", problems[0]) def test_another_kind_is_accepted(self) -> None: text = "Seeds sprout in spring.\n" root = write_root(self.tmp / "r", {"notes.md": text}, [entry("notes.md", text, kind="pdf-page", main_topic=False, url="file:///notes.pdf")]) self.assertEqual(sources.check(root), []) def test_a_bad_path_is_1(self) -> None: for name in ("Bread.md", "br ead.md", "bröd.md"): with self.subTest(name=name): text = "Rye.\n" root = write_root(self.tmp / sha(name.encode()), {name: text}, [entry(name, text, kind="docs", main_topic=True, url="https://example.org/x")]) self.assertTrue(any("path" in p for p in sources.check(root)), name) def test_sources_out_of_order_are_1(self) -> None: es = [entry(n, t, **META_SOURCES[n]) for n, t in sorted(FILES.items())] root = write_root(self.tmp / "r", FILES, list(reversed(es))) self.assertTrue(any("sorted" in p for p in sources.check(root))) def test_a_listed_file_with_ambiguous_lines_is_1(self) -> None: text = "Rye\x0cbread.\n" root = write_root(self.tmp / "r", {"docs-en-rye.md": text}, [entry("docs-en-rye.md", text, kind="docs", main_topic=True, url="https://example.org/r")]) problems = sources.check(root) self.assertEqual(len(problems), 1, problems) self.assertIn("ambiguous", problems[0]) def test_frozen_at_and_subset_of_have_their_form(self) -> None: for top in ({"frozen_at": "2026-10-08 12:00"}, {"subset_of": {"frozen_at": FROZEN}}, {"subset_of": "x"}, {"index": {"pages": "3"}}): with self.subTest(top=top): root = write_root(self.tmp / sha(repr(top).encode()), FILES, **top) self.assertEqual(len(sources.check(root)), 1, top) def test_the_index_may_be_absent(self) -> None: root = write_root(self.tmp / "r", FILES) doc = manifest(root) del doc["index"] (root / "source-manifest.json").write_text(sources.dump(doc), encoding="utf-8") self.assertEqual(sources.check(root), []) def test_unknown_keys_are_ignored(self) -> None: es = [{**entry(n, t, **META_SOURCES[n]), "x-later": 1} for n, t in sorted(FILES.items())] root = write_root(self.tmp / "r", FILES, es, **{"x-later": True}) self.assertEqual(sources.check(root), []) def test_no_manifest_or_an_unreadable_one_cannot_run(self) -> None: root = write_root(self.tmp / "r", FILES) for text in (None, "{", "[]", sources.dump({"schema": "other/1", "sources": []}), sources.dump({"schema": "okf-wiki/source-manifest/1", "sources": {}})): with self.subTest(text=text): m = root / "source-manifest.json" if text is None: m.unlink(missing_ok=True) else: m.write_text(text, encoding="utf-8") code, _ = self.check(root) self.assertEqual(code, 2) self.assertEqual(self.check(self.tmp / "nowhere")[0], 2) class FreezeTest(Base): def staging(self, files: dict[str, str] | None = None, meta_sources: dict[str, Any] | None = None, **meta_top: Any) -> tuple[Path, Path]: staging = self.tmp / "staging" staging.mkdir() for name, text in (FILES if files is None else files).items(): (staging / name).write_bytes(text.encode("utf-8")) meta = {"collection": "garden", "index": INDEX, "schema": "okf-wiki/source-meta/1", "sources": META_SOURCES if meta_sources is None else meta_sources, **meta_top} meta_path = self.tmp / "meta.json" meta_path.write_text(sources.dump(meta), encoding="utf-8") return staging, meta_path def freeze(self, staging: Path, meta: Path, out: Path, frozen_at: str = FROZEN) -> tuple[int, str]: code, o, e = run_main(["freeze", str(staging), "--meta", str(meta), "--out", str(out), "--frozen-at", frozen_at]) return code, o + e def test_freeze_writes_the_k1_manifest_and_read_only_copies(self) -> None: staging, meta = self.staging() out = self.tmp / "frozen" code, text = self.freeze(staging, meta, out) self.assertEqual(code, 0, text) expected = {"collection": "garden", "frozen_at": FROZEN, "index": INDEX, "schema": "okf-wiki/source-manifest/1", "subset_of": None, "sources": [entry(n, t, **META_SOURCES[n]) for n, t in sorted(FILES.items())]} self.assertEqual((out / "source-manifest.json").read_text(encoding="utf-8"), sources.dump(expected)) self.assertEqual(sorted(p.name for p in (out / "kilde").iterdir()), sorted(FILES)) for name, text in FILES.items(): p = out / "kilde" / name self.assertEqual(p.read_bytes(), text.encode("utf-8")) self.assertEqual(stat.S_IMODE(p.stat().st_mode) & 0o222, 0, f"{name} is writable") self.assertEqual(sources.check(out), []) self.assertEqual(run_main(["check", str(out)])[0], 0) def test_two_freezes_of_the_same_staging_are_byte_identical(self) -> None: staging, meta = self.staging() self.assertEqual(self.freeze(staging, meta, self.tmp / "a")[0], 0) self.assertEqual(self.freeze(staging, meta, self.tmp / "b")[0], 0) self.assertEqual((self.tmp / "a" / "source-manifest.json").read_bytes(), (self.tmp / "b" / "source-manifest.json").read_bytes()) def test_the_index_is_left_out_when_the_meta_has_none(self) -> None: staging, meta = self.staging() doc = json.loads(meta.read_text(encoding="utf-8")) del doc["index"] meta.write_text(sources.dump(doc), encoding="utf-8") out = self.tmp / "frozen" self.assertEqual(self.freeze(staging, meta, out)[0], 0) self.assertNotIn("index", manifest(out)) self.assertEqual(sources.check(out), []) def test_an_existing_out_is_refused_and_left_alone(self) -> None: staging, meta = self.staging() out = self.tmp / "frozen" out.mkdir() (out / "keep.txt").write_text("mine\n", encoding="utf-8") code, text = self.freeze(staging, meta, out) self.assertEqual(code, 2) self.assertIn("exists", text) self.assertEqual([p.name for p in out.iterdir()], ["keep.txt"]) def test_ambiguous_lines_are_refused_and_listed(self) -> None: files = {**FILES, "docs-en-bread.md": "# Bread\r\n\r\nFlour.\r\n", "docs-en-garden-soil.md": "Soil\u2028mix\n"} staging, meta = self.staging(files) out = self.tmp / "frozen" code, text = self.freeze(staging, meta, out) self.assertEqual(code, 1) self.assertIn("docs-en-bread.md", text) self.assertIn("docs-en-garden-soil.md", text) self.assertIn("ambiguous", text) self.assertFalse(out.exists()) def test_staging_and_meta_must_list_the_same_files(self) -> None: meta_sources = {**META_SOURCES, "docs-en-ghost.md": {"kind": "docs", "main_topic": True, "url": "https://example.org/ghost"}} staging, meta = self.staging(meta_sources=meta_sources) (staging / "stray.md").write_text("Weeds.\n", encoding="utf-8") out = self.tmp / "frozen" code, text = self.freeze(staging, meta, out) self.assertEqual(code, 1) self.assertIn("docs-en-ghost.md", text) self.assertIn("stray.md", text) self.assertFalse(out.exists()) def test_an_invalid_entry_in_the_meta_is_refused(self) -> None: for name, bad in (("docs-en-bread.md", {"main_topic": "yes"}), ("release-v1.0.0.md", {"version": None}), ("docs-en-bread.md", {"url": ""}), ("docs-en-bread.md", {"kind": ""})): with self.subTest(bad=bad): self._tmp.cleanup() self._tmp = tempfile.TemporaryDirectory() self.tmp = Path(self._tmp.name) meta_sources = {**META_SOURCES, name: {**META_SOURCES[name], **bad}} staging, meta = self.staging(meta_sources=meta_sources) out = self.tmp / "frozen" code, text = self.freeze(staging, meta, out) self.assertEqual(code, 1, text) self.assertIn(name, text) self.assertFalse(out.exists()) def test_a_bad_file_name_or_a_non_utf8_file_is_refused(self) -> None: staging, meta = self.staging() (staging / "docs-en-bread.md").write_bytes(b"Bread \xff\n") out = self.tmp / "frozen" code, text = self.freeze(staging, meta, out) self.assertEqual(code, 1) self.assertIn("UTF-8", text) self.assertFalse(out.exists()) def test_a_bad_frozen_at_cannot_run(self) -> None: staging, meta = self.staging() self.assertEqual(self.freeze(staging, meta, self.tmp / "frozen", "2026-10-08")[0], 2) self.assertFalse((self.tmp / "frozen").exists()) def test_frozen_at_defaults_to_now_in_utc(self) -> None: staging, meta = self.staging() out = self.tmp / "frozen" code, o, e = run_main(["freeze", str(staging), "--meta", str(meta), "--out", str(out)]) self.assertEqual(code, 0, o + e) self.assertRegex(manifest(out)["frozen_at"], r"^\d{4}-\d\d-\d\dT\d\d:\d\d:\d\dZ$") class SubsetTest(Base): def setUp(self) -> None: super().setUp() self.origin = write_root(self.tmp / "origin", FILES) def paths(self, *lines: str) -> Path: p = self.tmp / "paths.txt" p.write_text("".join(f"{x}\n" for x in lines), encoding="utf-8") return p def subset(self, paths: Path, out: Path, origin: Path | None = None) -> tuple[int, str]: code, o, e = run_main(["subset", str(origin or self.origin), "--paths", str(paths), "--out", str(out), "--frozen-at", LATER]) return code, o + e def test_a_subset_keeps_the_entries_and_points_at_its_origin(self) -> None: out = self.tmp / "part" code, text = self.subset(self.paths("kilde/docs-en-bread.md", "kilde/release-v1.0.0.md"), out) self.assertEqual(code, 0, text) origin = manifest(self.origin) part = manifest(out) self.assertEqual(part["sources"], [origin["sources"][0], origin["sources"][2]]) self.assertEqual(part["frozen_at"], LATER) self.assertEqual(part["index"], origin["index"]) self.assertEqual(part["collection"], origin["collection"]) self.assertEqual(part["subset_of"], {"frozen_at": FROZEN, "manifest_sha256": sha((self.origin / "source-manifest.json").read_bytes())}) self.assertEqual(sorted(p.name for p in (out / "kilde").iterdir()), ["docs-en-bread.md", "release-v1.0.0.md"]) for p in (out / "kilde").iterdir(): self.assertEqual(p.read_bytes(), (self.origin / "kilde" / p.name).read_bytes()) self.assertEqual(stat.S_IMODE(p.stat().st_mode) & 0o222, 0) self.assertEqual((out / "source-manifest.json").read_text(encoding="utf-8"), sources.dump(part)) self.assertEqual(sources.check(out), []) def test_the_paths_file_must_be_sorted_unique_and_known(self) -> None: for lines in (("kilde/release-v1.0.0.md", "kilde/docs-en-bread.md"), ("kilde/docs-en-bread.md", "kilde/docs-en-bread.md"), ("kilde/docs-en-rye.md",), ("docs-en-bread.md",), ("kilde/docs-en-bread.md", ""), ()): with self.subTest(lines=lines): out = self.tmp / ("part-" + sha(repr(lines).encode())[:8]) code, _ = self.subset(self.paths(*lines), out) self.assertEqual(code, 2, lines) self.assertFalse(out.exists()) def test_an_origin_that_fails_check_cannot_be_subset(self) -> None: os.chmod(self.origin / "kilde" / "docs-en-bread.md", 0o644) (self.origin / "kilde" / "docs-en-bread.md").write_text("changed\n", encoding="utf-8") out = self.tmp / "part" self.assertEqual(self.subset(self.paths("kilde/docs-en-garden-soil.md"), out)[0], 2) self.assertFalse(out.exists()) def test_an_existing_out_is_refused(self) -> None: out = self.tmp / "part" out.mkdir() self.assertEqual(self.subset(self.paths("kilde/docs-en-bread.md"), out)[0], 2) self.assertEqual(list(out.iterdir()), []) if __name__ == "__main__": unittest.main() class FolderBase(Base): def folder(self, files: dict[str, bytes]) -> Path: src = self.tmp / "Dokumenter" for rel, data in files.items(): p = src / rel p.parent.mkdir(parents=True, exist_ok=True) p.write_bytes(data) return src def run_folder(self, src: Path, out: Path) -> tuple[int, str]: code, o, e = run_main(["folder", str(src), "--collection", "hage", "--out", str(out), "--frozen-at", FROZEN]) return code, o + e class FolderTest(FolderBase): """`sources folder`: any folder of text documents becomes a frozen copy with no hand-written meta (motor step 2: names and subfolders, the meta made by code, line breaks and encodings, .txt kept, other files reported).""" def test_each_document_records_its_language_when_it_can_be_told(self) -> None: nb = "# Bier\n\nDet er viktig at bikubene står i le, og at de ikke blir for varme om sommeren.\n" * 2 en = "# Bees\n\nIt is important that the hives stand in shelter and that they do not get too hot.\n" * 2 src = self.folder({"bier.md": nb.encode(), "bees.md": en.encode(), "kort.md": b"# Kort\n\nBikuber.\n"}) out = self.tmp / "frozen" code, text = self.run_folder(src, out) self.assertEqual(code, 0, text) by_path = {e["path"]: e for e in manifest(out)["sources"]} self.assertEqual(by_path["kilde/bier.md"]["lang"], "no") self.assertEqual(by_path["kilde/bees.md"]["lang"], "en") self.assertNotIn("lang", by_path["kilde/kort.md"]) self.assertEqual(sources.check(out), []) def test_subfolders_capitals_and_nordic_letters_get_safe_names_and_keep_their_original(self) -> None: src = self.folder({"Møtereferat.md": "# Møte\n\nVi sådde.\n".encode(), "Årlig/Kontroll Plan.md": "# Kontroll\n\nSjekk jorda.\n".encode()}) out = self.tmp / "frozen" code, text = self.run_folder(src, out) self.assertEqual(code, 0, text) doc = manifest(out) by_path = {e["path"]: e for e in doc["sources"]} self.assertEqual(sorted(by_path), ["kilde/aarlig__kontroll-plan.md", "kilde/moetereferat.md"]) self.assertEqual(by_path["kilde/moetereferat.md"]["original"], "Møtereferat.md") self.assertEqual(by_path["kilde/aarlig__kontroll-plan.md"]["original"], "Årlig/Kontroll Plan.md") self.assertEqual(sources.check(out), []) def test_a_document_named_like_a_release_note_is_not_read_as_one(self) -> None: # The release-note rules (order, version, no skeleton, history) go by the kilde/ name; in a folder every # document has one main topic, so its safe name never has that form (motor B13). src = self.folder({"Release-v2.1.0.md": b"# Release v2.1.0\n\n## 2.1.0\n\nThe hive scale reads in grams.\n"}) out = self.tmp / "frozen" code, text = self.run_folder(src, out) self.assertEqual(code, 0, text) (entry,) = manifest(out)["sources"] name = entry["path"][len("kilde/"):] self.assertIsNone(kblocks.RELEASE.match(name), name) self.assertTrue(name.startswith("release-v2.1.0"), name) self.assertEqual(entry["original"], "Release-v2.1.0.md") self.assertEqual([b.version for b in kblocks.blocks(out / "kilde" / name)], [None]) self.assertEqual(sources.check(out), []) def test_the_meta_is_made_by_code(self) -> None: src = self.folder({"a.md": b"# A\n\nOne.\n"}) out = self.tmp / "frozen" self.assertEqual(self.run_folder(src, out)[0], 0) doc = manifest(out) self.assertEqual(doc["collection"], "hage") self.assertIsNone(doc["subset_of"]) e = doc["sources"][0] self.assertEqual((e["kind"], e["main_topic"]), ("document", True)) self.assertEqual(e["url"], (src / "a.md").resolve().as_uri()) def test_original_names_are_nfc(self) -> None: src = self.folder({"Möte.md": b"# M\n\nX.\n"}) # decomposed, as some file systems return it out = self.tmp / "frozen" self.assertEqual(self.run_folder(src, out)[0], 0) self.assertEqual(manifest(out)["sources"][0]["original"], "Möte.md") def test_windows_line_breaks_latin1_and_bom_are_normalised_and_recorded(self) -> None: src = self.folder({"crlf.md": b"# A\r\n\r\nOne.\r\n", "latin.txt": "Blåbær og øl.\n".encode("cp1252"), "bom.md": b"\xef\xbb\xbf# Tittel\n\nTekst.\n", "ff.txt": b"Side en.\x0cSide to.\n"}) out = self.tmp / "frozen" code, text = self.run_folder(src, out) self.assertEqual(code, 0, text) kilde = out / "kilde" self.assertEqual((kilde / "crlf.md").read_bytes(), b"# A\n\nOne.\n") self.assertEqual((kilde / "latin.txt").read_text(encoding="utf-8"), "Blåbær og øl.\n") self.assertEqual((kilde / "bom.md").read_bytes(), b"# Tittel\n\nTekst.\n") self.assertEqual((kilde / "ff.txt").read_bytes(), b"Side en.\nSide to.\n") normalised = {e["path"]: e["normalised"] for e in manifest(out)["sources"]} self.assertEqual(normalised, {"kilde/bom.md": ["bom"], "kilde/crlf.md": ["line-breaks"], "kilde/ff.txt": ["line-breaks"], "kilde/latin.txt": ["cp1252"]}) def test_a_clean_file_is_copied_byte_for_byte_and_records_nothing_normalised(self) -> None: src = self.folder({"a.md": b"# A\n\nOne.\n"}) out = self.tmp / "frozen" self.assertEqual(self.run_folder(src, out)[0], 0) self.assertEqual((out / "kilde" / "a.md").read_bytes(), b"# A\n\nOne.\n") self.assertEqual(manifest(out)["sources"][0]["normalised"], []) def test_other_files_hidden_files_and_empty_files_are_reported_not_dropped_silently(self) -> None: src = self.folder({"a.md": b"# A\n\nOne.\n", "bilde.png": b"\x89PNG", ".skjult.md": b"# H\n\nX.\n", ".git/config": b"x", "tom.txt": b" \n\n"}) out = self.tmp / "frozen" code, text = self.run_folder(src, out) self.assertEqual(code, 0, text) self.assertEqual([e["path"] for e in manifest(out)["sources"]], ["kilde/a.md"]) self.assertIn("1 sources", text) self.assertIn("bilde.png: skipped, no reader for .png", text) self.assertIn(".skjult.md: skipped, hidden", text) self.assertIn(".git: skipped, hidden", text) self.assertIn("tom.txt: skipped, empty", text) def test_two_originals_with_one_safe_name_are_refused(self) -> None: src = self.folder({"Møte.md": b"# P\n\nX.\n", "moete.md": b"# Q\n\nY.\n"}) out = self.tmp / "frozen" code, text = self.run_folder(src, out) self.assertEqual(code, 1, text) self.assertIn("moete.md: the safe name of Møte.md and moete.md", text) self.assertFalse(out.exists()) def test_a_folder_without_a_readable_document_cannot_run(self) -> None: src = self.folder({"x.pdf": b"%PDF"}) code, text = self.run_folder(src, self.tmp / "frozen") self.assertEqual(code, 2, text) def test_two_runs_are_byte_identical(self) -> None: src = self.folder({"Å.md": "# Å\n\nÆ.\n".encode(), "b/c.txt": b"C.\n"}) self.assertEqual(self.run_folder(src, self.tmp / "f1")[0], 0) self.assertEqual(self.run_folder(src, self.tmp / "f2")[0], 0) self.assertEqual((self.tmp / "f1" / "source-manifest.json").read_bytes(), (self.tmp / "f2" / "source-manifest.json").read_bytes()) class FolderOfficeTest(FolderBase): """Motor step 3 (A1, B11): PDF and Office files in a folder are frozen as the text a reader sees, with the place of every line (page, paragraph, cell, slide) in the manifest and the original's sha256 beside it.""" def office_folder(self) -> Path: from test_readers import docx, pdf, pptx, xlsx src = self.tmp / "Dokumenter" (src / "Kurs").mkdir(parents=True) pdf(src / "Rapport.pdf", ["Side en om bier", "Side to om honning"]) docx(src / "Plan.docx") xlsx(src / "kuber.xlsx") pptx(src / "Kurs" / "Våren.pptx") return src def test_pdf_and_office_files_are_frozen_as_their_rendering_with_a_place_for_every_line(self) -> None: src = self.office_folder() out = self.tmp / "frozen" code, text = self.run_folder(src, out) self.assertEqual(code, 0, text) by_path = {e["path"]: e for e in manifest(out)["sources"]} self.assertEqual(sorted(by_path), ["kilde/kuber.xlsx.md", "kilde/kurs__vaaren.pptx.md", "kilde/plan.docx.md", "kilde/rapport.pdf.md"]) e = by_path["kilde/rapport.pdf.md"] self.assertEqual((e["original"], e["reader"], e["kind"], e["main_topic"]), ("Rapport.pdf", "pdf", "document", True)) self.assertEqual(e["url"], (src / "Rapport.pdf").resolve().as_uri()) self.assertEqual(e["original_sha256"], hashlib.sha256((src / "Rapport.pdf").read_bytes()).hexdigest()) self.assertEqual(e["places"], [{"page": 1}, None, {"page": 2}]) self.assertEqual(e["original_copy"], "original/rapport.pdf") self.assertEqual((out / "original" / "rapport.pdf").read_bytes(), (src / "Rapport.pdf").read_bytes()) self.assertEqual((out / "kilde" / "rapport.pdf.md").read_text(encoding="utf-8"), readers.render(src / "Rapport.pdf").text) for path, entry in by_path.items(): self.assertEqual(len(entry["places"]), entry["lines"], path) self.assertEqual(by_path["kilde/kuber.xlsx.md"]["places"][2], {"sheet": "Plan", "cells": [["A1", 0], ["B1", 13]]}) self.assertEqual(sources.check(out), []) def test_hidden_content_left_out_is_reported_per_file(self) -> None: code, text = self.run_folder(self.office_folder(), self.tmp / "frozen") self.assertEqual(code, 0, text) self.assertIn("kuber.xlsx: 3 hidden parts left out", text) self.assertIn("Kurs/Våren.pptx: 2 hidden parts left out", text) self.assertEqual(manifest(self.tmp / "frozen")["sources"][0]["left_out"], 3) def test_a_broken_file_and_a_pdf_without_text_are_reported_not_dropped_silently(self) -> None: from test_readers import pdf src = self.folder({"a.md": b"# A\n\nOne.\n", "brutt.docx": b"not a zip"}) pdf(src / "skannet.pdf", [""]) code, text = self.run_folder(src, self.tmp / "frozen") self.assertEqual(code, 0, text) self.assertIn("brutt.docx: skipped, unreadable:", text) self.assertIn("skannet.pdf: skipped, no text (a scanned page needs text recognition first)", text) def test_a_pdf_without_the_pinned_library_stops_the_folder_with_the_install_command(self) -> None: from test_readers import pdf src = self.folder({"a.md": b"# A\n\nOne.\n"}) pdf(src / "r.pdf", ["Tekst"]) with mock.patch.object(readers, "PDF_PIN", "0.0.1"): code, text = self.run_folder(src, self.tmp / "frozen") self.assertEqual(code, 1, text) self.assertIn("pip install pypdf==0.0.1", text) self.assertFalse((self.tmp / "frozen").exists()) def test_check_refuses_an_original_copy_that_is_missing_or_changed(self) -> None: out = self.tmp / "frozen" self.assertEqual(self.run_folder(self.office_folder(), out)[0], 0) copy = out / "original" / "plan.docx" copy.chmod(0o644) copy.write_bytes(b"changed") self.assertIn("original/plan.docx: sha256 differs from original_sha256", "; ".join(sources.check(out))) copy.unlink() self.assertIn("original/plan.docx: missing", "; ".join(sources.check(out))) def test_check_refuses_places_that_do_not_match_the_lines(self) -> None: out = self.tmp / "frozen" self.assertEqual(self.run_folder(self.office_folder(), out)[0], 0) doc = manifest(out) doc["sources"][0]["places"].append(None) (out / "source-manifest.json").write_text(json.dumps(doc), encoding="utf-8") self.assertTrue(any("places" in p for p in sources.check(out)), sources.check(out))