okf-wiki/tests/test_sources.py
Kjell Tore Guttormsen 0fd86a23a3
okf-wiki 1.1.1
First public release, published as one snapshot. What changed between
versions is in CHANGELOG.md.

Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
2026-11-14 21:03:20 +01:00

638 lines
32 KiB
Python

"""Tests for okfwiki/sources.py: the source manifest (contract K1) with check, freeze and subset.
Every source here is synthetic (bread and gardening); no text from any documentation.
Run: python3 -m unittest discover -s tests -p 'test_sources.py'
"""
from __future__ import annotations
import hashlib
import io
import json
import os
import stat
import sys
import tempfile
import unittest
from contextlib import redirect_stderr, redirect_stdout
from pathlib import Path
from typing import Any
from unittest import mock
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT))
from okfwiki import kblocks, readers, sources # noqa: E402
FROZEN = "2026-10-08T12:00:00Z"
LATER = "2026-10-09T06:30:00Z"
FILES = {
"docs-en-bread.md": "# Bread\n\nFlour, water and salt.\nKnead for ten minutes.\n",
"docs-en-garden-soil.md": "# Soil\n\nCompost feeds the soil.\n",
"release-v1.0.0.md": '<Update label="1.0.0" description="May 1, 2026">\n * Added rye\n</Update>\n',
}
META_SOURCES: dict[str, dict[str, Any]] = {
"docs-en-bread.md": {"kind": "docs", "main_topic": True, "url": "https://example.org/docs/en/bread.md"},
"docs-en-garden-soil.md": {"kind": "docs", "main_topic": True,
"url": "https://example.org/docs/en/garden/soil.md"},
"release-v1.0.0.md": {"kind": "release-notes", "main_topic": False,
"url": "https://example.org/docs/en/changelog#1-0-0", "version": "1.0.0"},
}
INDEX = {"pages": 3, "sha256": "a" * 64, "url": "https://example.org/docs/llms.txt"}
def sha(data: bytes) -> str:
return hashlib.sha256(data).hexdigest()
def run_main(argv: list[str]) -> tuple[int, str, str]:
out, err = io.StringIO(), io.StringIO()
with redirect_stdout(out), redirect_stderr(err):
code = sources.main(argv)
return code, out.getvalue(), err.getvalue()
def entry(name: str, text: str, **meta: Any) -> dict[str, Any]:
data = text.encode("utf-8")
return {"bytes": len(data), "lines": len(text.splitlines()), "path": f"kilde/{name}", "sha256": sha(data), **meta}
def write_root(root: Path, files: dict[str, str], entries: list[dict[str, Any]] | None = None,
**top: Any) -> Path:
"""A K1 root written by hand, so check is tested apart from freeze."""
(root / "kilde").mkdir(parents=True)
for name, text in files.items():
(root / "kilde" / name).write_bytes(text.encode("utf-8"))
if entries is None:
entries = [entry(n, t, **META_SOURCES[n]) for n, t in sorted(files.items())]
doc = {"collection": "garden", "frozen_at": FROZEN, "index": INDEX, "schema": "okf-wiki/source-manifest/1",
"sources": entries, "subset_of": None, **top}
(root / "source-manifest.json").write_text(sources.dump(doc), encoding="utf-8")
return root
def manifest(root: Path) -> dict[str, Any]:
doc: dict[str, Any] = json.loads((root / "source-manifest.json").read_text(encoding="utf-8"))
return doc
class Base(unittest.TestCase):
def setUp(self) -> None:
self._tmp = tempfile.TemporaryDirectory()
self.tmp = Path(self._tmp.name)
def tearDown(self) -> None:
self._tmp.cleanup()
class DumpTest(unittest.TestCase):
def test_the_written_form_is_indent_1_sorted_keys_utf8_and_a_final_newline(self) -> None:
self.assertEqual(sources.dump({"b": "ø", "a": [1]}), '{\n "a": [\n 1\n ],\n "b": "ø"\n}\n')
class LinesTest(unittest.TestCase):
def test_plain_newlines_are_not_ambiguous(self) -> None:
for text in ("", "a", "a\n", "a\n\nb\n", "\n\n"):
self.assertFalse(sources.ambiguous_lines(text), repr(text))
def test_any_other_line_break_is_ambiguous(self) -> None:
for text in ("a\r\nb\n", "a\rb", "a\x0cb\n", "a\x0bb", "a\x1cb", "a\x85b", "a\u2028b", "a\u2029b"):
self.assertTrue(sources.ambiguous_lines(text), repr(text))
class CheckTest(Base):
def check(self, root: Path) -> tuple[int, str]:
code, out, err = run_main(["check", str(root)])
return code, out + err
def test_a_clean_root_is_0(self) -> None:
root = write_root(self.tmp / "r", FILES)
self.assertEqual(sources.check(root), [])
code, text = self.check(root)
self.assertEqual(code, 0, text)
self.assertIn("3 sources", text)
def test_a_missing_file_is_1_and_named(self) -> None:
root = write_root(self.tmp / "r", FILES)
(root / "kilde" / "docs-en-bread.md").unlink()
code, text = self.check(root)
self.assertEqual(code, 1)
self.assertIn("kilde/docs-en-bread.md", text)
self.assertIn("missing", text)
def test_one_changed_byte_is_1(self) -> None:
root = write_root(self.tmp / "r", FILES)
p = root / "kilde" / "docs-en-bread.md"
p.write_bytes(p.read_bytes().replace(b"salt", b"sale"))
problems = sources.check(root)
self.assertEqual(len(problems), 1, problems)
self.assertIn("kilde/docs-en-bread.md", problems[0])
self.assertIn("sha256", problems[0])
def test_a_wrong_byte_or_line_count_is_1(self) -> None:
good = entry("docs-en-bread.md", FILES["docs-en-bread.md"], **META_SOURCES["docs-en-bread.md"])
for key in ("bytes", "lines"):
with self.subTest(key=key):
root = write_root(self.tmp / key, {"docs-en-bread.md": FILES["docs-en-bread.md"]},
[{**good, key: good[key] + 1}])
problems = sources.check(root)
self.assertEqual(len(problems), 1, problems)
self.assertIn(key, problems[0])
def test_an_unlisted_file_is_1_also_in_a_subfolder(self) -> None:
root = write_root(self.tmp / "r", FILES)
(root / "kilde" / "stray.md").write_text("Tomatoes.\n", encoding="utf-8")
(root / "kilde" / "sub").mkdir()
(root / "kilde" / "sub" / "deep.md").write_text("Beans.\n", encoding="utf-8")
problems = sources.check(root)
self.assertEqual(len(problems), 2, problems)
self.assertTrue(any("kilde/stray.md" in p and "not listed" in p for p in problems))
self.assertTrue(any("kilde/sub/deep.md" in p and "not listed" in p for p in problems))
def test_a_double_path_is_1(self) -> None:
e = entry("docs-en-bread.md", FILES["docs-en-bread.md"], **META_SOURCES["docs-en-bread.md"])
root = write_root(self.tmp / "r", {"docs-en-bread.md": FILES["docs-en-bread.md"]}, [e, dict(e)])
problems = sources.check(root)
self.assertTrue(any("kilde/docs-en-bread.md" in p and "more than once" in p for p in problems), problems)
def test_an_invalid_main_topic_is_1(self) -> None:
for bad in ("true", 1, None):
with self.subTest(bad=bad):
e = entry("docs-en-bread.md", FILES["docs-en-bread.md"],
**{**META_SOURCES["docs-en-bread.md"], "main_topic": bad})
root = write_root(self.tmp / repr(bad), {"docs-en-bread.md": FILES["docs-en-bread.md"]}, [e])
problems = sources.check(root)
self.assertEqual(len(problems), 1, problems)
self.assertIn("main_topic", problems[0])
def test_a_missing_main_topic_is_1(self) -> None:
meta = {k: v for k, v in META_SOURCES["docs-en-bread.md"].items() if k != "main_topic"}
root = write_root(self.tmp / "r", {"docs-en-bread.md": FILES["docs-en-bread.md"]},
[entry("docs-en-bread.md", FILES["docs-en-bread.md"], **meta)])
self.assertTrue(any("main_topic" in p for p in sources.check(root)))
def test_release_notes_need_a_version(self) -> None:
meta = {k: v for k, v in META_SOURCES["release-v1.0.0.md"].items() if k != "version"}
text = FILES["release-v1.0.0.md"]
root = write_root(self.tmp / "r", {"release-v1.0.0.md": text}, [entry("release-v1.0.0.md", text, **meta)])
problems = sources.check(root)
self.assertEqual(len(problems), 1, problems)
self.assertIn("version", problems[0])
def test_another_kind_is_accepted(self) -> None:
text = "Seeds sprout in spring.\n"
root = write_root(self.tmp / "r", {"notes.md": text}, [entry("notes.md", text, kind="pdf-page",
main_topic=False, url="file:///notes.pdf")])
self.assertEqual(sources.check(root), [])
def test_a_bad_path_is_1(self) -> None:
for name in ("Bread.md", "br ead.md", "bröd.md"):
with self.subTest(name=name):
text = "Rye.\n"
root = write_root(self.tmp / sha(name.encode()), {name: text},
[entry(name, text, kind="docs", main_topic=True, url="https://example.org/x")])
self.assertTrue(any("path" in p for p in sources.check(root)), name)
def test_sources_out_of_order_are_1(self) -> None:
es = [entry(n, t, **META_SOURCES[n]) for n, t in sorted(FILES.items())]
root = write_root(self.tmp / "r", FILES, list(reversed(es)))
self.assertTrue(any("sorted" in p for p in sources.check(root)))
def test_a_listed_file_with_ambiguous_lines_is_1(self) -> None:
text = "Rye\x0cbread.\n"
root = write_root(self.tmp / "r", {"docs-en-rye.md": text},
[entry("docs-en-rye.md", text, kind="docs", main_topic=True, url="https://example.org/r")])
problems = sources.check(root)
self.assertEqual(len(problems), 1, problems)
self.assertIn("ambiguous", problems[0])
def test_frozen_at_and_subset_of_have_their_form(self) -> None:
for top in ({"frozen_at": "2026-10-08 12:00"}, {"subset_of": {"frozen_at": FROZEN}},
{"subset_of": "x"}, {"index": {"pages": "3"}}):
with self.subTest(top=top):
root = write_root(self.tmp / sha(repr(top).encode()), FILES, **top)
self.assertEqual(len(sources.check(root)), 1, top)
def test_the_index_may_be_absent(self) -> None:
root = write_root(self.tmp / "r", FILES)
doc = manifest(root)
del doc["index"]
(root / "source-manifest.json").write_text(sources.dump(doc), encoding="utf-8")
self.assertEqual(sources.check(root), [])
def test_unknown_keys_are_ignored(self) -> None:
es = [{**entry(n, t, **META_SOURCES[n]), "x-later": 1} for n, t in sorted(FILES.items())]
root = write_root(self.tmp / "r", FILES, es, **{"x-later": True})
self.assertEqual(sources.check(root), [])
def test_no_manifest_or_an_unreadable_one_cannot_run(self) -> None:
root = write_root(self.tmp / "r", FILES)
for text in (None, "{", "[]", sources.dump({"schema": "other/1", "sources": []}),
sources.dump({"schema": "okf-wiki/source-manifest/1", "sources": {}})):
with self.subTest(text=text):
m = root / "source-manifest.json"
if text is None:
m.unlink(missing_ok=True)
else:
m.write_text(text, encoding="utf-8")
code, _ = self.check(root)
self.assertEqual(code, 2)
self.assertEqual(self.check(self.tmp / "nowhere")[0], 2)
class FreezeTest(Base):
def staging(self, files: dict[str, str] | None = None, meta_sources: dict[str, Any] | None = None,
**meta_top: Any) -> tuple[Path, Path]:
staging = self.tmp / "staging"
staging.mkdir()
for name, text in (FILES if files is None else files).items():
(staging / name).write_bytes(text.encode("utf-8"))
meta = {"collection": "garden", "index": INDEX, "schema": "okf-wiki/source-meta/1",
"sources": META_SOURCES if meta_sources is None else meta_sources, **meta_top}
meta_path = self.tmp / "meta.json"
meta_path.write_text(sources.dump(meta), encoding="utf-8")
return staging, meta_path
def freeze(self, staging: Path, meta: Path, out: Path, frozen_at: str = FROZEN) -> tuple[int, str]:
code, o, e = run_main(["freeze", str(staging), "--meta", str(meta), "--out", str(out),
"--frozen-at", frozen_at])
return code, o + e
def test_freeze_writes_the_k1_manifest_and_read_only_copies(self) -> None:
staging, meta = self.staging()
out = self.tmp / "frozen"
code, text = self.freeze(staging, meta, out)
self.assertEqual(code, 0, text)
expected = {"collection": "garden", "frozen_at": FROZEN, "index": INDEX,
"schema": "okf-wiki/source-manifest/1", "subset_of": None,
"sources": [entry(n, t, **META_SOURCES[n]) for n, t in sorted(FILES.items())]}
self.assertEqual((out / "source-manifest.json").read_text(encoding="utf-8"), sources.dump(expected))
self.assertEqual(sorted(p.name for p in (out / "kilde").iterdir()), sorted(FILES))
for name, text in FILES.items():
p = out / "kilde" / name
self.assertEqual(p.read_bytes(), text.encode("utf-8"))
self.assertEqual(stat.S_IMODE(p.stat().st_mode) & 0o222, 0, f"{name} is writable")
self.assertEqual(sources.check(out), [])
self.assertEqual(run_main(["check", str(out)])[0], 0)
def test_two_freezes_of_the_same_staging_are_byte_identical(self) -> None:
staging, meta = self.staging()
self.assertEqual(self.freeze(staging, meta, self.tmp / "a")[0], 0)
self.assertEqual(self.freeze(staging, meta, self.tmp / "b")[0], 0)
self.assertEqual((self.tmp / "a" / "source-manifest.json").read_bytes(),
(self.tmp / "b" / "source-manifest.json").read_bytes())
def test_the_index_is_left_out_when_the_meta_has_none(self) -> None:
staging, meta = self.staging()
doc = json.loads(meta.read_text(encoding="utf-8"))
del doc["index"]
meta.write_text(sources.dump(doc), encoding="utf-8")
out = self.tmp / "frozen"
self.assertEqual(self.freeze(staging, meta, out)[0], 0)
self.assertNotIn("index", manifest(out))
self.assertEqual(sources.check(out), [])
def test_an_existing_out_is_refused_and_left_alone(self) -> None:
staging, meta = self.staging()
out = self.tmp / "frozen"
out.mkdir()
(out / "keep.txt").write_text("mine\n", encoding="utf-8")
code, text = self.freeze(staging, meta, out)
self.assertEqual(code, 2)
self.assertIn("exists", text)
self.assertEqual([p.name for p in out.iterdir()], ["keep.txt"])
def test_ambiguous_lines_are_refused_and_listed(self) -> None:
files = {**FILES, "docs-en-bread.md": "# Bread\r\n\r\nFlour.\r\n", "docs-en-garden-soil.md": "Soil\u2028mix\n"}
staging, meta = self.staging(files)
out = self.tmp / "frozen"
code, text = self.freeze(staging, meta, out)
self.assertEqual(code, 1)
self.assertIn("docs-en-bread.md", text)
self.assertIn("docs-en-garden-soil.md", text)
self.assertIn("ambiguous", text)
self.assertFalse(out.exists())
def test_staging_and_meta_must_list_the_same_files(self) -> None:
meta_sources = {**META_SOURCES, "docs-en-ghost.md": {"kind": "docs", "main_topic": True,
"url": "https://example.org/ghost"}}
staging, meta = self.staging(meta_sources=meta_sources)
(staging / "stray.md").write_text("Weeds.\n", encoding="utf-8")
out = self.tmp / "frozen"
code, text = self.freeze(staging, meta, out)
self.assertEqual(code, 1)
self.assertIn("docs-en-ghost.md", text)
self.assertIn("stray.md", text)
self.assertFalse(out.exists())
def test_an_invalid_entry_in_the_meta_is_refused(self) -> None:
for name, bad in (("docs-en-bread.md", {"main_topic": "yes"}), ("release-v1.0.0.md", {"version": None}),
("docs-en-bread.md", {"url": ""}), ("docs-en-bread.md", {"kind": ""})):
with self.subTest(bad=bad):
self._tmp.cleanup()
self._tmp = tempfile.TemporaryDirectory()
self.tmp = Path(self._tmp.name)
meta_sources = {**META_SOURCES, name: {**META_SOURCES[name], **bad}}
staging, meta = self.staging(meta_sources=meta_sources)
out = self.tmp / "frozen"
code, text = self.freeze(staging, meta, out)
self.assertEqual(code, 1, text)
self.assertIn(name, text)
self.assertFalse(out.exists())
def test_a_bad_file_name_or_a_non_utf8_file_is_refused(self) -> None:
staging, meta = self.staging()
(staging / "docs-en-bread.md").write_bytes(b"Bread \xff\n")
out = self.tmp / "frozen"
code, text = self.freeze(staging, meta, out)
self.assertEqual(code, 1)
self.assertIn("UTF-8", text)
self.assertFalse(out.exists())
def test_a_bad_frozen_at_cannot_run(self) -> None:
staging, meta = self.staging()
self.assertEqual(self.freeze(staging, meta, self.tmp / "frozen", "2026-10-08")[0], 2)
self.assertFalse((self.tmp / "frozen").exists())
def test_frozen_at_defaults_to_now_in_utc(self) -> None:
staging, meta = self.staging()
out = self.tmp / "frozen"
code, o, e = run_main(["freeze", str(staging), "--meta", str(meta), "--out", str(out)])
self.assertEqual(code, 0, o + e)
self.assertRegex(manifest(out)["frozen_at"], r"^\d{4}-\d\d-\d\dT\d\d:\d\d:\d\dZ$")
class SubsetTest(Base):
def setUp(self) -> None:
super().setUp()
self.origin = write_root(self.tmp / "origin", FILES)
def paths(self, *lines: str) -> Path:
p = self.tmp / "paths.txt"
p.write_text("".join(f"{x}\n" for x in lines), encoding="utf-8")
return p
def subset(self, paths: Path, out: Path, origin: Path | None = None) -> tuple[int, str]:
code, o, e = run_main(["subset", str(origin or self.origin), "--paths", str(paths), "--out", str(out),
"--frozen-at", LATER])
return code, o + e
def test_a_subset_keeps_the_entries_and_points_at_its_origin(self) -> None:
out = self.tmp / "part"
code, text = self.subset(self.paths("kilde/docs-en-bread.md", "kilde/release-v1.0.0.md"), out)
self.assertEqual(code, 0, text)
origin = manifest(self.origin)
part = manifest(out)
self.assertEqual(part["sources"], [origin["sources"][0], origin["sources"][2]])
self.assertEqual(part["frozen_at"], LATER)
self.assertEqual(part["index"], origin["index"])
self.assertEqual(part["collection"], origin["collection"])
self.assertEqual(part["subset_of"], {"frozen_at": FROZEN,
"manifest_sha256": sha((self.origin / "source-manifest.json").read_bytes())})
self.assertEqual(sorted(p.name for p in (out / "kilde").iterdir()), ["docs-en-bread.md", "release-v1.0.0.md"])
for p in (out / "kilde").iterdir():
self.assertEqual(p.read_bytes(), (self.origin / "kilde" / p.name).read_bytes())
self.assertEqual(stat.S_IMODE(p.stat().st_mode) & 0o222, 0)
self.assertEqual((out / "source-manifest.json").read_text(encoding="utf-8"), sources.dump(part))
self.assertEqual(sources.check(out), [])
def test_the_paths_file_must_be_sorted_unique_and_known(self) -> None:
for lines in (("kilde/release-v1.0.0.md", "kilde/docs-en-bread.md"),
("kilde/docs-en-bread.md", "kilde/docs-en-bread.md"),
("kilde/docs-en-rye.md",), ("docs-en-bread.md",), ("kilde/docs-en-bread.md", ""), ()):
with self.subTest(lines=lines):
out = self.tmp / ("part-" + sha(repr(lines).encode())[:8])
code, _ = self.subset(self.paths(*lines), out)
self.assertEqual(code, 2, lines)
self.assertFalse(out.exists())
def test_an_origin_that_fails_check_cannot_be_subset(self) -> None:
os.chmod(self.origin / "kilde" / "docs-en-bread.md", 0o644)
(self.origin / "kilde" / "docs-en-bread.md").write_text("changed\n", encoding="utf-8")
out = self.tmp / "part"
self.assertEqual(self.subset(self.paths("kilde/docs-en-garden-soil.md"), out)[0], 2)
self.assertFalse(out.exists())
def test_an_existing_out_is_refused(self) -> None:
out = self.tmp / "part"
out.mkdir()
self.assertEqual(self.subset(self.paths("kilde/docs-en-bread.md"), out)[0], 2)
self.assertEqual(list(out.iterdir()), [])
if __name__ == "__main__":
unittest.main()
class FolderBase(Base):
def folder(self, files: dict[str, bytes]) -> Path:
src = self.tmp / "Dokumenter"
for rel, data in files.items():
p = src / rel
p.parent.mkdir(parents=True, exist_ok=True)
p.write_bytes(data)
return src
def run_folder(self, src: Path, out: Path) -> tuple[int, str]:
code, o, e = run_main(["folder", str(src), "--collection", "hage", "--out", str(out), "--frozen-at", FROZEN])
return code, o + e
class FolderTest(FolderBase):
"""`sources folder`: any folder of text documents becomes a frozen copy with no hand-written meta (motor step 2:
names and subfolders, the meta made by code, line breaks and encodings, .txt kept, other files reported)."""
def test_each_document_records_its_language_when_it_can_be_told(self) -> None:
nb = "# Bier\n\nDet er viktig at bikubene står i le, og at de ikke blir for varme om sommeren.\n" * 2
en = "# Bees\n\nIt is important that the hives stand in shelter and that they do not get too hot.\n" * 2
src = self.folder({"bier.md": nb.encode(), "bees.md": en.encode(), "kort.md": b"# Kort\n\nBikuber.\n"})
out = self.tmp / "frozen"
code, text = self.run_folder(src, out)
self.assertEqual(code, 0, text)
by_path = {e["path"]: e for e in manifest(out)["sources"]}
self.assertEqual(by_path["kilde/bier.md"]["lang"], "no")
self.assertEqual(by_path["kilde/bees.md"]["lang"], "en")
self.assertNotIn("lang", by_path["kilde/kort.md"])
self.assertEqual(sources.check(out), [])
def test_subfolders_capitals_and_nordic_letters_get_safe_names_and_keep_their_original(self) -> None:
src = self.folder({"Møtereferat.md": "# Møte\n\nVi sådde.\n".encode(),
"Årlig/Kontroll Plan.md": "# Kontroll\n\nSjekk jorda.\n".encode()})
out = self.tmp / "frozen"
code, text = self.run_folder(src, out)
self.assertEqual(code, 0, text)
doc = manifest(out)
by_path = {e["path"]: e for e in doc["sources"]}
self.assertEqual(sorted(by_path), ["kilde/aarlig__kontroll-plan.md", "kilde/moetereferat.md"])
self.assertEqual(by_path["kilde/moetereferat.md"]["original"], "Møtereferat.md")
self.assertEqual(by_path["kilde/aarlig__kontroll-plan.md"]["original"], "Årlig/Kontroll Plan.md")
self.assertEqual(sources.check(out), [])
def test_a_document_named_like_a_release_note_is_not_read_as_one(self) -> None:
# The release-note rules (order, version, no skeleton, history) go by the kilde/ name; in a folder every
# document has one main topic, so its safe name never has that form (motor B13).
src = self.folder({"Release-v2.1.0.md": b"# Release v2.1.0\n\n## 2.1.0\n\nThe hive scale reads in grams.\n"})
out = self.tmp / "frozen"
code, text = self.run_folder(src, out)
self.assertEqual(code, 0, text)
(entry,) = manifest(out)["sources"]
name = entry["path"][len("kilde/"):]
self.assertIsNone(kblocks.RELEASE.match(name), name)
self.assertTrue(name.startswith("release-v2.1.0"), name)
self.assertEqual(entry["original"], "Release-v2.1.0.md")
self.assertEqual([b.version for b in kblocks.blocks(out / "kilde" / name)], [None])
self.assertEqual(sources.check(out), [])
def test_the_meta_is_made_by_code(self) -> None:
src = self.folder({"a.md": b"# A\n\nOne.\n"})
out = self.tmp / "frozen"
self.assertEqual(self.run_folder(src, out)[0], 0)
doc = manifest(out)
self.assertEqual(doc["collection"], "hage")
self.assertIsNone(doc["subset_of"])
e = doc["sources"][0]
self.assertEqual((e["kind"], e["main_topic"]), ("document", True))
self.assertEqual(e["url"], (src / "a.md").resolve().as_uri())
def test_original_names_are_nfc(self) -> None:
src = self.folder({"Möte.md": b"# M\n\nX.\n"}) # decomposed, as some file systems return it
out = self.tmp / "frozen"
self.assertEqual(self.run_folder(src, out)[0], 0)
self.assertEqual(manifest(out)["sources"][0]["original"], "Möte.md")
def test_windows_line_breaks_latin1_and_bom_are_normalised_and_recorded(self) -> None:
src = self.folder({"crlf.md": b"# A\r\n\r\nOne.\r\n", "latin.txt": "Blåbær og øl.\n".encode("cp1252"),
"bom.md": b"\xef\xbb\xbf# Tittel\n\nTekst.\n", "ff.txt": b"Side en.\x0cSide to.\n"})
out = self.tmp / "frozen"
code, text = self.run_folder(src, out)
self.assertEqual(code, 0, text)
kilde = out / "kilde"
self.assertEqual((kilde / "crlf.md").read_bytes(), b"# A\n\nOne.\n")
self.assertEqual((kilde / "latin.txt").read_text(encoding="utf-8"), "Blåbær og øl.\n")
self.assertEqual((kilde / "bom.md").read_bytes(), b"# Tittel\n\nTekst.\n")
self.assertEqual((kilde / "ff.txt").read_bytes(), b"Side en.\nSide to.\n")
normalised = {e["path"]: e["normalised"] for e in manifest(out)["sources"]}
self.assertEqual(normalised, {"kilde/bom.md": ["bom"], "kilde/crlf.md": ["line-breaks"],
"kilde/ff.txt": ["line-breaks"], "kilde/latin.txt": ["cp1252"]})
def test_a_clean_file_is_copied_byte_for_byte_and_records_nothing_normalised(self) -> None:
src = self.folder({"a.md": b"# A\n\nOne.\n"})
out = self.tmp / "frozen"
self.assertEqual(self.run_folder(src, out)[0], 0)
self.assertEqual((out / "kilde" / "a.md").read_bytes(), b"# A\n\nOne.\n")
self.assertEqual(manifest(out)["sources"][0]["normalised"], [])
def test_other_files_hidden_files_and_empty_files_are_reported_not_dropped_silently(self) -> None:
src = self.folder({"a.md": b"# A\n\nOne.\n", "bilde.png": b"\x89PNG", ".skjult.md": b"# H\n\nX.\n",
".git/config": b"x", "tom.txt": b" \n\n"})
out = self.tmp / "frozen"
code, text = self.run_folder(src, out)
self.assertEqual(code, 0, text)
self.assertEqual([e["path"] for e in manifest(out)["sources"]], ["kilde/a.md"])
self.assertIn("1 sources", text)
self.assertIn("bilde.png: skipped, no reader for .png", text)
self.assertIn(".skjult.md: skipped, hidden", text)
self.assertIn(".git: skipped, hidden", text)
self.assertIn("tom.txt: skipped, empty", text)
def test_two_originals_with_one_safe_name_are_refused(self) -> None:
src = self.folder({"Møte.md": b"# P\n\nX.\n", "moete.md": b"# Q\n\nY.\n"})
out = self.tmp / "frozen"
code, text = self.run_folder(src, out)
self.assertEqual(code, 1, text)
self.assertIn("moete.md: the safe name of Møte.md and moete.md", text)
self.assertFalse(out.exists())
def test_a_folder_without_a_readable_document_cannot_run(self) -> None:
src = self.folder({"x.pdf": b"%PDF"})
code, text = self.run_folder(src, self.tmp / "frozen")
self.assertEqual(code, 2, text)
def test_two_runs_are_byte_identical(self) -> None:
src = self.folder({"Å.md": "# Å\n\nÆ.\n".encode(), "b/c.txt": b"C.\n"})
self.assertEqual(self.run_folder(src, self.tmp / "f1")[0], 0)
self.assertEqual(self.run_folder(src, self.tmp / "f2")[0], 0)
self.assertEqual((self.tmp / "f1" / "source-manifest.json").read_bytes(),
(self.tmp / "f2" / "source-manifest.json").read_bytes())
class FolderOfficeTest(FolderBase):
"""Motor step 3 (A1, B11): PDF and Office files in a folder are frozen as the text a reader sees, with the place of
every line (page, paragraph, cell, slide) in the manifest and the original's sha256 beside it."""
def office_folder(self) -> Path:
from test_readers import docx, pdf, pptx, xlsx
src = self.tmp / "Dokumenter"
(src / "Kurs").mkdir(parents=True)
pdf(src / "Rapport.pdf", ["Side en om bier", "Side to om honning"])
docx(src / "Plan.docx")
xlsx(src / "kuber.xlsx")
pptx(src / "Kurs" / "Våren.pptx")
return src
def test_pdf_and_office_files_are_frozen_as_their_rendering_with_a_place_for_every_line(self) -> None:
src = self.office_folder()
out = self.tmp / "frozen"
code, text = self.run_folder(src, out)
self.assertEqual(code, 0, text)
by_path = {e["path"]: e for e in manifest(out)["sources"]}
self.assertEqual(sorted(by_path), ["kilde/kuber.xlsx.md", "kilde/kurs__vaaren.pptx.md", "kilde/plan.docx.md",
"kilde/rapport.pdf.md"])
e = by_path["kilde/rapport.pdf.md"]
self.assertEqual((e["original"], e["reader"], e["kind"], e["main_topic"]), ("Rapport.pdf", "pdf", "document", True))
self.assertEqual(e["url"], (src / "Rapport.pdf").resolve().as_uri())
self.assertEqual(e["original_sha256"], hashlib.sha256((src / "Rapport.pdf").read_bytes()).hexdigest())
self.assertEqual(e["places"], [{"page": 1}, None, {"page": 2}])
self.assertEqual(e["original_copy"], "original/rapport.pdf")
self.assertEqual((out / "original" / "rapport.pdf").read_bytes(), (src / "Rapport.pdf").read_bytes())
self.assertEqual((out / "kilde" / "rapport.pdf.md").read_text(encoding="utf-8"),
readers.render(src / "Rapport.pdf").text)
for path, entry in by_path.items():
self.assertEqual(len(entry["places"]), entry["lines"], path)
self.assertEqual(by_path["kilde/kuber.xlsx.md"]["places"][2],
{"sheet": "Plan", "cells": [["A1", 0], ["B1", 13]]})
self.assertEqual(sources.check(out), [])
def test_hidden_content_left_out_is_reported_per_file(self) -> None:
code, text = self.run_folder(self.office_folder(), self.tmp / "frozen")
self.assertEqual(code, 0, text)
self.assertIn("kuber.xlsx: 3 hidden parts left out", text)
self.assertIn("Kurs/Våren.pptx: 2 hidden parts left out", text)
self.assertEqual(manifest(self.tmp / "frozen")["sources"][0]["left_out"], 3)
def test_a_broken_file_and_a_pdf_without_text_are_reported_not_dropped_silently(self) -> None:
from test_readers import pdf
src = self.folder({"a.md": b"# A\n\nOne.\n", "brutt.docx": b"not a zip"})
pdf(src / "skannet.pdf", [""])
code, text = self.run_folder(src, self.tmp / "frozen")
self.assertEqual(code, 0, text)
self.assertIn("brutt.docx: skipped, unreadable:", text)
self.assertIn("skannet.pdf: skipped, no text (a scanned page needs text recognition first)", text)
def test_a_pdf_without_the_pinned_library_stops_the_folder_with_the_install_command(self) -> None:
from test_readers import pdf
src = self.folder({"a.md": b"# A\n\nOne.\n"})
pdf(src / "r.pdf", ["Tekst"])
with mock.patch.object(readers, "PDF_PIN", "0.0.1"):
code, text = self.run_folder(src, self.tmp / "frozen")
self.assertEqual(code, 1, text)
self.assertIn("pip install pypdf==0.0.1", text)
self.assertFalse((self.tmp / "frozen").exists())
def test_check_refuses_an_original_copy_that_is_missing_or_changed(self) -> None:
out = self.tmp / "frozen"
self.assertEqual(self.run_folder(self.office_folder(), out)[0], 0)
copy = out / "original" / "plan.docx"
copy.chmod(0o644)
copy.write_bytes(b"changed")
self.assertIn("original/plan.docx: sha256 differs from original_sha256", "; ".join(sources.check(out)))
copy.unlink()
self.assertIn("original/plan.docx: missing", "; ".join(sources.check(out)))
def test_check_refuses_places_that_do_not_match_the_lines(self) -> None:
out = self.tmp / "frozen"
self.assertEqual(self.run_folder(self.office_folder(), out)[0], 0)
doc = manifest(out)
doc["sources"][0]["places"].append(None)
(out / "source-manifest.json").write_text(json.dumps(doc), encoding="utf-8")
self.assertTrue(any("places" in p for p in sources.check(out)), sources.check(out))