First public release, published as one snapshot. What changed between versions is in CHANGELOG.md. Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
638 lines
32 KiB
Python
638 lines
32 KiB
Python
"""Tests for okfwiki/sources.py: the source manifest (contract K1) with check, freeze and subset.
|
|
|
|
Every source here is synthetic (bread and gardening); no text from any documentation.
|
|
Run: python3 -m unittest discover -s tests -p 'test_sources.py'
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import io
|
|
import json
|
|
import os
|
|
import stat
|
|
import sys
|
|
import tempfile
|
|
import unittest
|
|
from contextlib import redirect_stderr, redirect_stdout
|
|
from pathlib import Path
|
|
from typing import Any
|
|
from unittest import mock
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT))
|
|
|
|
from okfwiki import kblocks, readers, sources # noqa: E402
|
|
|
|
FROZEN = "2026-10-08T12:00:00Z"
|
|
LATER = "2026-10-09T06:30:00Z"
|
|
FILES = {
|
|
"docs-en-bread.md": "# Bread\n\nFlour, water and salt.\nKnead for ten minutes.\n",
|
|
"docs-en-garden-soil.md": "# Soil\n\nCompost feeds the soil.\n",
|
|
"release-v1.0.0.md": '<Update label="1.0.0" description="May 1, 2026">\n * Added rye\n</Update>\n',
|
|
}
|
|
META_SOURCES: dict[str, dict[str, Any]] = {
|
|
"docs-en-bread.md": {"kind": "docs", "main_topic": True, "url": "https://example.org/docs/en/bread.md"},
|
|
"docs-en-garden-soil.md": {"kind": "docs", "main_topic": True,
|
|
"url": "https://example.org/docs/en/garden/soil.md"},
|
|
"release-v1.0.0.md": {"kind": "release-notes", "main_topic": False,
|
|
"url": "https://example.org/docs/en/changelog#1-0-0", "version": "1.0.0"},
|
|
}
|
|
INDEX = {"pages": 3, "sha256": "a" * 64, "url": "https://example.org/docs/llms.txt"}
|
|
|
|
|
|
def sha(data: bytes) -> str:
|
|
return hashlib.sha256(data).hexdigest()
|
|
|
|
|
|
def run_main(argv: list[str]) -> tuple[int, str, str]:
|
|
out, err = io.StringIO(), io.StringIO()
|
|
with redirect_stdout(out), redirect_stderr(err):
|
|
code = sources.main(argv)
|
|
return code, out.getvalue(), err.getvalue()
|
|
|
|
|
|
def entry(name: str, text: str, **meta: Any) -> dict[str, Any]:
|
|
data = text.encode("utf-8")
|
|
return {"bytes": len(data), "lines": len(text.splitlines()), "path": f"kilde/{name}", "sha256": sha(data), **meta}
|
|
|
|
|
|
def write_root(root: Path, files: dict[str, str], entries: list[dict[str, Any]] | None = None,
|
|
**top: Any) -> Path:
|
|
"""A K1 root written by hand, so check is tested apart from freeze."""
|
|
(root / "kilde").mkdir(parents=True)
|
|
for name, text in files.items():
|
|
(root / "kilde" / name).write_bytes(text.encode("utf-8"))
|
|
if entries is None:
|
|
entries = [entry(n, t, **META_SOURCES[n]) for n, t in sorted(files.items())]
|
|
doc = {"collection": "garden", "frozen_at": FROZEN, "index": INDEX, "schema": "okf-wiki/source-manifest/1",
|
|
"sources": entries, "subset_of": None, **top}
|
|
(root / "source-manifest.json").write_text(sources.dump(doc), encoding="utf-8")
|
|
return root
|
|
|
|
|
|
def manifest(root: Path) -> dict[str, Any]:
|
|
doc: dict[str, Any] = json.loads((root / "source-manifest.json").read_text(encoding="utf-8"))
|
|
return doc
|
|
|
|
|
|
class Base(unittest.TestCase):
|
|
def setUp(self) -> None:
|
|
self._tmp = tempfile.TemporaryDirectory()
|
|
self.tmp = Path(self._tmp.name)
|
|
|
|
def tearDown(self) -> None:
|
|
self._tmp.cleanup()
|
|
|
|
|
|
class DumpTest(unittest.TestCase):
|
|
def test_the_written_form_is_indent_1_sorted_keys_utf8_and_a_final_newline(self) -> None:
|
|
self.assertEqual(sources.dump({"b": "ø", "a": [1]}), '{\n "a": [\n 1\n ],\n "b": "ø"\n}\n')
|
|
|
|
|
|
class LinesTest(unittest.TestCase):
|
|
def test_plain_newlines_are_not_ambiguous(self) -> None:
|
|
for text in ("", "a", "a\n", "a\n\nb\n", "\n\n"):
|
|
self.assertFalse(sources.ambiguous_lines(text), repr(text))
|
|
|
|
def test_any_other_line_break_is_ambiguous(self) -> None:
|
|
for text in ("a\r\nb\n", "a\rb", "a\x0cb\n", "a\x0bb", "a\x1cb", "a\x85b", "a\u2028b", "a\u2029b"):
|
|
self.assertTrue(sources.ambiguous_lines(text), repr(text))
|
|
|
|
|
|
class CheckTest(Base):
|
|
def check(self, root: Path) -> tuple[int, str]:
|
|
code, out, err = run_main(["check", str(root)])
|
|
return code, out + err
|
|
|
|
def test_a_clean_root_is_0(self) -> None:
|
|
root = write_root(self.tmp / "r", FILES)
|
|
self.assertEqual(sources.check(root), [])
|
|
code, text = self.check(root)
|
|
self.assertEqual(code, 0, text)
|
|
self.assertIn("3 sources", text)
|
|
|
|
def test_a_missing_file_is_1_and_named(self) -> None:
|
|
root = write_root(self.tmp / "r", FILES)
|
|
(root / "kilde" / "docs-en-bread.md").unlink()
|
|
code, text = self.check(root)
|
|
self.assertEqual(code, 1)
|
|
self.assertIn("kilde/docs-en-bread.md", text)
|
|
self.assertIn("missing", text)
|
|
|
|
def test_one_changed_byte_is_1(self) -> None:
|
|
root = write_root(self.tmp / "r", FILES)
|
|
p = root / "kilde" / "docs-en-bread.md"
|
|
p.write_bytes(p.read_bytes().replace(b"salt", b"sale"))
|
|
problems = sources.check(root)
|
|
self.assertEqual(len(problems), 1, problems)
|
|
self.assertIn("kilde/docs-en-bread.md", problems[0])
|
|
self.assertIn("sha256", problems[0])
|
|
|
|
def test_a_wrong_byte_or_line_count_is_1(self) -> None:
|
|
good = entry("docs-en-bread.md", FILES["docs-en-bread.md"], **META_SOURCES["docs-en-bread.md"])
|
|
for key in ("bytes", "lines"):
|
|
with self.subTest(key=key):
|
|
root = write_root(self.tmp / key, {"docs-en-bread.md": FILES["docs-en-bread.md"]},
|
|
[{**good, key: good[key] + 1}])
|
|
problems = sources.check(root)
|
|
self.assertEqual(len(problems), 1, problems)
|
|
self.assertIn(key, problems[0])
|
|
|
|
def test_an_unlisted_file_is_1_also_in_a_subfolder(self) -> None:
|
|
root = write_root(self.tmp / "r", FILES)
|
|
(root / "kilde" / "stray.md").write_text("Tomatoes.\n", encoding="utf-8")
|
|
(root / "kilde" / "sub").mkdir()
|
|
(root / "kilde" / "sub" / "deep.md").write_text("Beans.\n", encoding="utf-8")
|
|
problems = sources.check(root)
|
|
self.assertEqual(len(problems), 2, problems)
|
|
self.assertTrue(any("kilde/stray.md" in p and "not listed" in p for p in problems))
|
|
self.assertTrue(any("kilde/sub/deep.md" in p and "not listed" in p for p in problems))
|
|
|
|
def test_a_double_path_is_1(self) -> None:
|
|
e = entry("docs-en-bread.md", FILES["docs-en-bread.md"], **META_SOURCES["docs-en-bread.md"])
|
|
root = write_root(self.tmp / "r", {"docs-en-bread.md": FILES["docs-en-bread.md"]}, [e, dict(e)])
|
|
problems = sources.check(root)
|
|
self.assertTrue(any("kilde/docs-en-bread.md" in p and "more than once" in p for p in problems), problems)
|
|
|
|
def test_an_invalid_main_topic_is_1(self) -> None:
|
|
for bad in ("true", 1, None):
|
|
with self.subTest(bad=bad):
|
|
e = entry("docs-en-bread.md", FILES["docs-en-bread.md"],
|
|
**{**META_SOURCES["docs-en-bread.md"], "main_topic": bad})
|
|
root = write_root(self.tmp / repr(bad), {"docs-en-bread.md": FILES["docs-en-bread.md"]}, [e])
|
|
problems = sources.check(root)
|
|
self.assertEqual(len(problems), 1, problems)
|
|
self.assertIn("main_topic", problems[0])
|
|
|
|
def test_a_missing_main_topic_is_1(self) -> None:
|
|
meta = {k: v for k, v in META_SOURCES["docs-en-bread.md"].items() if k != "main_topic"}
|
|
root = write_root(self.tmp / "r", {"docs-en-bread.md": FILES["docs-en-bread.md"]},
|
|
[entry("docs-en-bread.md", FILES["docs-en-bread.md"], **meta)])
|
|
self.assertTrue(any("main_topic" in p for p in sources.check(root)))
|
|
|
|
def test_release_notes_need_a_version(self) -> None:
|
|
meta = {k: v for k, v in META_SOURCES["release-v1.0.0.md"].items() if k != "version"}
|
|
text = FILES["release-v1.0.0.md"]
|
|
root = write_root(self.tmp / "r", {"release-v1.0.0.md": text}, [entry("release-v1.0.0.md", text, **meta)])
|
|
problems = sources.check(root)
|
|
self.assertEqual(len(problems), 1, problems)
|
|
self.assertIn("version", problems[0])
|
|
|
|
def test_another_kind_is_accepted(self) -> None:
|
|
text = "Seeds sprout in spring.\n"
|
|
root = write_root(self.tmp / "r", {"notes.md": text}, [entry("notes.md", text, kind="pdf-page",
|
|
main_topic=False, url="file:///notes.pdf")])
|
|
self.assertEqual(sources.check(root), [])
|
|
|
|
def test_a_bad_path_is_1(self) -> None:
|
|
for name in ("Bread.md", "br ead.md", "bröd.md"):
|
|
with self.subTest(name=name):
|
|
text = "Rye.\n"
|
|
root = write_root(self.tmp / sha(name.encode()), {name: text},
|
|
[entry(name, text, kind="docs", main_topic=True, url="https://example.org/x")])
|
|
self.assertTrue(any("path" in p for p in sources.check(root)), name)
|
|
|
|
def test_sources_out_of_order_are_1(self) -> None:
|
|
es = [entry(n, t, **META_SOURCES[n]) for n, t in sorted(FILES.items())]
|
|
root = write_root(self.tmp / "r", FILES, list(reversed(es)))
|
|
self.assertTrue(any("sorted" in p for p in sources.check(root)))
|
|
|
|
def test_a_listed_file_with_ambiguous_lines_is_1(self) -> None:
|
|
text = "Rye\x0cbread.\n"
|
|
root = write_root(self.tmp / "r", {"docs-en-rye.md": text},
|
|
[entry("docs-en-rye.md", text, kind="docs", main_topic=True, url="https://example.org/r")])
|
|
problems = sources.check(root)
|
|
self.assertEqual(len(problems), 1, problems)
|
|
self.assertIn("ambiguous", problems[0])
|
|
|
|
def test_frozen_at_and_subset_of_have_their_form(self) -> None:
|
|
for top in ({"frozen_at": "2026-10-08 12:00"}, {"subset_of": {"frozen_at": FROZEN}},
|
|
{"subset_of": "x"}, {"index": {"pages": "3"}}):
|
|
with self.subTest(top=top):
|
|
root = write_root(self.tmp / sha(repr(top).encode()), FILES, **top)
|
|
self.assertEqual(len(sources.check(root)), 1, top)
|
|
|
|
def test_the_index_may_be_absent(self) -> None:
|
|
root = write_root(self.tmp / "r", FILES)
|
|
doc = manifest(root)
|
|
del doc["index"]
|
|
(root / "source-manifest.json").write_text(sources.dump(doc), encoding="utf-8")
|
|
self.assertEqual(sources.check(root), [])
|
|
|
|
def test_unknown_keys_are_ignored(self) -> None:
|
|
es = [{**entry(n, t, **META_SOURCES[n]), "x-later": 1} for n, t in sorted(FILES.items())]
|
|
root = write_root(self.tmp / "r", FILES, es, **{"x-later": True})
|
|
self.assertEqual(sources.check(root), [])
|
|
|
|
def test_no_manifest_or_an_unreadable_one_cannot_run(self) -> None:
|
|
root = write_root(self.tmp / "r", FILES)
|
|
for text in (None, "{", "[]", sources.dump({"schema": "other/1", "sources": []}),
|
|
sources.dump({"schema": "okf-wiki/source-manifest/1", "sources": {}})):
|
|
with self.subTest(text=text):
|
|
m = root / "source-manifest.json"
|
|
if text is None:
|
|
m.unlink(missing_ok=True)
|
|
else:
|
|
m.write_text(text, encoding="utf-8")
|
|
code, _ = self.check(root)
|
|
self.assertEqual(code, 2)
|
|
self.assertEqual(self.check(self.tmp / "nowhere")[0], 2)
|
|
|
|
|
|
class FreezeTest(Base):
|
|
def staging(self, files: dict[str, str] | None = None, meta_sources: dict[str, Any] | None = None,
|
|
**meta_top: Any) -> tuple[Path, Path]:
|
|
staging = self.tmp / "staging"
|
|
staging.mkdir()
|
|
for name, text in (FILES if files is None else files).items():
|
|
(staging / name).write_bytes(text.encode("utf-8"))
|
|
meta = {"collection": "garden", "index": INDEX, "schema": "okf-wiki/source-meta/1",
|
|
"sources": META_SOURCES if meta_sources is None else meta_sources, **meta_top}
|
|
meta_path = self.tmp / "meta.json"
|
|
meta_path.write_text(sources.dump(meta), encoding="utf-8")
|
|
return staging, meta_path
|
|
|
|
def freeze(self, staging: Path, meta: Path, out: Path, frozen_at: str = FROZEN) -> tuple[int, str]:
|
|
code, o, e = run_main(["freeze", str(staging), "--meta", str(meta), "--out", str(out),
|
|
"--frozen-at", frozen_at])
|
|
return code, o + e
|
|
|
|
def test_freeze_writes_the_k1_manifest_and_read_only_copies(self) -> None:
|
|
staging, meta = self.staging()
|
|
out = self.tmp / "frozen"
|
|
code, text = self.freeze(staging, meta, out)
|
|
self.assertEqual(code, 0, text)
|
|
expected = {"collection": "garden", "frozen_at": FROZEN, "index": INDEX,
|
|
"schema": "okf-wiki/source-manifest/1", "subset_of": None,
|
|
"sources": [entry(n, t, **META_SOURCES[n]) for n, t in sorted(FILES.items())]}
|
|
self.assertEqual((out / "source-manifest.json").read_text(encoding="utf-8"), sources.dump(expected))
|
|
self.assertEqual(sorted(p.name for p in (out / "kilde").iterdir()), sorted(FILES))
|
|
for name, text in FILES.items():
|
|
p = out / "kilde" / name
|
|
self.assertEqual(p.read_bytes(), text.encode("utf-8"))
|
|
self.assertEqual(stat.S_IMODE(p.stat().st_mode) & 0o222, 0, f"{name} is writable")
|
|
self.assertEqual(sources.check(out), [])
|
|
self.assertEqual(run_main(["check", str(out)])[0], 0)
|
|
|
|
def test_two_freezes_of_the_same_staging_are_byte_identical(self) -> None:
|
|
staging, meta = self.staging()
|
|
self.assertEqual(self.freeze(staging, meta, self.tmp / "a")[0], 0)
|
|
self.assertEqual(self.freeze(staging, meta, self.tmp / "b")[0], 0)
|
|
self.assertEqual((self.tmp / "a" / "source-manifest.json").read_bytes(),
|
|
(self.tmp / "b" / "source-manifest.json").read_bytes())
|
|
|
|
def test_the_index_is_left_out_when_the_meta_has_none(self) -> None:
|
|
staging, meta = self.staging()
|
|
doc = json.loads(meta.read_text(encoding="utf-8"))
|
|
del doc["index"]
|
|
meta.write_text(sources.dump(doc), encoding="utf-8")
|
|
out = self.tmp / "frozen"
|
|
self.assertEqual(self.freeze(staging, meta, out)[0], 0)
|
|
self.assertNotIn("index", manifest(out))
|
|
self.assertEqual(sources.check(out), [])
|
|
|
|
def test_an_existing_out_is_refused_and_left_alone(self) -> None:
|
|
staging, meta = self.staging()
|
|
out = self.tmp / "frozen"
|
|
out.mkdir()
|
|
(out / "keep.txt").write_text("mine\n", encoding="utf-8")
|
|
code, text = self.freeze(staging, meta, out)
|
|
self.assertEqual(code, 2)
|
|
self.assertIn("exists", text)
|
|
self.assertEqual([p.name for p in out.iterdir()], ["keep.txt"])
|
|
|
|
def test_ambiguous_lines_are_refused_and_listed(self) -> None:
|
|
files = {**FILES, "docs-en-bread.md": "# Bread\r\n\r\nFlour.\r\n", "docs-en-garden-soil.md": "Soil\u2028mix\n"}
|
|
staging, meta = self.staging(files)
|
|
out = self.tmp / "frozen"
|
|
code, text = self.freeze(staging, meta, out)
|
|
self.assertEqual(code, 1)
|
|
self.assertIn("docs-en-bread.md", text)
|
|
self.assertIn("docs-en-garden-soil.md", text)
|
|
self.assertIn("ambiguous", text)
|
|
self.assertFalse(out.exists())
|
|
|
|
def test_staging_and_meta_must_list_the_same_files(self) -> None:
|
|
meta_sources = {**META_SOURCES, "docs-en-ghost.md": {"kind": "docs", "main_topic": True,
|
|
"url": "https://example.org/ghost"}}
|
|
staging, meta = self.staging(meta_sources=meta_sources)
|
|
(staging / "stray.md").write_text("Weeds.\n", encoding="utf-8")
|
|
out = self.tmp / "frozen"
|
|
code, text = self.freeze(staging, meta, out)
|
|
self.assertEqual(code, 1)
|
|
self.assertIn("docs-en-ghost.md", text)
|
|
self.assertIn("stray.md", text)
|
|
self.assertFalse(out.exists())
|
|
|
|
def test_an_invalid_entry_in_the_meta_is_refused(self) -> None:
|
|
for name, bad in (("docs-en-bread.md", {"main_topic": "yes"}), ("release-v1.0.0.md", {"version": None}),
|
|
("docs-en-bread.md", {"url": ""}), ("docs-en-bread.md", {"kind": ""})):
|
|
with self.subTest(bad=bad):
|
|
self._tmp.cleanup()
|
|
self._tmp = tempfile.TemporaryDirectory()
|
|
self.tmp = Path(self._tmp.name)
|
|
meta_sources = {**META_SOURCES, name: {**META_SOURCES[name], **bad}}
|
|
staging, meta = self.staging(meta_sources=meta_sources)
|
|
out = self.tmp / "frozen"
|
|
code, text = self.freeze(staging, meta, out)
|
|
self.assertEqual(code, 1, text)
|
|
self.assertIn(name, text)
|
|
self.assertFalse(out.exists())
|
|
|
|
def test_a_bad_file_name_or_a_non_utf8_file_is_refused(self) -> None:
|
|
staging, meta = self.staging()
|
|
(staging / "docs-en-bread.md").write_bytes(b"Bread \xff\n")
|
|
out = self.tmp / "frozen"
|
|
code, text = self.freeze(staging, meta, out)
|
|
self.assertEqual(code, 1)
|
|
self.assertIn("UTF-8", text)
|
|
self.assertFalse(out.exists())
|
|
|
|
def test_a_bad_frozen_at_cannot_run(self) -> None:
|
|
staging, meta = self.staging()
|
|
self.assertEqual(self.freeze(staging, meta, self.tmp / "frozen", "2026-10-08")[0], 2)
|
|
self.assertFalse((self.tmp / "frozen").exists())
|
|
|
|
def test_frozen_at_defaults_to_now_in_utc(self) -> None:
|
|
staging, meta = self.staging()
|
|
out = self.tmp / "frozen"
|
|
code, o, e = run_main(["freeze", str(staging), "--meta", str(meta), "--out", str(out)])
|
|
self.assertEqual(code, 0, o + e)
|
|
self.assertRegex(manifest(out)["frozen_at"], r"^\d{4}-\d\d-\d\dT\d\d:\d\d:\d\dZ$")
|
|
|
|
|
|
class SubsetTest(Base):
|
|
def setUp(self) -> None:
|
|
super().setUp()
|
|
self.origin = write_root(self.tmp / "origin", FILES)
|
|
|
|
def paths(self, *lines: str) -> Path:
|
|
p = self.tmp / "paths.txt"
|
|
p.write_text("".join(f"{x}\n" for x in lines), encoding="utf-8")
|
|
return p
|
|
|
|
def subset(self, paths: Path, out: Path, origin: Path | None = None) -> tuple[int, str]:
|
|
code, o, e = run_main(["subset", str(origin or self.origin), "--paths", str(paths), "--out", str(out),
|
|
"--frozen-at", LATER])
|
|
return code, o + e
|
|
|
|
def test_a_subset_keeps_the_entries_and_points_at_its_origin(self) -> None:
|
|
out = self.tmp / "part"
|
|
code, text = self.subset(self.paths("kilde/docs-en-bread.md", "kilde/release-v1.0.0.md"), out)
|
|
self.assertEqual(code, 0, text)
|
|
origin = manifest(self.origin)
|
|
part = manifest(out)
|
|
self.assertEqual(part["sources"], [origin["sources"][0], origin["sources"][2]])
|
|
self.assertEqual(part["frozen_at"], LATER)
|
|
self.assertEqual(part["index"], origin["index"])
|
|
self.assertEqual(part["collection"], origin["collection"])
|
|
self.assertEqual(part["subset_of"], {"frozen_at": FROZEN,
|
|
"manifest_sha256": sha((self.origin / "source-manifest.json").read_bytes())})
|
|
self.assertEqual(sorted(p.name for p in (out / "kilde").iterdir()), ["docs-en-bread.md", "release-v1.0.0.md"])
|
|
for p in (out / "kilde").iterdir():
|
|
self.assertEqual(p.read_bytes(), (self.origin / "kilde" / p.name).read_bytes())
|
|
self.assertEqual(stat.S_IMODE(p.stat().st_mode) & 0o222, 0)
|
|
self.assertEqual((out / "source-manifest.json").read_text(encoding="utf-8"), sources.dump(part))
|
|
self.assertEqual(sources.check(out), [])
|
|
|
|
def test_the_paths_file_must_be_sorted_unique_and_known(self) -> None:
|
|
for lines in (("kilde/release-v1.0.0.md", "kilde/docs-en-bread.md"),
|
|
("kilde/docs-en-bread.md", "kilde/docs-en-bread.md"),
|
|
("kilde/docs-en-rye.md",), ("docs-en-bread.md",), ("kilde/docs-en-bread.md", ""), ()):
|
|
with self.subTest(lines=lines):
|
|
out = self.tmp / ("part-" + sha(repr(lines).encode())[:8])
|
|
code, _ = self.subset(self.paths(*lines), out)
|
|
self.assertEqual(code, 2, lines)
|
|
self.assertFalse(out.exists())
|
|
|
|
def test_an_origin_that_fails_check_cannot_be_subset(self) -> None:
|
|
os.chmod(self.origin / "kilde" / "docs-en-bread.md", 0o644)
|
|
(self.origin / "kilde" / "docs-en-bread.md").write_text("changed\n", encoding="utf-8")
|
|
out = self.tmp / "part"
|
|
self.assertEqual(self.subset(self.paths("kilde/docs-en-garden-soil.md"), out)[0], 2)
|
|
self.assertFalse(out.exists())
|
|
|
|
def test_an_existing_out_is_refused(self) -> None:
|
|
out = self.tmp / "part"
|
|
out.mkdir()
|
|
self.assertEqual(self.subset(self.paths("kilde/docs-en-bread.md"), out)[0], 2)
|
|
self.assertEqual(list(out.iterdir()), [])
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|
|
|
|
|
|
class FolderBase(Base):
|
|
def folder(self, files: dict[str, bytes]) -> Path:
|
|
src = self.tmp / "Dokumenter"
|
|
for rel, data in files.items():
|
|
p = src / rel
|
|
p.parent.mkdir(parents=True, exist_ok=True)
|
|
p.write_bytes(data)
|
|
return src
|
|
|
|
def run_folder(self, src: Path, out: Path) -> tuple[int, str]:
|
|
code, o, e = run_main(["folder", str(src), "--collection", "hage", "--out", str(out), "--frozen-at", FROZEN])
|
|
return code, o + e
|
|
|
|
|
|
class FolderTest(FolderBase):
|
|
"""`sources folder`: any folder of text documents becomes a frozen copy with no hand-written meta (motor step 2:
|
|
names and subfolders, the meta made by code, line breaks and encodings, .txt kept, other files reported)."""
|
|
|
|
def test_each_document_records_its_language_when_it_can_be_told(self) -> None:
|
|
nb = "# Bier\n\nDet er viktig at bikubene står i le, og at de ikke blir for varme om sommeren.\n" * 2
|
|
en = "# Bees\n\nIt is important that the hives stand in shelter and that they do not get too hot.\n" * 2
|
|
src = self.folder({"bier.md": nb.encode(), "bees.md": en.encode(), "kort.md": b"# Kort\n\nBikuber.\n"})
|
|
out = self.tmp / "frozen"
|
|
code, text = self.run_folder(src, out)
|
|
self.assertEqual(code, 0, text)
|
|
by_path = {e["path"]: e for e in manifest(out)["sources"]}
|
|
self.assertEqual(by_path["kilde/bier.md"]["lang"], "no")
|
|
self.assertEqual(by_path["kilde/bees.md"]["lang"], "en")
|
|
self.assertNotIn("lang", by_path["kilde/kort.md"])
|
|
self.assertEqual(sources.check(out), [])
|
|
|
|
def test_subfolders_capitals_and_nordic_letters_get_safe_names_and_keep_their_original(self) -> None:
|
|
src = self.folder({"Møtereferat.md": "# Møte\n\nVi sådde.\n".encode(),
|
|
"Årlig/Kontroll Plan.md": "# Kontroll\n\nSjekk jorda.\n".encode()})
|
|
out = self.tmp / "frozen"
|
|
code, text = self.run_folder(src, out)
|
|
self.assertEqual(code, 0, text)
|
|
doc = manifest(out)
|
|
by_path = {e["path"]: e for e in doc["sources"]}
|
|
self.assertEqual(sorted(by_path), ["kilde/aarlig__kontroll-plan.md", "kilde/moetereferat.md"])
|
|
self.assertEqual(by_path["kilde/moetereferat.md"]["original"], "Møtereferat.md")
|
|
self.assertEqual(by_path["kilde/aarlig__kontroll-plan.md"]["original"], "Årlig/Kontroll Plan.md")
|
|
self.assertEqual(sources.check(out), [])
|
|
|
|
def test_a_document_named_like_a_release_note_is_not_read_as_one(self) -> None:
|
|
# The release-note rules (order, version, no skeleton, history) go by the kilde/ name; in a folder every
|
|
# document has one main topic, so its safe name never has that form (motor B13).
|
|
src = self.folder({"Release-v2.1.0.md": b"# Release v2.1.0\n\n## 2.1.0\n\nThe hive scale reads in grams.\n"})
|
|
out = self.tmp / "frozen"
|
|
code, text = self.run_folder(src, out)
|
|
self.assertEqual(code, 0, text)
|
|
(entry,) = manifest(out)["sources"]
|
|
name = entry["path"][len("kilde/"):]
|
|
self.assertIsNone(kblocks.RELEASE.match(name), name)
|
|
self.assertTrue(name.startswith("release-v2.1.0"), name)
|
|
self.assertEqual(entry["original"], "Release-v2.1.0.md")
|
|
self.assertEqual([b.version for b in kblocks.blocks(out / "kilde" / name)], [None])
|
|
self.assertEqual(sources.check(out), [])
|
|
|
|
def test_the_meta_is_made_by_code(self) -> None:
|
|
src = self.folder({"a.md": b"# A\n\nOne.\n"})
|
|
out = self.tmp / "frozen"
|
|
self.assertEqual(self.run_folder(src, out)[0], 0)
|
|
doc = manifest(out)
|
|
self.assertEqual(doc["collection"], "hage")
|
|
self.assertIsNone(doc["subset_of"])
|
|
e = doc["sources"][0]
|
|
self.assertEqual((e["kind"], e["main_topic"]), ("document", True))
|
|
self.assertEqual(e["url"], (src / "a.md").resolve().as_uri())
|
|
|
|
def test_original_names_are_nfc(self) -> None:
|
|
src = self.folder({"Möte.md": b"# M\n\nX.\n"}) # decomposed, as some file systems return it
|
|
out = self.tmp / "frozen"
|
|
self.assertEqual(self.run_folder(src, out)[0], 0)
|
|
self.assertEqual(manifest(out)["sources"][0]["original"], "Möte.md")
|
|
|
|
def test_windows_line_breaks_latin1_and_bom_are_normalised_and_recorded(self) -> None:
|
|
src = self.folder({"crlf.md": b"# A\r\n\r\nOne.\r\n", "latin.txt": "Blåbær og øl.\n".encode("cp1252"),
|
|
"bom.md": b"\xef\xbb\xbf# Tittel\n\nTekst.\n", "ff.txt": b"Side en.\x0cSide to.\n"})
|
|
out = self.tmp / "frozen"
|
|
code, text = self.run_folder(src, out)
|
|
self.assertEqual(code, 0, text)
|
|
kilde = out / "kilde"
|
|
self.assertEqual((kilde / "crlf.md").read_bytes(), b"# A\n\nOne.\n")
|
|
self.assertEqual((kilde / "latin.txt").read_text(encoding="utf-8"), "Blåbær og øl.\n")
|
|
self.assertEqual((kilde / "bom.md").read_bytes(), b"# Tittel\n\nTekst.\n")
|
|
self.assertEqual((kilde / "ff.txt").read_bytes(), b"Side en.\nSide to.\n")
|
|
normalised = {e["path"]: e["normalised"] for e in manifest(out)["sources"]}
|
|
self.assertEqual(normalised, {"kilde/bom.md": ["bom"], "kilde/crlf.md": ["line-breaks"],
|
|
"kilde/ff.txt": ["line-breaks"], "kilde/latin.txt": ["cp1252"]})
|
|
|
|
def test_a_clean_file_is_copied_byte_for_byte_and_records_nothing_normalised(self) -> None:
|
|
src = self.folder({"a.md": b"# A\n\nOne.\n"})
|
|
out = self.tmp / "frozen"
|
|
self.assertEqual(self.run_folder(src, out)[0], 0)
|
|
self.assertEqual((out / "kilde" / "a.md").read_bytes(), b"# A\n\nOne.\n")
|
|
self.assertEqual(manifest(out)["sources"][0]["normalised"], [])
|
|
|
|
def test_other_files_hidden_files_and_empty_files_are_reported_not_dropped_silently(self) -> None:
|
|
src = self.folder({"a.md": b"# A\n\nOne.\n", "bilde.png": b"\x89PNG", ".skjult.md": b"# H\n\nX.\n",
|
|
".git/config": b"x", "tom.txt": b" \n\n"})
|
|
out = self.tmp / "frozen"
|
|
code, text = self.run_folder(src, out)
|
|
self.assertEqual(code, 0, text)
|
|
self.assertEqual([e["path"] for e in manifest(out)["sources"]], ["kilde/a.md"])
|
|
self.assertIn("1 sources", text)
|
|
self.assertIn("bilde.png: skipped, no reader for .png", text)
|
|
self.assertIn(".skjult.md: skipped, hidden", text)
|
|
self.assertIn(".git: skipped, hidden", text)
|
|
self.assertIn("tom.txt: skipped, empty", text)
|
|
|
|
def test_two_originals_with_one_safe_name_are_refused(self) -> None:
|
|
src = self.folder({"Møte.md": b"# P\n\nX.\n", "moete.md": b"# Q\n\nY.\n"})
|
|
out = self.tmp / "frozen"
|
|
code, text = self.run_folder(src, out)
|
|
self.assertEqual(code, 1, text)
|
|
self.assertIn("moete.md: the safe name of Møte.md and moete.md", text)
|
|
self.assertFalse(out.exists())
|
|
|
|
def test_a_folder_without_a_readable_document_cannot_run(self) -> None:
|
|
src = self.folder({"x.pdf": b"%PDF"})
|
|
code, text = self.run_folder(src, self.tmp / "frozen")
|
|
self.assertEqual(code, 2, text)
|
|
|
|
def test_two_runs_are_byte_identical(self) -> None:
|
|
src = self.folder({"Å.md": "# Å\n\nÆ.\n".encode(), "b/c.txt": b"C.\n"})
|
|
self.assertEqual(self.run_folder(src, self.tmp / "f1")[0], 0)
|
|
self.assertEqual(self.run_folder(src, self.tmp / "f2")[0], 0)
|
|
self.assertEqual((self.tmp / "f1" / "source-manifest.json").read_bytes(),
|
|
(self.tmp / "f2" / "source-manifest.json").read_bytes())
|
|
|
|
|
|
class FolderOfficeTest(FolderBase):
|
|
"""Motor step 3 (A1, B11): PDF and Office files in a folder are frozen as the text a reader sees, with the place of
|
|
every line (page, paragraph, cell, slide) in the manifest and the original's sha256 beside it."""
|
|
|
|
def office_folder(self) -> Path:
|
|
from test_readers import docx, pdf, pptx, xlsx
|
|
src = self.tmp / "Dokumenter"
|
|
(src / "Kurs").mkdir(parents=True)
|
|
pdf(src / "Rapport.pdf", ["Side en om bier", "Side to om honning"])
|
|
docx(src / "Plan.docx")
|
|
xlsx(src / "kuber.xlsx")
|
|
pptx(src / "Kurs" / "Våren.pptx")
|
|
return src
|
|
|
|
def test_pdf_and_office_files_are_frozen_as_their_rendering_with_a_place_for_every_line(self) -> None:
|
|
src = self.office_folder()
|
|
out = self.tmp / "frozen"
|
|
code, text = self.run_folder(src, out)
|
|
self.assertEqual(code, 0, text)
|
|
by_path = {e["path"]: e for e in manifest(out)["sources"]}
|
|
self.assertEqual(sorted(by_path), ["kilde/kuber.xlsx.md", "kilde/kurs__vaaren.pptx.md", "kilde/plan.docx.md",
|
|
"kilde/rapport.pdf.md"])
|
|
e = by_path["kilde/rapport.pdf.md"]
|
|
self.assertEqual((e["original"], e["reader"], e["kind"], e["main_topic"]), ("Rapport.pdf", "pdf", "document", True))
|
|
self.assertEqual(e["url"], (src / "Rapport.pdf").resolve().as_uri())
|
|
self.assertEqual(e["original_sha256"], hashlib.sha256((src / "Rapport.pdf").read_bytes()).hexdigest())
|
|
self.assertEqual(e["places"], [{"page": 1}, None, {"page": 2}])
|
|
self.assertEqual(e["original_copy"], "original/rapport.pdf")
|
|
self.assertEqual((out / "original" / "rapport.pdf").read_bytes(), (src / "Rapport.pdf").read_bytes())
|
|
self.assertEqual((out / "kilde" / "rapport.pdf.md").read_text(encoding="utf-8"),
|
|
readers.render(src / "Rapport.pdf").text)
|
|
for path, entry in by_path.items():
|
|
self.assertEqual(len(entry["places"]), entry["lines"], path)
|
|
self.assertEqual(by_path["kilde/kuber.xlsx.md"]["places"][2],
|
|
{"sheet": "Plan", "cells": [["A1", 0], ["B1", 13]]})
|
|
self.assertEqual(sources.check(out), [])
|
|
|
|
def test_hidden_content_left_out_is_reported_per_file(self) -> None:
|
|
code, text = self.run_folder(self.office_folder(), self.tmp / "frozen")
|
|
self.assertEqual(code, 0, text)
|
|
self.assertIn("kuber.xlsx: 3 hidden parts left out", text)
|
|
self.assertIn("Kurs/Våren.pptx: 2 hidden parts left out", text)
|
|
self.assertEqual(manifest(self.tmp / "frozen")["sources"][0]["left_out"], 3)
|
|
|
|
def test_a_broken_file_and_a_pdf_without_text_are_reported_not_dropped_silently(self) -> None:
|
|
from test_readers import pdf
|
|
src = self.folder({"a.md": b"# A\n\nOne.\n", "brutt.docx": b"not a zip"})
|
|
pdf(src / "skannet.pdf", [""])
|
|
code, text = self.run_folder(src, self.tmp / "frozen")
|
|
self.assertEqual(code, 0, text)
|
|
self.assertIn("brutt.docx: skipped, unreadable:", text)
|
|
self.assertIn("skannet.pdf: skipped, no text (a scanned page needs text recognition first)", text)
|
|
|
|
def test_a_pdf_without_the_pinned_library_stops_the_folder_with_the_install_command(self) -> None:
|
|
from test_readers import pdf
|
|
src = self.folder({"a.md": b"# A\n\nOne.\n"})
|
|
pdf(src / "r.pdf", ["Tekst"])
|
|
with mock.patch.object(readers, "PDF_PIN", "0.0.1"):
|
|
code, text = self.run_folder(src, self.tmp / "frozen")
|
|
self.assertEqual(code, 1, text)
|
|
self.assertIn("pip install pypdf==0.0.1", text)
|
|
self.assertFalse((self.tmp / "frozen").exists())
|
|
|
|
def test_check_refuses_an_original_copy_that_is_missing_or_changed(self) -> None:
|
|
out = self.tmp / "frozen"
|
|
self.assertEqual(self.run_folder(self.office_folder(), out)[0], 0)
|
|
copy = out / "original" / "plan.docx"
|
|
copy.chmod(0o644)
|
|
copy.write_bytes(b"changed")
|
|
self.assertIn("original/plan.docx: sha256 differs from original_sha256", "; ".join(sources.check(out)))
|
|
copy.unlink()
|
|
self.assertIn("original/plan.docx: missing", "; ".join(sources.check(out)))
|
|
|
|
def test_check_refuses_places_that_do_not_match_the_lines(self) -> None:
|
|
out = self.tmp / "frozen"
|
|
self.assertEqual(self.run_folder(self.office_folder(), out)[0], 0)
|
|
doc = manifest(out)
|
|
doc["sources"][0]["places"].append(None)
|
|
(out / "source-manifest.json").write_text(json.dumps(doc), encoding="utf-8")
|
|
self.assertTrue(any("places" in p for p in sources.check(out)), sources.check(out))
|