"""Eval for the change warning (contract K4): `okfwiki.sources diff` and `marks`, written RED before the code (Step 8). Two synthetic K1 roots, old and new, with planted changes: 2 new, 3 changed (another sha256), 1 gone and 4 unchanged sources, and a synthetic collection whose pages cite them. The denominator is the planted changes (6) and the pages that cite a changed or gone source (3 of the 5 concept pages); it comes from this file, never from the product. Every expectation is the WHOLE output as K4 describes it, and the exit code; two runs must be byte-identical. The commands run as `python3 -m okfwiki.sources` in a child process, exactly as a user runs them. Frozen in wave 2: the session that writes diff and marks never edits this file. Run: python3 -m unittest discover -s tests -p 'test_change_warning.py' """ from __future__ import annotations import hashlib import json import subprocess import sys import tempfile import unittest from pathlib import Path from typing import Any ROOT = Path(__file__).resolve().parents[1] OLD_AT = "2026-10-08T06:30:00Z" NEW_AT = "2026-10-09T06:30:00Z" SITE = "https://example.org/docs/en/" def dump(data: Any) -> str: """K4 item 3: the one written form.""" return json.dumps(data, indent=1, sort_keys=True, ensure_ascii=False) + "\n" def sha(data: bytes) -> str: return hashlib.sha256(data).hexdigest() def kind_of(name: str) -> str: return "release-notes" if name.startswith("release-") else "docs" UNCHANGED = { "docs-en-bread.md": "# Bread\n\nFlour, water and salt.\n", "docs-en-garden-soil.md": "# Soil\n\nCompost feeds the soil.\n", "docs-en-oven.md": "# Oven\n\nHeat the stone for an hour.\n", "release-v1.0.0.md": '\n * First tools\n\n', } CHANGED_OLD = { "docs-en-compost.md": "# Compost\n\nTurn the heap every week.\n", "docs-en-seeds.md": "# Seeds\n\nSow peas in April.\n", "release-v1.1.0.md": '\n * Added rake\n\n', } CHANGED_NEW = { "docs-en-compost.md": "# Compost\n\nTurn the heap every second week.\n", "docs-en-seeds.md": "# Seeds\n\nSow peas in March.\n", "release-v1.1.0.md": '\n * Added rake and hoe\n\n', } GONE = {"docs-en-spade.md": "# Spade\n\nDig before the frost.\n"} NEW = { "docs-en-rake.md": "# Rake\n\nRake the leaves in autumn.\n", "release-v1.2.0.md": '\n * Added watering can\n\n', } OLD_FILES = {**UNCHANGED, **CHANGED_OLD, **GONE} NEW_FILES = {**UNCHANGED, **CHANGED_NEW, **NEW} def write_root(root: Path, files: dict[str, str], frozen_at: str) -> bytes: """A K1 root written by hand (so the eval does not lean on freeze); returns the manifest's bytes.""" (root / "kilde").mkdir(parents=True) entries = [] for name, text in sorted(files.items()): data = text.encode("utf-8") (root / "kilde" / name).write_bytes(data) entry: dict[str, Any] = {"bytes": len(data), "kind": kind_of(name), "lines": len(text.splitlines()), "main_topic": kind_of(name) == "docs", "path": f"kilde/{name}", "sha256": sha(data), "url": SITE + name} if kind_of(name) == "release-notes": entry["version"] = name[len("release-v"):-len(".md")] entries.append(entry) doc = {"collection": "garden", "frozen_at": frozen_at, "schema": "okf-wiki/source-manifest/1", "sources": entries, "subset_of": None} text = dump(doc) (root / "source-manifest.json").write_text(text, encoding="utf-8") return text.encode("utf-8") def concept(title: str, resources: list[str]) -> str: cites = "".join(f" - id: s-{n}\n resource: /kilde/{r}\n title: Notes\n x-locator: {{ line: 3 }}\n" f" x-quote: 'x'\n" for n, r in enumerate(resources, 1)) notes = "".join(f"[^s-{n}]: Notes, line 3\n" for n in range(1, len(resources) + 1)) marks = "".join(f"[^s-{n}]" for n in range(1, len(resources) + 1)) return (f"---\ntype: Concept\ntitle: {title}\ndescription: About {title.lower()}.\nsources:\n{cites}" f"generated: {{ by: claude-code/claude-opus-5-5, at: 2026-10-08T09:00:00Z }}\n---\n\n# {title}\n\n" f"A sentence about {title.lower()}.{marks}\n\n{notes}") PAGES = { "bread.md": concept("Bread", ["docs-en-bread.md"]), # unchanged only: no mark "compost.md": concept("Compost", ["docs-en-garden-soil.md", "docs-en-compost.md"]), # one changed "seeds.md": concept("Seeds", ["release-v1.1.0.md", "docs-en-spade.md", "docs-en-seeds.md"]), # 2 changed, 1 gone "tools.md": concept("Tools", ["docs-en-spade.md", "docs-en-spade.md", "release-v1.0.0.md"]), # gone, cited twice "oven.md": concept("Oven", ["docs-en-oven.md"]), # unchanged only: no mark "index.md": "---\ntype: Index\n---\n\n# Garden\n\n- [Bread](bread.md)\n- [Compost](compost.md)\n", "log.md": "---\ntype: Log\n---\n\n# Log\n\n- 2026-10-08: built\n", } class ChangeWarningEval(unittest.TestCase): def setUp(self) -> None: self._tmp = tempfile.TemporaryDirectory() self.tmp = Path(self._tmp.name) self.old = self.tmp / "old" self.new = self.tmp / "new" self.old_manifest = write_root(self.old, OLD_FILES, OLD_AT) self.new_manifest = write_root(self.new, NEW_FILES, NEW_AT) self.wiki = self.tmp / "wiki" self.wiki.mkdir() for name, text in PAGES.items(): (self.wiki / name).write_text(text, encoding="utf-8") def tearDown(self) -> None: self._tmp.cleanup() def run_cli(self, *args: str) -> tuple[int, bytes]: p = subprocess.run([sys.executable, "-m", "okfwiki.sources", *args], cwd=ROOT, capture_output=True, timeout=120) return p.returncode, p.stdout def expected_diff(self) -> dict[str, Any]: return { "changed": [{"kind": kind_of(n), "new_sha256": sha(CHANGED_NEW[n].encode()), "old_sha256": sha(CHANGED_OLD[n].encode()), "path": f"kilde/{n}"} for n in sorted(CHANGED_OLD)], "gone": [{"kind": "docs", "path": "kilde/docs-en-spade.md"}], "new": {"frozen_at": NEW_AT, "manifest_sha256": sha(self.new_manifest)}, "new_sources": [{"kind": "docs", "path": "kilde/docs-en-rake.md"}, {"kind": "release-notes", "path": "kilde/release-v1.2.0.md"}], "old": {"frozen_at": OLD_AT, "manifest_sha256": sha(self.old_manifest)}, "schema": "okf-wiki/source-diff/1", "unchanged": 4, } EXPECTED_MARKS = { "pages": [ {"page": "compost.md", "sources": [{"change": "changed", "path": "kilde/docs-en-compost.md"}]}, {"page": "seeds.md", "sources": [{"change": "changed", "path": "kilde/docs-en-seeds.md"}, {"change": "gone", "path": "kilde/docs-en-spade.md"}, {"change": "changed", "path": "kilde/release-v1.1.0.md"}]}, {"page": "tools.md", "sources": [{"change": "gone", "path": "kilde/docs-en-spade.md"}]}, ], "schema": "okf-wiki/source-marks/1", } def test_00_the_planted_roots_are_sound_k1(self) -> None: """Precondition, green before Step 8: the eval's own roots pass the K1 check.""" for root in (self.old, self.new): code, out = self.run_cli("check", str(root)) self.assertEqual(code, 0, out.decode()) planted = {"new": len(NEW), "changed": len(CHANGED_OLD), "gone": len(GONE), "unchanged": len(UNCHANGED)} self.assertEqual(planted, {"new": 2, "changed": 3, "gone": 1, "unchanged": 4}) def test_diff_reports_every_planted_change_and_exits_1(self) -> None: code, out = self.run_cli("diff", str(self.old), str(self.new)) self.assertEqual(out.decode("utf-8"), dump(self.expected_diff())) self.assertEqual(code, 1) def test_diff_twice_is_byte_identical(self) -> None: first = self.run_cli("diff", str(self.old), str(self.new)) second = self.run_cli("diff", str(self.old), str(self.new)) self.assertEqual(first[0], 1) self.assertEqual(first, second) def test_diff_of_a_root_against_itself_is_empty_and_exits_0(self) -> None: code, out = self.run_cli("diff", str(self.old), str(self.old)) same = {"frozen_at": OLD_AT, "manifest_sha256": sha(self.old_manifest)} self.assertEqual(out.decode("utf-8"), dump({"changed": [], "gone": [], "new": same, "new_sources": [], "old": same, "schema": "okf-wiki/source-diff/1", "unchanged": 8})) self.assertEqual(code, 0) def test_each_kind_of_change_alone_exits_1(self) -> None: """K4: exit 1 on at least one change, whichever kind it is.""" bread = "docs-en-bread.md" alone = {"new": ({**UNCHANGED, **NEW}, "new_sources", [{"kind": "docs", "path": "kilde/docs-en-rake.md"}, {"kind": "release-notes", "path": "kilde/release-v1.2.0.md"}]), "changed": ({**UNCHANGED, bread: "# Bread\n\nFlour and water.\n"}, "changed", [{"kind": "docs", "new_sha256": sha(b"# Bread\n\nFlour and water.\n"), "old_sha256": sha(UNCHANGED[bread].encode()), "path": f"kilde/{bread}"}]), "gone": ({k: v for k, v in UNCHANGED.items() if k != bread}, "gone", [{"kind": "docs", "path": f"kilde/{bread}"}])} base = self.tmp / "base" base_manifest = write_root(base, UNCHANGED, OLD_AT) for name, (files, key, listed) in alone.items(): with self.subTest(kind=name): root = self.tmp / f"only-{name}" manifest = write_root(root, files, NEW_AT) expected: dict[str, Any] = { "changed": [], "gone": [], "new": {"frozen_at": NEW_AT, "manifest_sha256": sha(manifest)}, "new_sources": [], "old": {"frozen_at": OLD_AT, "manifest_sha256": sha(base_manifest)}, "schema": "okf-wiki/source-diff/1", "unchanged": 3 if name != "new" else 4, key: listed} code, out = self.run_cli("diff", str(base), str(root)) self.assertEqual(out.decode("utf-8"), dump(expected)) self.assertEqual(code, 1) def test_diff_cannot_run_on_a_root_that_fails_the_k1_check(self) -> None: # exit 2 alone would also be an unknown command; the sound pair must run first self.assertEqual(self.run_cli("diff", str(self.old), str(self.new))[0], 1) (self.new / "kilde" / "docs-en-bread.md").write_text("# Bread\n\nFlour and water.\n", encoding="utf-8") self.assertEqual(self.run_cli("diff", str(self.old), str(self.new))[0], 2) self.assertEqual(self.run_cli("diff", str(self.old), str(self.tmp / "nowhere"))[0], 2) def write_diff(self, data: dict[str, Any]) -> Path: p = self.tmp / "diff.json" p.write_text(dump(data), encoding="utf-8") return p def test_marks_lists_the_pages_built_on_changed_or_gone_sources_and_exits_1(self) -> None: code, out = self.run_cli("marks", str(self.wiki), "--diff", str(self.write_diff(self.expected_diff()))) self.assertEqual(out.decode("utf-8"), dump(self.EXPECTED_MARKS)) self.assertEqual(code, 1) def test_marks_twice_is_byte_identical(self) -> None: diff = self.write_diff(self.expected_diff()) first = self.run_cli("marks", str(self.wiki), "--diff", str(diff)) second = self.run_cli("marks", str(self.wiki), "--diff", str(diff)) self.assertEqual(first[0], 1) self.assertEqual(first, second) def test_marks_with_only_new_sources_marks_nothing_and_exits_0(self) -> None: only_new = {**self.expected_diff(), "changed": [], "gone": []} code, out = self.run_cli("marks", str(self.wiki), "--diff", str(self.write_diff(only_new))) self.assertEqual(out.decode("utf-8"), dump({"pages": [], "schema": "okf-wiki/source-marks/1"})) self.assertEqual(code, 0) def test_marks_cannot_run_without_a_readable_diff(self) -> None: # exit 2 alone would also be an unknown command; the sound diff must run first self.assertEqual(self.run_cli("marks", str(self.wiki), "--diff", str(self.write_diff(self.expected_diff())))[0], 1) bad = self.tmp / "bad.json" bad.write_text("{", encoding="utf-8") self.assertEqual(self.run_cli("marks", str(self.wiki), "--diff", str(bad))[0], 2) self.assertEqual(self.run_cli("marks", str(self.wiki), "--diff", str(self.tmp / "none.json"))[0], 2) if __name__ == "__main__": unittest.main()