okf-wiki/tests/test_change_warning.py
Kjell Tore Guttormsen 0fd86a23a3
okf-wiki 1.1.1
First public release, published as one snapshot. What changed between
versions is in CHANGELOG.md.

Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
2026-11-14 21:03:20 +01:00

245 lines
13 KiB
Python

"""Eval for the change warning (contract K4): `okfwiki.sources diff` and `marks`, written RED before the code (Step 8).
Two synthetic K1 roots, old and new, with planted changes: 2 new, 3 changed (another sha256), 1 gone and 4 unchanged
sources, and a synthetic collection whose pages cite them. The denominator is the planted changes (6) and the pages
that cite a changed or gone source (3 of the 5 concept pages); it comes from this file, never from the product. Every
expectation is the WHOLE output as K4 describes it, and the exit code; two runs must be byte-identical. The commands
run as `python3 -m okfwiki.sources` in a child process, exactly as a user runs them.
Frozen in wave 2: the session that writes diff and marks never edits this file.
Run: python3 -m unittest discover -s tests -p 'test_change_warning.py'
"""
from __future__ import annotations
import hashlib
import json
import subprocess
import sys
import tempfile
import unittest
from pathlib import Path
from typing import Any
ROOT = Path(__file__).resolve().parents[1]
OLD_AT = "2026-10-08T06:30:00Z"
NEW_AT = "2026-10-09T06:30:00Z"
SITE = "https://example.org/docs/en/"
def dump(data: Any) -> str:
"""K4 item 3: the one written form."""
return json.dumps(data, indent=1, sort_keys=True, ensure_ascii=False) + "\n"
def sha(data: bytes) -> str:
return hashlib.sha256(data).hexdigest()
def kind_of(name: str) -> str:
return "release-notes" if name.startswith("release-") else "docs"
UNCHANGED = {
"docs-en-bread.md": "# Bread\n\nFlour, water and salt.\n",
"docs-en-garden-soil.md": "# Soil\n\nCompost feeds the soil.\n",
"docs-en-oven.md": "# Oven\n\nHeat the stone for an hour.\n",
"release-v1.0.0.md": '<Update label="1.0.0" description="May 1, 2026">\n * First tools\n</Update>\n',
}
CHANGED_OLD = {
"docs-en-compost.md": "# Compost\n\nTurn the heap every week.\n",
"docs-en-seeds.md": "# Seeds\n\nSow peas in April.\n",
"release-v1.1.0.md": '<Update label="1.1.0" description="June 2, 2026">\n * Added rake\n</Update>\n',
}
CHANGED_NEW = {
"docs-en-compost.md": "# Compost\n\nTurn the heap every second week.\n",
"docs-en-seeds.md": "# Seeds\n\nSow peas in March.\n",
"release-v1.1.0.md": '<Update label="1.1.0" description="June 2, 2026">\n * Added rake and hoe\n</Update>\n',
}
GONE = {"docs-en-spade.md": "# Spade\n\nDig before the frost.\n"}
NEW = {
"docs-en-rake.md": "# Rake\n\nRake the leaves in autumn.\n",
"release-v1.2.0.md": '<Update label="1.2.0" description="July 3, 2026">\n * Added watering can\n</Update>\n',
}
OLD_FILES = {**UNCHANGED, **CHANGED_OLD, **GONE}
NEW_FILES = {**UNCHANGED, **CHANGED_NEW, **NEW}
def write_root(root: Path, files: dict[str, str], frozen_at: str) -> bytes:
"""A K1 root written by hand (so the eval does not lean on freeze); returns the manifest's bytes."""
(root / "kilde").mkdir(parents=True)
entries = []
for name, text in sorted(files.items()):
data = text.encode("utf-8")
(root / "kilde" / name).write_bytes(data)
entry: dict[str, Any] = {"bytes": len(data), "kind": kind_of(name), "lines": len(text.splitlines()),
"main_topic": kind_of(name) == "docs", "path": f"kilde/{name}", "sha256": sha(data),
"url": SITE + name}
if kind_of(name) == "release-notes":
entry["version"] = name[len("release-v"):-len(".md")]
entries.append(entry)
doc = {"collection": "garden", "frozen_at": frozen_at, "schema": "okf-wiki/source-manifest/1",
"sources": entries, "subset_of": None}
text = dump(doc)
(root / "source-manifest.json").write_text(text, encoding="utf-8")
return text.encode("utf-8")
def concept(title: str, resources: list[str]) -> str:
cites = "".join(f" - id: s-{n}\n resource: /kilde/{r}\n title: Notes\n x-locator: {{ line: 3 }}\n"
f" x-quote: 'x'\n" for n, r in enumerate(resources, 1))
notes = "".join(f"[^s-{n}]: Notes, line 3\n" for n in range(1, len(resources) + 1))
marks = "".join(f"[^s-{n}]" for n in range(1, len(resources) + 1))
return (f"---\ntype: Concept\ntitle: {title}\ndescription: About {title.lower()}.\nsources:\n{cites}"
f"generated: {{ by: claude-code/claude-opus-5-5, at: 2026-10-08T09:00:00Z }}\n---\n\n# {title}\n\n"
f"A sentence about {title.lower()}.{marks}\n\n{notes}")
PAGES = {
"bread.md": concept("Bread", ["docs-en-bread.md"]), # unchanged only: no mark
"compost.md": concept("Compost", ["docs-en-garden-soil.md", "docs-en-compost.md"]), # one changed
"seeds.md": concept("Seeds", ["release-v1.1.0.md", "docs-en-spade.md", "docs-en-seeds.md"]), # 2 changed, 1 gone
"tools.md": concept("Tools", ["docs-en-spade.md", "docs-en-spade.md", "release-v1.0.0.md"]), # gone, cited twice
"oven.md": concept("Oven", ["docs-en-oven.md"]), # unchanged only: no mark
"index.md": "---\ntype: Index\n---\n\n# Garden\n\n- [Bread](bread.md)\n- [Compost](compost.md)\n",
"log.md": "---\ntype: Log\n---\n\n# Log\n\n- 2026-10-08: built\n",
}
class ChangeWarningEval(unittest.TestCase):
def setUp(self) -> None:
self._tmp = tempfile.TemporaryDirectory()
self.tmp = Path(self._tmp.name)
self.old = self.tmp / "old"
self.new = self.tmp / "new"
self.old_manifest = write_root(self.old, OLD_FILES, OLD_AT)
self.new_manifest = write_root(self.new, NEW_FILES, NEW_AT)
self.wiki = self.tmp / "wiki"
self.wiki.mkdir()
for name, text in PAGES.items():
(self.wiki / name).write_text(text, encoding="utf-8")
def tearDown(self) -> None:
self._tmp.cleanup()
def run_cli(self, *args: str) -> tuple[int, bytes]:
p = subprocess.run([sys.executable, "-m", "okfwiki.sources", *args], cwd=ROOT, capture_output=True,
timeout=120)
return p.returncode, p.stdout
def expected_diff(self) -> dict[str, Any]:
return {
"changed": [{"kind": kind_of(n), "new_sha256": sha(CHANGED_NEW[n].encode()),
"old_sha256": sha(CHANGED_OLD[n].encode()), "path": f"kilde/{n}"} for n in sorted(CHANGED_OLD)],
"gone": [{"kind": "docs", "path": "kilde/docs-en-spade.md"}],
"new": {"frozen_at": NEW_AT, "manifest_sha256": sha(self.new_manifest)},
"new_sources": [{"kind": "docs", "path": "kilde/docs-en-rake.md"},
{"kind": "release-notes", "path": "kilde/release-v1.2.0.md"}],
"old": {"frozen_at": OLD_AT, "manifest_sha256": sha(self.old_manifest)},
"schema": "okf-wiki/source-diff/1",
"unchanged": 4,
}
EXPECTED_MARKS = {
"pages": [
{"page": "compost.md", "sources": [{"change": "changed", "path": "kilde/docs-en-compost.md"}]},
{"page": "seeds.md", "sources": [{"change": "changed", "path": "kilde/docs-en-seeds.md"},
{"change": "gone", "path": "kilde/docs-en-spade.md"},
{"change": "changed", "path": "kilde/release-v1.1.0.md"}]},
{"page": "tools.md", "sources": [{"change": "gone", "path": "kilde/docs-en-spade.md"}]},
],
"schema": "okf-wiki/source-marks/1",
}
def test_00_the_planted_roots_are_sound_k1(self) -> None:
"""Precondition, green before Step 8: the eval's own roots pass the K1 check."""
for root in (self.old, self.new):
code, out = self.run_cli("check", str(root))
self.assertEqual(code, 0, out.decode())
planted = {"new": len(NEW), "changed": len(CHANGED_OLD), "gone": len(GONE), "unchanged": len(UNCHANGED)}
self.assertEqual(planted, {"new": 2, "changed": 3, "gone": 1, "unchanged": 4})
def test_diff_reports_every_planted_change_and_exits_1(self) -> None:
code, out = self.run_cli("diff", str(self.old), str(self.new))
self.assertEqual(out.decode("utf-8"), dump(self.expected_diff()))
self.assertEqual(code, 1)
def test_diff_twice_is_byte_identical(self) -> None:
first = self.run_cli("diff", str(self.old), str(self.new))
second = self.run_cli("diff", str(self.old), str(self.new))
self.assertEqual(first[0], 1)
self.assertEqual(first, second)
def test_diff_of_a_root_against_itself_is_empty_and_exits_0(self) -> None:
code, out = self.run_cli("diff", str(self.old), str(self.old))
same = {"frozen_at": OLD_AT, "manifest_sha256": sha(self.old_manifest)}
self.assertEqual(out.decode("utf-8"), dump({"changed": [], "gone": [], "new": same, "new_sources": [],
"old": same, "schema": "okf-wiki/source-diff/1",
"unchanged": 8}))
self.assertEqual(code, 0)
def test_each_kind_of_change_alone_exits_1(self) -> None:
"""K4: exit 1 on at least one change, whichever kind it is."""
bread = "docs-en-bread.md"
alone = {"new": ({**UNCHANGED, **NEW}, "new_sources", [{"kind": "docs", "path": "kilde/docs-en-rake.md"},
{"kind": "release-notes",
"path": "kilde/release-v1.2.0.md"}]),
"changed": ({**UNCHANGED, bread: "# Bread\n\nFlour and water.\n"}, "changed",
[{"kind": "docs", "new_sha256": sha(b"# Bread\n\nFlour and water.\n"),
"old_sha256": sha(UNCHANGED[bread].encode()), "path": f"kilde/{bread}"}]),
"gone": ({k: v for k, v in UNCHANGED.items() if k != bread}, "gone",
[{"kind": "docs", "path": f"kilde/{bread}"}])}
base = self.tmp / "base"
base_manifest = write_root(base, UNCHANGED, OLD_AT)
for name, (files, key, listed) in alone.items():
with self.subTest(kind=name):
root = self.tmp / f"only-{name}"
manifest = write_root(root, files, NEW_AT)
expected: dict[str, Any] = {
"changed": [], "gone": [], "new": {"frozen_at": NEW_AT, "manifest_sha256": sha(manifest)},
"new_sources": [], "old": {"frozen_at": OLD_AT, "manifest_sha256": sha(base_manifest)},
"schema": "okf-wiki/source-diff/1", "unchanged": 3 if name != "new" else 4, key: listed}
code, out = self.run_cli("diff", str(base), str(root))
self.assertEqual(out.decode("utf-8"), dump(expected))
self.assertEqual(code, 1)
def test_diff_cannot_run_on_a_root_that_fails_the_k1_check(self) -> None:
# exit 2 alone would also be an unknown command; the sound pair must run first
self.assertEqual(self.run_cli("diff", str(self.old), str(self.new))[0], 1)
(self.new / "kilde" / "docs-en-bread.md").write_text("# Bread\n\nFlour and water.\n", encoding="utf-8")
self.assertEqual(self.run_cli("diff", str(self.old), str(self.new))[0], 2)
self.assertEqual(self.run_cli("diff", str(self.old), str(self.tmp / "nowhere"))[0], 2)
def write_diff(self, data: dict[str, Any]) -> Path:
p = self.tmp / "diff.json"
p.write_text(dump(data), encoding="utf-8")
return p
def test_marks_lists_the_pages_built_on_changed_or_gone_sources_and_exits_1(self) -> None:
code, out = self.run_cli("marks", str(self.wiki), "--diff", str(self.write_diff(self.expected_diff())))
self.assertEqual(out.decode("utf-8"), dump(self.EXPECTED_MARKS))
self.assertEqual(code, 1)
def test_marks_twice_is_byte_identical(self) -> None:
diff = self.write_diff(self.expected_diff())
first = self.run_cli("marks", str(self.wiki), "--diff", str(diff))
second = self.run_cli("marks", str(self.wiki), "--diff", str(diff))
self.assertEqual(first[0], 1)
self.assertEqual(first, second)
def test_marks_with_only_new_sources_marks_nothing_and_exits_0(self) -> None:
only_new = {**self.expected_diff(), "changed": [], "gone": []}
code, out = self.run_cli("marks", str(self.wiki), "--diff", str(self.write_diff(only_new)))
self.assertEqual(out.decode("utf-8"), dump({"pages": [], "schema": "okf-wiki/source-marks/1"}))
self.assertEqual(code, 0)
def test_marks_cannot_run_without_a_readable_diff(self) -> None:
# exit 2 alone would also be an unknown command; the sound diff must run first
self.assertEqual(self.run_cli("marks", str(self.wiki), "--diff", str(self.write_diff(self.expected_diff())))[0], 1)
bad = self.tmp / "bad.json"
bad.write_text("{", encoding="utf-8")
self.assertEqual(self.run_cli("marks", str(self.wiki), "--diff", str(bad))[0], 2)
self.assertEqual(self.run_cli("marks", str(self.wiki), "--diff", str(self.tmp / "none.json"))[0], 2)
if __name__ == "__main__":
unittest.main()