First public release, published as one snapshot. What changed between versions is in CHANGELOG.md. Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
245 lines
13 KiB
Python
245 lines
13 KiB
Python
"""Eval for the change warning (contract K4): `okfwiki.sources diff` and `marks`, written RED before the code (Step 8).
|
|
|
|
Two synthetic K1 roots, old and new, with planted changes: 2 new, 3 changed (another sha256), 1 gone and 4 unchanged
|
|
sources, and a synthetic collection whose pages cite them. The denominator is the planted changes (6) and the pages
|
|
that cite a changed or gone source (3 of the 5 concept pages); it comes from this file, never from the product. Every
|
|
expectation is the WHOLE output as K4 describes it, and the exit code; two runs must be byte-identical. The commands
|
|
run as `python3 -m okfwiki.sources` in a child process, exactly as a user runs them.
|
|
Frozen in wave 2: the session that writes diff and marks never edits this file.
|
|
Run: python3 -m unittest discover -s tests -p 'test_change_warning.py'
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import json
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import unittest
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
OLD_AT = "2026-10-08T06:30:00Z"
|
|
NEW_AT = "2026-10-09T06:30:00Z"
|
|
SITE = "https://example.org/docs/en/"
|
|
|
|
|
|
def dump(data: Any) -> str:
|
|
"""K4 item 3: the one written form."""
|
|
return json.dumps(data, indent=1, sort_keys=True, ensure_ascii=False) + "\n"
|
|
|
|
|
|
def sha(data: bytes) -> str:
|
|
return hashlib.sha256(data).hexdigest()
|
|
|
|
|
|
def kind_of(name: str) -> str:
|
|
return "release-notes" if name.startswith("release-") else "docs"
|
|
|
|
|
|
UNCHANGED = {
|
|
"docs-en-bread.md": "# Bread\n\nFlour, water and salt.\n",
|
|
"docs-en-garden-soil.md": "# Soil\n\nCompost feeds the soil.\n",
|
|
"docs-en-oven.md": "# Oven\n\nHeat the stone for an hour.\n",
|
|
"release-v1.0.0.md": '<Update label="1.0.0" description="May 1, 2026">\n * First tools\n</Update>\n',
|
|
}
|
|
CHANGED_OLD = {
|
|
"docs-en-compost.md": "# Compost\n\nTurn the heap every week.\n",
|
|
"docs-en-seeds.md": "# Seeds\n\nSow peas in April.\n",
|
|
"release-v1.1.0.md": '<Update label="1.1.0" description="June 2, 2026">\n * Added rake\n</Update>\n',
|
|
}
|
|
CHANGED_NEW = {
|
|
"docs-en-compost.md": "# Compost\n\nTurn the heap every second week.\n",
|
|
"docs-en-seeds.md": "# Seeds\n\nSow peas in March.\n",
|
|
"release-v1.1.0.md": '<Update label="1.1.0" description="June 2, 2026">\n * Added rake and hoe\n</Update>\n',
|
|
}
|
|
GONE = {"docs-en-spade.md": "# Spade\n\nDig before the frost.\n"}
|
|
NEW = {
|
|
"docs-en-rake.md": "# Rake\n\nRake the leaves in autumn.\n",
|
|
"release-v1.2.0.md": '<Update label="1.2.0" description="July 3, 2026">\n * Added watering can\n</Update>\n',
|
|
}
|
|
OLD_FILES = {**UNCHANGED, **CHANGED_OLD, **GONE}
|
|
NEW_FILES = {**UNCHANGED, **CHANGED_NEW, **NEW}
|
|
|
|
|
|
def write_root(root: Path, files: dict[str, str], frozen_at: str) -> bytes:
|
|
"""A K1 root written by hand (so the eval does not lean on freeze); returns the manifest's bytes."""
|
|
(root / "kilde").mkdir(parents=True)
|
|
entries = []
|
|
for name, text in sorted(files.items()):
|
|
data = text.encode("utf-8")
|
|
(root / "kilde" / name).write_bytes(data)
|
|
entry: dict[str, Any] = {"bytes": len(data), "kind": kind_of(name), "lines": len(text.splitlines()),
|
|
"main_topic": kind_of(name) == "docs", "path": f"kilde/{name}", "sha256": sha(data),
|
|
"url": SITE + name}
|
|
if kind_of(name) == "release-notes":
|
|
entry["version"] = name[len("release-v"):-len(".md")]
|
|
entries.append(entry)
|
|
doc = {"collection": "garden", "frozen_at": frozen_at, "schema": "okf-wiki/source-manifest/1",
|
|
"sources": entries, "subset_of": None}
|
|
text = dump(doc)
|
|
(root / "source-manifest.json").write_text(text, encoding="utf-8")
|
|
return text.encode("utf-8")
|
|
|
|
|
|
def concept(title: str, resources: list[str]) -> str:
|
|
cites = "".join(f" - id: s-{n}\n resource: /kilde/{r}\n title: Notes\n x-locator: {{ line: 3 }}\n"
|
|
f" x-quote: 'x'\n" for n, r in enumerate(resources, 1))
|
|
notes = "".join(f"[^s-{n}]: Notes, line 3\n" for n in range(1, len(resources) + 1))
|
|
marks = "".join(f"[^s-{n}]" for n in range(1, len(resources) + 1))
|
|
return (f"---\ntype: Concept\ntitle: {title}\ndescription: About {title.lower()}.\nsources:\n{cites}"
|
|
f"generated: {{ by: claude-code/claude-opus-5-5, at: 2026-10-08T09:00:00Z }}\n---\n\n# {title}\n\n"
|
|
f"A sentence about {title.lower()}.{marks}\n\n{notes}")
|
|
|
|
|
|
PAGES = {
|
|
"bread.md": concept("Bread", ["docs-en-bread.md"]), # unchanged only: no mark
|
|
"compost.md": concept("Compost", ["docs-en-garden-soil.md", "docs-en-compost.md"]), # one changed
|
|
"seeds.md": concept("Seeds", ["release-v1.1.0.md", "docs-en-spade.md", "docs-en-seeds.md"]), # 2 changed, 1 gone
|
|
"tools.md": concept("Tools", ["docs-en-spade.md", "docs-en-spade.md", "release-v1.0.0.md"]), # gone, cited twice
|
|
"oven.md": concept("Oven", ["docs-en-oven.md"]), # unchanged only: no mark
|
|
"index.md": "---\ntype: Index\n---\n\n# Garden\n\n- [Bread](bread.md)\n- [Compost](compost.md)\n",
|
|
"log.md": "---\ntype: Log\n---\n\n# Log\n\n- 2026-10-08: built\n",
|
|
}
|
|
|
|
|
|
class ChangeWarningEval(unittest.TestCase):
|
|
def setUp(self) -> None:
|
|
self._tmp = tempfile.TemporaryDirectory()
|
|
self.tmp = Path(self._tmp.name)
|
|
self.old = self.tmp / "old"
|
|
self.new = self.tmp / "new"
|
|
self.old_manifest = write_root(self.old, OLD_FILES, OLD_AT)
|
|
self.new_manifest = write_root(self.new, NEW_FILES, NEW_AT)
|
|
self.wiki = self.tmp / "wiki"
|
|
self.wiki.mkdir()
|
|
for name, text in PAGES.items():
|
|
(self.wiki / name).write_text(text, encoding="utf-8")
|
|
|
|
def tearDown(self) -> None:
|
|
self._tmp.cleanup()
|
|
|
|
def run_cli(self, *args: str) -> tuple[int, bytes]:
|
|
p = subprocess.run([sys.executable, "-m", "okfwiki.sources", *args], cwd=ROOT, capture_output=True,
|
|
timeout=120)
|
|
return p.returncode, p.stdout
|
|
|
|
def expected_diff(self) -> dict[str, Any]:
|
|
return {
|
|
"changed": [{"kind": kind_of(n), "new_sha256": sha(CHANGED_NEW[n].encode()),
|
|
"old_sha256": sha(CHANGED_OLD[n].encode()), "path": f"kilde/{n}"} for n in sorted(CHANGED_OLD)],
|
|
"gone": [{"kind": "docs", "path": "kilde/docs-en-spade.md"}],
|
|
"new": {"frozen_at": NEW_AT, "manifest_sha256": sha(self.new_manifest)},
|
|
"new_sources": [{"kind": "docs", "path": "kilde/docs-en-rake.md"},
|
|
{"kind": "release-notes", "path": "kilde/release-v1.2.0.md"}],
|
|
"old": {"frozen_at": OLD_AT, "manifest_sha256": sha(self.old_manifest)},
|
|
"schema": "okf-wiki/source-diff/1",
|
|
"unchanged": 4,
|
|
}
|
|
|
|
EXPECTED_MARKS = {
|
|
"pages": [
|
|
{"page": "compost.md", "sources": [{"change": "changed", "path": "kilde/docs-en-compost.md"}]},
|
|
{"page": "seeds.md", "sources": [{"change": "changed", "path": "kilde/docs-en-seeds.md"},
|
|
{"change": "gone", "path": "kilde/docs-en-spade.md"},
|
|
{"change": "changed", "path": "kilde/release-v1.1.0.md"}]},
|
|
{"page": "tools.md", "sources": [{"change": "gone", "path": "kilde/docs-en-spade.md"}]},
|
|
],
|
|
"schema": "okf-wiki/source-marks/1",
|
|
}
|
|
|
|
def test_00_the_planted_roots_are_sound_k1(self) -> None:
|
|
"""Precondition, green before Step 8: the eval's own roots pass the K1 check."""
|
|
for root in (self.old, self.new):
|
|
code, out = self.run_cli("check", str(root))
|
|
self.assertEqual(code, 0, out.decode())
|
|
planted = {"new": len(NEW), "changed": len(CHANGED_OLD), "gone": len(GONE), "unchanged": len(UNCHANGED)}
|
|
self.assertEqual(planted, {"new": 2, "changed": 3, "gone": 1, "unchanged": 4})
|
|
|
|
def test_diff_reports_every_planted_change_and_exits_1(self) -> None:
|
|
code, out = self.run_cli("diff", str(self.old), str(self.new))
|
|
self.assertEqual(out.decode("utf-8"), dump(self.expected_diff()))
|
|
self.assertEqual(code, 1)
|
|
|
|
def test_diff_twice_is_byte_identical(self) -> None:
|
|
first = self.run_cli("diff", str(self.old), str(self.new))
|
|
second = self.run_cli("diff", str(self.old), str(self.new))
|
|
self.assertEqual(first[0], 1)
|
|
self.assertEqual(first, second)
|
|
|
|
def test_diff_of_a_root_against_itself_is_empty_and_exits_0(self) -> None:
|
|
code, out = self.run_cli("diff", str(self.old), str(self.old))
|
|
same = {"frozen_at": OLD_AT, "manifest_sha256": sha(self.old_manifest)}
|
|
self.assertEqual(out.decode("utf-8"), dump({"changed": [], "gone": [], "new": same, "new_sources": [],
|
|
"old": same, "schema": "okf-wiki/source-diff/1",
|
|
"unchanged": 8}))
|
|
self.assertEqual(code, 0)
|
|
|
|
def test_each_kind_of_change_alone_exits_1(self) -> None:
|
|
"""K4: exit 1 on at least one change, whichever kind it is."""
|
|
bread = "docs-en-bread.md"
|
|
alone = {"new": ({**UNCHANGED, **NEW}, "new_sources", [{"kind": "docs", "path": "kilde/docs-en-rake.md"},
|
|
{"kind": "release-notes",
|
|
"path": "kilde/release-v1.2.0.md"}]),
|
|
"changed": ({**UNCHANGED, bread: "# Bread\n\nFlour and water.\n"}, "changed",
|
|
[{"kind": "docs", "new_sha256": sha(b"# Bread\n\nFlour and water.\n"),
|
|
"old_sha256": sha(UNCHANGED[bread].encode()), "path": f"kilde/{bread}"}]),
|
|
"gone": ({k: v for k, v in UNCHANGED.items() if k != bread}, "gone",
|
|
[{"kind": "docs", "path": f"kilde/{bread}"}])}
|
|
base = self.tmp / "base"
|
|
base_manifest = write_root(base, UNCHANGED, OLD_AT)
|
|
for name, (files, key, listed) in alone.items():
|
|
with self.subTest(kind=name):
|
|
root = self.tmp / f"only-{name}"
|
|
manifest = write_root(root, files, NEW_AT)
|
|
expected: dict[str, Any] = {
|
|
"changed": [], "gone": [], "new": {"frozen_at": NEW_AT, "manifest_sha256": sha(manifest)},
|
|
"new_sources": [], "old": {"frozen_at": OLD_AT, "manifest_sha256": sha(base_manifest)},
|
|
"schema": "okf-wiki/source-diff/1", "unchanged": 3 if name != "new" else 4, key: listed}
|
|
code, out = self.run_cli("diff", str(base), str(root))
|
|
self.assertEqual(out.decode("utf-8"), dump(expected))
|
|
self.assertEqual(code, 1)
|
|
|
|
def test_diff_cannot_run_on_a_root_that_fails_the_k1_check(self) -> None:
|
|
# exit 2 alone would also be an unknown command; the sound pair must run first
|
|
self.assertEqual(self.run_cli("diff", str(self.old), str(self.new))[0], 1)
|
|
(self.new / "kilde" / "docs-en-bread.md").write_text("# Bread\n\nFlour and water.\n", encoding="utf-8")
|
|
self.assertEqual(self.run_cli("diff", str(self.old), str(self.new))[0], 2)
|
|
self.assertEqual(self.run_cli("diff", str(self.old), str(self.tmp / "nowhere"))[0], 2)
|
|
|
|
def write_diff(self, data: dict[str, Any]) -> Path:
|
|
p = self.tmp / "diff.json"
|
|
p.write_text(dump(data), encoding="utf-8")
|
|
return p
|
|
|
|
def test_marks_lists_the_pages_built_on_changed_or_gone_sources_and_exits_1(self) -> None:
|
|
code, out = self.run_cli("marks", str(self.wiki), "--diff", str(self.write_diff(self.expected_diff())))
|
|
self.assertEqual(out.decode("utf-8"), dump(self.EXPECTED_MARKS))
|
|
self.assertEqual(code, 1)
|
|
|
|
def test_marks_twice_is_byte_identical(self) -> None:
|
|
diff = self.write_diff(self.expected_diff())
|
|
first = self.run_cli("marks", str(self.wiki), "--diff", str(diff))
|
|
second = self.run_cli("marks", str(self.wiki), "--diff", str(diff))
|
|
self.assertEqual(first[0], 1)
|
|
self.assertEqual(first, second)
|
|
|
|
def test_marks_with_only_new_sources_marks_nothing_and_exits_0(self) -> None:
|
|
only_new = {**self.expected_diff(), "changed": [], "gone": []}
|
|
code, out = self.run_cli("marks", str(self.wiki), "--diff", str(self.write_diff(only_new)))
|
|
self.assertEqual(out.decode("utf-8"), dump({"pages": [], "schema": "okf-wiki/source-marks/1"}))
|
|
self.assertEqual(code, 0)
|
|
|
|
def test_marks_cannot_run_without_a_readable_diff(self) -> None:
|
|
# exit 2 alone would also be an unknown command; the sound diff must run first
|
|
self.assertEqual(self.run_cli("marks", str(self.wiki), "--diff", str(self.write_diff(self.expected_diff())))[0], 1)
|
|
bad = self.tmp / "bad.json"
|
|
bad.write_text("{", encoding="utf-8")
|
|
self.assertEqual(self.run_cli("marks", str(self.wiki), "--diff", str(bad))[0], 2)
|
|
self.assertEqual(self.run_cli("marks", str(self.wiki), "--diff", str(self.tmp / "none.json"))[0], 2)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|