okf-wiki 1.1.1
First public release, published as one snapshot. What changed between versions is in CHANGELOG.md. Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
This commit is contained in:
commit
0fd86a23a3
227 changed files with 96077 additions and 0 deletions
338
tests/test_scripts.py
Normal file
338
tests/test_scripts.py
Normal file
|
|
@ -0,0 +1,338 @@
|
|||
"""The two v1 check scripts (V1 Step 3, rewritten in wave 2 order 7 for plan revision 5): tests/v1_repo.sh (repository
|
||||
checks) and tests/v1_candidate.sh (run outputs).
|
||||
|
||||
Both are bash 3.2 and ASCII only. The candidate script reads only folders under V1_ROOT and V1_PROJECT (and the source
|
||||
roots once a run folder exists), so with both empty every one of its lines is FAIL, and that stays true after later
|
||||
steps add the tools it calls. The fixtures are synthetic text about bread (CLAUDE.md rule 3).
|
||||
Run: python3 -m unittest discover -s tests -p test_scripts.py
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
REPO = ROOT / "tests" / "v1_repo.sh"
|
||||
CANDIDATE = ROOT / "tests" / "v1_candidate.sh"
|
||||
BASH = "/bin/bash" if Path("/bin/bash").is_file() else "bash" # macOS ships bash 3.2 there; the scripts must run on it
|
||||
CHECK_LINE = re.compile(r"^(PASS|FAIL) \S")
|
||||
TABLE_ROW = re.compile(r"^# (\S.*?)\s{2,}\S")
|
||||
TABLE_LINE = re.compile(r"^# (\S.*?)\s{2,}(\S.*)$")
|
||||
CHECK_NAME = re.compile(r'^\s*check "([^"]+)"', re.MULTILINE)
|
||||
# Per run folder (del1/, samling1/) and once for the whole candidate.
|
||||
CANDIDATE_LINES_PER_RUN = 13
|
||||
CANDIDATE_LINES_ONCE = 2
|
||||
RUNS = ("del1", "samling1")
|
||||
|
||||
|
||||
def table_and_checks(script: Path) -> tuple[list[str], list[str]]:
|
||||
"""The expected-state table in the script's header comment and the names of its check lines, in order."""
|
||||
text = script.read_text(encoding="ascii")
|
||||
rows = [m.group(1) for m in (TABLE_ROW.match(x) for x in text.splitlines()) if m]
|
||||
names = [n.replace("$RUN", "<run>") for n in CHECK_NAME.findall(text)]
|
||||
return rows, names
|
||||
|
||||
|
||||
def check_body(script: Path, prefix: str) -> str:
|
||||
"""The text of the check whose name starts with prefix, up to the next check line."""
|
||||
text = script.read_text(encoding="ascii")
|
||||
name = next(n for n in CHECK_NAME.findall(text) if n.startswith(prefix))
|
||||
body = text[text.index(name):]
|
||||
return body[:body.index("\ncheck ") if "\ncheck " in body else len(body)]
|
||||
|
||||
|
||||
def snippet(script: Path, prefix: str) -> str:
|
||||
"""The python code a check passes to py '...'."""
|
||||
body = check_body(script, prefix)
|
||||
start = body.index("py '\n") + len("py '\n")
|
||||
end = body.index("'", start)
|
||||
return body[start:end]
|
||||
|
||||
|
||||
def run_candidate(env: dict[str, str]) -> list[str]:
|
||||
res = subprocess.run([BASH, str(CANDIDATE)], env={**os.environ, **env}, capture_output=True, text=True, cwd=ROOT)
|
||||
return [x for x in res.stdout.splitlines() if x.strip()]
|
||||
|
||||
|
||||
def dump(data: Any) -> str:
|
||||
return json.dumps(data, indent=1, sort_keys=True) + "\n"
|
||||
|
||||
|
||||
class ScriptsTest(unittest.TestCase):
|
||||
def test_the_expected_state_table_names_every_check_in_order(self) -> None:
|
||||
for script in (REPO, CANDIDATE):
|
||||
rows, names = table_and_checks(script)
|
||||
self.assertEqual(len(rows), len(names), script.name)
|
||||
for row, name in zip(rows, names):
|
||||
self.assertTrue(name.startswith(row), f"{script.name}: table row {row!r} does not name {name!r}")
|
||||
|
||||
def test_bash_syntax_and_ascii(self) -> None:
|
||||
for script in (REPO, CANDIDATE):
|
||||
self.assertTrue(script.is_file(), f"{script.name} is missing")
|
||||
self.assertEqual(subprocess.run([BASH, "-n", str(script)]).returncode, 0, script.name)
|
||||
script.read_bytes().decode("ascii")
|
||||
|
||||
|
||||
class RepoScriptTest(unittest.TestCase):
|
||||
def test_the_repo_script_finds_uvx_on_path_and_types_the_tools(self) -> None:
|
||||
text = REPO.read_text(encoding="ascii")
|
||||
self.assertIn("command -v uvx", text)
|
||||
self.assertRegex(text, r"mypy --strict okfwiki gates tests tools/\*\.py")
|
||||
|
||||
def test_the_table_says_only_the_change_warning_eval_is_red(self) -> None:
|
||||
rows = {m.group(1): m.group(2) for m in (TABLE_LINE.match(x) for x in REPO.read_text(encoding="ascii").splitlines())
|
||||
if m}
|
||||
self.assertIn("test_change_warning.py", rows["unittest tests"])
|
||||
self.assertTrue(rows["package boundary"].startswith("PASS"), rows["package boundary"])
|
||||
self.assertTrue(rows["link rule on P1's K"].startswith("PASS"), rows["link rule on P1's K"])
|
||||
self.assertTrue(rows["rule 3"].startswith("PASS"), rows["rule 3"])
|
||||
self.assertIn("at least 4 of its 8", rows["rule 3"]) # what the rule counts, not what it leaves out
|
||||
|
||||
def test_the_sources_default_to_the_fresh_copy_and_p1_has_its_own(self) -> None:
|
||||
text = REPO.read_text(encoding="ascii")
|
||||
self.assertIn('V1_SOURCES="${V1_SOURCES:-$HOME/okf/benk/samling1/20261008}"', text)
|
||||
self.assertIn('P1_SOURCES="${P1_SOURCES:-$HOME/okf/benk/wiki}"', text)
|
||||
self.assertIn('P1_K="${P1_K:-$P1_SOURCES/prove-p1/K}"', text)
|
||||
|
||||
def test_the_link_rule_line_pins_the_number_its_name_states(self) -> None:
|
||||
text = REPO.read_text(encoding="ascii")
|
||||
name = next(n for n in CHECK_NAME.findall(text) if n.startswith("link rule on P1's K"))
|
||||
found = re.search(r"([0-9]+) of ([0-9]+)", name)
|
||||
assert found is not None, name
|
||||
body = check_body(REPO, "link rule on P1's K")
|
||||
self.assertIn(f'rule.get("pass") == {found.group(1)}', body)
|
||||
self.assertIn(f'rule.get("of") == {found.group(2)}', body)
|
||||
|
||||
def test_the_link_rule_line_reads_the_function_not_the_command(self) -> None:
|
||||
# P1's K has no K1 root, so the command exits 2 there; the rule of 04.10 is kept as a lock through the function.
|
||||
body = check_body(REPO, "link rule on P1's K")
|
||||
self.assertIn("link_rule_report", body)
|
||||
self.assertIn('os.environ["P1_SOURCES"]', body)
|
||||
self.assertNotIn('"-m", "okfwiki.audit"', body)
|
||||
|
||||
|
||||
RUN8 = "the starter needs flour and water every single morning"
|
||||
CSS = "box-shadow: 0 2px 8px rgba(0,0,0,.15);"
|
||||
|
||||
|
||||
class RuleThreeTest(unittest.TestCase):
|
||||
"""The rule 3 line (CLAUDE.md rule 3) on a planted tree. A run of eight tokens counts only when at least 4 of them
|
||||
are words of letters only (chosen in wave 2 order 7): it still catches a run of eight ordinary words, and no longer
|
||||
counts the CSS line of numbers, units and three letter words that collided with the fresh copy."""
|
||||
|
||||
def run_rule(self, mine: str, source: str | None, where: str = "tests/planted.txt") -> int:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
(root / "tests").mkdir()
|
||||
other = "Barley grows well in cold and wet northern fields each year.\n"
|
||||
(root / "tests" / "other.txt").write_text(other, encoding="utf-8")
|
||||
(root / where).parent.mkdir(parents=True, exist_ok=True)
|
||||
(root / where).write_text(mine + "\n", encoding="utf-8")
|
||||
(root / "evals").mkdir()
|
||||
(root / "evals" / "README.md").write_text("# Evals\n\nNothing shared here at all.\n", encoding="utf-8")
|
||||
(root / "src" / "kilde").mkdir(parents=True)
|
||||
if source is not None:
|
||||
(root / "src" / "kilde" / "a.md").write_text(source + "\n", encoding="utf-8")
|
||||
res = subprocess.run([sys.executable, "-c", snippet(REPO, "rule 3")], cwd=root, capture_output=True,
|
||||
env={**os.environ, "V1_SOURCES": str(root / "src")})
|
||||
return res.returncode
|
||||
|
||||
def test_a_run_of_eight_ordinary_words_is_caught(self) -> None:
|
||||
self.assertEqual(1, self.run_rule(f"A note: {RUN8}.", f"Docs say {RUN8} too."))
|
||||
|
||||
def test_a_css_line_of_numbers_units_and_css_words_is_not_counted(self) -> None:
|
||||
self.assertEqual(0, self.run_rule(f".pop {{ {CSS} }}", f"Set `{CSS}` on the card."))
|
||||
|
||||
def test_four_words_among_eight_tokens_still_count(self) -> None:
|
||||
run = "set 2 cups of 3 warm water 1"
|
||||
self.assertEqual(1, self.run_rule(f"A: {run}.", f"B: {run}."))
|
||||
|
||||
def test_the_tracked_agent_prompts_and_tools_are_scanned(self) -> None:
|
||||
# review finding 48 (order 39): .claude/agents holds prompts with no copy under agents/, and tools/ is tracked
|
||||
for where in (".claude/agents/planted.md", "tools/planted.py"):
|
||||
with self.subTest(where=where):
|
||||
self.assertEqual(1, self.run_rule(f"A note: {RUN8}.", f"Docs say {RUN8} too.", where))
|
||||
|
||||
def test_nothing_shared_passes(self) -> None:
|
||||
self.assertEqual(0, self.run_rule(f"A note: {RUN8}.", "Rye is a dense bread with a sour taste."))
|
||||
|
||||
def test_no_source_files_cannot_pass(self) -> None:
|
||||
self.assertEqual(1, self.run_rule(f"A note: {RUN8}.", None))
|
||||
|
||||
def run_in_git(self, name: str) -> int:
|
||||
"""The planted tree as a git repository that ignores *.local.*, with one more file under docs/."""
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
for d in ("tests", "docs", "evals", "src/kilde"):
|
||||
(root / d).mkdir(parents=True)
|
||||
(root / "tests" / "planted.txt").write_text("Rye is a dense bread with a sour taste.\n", encoding="utf-8")
|
||||
(root / "evals" / "README.md").write_text("# Evals\n\nNothing shared here at all.\n", encoding="utf-8")
|
||||
(root / "src" / "kilde" / "a.md").write_text(f"Docs say {RUN8} too.\n", encoding="utf-8")
|
||||
(root / ".gitignore").write_text("*.local.*\nsrc/\n", encoding="utf-8")
|
||||
(root / "docs" / name).write_text(f"A note: {RUN8}.\n", encoding="utf-8")
|
||||
subprocess.run(["git", "init", "-q"], cwd=root, check=True)
|
||||
res = subprocess.run([sys.executable, "-c", snippet(REPO, "rule 3")], cwd=root, capture_output=True,
|
||||
env={**os.environ, "V1_SOURCES": str(root / "src")})
|
||||
return res.returncode
|
||||
|
||||
def test_a_file_git_ignores_is_never_committed_and_is_not_counted(self) -> None:
|
||||
self.assertEqual(0, self.run_in_git("order.local.md"))
|
||||
|
||||
def test_a_new_file_git_would_add_is_counted(self) -> None:
|
||||
self.assertEqual(1, self.run_in_git("order.md"))
|
||||
|
||||
|
||||
class SharedIgnoreTest(unittest.TestCase):
|
||||
"""The shared .gitignore, not one clone's .git/info/exclude, keeps local files out of every clone's commits."""
|
||||
|
||||
def test_local_plans_and_the_order_folders_are_ignored_in_every_clone(self) -> None:
|
||||
# review finding 2 (order 39): .claude/projects/ (briefs and plans that may name Anthropic doc pages, CLAUDE.md
|
||||
# rule 3) was ignored only through this clone's unversioned exclude file, while .claude/agents/ is tracked
|
||||
lines = {x.strip() for x in (ROOT / ".gitignore").read_text(encoding="utf-8").splitlines()}
|
||||
for entry in ("STATE.md", "PLAN.md", ".claude/projects/"):
|
||||
with self.subTest(entry=entry):
|
||||
self.assertIn(entry, lines)
|
||||
|
||||
|
||||
class ReadmeTest(unittest.TestCase):
|
||||
"""README.md says what the code and the skill do (review findings 4, 5 and 47, order 39)."""
|
||||
|
||||
def setUp(self) -> None:
|
||||
self.text = (ROOT / "README.md").read_text(encoding="utf-8")
|
||||
|
||||
def test_the_types_command_is_the_repo_checks_and_names_what_it_needs(self) -> None:
|
||||
line = next(x.strip() for x in REPO.read_text(encoding="ascii").splitlines() if "mypy --strict" in x
|
||||
and x.strip().startswith('"$uvx_bin"'))
|
||||
command = line.replace('"$uvx_bin"', "uvx")
|
||||
self.assertIn(f"`{command}`", self.text)
|
||||
for need in ("uvx", "offline", "types-PyYAML", "PyYAML"):
|
||||
with self.subTest(need=need):
|
||||
self.assertIn(need, self.text)
|
||||
self.assertNotIn("Types: `mypy --strict okfwiki tests`", self.text)
|
||||
|
||||
def test_the_skill_section_is_the_k_build(self) -> None:
|
||||
skill = self.text.split("## The skill", 1)[1].split("\n## ", 1)[0]
|
||||
flat = " ".join(skill.split())
|
||||
for phrase in ("packs of five", "okf-wiki-compiler", "okf-wiki-linker", "okf-wiki-reader", "okf-wiki-repairer",
|
||||
"okf-wiki-merger", "main page", "home"):
|
||||
with self.subTest(phrase=phrase):
|
||||
self.assertIn(phrase, flat)
|
||||
for gone in ("x-outdated", "x-disputed", "one source at a time", "writer subagent", "Writing a source"):
|
||||
with self.subTest(gone=gone):
|
||||
self.assertNotIn(gone, self.text)
|
||||
|
||||
|
||||
class CandidateScriptTest(unittest.TestCase):
|
||||
def test_the_runs_are_the_first_part_and_collection_1(self) -> None:
|
||||
text = CANDIDATE.read_text(encoding="ascii")
|
||||
self.assertIn("for RUN in del1 samling1", text)
|
||||
self.assertNotIn("utsnitt", text)
|
||||
self.assertIn('DEL1_SOURCES="${DEL1_SOURCES:-$HOME/okf/benk/samling1/del1}"', text)
|
||||
self.assertIn('SAMLING1_SOURCES="${SAMLING1_SOURCES:-$HOME/okf/benk/samling1/20261008}"', text)
|
||||
|
||||
def test_google_viewer_comes_from_the_mirror_at_the_pinned_commit(self) -> None:
|
||||
text = CANDIDATE.read_text(encoding="ascii")
|
||||
self.assertIn('OKF_UPSTREAM="${OKF_UPSTREAM:-$HOME/repos/_okf-canonical}"', text)
|
||||
self.assertIn('OKF_UPSTREAM_REV="${OKF_UPSTREAM_REV:-ad30107}"', text)
|
||||
self.assertNotIn("_okf-upstream", text)
|
||||
self.assertIn("archive", text)
|
||||
|
||||
def test_every_table_row_names_the_step_that_turns_it_pass(self) -> None:
|
||||
rows = [m.group(2) for m in (TABLE_LINE.match(x) for x in CANDIDATE.read_text(encoding="ascii").splitlines()) if m]
|
||||
self.assertEqual(CANDIDATE_LINES_PER_RUN + CANDIDATE_LINES_ONCE, len(rows))
|
||||
for row in rows:
|
||||
self.assertRegex(row, r"Step 1[0-9]", row)
|
||||
|
||||
def test_the_link_rule_line_reads_the_main_page_report(self) -> None:
|
||||
body = check_body(CANDIDATE, "$RUN link rule")
|
||||
self.assertIn('"main_pages"', body)
|
||||
self.assertIn('"linked"', body)
|
||||
|
||||
def test_the_candidate_script_fails_every_line_on_an_empty_root(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
lines = run_candidate({"V1_ROOT": tmp, "V1_PROJECT": tmp})
|
||||
self.assertEqual(len(lines), len(RUNS) * CANDIDATE_LINES_PER_RUN + CANDIDATE_LINES_ONCE, lines)
|
||||
self.assertTrue(all(CHECK_LINE.match(x) for x in lines), lines)
|
||||
self.assertTrue(all(x.startswith("FAIL") for x in lines), lines)
|
||||
|
||||
|
||||
def results(arm: str, correct: dict[str, bool]) -> str:
|
||||
return dump({"schema": "okf-wiki/answer-results/1", "arm": arm, "set_sha256": "a" * 64,
|
||||
"results": [{"id": i, "group": "docs-2-3", "correct": c, "keys_held": 1, "keys": 1,
|
||||
"paths_cited": 2, "paths_needed": 2} for i, c in sorted(correct.items())]})
|
||||
|
||||
|
||||
class CriterionFourTest(unittest.TestCase):
|
||||
"""The answer-measurement line reads the result files of contract K5 per arm, under <run>/svar/."""
|
||||
|
||||
def lines(self, run: str, filsok: dict[str, bool], bundle: dict[str, bool]) -> list[str]:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
svar = Path(tmp) / run / "svar"
|
||||
svar.mkdir(parents=True)
|
||||
(svar / "filsok.json").write_text(results("filsok", filsok), encoding="utf-8")
|
||||
(svar / "bundle.json").write_text(results("bundle", bundle), encoding="utf-8")
|
||||
lines = run_candidate({"V1_ROOT": tmp, "V1_PROJECT": tmp})
|
||||
return [x for x in lines if x.split(" ", 2)[1] == run and " criterion 4" in x]
|
||||
|
||||
def test_the_first_part_passes_with_both_arms_measured_on_the_same_questions(self) -> None:
|
||||
ids = [f"p{n:02d}" for n in range(1, 13)]
|
||||
self.assertEqual(["PASS"], [x.split()[0] for x in self.lines("del1", {i: False for i in ids}, {i: False for i in ids})])
|
||||
|
||||
def test_collection_1_needs_the_bundle_arm_significantly_better(self) -> None:
|
||||
ids = [f"d{n:02d}" for n in range(1, 13)]
|
||||
better = self.lines("samling1", {i: False for i in ids}, {i: True for i in ids}) # 12 pairs only B: p < 0.05
|
||||
self.assertEqual(["PASS"], [x.split()[0] for x in better])
|
||||
level = self.lines("samling1", {i: False for i in ids}, {i: i in ids[:3] for i in ids}) # 3 only B: p 0.25
|
||||
self.assertEqual(["FAIL"], [x.split()[0] for x in level])
|
||||
|
||||
def test_arms_over_different_questions_fail(self) -> None:
|
||||
self.assertEqual(["FAIL"], [x.split()[0] for x in self.lines("del1", {"p01": True}, {"p02": True})])
|
||||
|
||||
|
||||
class ChangeWarningTest(unittest.TestCase):
|
||||
"""The change-warning line (Step 18a): a diff against the frozen copy of collection 1 and marks the view can show."""
|
||||
|
||||
def test_a_diff_of_the_frozen_copy_and_marks_that_fit_the_bundle_pass(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
src = root / "src"
|
||||
(src / "kilde").mkdir(parents=True)
|
||||
(src / "kilde" / "bread.md").write_text("# Bread\nSourdough needs a starter.\n", encoding="utf-8")
|
||||
(src / "source-manifest.json").write_text('{"schema": "okf-wiki/source-manifest/1", "sources": []}\n',
|
||||
encoding="utf-8")
|
||||
frozen = hashlib.sha256((src / "source-manifest.json").read_bytes()).hexdigest()
|
||||
run = root / "v1" / "samling1"
|
||||
(run / "endring").mkdir(parents=True)
|
||||
(run / "index.md").write_text("# Bread\n\n* [Sourdough](sourdough.md) - A bread.\n", encoding="utf-8")
|
||||
(run / "sourdough.md").write_text(
|
||||
"---\ntype: Concept\ntitle: Sourdough\nsources:\n - id: b-2\n resource: /kilde/bread.md\n"
|
||||
" x-locator: { line: 2 }\n x-quote: 'Sourdough needs a starter.'\n---\n\n# Sourdough\n\n"
|
||||
"Sourdough needs a starter.[^b-2]\n\n[^b-2]: bread.md, line 2\n", encoding="utf-8")
|
||||
diff: dict[str, Any] = {
|
||||
"schema": "okf-wiki/source-diff/1", "changed": [], "gone": [], "new_sources": [], "unchanged": 1,
|
||||
"old": {"frozen_at": "2026-10-08T12:00:00Z", "manifest_sha256": frozen},
|
||||
"new": {"frozen_at": "2026-11-01T12:00:00Z", "manifest_sha256": "b" * 64}}
|
||||
marks = {"schema": "okf-wiki/source-marks/1",
|
||||
"pages": [{"page": "sourdough.md", "sources": [{"change": "changed", "path": "kilde/bread.md"}]}]}
|
||||
(run / "endring" / "diff.json").write_text(dump(diff), encoding="utf-8")
|
||||
(run / "endring" / "marks.json").write_text(dump(marks), encoding="utf-8")
|
||||
env = {"V1_ROOT": str(root / "v1"), "V1_PROJECT": tmp, "SAMLING1_SOURCES": str(src)}
|
||||
line = next(x for x in run_candidate(env) if x.split(" ", 1)[1].startswith("change warning"))
|
||||
self.assertTrue(line.startswith("PASS"), line)
|
||||
diff["old"]["manifest_sha256"] = "c" * 64
|
||||
(run / "endring" / "diff.json").write_text(dump(diff), encoding="utf-8")
|
||||
line = next(x for x in run_candidate(env) if x.split(" ", 1)[1].startswith("change warning"))
|
||||
self.assertTrue(line.startswith("FAIL"), line)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Loading…
Add table
Add a link
Reference in a new issue