"""Tests for tools/diagnose.py and .claude/agents/okf-wiki-diagnoser.md: why a claim is not carried by its excerpt (wave 4 order 20, contract K9). The diagnosis explains; the reader judges; nothing here is a verdict. The fixture is a source of gardening notes, one wiki page and twelve claims, written by hand; every expected number is worked out by hand from it. The text is synthetic, about gardening (CLAUDE.md rule 3). Run: python3 -m unittest discover -s tests -p 'test_diagnose.py' """ from __future__ import annotations import contextlib import importlib.util import io import json import re import sys import tempfile import unittest from pathlib import Path from typing import Any from okfwiki import reader ROOT = Path(__file__).resolve().parents[1] AGENT = ROOT / ".claude" / "agents" / "okf-wiki-diagnoser.md" _SPEC = importlib.util.spec_from_file_location("diagnose", ROOT / "tools" / "diagnose.py") assert _SPEC is not None and _SPEC.loader is not None dg: Any = importlib.util.module_from_spec(_SPEC) sys.modules["diagnose"] = dg _SPEC.loader.exec_module(dg) SOURCE_LINES = [ "# Garden care", # 1 "", # 2 "Water the beds early in the day.", # 3 "", # 4 "## Tools", # 5 "", # 6 "```sh", # 7 "# sharpen the spade", # 8 "```", # 9 "", # 10 "A sharp spade cuts roots cleanly.", # 11 "", # 12 "## Compost", # 13 "", # 14 "Turn the heap every second week.", # 15 "", # 16 "Keep the heap as damp as a wrung sponge.", # 17 "", # 18 "### Leaves", # 19 "", # 20 "Shred autumn leaves before adding them.", # 21 ] SOURCE = "\n".join(SOURCE_LINES) QUOTES = {3: SOURCE_LINES[2], 11: SOURCE_LINES[10], 15: SOURCE_LINES[14], 17: SOURCE_LINES[16], 21: SOURCE_LINES[20]} LINES_OF = [3, 11, 15, 17, 21, 3, 11, 15, 17, 21, 3, 11] # the quoted line of claims g01 .. g12 UNSUPPORTED = {"g02", "g05", "g09"} def claim(n: int) -> dict[str, Any]: line = LINES_OF[n - 1] return {"id": f"g{n:02d}", "page": "garden.md", "line": 6, "claim": f"Gardeners follow rule number {n}.", "footnotes": ["s1"], "excerpts": [{"file": "docs-garden.md", "line": line, "text": QUOTES[line]}], "quote_ok": True, "excerpt_sha256": "0" * 64, "provenance": "merged" if n == 5 else "documented", "resources": ["/kilde/docs-garden.md"]} def run(argv: list[str]) -> tuple[int, str]: buf = io.StringIO() with contextlib.redirect_stdout(buf), contextlib.redirect_stderr(io.StringIO()): code = dg.main(argv) return code, buf.getvalue() def jsonl(path: Path, lines: list[dict[str, Any]]) -> None: path.parent.mkdir(parents=True, exist_ok=True) path.write_text("\n".join(json.dumps(x) for x in lines) + "\n", encoding="utf-8") class Fixture(unittest.TestCase): def setUp(self) -> None: self._tmp = tempfile.TemporaryDirectory() self.root = Path(self._tmp.name) self.sources = self.root / "src" (self.sources / "kilde").mkdir(parents=True) (self.sources / "kilde" / "docs-garden.md").write_text(SOURCE, encoding="utf-8") self.wiki = self.root / "wiki" self.wiki.mkdir() (self.wiki / "garden.md").write_text("---\ntype: Concept\ntitle: Garden upkeep\n---\n\nBody.\n", encoding="utf-8") self.claims = self.root / "claims.json" self.claims.write_text(json.dumps({"schema": "okf-wiki/claims/1", "claims": [claim(n) for n in range(1, 13)], "exceptions": [], "unkeyed": []}), encoding="utf-8") self.verdicts = self.root / "verdicts.json" self.verdicts.write_text(json.dumps({"reader": "okf-wiki-reader", "verdicts": [ {"id": f"g{n:02d}", "verdict": "unsupported" if f"g{n:02d}" in UNSUPPORTED else "supported"} for n in range(1, 13)]}), encoding="utf-8") self.m = self.root / "M" def tearDown(self) -> None: self._tmp.cleanup() def select(self, name: str = "utvalg.json", sample: int = 4) -> tuple[int, dict[str, Any]]: out = self.m / name code, _ = run(["select", "--claims", str(self.claims), "--verdicts", str(self.verdicts), "--out", str(out), "--sample", str(sample)]) return code, json.loads(out.read_text(encoding="utf-8")) if out.is_file() else {} def render(self, size: int = 3) -> int: return run(["render", "--utvalg", str(self.m / "utvalg.json"), "--claims", str(self.claims), "--sources", str(self.sources), "--wiki", str(self.wiki), "--out", str(self.m / "tasks"), "--size", str(size)])[0] class SelectTest(Fixture): def test_the_seed_gives_the_same_selection_twice(self) -> None: code, a = self.select("a.json") self.assertEqual(0, code) _, b = self.select("b.json") self.assertEqual((self.m / "a.json").read_bytes(), (self.m / "b.json").read_bytes()) self.assertEqual(7, len(a["ids"])) # 3 unsupported + 4 of the 9 supported self.assertTrue(UNSUPPORTED <= set(a["ids"])) self.assertEqual(7, len(set(a["ids"]))) self.assertEqual(dg.SEED, a["seed"]) self.assertEqual(20261012, dg.SEED) def test_the_selection_carries_no_verdict(self) -> None: _, doc = self.select() text = (self.m / "utvalg.json").read_text(encoding="utf-8") self.assertNotIn("unsupported", text) self.assertNotIn("verdict", text) self.assertNotIn("supported", text) self.assertEqual({"schema", "seed", "sample", "ids"}, set(doc)) def test_by_hand_the_sample_and_the_shuffle_use_one_generator(self) -> None: import random _, doc = self.select() rng = random.Random(20261012) supported = sorted(f"g{n:02d}" for n in range(1, 13) if f"g{n:02d}" not in UNSUPPORTED) picked = rng.sample(supported, 4) ids = sorted(UNSUPPORTED) + sorted(picked) rng.shuffle(ids) self.assertEqual(ids, doc["ids"]) def test_too_few_supported_or_a_claim_without_verdict_cannot_run(self) -> None: self.assertEqual(2, self.select(sample=10)[0]) doc = json.loads(self.verdicts.read_text(encoding="utf-8")) doc["verdicts"] = doc["verdicts"][:-1] self.verdicts.write_text(json.dumps(doc), encoding="utf-8") self.assertEqual(2, self.select("c.json")[0]) class RenderTest(Fixture): def setUp(self) -> None: super().setUp() self.select() self.ids = json.loads((self.m / "utvalg.json").read_text(encoding="utf-8"))["ids"] def test_one_task_per_call_of_at_most_size_claims_in_selection_order(self) -> None: self.assertEqual(0, self.render(size=3)) calls = json.loads((self.m / "tasks" / "calls.json").read_text(encoding="utf-8")) self.assertEqual(["d001", "d002", "d003"], [c["name"] for c in calls["calls"]]) self.assertEqual(self.ids, [i for c in calls["calls"] for i in c["ids"]]) self.assertEqual([3, 3, 1], [len(c["ids"]) for c in calls["calls"]]) self.assertEqual(["calls.json", "d001.md", "d002.md", "d003.md"], sorted(p.name for p in (self.m / "tasks").iterdir())) def task_of(self, cid: str, size: int = 10) -> tuple[str, int]: self.render(size=size) calls = json.loads((self.m / "tasks" / "calls.json").read_text(encoding="utf-8"))["calls"] for c in calls: if cid in c["ids"]: return (self.m / "tasks" / f"{c['name']}.md").read_text(encoding="utf-8"), c["ids"].index(cid) + 1 raise AssertionError(cid) def window(self, text: str, number: int) -> list[str]: block = text.split(f"## Claim {number}\n", 1)[1].split("\n## Claim ", 1)[0] body = block.split("", 1)[0] return body.split("\n") def test_the_window_stops_at_the_files_start_and_marks_the_quote_and_code(self) -> None: text, number = self.task_of("g02") # quotes line 11: lines 3..19, under Garden care > Tools head, *rows = self.window(text, number) self.assertIn('lines="3-19"', head) self.assertIn('headings="Garden care › Tools"', head) numbered = [r for r in rows if re.match(r"^\s*\d+ ", r)] self.assertEqual(list(range(3, 20)), [int(r.split()[0]) for r in numbered]) self.assertTrue(any(r.startswith(" 11 > |") for r in numbered), numbered) self.assertTrue(any(r.startswith(" 8 c|") for r in numbered), numbered) self.assertTrue(any(r.startswith(" 7 c|") for r in numbered), numbered) self.assertTrue(any(r.startswith(" 3 |") for r in numbered), numbered) text, number = self.task_of("g09") # quotes line 17: lines 9..21, the file ends at 21 head, *_ = self.window(text, number) self.assertIn('lines="9-21"', head) self.assertIn('headings="Garden care › Compost"', head) def test_the_task_shows_the_page_title_claim_and_excerpt_and_no_id_or_verdict(self) -> None: text, number = self.task_of("g05") self.assertIn("Page title: Garden upkeep", text) self.assertIn("Gardeners follow rule number 5.", text) self.assertIn(f'\n{QUOTES[21]}\n', text) for word in ("g05", "supported", "verdict", "merged", "documented"): self.assertNotIn(word, text) def test_an_existing_task_folder_is_never_written_into(self) -> None: self.assertEqual(0, self.render()) before = (self.m / "tasks" / "d001.md").read_bytes() self.assertEqual(2, self.render(size=2)) self.assertEqual(before, (self.m / "tasks" / "d001.md").read_bytes()) class CollectTest(Fixture): def setUp(self) -> None: super().setUp() self.select() self.render(size=3) self.calls = json.loads((self.m / "tasks" / "calls.json").read_text(encoding="utf-8"))["calls"] self.proj = self.root / "transcripts" def reply(self, run_name: str, call: str, text: str, workflow: str | None = None, handback: bool = False, agent_type: str = "okf-wiki-diagnoser") -> None: sub = self.proj / "s1" / "subagents" if workflow: sub = sub / "workflows" / workflow name = f"agent-{run_name}-{call}" block = ({"type": "tool_use", "name": "SubagentHandback", "input": {"message": text}} if handback else {"type": "text", "text": text}) jsonl(sub / f"{name}.jsonl", [{"type": "user", "message": {"content": "task"}}, {"type": "assistant", "message": {"id": f"m-{name}", "content": [block]}}]) (sub / f"{name}.meta.json").write_text(json.dumps({"agentType": agent_type, "description": f"V1-{run_name}-diagnose-{call}"})) def collect(self, *runs: str) -> tuple[int, dict[str, Any]]: out = self.m / f"diagnoses-{len(list(self.m.glob('diagnoses-*')))}.json" argv = ["collect", "--transcripts", str(self.proj), "--tasks", str(self.m / "tasks"), "--out", str(out)] for r in runs: argv += ["--run", r] code, _ = run(argv) return code, json.loads(out.read_text(encoding="utf-8")) if out.is_file() else {} def by_id(self, doc: dict[str, Any]) -> dict[str, dict[str, Any]]: return {d["id"]: d for d in doc["diagnoses"]} def test_one_line_per_claim_and_every_bad_line_is_missing(self) -> None: d1, d2, d3 = (c["ids"] for c in self.calls) self.reply("del1d", "d001", "1: L | the spade\n2: S |\n3: Q | wrong code") self.reply("del1d", "d002", "Some words first.\n1: A | " + "word " * 13 + "\n2: O | Compost\n2: U | twice\n" "3: B | every", workflow="wf_x", handback=True) code, doc = self.collect("del1d") self.assertEqual(1, code) # some are missing found = self.by_id(doc) self.assertEqual(("L", "the spade"), (found[d1[0]]["code"], found[d1[0]]["words"])) self.assertEqual(("S", ""), (found[d1[1]]["code"], found[d1[1]]["words"])) self.assertIsNone(found[d1[2]]["code"]) # Q is not a code self.assertIsNone(found[d2[0]]["code"]) # A with more than twelve words self.assertIsNone(found[d2[1]]["code"]) # number 2 given twice self.assertEqual(("B", "every"), (found[d2[2]]["code"], found[d2[2]]["words"])) self.assertIsNone(found[d3[0]]["code"]) # d003 never answered self.assertEqual(sorted([d1[2], d2[0], d2[1], d3[0]]), sorted(doc["missing"])) self.assertEqual([i for c in self.calls for i in c["ids"]], [d["id"] for d in doc["diagnoses"]]) def test_a_code_other_than_s_needs_the_missing_words(self) -> None: self.reply("del1d", "d001", "1: T |\n2: S\n3: F | the heap") _, doc = self.collect("del1d") found = self.by_id(doc) d1 = self.calls[0]["ids"] self.assertIsNone(found[d1[0]]["code"]) self.assertEqual(("S", "F"), (found[d1[1]]["code"], found[d1[2]]["code"])) def test_a_later_run_answers_a_call_again_and_two_transcripts_under_one_name_are_missing(self) -> None: d1, d2, d3 = (c["ids"] for c in self.calls) self.reply("del1d", "d001", "1: S |\n2: S |\n3: S |") self.reply("del1d", "d002", "1: S |\n2: S |\n3: S |") self.reply("del1d", "d003", "1: S |") self.reply("del1d2", "d002", "1: U | rain\n2: U | rain\n3: U | rain") self.reply("del1d2", "d003", "1: O | Leaves") self.reply("del1d2", "d003", "1: L | heap", workflow="wf_y") code, doc = self.collect("del1d", "del1d2") found = self.by_id(doc) self.assertEqual("S", found[d1[0]]["code"]) self.assertEqual(("U", "del1d2"), (found[d2[0]]["code"], found[d2[0]]["run"])) self.assertIsNone(found[d3[0]]["code"]) # del1d2 answered d003 twice self.assertEqual(1, code) def test_another_agent_type_is_not_a_diagnosis(self) -> None: self.reply("del1d", "d001", "1: S |\n2: S |\n3: S |", agent_type="general-purpose") _, doc = self.collect("del1d") self.assertEqual(7, len(doc["missing"])) def test_a_diagnoser_started_from_a_plugin_is_collected(self) -> None: self.reply("del1d", "d001", "1: S |\n2: S |\n3: S |", agent_type="okf-wiki:okf-wiki-diagnoser") _, doc = self.collect("del1d") self.assertEqual(4, len(doc["missing"])) class TallyTest(Fixture): def diagnoses(self, codes: dict[str, str | None]) -> Path: path = self.root / "diagnoses.json" path.write_text(json.dumps({"schema": dg.DIAGNOSES_SCHEMA, "diagnoses": [ {"id": cid, "call": "d001", "number": 1, "code": code, "words": "", "run": "del1d"} for cid, code in codes.items()], "missing": [c for c, v in codes.items() if v is None]}), encoding="utf-8") return path def tally(self, codes: dict[str, str | None]) -> tuple[int, dict[str, Any], str]: out = self.m / f"tally-{len(list(self.m.glob('tally-*')))}.json" code, text = run(["tally", "--diagnoses", str(self.diagnoses(codes)), "--verdicts", str(self.verdicts), "--claims", str(self.claims), "--out", str(out)]) return code, json.loads(out.read_text(encoding="utf-8")) if out.is_file() else {}, text def test_counts_per_code_agreement_and_groups_by_hand(self) -> None: # unsupported g02 (L), g05 (B, merged), g09 (S); supported sample g01 (S), g03 (T), g04 (S), g10 (missing) code, doc, text = self.tally({"g02": "L", "g05": "B", "g09": "S", "g01": "S", "g03": "T", "g04": "S", "g10": None}) self.assertEqual({"S": 1, "L": 1, "B": 1, "n": 3}, {k: v for k, v in doc["unsupported"].items() if v}) self.assertEqual({"S": 2, "T": 1, "missing": 1, "n": 4}, {k: v for k, v in doc["supported_sample"].items() if v}) self.assertEqual(set(dg.CODES) | {"missing", "n"}, set(doc["unsupported"])) a = doc["agreement"] self.assertEqual((1, 3), (a["unsupported_called_s"], a["unsupported_diagnosed"])) self.assertEqual((1, 3), (a["supported_not_carried"], a["supported_diagnosed"])) self.assertEqual({"S": 0, "L": 0, "B": 1, "missing": 0, "n": 1}, {k: v for k, v in doc["per_provenance"]["merged"]["unsupported"].items() if k in ("S", "L", "B", "missing", "n")}) self.assertEqual(3, doc["per_source"]["docs-garden.md"]["unsupported"]["n"]) self.assertEqual({"s": 1, "of": 3, "missing": 1, "stop": True}, doc["m1"]) # 1 of 3 is a third: stop self.assertEqual(1, code) self.assertIn("M1", text) def test_under_a_third_s_and_nothing_missing_passes_the_m1_rule(self) -> None: code, doc, _ = self.tally({"g02": "L", "g05": "B", "g09": "U", "g01": "S"}) self.assertEqual({"s": 0, "of": 3, "missing": 0, "stop": False}, doc["m1"]) self.assertEqual(0, code) def test_clopper_pearson_by_known_values(self) -> None: lo, hi = dg.clopper_pearson(0, 100) self.assertEqual(0.0, lo) self.assertAlmostEqual(1 - 0.025 ** (1 / 100), hi, places=6) lo, hi = dg.clopper_pearson(5, 100) self.assertAlmostEqual(0.01643, lo, places=4) self.assertAlmostEqual(0.11284, hi, places=4) self.assertEqual((0.0, 1.0), dg.clopper_pearson(0, 0)) class AgentFileTest(unittest.TestCase): def setUp(self) -> None: self.text = AGENT.read_text(encoding="utf-8") def test_frontmatter_names_the_type_read_only_and_opus(self) -> None: head = self.text.split("\n---\n", 1)[0] self.assertIn("name: okf-wiki-diagnoser", head) self.assertIn("tools: Read", head) self.assertIn("model: opus", head) self.assertNotIn("Write", head) def test_no_run_of_eight_words_from_the_readers_instructions(self) -> None: body = reader.instructions(self.text) pair = reader.Pair("agent", body, ()) self.assertEqual([], reader.shared_runs(reader.INSTRUCTIONS, [pair])) def test_it_names_every_code_and_the_reply_form(self) -> None: body = reader.instructions(self.text) for code in dg.CODES: self.assertRegex(body, rf"\*\*{code}\*\*") self.assertIn(": | ", body) self.assertNotIn("supported", body) # it never names the reader's verdicts class NoModelTest(unittest.TestCase): def test_the_tool_imports_no_network_or_model_client(self) -> None: source = (ROOT / "tools" / "diagnose.py").read_text(encoding="utf-8") for banned in ("anthropic", "openai", "urllib", "http.client", "requests", "subprocess", "socket"): self.assertNotIn(f"import {banned}", source) if __name__ == "__main__": unittest.main()