okf-wiki/tests/test_diagnose.py
Kjell Tore Guttormsen 0fd86a23a3
okf-wiki 1.1.1
First public release, published as one snapshot. What changed between
versions is in CHANGELOG.md.

Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
2026-11-14 21:03:20 +01:00

377 lines
19 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

"""Tests for tools/diagnose.py and .claude/agents/okf-wiki-diagnoser.md: why a claim is not carried by its excerpt
(wave 4 order 20, contract K9). The diagnosis explains; the reader judges; nothing here is a verdict.
The fixture is a source of gardening notes, one wiki page and twelve claims, written by hand; every expected number is
worked out by hand from it. The text is synthetic, about gardening (CLAUDE.md rule 3).
Run: python3 -m unittest discover -s tests -p 'test_diagnose.py'
"""
from __future__ import annotations
import contextlib
import importlib.util
import io
import json
import re
import sys
import tempfile
import unittest
from pathlib import Path
from typing import Any
from okfwiki import reader
ROOT = Path(__file__).resolve().parents[1]
AGENT = ROOT / ".claude" / "agents" / "okf-wiki-diagnoser.md"
_SPEC = importlib.util.spec_from_file_location("diagnose", ROOT / "tools" / "diagnose.py")
assert _SPEC is not None and _SPEC.loader is not None
dg: Any = importlib.util.module_from_spec(_SPEC)
sys.modules["diagnose"] = dg
_SPEC.loader.exec_module(dg)
SOURCE_LINES = [
"# Garden care", # 1
"", # 2
"Water the beds early in the day.", # 3
"", # 4
"## Tools", # 5
"", # 6
"```sh", # 7
"# sharpen the spade", # 8
"```", # 9
"", # 10
"A sharp spade cuts roots cleanly.", # 11
"", # 12
"## Compost", # 13
"", # 14
"Turn the heap every second week.", # 15
"", # 16
"Keep the heap as damp as a wrung sponge.", # 17
"", # 18
"### Leaves", # 19
"", # 20
"Shred autumn leaves before adding them.", # 21
]
SOURCE = "\n".join(SOURCE_LINES)
QUOTES = {3: SOURCE_LINES[2], 11: SOURCE_LINES[10], 15: SOURCE_LINES[14], 17: SOURCE_LINES[16], 21: SOURCE_LINES[20]}
LINES_OF = [3, 11, 15, 17, 21, 3, 11, 15, 17, 21, 3, 11] # the quoted line of claims g01 .. g12
UNSUPPORTED = {"g02", "g05", "g09"}
def claim(n: int) -> dict[str, Any]:
line = LINES_OF[n - 1]
return {"id": f"g{n:02d}", "page": "garden.md", "line": 6, "claim": f"Gardeners follow rule number {n}.",
"footnotes": ["s1"], "excerpts": [{"file": "docs-garden.md", "line": line, "text": QUOTES[line]}],
"quote_ok": True, "excerpt_sha256": "0" * 64, "provenance": "merged" if n == 5 else "documented",
"resources": ["/kilde/docs-garden.md"]}
def run(argv: list[str]) -> tuple[int, str]:
buf = io.StringIO()
with contextlib.redirect_stdout(buf), contextlib.redirect_stderr(io.StringIO()):
code = dg.main(argv)
return code, buf.getvalue()
def jsonl(path: Path, lines: list[dict[str, Any]]) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text("\n".join(json.dumps(x) for x in lines) + "\n", encoding="utf-8")
class Fixture(unittest.TestCase):
def setUp(self) -> None:
self._tmp = tempfile.TemporaryDirectory()
self.root = Path(self._tmp.name)
self.sources = self.root / "src"
(self.sources / "kilde").mkdir(parents=True)
(self.sources / "kilde" / "docs-garden.md").write_text(SOURCE, encoding="utf-8")
self.wiki = self.root / "wiki"
self.wiki.mkdir()
(self.wiki / "garden.md").write_text("---\ntype: Concept\ntitle: Garden upkeep\n---\n\nBody.\n", encoding="utf-8")
self.claims = self.root / "claims.json"
self.claims.write_text(json.dumps({"schema": "okf-wiki/claims/1", "claims": [claim(n) for n in range(1, 13)],
"exceptions": [], "unkeyed": []}), encoding="utf-8")
self.verdicts = self.root / "verdicts.json"
self.verdicts.write_text(json.dumps({"reader": "okf-wiki-reader", "verdicts": [
{"id": f"g{n:02d}", "verdict": "unsupported" if f"g{n:02d}" in UNSUPPORTED else "supported"}
for n in range(1, 13)]}), encoding="utf-8")
self.m = self.root / "M"
def tearDown(self) -> None:
self._tmp.cleanup()
def select(self, name: str = "utvalg.json", sample: int = 4) -> tuple[int, dict[str, Any]]:
out = self.m / name
code, _ = run(["select", "--claims", str(self.claims), "--verdicts", str(self.verdicts), "--out", str(out),
"--sample", str(sample)])
return code, json.loads(out.read_text(encoding="utf-8")) if out.is_file() else {}
def render(self, size: int = 3) -> int:
return run(["render", "--utvalg", str(self.m / "utvalg.json"), "--claims", str(self.claims), "--sources",
str(self.sources), "--wiki", str(self.wiki), "--out", str(self.m / "tasks"), "--size", str(size)])[0]
class SelectTest(Fixture):
def test_the_seed_gives_the_same_selection_twice(self) -> None:
code, a = self.select("a.json")
self.assertEqual(0, code)
_, b = self.select("b.json")
self.assertEqual((self.m / "a.json").read_bytes(), (self.m / "b.json").read_bytes())
self.assertEqual(7, len(a["ids"])) # 3 unsupported + 4 of the 9 supported
self.assertTrue(UNSUPPORTED <= set(a["ids"]))
self.assertEqual(7, len(set(a["ids"])))
self.assertEqual(dg.SEED, a["seed"])
self.assertEqual(20261012, dg.SEED)
def test_the_selection_carries_no_verdict(self) -> None:
_, doc = self.select()
text = (self.m / "utvalg.json").read_text(encoding="utf-8")
self.assertNotIn("unsupported", text)
self.assertNotIn("verdict", text)
self.assertNotIn("supported", text)
self.assertEqual({"schema", "seed", "sample", "ids"}, set(doc))
def test_by_hand_the_sample_and_the_shuffle_use_one_generator(self) -> None:
import random
_, doc = self.select()
rng = random.Random(20261012)
supported = sorted(f"g{n:02d}" for n in range(1, 13) if f"g{n:02d}" not in UNSUPPORTED)
picked = rng.sample(supported, 4)
ids = sorted(UNSUPPORTED) + sorted(picked)
rng.shuffle(ids)
self.assertEqual(ids, doc["ids"])
def test_too_few_supported_or_a_claim_without_verdict_cannot_run(self) -> None:
self.assertEqual(2, self.select(sample=10)[0])
doc = json.loads(self.verdicts.read_text(encoding="utf-8"))
doc["verdicts"] = doc["verdicts"][:-1]
self.verdicts.write_text(json.dumps(doc), encoding="utf-8")
self.assertEqual(2, self.select("c.json")[0])
class RenderTest(Fixture):
def setUp(self) -> None:
super().setUp()
self.select()
self.ids = json.loads((self.m / "utvalg.json").read_text(encoding="utf-8"))["ids"]
def test_one_task_per_call_of_at_most_size_claims_in_selection_order(self) -> None:
self.assertEqual(0, self.render(size=3))
calls = json.loads((self.m / "tasks" / "calls.json").read_text(encoding="utf-8"))
self.assertEqual(["d001", "d002", "d003"], [c["name"] for c in calls["calls"]])
self.assertEqual(self.ids, [i for c in calls["calls"] for i in c["ids"]])
self.assertEqual([3, 3, 1], [len(c["ids"]) for c in calls["calls"]])
self.assertEqual(["calls.json", "d001.md", "d002.md", "d003.md"],
sorted(p.name for p in (self.m / "tasks").iterdir()))
def task_of(self, cid: str, size: int = 10) -> tuple[str, int]:
self.render(size=size)
calls = json.loads((self.m / "tasks" / "calls.json").read_text(encoding="utf-8"))["calls"]
for c in calls:
if cid in c["ids"]:
return (self.m / "tasks" / f"{c['name']}.md").read_text(encoding="utf-8"), c["ids"].index(cid) + 1
raise AssertionError(cid)
def window(self, text: str, number: int) -> list[str]:
block = text.split(f"## Claim {number}\n", 1)[1].split("\n## Claim ", 1)[0]
body = block.split("<window", 1)[1].split("</window>", 1)[0]
return body.split("\n")
def test_the_window_stops_at_the_files_start_and_marks_the_quote_and_code(self) -> None:
text, number = self.task_of("g02") # quotes line 11: lines 3..19, under Garden care > Tools
head, *rows = self.window(text, number)
self.assertIn('lines="3-19"', head)
self.assertIn('headings="Garden care › Tools"', head)
numbered = [r for r in rows if re.match(r"^\s*\d+ ", r)]
self.assertEqual(list(range(3, 20)), [int(r.split()[0]) for r in numbered])
self.assertTrue(any(r.startswith(" 11 > |") for r in numbered), numbered)
self.assertTrue(any(r.startswith(" 8 c|") for r in numbered), numbered)
self.assertTrue(any(r.startswith(" 7 c|") for r in numbered), numbered)
self.assertTrue(any(r.startswith(" 3 |") for r in numbered), numbered)
text, number = self.task_of("g09") # quotes line 17: lines 9..21, the file ends at 21
head, *_ = self.window(text, number)
self.assertIn('lines="9-21"', head)
self.assertIn('headings="Garden care › Compost"', head)
def test_the_task_shows_the_page_title_claim_and_excerpt_and_no_id_or_verdict(self) -> None:
text, number = self.task_of("g05")
self.assertIn("Page title: Garden upkeep", text)
self.assertIn("<claim>Gardeners follow rule number 5.</claim>", text)
self.assertIn(f'<excerpt file="docs-garden.md" line="21">\n{QUOTES[21]}\n</excerpt>', text)
for word in ("g05", "supported", "verdict", "merged", "documented"):
self.assertNotIn(word, text)
def test_an_existing_task_folder_is_never_written_into(self) -> None:
self.assertEqual(0, self.render())
before = (self.m / "tasks" / "d001.md").read_bytes()
self.assertEqual(2, self.render(size=2))
self.assertEqual(before, (self.m / "tasks" / "d001.md").read_bytes())
class CollectTest(Fixture):
def setUp(self) -> None:
super().setUp()
self.select()
self.render(size=3)
self.calls = json.loads((self.m / "tasks" / "calls.json").read_text(encoding="utf-8"))["calls"]
self.proj = self.root / "transcripts"
def reply(self, run_name: str, call: str, text: str, workflow: str | None = None, handback: bool = False,
agent_type: str = "okf-wiki-diagnoser") -> None:
sub = self.proj / "s1" / "subagents"
if workflow:
sub = sub / "workflows" / workflow
name = f"agent-{run_name}-{call}"
block = ({"type": "tool_use", "name": "SubagentHandback", "input": {"message": text}} if handback
else {"type": "text", "text": text})
jsonl(sub / f"{name}.jsonl", [{"type": "user", "message": {"content": "task"}},
{"type": "assistant", "message": {"id": f"m-{name}", "content": [block]}}])
(sub / f"{name}.meta.json").write_text(json.dumps({"agentType": agent_type,
"description": f"V1-{run_name}-diagnose-{call}"}))
def collect(self, *runs: str) -> tuple[int, dict[str, Any]]:
out = self.m / f"diagnoses-{len(list(self.m.glob('diagnoses-*')))}.json"
argv = ["collect", "--transcripts", str(self.proj), "--tasks", str(self.m / "tasks"), "--out", str(out)]
for r in runs:
argv += ["--run", r]
code, _ = run(argv)
return code, json.loads(out.read_text(encoding="utf-8")) if out.is_file() else {}
def by_id(self, doc: dict[str, Any]) -> dict[str, dict[str, Any]]:
return {d["id"]: d for d in doc["diagnoses"]}
def test_one_line_per_claim_and_every_bad_line_is_missing(self) -> None:
d1, d2, d3 = (c["ids"] for c in self.calls)
self.reply("del1d", "d001", "1: L | the spade\n2: S |\n3: Q | wrong code")
self.reply("del1d", "d002", "Some words first.\n1: A | " + "word " * 13 + "\n2: O | Compost\n2: U | twice\n"
"3: B | every", workflow="wf_x", handback=True)
code, doc = self.collect("del1d")
self.assertEqual(1, code) # some are missing
found = self.by_id(doc)
self.assertEqual(("L", "the spade"), (found[d1[0]]["code"], found[d1[0]]["words"]))
self.assertEqual(("S", ""), (found[d1[1]]["code"], found[d1[1]]["words"]))
self.assertIsNone(found[d1[2]]["code"]) # Q is not a code
self.assertIsNone(found[d2[0]]["code"]) # A with more than twelve words
self.assertIsNone(found[d2[1]]["code"]) # number 2 given twice
self.assertEqual(("B", "every"), (found[d2[2]]["code"], found[d2[2]]["words"]))
self.assertIsNone(found[d3[0]]["code"]) # d003 never answered
self.assertEqual(sorted([d1[2], d2[0], d2[1], d3[0]]), sorted(doc["missing"]))
self.assertEqual([i for c in self.calls for i in c["ids"]], [d["id"] for d in doc["diagnoses"]])
def test_a_code_other_than_s_needs_the_missing_words(self) -> None:
self.reply("del1d", "d001", "1: T |\n2: S\n3: F | the heap")
_, doc = self.collect("del1d")
found = self.by_id(doc)
d1 = self.calls[0]["ids"]
self.assertIsNone(found[d1[0]]["code"])
self.assertEqual(("S", "F"), (found[d1[1]]["code"], found[d1[2]]["code"]))
def test_a_later_run_answers_a_call_again_and_two_transcripts_under_one_name_are_missing(self) -> None:
d1, d2, d3 = (c["ids"] for c in self.calls)
self.reply("del1d", "d001", "1: S |\n2: S |\n3: S |")
self.reply("del1d", "d002", "1: S |\n2: S |\n3: S |")
self.reply("del1d", "d003", "1: S |")
self.reply("del1d2", "d002", "1: U | rain\n2: U | rain\n3: U | rain")
self.reply("del1d2", "d003", "1: O | Leaves")
self.reply("del1d2", "d003", "1: L | heap", workflow="wf_y")
code, doc = self.collect("del1d", "del1d2")
found = self.by_id(doc)
self.assertEqual("S", found[d1[0]]["code"])
self.assertEqual(("U", "del1d2"), (found[d2[0]]["code"], found[d2[0]]["run"]))
self.assertIsNone(found[d3[0]]["code"]) # del1d2 answered d003 twice
self.assertEqual(1, code)
def test_another_agent_type_is_not_a_diagnosis(self) -> None:
self.reply("del1d", "d001", "1: S |\n2: S |\n3: S |", agent_type="general-purpose")
_, doc = self.collect("del1d")
self.assertEqual(7, len(doc["missing"]))
def test_a_diagnoser_started_from_a_plugin_is_collected(self) -> None:
self.reply("del1d", "d001", "1: S |\n2: S |\n3: S |", agent_type="okf-wiki:okf-wiki-diagnoser")
_, doc = self.collect("del1d")
self.assertEqual(4, len(doc["missing"]))
class TallyTest(Fixture):
def diagnoses(self, codes: dict[str, str | None]) -> Path:
path = self.root / "diagnoses.json"
path.write_text(json.dumps({"schema": dg.DIAGNOSES_SCHEMA, "diagnoses": [
{"id": cid, "call": "d001", "number": 1, "code": code, "words": "", "run": "del1d"}
for cid, code in codes.items()], "missing": [c for c, v in codes.items() if v is None]}), encoding="utf-8")
return path
def tally(self, codes: dict[str, str | None]) -> tuple[int, dict[str, Any], str]:
out = self.m / f"tally-{len(list(self.m.glob('tally-*')))}.json"
code, text = run(["tally", "--diagnoses", str(self.diagnoses(codes)), "--verdicts", str(self.verdicts),
"--claims", str(self.claims), "--out", str(out)])
return code, json.loads(out.read_text(encoding="utf-8")) if out.is_file() else {}, text
def test_counts_per_code_agreement_and_groups_by_hand(self) -> None:
# unsupported g02 (L), g05 (B, merged), g09 (S); supported sample g01 (S), g03 (T), g04 (S), g10 (missing)
code, doc, text = self.tally({"g02": "L", "g05": "B", "g09": "S", "g01": "S", "g03": "T", "g04": "S",
"g10": None})
self.assertEqual({"S": 1, "L": 1, "B": 1, "n": 3},
{k: v for k, v in doc["unsupported"].items() if v})
self.assertEqual({"S": 2, "T": 1, "missing": 1, "n": 4}, {k: v for k, v in doc["supported_sample"].items() if v})
self.assertEqual(set(dg.CODES) | {"missing", "n"}, set(doc["unsupported"]))
a = doc["agreement"]
self.assertEqual((1, 3), (a["unsupported_called_s"], a["unsupported_diagnosed"]))
self.assertEqual((1, 3), (a["supported_not_carried"], a["supported_diagnosed"]))
self.assertEqual({"S": 0, "L": 0, "B": 1, "missing": 0, "n": 1},
{k: v for k, v in doc["per_provenance"]["merged"]["unsupported"].items()
if k in ("S", "L", "B", "missing", "n")})
self.assertEqual(3, doc["per_source"]["docs-garden.md"]["unsupported"]["n"])
self.assertEqual({"s": 1, "of": 3, "missing": 1, "stop": True}, doc["m1"]) # 1 of 3 is a third: stop
self.assertEqual(1, code)
self.assertIn("M1", text)
def test_under_a_third_s_and_nothing_missing_passes_the_m1_rule(self) -> None:
code, doc, _ = self.tally({"g02": "L", "g05": "B", "g09": "U", "g01": "S"})
self.assertEqual({"s": 0, "of": 3, "missing": 0, "stop": False}, doc["m1"])
self.assertEqual(0, code)
def test_clopper_pearson_by_known_values(self) -> None:
lo, hi = dg.clopper_pearson(0, 100)
self.assertEqual(0.0, lo)
self.assertAlmostEqual(1 - 0.025 ** (1 / 100), hi, places=6)
lo, hi = dg.clopper_pearson(5, 100)
self.assertAlmostEqual(0.01643, lo, places=4)
self.assertAlmostEqual(0.11284, hi, places=4)
self.assertEqual((0.0, 1.0), dg.clopper_pearson(0, 0))
class AgentFileTest(unittest.TestCase):
def setUp(self) -> None:
self.text = AGENT.read_text(encoding="utf-8")
def test_frontmatter_names_the_type_read_only_and_opus(self) -> None:
head = self.text.split("\n---\n", 1)[0]
self.assertIn("name: okf-wiki-diagnoser", head)
self.assertIn("tools: Read", head)
self.assertIn("model: opus", head)
self.assertNotIn("Write", head)
def test_no_run_of_eight_words_from_the_readers_instructions(self) -> None:
body = reader.instructions(self.text)
pair = reader.Pair("agent", body, ())
self.assertEqual([], reader.shared_runs(reader.INSTRUCTIONS, [pair]))
def test_it_names_every_code_and_the_reply_form(self) -> None:
body = reader.instructions(self.text)
for code in dg.CODES:
self.assertRegex(body, rf"\*\*{code}\*\*")
self.assertIn("<number>: <code> | <words>", body)
self.assertNotIn("supported", body) # it never names the reader's verdicts
class NoModelTest(unittest.TestCase):
def test_the_tool_imports_no_network_or_model_client(self) -> None:
source = (ROOT / "tools" / "diagnose.py").read_text(encoding="utf-8")
for banned in ("anthropic", "openai", "urllib", "http.client", "requests", "subprocess", "socket"):
self.assertNotIn(f"import {banned}", source)
if __name__ == "__main__":
unittest.main()