First public release, published as one snapshot. What changed between versions is in CHANGELOG.md. Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
377 lines
19 KiB
Python
377 lines
19 KiB
Python
"""Tests for tools/diagnose.py and .claude/agents/okf-wiki-diagnoser.md: why a claim is not carried by its excerpt
|
||
(wave 4 order 20, contract K9). The diagnosis explains; the reader judges; nothing here is a verdict.
|
||
|
||
The fixture is a source of gardening notes, one wiki page and twelve claims, written by hand; every expected number is
|
||
worked out by hand from it. The text is synthetic, about gardening (CLAUDE.md rule 3).
|
||
Run: python3 -m unittest discover -s tests -p 'test_diagnose.py'
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import contextlib
|
||
import importlib.util
|
||
import io
|
||
import json
|
||
import re
|
||
import sys
|
||
import tempfile
|
||
import unittest
|
||
from pathlib import Path
|
||
from typing import Any
|
||
|
||
from okfwiki import reader
|
||
|
||
ROOT = Path(__file__).resolve().parents[1]
|
||
AGENT = ROOT / ".claude" / "agents" / "okf-wiki-diagnoser.md"
|
||
|
||
_SPEC = importlib.util.spec_from_file_location("diagnose", ROOT / "tools" / "diagnose.py")
|
||
assert _SPEC is not None and _SPEC.loader is not None
|
||
dg: Any = importlib.util.module_from_spec(_SPEC)
|
||
sys.modules["diagnose"] = dg
|
||
_SPEC.loader.exec_module(dg)
|
||
|
||
SOURCE_LINES = [
|
||
"# Garden care", # 1
|
||
"", # 2
|
||
"Water the beds early in the day.", # 3
|
||
"", # 4
|
||
"## Tools", # 5
|
||
"", # 6
|
||
"```sh", # 7
|
||
"# sharpen the spade", # 8
|
||
"```", # 9
|
||
"", # 10
|
||
"A sharp spade cuts roots cleanly.", # 11
|
||
"", # 12
|
||
"## Compost", # 13
|
||
"", # 14
|
||
"Turn the heap every second week.", # 15
|
||
"", # 16
|
||
"Keep the heap as damp as a wrung sponge.", # 17
|
||
"", # 18
|
||
"### Leaves", # 19
|
||
"", # 20
|
||
"Shred autumn leaves before adding them.", # 21
|
||
]
|
||
SOURCE = "\n".join(SOURCE_LINES)
|
||
QUOTES = {3: SOURCE_LINES[2], 11: SOURCE_LINES[10], 15: SOURCE_LINES[14], 17: SOURCE_LINES[16], 21: SOURCE_LINES[20]}
|
||
LINES_OF = [3, 11, 15, 17, 21, 3, 11, 15, 17, 21, 3, 11] # the quoted line of claims g01 .. g12
|
||
UNSUPPORTED = {"g02", "g05", "g09"}
|
||
|
||
|
||
def claim(n: int) -> dict[str, Any]:
|
||
line = LINES_OF[n - 1]
|
||
return {"id": f"g{n:02d}", "page": "garden.md", "line": 6, "claim": f"Gardeners follow rule number {n}.",
|
||
"footnotes": ["s1"], "excerpts": [{"file": "docs-garden.md", "line": line, "text": QUOTES[line]}],
|
||
"quote_ok": True, "excerpt_sha256": "0" * 64, "provenance": "merged" if n == 5 else "documented",
|
||
"resources": ["/kilde/docs-garden.md"]}
|
||
|
||
|
||
def run(argv: list[str]) -> tuple[int, str]:
|
||
buf = io.StringIO()
|
||
with contextlib.redirect_stdout(buf), contextlib.redirect_stderr(io.StringIO()):
|
||
code = dg.main(argv)
|
||
return code, buf.getvalue()
|
||
|
||
|
||
def jsonl(path: Path, lines: list[dict[str, Any]]) -> None:
|
||
path.parent.mkdir(parents=True, exist_ok=True)
|
||
path.write_text("\n".join(json.dumps(x) for x in lines) + "\n", encoding="utf-8")
|
||
|
||
|
||
class Fixture(unittest.TestCase):
|
||
def setUp(self) -> None:
|
||
self._tmp = tempfile.TemporaryDirectory()
|
||
self.root = Path(self._tmp.name)
|
||
self.sources = self.root / "src"
|
||
(self.sources / "kilde").mkdir(parents=True)
|
||
(self.sources / "kilde" / "docs-garden.md").write_text(SOURCE, encoding="utf-8")
|
||
self.wiki = self.root / "wiki"
|
||
self.wiki.mkdir()
|
||
(self.wiki / "garden.md").write_text("---\ntype: Concept\ntitle: Garden upkeep\n---\n\nBody.\n", encoding="utf-8")
|
||
self.claims = self.root / "claims.json"
|
||
self.claims.write_text(json.dumps({"schema": "okf-wiki/claims/1", "claims": [claim(n) for n in range(1, 13)],
|
||
"exceptions": [], "unkeyed": []}), encoding="utf-8")
|
||
self.verdicts = self.root / "verdicts.json"
|
||
self.verdicts.write_text(json.dumps({"reader": "okf-wiki-reader", "verdicts": [
|
||
{"id": f"g{n:02d}", "verdict": "unsupported" if f"g{n:02d}" in UNSUPPORTED else "supported"}
|
||
for n in range(1, 13)]}), encoding="utf-8")
|
||
self.m = self.root / "M"
|
||
|
||
def tearDown(self) -> None:
|
||
self._tmp.cleanup()
|
||
|
||
def select(self, name: str = "utvalg.json", sample: int = 4) -> tuple[int, dict[str, Any]]:
|
||
out = self.m / name
|
||
code, _ = run(["select", "--claims", str(self.claims), "--verdicts", str(self.verdicts), "--out", str(out),
|
||
"--sample", str(sample)])
|
||
return code, json.loads(out.read_text(encoding="utf-8")) if out.is_file() else {}
|
||
|
||
def render(self, size: int = 3) -> int:
|
||
return run(["render", "--utvalg", str(self.m / "utvalg.json"), "--claims", str(self.claims), "--sources",
|
||
str(self.sources), "--wiki", str(self.wiki), "--out", str(self.m / "tasks"), "--size", str(size)])[0]
|
||
|
||
|
||
class SelectTest(Fixture):
|
||
def test_the_seed_gives_the_same_selection_twice(self) -> None:
|
||
code, a = self.select("a.json")
|
||
self.assertEqual(0, code)
|
||
_, b = self.select("b.json")
|
||
self.assertEqual((self.m / "a.json").read_bytes(), (self.m / "b.json").read_bytes())
|
||
self.assertEqual(7, len(a["ids"])) # 3 unsupported + 4 of the 9 supported
|
||
self.assertTrue(UNSUPPORTED <= set(a["ids"]))
|
||
self.assertEqual(7, len(set(a["ids"])))
|
||
self.assertEqual(dg.SEED, a["seed"])
|
||
self.assertEqual(20261012, dg.SEED)
|
||
|
||
def test_the_selection_carries_no_verdict(self) -> None:
|
||
_, doc = self.select()
|
||
text = (self.m / "utvalg.json").read_text(encoding="utf-8")
|
||
self.assertNotIn("unsupported", text)
|
||
self.assertNotIn("verdict", text)
|
||
self.assertNotIn("supported", text)
|
||
self.assertEqual({"schema", "seed", "sample", "ids"}, set(doc))
|
||
|
||
def test_by_hand_the_sample_and_the_shuffle_use_one_generator(self) -> None:
|
||
import random
|
||
|
||
_, doc = self.select()
|
||
rng = random.Random(20261012)
|
||
supported = sorted(f"g{n:02d}" for n in range(1, 13) if f"g{n:02d}" not in UNSUPPORTED)
|
||
picked = rng.sample(supported, 4)
|
||
ids = sorted(UNSUPPORTED) + sorted(picked)
|
||
rng.shuffle(ids)
|
||
self.assertEqual(ids, doc["ids"])
|
||
|
||
def test_too_few_supported_or_a_claim_without_verdict_cannot_run(self) -> None:
|
||
self.assertEqual(2, self.select(sample=10)[0])
|
||
doc = json.loads(self.verdicts.read_text(encoding="utf-8"))
|
||
doc["verdicts"] = doc["verdicts"][:-1]
|
||
self.verdicts.write_text(json.dumps(doc), encoding="utf-8")
|
||
self.assertEqual(2, self.select("c.json")[0])
|
||
|
||
|
||
class RenderTest(Fixture):
|
||
def setUp(self) -> None:
|
||
super().setUp()
|
||
self.select()
|
||
self.ids = json.loads((self.m / "utvalg.json").read_text(encoding="utf-8"))["ids"]
|
||
|
||
def test_one_task_per_call_of_at_most_size_claims_in_selection_order(self) -> None:
|
||
self.assertEqual(0, self.render(size=3))
|
||
calls = json.loads((self.m / "tasks" / "calls.json").read_text(encoding="utf-8"))
|
||
self.assertEqual(["d001", "d002", "d003"], [c["name"] for c in calls["calls"]])
|
||
self.assertEqual(self.ids, [i for c in calls["calls"] for i in c["ids"]])
|
||
self.assertEqual([3, 3, 1], [len(c["ids"]) for c in calls["calls"]])
|
||
self.assertEqual(["calls.json", "d001.md", "d002.md", "d003.md"],
|
||
sorted(p.name for p in (self.m / "tasks").iterdir()))
|
||
|
||
def task_of(self, cid: str, size: int = 10) -> tuple[str, int]:
|
||
self.render(size=size)
|
||
calls = json.loads((self.m / "tasks" / "calls.json").read_text(encoding="utf-8"))["calls"]
|
||
for c in calls:
|
||
if cid in c["ids"]:
|
||
return (self.m / "tasks" / f"{c['name']}.md").read_text(encoding="utf-8"), c["ids"].index(cid) + 1
|
||
raise AssertionError(cid)
|
||
|
||
def window(self, text: str, number: int) -> list[str]:
|
||
block = text.split(f"## Claim {number}\n", 1)[1].split("\n## Claim ", 1)[0]
|
||
body = block.split("<window", 1)[1].split("</window>", 1)[0]
|
||
return body.split("\n")
|
||
|
||
def test_the_window_stops_at_the_files_start_and_marks_the_quote_and_code(self) -> None:
|
||
text, number = self.task_of("g02") # quotes line 11: lines 3..19, under Garden care > Tools
|
||
head, *rows = self.window(text, number)
|
||
self.assertIn('lines="3-19"', head)
|
||
self.assertIn('headings="Garden care › Tools"', head)
|
||
numbered = [r for r in rows if re.match(r"^\s*\d+ ", r)]
|
||
self.assertEqual(list(range(3, 20)), [int(r.split()[0]) for r in numbered])
|
||
self.assertTrue(any(r.startswith(" 11 > |") for r in numbered), numbered)
|
||
self.assertTrue(any(r.startswith(" 8 c|") for r in numbered), numbered)
|
||
self.assertTrue(any(r.startswith(" 7 c|") for r in numbered), numbered)
|
||
self.assertTrue(any(r.startswith(" 3 |") for r in numbered), numbered)
|
||
text, number = self.task_of("g09") # quotes line 17: lines 9..21, the file ends at 21
|
||
head, *_ = self.window(text, number)
|
||
self.assertIn('lines="9-21"', head)
|
||
self.assertIn('headings="Garden care › Compost"', head)
|
||
|
||
def test_the_task_shows_the_page_title_claim_and_excerpt_and_no_id_or_verdict(self) -> None:
|
||
text, number = self.task_of("g05")
|
||
self.assertIn("Page title: Garden upkeep", text)
|
||
self.assertIn("<claim>Gardeners follow rule number 5.</claim>", text)
|
||
self.assertIn(f'<excerpt file="docs-garden.md" line="21">\n{QUOTES[21]}\n</excerpt>', text)
|
||
for word in ("g05", "supported", "verdict", "merged", "documented"):
|
||
self.assertNotIn(word, text)
|
||
|
||
def test_an_existing_task_folder_is_never_written_into(self) -> None:
|
||
self.assertEqual(0, self.render())
|
||
before = (self.m / "tasks" / "d001.md").read_bytes()
|
||
self.assertEqual(2, self.render(size=2))
|
||
self.assertEqual(before, (self.m / "tasks" / "d001.md").read_bytes())
|
||
|
||
|
||
class CollectTest(Fixture):
|
||
def setUp(self) -> None:
|
||
super().setUp()
|
||
self.select()
|
||
self.render(size=3)
|
||
self.calls = json.loads((self.m / "tasks" / "calls.json").read_text(encoding="utf-8"))["calls"]
|
||
self.proj = self.root / "transcripts"
|
||
|
||
def reply(self, run_name: str, call: str, text: str, workflow: str | None = None, handback: bool = False,
|
||
agent_type: str = "okf-wiki-diagnoser") -> None:
|
||
sub = self.proj / "s1" / "subagents"
|
||
if workflow:
|
||
sub = sub / "workflows" / workflow
|
||
name = f"agent-{run_name}-{call}"
|
||
block = ({"type": "tool_use", "name": "SubagentHandback", "input": {"message": text}} if handback
|
||
else {"type": "text", "text": text})
|
||
jsonl(sub / f"{name}.jsonl", [{"type": "user", "message": {"content": "task"}},
|
||
{"type": "assistant", "message": {"id": f"m-{name}", "content": [block]}}])
|
||
(sub / f"{name}.meta.json").write_text(json.dumps({"agentType": agent_type,
|
||
"description": f"V1-{run_name}-diagnose-{call}"}))
|
||
|
||
def collect(self, *runs: str) -> tuple[int, dict[str, Any]]:
|
||
out = self.m / f"diagnoses-{len(list(self.m.glob('diagnoses-*')))}.json"
|
||
argv = ["collect", "--transcripts", str(self.proj), "--tasks", str(self.m / "tasks"), "--out", str(out)]
|
||
for r in runs:
|
||
argv += ["--run", r]
|
||
code, _ = run(argv)
|
||
return code, json.loads(out.read_text(encoding="utf-8")) if out.is_file() else {}
|
||
|
||
def by_id(self, doc: dict[str, Any]) -> dict[str, dict[str, Any]]:
|
||
return {d["id"]: d for d in doc["diagnoses"]}
|
||
|
||
def test_one_line_per_claim_and_every_bad_line_is_missing(self) -> None:
|
||
d1, d2, d3 = (c["ids"] for c in self.calls)
|
||
self.reply("del1d", "d001", "1: L | the spade\n2: S |\n3: Q | wrong code")
|
||
self.reply("del1d", "d002", "Some words first.\n1: A | " + "word " * 13 + "\n2: O | Compost\n2: U | twice\n"
|
||
"3: B | every", workflow="wf_x", handback=True)
|
||
code, doc = self.collect("del1d")
|
||
self.assertEqual(1, code) # some are missing
|
||
found = self.by_id(doc)
|
||
self.assertEqual(("L", "the spade"), (found[d1[0]]["code"], found[d1[0]]["words"]))
|
||
self.assertEqual(("S", ""), (found[d1[1]]["code"], found[d1[1]]["words"]))
|
||
self.assertIsNone(found[d1[2]]["code"]) # Q is not a code
|
||
self.assertIsNone(found[d2[0]]["code"]) # A with more than twelve words
|
||
self.assertIsNone(found[d2[1]]["code"]) # number 2 given twice
|
||
self.assertEqual(("B", "every"), (found[d2[2]]["code"], found[d2[2]]["words"]))
|
||
self.assertIsNone(found[d3[0]]["code"]) # d003 never answered
|
||
self.assertEqual(sorted([d1[2], d2[0], d2[1], d3[0]]), sorted(doc["missing"]))
|
||
self.assertEqual([i for c in self.calls for i in c["ids"]], [d["id"] for d in doc["diagnoses"]])
|
||
|
||
def test_a_code_other_than_s_needs_the_missing_words(self) -> None:
|
||
self.reply("del1d", "d001", "1: T |\n2: S\n3: F | the heap")
|
||
_, doc = self.collect("del1d")
|
||
found = self.by_id(doc)
|
||
d1 = self.calls[0]["ids"]
|
||
self.assertIsNone(found[d1[0]]["code"])
|
||
self.assertEqual(("S", "F"), (found[d1[1]]["code"], found[d1[2]]["code"]))
|
||
|
||
def test_a_later_run_answers_a_call_again_and_two_transcripts_under_one_name_are_missing(self) -> None:
|
||
d1, d2, d3 = (c["ids"] for c in self.calls)
|
||
self.reply("del1d", "d001", "1: S |\n2: S |\n3: S |")
|
||
self.reply("del1d", "d002", "1: S |\n2: S |\n3: S |")
|
||
self.reply("del1d", "d003", "1: S |")
|
||
self.reply("del1d2", "d002", "1: U | rain\n2: U | rain\n3: U | rain")
|
||
self.reply("del1d2", "d003", "1: O | Leaves")
|
||
self.reply("del1d2", "d003", "1: L | heap", workflow="wf_y")
|
||
code, doc = self.collect("del1d", "del1d2")
|
||
found = self.by_id(doc)
|
||
self.assertEqual("S", found[d1[0]]["code"])
|
||
self.assertEqual(("U", "del1d2"), (found[d2[0]]["code"], found[d2[0]]["run"]))
|
||
self.assertIsNone(found[d3[0]]["code"]) # del1d2 answered d003 twice
|
||
self.assertEqual(1, code)
|
||
|
||
def test_another_agent_type_is_not_a_diagnosis(self) -> None:
|
||
self.reply("del1d", "d001", "1: S |\n2: S |\n3: S |", agent_type="general-purpose")
|
||
_, doc = self.collect("del1d")
|
||
self.assertEqual(7, len(doc["missing"]))
|
||
|
||
def test_a_diagnoser_started_from_a_plugin_is_collected(self) -> None:
|
||
self.reply("del1d", "d001", "1: S |\n2: S |\n3: S |", agent_type="okf-wiki:okf-wiki-diagnoser")
|
||
_, doc = self.collect("del1d")
|
||
self.assertEqual(4, len(doc["missing"]))
|
||
|
||
|
||
class TallyTest(Fixture):
|
||
def diagnoses(self, codes: dict[str, str | None]) -> Path:
|
||
path = self.root / "diagnoses.json"
|
||
path.write_text(json.dumps({"schema": dg.DIAGNOSES_SCHEMA, "diagnoses": [
|
||
{"id": cid, "call": "d001", "number": 1, "code": code, "words": "", "run": "del1d"}
|
||
for cid, code in codes.items()], "missing": [c for c, v in codes.items() if v is None]}), encoding="utf-8")
|
||
return path
|
||
|
||
def tally(self, codes: dict[str, str | None]) -> tuple[int, dict[str, Any], str]:
|
||
out = self.m / f"tally-{len(list(self.m.glob('tally-*')))}.json"
|
||
code, text = run(["tally", "--diagnoses", str(self.diagnoses(codes)), "--verdicts", str(self.verdicts),
|
||
"--claims", str(self.claims), "--out", str(out)])
|
||
return code, json.loads(out.read_text(encoding="utf-8")) if out.is_file() else {}, text
|
||
|
||
def test_counts_per_code_agreement_and_groups_by_hand(self) -> None:
|
||
# unsupported g02 (L), g05 (B, merged), g09 (S); supported sample g01 (S), g03 (T), g04 (S), g10 (missing)
|
||
code, doc, text = self.tally({"g02": "L", "g05": "B", "g09": "S", "g01": "S", "g03": "T", "g04": "S",
|
||
"g10": None})
|
||
self.assertEqual({"S": 1, "L": 1, "B": 1, "n": 3},
|
||
{k: v for k, v in doc["unsupported"].items() if v})
|
||
self.assertEqual({"S": 2, "T": 1, "missing": 1, "n": 4}, {k: v for k, v in doc["supported_sample"].items() if v})
|
||
self.assertEqual(set(dg.CODES) | {"missing", "n"}, set(doc["unsupported"]))
|
||
a = doc["agreement"]
|
||
self.assertEqual((1, 3), (a["unsupported_called_s"], a["unsupported_diagnosed"]))
|
||
self.assertEqual((1, 3), (a["supported_not_carried"], a["supported_diagnosed"]))
|
||
self.assertEqual({"S": 0, "L": 0, "B": 1, "missing": 0, "n": 1},
|
||
{k: v for k, v in doc["per_provenance"]["merged"]["unsupported"].items()
|
||
if k in ("S", "L", "B", "missing", "n")})
|
||
self.assertEqual(3, doc["per_source"]["docs-garden.md"]["unsupported"]["n"])
|
||
self.assertEqual({"s": 1, "of": 3, "missing": 1, "stop": True}, doc["m1"]) # 1 of 3 is a third: stop
|
||
self.assertEqual(1, code)
|
||
self.assertIn("M1", text)
|
||
|
||
def test_under_a_third_s_and_nothing_missing_passes_the_m1_rule(self) -> None:
|
||
code, doc, _ = self.tally({"g02": "L", "g05": "B", "g09": "U", "g01": "S"})
|
||
self.assertEqual({"s": 0, "of": 3, "missing": 0, "stop": False}, doc["m1"])
|
||
self.assertEqual(0, code)
|
||
|
||
def test_clopper_pearson_by_known_values(self) -> None:
|
||
lo, hi = dg.clopper_pearson(0, 100)
|
||
self.assertEqual(0.0, lo)
|
||
self.assertAlmostEqual(1 - 0.025 ** (1 / 100), hi, places=6)
|
||
lo, hi = dg.clopper_pearson(5, 100)
|
||
self.assertAlmostEqual(0.01643, lo, places=4)
|
||
self.assertAlmostEqual(0.11284, hi, places=4)
|
||
self.assertEqual((0.0, 1.0), dg.clopper_pearson(0, 0))
|
||
|
||
|
||
class AgentFileTest(unittest.TestCase):
|
||
def setUp(self) -> None:
|
||
self.text = AGENT.read_text(encoding="utf-8")
|
||
|
||
def test_frontmatter_names_the_type_read_only_and_opus(self) -> None:
|
||
head = self.text.split("\n---\n", 1)[0]
|
||
self.assertIn("name: okf-wiki-diagnoser", head)
|
||
self.assertIn("tools: Read", head)
|
||
self.assertIn("model: opus", head)
|
||
self.assertNotIn("Write", head)
|
||
|
||
def test_no_run_of_eight_words_from_the_readers_instructions(self) -> None:
|
||
body = reader.instructions(self.text)
|
||
pair = reader.Pair("agent", body, ())
|
||
self.assertEqual([], reader.shared_runs(reader.INSTRUCTIONS, [pair]))
|
||
|
||
def test_it_names_every_code_and_the_reply_form(self) -> None:
|
||
body = reader.instructions(self.text)
|
||
for code in dg.CODES:
|
||
self.assertRegex(body, rf"\*\*{code}\*\*")
|
||
self.assertIn("<number>: <code> | <words>", body)
|
||
self.assertNotIn("supported", body) # it never names the reader's verdicts
|
||
|
||
|
||
class NoModelTest(unittest.TestCase):
|
||
def test_the_tool_imports_no_network_or_model_client(self) -> None:
|
||
source = (ROOT / "tools" / "diagnose.py").read_text(encoding="utf-8")
|
||
for banned in ("anthropic", "openai", "urllib", "http.client", "requests", "subprocess", "socket"):
|
||
self.assertNotIn(f"import {banned}", source)
|
||
|
||
|
||
if __name__ == "__main__":
|
||
unittest.main()
|