"""Tests for okfwiki/reader.py, the tool around the reader pass (PLAN.md criterion 2). It calls no model. The reader judges up to ten claims per call and gives one verdict per claim (K3 in the wave-1 contracts). `render` writes one task per call of a call layout: the reader's instructions, then the claims numbered 1. to n. in layout order, each with its excerpts, and nothing else. `collect` reads the readers' replies from the subagent transcripts and turns them into the verdicts file that gates/calibration.py scores, by one fail-closed rule. Every fixture is synthetic text about baking bread (CLAUDE.md rule 3). Run: python3 -m unittest discover -s tests -p 'test_reader.py' """ from __future__ import annotations import contextlib import hashlib import importlib.util import io import json import sys import tempfile import unittest from pathlib import Path from typing import Any from unittest import mock ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT)) from okfwiki import reader # noqa: E402 _SPEC = importlib.util.spec_from_file_location("calibration", ROOT / "gates" / "calibration.py") assert _SPEC is not None and _SPEC.loader is not None calibration = importlib.util.module_from_spec(_SPEC) sys.modules["calibration"] = calibration _SPEC.loader.exec_module(calibration) AGENT = """--- name: test-reader description: A reader used in tests. tools: Read --- Read the one task file you are given and follow it. """ PAIRS: list[dict[str, Any]] = [ { "id": "c001", "claim": "The dough rests for half a day before baking.", "excerpts": [{"file": "bread.md", "line": 3, "text": "The dough rests for twelve hours before baking."}], "label": "supported", "kind": "paraphrase", "why": "twelve hours is half a day", }, { "id": "c002", "claim": "Rye flour and a hot oven both give a dark crust.", "excerpts": [ {"file": "bread.md", "line": 4, "text": "A hot oven gives a dark crust."}, {"file": "flour.md", "line": 2, "text": "Rye flour absorbs more water than wheat flour."}, ], }, { "id": "c003", "claim": "A starter is fed once a day.", "excerpts": [{"file": "bread.md", "line": 2, "text": "Sourdough needs a starter that is fed daily."}], }, ] def write_json(path: Path, data: Any) -> Path: path.write_text(json.dumps(data), encoding="utf-8") return path def sha(path: Path) -> str: return hashlib.sha256(path.read_bytes()).hexdigest() def layout(set_path: Path, calls: list[tuple[str, list[str]]], size_max: int = 10) -> dict[str, Any]: return {"call_size_max": size_max, "calls": [{"call": c, "ids": ids} for c, ids in calls], "schema": "okf-wiki/reader-calls/1", "set_sha256": sha(set_path)} class Bench: """A set, a layout and an agent file in a temporary directory.""" def __init__(self, tmp: str, calls: list[tuple[str, list[str]]] | None = None) -> None: self.dir = Path(tmp) self.set = write_json(self.dir / "set.json", {"schema": "x", "pairs": PAIRS}) self.calls = write_json(self.dir / "calls.json", layout(self.set, calls if calls is not None else [("k001", ["c002", "c001"]), ("k002", ["c003"])])) self.agent = self.dir / "agent.md" self.agent.write_text(AGENT, encoding="utf-8") self.tasks = self.dir / "tasks" def render(self) -> tuple[int, str]: err = io.StringIO() with contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(err): code = reader.main(["render", "--set", str(self.set), "--calls", str(self.calls), "--agent", str(self.agent), "--out-dir", str(self.tasks)]) return code, err.getvalue() class InstructionsTest(unittest.TestCase): def test_frontmatter_is_stripped(self) -> None: body = reader.instructions(AGENT) self.assertTrue(body.startswith("Read the one task file")) self.assertNotIn("name: test-reader", body) def test_text_without_frontmatter_is_kept(self) -> None: self.assertEqual(reader.instructions("Only a body.\n"), "Only a body.") def test_unclosed_frontmatter_is_an_error(self) -> None: with self.assertRaises(reader.RunError): reader.instructions("---\nname: x\nNo end.\n") def test_the_reader_instructions_name_the_six_unsupported_kinds_by_their_idea(self) -> None: text = reader.INSTRUCTIONS.lower() for idea in ("broader", "number", "cause", "not in the excerpts", "negat", "different file"): self.assertIn(idea, text, idea) def test_the_reader_instructions_make_the_reader_work_out_a_derived_figure_before_judging(self) -> None: # Folder set, first reading (08.11.2026): one wrong-number claim stated a difference the excerpt did not give # and the reader accepted it; a figure the claim derives must be computed, not trusted. text = reader.INSTRUCTIONS.lower() self.assertIn("work it out yourself", text) for kind in ("difference", "sum", "share"): self.assertIn(kind, text, kind) def test_the_reader_instructions_ask_for_one_line_per_claim_and_judge_each_claim_alone(self) -> None: text = reader.INSTRUCTIONS self.assertIn(": supported", text) self.assertIn(": unsupported", text) self.assertIn("on its own", text) def test_the_reader_instructions_say_nothing_about_how_the_reader_is_tested(self) -> None: text = reader.INSTRUCTIONS.lower() for word in ("calibrat", "gate", "planted", "filler", "key", "label", "position"): self.assertNotIn(word, text, word) class ShippedAgentTest(unittest.TestCase): """The agent file is thin: its body (the subagent's system text) only says to read the task file and follow it, so the instructions that change between gate runs reach the reader in the task, whatever the session loaded.""" def setUp(self) -> None: self.text = (ROOT / "agents" / "okf-wiki-reader.md").read_text(encoding="utf-8") head, _, _ = self.text[4:].partition("\n---\n") self.fields = dict(line.split(": ", 1) for line in head.splitlines()) self.body = reader.instructions(self.text) def test_frontmatter_names_the_reader_with_read_only_and_opus(self) -> None: self.assertTrue(self.text.startswith("---\nname: okf-wiki-reader\n")) self.assertEqual(self.fields["tools"], "Read") self.assertEqual(self.fields["model"], "opus") self.assertEqual(self.fields["omitClaudeMd"], "true") def test_body_is_thin_and_points_to_the_task_file(self) -> None: self.assertLess(len(self.body), 1200) self.assertIn("task file", self.body) self.assertNotIn("## ", self.body) self.assertNotIn(reader.INSTRUCTIONS[:200], self.body) def test_body_says_nothing_about_how_the_reader_is_tested(self) -> None: text = self.body.lower() for word in ("calibrat", "gate", "planted", "filler", "key", "label", "position"): self.assertNotIn(word, text, word) def test_project_agent_is_the_shipped_reader_byte_for_byte(self) -> None: # A session started in this repository loads .claude/agents/; the reader it gets must be the one the # calibration gate measured, so the two files may never drift apart. shipped = (ROOT / "agents" / "okf-wiki-reader.md").read_bytes() self.assertEqual(shipped, (ROOT / ".claude" / "agents" / "okf-wiki-reader.md").read_bytes()) class LoadPairsTest(unittest.TestCase): def test_only_id_claim_and_excerpts_survive(self) -> None: with tempfile.TemporaryDirectory() as tmp: pairs = reader.load_pairs(write_json(Path(tmp) / "set.json", {"schema": "x", "pairs": PAIRS})) self.assertEqual([p.id for p in pairs], ["c001", "c002", "c003"]) self.assertEqual(pairs[1].excerpts[1].file, "flour.md") self.assertEqual(pairs[1].excerpts[1].line, 2) def test_a_pair_without_excerpts_is_an_error(self) -> None: bad = {"pairs": [{"id": "c009", "claim": "Bread is good.", "excerpts": []}]} with tempfile.TemporaryDirectory() as tmp: with self.assertRaises(reader.RunError): reader.load_pairs(write_json(Path(tmp) / "set.json", bad)) def test_duplicate_ids_are_an_error(self) -> None: with tempfile.TemporaryDirectory() as tmp: with self.assertRaises(reader.RunError): reader.load_pairs(write_json(Path(tmp) / "set.json", {"pairs": [PAIRS[0], PAIRS[0]]})) class LoadCallsTest(unittest.TestCase): def test_calls_keep_layout_order(self) -> None: with tempfile.TemporaryDirectory() as tmp: b = Bench(tmp) calls = reader.load_calls(b.calls, b.set) self.assertEqual([(c.name, c.ids) for c in calls], [("k001", ("c002", "c001")), ("k002", ("c003",))]) def test_a_layout_may_hold_part_of_the_set(self) -> None: with tempfile.TemporaryDirectory() as tmp: b = Bench(tmp, [("k001", ["c003"])]) self.assertEqual(len(reader.load_calls(b.calls, b.set)), 1) def test_layouts_that_break_k3_are_refused(self) -> None: cases: dict[str, list[tuple[str, list[str]]]] = { "unknown id": [("k001", ["c001", "c404"])], "id twice": [("k001", ["c001"]), ("k002", ["c001"])], "call name twice": [("k001", ["c001"]), ("k001", ["c002"])], "empty call": [("k001", [])], "unsafe call name": [("../k001", ["c001"])], } for what, calls in cases.items(): with self.subTest(what), tempfile.TemporaryDirectory() as tmp: b = Bench(tmp, calls) with self.assertRaises(reader.RunError): reader.load_calls(b.calls, b.set) def test_a_call_over_the_size_limit_is_refused(self) -> None: with tempfile.TemporaryDirectory() as tmp: b = Bench(tmp) write_json(b.calls, layout(b.set, [("k001", ["c001", "c002", "c003"])], size_max=2)) with self.assertRaises(reader.RunError): reader.load_calls(b.calls, b.set) write_json(b.calls, layout(b.set, [("k001", ["c001"])], size_max=11)) with self.assertRaises(reader.RunError): reader.load_calls(b.calls, b.set) def test_a_layout_bound_to_another_set_is_refused(self) -> None: with tempfile.TemporaryDirectory() as tmp: b = Bench(tmp) data = json.loads(b.calls.read_text()) data["set_sha256"] = "0" * 64 write_json(b.calls, data) with self.assertRaises(reader.RunError): reader.load_calls(b.calls, b.set) def test_a_layout_with_another_schema_is_refused(self) -> None: with tempfile.TemporaryDirectory() as tmp: b = Bench(tmp) data = json.loads(b.calls.read_text()) data["schema"] = "okf-wiki/reader-calls/0" write_json(b.calls, data) with self.assertRaises(reader.RunError): reader.load_calls(b.calls, b.set) class RenderTest(unittest.TestCase): def test_render_writes_one_task_per_call_named_after_the_call(self) -> None: with tempfile.TemporaryDirectory() as tmp: b = Bench(tmp) code, _ = b.render() self.assertEqual(code, 0) self.assertEqual(sorted(p.name for p in b.tasks.iterdir()), ["k001.md", "k002.md"]) def test_task_is_the_instructions_then_the_claims_numbered_in_layout_order_with_their_excerpts(self) -> None: with tempfile.TemporaryDirectory() as tmp: b = Bench(tmp) b.render() task = (b.tasks / "k001.md").read_text(encoding="utf-8") self.assertTrue(task.startswith(reader.INSTRUCTIONS.strip())) first, second = task.index("1. " + PAIRS[1]["claim"]), task.index("2. " + PAIRS[0]["claim"]) self.assertLess(task.index(reader.INSTRUCTIONS.strip()[-40:]), first) self.assertLess(first, second) for e in PAIRS[1]["excerpts"]: at = task.index(f'\n{e["text"]}\n') self.assertTrue(first < at < second) self.assertIn(f'\n{PAIRS[0]["excerpts"][0]["text"]}\n', task[second:]) self.assertNotIn("3. ", task) def test_task_never_carries_an_id_the_key_the_call_name_or_another_call(self) -> None: with tempfile.TemporaryDirectory() as tmp: b = Bench(tmp) b.render() task = (b.tasks / "k001.md").read_text(encoding="utf-8") for leak in ("c001", "c002", "k001", "paraphrase", "twelve hours is half a day", PAIRS[2]["claim"], PAIRS[2]["excerpts"][0]["text"]): self.assertNotIn(leak, task) def test_the_system_text_sharing_eight_words_with_a_claim_or_excerpt_is_refused(self) -> None: for leak in ("For example: the dough rests for half a day before baking.", "Note that rye flour absorbs more water than wheat flour does."): with self.subTest(leak), tempfile.TemporaryDirectory() as tmp: b = Bench(tmp) b.agent.write_text(AGENT + "\n" + leak + "\n", encoding="utf-8") code, err = b.render() self.assertEqual(code, 1) self.assertIn("share a run of 8 words", err) self.assertFalse(b.tasks.exists()) def test_the_instructions_in_the_task_sharing_eight_words_with_a_claim_or_excerpt_are_refused(self) -> None: leaky = reader.INSTRUCTIONS + "\nFor example: sourdough needs a starter that is fed daily.\n" with tempfile.TemporaryDirectory() as tmp, mock.patch.object(reader, "INSTRUCTIONS", leaky): b = Bench(tmp) code, err = b.render() self.assertEqual(code, 1) self.assertIn("share a run of 8 words", err) self.assertFalse(b.tasks.exists()) def test_shared_runs_count_words_not_layout(self) -> None: with tempfile.TemporaryDirectory() as tmp: pairs = reader.load_pairs(write_json(Path(tmp) / "set.json", {"pairs": PAIRS})) self.assertEqual(reader.shared_runs("THE DOUGH rests\nfor twelve hours before baking", pairs), ["c001"]) self.assertEqual(reader.shared_runs("the dough rests for twelve hours before", pairs), []) def test_render_refuses_to_change_a_task_already_written(self) -> None: with tempfile.TemporaryDirectory() as tmp: b = Bench(tmp) self.assertEqual(b.render()[0], 0) self.assertEqual(b.render()[0], 0) (b.tasks / "k002.md").write_text("edited\n", encoding="utf-8") code, err = b.render() self.assertEqual(code, 1) self.assertIn("k002.md", err) self.assertEqual((b.tasks / "k002.md").read_text(encoding="utf-8"), "edited\n") class ParseReplyTest(unittest.TestCase): """K3 point 3: one rule, fail-closed.""" def test_lines_in_any_order(self) -> None: self.assertEqual(reader.parse_reply("2: unsupported\n1: supported\n3: supported", 3), {1: "supported", 2: "unsupported", 3: "supported"}) def test_lines_are_stripped_and_empty_lines_dropped(self) -> None: self.assertEqual(reader.parse_reply("\n 1: supported \n\n2 :unsupported\n10:supported\n", 10), {1: "supported", 2: "unsupported", 10: "supported"}) def test_one_line_that_does_not_fit_gives_no_verdict_for_any_claim(self) -> None: for reply in ("1: supported\n2: unsupported\nThat is all.", "Verdicts:\n1: supported\n2: unsupported", "1: supported (the excerpt says so)\n2: unsupported", "```\n1: supported\n2: unsupported\n```", "**1: supported**\n2: unsupported", "1: Supported\n2: unsupported", "1. supported\n2: unsupported", "01: supported\n2: unsupported", "0: supported\n1: supported\n2: unsupported", "1: supported, 2: unsupported"): with self.subTest(reply): self.assertEqual(reader.parse_reply(reply, 2), {}) def test_a_number_outside_one_to_n_gives_no_verdict_for_any_claim(self) -> None: self.assertEqual(reader.parse_reply("1: supported\n2: supported\n3: unsupported", 2), {}) def test_a_number_given_twice_gives_no_verdict_for_that_claim(self) -> None: self.assertEqual(reader.parse_reply("1: supported\n1: unsupported\n2: supported", 2), {2: "supported"}) self.assertEqual(reader.parse_reply("1: supported\n1: supported\n2: supported", 2), {2: "supported"}) def test_a_missing_number_gives_no_verdict_for_that_claim(self) -> None: self.assertEqual(reader.parse_reply("1: supported\n3: unsupported", 3), {1: "supported", 3: "unsupported"}) def test_an_empty_reply_gives_no_verdict(self) -> None: self.assertEqual(reader.parse_reply("", 3), {}) self.assertEqual(reader.parse_reply("\n \n", 3), {}) MODEL = "claude-opus-5-5" def assistant(mid: str, *blocks: dict[str, Any], model: str | None = MODEL) -> list[dict[str, Any]]: """One assistant message as the transcript writes it: one record per content block, all with the message id and, as Claude Code writes it, the model that wrote the message (left out when model is None).""" named: dict[str, Any] = {} if model is None else {"model": model} return [{"type": "assistant", "message": {"id": mid, "role": "assistant", "content": [b], **named, "stop_reason": "end_turn" if i == len(blocks) - 1 else None}} for i, b in enumerate(blocks)] def text(t: str) -> dict[str, Any]: return {"type": "text", "text": t} def subagent(root: Path, session: str, agent: str, description: str, records: list[dict[str, Any]], agent_type: str = "okf-wiki-reader", where: str = "subagents") -> None: d = root / session / where d.mkdir(parents=True, exist_ok=True) (d / f"agent-{agent}.meta.json").write_text(json.dumps({"agentType": agent_type, "description": description}), encoding="utf-8") (d / f"agent-{agent}.jsonl").write_text("".join(json.dumps(r) + "\n" for r in records), encoding="utf-8") def answered(reply: str, mid: str = "m2", model: str | None = MODEL) -> list[dict[str, Any]]: """A reader that reads its task, says a word before the tool call, thinks, then replies.""" return [{"type": "user", "message": {"role": "user", "content": "Task file: /bench/tasks/k001.md"}}, *assistant("m1", text("I will read the task."), {"type": "tool_use", "id": "t1", "name": "Read", "input": {"file_path": "/bench/k001.md"}}, model=model), {"type": "user", "message": {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "t1", "content": "the task"}]}}, *assistant(mid, {"type": "thinking", "thinking": "1: unsupported\nweighing claim one"}, text(reply), model=model)] def handed_back(reply: Any, mid: str = "m2", model: str | None = MODEL) -> list[dict[str, Any]]: """A reader that reads its task, thinks, and delivers its reply through the hand-back tool (Claude Code 2.1.295).""" return answered("", mid, model)[:-2] + assistant(mid, {"type": "thinking", "thinking": "2: unsupported"}, {"type": "tool_use", "id": "t2", "name": "SubagentHandback", "input": {"message": reply}}, model=model) class FinalReplyTest(unittest.TestCase): def test_the_reply_is_the_hand_back_message_when_the_reader_hands_back(self) -> None: with tempfile.TemporaryDirectory() as tmp: subagent(Path(tmp), "s1", "a1", "V1-kal9-read-k001", handed_back("1: supported\n2: unsupported")) path = Path(tmp) / "s1" / "subagents" / "agent-a1.jsonl" self.assertEqual(reader.final_reply(path), "1: supported\n2: unsupported") def test_the_last_hand_back_wins_and_text_beside_it_is_not_the_reply(self) -> None: records = handed_back("1: unsupported") + assistant("m3", text("Here are my verdicts."), { "type": "tool_use", "id": "t3", "name": "SubagentHandback", "input": {"message": "1: supported"}}) with tempfile.TemporaryDirectory() as tmp: subagent(Path(tmp), "s1", "a1", "V1-kal9-read-k001", records) path = Path(tmp) / "s1" / "subagents" / "agent-a1.jsonl" self.assertEqual(reader.final_reply(path), "1: supported") def test_a_hand_back_without_a_text_message_is_empty(self) -> None: with tempfile.TemporaryDirectory() as tmp: subagent(Path(tmp), "s1", "a1", "V1-kal9-read-k001", handed_back(["1: supported"])) path = Path(tmp) / "s1" / "subagents" / "agent-a1.jsonl" self.assertEqual(reader.final_reply(path), "") def test_the_reply_is_the_text_of_the_last_assistant_message_without_thinking(self) -> None: with tempfile.TemporaryDirectory() as tmp: subagent(Path(tmp), "s1", "a1", "V1-kal9-read-k001", answered("1: supported\n2: supported")) path = Path(tmp) / "s1" / "subagents" / "agent-a1.jsonl" self.assertEqual(reader.final_reply(path), "1: supported\n2: supported") def test_text_blocks_of_the_last_message_are_joined_in_order(self) -> None: records = answered("1: supported") + assistant("m3", text("2: unsupported"), text("1: supported")) with tempfile.TemporaryDirectory() as tmp: subagent(Path(tmp), "s1", "a1", "V1-kal9-read-k001", records) path = Path(tmp) / "s1" / "subagents" / "agent-a1.jsonl" self.assertEqual(reader.final_reply(path), "2: unsupported\n1: supported") def test_a_transcript_without_a_reply_is_empty(self) -> None: with tempfile.TemporaryDirectory() as tmp: subagent(Path(tmp), "s1", "a1", "V1-kal9-read-k001", answered("x")[:2]) path = Path(tmp) / "s1" / "subagents" / "agent-a1.jsonl" self.assertEqual(reader.final_reply(path), "I will read the task.") subagent(Path(tmp), "s1", "a2", "V1-kal9-read-k002", answered("x")[:1]) self.assertEqual(reader.final_reply(Path(tmp) / "s1" / "subagents" / "agent-a2.jsonl"), "") class CollectTest(unittest.TestCase): def collect(self, b: Bench, transcripts: Path, run: str = "kal9", check: tuple[str, ...] = ("--calibration",)) -> tuple[int, str]: out = io.StringIO() with contextlib.redirect_stdout(out), contextlib.redirect_stderr(io.StringIO()): code = reader.main(["collect", "--set", str(b.set), "--calls", str(b.calls), "--transcripts", str(transcripts), "--run", run, "--reader", "r1", "--out", str(b.dir / "v.json"), *check]) return code, out.getvalue() def verdicts(self, b: Bench) -> tuple[str, dict[str, str], str]: result: tuple[str, dict[str, str], str] = calibration.read_verdicts(b.dir / "v.json") return result def test_the_gate_reads_what_collect_writes_from_the_transcripts(self) -> None: with tempfile.TemporaryDirectory() as tmp: b, t = Bench(tmp), Path(tmp) / "transcripts" subagent(t, "s1", "a1", "V1-kal9-read-k001", answered("2: supported\n1: unsupported")) subagent(t, "s1", "a2", "V1-kal9-read-k002", answered("1: supported")) code, out = self.collect(b, t) self.assertEqual(code, 0) self.assertIn("3 of 3 verdicts", out) name, verdicts, fail = self.verdicts(b) doc = json.loads((b.dir / "v.json").read_text(encoding="utf-8")) self.assertEqual((name, fail), ("r1", "")) self.assertEqual(verdicts, {"c002": "unsupported", "c001": "supported", "c003": "supported"}) self.assertEqual([v["id"] for v in doc["verdicts"]], ["c002", "c001", "c003"]) def test_a_call_with_no_transcript_leaves_its_claims_without_verdict_and_exits_1(self) -> None: with tempfile.TemporaryDirectory() as tmp: b, t = Bench(tmp), Path(tmp) / "transcripts" subagent(t, "s1", "a1", "V1-kal9-read-k001", answered("1: supported\n2: supported")) code, out = self.collect(b, t) self.assertEqual(code, 1) self.assertIn("2 of 3 verdicts", out) self.assertEqual(set(self.verdicts(b)[1]), {"c001", "c002"}) def test_replies_handed_back_are_collected(self) -> None: with tempfile.TemporaryDirectory() as tmp: b, t = Bench(tmp), Path(tmp) / "transcripts" subagent(t, "s1", "a1", "V1-kal9-read-k001", handed_back("1: unsupported\n2: supported")) subagent(t, "s1", "a2", "V1-kal9-read-k002", handed_back("1: unsupported")) code, out = self.collect(b, t) self.assertEqual(code, 0) self.assertEqual(self.verdicts(b)[1], {"c002": "unsupported", "c001": "supported", "c003": "unsupported"}) def test_a_reply_that_breaks_the_rule_loses_its_whole_call(self) -> None: with tempfile.TemporaryDirectory() as tmp: b, t = Bench(tmp), Path(tmp) / "transcripts" subagent(t, "s1", "a1", "V1-kal9-read-k001", answered("1: supported\n2: supported\nDone.")) subagent(t, "s1", "a2", "V1-kal9-read-k002", answered("1: supported")) code, out = self.collect(b, t) self.assertEqual(code, 1) self.assertIn("1 broke the reply rule", out) self.assertEqual(self.verdicts(b)[1], {"c003": "supported"}) def test_a_call_answered_by_two_readers_gets_no_verdict(self) -> None: with tempfile.TemporaryDirectory() as tmp: b, t = Bench(tmp), Path(tmp) / "transcripts" subagent(t, "s1", "a1", "V1-kal9-read-k001", answered("1: supported\n2: supported")) subagent(t, "s2", "a9", "V1-kal9-read-k001", answered("1: supported\n2: supported")) subagent(t, "s1", "a2", "V1-kal9-read-k002", answered("1: supported")) code, out = self.collect(b, t) self.assertEqual(code, 1) self.assertIn("1 answered twice", out) self.assertEqual(self.verdicts(b)[1], {"c003": "supported"}) def test_other_runs_other_lines_and_other_agents_are_ignored(self) -> None: with tempfile.TemporaryDirectory() as tmp: b, t = Bench(tmp), Path(tmp) / "transcripts" subagent(t, "s1", "a1", "V1-kal9-read-k001", answered("1: supported\n2: supported")) subagent(t, "s1", "a2", "V1-kal9-read-k002", answered("1: supported")) subagent(t, "s1", "a3", "V1-kal8-read-k001", answered("1: unsupported\n2: unsupported")) subagent(t, "s1", "a4", "V1-kal9-pack-k002", answered("1: unsupported")) subagent(t, "s1", "a5", "a helper", answered("1: unsupported")) code, out = self.collect(b, t) self.assertEqual(code, 0) self.assertEqual(set(self.verdicts(b)[1].values()), {"supported"}) def test_a_transcript_for_a_call_not_in_the_layout_is_counted(self) -> None: with tempfile.TemporaryDirectory() as tmp: b, t = Bench(tmp), Path(tmp) / "transcripts" subagent(t, "s1", "a1", "V1-kal9-read-k001", answered("1: supported\n2: supported")) subagent(t, "s1", "a2", "V1-kal9-read-k002", answered("1: supported")) subagent(t, "s1", "a3", "V1-kal9-read-k777", answered("1: supported")) code, out = self.collect(b, t) self.assertEqual(code, 0) self.assertIn("1 transcript for a call not in the layout", out) def test_readers_run_as_a_workflow_are_collected(self) -> None: # A workflow writes its agents' transcripts one level down, under subagents/workflows//. with tempfile.TemporaryDirectory() as tmp: b, t = Bench(tmp), Path(tmp) / "transcripts" subagent(t, "s1", "a1", "V1-kal9-read-k001", answered("1: supported\n2: supported")) subagent(t, "s1", "a2", "V1-kal9-read-k002", answered("1: unsupported"), where="subagents/workflows/wf_1") code, out = self.collect(b, t) self.assertEqual(code, 0) self.assertEqual(self.verdicts(b)[1]["c003"], "unsupported") def test_a_reader_description_on_another_agent_type_is_not_collected(self) -> None: with tempfile.TemporaryDirectory() as tmp: b, t = Bench(tmp), Path(tmp) / "transcripts" subagent(t, "s1", "a1", "V1-kal9-read-k001", answered("1: supported\n2: supported")) subagent(t, "s1", "a2", "V1-kal9-read-k002", answered("1: supported"), agent_type="general-purpose") code, out = self.collect(b, t) self.assertEqual(code, 1) self.assertIn("1 not answered", out) self.assertNotIn("c003", self.verdicts(b)[1]) def test_a_set_that_cannot_be_read_fails_with_a_message_and_writes_nothing(self) -> None: with tempfile.TemporaryDirectory() as tmp: b = Bench(tmp) b.set.unlink() err = io.StringIO() with contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(err): code = reader.main(["collect", "--set", str(b.set), "--calls", str(b.calls), "--transcripts", tmp, "--run", "kal9", "--reader", "r1", "--out", str(b.dir / "v.json"), "--calibration"]) self.assertEqual(code, 1) self.assertIn("FAIL:", err.getvalue()) self.assertFalse((b.dir / "v.json").exists()) def test_collect_needs_a_transcripts_directory(self) -> None: with tempfile.TemporaryDirectory() as tmp: b = Bench(tmp) code, _ = self.collect(b, Path(tmp) / "nowhere") self.assertEqual(code, 1) self.assertFalse((b.dir / "v.json").exists()) SET_SHA, KEY_SHA = "a" * 64, "b" * 64 # the set and key the test manifest names def passed_record(models: list[str], set_sha: str = SET_SHA, key_sha: str = KEY_SHA) -> dict[str, Any]: """A record of green calibration runs (okf-wiki/calibration-passed/1) with one entry per model.""" return {"schema": "okf-wiki/calibration-passed/1", "passed": [{"model": m, "set_sha256": set_sha, "key_sha256": key_sha, "verdicts_sha256": "c" * 64, "reader": "r0"} for m in models]} def test_manifest(directory: Path) -> Path: """A calibration manifest that names the set SET_SHA and the key KEY_SHA.""" return write_json(directory / "manifest.json", {"schema": "okf-wiki/calibration-manifest/2", "set": {"file": "s.json", "sha256": SET_SHA}, "key": {"file": "k.json", "sha256": KEY_SHA}}) class CollectModelTest(unittest.TestCase): """PLAN ยง Rammer 23: collect names the model the readers ran on, and turns a reading into verdicts only when that model has a recorded green calibration run (--gate-passed); the calibration run itself says --calibration.""" def run_collect(self, b: Bench, transcripts: Path, *check: str) -> tuple[int, str]: out = io.StringIO() with contextlib.redirect_stdout(out), contextlib.redirect_stderr(out): try: manifest = ["--manifest", str(test_manifest(b.dir))] if "--gate-passed" in check else [] code = reader.main(["collect", "--set", str(b.set), "--calls", str(b.calls), "--transcripts", str(transcripts), "--run", "kal9", "--reader", "r1", "--out", str(b.dir / "v.json"), *check, *manifest]) except SystemExit as exc: # argparse refuses the options code = exc.code if isinstance(exc.code, int) else 2 return code, out.getvalue() def collect(self, b: Bench, transcripts: Path) -> tuple[int, str]: return self.run_collect(b, transcripts, "--calibration") def two_readers(self, t: Path, first: str | None = MODEL, second: str | None = MODEL) -> None: subagent(t, "s1", "a1", "V1-kal9-read-k001", answered("1: supported\n2: supported", model=first)) subagent(t, "s1", "a2", "V1-kal9-read-k002", answered("1: supported", model=second)) def doc(self, b: Bench) -> dict[str, Any]: data: dict[str, Any] = json.loads((b.dir / "v.json").read_text(encoding="utf-8")) return data def test_collect_writes_and_prints_the_sorted_distinct_models_of_its_readers(self) -> None: with tempfile.TemporaryDirectory() as tmp: b, t = Bench(tmp), Path(tmp) / "transcripts" self.two_readers(t, "claude-test-b", "claude-test-a") code, out = self.collect(b, t) self.assertEqual(code, 0, out) doc = self.doc(b) self.assertEqual(doc["models"], ["claude-test-a", "claude-test-b"]) self.assertEqual(list(doc), ["reader", "models", "verdicts"]) self.assertIn("models: claude-test-a, claude-test-b\n", out) def test_one_model_on_every_reader_is_named_once(self) -> None: with tempfile.TemporaryDirectory() as tmp: b, t = Bench(tmp), Path(tmp) / "transcripts" self.two_readers(t) code, out = self.collect(b, t) self.assertEqual(self.doc(b)["models"], [MODEL]) self.assertIn(f"models: {MODEL}\n", out) def test_records_without_a_model_and_synthetic_ones_are_ignored(self) -> None: records = answered("1: supported\n2: supported", model=None) + assistant( "m3", text("1: supported\n2: supported"), model="") records[2]["message"]["model"] = MODEL # one record of the reader's own turns names the model with tempfile.TemporaryDirectory() as tmp: b, t = Bench(tmp), Path(tmp) / "transcripts" subagent(t, "s1", "a1", "V1-kal9-read-k001", records) subagent(t, "s1", "a2", "V1-kal9-read-k002", answered("1: supported", model=None)) code, out = self.collect(b, t) self.assertEqual(code, 0, out) self.assertEqual(self.doc(b)["models"], [MODEL]) def test_readers_that_name_no_model_give_an_empty_list_and_none(self) -> None: with tempfile.TemporaryDirectory() as tmp: b, t = Bench(tmp), Path(tmp) / "transcripts" self.two_readers(t, None, None) code, out = self.collect(b, t) self.assertEqual(self.doc(b)["models"], []) self.assertIn("models: (none)\n", out) def test_transcripts_collect_does_not_use_do_not_name_a_model(self) -> None: with tempfile.TemporaryDirectory() as tmp: b, t = Bench(tmp), Path(tmp) / "transcripts" self.two_readers(t) subagent(t, "s1", "a3", "V1-kal8-read-k001", answered("1: supported", model="claude-other-run")) subagent(t, "s1", "a4", "V1-kal9-read-k777", answered("1: supported", model="claude-stray")) subagent(t, "s1", "a5", "V1-kal9-read-k002", answered("1: supported", model="claude-helper"), agent_type="general-purpose") code, out = self.collect(b, t) self.assertEqual(self.doc(b)["models"], [MODEL]) def test_readers_started_from_a_plugin_are_collected_by_their_namespaced_type(self) -> None: with tempfile.TemporaryDirectory() as tmp: b, t = Bench(tmp), Path(tmp) / "transcripts" subagent(t, "s1", "a1", "V1-kal9-read-k001", answered("1: supported\n2: supported"), agent_type="okf-wiki:okf-wiki-reader") subagent(t, "s1", "a2", "V1-kal9-read-k002", answered("1: supported"), agent_type="okf-wiki-reader") for i, other in enumerate(("okf-wiki:okf-wiki-reader-old", "okf-wiki-reader:helper", ":okf-wiki-reader")): subagent(t, "s2", f"x{i}", "V1-kal9-read-k001", answered("1: unsupported\n2: unsupported"), agent_type=other) code, out = self.collect(b, t) self.assertEqual(code, 0, out) self.assertIn("3 of 3 verdicts", out) def test_collect_needs_exactly_one_of_calibration_and_gate_passed(self) -> None: with tempfile.TemporaryDirectory() as tmp: b, t = Bench(tmp), Path(tmp) / "transcripts" self.two_readers(t) passed = write_json(b.dir / "passed.json", passed_record([MODEL])) for check in ((), ("--calibration", "--gate-passed", str(passed))): with self.subTest(check=check): code, _ = self.run_collect(b, t, *check) self.assertEqual(code, 2) self.assertFalse((b.dir / "v.json").exists()) def test_the_tracked_record_holds_a_green_run_on_the_tracked_set_and_key(self) -> None: # The set itself stays out of git (rule 3), so a fresh clone could not run the gate and collect refused every # reading (motor B16); the record holds only models and sha256, no text. self.assertIn("claude-opus-5-5", reader.passed_models(reader.CALIBRATION_PASSED, reader.CALIBRATION_MANIFEST)) def test_gate_passed_without_a_file_reads_the_tracked_record(self) -> None: args = reader.build_parser().parse_args(["collect", "--set", "s", "--calls", "c", "--transcripts", "t", "--run", "r", "--reader", "n", "--out", "o", "--gate-passed"]) self.assertEqual(args.gate_passed, str(reader.CALIBRATION_PASSED)) def test_a_model_with_a_green_run_on_record_is_collected(self) -> None: with tempfile.TemporaryDirectory() as tmp: b, t = Bench(tmp), Path(tmp) / "transcripts" self.two_readers(t) passed = write_json(b.dir / "passed.json", passed_record(["claude-older", MODEL])) code, out = self.run_collect(b, t, "--gate-passed", str(passed)) self.assertEqual(code, 0, out) self.assertEqual(self.doc(b)["models"], [MODEL]) self.assertIn("3 of 3 verdicts", out) def test_a_model_without_a_green_run_writes_nothing_and_exits_1(self) -> None: with tempfile.TemporaryDirectory() as tmp: b, t = Bench(tmp), Path(tmp) / "transcripts" self.two_readers(t, "claude-new", MODEL) passed = write_json(b.dir / "passed.json", passed_record([MODEL])) code, out = self.run_collect(b, t, "--gate-passed", str(passed)) self.assertEqual(code, 1, out) self.assertFalse((b.dir / "v.json").exists()) self.assertIn(f"collect cannot run: model(s) claude-new have no green calibration run in {passed} on the " f"set and key of {b.dir / 'manifest.json'}; run the calibration gate on them first", out) def test_readers_that_name_no_model_are_refused_against_the_record(self) -> None: with tempfile.TemporaryDirectory() as tmp: b, t = Bench(tmp), Path(tmp) / "transcripts" self.two_readers(t, None, None) passed = write_json(b.dir / "passed.json", passed_record([MODEL])) code, out = self.run_collect(b, t, "--gate-passed", str(passed)) self.assertEqual(code, 1, out) self.assertFalse((b.dir / "v.json").exists()) self.assertIn(f"collect cannot run: model(s) (none) have no green calibration run in {passed}", out) def test_an_unreadable_or_malformed_record_writes_nothing_and_exits_1(self) -> None: bad: dict[str, Any] = { "missing": None, "not json": "{", "other schema": {"schema": "okf-wiki/other/1", "passed": [{"model": MODEL}]}, "no list": {"schema": "okf-wiki/calibration-passed/1", "passed": {"model": MODEL}}, "entry without a model": {"schema": "okf-wiki/calibration-passed/1", "passed": [{"reader": "r0"}]}, "entry without a set": {"schema": "okf-wiki/calibration-passed/1", "passed": [{"model": MODEL, "key_sha256": KEY_SHA}]}, } for what, content in bad.items(): with self.subTest(what=what), tempfile.TemporaryDirectory() as tmp: b, t = Bench(tmp), Path(tmp) / "transcripts" self.two_readers(t) passed = b.dir / "passed.json" if isinstance(content, str): passed.write_text(content, encoding="utf-8") elif content is not None: write_json(passed, content) code, out = self.run_collect(b, t, "--gate-passed", str(passed)) self.assertEqual(code, 1, out) self.assertFalse((b.dir / "v.json").exists()) self.assertIn(str(passed), out) def test_the_record_the_gate_writes_is_the_record_collect_reads(self) -> None: with tempfile.TemporaryDirectory() as tmp: b, t = Bench(tmp), Path(tmp) / "transcripts" self.two_readers(t) passed = b.dir / "passed.json" calibration.record_passed(passed, {"model": MODEL, "set_sha256": SET_SHA, "key_sha256": KEY_SHA, "verdicts_sha256": "c" * 64, "reader": "r0"}) code, out = self.run_collect(b, t, "--gate-passed", str(passed)) self.assertEqual(code, 0, out) def test_a_green_run_on_another_set_or_key_does_not_clear_the_model(self) -> None: for what, set_sha, key_sha in (("set", "d" * 64, KEY_SHA), ("key", SET_SHA, "e" * 64)): with self.subTest(what=what), tempfile.TemporaryDirectory() as tmp: b, t = Bench(tmp), Path(tmp) / "transcripts" self.two_readers(t) passed = write_json(b.dir / "passed.json", passed_record([MODEL], set_sha, key_sha)) code, out = self.run_collect(b, t, "--gate-passed", str(passed)) self.assertEqual(code, 1, out) self.assertFalse((b.dir / "v.json").exists()) self.assertIn(f"model(s) {MODEL} have no green calibration run", out) def test_the_default_manifest_is_the_tracked_calibration_manifest(self) -> None: self.assertEqual(ROOT / "gates" / "calibration.manifest.json", reader.CALIBRATION_MANIFEST) self.assertTrue(reader.CALIBRATION_MANIFEST.is_file()) def test_the_default_manifests_are_the_docs_set_and_the_folder_set(self) -> None: # B16: a reader judges folder documents (Norwegian and English PDF/Office) only after it passed a set made # of them, so a model must have a green run on both tracked sets before any reading counts. args = reader.build_parser().parse_args(["collect", "--set", "s", "--calls", "c", "--transcripts", "t", "--run", "r", "--reader", "n", "--out", "o", "--gate-passed"]) self.assertEqual(reader.manifests_of(args), [reader.CALIBRATION_MANIFEST, reader.CALIBRATION_FOLDER_MANIFEST]) self.assertEqual(ROOT / "gates" / "calibration-folder.manifest.json", reader.CALIBRATION_FOLDER_MANIFEST) self.assertTrue(reader.CALIBRATION_FOLDER_MANIFEST.is_file()) def test_a_model_needs_a_green_run_on_every_manifest_given(self) -> None: other_set, other_key = "d" * 64, "e" * 64 for models_on_other, expected in (([], 1), ([MODEL], 0)): with self.subTest(models_on_other=models_on_other), tempfile.TemporaryDirectory() as tmp: b, t = Bench(tmp), Path(tmp) / "transcripts" self.two_readers(t) second = write_json(b.dir / "manifest-2.json", {"schema": "okf-wiki/calibration-manifest/2", "set": {"file": "s2.json", "sha256": other_set}, "key": {"file": "k2.json", "sha256": other_key}}) record = passed_record([MODEL]) record["passed"] += passed_record(models_on_other, other_set, other_key)["passed"] passed = write_json(b.dir / "passed.json", record) code, out = self.run_collect(b, t, "--gate-passed", str(passed), "--manifest", str(second)) self.assertEqual(code, expected, out) self.assertEqual((b.dir / "v.json").exists(), expected == 0) if expected: self.assertIn(f"model(s) {MODEL} have no green calibration run in {passed} on the set and key " f"of {second}", out) def test_the_tracked_record_holds_a_green_run_on_the_folder_set(self) -> None: self.assertIn("claude-opus-5-5", reader.passed_models(reader.CALIBRATION_PASSED, reader.CALIBRATION_FOLDER_MANIFEST)) def test_the_gate_refuses_to_add_to_a_record_collect_cannot_read(self) -> None: with tempfile.TemporaryDirectory() as tmp: passed = write_json(Path(tmp) / "passed.json", {"schema": "okf-wiki/calibration-passed/1", "passed": [{"reader": "r0"}]}) with self.assertRaises(calibration.RunError): calibration.record_passed(passed, {"model": MODEL, "set_sha256": SET_SHA, "key_sha256": KEY_SHA, "verdicts_sha256": "c" * 64, "reader": "r0"}) class NoModelTest(unittest.TestCase): def test_tool_imports_no_network_process_or_model_client(self) -> None: source = (ROOT / "okfwiki" / "reader.py").read_text(encoding="utf-8") for banned in ("anthropic", "openai", "urllib", "http.client", "requests", "subprocess", "socket", "asyncio"): self.assertNotIn(f"import {banned}", source) self.assertNotIn(f"from {banned}", source) for call in ("os.system", "os.popen", "os.exec", "os.spawn", "os.posix_spawn", "create_subprocess"): self.assertNotIn(call, source) if __name__ == "__main__": unittest.main()