First public release, published as one snapshot. What changed between versions is in CHANGELOG.md. Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
840 lines
46 KiB
Python
840 lines
46 KiB
Python
"""Tests for okfwiki/reader.py, the tool around the reader pass (PLAN.md criterion 2). It calls no model.
|
|
|
|
The reader judges up to ten claims per call and gives one verdict per claim (K3 in the wave-1 contracts). `render`
|
|
writes one task per call of a call layout: the reader's instructions, then the claims numbered 1. to n. in layout
|
|
order, each with its excerpts, and nothing else. `collect` reads the readers' replies from the subagent transcripts
|
|
and turns them into the verdicts file that gates/calibration.py scores, by one fail-closed rule. Every fixture is
|
|
synthetic text about baking bread (CLAUDE.md rule 3).
|
|
Run: python3 -m unittest discover -s tests -p 'test_reader.py'
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import contextlib
|
|
import hashlib
|
|
import importlib.util
|
|
import io
|
|
import json
|
|
import sys
|
|
import tempfile
|
|
import unittest
|
|
from pathlib import Path
|
|
from typing import Any
|
|
from unittest import mock
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT))
|
|
|
|
from okfwiki import reader # noqa: E402
|
|
|
|
_SPEC = importlib.util.spec_from_file_location("calibration", ROOT / "gates" / "calibration.py")
|
|
assert _SPEC is not None and _SPEC.loader is not None
|
|
calibration = importlib.util.module_from_spec(_SPEC)
|
|
sys.modules["calibration"] = calibration
|
|
_SPEC.loader.exec_module(calibration)
|
|
|
|
AGENT = """---
|
|
name: test-reader
|
|
description: A reader used in tests.
|
|
tools: Read
|
|
---
|
|
|
|
Read the one task file you are given and follow it.
|
|
"""
|
|
|
|
PAIRS: list[dict[str, Any]] = [
|
|
{
|
|
"id": "c001",
|
|
"claim": "The dough rests for half a day before baking.",
|
|
"excerpts": [{"file": "bread.md", "line": 3, "text": "The dough rests for twelve hours before baking."}],
|
|
"label": "supported",
|
|
"kind": "paraphrase",
|
|
"why": "twelve hours is half a day",
|
|
},
|
|
{
|
|
"id": "c002",
|
|
"claim": "Rye flour and a hot oven both give a dark crust.",
|
|
"excerpts": [
|
|
{"file": "bread.md", "line": 4, "text": "A hot oven gives a dark crust."},
|
|
{"file": "flour.md", "line": 2, "text": "Rye flour absorbs more water than wheat flour."},
|
|
],
|
|
},
|
|
{
|
|
"id": "c003",
|
|
"claim": "A starter is fed once a day.",
|
|
"excerpts": [{"file": "bread.md", "line": 2, "text": "Sourdough needs a starter that is fed daily."}],
|
|
},
|
|
]
|
|
|
|
|
|
def write_json(path: Path, data: Any) -> Path:
|
|
path.write_text(json.dumps(data), encoding="utf-8")
|
|
return path
|
|
|
|
|
|
def sha(path: Path) -> str:
|
|
return hashlib.sha256(path.read_bytes()).hexdigest()
|
|
|
|
|
|
def layout(set_path: Path, calls: list[tuple[str, list[str]]], size_max: int = 10) -> dict[str, Any]:
|
|
return {"call_size_max": size_max, "calls": [{"call": c, "ids": ids} for c, ids in calls],
|
|
"schema": "okf-wiki/reader-calls/1", "set_sha256": sha(set_path)}
|
|
|
|
|
|
class Bench:
|
|
"""A set, a layout and an agent file in a temporary directory."""
|
|
|
|
def __init__(self, tmp: str, calls: list[tuple[str, list[str]]] | None = None) -> None:
|
|
self.dir = Path(tmp)
|
|
self.set = write_json(self.dir / "set.json", {"schema": "x", "pairs": PAIRS})
|
|
self.calls = write_json(self.dir / "calls.json",
|
|
layout(self.set, calls if calls is not None else [("k001", ["c002", "c001"]),
|
|
("k002", ["c003"])]))
|
|
self.agent = self.dir / "agent.md"
|
|
self.agent.write_text(AGENT, encoding="utf-8")
|
|
self.tasks = self.dir / "tasks"
|
|
|
|
def render(self) -> tuple[int, str]:
|
|
err = io.StringIO()
|
|
with contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(err):
|
|
code = reader.main(["render", "--set", str(self.set), "--calls", str(self.calls), "--agent",
|
|
str(self.agent), "--out-dir", str(self.tasks)])
|
|
return code, err.getvalue()
|
|
|
|
|
|
class InstructionsTest(unittest.TestCase):
|
|
def test_frontmatter_is_stripped(self) -> None:
|
|
body = reader.instructions(AGENT)
|
|
self.assertTrue(body.startswith("Read the one task file"))
|
|
self.assertNotIn("name: test-reader", body)
|
|
|
|
def test_text_without_frontmatter_is_kept(self) -> None:
|
|
self.assertEqual(reader.instructions("Only a body.\n"), "Only a body.")
|
|
|
|
def test_unclosed_frontmatter_is_an_error(self) -> None:
|
|
with self.assertRaises(reader.RunError):
|
|
reader.instructions("---\nname: x\nNo end.\n")
|
|
|
|
def test_the_reader_instructions_name_the_six_unsupported_kinds_by_their_idea(self) -> None:
|
|
text = reader.INSTRUCTIONS.lower()
|
|
for idea in ("broader", "number", "cause", "not in the excerpts", "negat", "different file"):
|
|
self.assertIn(idea, text, idea)
|
|
|
|
def test_the_reader_instructions_make_the_reader_work_out_a_derived_figure_before_judging(self) -> None:
|
|
# Folder set, first reading (08.11.2026): one wrong-number claim stated a difference the excerpt did not give
|
|
# and the reader accepted it; a figure the claim derives must be computed, not trusted.
|
|
text = reader.INSTRUCTIONS.lower()
|
|
self.assertIn("work it out yourself", text)
|
|
for kind in ("difference", "sum", "share"):
|
|
self.assertIn(kind, text, kind)
|
|
|
|
def test_the_reader_instructions_ask_for_one_line_per_claim_and_judge_each_claim_alone(self) -> None:
|
|
text = reader.INSTRUCTIONS
|
|
self.assertIn("<number>: supported", text)
|
|
self.assertIn("<number>: unsupported", text)
|
|
self.assertIn("on its own", text)
|
|
|
|
def test_the_reader_instructions_say_nothing_about_how_the_reader_is_tested(self) -> None:
|
|
text = reader.INSTRUCTIONS.lower()
|
|
for word in ("calibrat", "gate", "planted", "filler", "key", "label", "position"):
|
|
self.assertNotIn(word, text, word)
|
|
|
|
|
|
class ShippedAgentTest(unittest.TestCase):
|
|
"""The agent file is thin: its body (the subagent's system text) only says to read the task file and follow it, so
|
|
the instructions that change between gate runs reach the reader in the task, whatever the session loaded."""
|
|
|
|
def setUp(self) -> None:
|
|
self.text = (ROOT / "agents" / "okf-wiki-reader.md").read_text(encoding="utf-8")
|
|
head, _, _ = self.text[4:].partition("\n---\n")
|
|
self.fields = dict(line.split(": ", 1) for line in head.splitlines())
|
|
self.body = reader.instructions(self.text)
|
|
|
|
def test_frontmatter_names_the_reader_with_read_only_and_opus(self) -> None:
|
|
self.assertTrue(self.text.startswith("---\nname: okf-wiki-reader\n"))
|
|
self.assertEqual(self.fields["tools"], "Read")
|
|
self.assertEqual(self.fields["model"], "opus")
|
|
self.assertEqual(self.fields["omitClaudeMd"], "true")
|
|
|
|
def test_body_is_thin_and_points_to_the_task_file(self) -> None:
|
|
self.assertLess(len(self.body), 1200)
|
|
self.assertIn("task file", self.body)
|
|
self.assertNotIn("## ", self.body)
|
|
self.assertNotIn(reader.INSTRUCTIONS[:200], self.body)
|
|
|
|
def test_body_says_nothing_about_how_the_reader_is_tested(self) -> None:
|
|
text = self.body.lower()
|
|
for word in ("calibrat", "gate", "planted", "filler", "key", "label", "position"):
|
|
self.assertNotIn(word, text, word)
|
|
|
|
def test_project_agent_is_the_shipped_reader_byte_for_byte(self) -> None:
|
|
# A session started in this repository loads .claude/agents/; the reader it gets must be the one the
|
|
# calibration gate measured, so the two files may never drift apart.
|
|
shipped = (ROOT / "agents" / "okf-wiki-reader.md").read_bytes()
|
|
self.assertEqual(shipped, (ROOT / ".claude" / "agents" / "okf-wiki-reader.md").read_bytes())
|
|
|
|
|
|
class LoadPairsTest(unittest.TestCase):
|
|
def test_only_id_claim_and_excerpts_survive(self) -> None:
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
pairs = reader.load_pairs(write_json(Path(tmp) / "set.json", {"schema": "x", "pairs": PAIRS}))
|
|
self.assertEqual([p.id for p in pairs], ["c001", "c002", "c003"])
|
|
self.assertEqual(pairs[1].excerpts[1].file, "flour.md")
|
|
self.assertEqual(pairs[1].excerpts[1].line, 2)
|
|
|
|
def test_a_pair_without_excerpts_is_an_error(self) -> None:
|
|
bad = {"pairs": [{"id": "c009", "claim": "Bread is good.", "excerpts": []}]}
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
with self.assertRaises(reader.RunError):
|
|
reader.load_pairs(write_json(Path(tmp) / "set.json", bad))
|
|
|
|
def test_duplicate_ids_are_an_error(self) -> None:
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
with self.assertRaises(reader.RunError):
|
|
reader.load_pairs(write_json(Path(tmp) / "set.json", {"pairs": [PAIRS[0], PAIRS[0]]}))
|
|
|
|
|
|
class LoadCallsTest(unittest.TestCase):
|
|
def test_calls_keep_layout_order(self) -> None:
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
b = Bench(tmp)
|
|
calls = reader.load_calls(b.calls, b.set)
|
|
self.assertEqual([(c.name, c.ids) for c in calls], [("k001", ("c002", "c001")), ("k002", ("c003",))])
|
|
|
|
def test_a_layout_may_hold_part_of_the_set(self) -> None:
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
b = Bench(tmp, [("k001", ["c003"])])
|
|
self.assertEqual(len(reader.load_calls(b.calls, b.set)), 1)
|
|
|
|
def test_layouts_that_break_k3_are_refused(self) -> None:
|
|
cases: dict[str, list[tuple[str, list[str]]]] = {
|
|
"unknown id": [("k001", ["c001", "c404"])],
|
|
"id twice": [("k001", ["c001"]), ("k002", ["c001"])],
|
|
"call name twice": [("k001", ["c001"]), ("k001", ["c002"])],
|
|
"empty call": [("k001", [])],
|
|
"unsafe call name": [("../k001", ["c001"])],
|
|
}
|
|
for what, calls in cases.items():
|
|
with self.subTest(what), tempfile.TemporaryDirectory() as tmp:
|
|
b = Bench(tmp, calls)
|
|
with self.assertRaises(reader.RunError):
|
|
reader.load_calls(b.calls, b.set)
|
|
|
|
def test_a_call_over_the_size_limit_is_refused(self) -> None:
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
b = Bench(tmp)
|
|
write_json(b.calls, layout(b.set, [("k001", ["c001", "c002", "c003"])], size_max=2))
|
|
with self.assertRaises(reader.RunError):
|
|
reader.load_calls(b.calls, b.set)
|
|
write_json(b.calls, layout(b.set, [("k001", ["c001"])], size_max=11))
|
|
with self.assertRaises(reader.RunError):
|
|
reader.load_calls(b.calls, b.set)
|
|
|
|
def test_a_layout_bound_to_another_set_is_refused(self) -> None:
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
b = Bench(tmp)
|
|
data = json.loads(b.calls.read_text())
|
|
data["set_sha256"] = "0" * 64
|
|
write_json(b.calls, data)
|
|
with self.assertRaises(reader.RunError):
|
|
reader.load_calls(b.calls, b.set)
|
|
|
|
def test_a_layout_with_another_schema_is_refused(self) -> None:
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
b = Bench(tmp)
|
|
data = json.loads(b.calls.read_text())
|
|
data["schema"] = "okf-wiki/reader-calls/0"
|
|
write_json(b.calls, data)
|
|
with self.assertRaises(reader.RunError):
|
|
reader.load_calls(b.calls, b.set)
|
|
|
|
|
|
class RenderTest(unittest.TestCase):
|
|
def test_render_writes_one_task_per_call_named_after_the_call(self) -> None:
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
b = Bench(tmp)
|
|
code, _ = b.render()
|
|
self.assertEqual(code, 0)
|
|
self.assertEqual(sorted(p.name for p in b.tasks.iterdir()), ["k001.md", "k002.md"])
|
|
|
|
def test_task_is_the_instructions_then_the_claims_numbered_in_layout_order_with_their_excerpts(self) -> None:
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
b = Bench(tmp)
|
|
b.render()
|
|
task = (b.tasks / "k001.md").read_text(encoding="utf-8")
|
|
self.assertTrue(task.startswith(reader.INSTRUCTIONS.strip()))
|
|
first, second = task.index("1. <claim>" + PAIRS[1]["claim"]), task.index("2. <claim>" + PAIRS[0]["claim"])
|
|
self.assertLess(task.index(reader.INSTRUCTIONS.strip()[-40:]), first)
|
|
self.assertLess(first, second)
|
|
for e in PAIRS[1]["excerpts"]:
|
|
at = task.index(f'<excerpt file="{e["file"]}" line="{e["line"]}">\n{e["text"]}\n</excerpt>')
|
|
self.assertTrue(first < at < second)
|
|
self.assertIn(f'<excerpt file="bread.md" line="3">\n{PAIRS[0]["excerpts"][0]["text"]}\n</excerpt>',
|
|
task[second:])
|
|
self.assertNotIn("3. <claim>", task)
|
|
|
|
def test_task_never_carries_an_id_the_key_the_call_name_or_another_call(self) -> None:
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
b = Bench(tmp)
|
|
b.render()
|
|
task = (b.tasks / "k001.md").read_text(encoding="utf-8")
|
|
for leak in ("c001", "c002", "k001", "paraphrase", "twelve hours is half a day", PAIRS[2]["claim"],
|
|
PAIRS[2]["excerpts"][0]["text"]):
|
|
self.assertNotIn(leak, task)
|
|
|
|
def test_the_system_text_sharing_eight_words_with_a_claim_or_excerpt_is_refused(self) -> None:
|
|
for leak in ("For example: the dough rests for half a day before baking.",
|
|
"Note that rye flour absorbs more water than wheat flour does."):
|
|
with self.subTest(leak), tempfile.TemporaryDirectory() as tmp:
|
|
b = Bench(tmp)
|
|
b.agent.write_text(AGENT + "\n" + leak + "\n", encoding="utf-8")
|
|
code, err = b.render()
|
|
self.assertEqual(code, 1)
|
|
self.assertIn("share a run of 8 words", err)
|
|
self.assertFalse(b.tasks.exists())
|
|
|
|
def test_the_instructions_in_the_task_sharing_eight_words_with_a_claim_or_excerpt_are_refused(self) -> None:
|
|
leaky = reader.INSTRUCTIONS + "\nFor example: sourdough needs a starter that is fed daily.\n"
|
|
with tempfile.TemporaryDirectory() as tmp, mock.patch.object(reader, "INSTRUCTIONS", leaky):
|
|
b = Bench(tmp)
|
|
code, err = b.render()
|
|
self.assertEqual(code, 1)
|
|
self.assertIn("share a run of 8 words", err)
|
|
self.assertFalse(b.tasks.exists())
|
|
|
|
def test_shared_runs_count_words_not_layout(self) -> None:
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
pairs = reader.load_pairs(write_json(Path(tmp) / "set.json", {"pairs": PAIRS}))
|
|
self.assertEqual(reader.shared_runs("THE DOUGH rests\nfor twelve hours before baking", pairs), ["c001"])
|
|
self.assertEqual(reader.shared_runs("the dough rests for twelve hours before", pairs), [])
|
|
|
|
def test_render_refuses_to_change_a_task_already_written(self) -> None:
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
b = Bench(tmp)
|
|
self.assertEqual(b.render()[0], 0)
|
|
self.assertEqual(b.render()[0], 0)
|
|
(b.tasks / "k002.md").write_text("edited\n", encoding="utf-8")
|
|
code, err = b.render()
|
|
self.assertEqual(code, 1)
|
|
self.assertIn("k002.md", err)
|
|
self.assertEqual((b.tasks / "k002.md").read_text(encoding="utf-8"), "edited\n")
|
|
|
|
|
|
class ParseReplyTest(unittest.TestCase):
|
|
"""K3 point 3: one rule, fail-closed."""
|
|
|
|
def test_lines_in_any_order(self) -> None:
|
|
self.assertEqual(reader.parse_reply("2: unsupported\n1: supported\n3: supported", 3),
|
|
{1: "supported", 2: "unsupported", 3: "supported"})
|
|
|
|
def test_lines_are_stripped_and_empty_lines_dropped(self) -> None:
|
|
self.assertEqual(reader.parse_reply("\n 1: supported \n\n2 :unsupported\n10:supported\n", 10),
|
|
{1: "supported", 2: "unsupported", 10: "supported"})
|
|
|
|
def test_one_line_that_does_not_fit_gives_no_verdict_for_any_claim(self) -> None:
|
|
for reply in ("1: supported\n2: unsupported\nThat is all.",
|
|
"Verdicts:\n1: supported\n2: unsupported",
|
|
"1: supported (the excerpt says so)\n2: unsupported",
|
|
"```\n1: supported\n2: unsupported\n```",
|
|
"**1: supported**\n2: unsupported",
|
|
"1: Supported\n2: unsupported",
|
|
"1. supported\n2: unsupported",
|
|
"01: supported\n2: unsupported",
|
|
"0: supported\n1: supported\n2: unsupported",
|
|
"1: supported, 2: unsupported"):
|
|
with self.subTest(reply):
|
|
self.assertEqual(reader.parse_reply(reply, 2), {})
|
|
|
|
def test_a_number_outside_one_to_n_gives_no_verdict_for_any_claim(self) -> None:
|
|
self.assertEqual(reader.parse_reply("1: supported\n2: supported\n3: unsupported", 2), {})
|
|
|
|
def test_a_number_given_twice_gives_no_verdict_for_that_claim(self) -> None:
|
|
self.assertEqual(reader.parse_reply("1: supported\n1: unsupported\n2: supported", 2), {2: "supported"})
|
|
self.assertEqual(reader.parse_reply("1: supported\n1: supported\n2: supported", 2), {2: "supported"})
|
|
|
|
def test_a_missing_number_gives_no_verdict_for_that_claim(self) -> None:
|
|
self.assertEqual(reader.parse_reply("1: supported\n3: unsupported", 3), {1: "supported", 3: "unsupported"})
|
|
|
|
def test_an_empty_reply_gives_no_verdict(self) -> None:
|
|
self.assertEqual(reader.parse_reply("", 3), {})
|
|
self.assertEqual(reader.parse_reply("\n \n", 3), {})
|
|
|
|
|
|
MODEL = "claude-opus-5-5"
|
|
|
|
|
|
def assistant(mid: str, *blocks: dict[str, Any], model: str | None = MODEL) -> list[dict[str, Any]]:
|
|
"""One assistant message as the transcript writes it: one record per content block, all with the message id and,
|
|
as Claude Code writes it, the model that wrote the message (left out when model is None)."""
|
|
named: dict[str, Any] = {} if model is None else {"model": model}
|
|
return [{"type": "assistant", "message": {"id": mid, "role": "assistant", "content": [b], **named,
|
|
"stop_reason": "end_turn" if i == len(blocks) - 1 else None}}
|
|
for i, b in enumerate(blocks)]
|
|
|
|
|
|
def text(t: str) -> dict[str, Any]:
|
|
return {"type": "text", "text": t}
|
|
|
|
|
|
def subagent(root: Path, session: str, agent: str, description: str, records: list[dict[str, Any]],
|
|
agent_type: str = "okf-wiki-reader", where: str = "subagents") -> None:
|
|
d = root / session / where
|
|
d.mkdir(parents=True, exist_ok=True)
|
|
(d / f"agent-{agent}.meta.json").write_text(json.dumps({"agentType": agent_type,
|
|
"description": description}), encoding="utf-8")
|
|
(d / f"agent-{agent}.jsonl").write_text("".join(json.dumps(r) + "\n" for r in records), encoding="utf-8")
|
|
|
|
|
|
def answered(reply: str, mid: str = "m2", model: str | None = MODEL) -> list[dict[str, Any]]:
|
|
"""A reader that reads its task, says a word before the tool call, thinks, then replies."""
|
|
return [{"type": "user", "message": {"role": "user", "content": "Task file: /bench/tasks/k001.md"}},
|
|
*assistant("m1", text("I will read the task."),
|
|
{"type": "tool_use", "id": "t1", "name": "Read", "input": {"file_path": "/bench/k001.md"}},
|
|
model=model),
|
|
{"type": "user", "message": {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "t1",
|
|
"content": "the task"}]}},
|
|
*assistant(mid, {"type": "thinking", "thinking": "1: unsupported\nweighing claim one"}, text(reply),
|
|
model=model)]
|
|
|
|
|
|
def handed_back(reply: Any, mid: str = "m2", model: str | None = MODEL) -> list[dict[str, Any]]:
|
|
"""A reader that reads its task, thinks, and delivers its reply through the hand-back tool (Claude Code 2.1.295)."""
|
|
return answered("", mid, model)[:-2] + assistant(mid, {"type": "thinking", "thinking": "2: unsupported"},
|
|
{"type": "tool_use", "id": "t2", "name": "SubagentHandback",
|
|
"input": {"message": reply}}, model=model)
|
|
|
|
|
|
class FinalReplyTest(unittest.TestCase):
|
|
def test_the_reply_is_the_hand_back_message_when_the_reader_hands_back(self) -> None:
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
subagent(Path(tmp), "s1", "a1", "V1-kal9-read-k001", handed_back("1: supported\n2: unsupported"))
|
|
path = Path(tmp) / "s1" / "subagents" / "agent-a1.jsonl"
|
|
self.assertEqual(reader.final_reply(path), "1: supported\n2: unsupported")
|
|
|
|
def test_the_last_hand_back_wins_and_text_beside_it_is_not_the_reply(self) -> None:
|
|
records = handed_back("1: unsupported") + assistant("m3", text("Here are my verdicts."), {
|
|
"type": "tool_use", "id": "t3", "name": "SubagentHandback", "input": {"message": "1: supported"}})
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
subagent(Path(tmp), "s1", "a1", "V1-kal9-read-k001", records)
|
|
path = Path(tmp) / "s1" / "subagents" / "agent-a1.jsonl"
|
|
self.assertEqual(reader.final_reply(path), "1: supported")
|
|
|
|
def test_a_hand_back_without_a_text_message_is_empty(self) -> None:
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
subagent(Path(tmp), "s1", "a1", "V1-kal9-read-k001", handed_back(["1: supported"]))
|
|
path = Path(tmp) / "s1" / "subagents" / "agent-a1.jsonl"
|
|
self.assertEqual(reader.final_reply(path), "")
|
|
|
|
def test_the_reply_is_the_text_of_the_last_assistant_message_without_thinking(self) -> None:
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
subagent(Path(tmp), "s1", "a1", "V1-kal9-read-k001", answered("1: supported\n2: supported"))
|
|
path = Path(tmp) / "s1" / "subagents" / "agent-a1.jsonl"
|
|
self.assertEqual(reader.final_reply(path), "1: supported\n2: supported")
|
|
|
|
def test_text_blocks_of_the_last_message_are_joined_in_order(self) -> None:
|
|
records = answered("1: supported") + assistant("m3", text("2: unsupported"), text("1: supported"))
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
subagent(Path(tmp), "s1", "a1", "V1-kal9-read-k001", records)
|
|
path = Path(tmp) / "s1" / "subagents" / "agent-a1.jsonl"
|
|
self.assertEqual(reader.final_reply(path), "2: unsupported\n1: supported")
|
|
|
|
def test_a_transcript_without_a_reply_is_empty(self) -> None:
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
subagent(Path(tmp), "s1", "a1", "V1-kal9-read-k001", answered("x")[:2])
|
|
path = Path(tmp) / "s1" / "subagents" / "agent-a1.jsonl"
|
|
self.assertEqual(reader.final_reply(path), "I will read the task.")
|
|
subagent(Path(tmp), "s1", "a2", "V1-kal9-read-k002", answered("x")[:1])
|
|
self.assertEqual(reader.final_reply(Path(tmp) / "s1" / "subagents" / "agent-a2.jsonl"), "")
|
|
|
|
|
|
class CollectTest(unittest.TestCase):
|
|
def collect(self, b: Bench, transcripts: Path, run: str = "kal9",
|
|
check: tuple[str, ...] = ("--calibration",)) -> tuple[int, str]:
|
|
out = io.StringIO()
|
|
with contextlib.redirect_stdout(out), contextlib.redirect_stderr(io.StringIO()):
|
|
code = reader.main(["collect", "--set", str(b.set), "--calls", str(b.calls), "--transcripts",
|
|
str(transcripts), "--run", run, "--reader", "r1", "--out", str(b.dir / "v.json"),
|
|
*check])
|
|
return code, out.getvalue()
|
|
|
|
def verdicts(self, b: Bench) -> tuple[str, dict[str, str], str]:
|
|
result: tuple[str, dict[str, str], str] = calibration.read_verdicts(b.dir / "v.json")
|
|
return result
|
|
|
|
def test_the_gate_reads_what_collect_writes_from_the_transcripts(self) -> None:
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
b, t = Bench(tmp), Path(tmp) / "transcripts"
|
|
subagent(t, "s1", "a1", "V1-kal9-read-k001", answered("2: supported\n1: unsupported"))
|
|
subagent(t, "s1", "a2", "V1-kal9-read-k002", answered("1: supported"))
|
|
code, out = self.collect(b, t)
|
|
self.assertEqual(code, 0)
|
|
self.assertIn("3 of 3 verdicts", out)
|
|
name, verdicts, fail = self.verdicts(b)
|
|
doc = json.loads((b.dir / "v.json").read_text(encoding="utf-8"))
|
|
self.assertEqual((name, fail), ("r1", ""))
|
|
self.assertEqual(verdicts, {"c002": "unsupported", "c001": "supported", "c003": "supported"})
|
|
self.assertEqual([v["id"] for v in doc["verdicts"]], ["c002", "c001", "c003"])
|
|
|
|
def test_a_call_with_no_transcript_leaves_its_claims_without_verdict_and_exits_1(self) -> None:
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
b, t = Bench(tmp), Path(tmp) / "transcripts"
|
|
subagent(t, "s1", "a1", "V1-kal9-read-k001", answered("1: supported\n2: supported"))
|
|
code, out = self.collect(b, t)
|
|
self.assertEqual(code, 1)
|
|
self.assertIn("2 of 3 verdicts", out)
|
|
self.assertEqual(set(self.verdicts(b)[1]), {"c001", "c002"})
|
|
|
|
def test_replies_handed_back_are_collected(self) -> None:
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
b, t = Bench(tmp), Path(tmp) / "transcripts"
|
|
subagent(t, "s1", "a1", "V1-kal9-read-k001", handed_back("1: unsupported\n2: supported"))
|
|
subagent(t, "s1", "a2", "V1-kal9-read-k002", handed_back("1: unsupported"))
|
|
code, out = self.collect(b, t)
|
|
self.assertEqual(code, 0)
|
|
self.assertEqual(self.verdicts(b)[1], {"c002": "unsupported", "c001": "supported", "c003": "unsupported"})
|
|
|
|
def test_a_reply_that_breaks_the_rule_loses_its_whole_call(self) -> None:
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
b, t = Bench(tmp), Path(tmp) / "transcripts"
|
|
subagent(t, "s1", "a1", "V1-kal9-read-k001", answered("1: supported\n2: supported\nDone."))
|
|
subagent(t, "s1", "a2", "V1-kal9-read-k002", answered("1: supported"))
|
|
code, out = self.collect(b, t)
|
|
self.assertEqual(code, 1)
|
|
self.assertIn("1 broke the reply rule", out)
|
|
self.assertEqual(self.verdicts(b)[1], {"c003": "supported"})
|
|
|
|
def test_a_call_answered_by_two_readers_gets_no_verdict(self) -> None:
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
b, t = Bench(tmp), Path(tmp) / "transcripts"
|
|
subagent(t, "s1", "a1", "V1-kal9-read-k001", answered("1: supported\n2: supported"))
|
|
subagent(t, "s2", "a9", "V1-kal9-read-k001", answered("1: supported\n2: supported"))
|
|
subagent(t, "s1", "a2", "V1-kal9-read-k002", answered("1: supported"))
|
|
code, out = self.collect(b, t)
|
|
self.assertEqual(code, 1)
|
|
self.assertIn("1 answered twice", out)
|
|
self.assertEqual(self.verdicts(b)[1], {"c003": "supported"})
|
|
|
|
def test_other_runs_other_lines_and_other_agents_are_ignored(self) -> None:
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
b, t = Bench(tmp), Path(tmp) / "transcripts"
|
|
subagent(t, "s1", "a1", "V1-kal9-read-k001", answered("1: supported\n2: supported"))
|
|
subagent(t, "s1", "a2", "V1-kal9-read-k002", answered("1: supported"))
|
|
subagent(t, "s1", "a3", "V1-kal8-read-k001", answered("1: unsupported\n2: unsupported"))
|
|
subagent(t, "s1", "a4", "V1-kal9-pack-k002", answered("1: unsupported"))
|
|
subagent(t, "s1", "a5", "a helper", answered("1: unsupported"))
|
|
code, out = self.collect(b, t)
|
|
self.assertEqual(code, 0)
|
|
self.assertEqual(set(self.verdicts(b)[1].values()), {"supported"})
|
|
|
|
def test_a_transcript_for_a_call_not_in_the_layout_is_counted(self) -> None:
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
b, t = Bench(tmp), Path(tmp) / "transcripts"
|
|
subagent(t, "s1", "a1", "V1-kal9-read-k001", answered("1: supported\n2: supported"))
|
|
subagent(t, "s1", "a2", "V1-kal9-read-k002", answered("1: supported"))
|
|
subagent(t, "s1", "a3", "V1-kal9-read-k777", answered("1: supported"))
|
|
code, out = self.collect(b, t)
|
|
self.assertEqual(code, 0)
|
|
self.assertIn("1 transcript for a call not in the layout", out)
|
|
|
|
def test_readers_run_as_a_workflow_are_collected(self) -> None:
|
|
# A workflow writes its agents' transcripts one level down, under subagents/workflows/<run id>/.
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
b, t = Bench(tmp), Path(tmp) / "transcripts"
|
|
subagent(t, "s1", "a1", "V1-kal9-read-k001", answered("1: supported\n2: supported"))
|
|
subagent(t, "s1", "a2", "V1-kal9-read-k002", answered("1: unsupported"), where="subagents/workflows/wf_1")
|
|
code, out = self.collect(b, t)
|
|
self.assertEqual(code, 0)
|
|
self.assertEqual(self.verdicts(b)[1]["c003"], "unsupported")
|
|
|
|
def test_a_reader_description_on_another_agent_type_is_not_collected(self) -> None:
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
b, t = Bench(tmp), Path(tmp) / "transcripts"
|
|
subagent(t, "s1", "a1", "V1-kal9-read-k001", answered("1: supported\n2: supported"))
|
|
subagent(t, "s1", "a2", "V1-kal9-read-k002", answered("1: supported"), agent_type="general-purpose")
|
|
code, out = self.collect(b, t)
|
|
self.assertEqual(code, 1)
|
|
self.assertIn("1 not answered", out)
|
|
self.assertNotIn("c003", self.verdicts(b)[1])
|
|
|
|
def test_a_set_that_cannot_be_read_fails_with_a_message_and_writes_nothing(self) -> None:
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
b = Bench(tmp)
|
|
b.set.unlink()
|
|
err = io.StringIO()
|
|
with contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(err):
|
|
code = reader.main(["collect", "--set", str(b.set), "--calls", str(b.calls), "--transcripts", tmp,
|
|
"--run", "kal9", "--reader", "r1", "--out", str(b.dir / "v.json"),
|
|
"--calibration"])
|
|
self.assertEqual(code, 1)
|
|
self.assertIn("FAIL:", err.getvalue())
|
|
self.assertFalse((b.dir / "v.json").exists())
|
|
|
|
def test_collect_needs_a_transcripts_directory(self) -> None:
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
b = Bench(tmp)
|
|
code, _ = self.collect(b, Path(tmp) / "nowhere")
|
|
self.assertEqual(code, 1)
|
|
self.assertFalse((b.dir / "v.json").exists())
|
|
|
|
|
|
SET_SHA, KEY_SHA = "a" * 64, "b" * 64 # the set and key the test manifest names
|
|
|
|
|
|
def passed_record(models: list[str], set_sha: str = SET_SHA, key_sha: str = KEY_SHA) -> dict[str, Any]:
|
|
"""A record of green calibration runs (okf-wiki/calibration-passed/1) with one entry per model."""
|
|
return {"schema": "okf-wiki/calibration-passed/1",
|
|
"passed": [{"model": m, "set_sha256": set_sha, "key_sha256": key_sha, "verdicts_sha256": "c" * 64,
|
|
"reader": "r0"} for m in models]}
|
|
|
|
|
|
def test_manifest(directory: Path) -> Path:
|
|
"""A calibration manifest that names the set SET_SHA and the key KEY_SHA."""
|
|
return write_json(directory / "manifest.json", {"schema": "okf-wiki/calibration-manifest/2",
|
|
"set": {"file": "s.json", "sha256": SET_SHA},
|
|
"key": {"file": "k.json", "sha256": KEY_SHA}})
|
|
|
|
|
|
class CollectModelTest(unittest.TestCase):
|
|
"""PLAN § Rammer 23: collect names the model the readers ran on, and turns a reading into verdicts only when that
|
|
model has a recorded green calibration run (--gate-passed); the calibration run itself says --calibration."""
|
|
|
|
def run_collect(self, b: Bench, transcripts: Path, *check: str) -> tuple[int, str]:
|
|
out = io.StringIO()
|
|
with contextlib.redirect_stdout(out), contextlib.redirect_stderr(out):
|
|
try:
|
|
manifest = ["--manifest", str(test_manifest(b.dir))] if "--gate-passed" in check else []
|
|
code = reader.main(["collect", "--set", str(b.set), "--calls", str(b.calls), "--transcripts",
|
|
str(transcripts), "--run", "kal9", "--reader", "r1", "--out",
|
|
str(b.dir / "v.json"), *check, *manifest])
|
|
except SystemExit as exc: # argparse refuses the options
|
|
code = exc.code if isinstance(exc.code, int) else 2
|
|
return code, out.getvalue()
|
|
|
|
def collect(self, b: Bench, transcripts: Path) -> tuple[int, str]:
|
|
return self.run_collect(b, transcripts, "--calibration")
|
|
|
|
def two_readers(self, t: Path, first: str | None = MODEL, second: str | None = MODEL) -> None:
|
|
subagent(t, "s1", "a1", "V1-kal9-read-k001", answered("1: supported\n2: supported", model=first))
|
|
subagent(t, "s1", "a2", "V1-kal9-read-k002", answered("1: supported", model=second))
|
|
|
|
def doc(self, b: Bench) -> dict[str, Any]:
|
|
data: dict[str, Any] = json.loads((b.dir / "v.json").read_text(encoding="utf-8"))
|
|
return data
|
|
|
|
def test_collect_writes_and_prints_the_sorted_distinct_models_of_its_readers(self) -> None:
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
b, t = Bench(tmp), Path(tmp) / "transcripts"
|
|
self.two_readers(t, "claude-test-b", "claude-test-a")
|
|
code, out = self.collect(b, t)
|
|
self.assertEqual(code, 0, out)
|
|
doc = self.doc(b)
|
|
self.assertEqual(doc["models"], ["claude-test-a", "claude-test-b"])
|
|
self.assertEqual(list(doc), ["reader", "models", "verdicts"])
|
|
self.assertIn("models: claude-test-a, claude-test-b\n", out)
|
|
|
|
def test_one_model_on_every_reader_is_named_once(self) -> None:
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
b, t = Bench(tmp), Path(tmp) / "transcripts"
|
|
self.two_readers(t)
|
|
code, out = self.collect(b, t)
|
|
self.assertEqual(self.doc(b)["models"], [MODEL])
|
|
self.assertIn(f"models: {MODEL}\n", out)
|
|
|
|
def test_records_without_a_model_and_synthetic_ones_are_ignored(self) -> None:
|
|
records = answered("1: supported\n2: supported", model=None) + assistant(
|
|
"m3", text("1: supported\n2: supported"), model="<synthetic>")
|
|
records[2]["message"]["model"] = MODEL # one record of the reader's own turns names the model
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
b, t = Bench(tmp), Path(tmp) / "transcripts"
|
|
subagent(t, "s1", "a1", "V1-kal9-read-k001", records)
|
|
subagent(t, "s1", "a2", "V1-kal9-read-k002", answered("1: supported", model=None))
|
|
code, out = self.collect(b, t)
|
|
self.assertEqual(code, 0, out)
|
|
self.assertEqual(self.doc(b)["models"], [MODEL])
|
|
|
|
def test_readers_that_name_no_model_give_an_empty_list_and_none(self) -> None:
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
b, t = Bench(tmp), Path(tmp) / "transcripts"
|
|
self.two_readers(t, None, None)
|
|
code, out = self.collect(b, t)
|
|
self.assertEqual(self.doc(b)["models"], [])
|
|
self.assertIn("models: (none)\n", out)
|
|
|
|
def test_transcripts_collect_does_not_use_do_not_name_a_model(self) -> None:
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
b, t = Bench(tmp), Path(tmp) / "transcripts"
|
|
self.two_readers(t)
|
|
subagent(t, "s1", "a3", "V1-kal8-read-k001", answered("1: supported", model="claude-other-run"))
|
|
subagent(t, "s1", "a4", "V1-kal9-read-k777", answered("1: supported", model="claude-stray"))
|
|
subagent(t, "s1", "a5", "V1-kal9-read-k002", answered("1: supported", model="claude-helper"),
|
|
agent_type="general-purpose")
|
|
code, out = self.collect(b, t)
|
|
self.assertEqual(self.doc(b)["models"], [MODEL])
|
|
|
|
def test_readers_started_from_a_plugin_are_collected_by_their_namespaced_type(self) -> None:
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
b, t = Bench(tmp), Path(tmp) / "transcripts"
|
|
subagent(t, "s1", "a1", "V1-kal9-read-k001", answered("1: supported\n2: supported"),
|
|
agent_type="okf-wiki:okf-wiki-reader")
|
|
subagent(t, "s1", "a2", "V1-kal9-read-k002", answered("1: supported"), agent_type="okf-wiki-reader")
|
|
for i, other in enumerate(("okf-wiki:okf-wiki-reader-old", "okf-wiki-reader:helper", ":okf-wiki-reader")):
|
|
subagent(t, "s2", f"x{i}", "V1-kal9-read-k001", answered("1: unsupported\n2: unsupported"),
|
|
agent_type=other)
|
|
code, out = self.collect(b, t)
|
|
self.assertEqual(code, 0, out)
|
|
self.assertIn("3 of 3 verdicts", out)
|
|
|
|
def test_collect_needs_exactly_one_of_calibration_and_gate_passed(self) -> None:
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
b, t = Bench(tmp), Path(tmp) / "transcripts"
|
|
self.two_readers(t)
|
|
passed = write_json(b.dir / "passed.json", passed_record([MODEL]))
|
|
for check in ((), ("--calibration", "--gate-passed", str(passed))):
|
|
with self.subTest(check=check):
|
|
code, _ = self.run_collect(b, t, *check)
|
|
self.assertEqual(code, 2)
|
|
self.assertFalse((b.dir / "v.json").exists())
|
|
|
|
def test_the_tracked_record_holds_a_green_run_on_the_tracked_set_and_key(self) -> None:
|
|
# The set itself stays out of git (rule 3), so a fresh clone could not run the gate and collect refused every
|
|
# reading (motor B16); the record holds only models and sha256, no text.
|
|
self.assertIn("claude-opus-5-5", reader.passed_models(reader.CALIBRATION_PASSED, reader.CALIBRATION_MANIFEST))
|
|
|
|
def test_gate_passed_without_a_file_reads_the_tracked_record(self) -> None:
|
|
args = reader.build_parser().parse_args(["collect", "--set", "s", "--calls", "c", "--transcripts", "t",
|
|
"--run", "r", "--reader", "n", "--out", "o", "--gate-passed"])
|
|
self.assertEqual(args.gate_passed, str(reader.CALIBRATION_PASSED))
|
|
|
|
def test_a_model_with_a_green_run_on_record_is_collected(self) -> None:
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
b, t = Bench(tmp), Path(tmp) / "transcripts"
|
|
self.two_readers(t)
|
|
passed = write_json(b.dir / "passed.json", passed_record(["claude-older", MODEL]))
|
|
code, out = self.run_collect(b, t, "--gate-passed", str(passed))
|
|
self.assertEqual(code, 0, out)
|
|
self.assertEqual(self.doc(b)["models"], [MODEL])
|
|
self.assertIn("3 of 3 verdicts", out)
|
|
|
|
def test_a_model_without_a_green_run_writes_nothing_and_exits_1(self) -> None:
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
b, t = Bench(tmp), Path(tmp) / "transcripts"
|
|
self.two_readers(t, "claude-new", MODEL)
|
|
passed = write_json(b.dir / "passed.json", passed_record([MODEL]))
|
|
code, out = self.run_collect(b, t, "--gate-passed", str(passed))
|
|
self.assertEqual(code, 1, out)
|
|
self.assertFalse((b.dir / "v.json").exists())
|
|
self.assertIn(f"collect cannot run: model(s) claude-new have no green calibration run in {passed} on the "
|
|
f"set and key of {b.dir / 'manifest.json'}; run the calibration gate on them first", out)
|
|
|
|
def test_readers_that_name_no_model_are_refused_against_the_record(self) -> None:
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
b, t = Bench(tmp), Path(tmp) / "transcripts"
|
|
self.two_readers(t, None, None)
|
|
passed = write_json(b.dir / "passed.json", passed_record([MODEL]))
|
|
code, out = self.run_collect(b, t, "--gate-passed", str(passed))
|
|
self.assertEqual(code, 1, out)
|
|
self.assertFalse((b.dir / "v.json").exists())
|
|
self.assertIn(f"collect cannot run: model(s) (none) have no green calibration run in {passed}", out)
|
|
|
|
def test_an_unreadable_or_malformed_record_writes_nothing_and_exits_1(self) -> None:
|
|
bad: dict[str, Any] = {
|
|
"missing": None,
|
|
"not json": "{",
|
|
"other schema": {"schema": "okf-wiki/other/1", "passed": [{"model": MODEL}]},
|
|
"no list": {"schema": "okf-wiki/calibration-passed/1", "passed": {"model": MODEL}},
|
|
"entry without a model": {"schema": "okf-wiki/calibration-passed/1", "passed": [{"reader": "r0"}]},
|
|
"entry without a set": {"schema": "okf-wiki/calibration-passed/1",
|
|
"passed": [{"model": MODEL, "key_sha256": KEY_SHA}]},
|
|
}
|
|
for what, content in bad.items():
|
|
with self.subTest(what=what), tempfile.TemporaryDirectory() as tmp:
|
|
b, t = Bench(tmp), Path(tmp) / "transcripts"
|
|
self.two_readers(t)
|
|
passed = b.dir / "passed.json"
|
|
if isinstance(content, str):
|
|
passed.write_text(content, encoding="utf-8")
|
|
elif content is not None:
|
|
write_json(passed, content)
|
|
code, out = self.run_collect(b, t, "--gate-passed", str(passed))
|
|
self.assertEqual(code, 1, out)
|
|
self.assertFalse((b.dir / "v.json").exists())
|
|
self.assertIn(str(passed), out)
|
|
|
|
def test_the_record_the_gate_writes_is_the_record_collect_reads(self) -> None:
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
b, t = Bench(tmp), Path(tmp) / "transcripts"
|
|
self.two_readers(t)
|
|
passed = b.dir / "passed.json"
|
|
calibration.record_passed(passed, {"model": MODEL, "set_sha256": SET_SHA, "key_sha256": KEY_SHA,
|
|
"verdicts_sha256": "c" * 64, "reader": "r0"})
|
|
code, out = self.run_collect(b, t, "--gate-passed", str(passed))
|
|
self.assertEqual(code, 0, out)
|
|
|
|
def test_a_green_run_on_another_set_or_key_does_not_clear_the_model(self) -> None:
|
|
for what, set_sha, key_sha in (("set", "d" * 64, KEY_SHA), ("key", SET_SHA, "e" * 64)):
|
|
with self.subTest(what=what), tempfile.TemporaryDirectory() as tmp:
|
|
b, t = Bench(tmp), Path(tmp) / "transcripts"
|
|
self.two_readers(t)
|
|
passed = write_json(b.dir / "passed.json", passed_record([MODEL], set_sha, key_sha))
|
|
code, out = self.run_collect(b, t, "--gate-passed", str(passed))
|
|
self.assertEqual(code, 1, out)
|
|
self.assertFalse((b.dir / "v.json").exists())
|
|
self.assertIn(f"model(s) {MODEL} have no green calibration run", out)
|
|
|
|
def test_the_default_manifest_is_the_tracked_calibration_manifest(self) -> None:
|
|
self.assertEqual(ROOT / "gates" / "calibration.manifest.json", reader.CALIBRATION_MANIFEST)
|
|
self.assertTrue(reader.CALIBRATION_MANIFEST.is_file())
|
|
|
|
def test_the_default_manifests_are_the_docs_set_and_the_folder_set(self) -> None:
|
|
# B16: a reader judges folder documents (Norwegian and English PDF/Office) only after it passed a set made
|
|
# of them, so a model must have a green run on both tracked sets before any reading counts.
|
|
args = reader.build_parser().parse_args(["collect", "--set", "s", "--calls", "c", "--transcripts", "t",
|
|
"--run", "r", "--reader", "n", "--out", "o", "--gate-passed"])
|
|
self.assertEqual(reader.manifests_of(args), [reader.CALIBRATION_MANIFEST, reader.CALIBRATION_FOLDER_MANIFEST])
|
|
self.assertEqual(ROOT / "gates" / "calibration-folder.manifest.json", reader.CALIBRATION_FOLDER_MANIFEST)
|
|
self.assertTrue(reader.CALIBRATION_FOLDER_MANIFEST.is_file())
|
|
|
|
def test_a_model_needs_a_green_run_on_every_manifest_given(self) -> None:
|
|
other_set, other_key = "d" * 64, "e" * 64
|
|
for models_on_other, expected in (([], 1), ([MODEL], 0)):
|
|
with self.subTest(models_on_other=models_on_other), tempfile.TemporaryDirectory() as tmp:
|
|
b, t = Bench(tmp), Path(tmp) / "transcripts"
|
|
self.two_readers(t)
|
|
second = write_json(b.dir / "manifest-2.json", {"schema": "okf-wiki/calibration-manifest/2",
|
|
"set": {"file": "s2.json", "sha256": other_set},
|
|
"key": {"file": "k2.json", "sha256": other_key}})
|
|
record = passed_record([MODEL])
|
|
record["passed"] += passed_record(models_on_other, other_set, other_key)["passed"]
|
|
passed = write_json(b.dir / "passed.json", record)
|
|
code, out = self.run_collect(b, t, "--gate-passed", str(passed), "--manifest", str(second))
|
|
self.assertEqual(code, expected, out)
|
|
self.assertEqual((b.dir / "v.json").exists(), expected == 0)
|
|
if expected:
|
|
self.assertIn(f"model(s) {MODEL} have no green calibration run in {passed} on the set and key "
|
|
f"of {second}", out)
|
|
|
|
def test_the_tracked_record_holds_a_green_run_on_the_folder_set(self) -> None:
|
|
self.assertIn("claude-opus-5-5",
|
|
reader.passed_models(reader.CALIBRATION_PASSED, reader.CALIBRATION_FOLDER_MANIFEST))
|
|
|
|
def test_the_gate_refuses_to_add_to_a_record_collect_cannot_read(self) -> None:
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
passed = write_json(Path(tmp) / "passed.json",
|
|
{"schema": "okf-wiki/calibration-passed/1", "passed": [{"reader": "r0"}]})
|
|
with self.assertRaises(calibration.RunError):
|
|
calibration.record_passed(passed, {"model": MODEL, "set_sha256": SET_SHA, "key_sha256": KEY_SHA,
|
|
"verdicts_sha256": "c" * 64, "reader": "r0"})
|
|
|
|
|
|
class NoModelTest(unittest.TestCase):
|
|
def test_tool_imports_no_network_process_or_model_client(self) -> None:
|
|
source = (ROOT / "okfwiki" / "reader.py").read_text(encoding="utf-8")
|
|
for banned in ("anthropic", "openai", "urllib", "http.client", "requests", "subprocess", "socket", "asyncio"):
|
|
self.assertNotIn(f"import {banned}", source)
|
|
self.assertNotIn(f"from {banned}", source)
|
|
for call in ("os.system", "os.popen", "os.exec", "os.spawn", "os.posix_spawn", "create_subprocess"):
|
|
self.assertNotIn(call, source)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|