First public release, published as one snapshot. What changed between versions is in CHANGELOG.md. Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
355 lines
19 KiB
Python
355 lines
19 KiB
Python
"""Tests for evals/direction_probe.py: the direction probe R (PLAN.md § Leveranser, row R).
|
|
|
|
Three arms answer the same questions from the same sources: file search alone, the okf-wiki plus file search, and a
|
|
wiki built with Astro-Han's karpathy-llm-wiki skill. The module sets up each arm's basis and task files, collects the
|
|
answers into the gates' format, harvests tokens and isolation breaches from the subagent transcripts, and prints the
|
|
table. It never calls a model. The fixtures are synthetic text about gardening (CLAUDE.md rule 3).
|
|
Run: python3 -m unittest discover -s evals/tests
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import sys
|
|
import tempfile
|
|
import unittest
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
sys.path.insert(0, str(ROOT / "evals"))
|
|
|
|
import direction_probe as dp # noqa: E402
|
|
|
|
TOMATO = "# Tomatoes\nTomatoes need full sun.\nWater them every morning.\n"
|
|
BASIL = "# Basil\nBasil likes warm soil.\n"
|
|
MINT = "# Mint\nMint spreads through runners.\n"
|
|
NOT_IN_CUT = "# Cactus\nCacti need little water.\n"
|
|
|
|
MD_SET: dict[str, Any] = {
|
|
"schema": "okf-wiki/multidoc-set/1",
|
|
"questions": [
|
|
{"id": "d01", "question": "What do tomatoes and basil need?",
|
|
"answer_keys": [{"file": "tomato.md", "text": "SECRETKEYSUN"}],
|
|
"evidence": [{"file": "tomato.md", "span": "SECRETSPANTOMATO"}]},
|
|
{"id": "d02", "question": "A question outside the probe.",
|
|
"answer_keys": [{"file": "mint.md", "text": "OTHERKEY"}],
|
|
"evidence": [{"file": "mint.md", "span": "OTHERSPAN"}]},
|
|
],
|
|
}
|
|
|
|
WD_SET: dict[str, Any] = {
|
|
"schema": "okf-wiki/wholedoc-set/1",
|
|
"questions": [
|
|
{"id": "d01", "type": "whole", "file": "tomato.md", "question": "What does the tomato page say overall?",
|
|
"answer_keys": [{"text": "SECRETWHOLEKEY"}], "evidence": [{"span": "SECRETWHOLESPAN"}]},
|
|
{"id": "d02", "type": "count", "file": "basil.md", "question": "How many basil lines are there?",
|
|
"count": 424242, "item_pattern": "SECRETPATTERN", "evidence": [{"span": "SECRETCOUNTSPAN"}]},
|
|
],
|
|
}
|
|
|
|
SECRETS = ("SECRETKEYSUN", "SECRETSPANTOMATO", "SECRETWHOLEKEY", "SECRETWHOLESPAN", "424242", "SECRETPATTERN",
|
|
"SECRETCOUNTSPAN", "OTHERKEY", "A question outside the probe")
|
|
|
|
|
|
class Fixture:
|
|
def __init__(self, root: Path) -> None:
|
|
self.root = root
|
|
self.sources = root / "kilde"
|
|
self.sources.mkdir()
|
|
for name, text in (("tomato.md", TOMATO), ("basil.md", BASIL), ("mint.md", MINT), ("cactus.md", NOT_IN_CUT)):
|
|
(self.sources / name).write_text(text, encoding="utf-8")
|
|
self.names = ["basil.md", "mint.md", "tomato.md"]
|
|
self.md_set = root / "md.json"
|
|
self.md_set.write_text(json.dumps(MD_SET), encoding="utf-8")
|
|
self.wd_set = root / "wd.json"
|
|
self.wd_set.write_text(json.dumps(WD_SET), encoding="utf-8")
|
|
self.wiki = root / "wiki-src"
|
|
self.wiki.mkdir()
|
|
(self.wiki / "index.md").write_text("# Index\n* [Tomatoes](tomatoes.md)\n", encoding="utf-8")
|
|
(self.wiki / "tomatoes.md").write_text("Tomatoes need full sun.\n", encoding="utf-8")
|
|
self.run = root / "run"
|
|
|
|
def setup(self, arm: str, **extra: Path) -> list[Path]:
|
|
return dp.setup_arm(run=self.run, arm=arm, sources=self.sources, names=self.names, md_set=self.md_set,
|
|
md_ids=["d01"], wd_set=self.wd_set, **extra)
|
|
|
|
|
|
class FetchAstroTest(unittest.TestCase):
|
|
def test_raw_file_keeps_the_name_and_the_body_byte_exact_under_the_template_header(self) -> None:
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
fx = Fixture(Path(tmp))
|
|
project = Path(tmp) / "astro"
|
|
written = dp.fetch_astro(sources=fx.sources, names=fx.names, project=project, topic="garden",
|
|
collected="2026-09-27")
|
|
self.assertEqual([p.name for p in written], fx.names)
|
|
raw = (project / "raw" / "garden" / "tomato.md").read_text(encoding="utf-8")
|
|
self.assertTrue(raw.startswith("# tomato\n\n> Source: kilde/tomato.md\n> Collected: 2026-09-27\n"
|
|
"> Published: Unknown\n\n"))
|
|
self.assertTrue(raw.endswith(TOMATO))
|
|
self.assertFalse((project / "raw" / "garden" / "cactus.md").exists())
|
|
|
|
def test_an_existing_raw_file_is_never_overwritten(self) -> None:
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
fx = Fixture(Path(tmp))
|
|
project = Path(tmp) / "astro"
|
|
dp.fetch_astro(sources=fx.sources, names=fx.names, project=project, topic="garden", collected="2026-09-27")
|
|
with self.assertRaises(dp.ProbeError):
|
|
dp.fetch_astro(sources=fx.sources, names=fx.names, project=project, topic="garden",
|
|
collected="2026-09-27")
|
|
|
|
|
|
class SetupTest(unittest.TestCase):
|
|
def test_file_search_basis_holds_exactly_the_cut(self) -> None:
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
fx = Fixture(Path(tmp))
|
|
fx.setup("filsok")
|
|
basis = fx.run / "arms" / "filsok"
|
|
self.assertEqual(sorted(p.name for p in basis.iterdir()), fx.names)
|
|
self.assertEqual((basis / "tomato.md").read_text(encoding="utf-8"), TOMATO)
|
|
|
|
def test_one_task_per_probe_question_and_none_for_the_rest(self) -> None:
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
fx = Fixture(Path(tmp))
|
|
tasks = fx.setup("filsok")
|
|
self.assertEqual(sorted(p.name for p in tasks), ["md-d01.md", "wd-d01.md", "wd-d02.md"])
|
|
self.assertTrue(all(p.parent == fx.run / "tasks" / "filsok" for p in tasks))
|
|
|
|
def test_a_task_carries_the_question_the_basis_and_the_answer_path_but_nothing_from_the_key(self) -> None:
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
fx = Fixture(Path(tmp))
|
|
tasks = {p.name: p.read_text(encoding="utf-8") for p in fx.setup("filsok")}
|
|
md = tasks["md-d01.md"]
|
|
self.assertIn("What do tomatoes and basil need?", md)
|
|
self.assertIn(str(fx.run / "arms" / "filsok"), md)
|
|
self.assertIn(str(fx.run / "answers" / "filsok" / "md-d01.json"), md)
|
|
self.assertIn('"count"', tasks["wd-d02.md"])
|
|
self.assertNotIn('"count"', md)
|
|
for text in tasks.values():
|
|
for secret in SECRETS:
|
|
self.assertNotIn(secret, text)
|
|
|
|
def test_the_wiki_arm_gets_the_wiki_and_the_cut_and_the_router_order(self) -> None:
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
fx = Fixture(Path(tmp))
|
|
tasks = fx.setup("wiki", wiki=fx.wiki)
|
|
basis = fx.run / "arms" / "wiki"
|
|
self.assertEqual(sorted(p.name for p in (basis / "kilde").iterdir()), fx.names)
|
|
self.assertTrue((basis / "wiki" / "index.md").is_file())
|
|
text = tasks[0].read_text(encoding="utf-8")
|
|
self.assertLess(text.index("index.md"), text.index("x-locator"))
|
|
|
|
def test_the_astro_arm_gets_raw_and_wiki_from_the_built_project(self) -> None:
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
fx = Fixture(Path(tmp))
|
|
project = Path(tmp) / "astro"
|
|
dp.fetch_astro(sources=fx.sources, names=fx.names, project=project, topic="garden", collected="2026-09-27")
|
|
(project / "wiki").mkdir()
|
|
(project / "wiki" / "index.md").write_text("# Knowledge Base Index\n", encoding="utf-8")
|
|
tasks = fx.setup("astro", astro=project)
|
|
basis = fx.run / "arms" / "astro"
|
|
self.assertTrue((basis / "raw" / "garden" / "basil.md").is_file())
|
|
self.assertTrue((basis / "wiki" / "index.md").is_file())
|
|
self.assertIn("Raw", tasks[0].read_text(encoding="utf-8"))
|
|
|
|
def test_an_arm_without_its_basis_or_an_existing_arm_is_refused(self) -> None:
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
fx = Fixture(Path(tmp))
|
|
with self.assertRaises(dp.ProbeError):
|
|
fx.setup("wiki")
|
|
fx.setup("filsok")
|
|
with self.assertRaises(dp.ProbeError):
|
|
fx.setup("filsok")
|
|
with self.assertRaises(dp.ProbeError):
|
|
fx.setup("grep-only")
|
|
|
|
def test_a_probe_id_missing_from_the_set_is_refused(self) -> None:
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
fx = Fixture(Path(tmp))
|
|
with self.assertRaises(dp.ProbeError):
|
|
dp.setup_arm(run=fx.run, arm="filsok", sources=fx.sources, names=fx.names, md_set=fx.md_set,
|
|
md_ids=["d09"], wd_set=fx.wd_set)
|
|
|
|
|
|
class CollectTest(unittest.TestCase):
|
|
def test_answers_are_gathered_per_gate_with_ids_from_the_file_names(self) -> None:
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
run = Path(tmp)
|
|
out = run / "answers" / "wiki"
|
|
out.mkdir(parents=True)
|
|
(out / "md-d01.json").write_text(json.dumps({"id": "wrong", "answer": "a", "citations": []}))
|
|
(out / "wd-d02.json").write_text(json.dumps({"answer": "7", "count": 7, "citations": []}))
|
|
(out / "wd-d03.json").write_text("not json")
|
|
md_path, wd_path, problems = dp.collect(run, "wiki")
|
|
md = json.loads(md_path.read_text(encoding="utf-8"))
|
|
wd = json.loads(wd_path.read_text(encoding="utf-8"))
|
|
self.assertEqual(md, {"arm": "wiki", "answers": [{"id": "d01", "answer": "a", "citations": []}]})
|
|
self.assertEqual(wd["answers"], [{"id": "d02", "answer": "7", "count": 7, "citations": []}])
|
|
self.assertEqual(problems, ["wd-d03.json: not a JSON object"])
|
|
|
|
|
|
def transcript(lines: list[dict[str, Any]]) -> str:
|
|
return "".join(json.dumps(line) + "\n" for line in lines)
|
|
|
|
|
|
def assistant(msg_id: str, tokens: tuple[int, int, int, int], *tools: tuple[str, dict[str, Any]]) -> dict[str, Any]:
|
|
i, cc, cr, o = tokens
|
|
content = [{"type": "tool_use", "name": name, "input": inp} for name, inp in tools]
|
|
return {"type": "assistant", "message": {"id": msg_id, "content": content, "usage": {
|
|
"input_tokens": i, "cache_creation_input_tokens": cc, "cache_read_input_tokens": cr, "output_tokens": o}}}
|
|
|
|
|
|
class HarvestTest(unittest.TestCase):
|
|
def write(self, folder: Path, name: str, lines: list[dict[str, Any]]) -> None:
|
|
(folder / name).write_text(transcript(lines), encoding="utf-8")
|
|
|
|
def test_tokens_count_each_message_once_and_breaches_are_named(self) -> None:
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
run, folder = Path(tmp) / "run", Path(tmp) / "subagents"
|
|
folder.mkdir()
|
|
task = run / "tasks" / "wiki" / "md-d01.md"
|
|
basis = run / "arms" / "wiki"
|
|
answer = run / "answers" / "wiki" / "md-d01.json"
|
|
first = {"type": "user", "message": {"content": f"Read {task} and follow it."}}
|
|
self.write(folder, "agent-a.jsonl", [
|
|
first,
|
|
assistant("m1", (2, 100, 1000, 5), ("Read", {"file_path": str(task)})),
|
|
assistant("m1", (2, 100, 1000, 5)),
|
|
assistant("m2", (1, 10, 2000, 50), ("Grep", {"pattern": "sun", "path": str(basis / "kilde")}),
|
|
("Glob", {"pattern": "*.md", "path": str(basis / "wiki")})),
|
|
assistant("m3", (1, 0, 3000, 80), ("Write", {"file_path": str(answer), "content": "{}"}),
|
|
("SubagentHandback", {"message": "done"})),
|
|
])
|
|
self.write(folder, "agent-b.jsonl", [
|
|
{"type": "user", "message": {"content": [{"type": "text", "text": f"Read {run}/tasks/filsok/wd-d02.md"}]}},
|
|
assistant("n1", (1, 1, 1, 1), ("Read", {"file_path": str(run / "arms" / "wiki" / "wiki" / "x.md")}),
|
|
("Grep", {"pattern": "sun"}), ("Bash", {"command": "ls"}),
|
|
("Read", {"file_path": str(run / "arms" / "filsok" / ".." / "wiki" / "y.md")})),
|
|
])
|
|
self.write(folder, "agent-c.jsonl", [{"type": "user", "message": {"content": "Something else."}}])
|
|
rows = dp.harvest(folder, run)
|
|
self.assertEqual(sorted(rows), ["filsok/wd-d02", "wiki/md-d01"])
|
|
wiki = rows["wiki/md-d01"]
|
|
self.assertEqual(wiki["tokens"], 1107 + 2061 + 3081)
|
|
self.assertEqual(wiki["breaches"], [])
|
|
self.assertEqual(wiki["transcript"], "agent-a.jsonl")
|
|
breaches = rows["filsok/wd-d02"]["breaches"]
|
|
self.assertEqual(len(breaches), 4)
|
|
self.assertTrue(any(b.startswith("Bash") for b in breaches))
|
|
self.assertTrue(any(b.startswith("Grep without a path") for b in breaches))
|
|
|
|
def test_bash_is_read_only_search_that_starts_in_the_basis(self) -> None:
|
|
basis = Path("/runs/r/arms/filsok")
|
|
task, answer = Path("/runs/r/tasks/filsok/md-d01.md"), Path("/runs/r/answers/filsok/md-d01.json")
|
|
allowed = [
|
|
f"cd {basis} && grep -rn 'full sun' .",
|
|
f"cd {basis} && grep -n sun tomato.md | head -5",
|
|
f"cd {basis} && find . -name '*.md' | sort && wc -l tomato.md 2>/dev/null",
|
|
f"cd '{basis}' && sed -n '1,40p' tomato.md",
|
|
f"cd {basis}/kilde && ls",
|
|
]
|
|
refused = [
|
|
"grep -rn sun /runs/r/arms/filsok",
|
|
f"cd {basis} && cat ../wiki/x.md",
|
|
f"cd {basis} && grep sun /runs/r/tasks/filsok/other.md",
|
|
f"cd {basis} && rm tomato.md",
|
|
f"cd {basis} && grep sun tomato.md > copy.txt",
|
|
f"cd {basis} && find . -name '*.md' -exec cat {{}} ;",
|
|
f"cd {basis} && sed -i 's/a/b/' tomato.md",
|
|
f"cd {basis} && cat $(echo /etc/hosts)",
|
|
f"cd {basis} && cd / && ls",
|
|
f"cd {basis} && ls ~",
|
|
"cd /runs/r/arms/wiki && ls",
|
|
]
|
|
for command in allowed:
|
|
self.assertEqual(dp.breaches_of([("Bash", {"command": command})], basis, task, answer), [], command)
|
|
for command in refused:
|
|
self.assertEqual(len(dp.breaches_of([("Bash", {"command": command})], basis, task, answer)), 1, command)
|
|
|
|
def test_the_task_names_bash_as_the_search_tool_and_its_rule(self) -> None:
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
fx = Fixture(Path(tmp))
|
|
text = fx.setup("filsok")[0].read_text(encoding="utf-8")
|
|
self.assertIn(f"cd {fx.run / 'arms' / 'filsok'} && ", text)
|
|
self.assertIn("grep", text)
|
|
|
|
def test_two_transcripts_for_one_task_keep_the_latest_and_report_the_rerun(self) -> None:
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
run, folder = Path(tmp) / "run", Path(tmp) / "subagents"
|
|
folder.mkdir()
|
|
task = run / "tasks" / "filsok" / "md-d01.md"
|
|
first = {"type": "user", "timestamp": "2026-09-27T10:00:00Z", "message": {"content": f"Read {task}"}}
|
|
later = {"type": "user", "timestamp": "2026-09-27T11:00:00Z", "message": {"content": f"Read {task}"}}
|
|
self.write(folder, "agent-old.jsonl", [first, assistant("o", (1, 1, 1, 1))])
|
|
self.write(folder, "agent-new.jsonl", [later, assistant("n", (2, 2, 2, 2))])
|
|
rows = dp.harvest(folder, run)
|
|
self.assertEqual(rows["filsok/md-d01"]["transcript"], "agent-new.jsonl")
|
|
self.assertEqual(rows["filsok/md-d01"]["tokens"], 8)
|
|
self.assertEqual(rows["filsok/md-d01"]["runs"], 2)
|
|
|
|
|
|
def results(arm: str, correct: dict[str, bool], kind: str) -> dict[str, Any]:
|
|
rows = []
|
|
for qid, ok in correct.items():
|
|
row: dict[str, Any] = {"id": qid, "answered": True, "answer_ok": ok, "source_ok": True, "correct": ok}
|
|
if kind == "wd":
|
|
row["type"] = "whole" if qid in ("d01", "d02") else "count"
|
|
rows.append(row)
|
|
return {"arm": arm, "set_sha256": "x", "results": rows}
|
|
|
|
|
|
class ScoreTest(unittest.TestCase):
|
|
def test_groups_are_counted_from_the_gate_results_and_pairs_compared(self) -> None:
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
run = Path(tmp)
|
|
res = run / "results"
|
|
res.mkdir()
|
|
md = {"filsok": {"d01": True, "d02": False, "d99": True}, "wiki": {"d01": True, "d02": True, "d99": False}}
|
|
wd = {"filsok": {"d01": True, "d02": False, "d03": True}, "wiki": {"d01": False, "d02": True, "d03": True}}
|
|
for arm in ("filsok", "wiki"):
|
|
(res / f"{arm}.md.json").write_text(json.dumps(results(arm, md[arm], "md")))
|
|
(res / f"{arm}.wd.json").write_text(json.dumps(results(arm, wd[arm], "wd")))
|
|
(run / "tokens.json").write_text(json.dumps({
|
|
"filsok/md-d01": {"tokens": 100, "breaches": [], "transcript": "a", "runs": 1},
|
|
"wiki/md-d01": {"tokens": 300, "breaches": [], "transcript": "b", "runs": 1}}))
|
|
table = dp.score(run, ["filsok", "wiki"], md_ids=["d01", "d02"])
|
|
self.assertEqual(table["arms"]["filsok"]["groups"]["flerdokument"], [1, 2])
|
|
self.assertEqual(table["arms"]["wiki"]["groups"]["flerdokument"], [2, 2])
|
|
self.assertEqual(table["arms"]["wiki"]["groups"]["helt dokument"], [1, 2])
|
|
self.assertEqual(table["arms"]["filsok"]["groups"]["telling"], [1, 1])
|
|
self.assertEqual(table["arms"]["wiki"]["total"], [4, 5])
|
|
self.assertEqual(table["arms"]["wiki"]["tokens"], {"sum": 300, "questions": 1})
|
|
pair = table["pairs"]["wiki vs filsok"]["flerdokument"]
|
|
self.assertEqual(pair, {"both": 1, "only wiki": 1, "only filsok": 0, "neither": 0})
|
|
self.assertEqual(table["pairs"]["wiki vs filsok"]["helt dokument"]["only filsok"], 1)
|
|
self.assertIn("| wiki |", dp.render_table(table))
|
|
|
|
def test_exact_mcnemar_is_the_two_sided_binomial_over_the_discordant_pairs(self) -> None:
|
|
self.assertEqual(dp.mcnemar_exact(0, 0), 1.0)
|
|
self.assertEqual(dp.mcnemar_exact(2, 0), 0.5)
|
|
self.assertAlmostEqual(dp.mcnemar_exact(6, 0), 0.03125)
|
|
self.assertAlmostEqual(dp.mcnemar_exact(1, 5), 0.21875)
|
|
self.assertEqual(dp.mcnemar_exact(3, 3), 1.0)
|
|
|
|
def test_every_pair_carries_its_mcnemar_p_over_all_groups(self) -> None:
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
run = Path(tmp)
|
|
res = run / "results"
|
|
res.mkdir()
|
|
md = {"filsok": {"d01": False, "d02": False}, "wiki": {"d01": True, "d02": True}}
|
|
for arm in ("filsok", "wiki"):
|
|
(res / f"{arm}.md.json").write_text(json.dumps(results(arm, md[arm], "md")))
|
|
(res / f"{arm}.wd.json").write_text(json.dumps(results(arm, {}, "wd")))
|
|
table = dp.score(run, ["filsok", "wiki"], md_ids=["d01", "d02"])
|
|
self.assertEqual(table["mcnemar"]["wiki vs filsok"], {"only wiki": 2, "only filsok": 0, "p": 0.5})
|
|
|
|
def test_a_missing_result_file_is_a_failure_not_a_zero(self) -> None:
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
run = Path(tmp)
|
|
(run / "results").mkdir()
|
|
with self.assertRaises(dp.ProbeError):
|
|
dp.score(run, ["filsok"], md_ids=["d01"])
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|