okf-wiki/evals/tests/test_direction_probe.py
Kjell Tore Guttormsen 0fd86a23a3
okf-wiki 1.1.1
First public release, published as one snapshot. What changed between
versions is in CHANGELOG.md.

Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
2026-11-14 21:03:20 +01:00

355 lines
19 KiB
Python

"""Tests for evals/direction_probe.py: the direction probe R (PLAN.md § Leveranser, row R).
Three arms answer the same questions from the same sources: file search alone, the okf-wiki plus file search, and a
wiki built with Astro-Han's karpathy-llm-wiki skill. The module sets up each arm's basis and task files, collects the
answers into the gates' format, harvests tokens and isolation breaches from the subagent transcripts, and prints the
table. It never calls a model. The fixtures are synthetic text about gardening (CLAUDE.md rule 3).
Run: python3 -m unittest discover -s evals/tests
"""
from __future__ import annotations
import json
import sys
import tempfile
import unittest
from pathlib import Path
from typing import Any
ROOT = Path(__file__).resolve().parents[2]
sys.path.insert(0, str(ROOT / "evals"))
import direction_probe as dp # noqa: E402
TOMATO = "# Tomatoes\nTomatoes need full sun.\nWater them every morning.\n"
BASIL = "# Basil\nBasil likes warm soil.\n"
MINT = "# Mint\nMint spreads through runners.\n"
NOT_IN_CUT = "# Cactus\nCacti need little water.\n"
MD_SET: dict[str, Any] = {
"schema": "okf-wiki/multidoc-set/1",
"questions": [
{"id": "d01", "question": "What do tomatoes and basil need?",
"answer_keys": [{"file": "tomato.md", "text": "SECRETKEYSUN"}],
"evidence": [{"file": "tomato.md", "span": "SECRETSPANTOMATO"}]},
{"id": "d02", "question": "A question outside the probe.",
"answer_keys": [{"file": "mint.md", "text": "OTHERKEY"}],
"evidence": [{"file": "mint.md", "span": "OTHERSPAN"}]},
],
}
WD_SET: dict[str, Any] = {
"schema": "okf-wiki/wholedoc-set/1",
"questions": [
{"id": "d01", "type": "whole", "file": "tomato.md", "question": "What does the tomato page say overall?",
"answer_keys": [{"text": "SECRETWHOLEKEY"}], "evidence": [{"span": "SECRETWHOLESPAN"}]},
{"id": "d02", "type": "count", "file": "basil.md", "question": "How many basil lines are there?",
"count": 424242, "item_pattern": "SECRETPATTERN", "evidence": [{"span": "SECRETCOUNTSPAN"}]},
],
}
SECRETS = ("SECRETKEYSUN", "SECRETSPANTOMATO", "SECRETWHOLEKEY", "SECRETWHOLESPAN", "424242", "SECRETPATTERN",
"SECRETCOUNTSPAN", "OTHERKEY", "A question outside the probe")
class Fixture:
def __init__(self, root: Path) -> None:
self.root = root
self.sources = root / "kilde"
self.sources.mkdir()
for name, text in (("tomato.md", TOMATO), ("basil.md", BASIL), ("mint.md", MINT), ("cactus.md", NOT_IN_CUT)):
(self.sources / name).write_text(text, encoding="utf-8")
self.names = ["basil.md", "mint.md", "tomato.md"]
self.md_set = root / "md.json"
self.md_set.write_text(json.dumps(MD_SET), encoding="utf-8")
self.wd_set = root / "wd.json"
self.wd_set.write_text(json.dumps(WD_SET), encoding="utf-8")
self.wiki = root / "wiki-src"
self.wiki.mkdir()
(self.wiki / "index.md").write_text("# Index\n* [Tomatoes](tomatoes.md)\n", encoding="utf-8")
(self.wiki / "tomatoes.md").write_text("Tomatoes need full sun.\n", encoding="utf-8")
self.run = root / "run"
def setup(self, arm: str, **extra: Path) -> list[Path]:
return dp.setup_arm(run=self.run, arm=arm, sources=self.sources, names=self.names, md_set=self.md_set,
md_ids=["d01"], wd_set=self.wd_set, **extra)
class FetchAstroTest(unittest.TestCase):
def test_raw_file_keeps_the_name_and_the_body_byte_exact_under_the_template_header(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
fx = Fixture(Path(tmp))
project = Path(tmp) / "astro"
written = dp.fetch_astro(sources=fx.sources, names=fx.names, project=project, topic="garden",
collected="2026-09-27")
self.assertEqual([p.name for p in written], fx.names)
raw = (project / "raw" / "garden" / "tomato.md").read_text(encoding="utf-8")
self.assertTrue(raw.startswith("# tomato\n\n> Source: kilde/tomato.md\n> Collected: 2026-09-27\n"
"> Published: Unknown\n\n"))
self.assertTrue(raw.endswith(TOMATO))
self.assertFalse((project / "raw" / "garden" / "cactus.md").exists())
def test_an_existing_raw_file_is_never_overwritten(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
fx = Fixture(Path(tmp))
project = Path(tmp) / "astro"
dp.fetch_astro(sources=fx.sources, names=fx.names, project=project, topic="garden", collected="2026-09-27")
with self.assertRaises(dp.ProbeError):
dp.fetch_astro(sources=fx.sources, names=fx.names, project=project, topic="garden",
collected="2026-09-27")
class SetupTest(unittest.TestCase):
def test_file_search_basis_holds_exactly_the_cut(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
fx = Fixture(Path(tmp))
fx.setup("filsok")
basis = fx.run / "arms" / "filsok"
self.assertEqual(sorted(p.name for p in basis.iterdir()), fx.names)
self.assertEqual((basis / "tomato.md").read_text(encoding="utf-8"), TOMATO)
def test_one_task_per_probe_question_and_none_for_the_rest(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
fx = Fixture(Path(tmp))
tasks = fx.setup("filsok")
self.assertEqual(sorted(p.name for p in tasks), ["md-d01.md", "wd-d01.md", "wd-d02.md"])
self.assertTrue(all(p.parent == fx.run / "tasks" / "filsok" for p in tasks))
def test_a_task_carries_the_question_the_basis_and_the_answer_path_but_nothing_from_the_key(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
fx = Fixture(Path(tmp))
tasks = {p.name: p.read_text(encoding="utf-8") for p in fx.setup("filsok")}
md = tasks["md-d01.md"]
self.assertIn("What do tomatoes and basil need?", md)
self.assertIn(str(fx.run / "arms" / "filsok"), md)
self.assertIn(str(fx.run / "answers" / "filsok" / "md-d01.json"), md)
self.assertIn('"count"', tasks["wd-d02.md"])
self.assertNotIn('"count"', md)
for text in tasks.values():
for secret in SECRETS:
self.assertNotIn(secret, text)
def test_the_wiki_arm_gets_the_wiki_and_the_cut_and_the_router_order(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
fx = Fixture(Path(tmp))
tasks = fx.setup("wiki", wiki=fx.wiki)
basis = fx.run / "arms" / "wiki"
self.assertEqual(sorted(p.name for p in (basis / "kilde").iterdir()), fx.names)
self.assertTrue((basis / "wiki" / "index.md").is_file())
text = tasks[0].read_text(encoding="utf-8")
self.assertLess(text.index("index.md"), text.index("x-locator"))
def test_the_astro_arm_gets_raw_and_wiki_from_the_built_project(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
fx = Fixture(Path(tmp))
project = Path(tmp) / "astro"
dp.fetch_astro(sources=fx.sources, names=fx.names, project=project, topic="garden", collected="2026-09-27")
(project / "wiki").mkdir()
(project / "wiki" / "index.md").write_text("# Knowledge Base Index\n", encoding="utf-8")
tasks = fx.setup("astro", astro=project)
basis = fx.run / "arms" / "astro"
self.assertTrue((basis / "raw" / "garden" / "basil.md").is_file())
self.assertTrue((basis / "wiki" / "index.md").is_file())
self.assertIn("Raw", tasks[0].read_text(encoding="utf-8"))
def test_an_arm_without_its_basis_or_an_existing_arm_is_refused(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
fx = Fixture(Path(tmp))
with self.assertRaises(dp.ProbeError):
fx.setup("wiki")
fx.setup("filsok")
with self.assertRaises(dp.ProbeError):
fx.setup("filsok")
with self.assertRaises(dp.ProbeError):
fx.setup("grep-only")
def test_a_probe_id_missing_from_the_set_is_refused(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
fx = Fixture(Path(tmp))
with self.assertRaises(dp.ProbeError):
dp.setup_arm(run=fx.run, arm="filsok", sources=fx.sources, names=fx.names, md_set=fx.md_set,
md_ids=["d09"], wd_set=fx.wd_set)
class CollectTest(unittest.TestCase):
def test_answers_are_gathered_per_gate_with_ids_from_the_file_names(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
run = Path(tmp)
out = run / "answers" / "wiki"
out.mkdir(parents=True)
(out / "md-d01.json").write_text(json.dumps({"id": "wrong", "answer": "a", "citations": []}))
(out / "wd-d02.json").write_text(json.dumps({"answer": "7", "count": 7, "citations": []}))
(out / "wd-d03.json").write_text("not json")
md_path, wd_path, problems = dp.collect(run, "wiki")
md = json.loads(md_path.read_text(encoding="utf-8"))
wd = json.loads(wd_path.read_text(encoding="utf-8"))
self.assertEqual(md, {"arm": "wiki", "answers": [{"id": "d01", "answer": "a", "citations": []}]})
self.assertEqual(wd["answers"], [{"id": "d02", "answer": "7", "count": 7, "citations": []}])
self.assertEqual(problems, ["wd-d03.json: not a JSON object"])
def transcript(lines: list[dict[str, Any]]) -> str:
return "".join(json.dumps(line) + "\n" for line in lines)
def assistant(msg_id: str, tokens: tuple[int, int, int, int], *tools: tuple[str, dict[str, Any]]) -> dict[str, Any]:
i, cc, cr, o = tokens
content = [{"type": "tool_use", "name": name, "input": inp} for name, inp in tools]
return {"type": "assistant", "message": {"id": msg_id, "content": content, "usage": {
"input_tokens": i, "cache_creation_input_tokens": cc, "cache_read_input_tokens": cr, "output_tokens": o}}}
class HarvestTest(unittest.TestCase):
def write(self, folder: Path, name: str, lines: list[dict[str, Any]]) -> None:
(folder / name).write_text(transcript(lines), encoding="utf-8")
def test_tokens_count_each_message_once_and_breaches_are_named(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
run, folder = Path(tmp) / "run", Path(tmp) / "subagents"
folder.mkdir()
task = run / "tasks" / "wiki" / "md-d01.md"
basis = run / "arms" / "wiki"
answer = run / "answers" / "wiki" / "md-d01.json"
first = {"type": "user", "message": {"content": f"Read {task} and follow it."}}
self.write(folder, "agent-a.jsonl", [
first,
assistant("m1", (2, 100, 1000, 5), ("Read", {"file_path": str(task)})),
assistant("m1", (2, 100, 1000, 5)),
assistant("m2", (1, 10, 2000, 50), ("Grep", {"pattern": "sun", "path": str(basis / "kilde")}),
("Glob", {"pattern": "*.md", "path": str(basis / "wiki")})),
assistant("m3", (1, 0, 3000, 80), ("Write", {"file_path": str(answer), "content": "{}"}),
("SubagentHandback", {"message": "done"})),
])
self.write(folder, "agent-b.jsonl", [
{"type": "user", "message": {"content": [{"type": "text", "text": f"Read {run}/tasks/filsok/wd-d02.md"}]}},
assistant("n1", (1, 1, 1, 1), ("Read", {"file_path": str(run / "arms" / "wiki" / "wiki" / "x.md")}),
("Grep", {"pattern": "sun"}), ("Bash", {"command": "ls"}),
("Read", {"file_path": str(run / "arms" / "filsok" / ".." / "wiki" / "y.md")})),
])
self.write(folder, "agent-c.jsonl", [{"type": "user", "message": {"content": "Something else."}}])
rows = dp.harvest(folder, run)
self.assertEqual(sorted(rows), ["filsok/wd-d02", "wiki/md-d01"])
wiki = rows["wiki/md-d01"]
self.assertEqual(wiki["tokens"], 1107 + 2061 + 3081)
self.assertEqual(wiki["breaches"], [])
self.assertEqual(wiki["transcript"], "agent-a.jsonl")
breaches = rows["filsok/wd-d02"]["breaches"]
self.assertEqual(len(breaches), 4)
self.assertTrue(any(b.startswith("Bash") for b in breaches))
self.assertTrue(any(b.startswith("Grep without a path") for b in breaches))
def test_bash_is_read_only_search_that_starts_in_the_basis(self) -> None:
basis = Path("/runs/r/arms/filsok")
task, answer = Path("/runs/r/tasks/filsok/md-d01.md"), Path("/runs/r/answers/filsok/md-d01.json")
allowed = [
f"cd {basis} && grep -rn 'full sun' .",
f"cd {basis} && grep -n sun tomato.md | head -5",
f"cd {basis} && find . -name '*.md' | sort && wc -l tomato.md 2>/dev/null",
f"cd '{basis}' && sed -n '1,40p' tomato.md",
f"cd {basis}/kilde && ls",
]
refused = [
"grep -rn sun /runs/r/arms/filsok",
f"cd {basis} && cat ../wiki/x.md",
f"cd {basis} && grep sun /runs/r/tasks/filsok/other.md",
f"cd {basis} && rm tomato.md",
f"cd {basis} && grep sun tomato.md > copy.txt",
f"cd {basis} && find . -name '*.md' -exec cat {{}} ;",
f"cd {basis} && sed -i 's/a/b/' tomato.md",
f"cd {basis} && cat $(echo /etc/hosts)",
f"cd {basis} && cd / && ls",
f"cd {basis} && ls ~",
"cd /runs/r/arms/wiki && ls",
]
for command in allowed:
self.assertEqual(dp.breaches_of([("Bash", {"command": command})], basis, task, answer), [], command)
for command in refused:
self.assertEqual(len(dp.breaches_of([("Bash", {"command": command})], basis, task, answer)), 1, command)
def test_the_task_names_bash_as_the_search_tool_and_its_rule(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
fx = Fixture(Path(tmp))
text = fx.setup("filsok")[0].read_text(encoding="utf-8")
self.assertIn(f"cd {fx.run / 'arms' / 'filsok'} && ", text)
self.assertIn("grep", text)
def test_two_transcripts_for_one_task_keep_the_latest_and_report_the_rerun(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
run, folder = Path(tmp) / "run", Path(tmp) / "subagents"
folder.mkdir()
task = run / "tasks" / "filsok" / "md-d01.md"
first = {"type": "user", "timestamp": "2026-09-27T10:00:00Z", "message": {"content": f"Read {task}"}}
later = {"type": "user", "timestamp": "2026-09-27T11:00:00Z", "message": {"content": f"Read {task}"}}
self.write(folder, "agent-old.jsonl", [first, assistant("o", (1, 1, 1, 1))])
self.write(folder, "agent-new.jsonl", [later, assistant("n", (2, 2, 2, 2))])
rows = dp.harvest(folder, run)
self.assertEqual(rows["filsok/md-d01"]["transcript"], "agent-new.jsonl")
self.assertEqual(rows["filsok/md-d01"]["tokens"], 8)
self.assertEqual(rows["filsok/md-d01"]["runs"], 2)
def results(arm: str, correct: dict[str, bool], kind: str) -> dict[str, Any]:
rows = []
for qid, ok in correct.items():
row: dict[str, Any] = {"id": qid, "answered": True, "answer_ok": ok, "source_ok": True, "correct": ok}
if kind == "wd":
row["type"] = "whole" if qid in ("d01", "d02") else "count"
rows.append(row)
return {"arm": arm, "set_sha256": "x", "results": rows}
class ScoreTest(unittest.TestCase):
def test_groups_are_counted_from_the_gate_results_and_pairs_compared(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
run = Path(tmp)
res = run / "results"
res.mkdir()
md = {"filsok": {"d01": True, "d02": False, "d99": True}, "wiki": {"d01": True, "d02": True, "d99": False}}
wd = {"filsok": {"d01": True, "d02": False, "d03": True}, "wiki": {"d01": False, "d02": True, "d03": True}}
for arm in ("filsok", "wiki"):
(res / f"{arm}.md.json").write_text(json.dumps(results(arm, md[arm], "md")))
(res / f"{arm}.wd.json").write_text(json.dumps(results(arm, wd[arm], "wd")))
(run / "tokens.json").write_text(json.dumps({
"filsok/md-d01": {"tokens": 100, "breaches": [], "transcript": "a", "runs": 1},
"wiki/md-d01": {"tokens": 300, "breaches": [], "transcript": "b", "runs": 1}}))
table = dp.score(run, ["filsok", "wiki"], md_ids=["d01", "d02"])
self.assertEqual(table["arms"]["filsok"]["groups"]["flerdokument"], [1, 2])
self.assertEqual(table["arms"]["wiki"]["groups"]["flerdokument"], [2, 2])
self.assertEqual(table["arms"]["wiki"]["groups"]["helt dokument"], [1, 2])
self.assertEqual(table["arms"]["filsok"]["groups"]["telling"], [1, 1])
self.assertEqual(table["arms"]["wiki"]["total"], [4, 5])
self.assertEqual(table["arms"]["wiki"]["tokens"], {"sum": 300, "questions": 1})
pair = table["pairs"]["wiki vs filsok"]["flerdokument"]
self.assertEqual(pair, {"both": 1, "only wiki": 1, "only filsok": 0, "neither": 0})
self.assertEqual(table["pairs"]["wiki vs filsok"]["helt dokument"]["only filsok"], 1)
self.assertIn("| wiki |", dp.render_table(table))
def test_exact_mcnemar_is_the_two_sided_binomial_over_the_discordant_pairs(self) -> None:
self.assertEqual(dp.mcnemar_exact(0, 0), 1.0)
self.assertEqual(dp.mcnemar_exact(2, 0), 0.5)
self.assertAlmostEqual(dp.mcnemar_exact(6, 0), 0.03125)
self.assertAlmostEqual(dp.mcnemar_exact(1, 5), 0.21875)
self.assertEqual(dp.mcnemar_exact(3, 3), 1.0)
def test_every_pair_carries_its_mcnemar_p_over_all_groups(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
run = Path(tmp)
res = run / "results"
res.mkdir()
md = {"filsok": {"d01": False, "d02": False}, "wiki": {"d01": True, "d02": True}}
for arm in ("filsok", "wiki"):
(res / f"{arm}.md.json").write_text(json.dumps(results(arm, md[arm], "md")))
(res / f"{arm}.wd.json").write_text(json.dumps(results(arm, {}, "wd")))
table = dp.score(run, ["filsok", "wiki"], md_ids=["d01", "d02"])
self.assertEqual(table["mcnemar"]["wiki vs filsok"], {"only wiki": 2, "only filsok": 0, "p": 0.5})
def test_a_missing_result_file_is_a_failure_not_a_zero(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
run = Path(tmp)
(run / "results").mkdir()
with self.assertRaises(dp.ProbeError):
dp.score(run, ["filsok"], md_ids=["d01"])
if __name__ == "__main__":
unittest.main()