First public release, published as one snapshot. What changed between versions is in CHANGELOG.md. Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
752 lines
39 KiB
Python
752 lines
39 KiB
Python
"""Tests for gates/answers.py: the answer harness of contract K5 (setup, collect, harvest, score).
|
|
|
|
Synthetic K1 roots about bread and gardens, synthetic transcripts and synthetic result files. No corpus text is used
|
|
(CLAUDE.md rule 3), and nothing here starts a model.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import contextlib
|
|
import io
|
|
import json
|
|
import sys
|
|
import tempfile
|
|
import unittest
|
|
from pathlib import Path
|
|
from typing import Any, Callable
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
|
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
|
|
|
import answers # noqa: E402
|
|
from test_answerset import ( # noqa: E402
|
|
KEY, OVEN, RISING, SOIL, SPAN, TEXTS, WATER, dump, make_set, q4, q23, root_ref, sha, write_root,
|
|
)
|
|
|
|
# Newcombe (1998), Statist. Med. 17:2635-2650, Table III, method 10: (e, f, g, h) -> 95 % interval for
|
|
# (e + f)/n - (e + g)/n, as printed (four decimals).
|
|
NEWCOMBE_TABLE_III = [
|
|
((36, 12, 2, 0), (0.0569, 0.3404)),
|
|
((20, 12, 2, 16), (0.0562, 0.3292)),
|
|
((18, 12, 2, 18), (0.0562, 0.3290)),
|
|
((35, 14, 0, 1), (0.1461, 0.4175)),
|
|
((18, 14, 0, 18), (0.1441, 0.3963)),
|
|
((1, 97, 1, 1), (0.8736, 0.9850)),
|
|
((0, 30, 0, 0), (0.8395, 1.0)),
|
|
((1, 98, 0, 1), (0.9171, 0.9916)),
|
|
((54, 0, 0, 0), (-0.0664, 0.0664)),
|
|
((53, 0, 0, 1), (-0.0729, 0.0729)),
|
|
((27, 0, 0, 27), (-0.0351, 0.0351)),
|
|
]
|
|
|
|
|
|
def run(argv: list[str]) -> tuple[int, str]:
|
|
out, err = io.StringIO(), io.StringIO()
|
|
with contextlib.redirect_stdout(out), contextlib.redirect_stderr(err):
|
|
try:
|
|
code = answers.main(argv)
|
|
except SystemExit as exc: # argparse refuses an option it does not know
|
|
code = int(exc.code or 0)
|
|
return code, out.getvalue() + err.getvalue()
|
|
|
|
|
|
class Base(unittest.TestCase):
|
|
def setUp(self) -> None:
|
|
self._tmp = tempfile.TemporaryDirectory()
|
|
self.tmp = Path(self._tmp.name).resolve()
|
|
self.root = write_root(self.tmp / "copy")
|
|
self.set = self.tmp / "set.json"
|
|
self.set.write_bytes(dump(make_set(self.root, [q23("d01"), q4("d02")])))
|
|
self.run_dir = self.tmp / "run"
|
|
|
|
def tearDown(self) -> None:
|
|
self._tmp.cleanup()
|
|
|
|
def setup_arm(self, arm: str = "filsok", extra: list[str] | None = None) -> tuple[int, str]:
|
|
return run(["setup", "--run", str(self.run_dir), "--name", "r1", "--arm", arm, "--set", str(self.set),
|
|
"--sources", str(self.root)] + (extra or []))
|
|
|
|
|
|
class SetupTest(Base):
|
|
def test_setup_copies_the_sources_as_the_arm_basis(self) -> None:
|
|
code, out = self.setup_arm()
|
|
self.assertEqual(code, 0, out)
|
|
basis = self.run_dir / "arms" / "filsok"
|
|
for path in TEXTS:
|
|
self.assertEqual((basis / path).read_bytes(), (self.root / path).read_bytes())
|
|
self.assertEqual(sorted(p.relative_to(basis).as_posix() for p in basis.rglob("*") if p.is_file()),
|
|
sorted(TEXTS))
|
|
|
|
def test_one_task_per_question_without_keys_spans_or_counts(self) -> None:
|
|
code, out = self.setup_arm()
|
|
self.assertEqual(code, 0, out)
|
|
basis = self.run_dir / "arms" / "filsok"
|
|
for qid in ("d01", "d02"):
|
|
task = self.run_dir / "tasks" / "filsok" / f"{qid}.md"
|
|
text = task.read_text(encoding="utf-8")
|
|
self.assertIn("How do the synthetic notes about bread and gardens fit together?", text)
|
|
self.assertIn(str(basis), text)
|
|
self.assertIn(str(self.run_dir / "answers" / "filsok" / f"{qid}.json"), text)
|
|
for value in list(SPAN.values()) + list(KEY.values()):
|
|
self.assertNotIn(value, text)
|
|
for word in ("answer_keys", "evidence", "docs-2-3", "docs-4", "history", "four documents"):
|
|
self.assertNotIn(word, text)
|
|
self.assertIn(f"S1-r1-filsok-{qid}\t{task}", out)
|
|
self.assertTrue((self.run_dir / "answers" / "filsok").is_dir())
|
|
|
|
def test_the_task_asks_for_k1_paths_in_the_citations(self) -> None:
|
|
self.setup_arm()
|
|
text = (self.run_dir / "tasks" / "filsok" / "d01.md").read_text(encoding="utf-8")
|
|
self.assertIn('"path": "kilde/', text)
|
|
self.assertIn('"quote"', text)
|
|
|
|
def test_an_arm_is_set_up_once(self) -> None:
|
|
self.assertEqual(self.setup_arm()[0], 0)
|
|
code, out = self.setup_arm()
|
|
self.assertEqual(code, 1)
|
|
self.assertIn("exists", out)
|
|
|
|
def test_the_bundle_arm_needs_a_bundle_and_gets_the_sources_beside_it(self) -> None:
|
|
self.assertEqual(self.setup_arm("bundle")[0], 1)
|
|
bundle = self.tmp / "bundle"
|
|
bundle.mkdir()
|
|
(bundle / "index.md").write_text("# Index\n", encoding="utf-8")
|
|
code, out = self.setup_arm("bundle", ["--bundle", str(bundle)])
|
|
self.assertEqual(code, 0, out)
|
|
basis = self.run_dir / "arms" / "bundle"
|
|
self.assertEqual((basis / "wiki" / "index.md").read_text(encoding="utf-8"), "# Index\n")
|
|
self.assertEqual((basis / "kilde" / "docs-bread-oven.md").read_bytes(),
|
|
(self.root / "kilde" / "docs-bread-oven.md").read_bytes())
|
|
text = (self.run_dir / "tasks" / "bundle" / "d01.md").read_text(encoding="utf-8")
|
|
self.assertIn("wiki/index.md", text)
|
|
self.assertNotIn("Read `BASIS/wiki/index.md` first", text)
|
|
search, main, links = (text.index("Search `BASIS/wiki/index.md`"), text.index("Open the main page"),
|
|
text.index("follow its links"))
|
|
self.assertLess(search, main)
|
|
self.assertLess(main, links)
|
|
self.assertIn("never read whole", text)
|
|
|
|
def test_only_the_fixed_arms(self) -> None:
|
|
code, out = self.setup_arm("server")
|
|
self.assertEqual(code, 1)
|
|
self.assertIn("filsok", out)
|
|
|
|
def test_the_root_must_be_the_one_the_set_names(self) -> None:
|
|
other = write_root(self.tmp / "other", frozen_at="2026-10-09T12:00:00Z")
|
|
code, out = run(["setup", "--run", str(self.run_dir), "--name", "r1", "--arm", "filsok",
|
|
"--set", str(self.set), "--sources", str(other)])
|
|
self.assertEqual(code, 1)
|
|
self.assertIn("manifest_sha256", out)
|
|
|
|
def test_a_run_name_with_a_dash_or_space_is_refused(self) -> None:
|
|
code, _ = run(["setup", "--run", str(self.run_dir), "--name", "r 1", "--arm", "filsok",
|
|
"--set", str(self.set), "--sources", str(self.root)])
|
|
self.assertEqual(code, 1)
|
|
|
|
|
|
class PartRootTest(Base):
|
|
"""Order 31 A3: an arm searches the whole copy for a set made from a part of it (`--part-root`)."""
|
|
|
|
PART = (RISING, OVEN, SOIL, WATER)
|
|
|
|
def make_part(self, copy: Path, change: Callable[[dict[str, Any]], None] | None = None) -> Path:
|
|
part = write_root(self.tmp / "part", self.PART, subset_of=root_ref(copy))
|
|
if change is not None:
|
|
manifest = part / "source-manifest.json"
|
|
data = json.loads(manifest.read_bytes())
|
|
change(data)
|
|
manifest.write_bytes(dump(data))
|
|
self.set.write_bytes(dump(make_set(part, [q23("d01"), q4("d02")])))
|
|
return part
|
|
|
|
def setup_part(self, part: Path | None, sources: Path | None = None) -> tuple[int, str]:
|
|
extra = ["--part-root", str(part)] if part is not None else []
|
|
return run(["setup", "--run", str(self.run_dir), "--name", "r1", "--arm", "filsok", "--set", str(self.set),
|
|
"--sources", str(sources or self.root)] + extra)
|
|
|
|
def test_a_set_of_the_part_is_set_up_on_the_whole_copy(self) -> None:
|
|
code, out = self.setup_part(self.make_part(self.root))
|
|
self.assertEqual(code, 0, out)
|
|
basis = self.run_dir / "arms" / "filsok"
|
|
self.assertEqual(sorted(p.relative_to(basis).as_posix() for p in basis.rglob("*") if p.is_file()),
|
|
sorted(TEXTS))
|
|
|
|
def test_a_part_that_is_not_a_subset_of_the_root_is_refused(self) -> None:
|
|
def other(data: dict[str, Any]) -> None:
|
|
data["subset_of"]["manifest_sha256"] = "0" * 64
|
|
code, out = self.setup_part(self.make_part(self.root, other))
|
|
self.assertEqual(code, 1)
|
|
self.assertIn("subset_of", out)
|
|
self.assertFalse((self.run_dir / "arms").exists())
|
|
|
|
def test_a_part_source_with_another_sha256_than_the_root_is_refused(self) -> None:
|
|
def changed(data: dict[str, Any]) -> None:
|
|
data["sources"][0]["sha256"] = "0" * 64
|
|
code, out = self.setup_part(self.make_part(self.root, changed))
|
|
self.assertEqual(code, 1)
|
|
self.assertIn("sha256", out)
|
|
self.assertFalse((self.run_dir / "arms").exists())
|
|
|
|
def test_a_part_source_missing_in_the_root_is_refused(self) -> None:
|
|
smaller = write_root(self.tmp / "smaller", (RISING, OVEN, SOIL))
|
|
code, out = self.setup_part(self.make_part(smaller), smaller)
|
|
self.assertEqual(code, 1)
|
|
self.assertIn(WATER, out)
|
|
self.assertFalse((self.run_dir / "arms").exists())
|
|
|
|
def test_a_part_root_that_is_not_the_sets_root_is_refused(self) -> None:
|
|
part = self.make_part(self.root)
|
|
self.set.write_bytes(dump(make_set(self.root, [q23("d01"), q4("d02")])))
|
|
code, out = self.setup_part(part)
|
|
self.assertEqual(code, 1)
|
|
self.assertIn("--part-root", out)
|
|
|
|
def test_without_part_root_a_set_of_the_part_is_still_refused_on_the_copy(self) -> None:
|
|
self.make_part(self.root)
|
|
code, out = self.setup_part(None)
|
|
self.assertEqual(code, 1)
|
|
self.assertIn("manifest_sha256", out)
|
|
|
|
|
|
class CollectTest(Base):
|
|
def test_collect_writes_the_k5_answers_file_with_the_id_from_the_file_name(self) -> None:
|
|
self.setup_arm()
|
|
folder = self.run_dir / "answers" / "filsok"
|
|
(folder / "d02.json").write_text(json.dumps({"id": "zz", "answer": "B", "citations": []}), encoding="utf-8")
|
|
(folder / "d01.json").write_text(json.dumps({"answer": "A", "citations": [{"path": "kilde/x.md",
|
|
"quote": "q"}]}),
|
|
encoding="utf-8")
|
|
code, out = run(["collect", "--run", str(self.run_dir), "--arm", "filsok"])
|
|
self.assertEqual(code, 0, out)
|
|
path = self.run_dir / "answers" / "filsok.json"
|
|
doc = json.loads(path.read_text(encoding="utf-8"))
|
|
self.assertEqual(doc["schema"], "okf-wiki/answers/1")
|
|
self.assertEqual(doc["arm"], "filsok")
|
|
self.assertEqual([a["id"] for a in doc["answers"]], ["d01", "d02"])
|
|
self.assertEqual(doc["answers"][0]["citations"], [{"path": "kilde/x.md", "quote": "q"}])
|
|
self.assertEqual(path.read_bytes(), dump(doc))
|
|
|
|
def test_an_answer_file_that_is_not_a_json_object_is_a_problem(self) -> None:
|
|
self.setup_arm()
|
|
folder = self.run_dir / "answers" / "filsok"
|
|
(folder / "d01.json").write_text("{not json", encoding="utf-8")
|
|
(folder / "d02.json").write_text(json.dumps({"answer": "B", "citations": []}), encoding="utf-8")
|
|
code, out = run(["collect", "--run", str(self.run_dir), "--arm", "filsok"])
|
|
self.assertEqual(code, 1)
|
|
self.assertIn("d01.json", out)
|
|
doc = json.loads((self.run_dir / "answers" / "filsok.json").read_text(encoding="utf-8"))
|
|
self.assertEqual([a["id"] for a in doc["answers"]], ["d02"])
|
|
|
|
|
|
def usage(inp: int, cc: int, cr: int, out: int) -> dict[str, int]:
|
|
return {"input_tokens": inp, "cache_creation_input_tokens": cc, "cache_read_input_tokens": cr,
|
|
"output_tokens": out}
|
|
|
|
|
|
def tool(name: str, **inp: Any) -> dict[str, Any]:
|
|
return {"type": "tool_use", "id": f"t-{name}-{len(inp)}", "name": name, "input": inp}
|
|
|
|
|
|
def assistant(mid: str, use: dict[str, int], blocks: list[dict[str, Any]], stop: str | None = "tool_use") -> dict[str, Any]:
|
|
message: dict[str, Any] = {"id": mid, "role": "assistant", "usage": use, "content": blocks}
|
|
if stop is not None:
|
|
message["stop_reason"] = stop
|
|
return {"type": "assistant", "message": message}
|
|
|
|
|
|
def user(text: str, stamp: str = "2026-10-09T10:00:00Z") -> dict[str, Any]:
|
|
return {"type": "user", "timestamp": stamp, "message": {"role": "user", "content": text}}
|
|
|
|
|
|
def write_transcript(path: Path, lines: list[dict[str, Any]], description: str | None = None) -> None:
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
path.write_text("".join(json.dumps(line) + "\n" for line in lines) + "not json\n", encoding="utf-8")
|
|
if description is not None:
|
|
meta = path.with_name(path.stem + ".meta.json")
|
|
meta.write_text(json.dumps({"agentType": "general-purpose", "description": description}), encoding="utf-8")
|
|
|
|
|
|
class HarvestTest(Base):
|
|
def setUp(self) -> None:
|
|
super().setUp()
|
|
self.assertEqual(self.setup_arm()[0], 0)
|
|
self.basis = self.run_dir / "arms" / "filsok"
|
|
self.transcripts = self.tmp / "transcripts"
|
|
self.subagents = self.transcripts / "session-1" / "subagents"
|
|
|
|
def task(self, qid: str) -> Path:
|
|
return self.run_dir / "tasks" / "filsok" / f"{qid}.md"
|
|
|
|
def answer(self, qid: str) -> Path:
|
|
return self.run_dir / "answers" / "filsok" / f"{qid}.json"
|
|
|
|
def good_lines(self, qid: str, stamp: str = "2026-10-09T10:00:00Z") -> list[dict[str, Any]]:
|
|
return [
|
|
user(f"Read {self.task(qid)} and do what it says.", stamp),
|
|
assistant("m1", usage(10, 100, 1000, 5), [tool("Read", file_path=str(self.task(qid)))]),
|
|
assistant("m1", usage(10, 100, 1000, 7), []),
|
|
assistant("m2", usage(3, 0, 1100, 20), [tool("Bash", command=f"cd {self.basis} && grep -rn dough .")]),
|
|
assistant("m3", usage(2, 0, 1200, 30), [tool("Write", file_path=str(self.answer(qid)), content="{}")],
|
|
"end_turn"),
|
|
]
|
|
|
|
def harvest(self) -> tuple[int, str, dict[str, Any]]:
|
|
code, out = run(["harvest", "--run", str(self.run_dir), "--name", "r1", "--arm", "filsok",
|
|
"--transcripts", str(self.transcripts)])
|
|
path = self.run_dir / "tokens" / "filsok.json"
|
|
return code, out, json.loads(path.read_text(encoding="utf-8")) if path.is_file() else {}
|
|
|
|
def breaches(self, calls: list[dict[str, Any]]) -> list[str]:
|
|
lines = self.good_lines("d01")
|
|
lines.insert(3, assistant("m9", usage(1, 0, 0, 1), calls))
|
|
write_transcript(self.subagents / "agent-a1.jsonl", lines, "S1-r1-filsok-d01")
|
|
return list(self.harvest()[2]["questions"][0]["breach_detail"])
|
|
|
|
def test_tokens_per_question_each_message_once_all_four_fields(self) -> None:
|
|
write_transcript(self.subagents / "agent-a1.jsonl", self.good_lines("d01"), "S1-r1-filsok-d01")
|
|
write_transcript(self.subagents / "agent-a2.jsonl", self.good_lines("d02"), "S1-r1-filsok-d02")
|
|
code, out, doc = self.harvest()
|
|
self.assertEqual(code, 0, out)
|
|
self.assertEqual(doc["schema"], "okf-wiki/answer-tokens/1")
|
|
self.assertEqual(doc["arm"], "filsok")
|
|
row = doc["questions"][0]
|
|
self.assertEqual(row["id"], "d01")
|
|
self.assertEqual((row["input_tokens"], row["cache_creation_input_tokens"], row["cache_read_input_tokens"],
|
|
row["output_tokens"]), (15, 100, 3300, 57))
|
|
self.assertEqual(row["isolation_breaches"], 0)
|
|
self.assertEqual([r["id"] for r in doc["questions"]], ["d01", "d02"])
|
|
path = self.run_dir / "tokens" / "filsok.json"
|
|
self.assertEqual(path.read_bytes(), dump(doc))
|
|
|
|
def test_isolation_breaches_are_counted(self) -> None:
|
|
lines = self.good_lines("d01")
|
|
lines.insert(3, assistant("m9", usage(1, 0, 0, 1), [
|
|
tool("Read", file_path="/etc/hosts"),
|
|
tool("Bash", command="cat /etc/passwd"),
|
|
tool("Grep", pattern="dough", path=str(self.run_dir / "arms" / "bundle")),
|
|
tool("Glob", pattern="*.md"),
|
|
tool("Read", file_path=str(self.basis / "kilde" / "docs-bread-oven.md")),
|
|
tool("Bash", command=f"cd {self.basis} && grep -rn crust ../.."),
|
|
tool("WebFetch", url="https://example.invalid/"),
|
|
tool("Write", file_path=str(self.basis / "notes.md"), content="x"),
|
|
]))
|
|
write_transcript(self.subagents / "agent-a1.jsonl", lines, "S1-r1-filsok-d01")
|
|
write_transcript(self.subagents / "agent-a2.jsonl", self.good_lines("d02"), "S1-r1-filsok-d02")
|
|
code, out, doc = self.harvest()
|
|
row = doc["questions"][0]
|
|
self.assertEqual(row["isolation_breaches"], 7, row)
|
|
self.assertEqual(len(row["breach_detail"]), 7)
|
|
self.assertIn("breaches: 7", out)
|
|
|
|
def test_echo_of_literal_words_in_a_chain_into_the_basis_is_not_a_breach(self) -> None:
|
|
lines = self.good_lines("d01")
|
|
lines.insert(3, assistant("m9", usage(1, 0, 0, 1), [
|
|
tool("Bash", command=f"cd {self.basis} && grep -rn dough kilde && echo --- && grep -rn crust kilde"),
|
|
tool("Bash", command=f"cd {self.basis} && grep -rn dough kilde | head -5; echo ----; ls kilde"),
|
|
tool("Bash", command=f'cd {self.basis} && echo "next file" && sed -n 1,5p kilde/docs-bread-oven.md'),
|
|
]))
|
|
write_transcript(self.subagents / "agent-a1.jsonl", lines, "S1-r1-filsok-d01")
|
|
code, out, doc = self.harvest()
|
|
self.assertEqual(doc["questions"][0]["isolation_breaches"], 0, doc["questions"][0]["breach_detail"])
|
|
self.assertNotIn("echo", self.task("d01").read_text(encoding="utf-8"))
|
|
|
|
def test_echo_outside_the_basis_or_with_an_expansion_is_still_a_breach(self) -> None:
|
|
lines = self.good_lines("d01")
|
|
lines.insert(3, assistant("m9", usage(1, 0, 0, 1), [
|
|
tool("Bash", command=f"cd {self.basis} && echo /outside/*"),
|
|
tool("Bash", command=f"cd {self.basis} && echo ~/*"),
|
|
tool("Bash", command=f"cd {self.basis} && echo ../../*"),
|
|
tool("Bash", command=f"cd {self.basis} && echo x > kilde/notes.md"),
|
|
tool("Bash", command=f"cd {self.basis} && echo $HOME"),
|
|
tool("Bash", command=f"cd {self.basis} && echo `id`"),
|
|
tool("Bash", command="echo --- && grep -rn dough kilde"),
|
|
]))
|
|
write_transcript(self.subagents / "agent-a1.jsonl", lines, "S1-r1-filsok-d01")
|
|
code, out, doc = self.harvest()
|
|
self.assertEqual(doc["questions"][0]["isolation_breaches"], 7, doc["questions"][0]["breach_detail"])
|
|
|
|
def test_stderr_sent_away_right_before_a_separator_is_not_a_breach(self) -> None:
|
|
b = self.basis
|
|
self.assertEqual(self.breaches([
|
|
tool("Bash", command=f"cd {b} && grep -rn dough kilde 2>/dev/null; ls kilde"),
|
|
tool("Bash", command=f"cd {b} && grep -rn dough kilde 2>/dev/null| head -3"),
|
|
tool("Bash", command=f"cd {b} && grep -rn dough kilde 2>/dev/null&& ls kilde"),
|
|
tool("Bash", command=f"cd {b} && grep -rn dough kilde 2>/dev/null|| ls kilde"),
|
|
]), [])
|
|
|
|
def test_other_redirections_before_a_separator_are_still_breaches(self) -> None:
|
|
b = self.basis
|
|
found = self.breaches([
|
|
tool("Bash", command=f"cd {b} && grep -rn dough kilde > kilde/out.md; ls kilde"),
|
|
tool("Bash", command=f"cd {b} && grep -rn dough kilde >/dev/null; ls kilde"),
|
|
tool("Bash", command=f"cd {b} && grep -rn dough kilde 2>&1; ls kilde"),
|
|
tool("Bash", command=f"cd {b} && grep -rn dough kilde 2>/outside/err; ls kilde"),
|
|
tool("Bash", command=f"cd {b} && grep -rn dough kilde 2>/dev/nullx; ls kilde"),
|
|
])
|
|
self.assertEqual(len(found), 5, found)
|
|
|
|
def test_a_search_pattern_or_sed_script_starting_with_a_slash_is_not_a_path(self) -> None:
|
|
b, oven = self.basis, "kilde/docs-bread-oven.md"
|
|
self.assertEqual(self.breaches([
|
|
tool("Bash", command=f"cd {b} && grep -rn /oven kilde"),
|
|
tool("Bash", command=f'cd {b} && grep -rlni "/oven" kilde | head -3'),
|
|
tool("Bash", command=f"cd {b} && egrep -n -e /oven -e /crust {oven}"),
|
|
tool("Bash", command=f"cd {b} && fgrep -A 3 -m 2 /oven {oven}"),
|
|
tool("Bash", command=f"cd {b} && grep -rn -- /oven kilde"),
|
|
tool("Bash", command=f"cd {b} && rg --regexp=/oven kilde"),
|
|
tool("Bash", command=f"cd {b} && rg -n --regexp /oven -g '*.md' kilde"),
|
|
tool("Bash", command=f"cd {b} && sed -n '/^oven:/p' {oven}"),
|
|
tool("Bash", command=f"cd {b} && sed -n -e '/^oven:/,/^crust:/p' {oven}"),
|
|
]), [])
|
|
|
|
def test_a_file_path_after_the_pattern_or_in_a_file_option_is_still_a_breach(self) -> None:
|
|
b, oven = self.basis, "kilde/docs-bread-oven.md"
|
|
found = self.breaches([
|
|
tool("Bash", command=f"cd {b} && grep -rn oven /outside/notes.md"),
|
|
tool("Bash", command=f"cd {b} && grep -e oven /outside/notes.md"),
|
|
tool("Bash", command=f"cd {b} && grep -e /oven /outside/notes.md"),
|
|
tool("Bash", command=f"cd {b} && grep -f /outside/words kilde"),
|
|
tool("Bash", command=f"cd {b} && grep -rnf /outside/words kilde"),
|
|
tool("Bash", command=f"cd {b} && grep --file=/outside/words kilde"),
|
|
tool("Bash", command=f"cd {b} && rg -e oven -- /outside/notes.md"),
|
|
tool("Bash", command=f"cd {b} && sed -f /outside/script {oven}"),
|
|
tool("Bash", command=f"cd {b} && sed -n 1p /outside/notes.md"),
|
|
tool("Bash", command=f"cd {b} && sed -n -e 1p -e 2p /outside/notes.md"),
|
|
tool("Bash", command=f"cd {b} && grep -rn oven ~/notes"),
|
|
tool("Bash", command=f"cd {b} && grep -rn oven ../.."),
|
|
])
|
|
self.assertEqual(len(found), 12, found)
|
|
|
|
def test_a_dollar_or_backtick_inside_single_quotes_is_not_an_expansion(self) -> None:
|
|
b, oven = self.basis, "kilde/docs-bread-oven.md"
|
|
self.assertEqual(self.breaches([
|
|
tool("Bash", command=f"cd {b} && sed -n '1,$p' {oven}"),
|
|
tool("Bash", command=f"cd {b} && grep -n '`oven`' {oven}"),
|
|
tool("Bash", command=f"cd {b} && grep -n '\"$HOME\"' {oven}"),
|
|
tool("Bash", command=f"cd {b} && grep -n \"it's\" {oven} | grep -c 'a$(b)'"),
|
|
]), [])
|
|
|
|
def test_a_dollar_or_backtick_outside_single_quotes_is_still_an_expansion(self) -> None:
|
|
b, oven = self.basis, "kilde/docs-bread-oven.md"
|
|
found = self.breaches([
|
|
tool("Bash", command=f'cd {b} && sed -n "1,$p" {oven}'),
|
|
tool("Bash", command=f'cd {b} && grep -n "\\`oven\\`" {oven}'),
|
|
tool("Bash", command=f"cd {b} && grep -n $HOME {oven}"),
|
|
tool("Bash", command=f"cd {b} && grep -n 'a'$HOME'b' {oven}"),
|
|
tool("Bash", command=f"cd {b} && grep -n \"'$HOME'\" {oven}"),
|
|
tool("Bash", command=f"cd {b} && grep -n `id` {oven}"),
|
|
tool("Bash", command=f"cd {b} && grep -n \\'$HOME {oven}"),
|
|
])
|
|
self.assertEqual(found, ["Bash with an expansion"] * 7, found)
|
|
|
|
def test_reading_the_own_answer_file_is_not_a_breach(self) -> None:
|
|
b, own = self.basis, self.answer("d01")
|
|
self.assertEqual(self.breaches([
|
|
tool("Read", file_path=str(own)),
|
|
tool("Bash", command=f"cd {own.parent} && jq -r '.citations|length' {own.name}"),
|
|
tool("Bash", command=f"cd {own.parent}/ && jq -r .id {own}"),
|
|
tool("Bash", command=f"jq -e '.citations|length' {own}"),
|
|
tool("Bash", command=f"cd {b} && jq . {own} && grep -rn dough kilde"),
|
|
tool("Bash", command=f"cd {b} && grep -c quote {own}"),
|
|
]), [])
|
|
|
|
def test_another_file_beside_the_answer_or_another_command_is_still_a_breach(self) -> None:
|
|
b, own, other = self.basis, self.answer("d01"), self.answer("d02")
|
|
found = self.breaches([
|
|
tool("Read", file_path=str(other)),
|
|
tool("Bash", command=f"cd {own.parent} && jq . {other.name}"),
|
|
tool("Bash", command=f"cd {own.parent} && ls"),
|
|
tool("Bash", command=f"cd {own.parent} && jq . {own.name} && cat {other.name}"),
|
|
tool("Bash", command=f"cd {own.parent} && jq . *.json"),
|
|
tool("Bash", command=f"cd {own.parent} && jq -f filter.jq {own.name}"),
|
|
tool("Bash", command=f"cd {own.parent} && jq . ../bundle/d01.json"),
|
|
tool("Bash", command=f"jq . {other}"),
|
|
tool("Bash", command=f"jq . {own.name}"),
|
|
tool("Bash", command=f"jq --slurpfile x {other} . {own}"),
|
|
tool("Bash", command=f"jq . {own} > {own}.bak"),
|
|
tool("Bash", command=f'cd {b} && python3 -c "import json;json.load(open(\'{own}\'))"'),
|
|
tool("Bash", command=f"python3 -m json.tool {own}"),
|
|
tool("Bash", command=f"cd {b} && jq . kilde/docs-bread-oven.md"),
|
|
tool("Bash", command=f"cd {self.run_dir / 'answers'} && jq . filsok/{own.name}"),
|
|
])
|
|
self.assertEqual(len(found), 15, found)
|
|
|
|
def test_other_arms_runs_and_unrelated_agents_are_ignored(self) -> None:
|
|
write_transcript(self.subagents / "agent-a1.jsonl", self.good_lines("d01"), "S1-r1-filsok-d01")
|
|
write_transcript(self.subagents / "agent-a2.jsonl", self.good_lines("d02"), "S1-r1-filsok-d02")
|
|
write_transcript(self.subagents / "agent-b1.jsonl", self.good_lines("d01"), "S1-r1-bundle-d01")
|
|
write_transcript(self.subagents / "agent-b2.jsonl", self.good_lines("d01"), "S1-r2-filsok-d01")
|
|
write_transcript(self.subagents / "agent-c1.jsonl", self.good_lines("d01"), "V1-r1-read-c001")
|
|
code, out, doc = self.harvest()
|
|
self.assertEqual(code, 0, out)
|
|
self.assertEqual([r["runs"] for r in doc["questions"]], [1, 1])
|
|
|
|
def test_a_retried_question_sums_its_runs(self) -> None:
|
|
write_transcript(self.subagents / "agent-a1.jsonl", self.good_lines("d01"), "S1-r1-filsok-d01")
|
|
write_transcript(self.subagents / "agent-a3.jsonl", self.good_lines("d01", "2026-10-09T11:00:00Z"),
|
|
"S1-r1-filsok-d01")
|
|
write_transcript(self.subagents / "agent-a2.jsonl", self.good_lines("d02"), "S1-r1-filsok-d02")
|
|
code, out, doc = self.harvest()
|
|
row = doc["questions"][0]
|
|
self.assertEqual((row["runs"], row["input_tokens"], row["output_tokens"]), (2, 30, 114))
|
|
|
|
def test_a_transcript_without_a_description_is_matched_by_its_task_path(self) -> None:
|
|
write_transcript(self.subagents / "agent-a1.jsonl", self.good_lines("d01"), "S1-r1-filsok-d01")
|
|
write_transcript(self.transcripts / "workflow-x" / "agent-w1.jsonl", self.good_lines("d02"))
|
|
code, out, doc = self.harvest()
|
|
self.assertEqual(code, 0, out)
|
|
self.assertEqual([r["id"] for r in doc["questions"]], ["d01", "d02"])
|
|
|
|
def test_a_task_without_a_transcript_is_listed_and_exits_1(self) -> None:
|
|
write_transcript(self.subagents / "agent-a1.jsonl", self.good_lines("d01"), "S1-r1-filsok-d01")
|
|
code, out, doc = self.harvest()
|
|
self.assertEqual(code, 1)
|
|
self.assertIn("d02", out)
|
|
self.assertEqual([r["id"] for r in doc["questions"]], ["d01"])
|
|
|
|
def test_an_output_never_recorded_is_derived_from_the_next_input_as_a_lower_bound(self) -> None:
|
|
lines = [
|
|
user(f"Read {self.task('d01')} and do what it says."),
|
|
assistant("m1", usage(1, 0, 1000, 2), [tool("Read", file_path=str(self.task("d01")))], None),
|
|
assistant("m2", usage(1, 5000, 1000, 10), [], "end_turn"),
|
|
]
|
|
write_transcript(self.subagents / "agent-a1.jsonl", lines, "S1-r1-filsok-d01")
|
|
write_transcript(self.subagents / "agent-a2.jsonl", self.good_lines("d02"), "S1-r1-filsok-d02")
|
|
_, _, doc = self.harvest()
|
|
row = doc["questions"][0]
|
|
self.assertEqual(row["output_tokens"], 12)
|
|
self.assertEqual(row["output_tokens_derived"], 4998)
|
|
|
|
|
|
def result(qid: str, group: str, correct: bool, paths: int = 2) -> dict[str, Any]:
|
|
return {"id": qid, "group": group, "correct": correct, "keys_held": 2 if correct else 0, "keys": 2,
|
|
"paths_cited": paths if correct else 0, "paths_needed": paths}
|
|
|
|
|
|
def results_doc(arm: str, rows: list[dict[str, Any]], set_sha: str = "a" * 64) -> dict[str, Any]:
|
|
return {"schema": "okf-wiki/answer-results/1", "arm": arm, "set_sha256": set_sha, "results": rows}
|
|
|
|
|
|
# Ten questions: both right 2, only filsok 1, only bundle 5, neither 2.
|
|
PATTERN = [ # (id, group, paths, filsok, bundle)
|
|
("d01", "docs-2-3", 2, True, True), ("d02", "docs-2-3", 3, True, True), ("d03", "docs-2-3", 2, True, False),
|
|
("d04", "docs-4", 4, False, True), ("d05", "docs-4", 5, False, True), ("d06", "docs-4", 4, False, True),
|
|
("d07", "history", 2, False, True), ("d08", "history", 4, False, True), ("d09", "docs-4", 4, False, False),
|
|
("d10", "history", 3, False, False),
|
|
]
|
|
|
|
|
|
class BashBreachTest(unittest.TestCase):
|
|
"""Review findings 8-11 (order 39): option values, sed scripts, comments and newlines in the breach parser."""
|
|
|
|
basis = Path("/w/arms/filsok")
|
|
|
|
def breach(self, rest: str) -> str | None:
|
|
return answers.bash_breach(f"cd {self.basis} && {rest}", self.basis)
|
|
|
|
def assertClean(self, rest: str) -> None:
|
|
self.assertIsNone(self.breach(rest), rest)
|
|
|
|
def assertBreach(self, rest: str) -> None:
|
|
self.assertIsNotNone(self.breach(rest), rest)
|
|
|
|
# finding 9: grep -C and long options with a separate value
|
|
def test_grep_context_and_long_option_values_are_not_the_search_word(self) -> None:
|
|
for rest in ('grep -C 2 "/kilde/docs-x.md" wiki/index.md',
|
|
'grep -rn --include "*.md" "/kilde/x" wiki',
|
|
'grep -rn --exclude-dir node_modules /kilde/x wiki',
|
|
'grep --after-context 3 /kilde/x wiki/index.md',
|
|
'grep --max-count 1 -e /kilde/x wiki/index.md',
|
|
"rg --glob '*.md' /kilde/x wiki",
|
|
"rg -n --max-depth 2 /kilde/x wiki"):
|
|
with self.subTest(rest=rest):
|
|
self.assertClean(rest)
|
|
|
|
def test_grep_context_with_a_separate_value_stays_strict(self) -> None:
|
|
# BSD grep's --context takes its value only after '=': `--context 1 /x f` makes 1 the word and /x a file.
|
|
self.assertBreach("grep --context 1 /outside/notes.md wiki")
|
|
self.assertClean("grep --context=1 /kilde/x wiki")
|
|
|
|
# finding 8: options whose value is a file are checked as paths
|
|
def test_a_file_option_value_outside_the_basis_is_a_breach(self) -> None:
|
|
for rest in ("grep --exclude-from /outside/list -rn dough kilde",
|
|
"grep --exclude-from=/outside/list -rn dough kilde",
|
|
"rg --ignore-file /outside/x dough kilde",
|
|
"grep -rn --include /outside/x dough kilde",
|
|
"rg --pre cat dough kilde",
|
|
"rg --pre=cat dough kilde"):
|
|
with self.subTest(rest=rest):
|
|
self.assertBreach(rest)
|
|
|
|
# finding 11 and 8: sed scripts that read, write or run, and clustered in-place flags
|
|
def test_sed_scripts_that_write_read_outside_or_run_are_breaches(self) -> None:
|
|
for rest in ("sed -n '/x/w /tmp/out' wiki/index.md",
|
|
"sed '1e touch /tmp/p' wiki/index.md",
|
|
"sed -n '1r /outside/notes.md' wiki/index.md",
|
|
"sed -n '1R /outside/notes.md' wiki/index.md",
|
|
"sed -n 'W out.txt' wiki/index.md",
|
|
"sed 's/a/b/w out.txt' wiki/index.md",
|
|
"sed 's/a/b/e' wiki/index.md",
|
|
"sed -n -e 1p -e '2w out' wiki/index.md",
|
|
"sed -n --expression='2w out' wiki/index.md",
|
|
"sed -ne'2w out' wiki/index.md",
|
|
"sed -n '1{p;w out\n}' wiki/index.md",
|
|
"sed -n '1r ../../x' wiki/index.md",
|
|
"sed -ni s/a/b/ wiki/index.md",
|
|
"sed -Ei s/a/b/ wiki/index.md",
|
|
"sed -I '' s/a/b/ wiki/index.md",
|
|
"sed -n '1q;}' wiki/index.md"):
|
|
with self.subTest(rest=rest):
|
|
self.assertBreach(rest)
|
|
|
|
def test_read_only_sed_scripts_stay_clean(self) -> None:
|
|
for rest in ("sed -n '/^oven:/p' kilde/a.md",
|
|
"sed -n '/write/,/^## /p' kilde/a.md",
|
|
"sed -n '/^## Reading/,/^## /{p;}' kilde/a.md",
|
|
"sed -n '10,20p;25q' kilde/a.md",
|
|
"sed -n '$=' kilde/a.md",
|
|
"sed 's/where/w/g' kilde/a.md",
|
|
"sed -E 's#/kilde/(.*)#\\1#' kilde/a.md",
|
|
"sed -n '\\,/kilde/,p' kilde/a.md",
|
|
"sed -n '/x/I p' kilde/a.md",
|
|
"sed -n '1r kilde/b.md' kilde/a.md",
|
|
"sed -n -e '/oven/!d' -e p kilde/a.md",
|
|
"sed -n 'y/abc/xyz/;p' kilde/a.md"):
|
|
with self.subTest(rest=rest):
|
|
self.assertClean(rest)
|
|
|
|
# finding 10: '#' inside a word and a newline between commands
|
|
def test_a_hash_inside_a_word_does_not_hide_the_rest_of_the_line(self) -> None:
|
|
self.assertEqual("Bash command not allowed: python3", self.breach("grep -c a#b kilde && python3 -c 'print(1)'"))
|
|
|
|
def test_a_newline_separates_commands(self) -> None:
|
|
self.assertEqual("Bash command not allowed: python3", self.breach("grep -c a kilde\npython3 -c 'print(1)'"))
|
|
self.assertEqual("Bash command not allowed: python3", self.breach("grep -c a kilde &&\npython3 x"))
|
|
self.assertClean("grep -c a kilde\nls kilde\n")
|
|
self.assertClean("grep -rn a kilde |\nhead -3")
|
|
self.assertClean('grep -c "a\nb" kilde')
|
|
|
|
|
|
class ScoreTest(unittest.TestCase):
|
|
def setUp(self) -> None:
|
|
self._tmp = tempfile.TemporaryDirectory()
|
|
self.tmp = Path(self._tmp.name)
|
|
self.a = self.write("filsok", [result(i, g, f, p) for i, g, p, f, _ in PATTERN])
|
|
self.b = self.write("bundle", [result(i, g, b, p) for i, g, p, _, b in PATTERN])
|
|
|
|
def tearDown(self) -> None:
|
|
self._tmp.cleanup()
|
|
|
|
def write(self, arm: str, rows: list[dict[str, Any]], name: str = "", set_sha: str = "a" * 64) -> Path:
|
|
path = self.tmp / (name or f"{arm}.json")
|
|
path.write_bytes(dump(results_doc(arm, rows, set_sha)))
|
|
return path
|
|
|
|
def score(self, *extra: str) -> tuple[int, str]:
|
|
return run(["score", "--results", str(self.a), "--results", str(self.b), *extra])
|
|
|
|
def test_per_arm_and_group_and_by_paths(self) -> None:
|
|
code, out = self.score()
|
|
self.assertEqual(code, 0, out)
|
|
self.assertIn("filsok: docs-2-3 3 of 3, docs-4 0 of 4, history 0 of 3, all 3 of 10", out)
|
|
self.assertIn("bundle: docs-2-3 2 of 3, docs-4 3 of 4, history 2 of 3, all 7 of 10", out)
|
|
self.assertIn("filsok by paths: 2-3 3 of 5, 4+ 0 of 5", out)
|
|
self.assertIn("bundle by paths: 2-3 3 of 5, 4+ 4 of 5", out)
|
|
|
|
def test_pairs_mcnemar_difference_and_smallest_visible_difference(self) -> None:
|
|
out_path = self.tmp / "score.json"
|
|
code, out = self.score("--out", str(out_path))
|
|
self.assertEqual(code, 0, out)
|
|
self.assertIn("pairs: both 2, only filsok 1, only bundle 5, neither 2", out)
|
|
self.assertIn("exact McNemar p = 0.219", out)
|
|
doc = json.loads(out_path.read_text(encoding="utf-8"))
|
|
self.assertEqual(doc["schema"], "okf-wiki/answer-score/1")
|
|
self.assertEqual(doc["pairs"]["all"], {"both": 2, "only filsok": 1, "only bundle": 5, "neither": 2})
|
|
self.assertAlmostEqual(doc["mcnemar_p"], 0.21875)
|
|
low, high = answers.paired_difference_ci(2, 5, 1, 2)
|
|
self.assertAlmostEqual(doc["difference"]["point"], 0.4)
|
|
self.assertAlmostEqual(doc["difference"]["low"], low)
|
|
self.assertAlmostEqual(doc["difference"]["high"], high)
|
|
self.assertAlmostEqual(doc["smallest_visible"]["discordant_share"], 0.6)
|
|
self.assertEqual(doc["smallest_visible"]["difference"], answers.min_detectable(10, 0.6))
|
|
self.assertIn("bundle - filsok = 0.400, 95 % CI", out)
|
|
self.assertIn("smallest difference visible at 80 % power", out)
|
|
self.assertEqual(out_path.read_bytes(), dump(doc))
|
|
|
|
def test_no_line_per_question(self) -> None:
|
|
_, out = self.score()
|
|
for qid, *_ in PATTERN:
|
|
self.assertNotIn(qid, out)
|
|
|
|
def test_tokens_per_question_are_reported_beside(self) -> None:
|
|
tok = {"schema": "okf-wiki/answer-tokens/1", "arm": "filsok", "questions": [
|
|
{"id": "d01", "input_tokens": 100, "cache_creation_input_tokens": 200, "cache_read_input_tokens": 600,
|
|
"output_tokens": 50, "output_tokens_derived": 50, "isolation_breaches": 1},
|
|
{"id": "d02", "input_tokens": 100, "cache_creation_input_tokens": 200, "cache_read_input_tokens": 600,
|
|
"output_tokens": 100, "output_tokens_derived": 0, "isolation_breaches": 0}]}
|
|
path = self.tmp / "tok-filsok.json"
|
|
path.write_bytes(dump(tok))
|
|
code, out = self.score("--tokens", str(path))
|
|
self.assertEqual(code, 0, out)
|
|
self.assertIn("filsok tokens: 2 questions harvested, 1000 per question, isolation breaches 1", out)
|
|
|
|
def test_two_result_files_for_different_sets_cannot_run(self) -> None:
|
|
self.b = self.write("bundle", [result(i, g, b, p) for i, g, p, _, b in PATTERN], set_sha="b" * 64)
|
|
self.assertEqual(self.score()[0], 1)
|
|
|
|
def test_two_result_files_with_different_questions_cannot_run(self) -> None:
|
|
self.b = self.write("bundle", [result(i, g, b, p) for i, g, p, _, b in PATTERN[:-1]])
|
|
self.assertEqual(self.score()[0], 1)
|
|
|
|
def test_the_arms_are_filsok_and_bundle(self) -> None:
|
|
self.b = self.write("server", [result(i, g, b, p) for i, g, p, _, b in PATTERN])
|
|
self.assertEqual(self.score()[0], 1)
|
|
|
|
def test_a_missing_result_file_cannot_run(self) -> None:
|
|
self.b = self.tmp / "none.json"
|
|
self.assertEqual(self.score()[0], 1)
|
|
|
|
|
|
class StatisticsTest(unittest.TestCase):
|
|
def test_exact_mcnemar_against_known_values(self) -> None:
|
|
self.assertAlmostEqual(answers.mcnemar_exact(0, 6), 0.03125)
|
|
self.assertAlmostEqual(answers.mcnemar_exact(6, 0), 0.03125)
|
|
self.assertAlmostEqual(answers.mcnemar_exact(1, 9), 0.021484375)
|
|
self.assertAlmostEqual(answers.mcnemar_exact(2, 0), 0.5)
|
|
self.assertEqual(answers.mcnemar_exact(0, 0), 1.0)
|
|
self.assertEqual(answers.mcnemar_exact(5, 5), 1.0)
|
|
|
|
def test_paired_interval_against_newcombe_table_iii_method_10(self) -> None:
|
|
for cells, (low, high) in NEWCOMBE_TABLE_III:
|
|
got = answers.paired_difference_ci(*cells)
|
|
self.assertAlmostEqual(got[0], low, delta=1e-4, msg=str(cells))
|
|
self.assertAlmostEqual(got[1], high, delta=1e-4, msg=str(cells))
|
|
|
|
def test_paired_interval_is_antisymmetric_in_the_discordant_cells(self) -> None:
|
|
low, high = answers.paired_difference_ci(20, 12, 2, 16)
|
|
back = answers.paired_difference_ci(20, 2, 12, 16)
|
|
self.assertAlmostEqual(back[0], -high)
|
|
self.assertAlmostEqual(back[1], -low)
|
|
|
|
def test_smallest_visible_difference_against_the_plans_sample_sizes(self) -> None:
|
|
# PLAN.md § Styrke: 37-57 questions see 20 points, 155-234 see 10 points, at 20-30 % discordant pairs.
|
|
for n, share, diff in ((37, 0.2, 0.2), (57, 0.3, 0.2), (155, 0.2, 0.1), (234, 0.3, 0.1)):
|
|
visible = answers.min_detectable(n, share)
|
|
assert visible is not None
|
|
self.assertLessEqual(visible, diff, (n, share))
|
|
fewer = answers.min_detectable(n - 1, share)
|
|
self.assertTrue(fewer is None or fewer > diff, (n - 1, share, fewer))
|
|
|
|
def test_no_discordant_pairs_has_no_smallest_difference(self) -> None:
|
|
self.assertIsNone(answers.min_detectable(60, 0.0))
|
|
|
|
def test_too_few_pairs_for_any_difference_up_to_the_discordant_share(self) -> None:
|
|
# n = 10, psi = 0.6: even d = psi needs (z_a sqrt(.6) + z_b sqrt(.6 - .36))^2 / .36 = 10.35 pairs.
|
|
self.assertIsNone(answers.min_detectable(10, 0.6))
|
|
visible = answers.min_detectable(11, 0.6)
|
|
assert visible is not None
|
|
self.assertLessEqual(visible, 0.6)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|