"""Tests for gates/answers.py: the answer harness of contract K5 (setup, collect, harvest, score). Synthetic K1 roots about bread and gardens, synthetic transcripts and synthetic result files. No corpus text is used (CLAUDE.md rule 3), and nothing here starts a model. """ from __future__ import annotations import contextlib import io import json import sys import tempfile import unittest from pathlib import Path from typing import Any, Callable sys.path.insert(0, str(Path(__file__).resolve().parents[1])) sys.path.insert(0, str(Path(__file__).resolve().parent)) import answers # noqa: E402 from test_answerset import ( # noqa: E402 KEY, OVEN, RISING, SOIL, SPAN, TEXTS, WATER, dump, make_set, q4, q23, root_ref, sha, write_root, ) # Newcombe (1998), Statist. Med. 17:2635-2650, Table III, method 10: (e, f, g, h) -> 95 % interval for # (e + f)/n - (e + g)/n, as printed (four decimals). NEWCOMBE_TABLE_III = [ ((36, 12, 2, 0), (0.0569, 0.3404)), ((20, 12, 2, 16), (0.0562, 0.3292)), ((18, 12, 2, 18), (0.0562, 0.3290)), ((35, 14, 0, 1), (0.1461, 0.4175)), ((18, 14, 0, 18), (0.1441, 0.3963)), ((1, 97, 1, 1), (0.8736, 0.9850)), ((0, 30, 0, 0), (0.8395, 1.0)), ((1, 98, 0, 1), (0.9171, 0.9916)), ((54, 0, 0, 0), (-0.0664, 0.0664)), ((53, 0, 0, 1), (-0.0729, 0.0729)), ((27, 0, 0, 27), (-0.0351, 0.0351)), ] def run(argv: list[str]) -> tuple[int, str]: out, err = io.StringIO(), io.StringIO() with contextlib.redirect_stdout(out), contextlib.redirect_stderr(err): try: code = answers.main(argv) except SystemExit as exc: # argparse refuses an option it does not know code = int(exc.code or 0) return code, out.getvalue() + err.getvalue() class Base(unittest.TestCase): def setUp(self) -> None: self._tmp = tempfile.TemporaryDirectory() self.tmp = Path(self._tmp.name).resolve() self.root = write_root(self.tmp / "copy") self.set = self.tmp / "set.json" self.set.write_bytes(dump(make_set(self.root, [q23("d01"), q4("d02")]))) self.run_dir = self.tmp / "run" def tearDown(self) -> None: self._tmp.cleanup() def setup_arm(self, arm: str = "filsok", extra: list[str] | None = None) -> tuple[int, str]: return run(["setup", "--run", str(self.run_dir), "--name", "r1", "--arm", arm, "--set", str(self.set), "--sources", str(self.root)] + (extra or [])) class SetupTest(Base): def test_setup_copies_the_sources_as_the_arm_basis(self) -> None: code, out = self.setup_arm() self.assertEqual(code, 0, out) basis = self.run_dir / "arms" / "filsok" for path in TEXTS: self.assertEqual((basis / path).read_bytes(), (self.root / path).read_bytes()) self.assertEqual(sorted(p.relative_to(basis).as_posix() for p in basis.rglob("*") if p.is_file()), sorted(TEXTS)) def test_one_task_per_question_without_keys_spans_or_counts(self) -> None: code, out = self.setup_arm() self.assertEqual(code, 0, out) basis = self.run_dir / "arms" / "filsok" for qid in ("d01", "d02"): task = self.run_dir / "tasks" / "filsok" / f"{qid}.md" text = task.read_text(encoding="utf-8") self.assertIn("How do the synthetic notes about bread and gardens fit together?", text) self.assertIn(str(basis), text) self.assertIn(str(self.run_dir / "answers" / "filsok" / f"{qid}.json"), text) for value in list(SPAN.values()) + list(KEY.values()): self.assertNotIn(value, text) for word in ("answer_keys", "evidence", "docs-2-3", "docs-4", "history", "four documents"): self.assertNotIn(word, text) self.assertIn(f"S1-r1-filsok-{qid}\t{task}", out) self.assertTrue((self.run_dir / "answers" / "filsok").is_dir()) def test_the_task_asks_for_k1_paths_in_the_citations(self) -> None: self.setup_arm() text = (self.run_dir / "tasks" / "filsok" / "d01.md").read_text(encoding="utf-8") self.assertIn('"path": "kilde/', text) self.assertIn('"quote"', text) def test_an_arm_is_set_up_once(self) -> None: self.assertEqual(self.setup_arm()[0], 0) code, out = self.setup_arm() self.assertEqual(code, 1) self.assertIn("exists", out) def test_the_bundle_arm_needs_a_bundle_and_gets_the_sources_beside_it(self) -> None: self.assertEqual(self.setup_arm("bundle")[0], 1) bundle = self.tmp / "bundle" bundle.mkdir() (bundle / "index.md").write_text("# Index\n", encoding="utf-8") code, out = self.setup_arm("bundle", ["--bundle", str(bundle)]) self.assertEqual(code, 0, out) basis = self.run_dir / "arms" / "bundle" self.assertEqual((basis / "wiki" / "index.md").read_text(encoding="utf-8"), "# Index\n") self.assertEqual((basis / "kilde" / "docs-bread-oven.md").read_bytes(), (self.root / "kilde" / "docs-bread-oven.md").read_bytes()) text = (self.run_dir / "tasks" / "bundle" / "d01.md").read_text(encoding="utf-8") self.assertIn("wiki/index.md", text) self.assertNotIn("Read `BASIS/wiki/index.md` first", text) search, main, links = (text.index("Search `BASIS/wiki/index.md`"), text.index("Open the main page"), text.index("follow its links")) self.assertLess(search, main) self.assertLess(main, links) self.assertIn("never read whole", text) def test_only_the_fixed_arms(self) -> None: code, out = self.setup_arm("server") self.assertEqual(code, 1) self.assertIn("filsok", out) def test_the_root_must_be_the_one_the_set_names(self) -> None: other = write_root(self.tmp / "other", frozen_at="2026-10-09T12:00:00Z") code, out = run(["setup", "--run", str(self.run_dir), "--name", "r1", "--arm", "filsok", "--set", str(self.set), "--sources", str(other)]) self.assertEqual(code, 1) self.assertIn("manifest_sha256", out) def test_a_run_name_with_a_dash_or_space_is_refused(self) -> None: code, _ = run(["setup", "--run", str(self.run_dir), "--name", "r 1", "--arm", "filsok", "--set", str(self.set), "--sources", str(self.root)]) self.assertEqual(code, 1) class PartRootTest(Base): """Order 31 A3: an arm searches the whole copy for a set made from a part of it (`--part-root`).""" PART = (RISING, OVEN, SOIL, WATER) def make_part(self, copy: Path, change: Callable[[dict[str, Any]], None] | None = None) -> Path: part = write_root(self.tmp / "part", self.PART, subset_of=root_ref(copy)) if change is not None: manifest = part / "source-manifest.json" data = json.loads(manifest.read_bytes()) change(data) manifest.write_bytes(dump(data)) self.set.write_bytes(dump(make_set(part, [q23("d01"), q4("d02")]))) return part def setup_part(self, part: Path | None, sources: Path | None = None) -> tuple[int, str]: extra = ["--part-root", str(part)] if part is not None else [] return run(["setup", "--run", str(self.run_dir), "--name", "r1", "--arm", "filsok", "--set", str(self.set), "--sources", str(sources or self.root)] + extra) def test_a_set_of_the_part_is_set_up_on_the_whole_copy(self) -> None: code, out = self.setup_part(self.make_part(self.root)) self.assertEqual(code, 0, out) basis = self.run_dir / "arms" / "filsok" self.assertEqual(sorted(p.relative_to(basis).as_posix() for p in basis.rglob("*") if p.is_file()), sorted(TEXTS)) def test_a_part_that_is_not_a_subset_of_the_root_is_refused(self) -> None: def other(data: dict[str, Any]) -> None: data["subset_of"]["manifest_sha256"] = "0" * 64 code, out = self.setup_part(self.make_part(self.root, other)) self.assertEqual(code, 1) self.assertIn("subset_of", out) self.assertFalse((self.run_dir / "arms").exists()) def test_a_part_source_with_another_sha256_than_the_root_is_refused(self) -> None: def changed(data: dict[str, Any]) -> None: data["sources"][0]["sha256"] = "0" * 64 code, out = self.setup_part(self.make_part(self.root, changed)) self.assertEqual(code, 1) self.assertIn("sha256", out) self.assertFalse((self.run_dir / "arms").exists()) def test_a_part_source_missing_in_the_root_is_refused(self) -> None: smaller = write_root(self.tmp / "smaller", (RISING, OVEN, SOIL)) code, out = self.setup_part(self.make_part(smaller), smaller) self.assertEqual(code, 1) self.assertIn(WATER, out) self.assertFalse((self.run_dir / "arms").exists()) def test_a_part_root_that_is_not_the_sets_root_is_refused(self) -> None: part = self.make_part(self.root) self.set.write_bytes(dump(make_set(self.root, [q23("d01"), q4("d02")]))) code, out = self.setup_part(part) self.assertEqual(code, 1) self.assertIn("--part-root", out) def test_without_part_root_a_set_of_the_part_is_still_refused_on_the_copy(self) -> None: self.make_part(self.root) code, out = self.setup_part(None) self.assertEqual(code, 1) self.assertIn("manifest_sha256", out) class CollectTest(Base): def test_collect_writes_the_k5_answers_file_with_the_id_from_the_file_name(self) -> None: self.setup_arm() folder = self.run_dir / "answers" / "filsok" (folder / "d02.json").write_text(json.dumps({"id": "zz", "answer": "B", "citations": []}), encoding="utf-8") (folder / "d01.json").write_text(json.dumps({"answer": "A", "citations": [{"path": "kilde/x.md", "quote": "q"}]}), encoding="utf-8") code, out = run(["collect", "--run", str(self.run_dir), "--arm", "filsok"]) self.assertEqual(code, 0, out) path = self.run_dir / "answers" / "filsok.json" doc = json.loads(path.read_text(encoding="utf-8")) self.assertEqual(doc["schema"], "okf-wiki/answers/1") self.assertEqual(doc["arm"], "filsok") self.assertEqual([a["id"] for a in doc["answers"]], ["d01", "d02"]) self.assertEqual(doc["answers"][0]["citations"], [{"path": "kilde/x.md", "quote": "q"}]) self.assertEqual(path.read_bytes(), dump(doc)) def test_an_answer_file_that_is_not_a_json_object_is_a_problem(self) -> None: self.setup_arm() folder = self.run_dir / "answers" / "filsok" (folder / "d01.json").write_text("{not json", encoding="utf-8") (folder / "d02.json").write_text(json.dumps({"answer": "B", "citations": []}), encoding="utf-8") code, out = run(["collect", "--run", str(self.run_dir), "--arm", "filsok"]) self.assertEqual(code, 1) self.assertIn("d01.json", out) doc = json.loads((self.run_dir / "answers" / "filsok.json").read_text(encoding="utf-8")) self.assertEqual([a["id"] for a in doc["answers"]], ["d02"]) def usage(inp: int, cc: int, cr: int, out: int) -> dict[str, int]: return {"input_tokens": inp, "cache_creation_input_tokens": cc, "cache_read_input_tokens": cr, "output_tokens": out} def tool(name: str, **inp: Any) -> dict[str, Any]: return {"type": "tool_use", "id": f"t-{name}-{len(inp)}", "name": name, "input": inp} def assistant(mid: str, use: dict[str, int], blocks: list[dict[str, Any]], stop: str | None = "tool_use") -> dict[str, Any]: message: dict[str, Any] = {"id": mid, "role": "assistant", "usage": use, "content": blocks} if stop is not None: message["stop_reason"] = stop return {"type": "assistant", "message": message} def user(text: str, stamp: str = "2026-10-09T10:00:00Z") -> dict[str, Any]: return {"type": "user", "timestamp": stamp, "message": {"role": "user", "content": text}} def write_transcript(path: Path, lines: list[dict[str, Any]], description: str | None = None) -> None: path.parent.mkdir(parents=True, exist_ok=True) path.write_text("".join(json.dumps(line) + "\n" for line in lines) + "not json\n", encoding="utf-8") if description is not None: meta = path.with_name(path.stem + ".meta.json") meta.write_text(json.dumps({"agentType": "general-purpose", "description": description}), encoding="utf-8") class HarvestTest(Base): def setUp(self) -> None: super().setUp() self.assertEqual(self.setup_arm()[0], 0) self.basis = self.run_dir / "arms" / "filsok" self.transcripts = self.tmp / "transcripts" self.subagents = self.transcripts / "session-1" / "subagents" def task(self, qid: str) -> Path: return self.run_dir / "tasks" / "filsok" / f"{qid}.md" def answer(self, qid: str) -> Path: return self.run_dir / "answers" / "filsok" / f"{qid}.json" def good_lines(self, qid: str, stamp: str = "2026-10-09T10:00:00Z") -> list[dict[str, Any]]: return [ user(f"Read {self.task(qid)} and do what it says.", stamp), assistant("m1", usage(10, 100, 1000, 5), [tool("Read", file_path=str(self.task(qid)))]), assistant("m1", usage(10, 100, 1000, 7), []), assistant("m2", usage(3, 0, 1100, 20), [tool("Bash", command=f"cd {self.basis} && grep -rn dough .")]), assistant("m3", usage(2, 0, 1200, 30), [tool("Write", file_path=str(self.answer(qid)), content="{}")], "end_turn"), ] def harvest(self) -> tuple[int, str, dict[str, Any]]: code, out = run(["harvest", "--run", str(self.run_dir), "--name", "r1", "--arm", "filsok", "--transcripts", str(self.transcripts)]) path = self.run_dir / "tokens" / "filsok.json" return code, out, json.loads(path.read_text(encoding="utf-8")) if path.is_file() else {} def breaches(self, calls: list[dict[str, Any]]) -> list[str]: lines = self.good_lines("d01") lines.insert(3, assistant("m9", usage(1, 0, 0, 1), calls)) write_transcript(self.subagents / "agent-a1.jsonl", lines, "S1-r1-filsok-d01") return list(self.harvest()[2]["questions"][0]["breach_detail"]) def test_tokens_per_question_each_message_once_all_four_fields(self) -> None: write_transcript(self.subagents / "agent-a1.jsonl", self.good_lines("d01"), "S1-r1-filsok-d01") write_transcript(self.subagents / "agent-a2.jsonl", self.good_lines("d02"), "S1-r1-filsok-d02") code, out, doc = self.harvest() self.assertEqual(code, 0, out) self.assertEqual(doc["schema"], "okf-wiki/answer-tokens/1") self.assertEqual(doc["arm"], "filsok") row = doc["questions"][0] self.assertEqual(row["id"], "d01") self.assertEqual((row["input_tokens"], row["cache_creation_input_tokens"], row["cache_read_input_tokens"], row["output_tokens"]), (15, 100, 3300, 57)) self.assertEqual(row["isolation_breaches"], 0) self.assertEqual([r["id"] for r in doc["questions"]], ["d01", "d02"]) path = self.run_dir / "tokens" / "filsok.json" self.assertEqual(path.read_bytes(), dump(doc)) def test_isolation_breaches_are_counted(self) -> None: lines = self.good_lines("d01") lines.insert(3, assistant("m9", usage(1, 0, 0, 1), [ tool("Read", file_path="/etc/hosts"), tool("Bash", command="cat /etc/passwd"), tool("Grep", pattern="dough", path=str(self.run_dir / "arms" / "bundle")), tool("Glob", pattern="*.md"), tool("Read", file_path=str(self.basis / "kilde" / "docs-bread-oven.md")), tool("Bash", command=f"cd {self.basis} && grep -rn crust ../.."), tool("WebFetch", url="https://example.invalid/"), tool("Write", file_path=str(self.basis / "notes.md"), content="x"), ])) write_transcript(self.subagents / "agent-a1.jsonl", lines, "S1-r1-filsok-d01") write_transcript(self.subagents / "agent-a2.jsonl", self.good_lines("d02"), "S1-r1-filsok-d02") code, out, doc = self.harvest() row = doc["questions"][0] self.assertEqual(row["isolation_breaches"], 7, row) self.assertEqual(len(row["breach_detail"]), 7) self.assertIn("breaches: 7", out) def test_echo_of_literal_words_in_a_chain_into_the_basis_is_not_a_breach(self) -> None: lines = self.good_lines("d01") lines.insert(3, assistant("m9", usage(1, 0, 0, 1), [ tool("Bash", command=f"cd {self.basis} && grep -rn dough kilde && echo --- && grep -rn crust kilde"), tool("Bash", command=f"cd {self.basis} && grep -rn dough kilde | head -5; echo ----; ls kilde"), tool("Bash", command=f'cd {self.basis} && echo "next file" && sed -n 1,5p kilde/docs-bread-oven.md'), ])) write_transcript(self.subagents / "agent-a1.jsonl", lines, "S1-r1-filsok-d01") code, out, doc = self.harvest() self.assertEqual(doc["questions"][0]["isolation_breaches"], 0, doc["questions"][0]["breach_detail"]) self.assertNotIn("echo", self.task("d01").read_text(encoding="utf-8")) def test_echo_outside_the_basis_or_with_an_expansion_is_still_a_breach(self) -> None: lines = self.good_lines("d01") lines.insert(3, assistant("m9", usage(1, 0, 0, 1), [ tool("Bash", command=f"cd {self.basis} && echo /outside/*"), tool("Bash", command=f"cd {self.basis} && echo ~/*"), tool("Bash", command=f"cd {self.basis} && echo ../../*"), tool("Bash", command=f"cd {self.basis} && echo x > kilde/notes.md"), tool("Bash", command=f"cd {self.basis} && echo $HOME"), tool("Bash", command=f"cd {self.basis} && echo `id`"), tool("Bash", command="echo --- && grep -rn dough kilde"), ])) write_transcript(self.subagents / "agent-a1.jsonl", lines, "S1-r1-filsok-d01") code, out, doc = self.harvest() self.assertEqual(doc["questions"][0]["isolation_breaches"], 7, doc["questions"][0]["breach_detail"]) def test_stderr_sent_away_right_before_a_separator_is_not_a_breach(self) -> None: b = self.basis self.assertEqual(self.breaches([ tool("Bash", command=f"cd {b} && grep -rn dough kilde 2>/dev/null; ls kilde"), tool("Bash", command=f"cd {b} && grep -rn dough kilde 2>/dev/null| head -3"), tool("Bash", command=f"cd {b} && grep -rn dough kilde 2>/dev/null&& ls kilde"), tool("Bash", command=f"cd {b} && grep -rn dough kilde 2>/dev/null|| ls kilde"), ]), []) def test_other_redirections_before_a_separator_are_still_breaches(self) -> None: b = self.basis found = self.breaches([ tool("Bash", command=f"cd {b} && grep -rn dough kilde > kilde/out.md; ls kilde"), tool("Bash", command=f"cd {b} && grep -rn dough kilde >/dev/null; ls kilde"), tool("Bash", command=f"cd {b} && grep -rn dough kilde 2>&1; ls kilde"), tool("Bash", command=f"cd {b} && grep -rn dough kilde 2>/outside/err; ls kilde"), tool("Bash", command=f"cd {b} && grep -rn dough kilde 2>/dev/nullx; ls kilde"), ]) self.assertEqual(len(found), 5, found) def test_a_search_pattern_or_sed_script_starting_with_a_slash_is_not_a_path(self) -> None: b, oven = self.basis, "kilde/docs-bread-oven.md" self.assertEqual(self.breaches([ tool("Bash", command=f"cd {b} && grep -rn /oven kilde"), tool("Bash", command=f'cd {b} && grep -rlni "/oven" kilde | head -3'), tool("Bash", command=f"cd {b} && egrep -n -e /oven -e /crust {oven}"), tool("Bash", command=f"cd {b} && fgrep -A 3 -m 2 /oven {oven}"), tool("Bash", command=f"cd {b} && grep -rn -- /oven kilde"), tool("Bash", command=f"cd {b} && rg --regexp=/oven kilde"), tool("Bash", command=f"cd {b} && rg -n --regexp /oven -g '*.md' kilde"), tool("Bash", command=f"cd {b} && sed -n '/^oven:/p' {oven}"), tool("Bash", command=f"cd {b} && sed -n -e '/^oven:/,/^crust:/p' {oven}"), ]), []) def test_a_file_path_after_the_pattern_or_in_a_file_option_is_still_a_breach(self) -> None: b, oven = self.basis, "kilde/docs-bread-oven.md" found = self.breaches([ tool("Bash", command=f"cd {b} && grep -rn oven /outside/notes.md"), tool("Bash", command=f"cd {b} && grep -e oven /outside/notes.md"), tool("Bash", command=f"cd {b} && grep -e /oven /outside/notes.md"), tool("Bash", command=f"cd {b} && grep -f /outside/words kilde"), tool("Bash", command=f"cd {b} && grep -rnf /outside/words kilde"), tool("Bash", command=f"cd {b} && grep --file=/outside/words kilde"), tool("Bash", command=f"cd {b} && rg -e oven -- /outside/notes.md"), tool("Bash", command=f"cd {b} && sed -f /outside/script {oven}"), tool("Bash", command=f"cd {b} && sed -n 1p /outside/notes.md"), tool("Bash", command=f"cd {b} && sed -n -e 1p -e 2p /outside/notes.md"), tool("Bash", command=f"cd {b} && grep -rn oven ~/notes"), tool("Bash", command=f"cd {b} && grep -rn oven ../.."), ]) self.assertEqual(len(found), 12, found) def test_a_dollar_or_backtick_inside_single_quotes_is_not_an_expansion(self) -> None: b, oven = self.basis, "kilde/docs-bread-oven.md" self.assertEqual(self.breaches([ tool("Bash", command=f"cd {b} && sed -n '1,$p' {oven}"), tool("Bash", command=f"cd {b} && grep -n '`oven`' {oven}"), tool("Bash", command=f"cd {b} && grep -n '\"$HOME\"' {oven}"), tool("Bash", command=f"cd {b} && grep -n \"it's\" {oven} | grep -c 'a$(b)'"), ]), []) def test_a_dollar_or_backtick_outside_single_quotes_is_still_an_expansion(self) -> None: b, oven = self.basis, "kilde/docs-bread-oven.md" found = self.breaches([ tool("Bash", command=f'cd {b} && sed -n "1,$p" {oven}'), tool("Bash", command=f'cd {b} && grep -n "\\`oven\\`" {oven}'), tool("Bash", command=f"cd {b} && grep -n $HOME {oven}"), tool("Bash", command=f"cd {b} && grep -n 'a'$HOME'b' {oven}"), tool("Bash", command=f"cd {b} && grep -n \"'$HOME'\" {oven}"), tool("Bash", command=f"cd {b} && grep -n `id` {oven}"), tool("Bash", command=f"cd {b} && grep -n \\'$HOME {oven}"), ]) self.assertEqual(found, ["Bash with an expansion"] * 7, found) def test_reading_the_own_answer_file_is_not_a_breach(self) -> None: b, own = self.basis, self.answer("d01") self.assertEqual(self.breaches([ tool("Read", file_path=str(own)), tool("Bash", command=f"cd {own.parent} && jq -r '.citations|length' {own.name}"), tool("Bash", command=f"cd {own.parent}/ && jq -r .id {own}"), tool("Bash", command=f"jq -e '.citations|length' {own}"), tool("Bash", command=f"cd {b} && jq . {own} && grep -rn dough kilde"), tool("Bash", command=f"cd {b} && grep -c quote {own}"), ]), []) def test_another_file_beside_the_answer_or_another_command_is_still_a_breach(self) -> None: b, own, other = self.basis, self.answer("d01"), self.answer("d02") found = self.breaches([ tool("Read", file_path=str(other)), tool("Bash", command=f"cd {own.parent} && jq . {other.name}"), tool("Bash", command=f"cd {own.parent} && ls"), tool("Bash", command=f"cd {own.parent} && jq . {own.name} && cat {other.name}"), tool("Bash", command=f"cd {own.parent} && jq . *.json"), tool("Bash", command=f"cd {own.parent} && jq -f filter.jq {own.name}"), tool("Bash", command=f"cd {own.parent} && jq . ../bundle/d01.json"), tool("Bash", command=f"jq . {other}"), tool("Bash", command=f"jq . {own.name}"), tool("Bash", command=f"jq --slurpfile x {other} . {own}"), tool("Bash", command=f"jq . {own} > {own}.bak"), tool("Bash", command=f'cd {b} && python3 -c "import json;json.load(open(\'{own}\'))"'), tool("Bash", command=f"python3 -m json.tool {own}"), tool("Bash", command=f"cd {b} && jq . kilde/docs-bread-oven.md"), tool("Bash", command=f"cd {self.run_dir / 'answers'} && jq . filsok/{own.name}"), ]) self.assertEqual(len(found), 15, found) def test_other_arms_runs_and_unrelated_agents_are_ignored(self) -> None: write_transcript(self.subagents / "agent-a1.jsonl", self.good_lines("d01"), "S1-r1-filsok-d01") write_transcript(self.subagents / "agent-a2.jsonl", self.good_lines("d02"), "S1-r1-filsok-d02") write_transcript(self.subagents / "agent-b1.jsonl", self.good_lines("d01"), "S1-r1-bundle-d01") write_transcript(self.subagents / "agent-b2.jsonl", self.good_lines("d01"), "S1-r2-filsok-d01") write_transcript(self.subagents / "agent-c1.jsonl", self.good_lines("d01"), "V1-r1-read-c001") code, out, doc = self.harvest() self.assertEqual(code, 0, out) self.assertEqual([r["runs"] for r in doc["questions"]], [1, 1]) def test_a_retried_question_sums_its_runs(self) -> None: write_transcript(self.subagents / "agent-a1.jsonl", self.good_lines("d01"), "S1-r1-filsok-d01") write_transcript(self.subagents / "agent-a3.jsonl", self.good_lines("d01", "2026-10-09T11:00:00Z"), "S1-r1-filsok-d01") write_transcript(self.subagents / "agent-a2.jsonl", self.good_lines("d02"), "S1-r1-filsok-d02") code, out, doc = self.harvest() row = doc["questions"][0] self.assertEqual((row["runs"], row["input_tokens"], row["output_tokens"]), (2, 30, 114)) def test_a_transcript_without_a_description_is_matched_by_its_task_path(self) -> None: write_transcript(self.subagents / "agent-a1.jsonl", self.good_lines("d01"), "S1-r1-filsok-d01") write_transcript(self.transcripts / "workflow-x" / "agent-w1.jsonl", self.good_lines("d02")) code, out, doc = self.harvest() self.assertEqual(code, 0, out) self.assertEqual([r["id"] for r in doc["questions"]], ["d01", "d02"]) def test_a_task_without_a_transcript_is_listed_and_exits_1(self) -> None: write_transcript(self.subagents / "agent-a1.jsonl", self.good_lines("d01"), "S1-r1-filsok-d01") code, out, doc = self.harvest() self.assertEqual(code, 1) self.assertIn("d02", out) self.assertEqual([r["id"] for r in doc["questions"]], ["d01"]) def test_an_output_never_recorded_is_derived_from_the_next_input_as_a_lower_bound(self) -> None: lines = [ user(f"Read {self.task('d01')} and do what it says."), assistant("m1", usage(1, 0, 1000, 2), [tool("Read", file_path=str(self.task("d01")))], None), assistant("m2", usage(1, 5000, 1000, 10), [], "end_turn"), ] write_transcript(self.subagents / "agent-a1.jsonl", lines, "S1-r1-filsok-d01") write_transcript(self.subagents / "agent-a2.jsonl", self.good_lines("d02"), "S1-r1-filsok-d02") _, _, doc = self.harvest() row = doc["questions"][0] self.assertEqual(row["output_tokens"], 12) self.assertEqual(row["output_tokens_derived"], 4998) def result(qid: str, group: str, correct: bool, paths: int = 2) -> dict[str, Any]: return {"id": qid, "group": group, "correct": correct, "keys_held": 2 if correct else 0, "keys": 2, "paths_cited": paths if correct else 0, "paths_needed": paths} def results_doc(arm: str, rows: list[dict[str, Any]], set_sha: str = "a" * 64) -> dict[str, Any]: return {"schema": "okf-wiki/answer-results/1", "arm": arm, "set_sha256": set_sha, "results": rows} # Ten questions: both right 2, only filsok 1, only bundle 5, neither 2. PATTERN = [ # (id, group, paths, filsok, bundle) ("d01", "docs-2-3", 2, True, True), ("d02", "docs-2-3", 3, True, True), ("d03", "docs-2-3", 2, True, False), ("d04", "docs-4", 4, False, True), ("d05", "docs-4", 5, False, True), ("d06", "docs-4", 4, False, True), ("d07", "history", 2, False, True), ("d08", "history", 4, False, True), ("d09", "docs-4", 4, False, False), ("d10", "history", 3, False, False), ] class BashBreachTest(unittest.TestCase): """Review findings 8-11 (order 39): option values, sed scripts, comments and newlines in the breach parser.""" basis = Path("/w/arms/filsok") def breach(self, rest: str) -> str | None: return answers.bash_breach(f"cd {self.basis} && {rest}", self.basis) def assertClean(self, rest: str) -> None: self.assertIsNone(self.breach(rest), rest) def assertBreach(self, rest: str) -> None: self.assertIsNotNone(self.breach(rest), rest) # finding 9: grep -C and long options with a separate value def test_grep_context_and_long_option_values_are_not_the_search_word(self) -> None: for rest in ('grep -C 2 "/kilde/docs-x.md" wiki/index.md', 'grep -rn --include "*.md" "/kilde/x" wiki', 'grep -rn --exclude-dir node_modules /kilde/x wiki', 'grep --after-context 3 /kilde/x wiki/index.md', 'grep --max-count 1 -e /kilde/x wiki/index.md', "rg --glob '*.md' /kilde/x wiki", "rg -n --max-depth 2 /kilde/x wiki"): with self.subTest(rest=rest): self.assertClean(rest) def test_grep_context_with_a_separate_value_stays_strict(self) -> None: # BSD grep's --context takes its value only after '=': `--context 1 /x f` makes 1 the word and /x a file. self.assertBreach("grep --context 1 /outside/notes.md wiki") self.assertClean("grep --context=1 /kilde/x wiki") # finding 8: options whose value is a file are checked as paths def test_a_file_option_value_outside_the_basis_is_a_breach(self) -> None: for rest in ("grep --exclude-from /outside/list -rn dough kilde", "grep --exclude-from=/outside/list -rn dough kilde", "rg --ignore-file /outside/x dough kilde", "grep -rn --include /outside/x dough kilde", "rg --pre cat dough kilde", "rg --pre=cat dough kilde"): with self.subTest(rest=rest): self.assertBreach(rest) # finding 11 and 8: sed scripts that read, write or run, and clustered in-place flags def test_sed_scripts_that_write_read_outside_or_run_are_breaches(self) -> None: for rest in ("sed -n '/x/w /tmp/out' wiki/index.md", "sed '1e touch /tmp/p' wiki/index.md", "sed -n '1r /outside/notes.md' wiki/index.md", "sed -n '1R /outside/notes.md' wiki/index.md", "sed -n 'W out.txt' wiki/index.md", "sed 's/a/b/w out.txt' wiki/index.md", "sed 's/a/b/e' wiki/index.md", "sed -n -e 1p -e '2w out' wiki/index.md", "sed -n --expression='2w out' wiki/index.md", "sed -ne'2w out' wiki/index.md", "sed -n '1{p;w out\n}' wiki/index.md", "sed -n '1r ../../x' wiki/index.md", "sed -ni s/a/b/ wiki/index.md", "sed -Ei s/a/b/ wiki/index.md", "sed -I '' s/a/b/ wiki/index.md", "sed -n '1q;}' wiki/index.md"): with self.subTest(rest=rest): self.assertBreach(rest) def test_read_only_sed_scripts_stay_clean(self) -> None: for rest in ("sed -n '/^oven:/p' kilde/a.md", "sed -n '/write/,/^## /p' kilde/a.md", "sed -n '/^## Reading/,/^## /{p;}' kilde/a.md", "sed -n '10,20p;25q' kilde/a.md", "sed -n '$=' kilde/a.md", "sed 's/where/w/g' kilde/a.md", "sed -E 's#/kilde/(.*)#\\1#' kilde/a.md", "sed -n '\\,/kilde/,p' kilde/a.md", "sed -n '/x/I p' kilde/a.md", "sed -n '1r kilde/b.md' kilde/a.md", "sed -n -e '/oven/!d' -e p kilde/a.md", "sed -n 'y/abc/xyz/;p' kilde/a.md"): with self.subTest(rest=rest): self.assertClean(rest) # finding 10: '#' inside a word and a newline between commands def test_a_hash_inside_a_word_does_not_hide_the_rest_of_the_line(self) -> None: self.assertEqual("Bash command not allowed: python3", self.breach("grep -c a#b kilde && python3 -c 'print(1)'")) def test_a_newline_separates_commands(self) -> None: self.assertEqual("Bash command not allowed: python3", self.breach("grep -c a kilde\npython3 -c 'print(1)'")) self.assertEqual("Bash command not allowed: python3", self.breach("grep -c a kilde &&\npython3 x")) self.assertClean("grep -c a kilde\nls kilde\n") self.assertClean("grep -rn a kilde |\nhead -3") self.assertClean('grep -c "a\nb" kilde') class ScoreTest(unittest.TestCase): def setUp(self) -> None: self._tmp = tempfile.TemporaryDirectory() self.tmp = Path(self._tmp.name) self.a = self.write("filsok", [result(i, g, f, p) for i, g, p, f, _ in PATTERN]) self.b = self.write("bundle", [result(i, g, b, p) for i, g, p, _, b in PATTERN]) def tearDown(self) -> None: self._tmp.cleanup() def write(self, arm: str, rows: list[dict[str, Any]], name: str = "", set_sha: str = "a" * 64) -> Path: path = self.tmp / (name or f"{arm}.json") path.write_bytes(dump(results_doc(arm, rows, set_sha))) return path def score(self, *extra: str) -> tuple[int, str]: return run(["score", "--results", str(self.a), "--results", str(self.b), *extra]) def test_per_arm_and_group_and_by_paths(self) -> None: code, out = self.score() self.assertEqual(code, 0, out) self.assertIn("filsok: docs-2-3 3 of 3, docs-4 0 of 4, history 0 of 3, all 3 of 10", out) self.assertIn("bundle: docs-2-3 2 of 3, docs-4 3 of 4, history 2 of 3, all 7 of 10", out) self.assertIn("filsok by paths: 2-3 3 of 5, 4+ 0 of 5", out) self.assertIn("bundle by paths: 2-3 3 of 5, 4+ 4 of 5", out) def test_pairs_mcnemar_difference_and_smallest_visible_difference(self) -> None: out_path = self.tmp / "score.json" code, out = self.score("--out", str(out_path)) self.assertEqual(code, 0, out) self.assertIn("pairs: both 2, only filsok 1, only bundle 5, neither 2", out) self.assertIn("exact McNemar p = 0.219", out) doc = json.loads(out_path.read_text(encoding="utf-8")) self.assertEqual(doc["schema"], "okf-wiki/answer-score/1") self.assertEqual(doc["pairs"]["all"], {"both": 2, "only filsok": 1, "only bundle": 5, "neither": 2}) self.assertAlmostEqual(doc["mcnemar_p"], 0.21875) low, high = answers.paired_difference_ci(2, 5, 1, 2) self.assertAlmostEqual(doc["difference"]["point"], 0.4) self.assertAlmostEqual(doc["difference"]["low"], low) self.assertAlmostEqual(doc["difference"]["high"], high) self.assertAlmostEqual(doc["smallest_visible"]["discordant_share"], 0.6) self.assertEqual(doc["smallest_visible"]["difference"], answers.min_detectable(10, 0.6)) self.assertIn("bundle - filsok = 0.400, 95 % CI", out) self.assertIn("smallest difference visible at 80 % power", out) self.assertEqual(out_path.read_bytes(), dump(doc)) def test_no_line_per_question(self) -> None: _, out = self.score() for qid, *_ in PATTERN: self.assertNotIn(qid, out) def test_tokens_per_question_are_reported_beside(self) -> None: tok = {"schema": "okf-wiki/answer-tokens/1", "arm": "filsok", "questions": [ {"id": "d01", "input_tokens": 100, "cache_creation_input_tokens": 200, "cache_read_input_tokens": 600, "output_tokens": 50, "output_tokens_derived": 50, "isolation_breaches": 1}, {"id": "d02", "input_tokens": 100, "cache_creation_input_tokens": 200, "cache_read_input_tokens": 600, "output_tokens": 100, "output_tokens_derived": 0, "isolation_breaches": 0}]} path = self.tmp / "tok-filsok.json" path.write_bytes(dump(tok)) code, out = self.score("--tokens", str(path)) self.assertEqual(code, 0, out) self.assertIn("filsok tokens: 2 questions harvested, 1000 per question, isolation breaches 1", out) def test_two_result_files_for_different_sets_cannot_run(self) -> None: self.b = self.write("bundle", [result(i, g, b, p) for i, g, p, _, b in PATTERN], set_sha="b" * 64) self.assertEqual(self.score()[0], 1) def test_two_result_files_with_different_questions_cannot_run(self) -> None: self.b = self.write("bundle", [result(i, g, b, p) for i, g, p, _, b in PATTERN[:-1]]) self.assertEqual(self.score()[0], 1) def test_the_arms_are_filsok_and_bundle(self) -> None: self.b = self.write("server", [result(i, g, b, p) for i, g, p, _, b in PATTERN]) self.assertEqual(self.score()[0], 1) def test_a_missing_result_file_cannot_run(self) -> None: self.b = self.tmp / "none.json" self.assertEqual(self.score()[0], 1) class StatisticsTest(unittest.TestCase): def test_exact_mcnemar_against_known_values(self) -> None: self.assertAlmostEqual(answers.mcnemar_exact(0, 6), 0.03125) self.assertAlmostEqual(answers.mcnemar_exact(6, 0), 0.03125) self.assertAlmostEqual(answers.mcnemar_exact(1, 9), 0.021484375) self.assertAlmostEqual(answers.mcnemar_exact(2, 0), 0.5) self.assertEqual(answers.mcnemar_exact(0, 0), 1.0) self.assertEqual(answers.mcnemar_exact(5, 5), 1.0) def test_paired_interval_against_newcombe_table_iii_method_10(self) -> None: for cells, (low, high) in NEWCOMBE_TABLE_III: got = answers.paired_difference_ci(*cells) self.assertAlmostEqual(got[0], low, delta=1e-4, msg=str(cells)) self.assertAlmostEqual(got[1], high, delta=1e-4, msg=str(cells)) def test_paired_interval_is_antisymmetric_in_the_discordant_cells(self) -> None: low, high = answers.paired_difference_ci(20, 12, 2, 16) back = answers.paired_difference_ci(20, 2, 12, 16) self.assertAlmostEqual(back[0], -high) self.assertAlmostEqual(back[1], -low) def test_smallest_visible_difference_against_the_plans_sample_sizes(self) -> None: # PLAN.md ยง Styrke: 37-57 questions see 20 points, 155-234 see 10 points, at 20-30 % discordant pairs. for n, share, diff in ((37, 0.2, 0.2), (57, 0.3, 0.2), (155, 0.2, 0.1), (234, 0.3, 0.1)): visible = answers.min_detectable(n, share) assert visible is not None self.assertLessEqual(visible, diff, (n, share)) fewer = answers.min_detectable(n - 1, share) self.assertTrue(fewer is None or fewer > diff, (n - 1, share, fewer)) def test_no_discordant_pairs_has_no_smallest_difference(self) -> None: self.assertIsNone(answers.min_detectable(60, 0.0)) def test_too_few_pairs_for_any_difference_up_to_the_discordant_share(self) -> None: # n = 10, psi = 0.6: even d = psi needs (z_a sqrt(.6) + z_b sqrt(.6 - .36))^2 / .36 = 10.35 pairs. self.assertIsNone(answers.min_detectable(10, 0.6)) visible = answers.min_detectable(11, 0.6) assert visible is not None self.assertLessEqual(visible, 0.6) if __name__ == "__main__": unittest.main()