okf-wiki 1.1.1

First public release, published as one snapshot. What changed between
versions is in CHANGELOG.md.

Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
This commit is contained in:
Kjell Tore Guttormsen 2026-11-14 21:03:20 +01:00
commit 0fd86a23a3
Signed by: ktg
SSH key fingerprint: SHA256:JakMjO6FTBBzN0Bhfj9saOoEjaFxlSdYuZQQpM/lF9Q
227 changed files with 96077 additions and 0 deletions

752
gates/tests/test_answers.py Normal file
View file

@ -0,0 +1,752 @@
"""Tests for gates/answers.py: the answer harness of contract K5 (setup, collect, harvest, score).
Synthetic K1 roots about bread and gardens, synthetic transcripts and synthetic result files. No corpus text is used
(CLAUDE.md rule 3), and nothing here starts a model.
"""
from __future__ import annotations
import contextlib
import io
import json
import sys
import tempfile
import unittest
from pathlib import Path
from typing import Any, Callable
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
sys.path.insert(0, str(Path(__file__).resolve().parent))
import answers # noqa: E402
from test_answerset import ( # noqa: E402
KEY, OVEN, RISING, SOIL, SPAN, TEXTS, WATER, dump, make_set, q4, q23, root_ref, sha, write_root,
)
# Newcombe (1998), Statist. Med. 17:2635-2650, Table III, method 10: (e, f, g, h) -> 95 % interval for
# (e + f)/n - (e + g)/n, as printed (four decimals).
NEWCOMBE_TABLE_III = [
((36, 12, 2, 0), (0.0569, 0.3404)),
((20, 12, 2, 16), (0.0562, 0.3292)),
((18, 12, 2, 18), (0.0562, 0.3290)),
((35, 14, 0, 1), (0.1461, 0.4175)),
((18, 14, 0, 18), (0.1441, 0.3963)),
((1, 97, 1, 1), (0.8736, 0.9850)),
((0, 30, 0, 0), (0.8395, 1.0)),
((1, 98, 0, 1), (0.9171, 0.9916)),
((54, 0, 0, 0), (-0.0664, 0.0664)),
((53, 0, 0, 1), (-0.0729, 0.0729)),
((27, 0, 0, 27), (-0.0351, 0.0351)),
]
def run(argv: list[str]) -> tuple[int, str]:
out, err = io.StringIO(), io.StringIO()
with contextlib.redirect_stdout(out), contextlib.redirect_stderr(err):
try:
code = answers.main(argv)
except SystemExit as exc: # argparse refuses an option it does not know
code = int(exc.code or 0)
return code, out.getvalue() + err.getvalue()
class Base(unittest.TestCase):
def setUp(self) -> None:
self._tmp = tempfile.TemporaryDirectory()
self.tmp = Path(self._tmp.name).resolve()
self.root = write_root(self.tmp / "copy")
self.set = self.tmp / "set.json"
self.set.write_bytes(dump(make_set(self.root, [q23("d01"), q4("d02")])))
self.run_dir = self.tmp / "run"
def tearDown(self) -> None:
self._tmp.cleanup()
def setup_arm(self, arm: str = "filsok", extra: list[str] | None = None) -> tuple[int, str]:
return run(["setup", "--run", str(self.run_dir), "--name", "r1", "--arm", arm, "--set", str(self.set),
"--sources", str(self.root)] + (extra or []))
class SetupTest(Base):
def test_setup_copies_the_sources_as_the_arm_basis(self) -> None:
code, out = self.setup_arm()
self.assertEqual(code, 0, out)
basis = self.run_dir / "arms" / "filsok"
for path in TEXTS:
self.assertEqual((basis / path).read_bytes(), (self.root / path).read_bytes())
self.assertEqual(sorted(p.relative_to(basis).as_posix() for p in basis.rglob("*") if p.is_file()),
sorted(TEXTS))
def test_one_task_per_question_without_keys_spans_or_counts(self) -> None:
code, out = self.setup_arm()
self.assertEqual(code, 0, out)
basis = self.run_dir / "arms" / "filsok"
for qid in ("d01", "d02"):
task = self.run_dir / "tasks" / "filsok" / f"{qid}.md"
text = task.read_text(encoding="utf-8")
self.assertIn("How do the synthetic notes about bread and gardens fit together?", text)
self.assertIn(str(basis), text)
self.assertIn(str(self.run_dir / "answers" / "filsok" / f"{qid}.json"), text)
for value in list(SPAN.values()) + list(KEY.values()):
self.assertNotIn(value, text)
for word in ("answer_keys", "evidence", "docs-2-3", "docs-4", "history", "four documents"):
self.assertNotIn(word, text)
self.assertIn(f"S1-r1-filsok-{qid}\t{task}", out)
self.assertTrue((self.run_dir / "answers" / "filsok").is_dir())
def test_the_task_asks_for_k1_paths_in_the_citations(self) -> None:
self.setup_arm()
text = (self.run_dir / "tasks" / "filsok" / "d01.md").read_text(encoding="utf-8")
self.assertIn('"path": "kilde/', text)
self.assertIn('"quote"', text)
def test_an_arm_is_set_up_once(self) -> None:
self.assertEqual(self.setup_arm()[0], 0)
code, out = self.setup_arm()
self.assertEqual(code, 1)
self.assertIn("exists", out)
def test_the_bundle_arm_needs_a_bundle_and_gets_the_sources_beside_it(self) -> None:
self.assertEqual(self.setup_arm("bundle")[0], 1)
bundle = self.tmp / "bundle"
bundle.mkdir()
(bundle / "index.md").write_text("# Index\n", encoding="utf-8")
code, out = self.setup_arm("bundle", ["--bundle", str(bundle)])
self.assertEqual(code, 0, out)
basis = self.run_dir / "arms" / "bundle"
self.assertEqual((basis / "wiki" / "index.md").read_text(encoding="utf-8"), "# Index\n")
self.assertEqual((basis / "kilde" / "docs-bread-oven.md").read_bytes(),
(self.root / "kilde" / "docs-bread-oven.md").read_bytes())
text = (self.run_dir / "tasks" / "bundle" / "d01.md").read_text(encoding="utf-8")
self.assertIn("wiki/index.md", text)
self.assertNotIn("Read `BASIS/wiki/index.md` first", text)
search, main, links = (text.index("Search `BASIS/wiki/index.md`"), text.index("Open the main page"),
text.index("follow its links"))
self.assertLess(search, main)
self.assertLess(main, links)
self.assertIn("never read whole", text)
def test_only_the_fixed_arms(self) -> None:
code, out = self.setup_arm("server")
self.assertEqual(code, 1)
self.assertIn("filsok", out)
def test_the_root_must_be_the_one_the_set_names(self) -> None:
other = write_root(self.tmp / "other", frozen_at="2026-10-09T12:00:00Z")
code, out = run(["setup", "--run", str(self.run_dir), "--name", "r1", "--arm", "filsok",
"--set", str(self.set), "--sources", str(other)])
self.assertEqual(code, 1)
self.assertIn("manifest_sha256", out)
def test_a_run_name_with_a_dash_or_space_is_refused(self) -> None:
code, _ = run(["setup", "--run", str(self.run_dir), "--name", "r 1", "--arm", "filsok",
"--set", str(self.set), "--sources", str(self.root)])
self.assertEqual(code, 1)
class PartRootTest(Base):
"""Order 31 A3: an arm searches the whole copy for a set made from a part of it (`--part-root`)."""
PART = (RISING, OVEN, SOIL, WATER)
def make_part(self, copy: Path, change: Callable[[dict[str, Any]], None] | None = None) -> Path:
part = write_root(self.tmp / "part", self.PART, subset_of=root_ref(copy))
if change is not None:
manifest = part / "source-manifest.json"
data = json.loads(manifest.read_bytes())
change(data)
manifest.write_bytes(dump(data))
self.set.write_bytes(dump(make_set(part, [q23("d01"), q4("d02")])))
return part
def setup_part(self, part: Path | None, sources: Path | None = None) -> tuple[int, str]:
extra = ["--part-root", str(part)] if part is not None else []
return run(["setup", "--run", str(self.run_dir), "--name", "r1", "--arm", "filsok", "--set", str(self.set),
"--sources", str(sources or self.root)] + extra)
def test_a_set_of_the_part_is_set_up_on_the_whole_copy(self) -> None:
code, out = self.setup_part(self.make_part(self.root))
self.assertEqual(code, 0, out)
basis = self.run_dir / "arms" / "filsok"
self.assertEqual(sorted(p.relative_to(basis).as_posix() for p in basis.rglob("*") if p.is_file()),
sorted(TEXTS))
def test_a_part_that_is_not_a_subset_of_the_root_is_refused(self) -> None:
def other(data: dict[str, Any]) -> None:
data["subset_of"]["manifest_sha256"] = "0" * 64
code, out = self.setup_part(self.make_part(self.root, other))
self.assertEqual(code, 1)
self.assertIn("subset_of", out)
self.assertFalse((self.run_dir / "arms").exists())
def test_a_part_source_with_another_sha256_than_the_root_is_refused(self) -> None:
def changed(data: dict[str, Any]) -> None:
data["sources"][0]["sha256"] = "0" * 64
code, out = self.setup_part(self.make_part(self.root, changed))
self.assertEqual(code, 1)
self.assertIn("sha256", out)
self.assertFalse((self.run_dir / "arms").exists())
def test_a_part_source_missing_in_the_root_is_refused(self) -> None:
smaller = write_root(self.tmp / "smaller", (RISING, OVEN, SOIL))
code, out = self.setup_part(self.make_part(smaller), smaller)
self.assertEqual(code, 1)
self.assertIn(WATER, out)
self.assertFalse((self.run_dir / "arms").exists())
def test_a_part_root_that_is_not_the_sets_root_is_refused(self) -> None:
part = self.make_part(self.root)
self.set.write_bytes(dump(make_set(self.root, [q23("d01"), q4("d02")])))
code, out = self.setup_part(part)
self.assertEqual(code, 1)
self.assertIn("--part-root", out)
def test_without_part_root_a_set_of_the_part_is_still_refused_on_the_copy(self) -> None:
self.make_part(self.root)
code, out = self.setup_part(None)
self.assertEqual(code, 1)
self.assertIn("manifest_sha256", out)
class CollectTest(Base):
def test_collect_writes_the_k5_answers_file_with_the_id_from_the_file_name(self) -> None:
self.setup_arm()
folder = self.run_dir / "answers" / "filsok"
(folder / "d02.json").write_text(json.dumps({"id": "zz", "answer": "B", "citations": []}), encoding="utf-8")
(folder / "d01.json").write_text(json.dumps({"answer": "A", "citations": [{"path": "kilde/x.md",
"quote": "q"}]}),
encoding="utf-8")
code, out = run(["collect", "--run", str(self.run_dir), "--arm", "filsok"])
self.assertEqual(code, 0, out)
path = self.run_dir / "answers" / "filsok.json"
doc = json.loads(path.read_text(encoding="utf-8"))
self.assertEqual(doc["schema"], "okf-wiki/answers/1")
self.assertEqual(doc["arm"], "filsok")
self.assertEqual([a["id"] for a in doc["answers"]], ["d01", "d02"])
self.assertEqual(doc["answers"][0]["citations"], [{"path": "kilde/x.md", "quote": "q"}])
self.assertEqual(path.read_bytes(), dump(doc))
def test_an_answer_file_that_is_not_a_json_object_is_a_problem(self) -> None:
self.setup_arm()
folder = self.run_dir / "answers" / "filsok"
(folder / "d01.json").write_text("{not json", encoding="utf-8")
(folder / "d02.json").write_text(json.dumps({"answer": "B", "citations": []}), encoding="utf-8")
code, out = run(["collect", "--run", str(self.run_dir), "--arm", "filsok"])
self.assertEqual(code, 1)
self.assertIn("d01.json", out)
doc = json.loads((self.run_dir / "answers" / "filsok.json").read_text(encoding="utf-8"))
self.assertEqual([a["id"] for a in doc["answers"]], ["d02"])
def usage(inp: int, cc: int, cr: int, out: int) -> dict[str, int]:
return {"input_tokens": inp, "cache_creation_input_tokens": cc, "cache_read_input_tokens": cr,
"output_tokens": out}
def tool(name: str, **inp: Any) -> dict[str, Any]:
return {"type": "tool_use", "id": f"t-{name}-{len(inp)}", "name": name, "input": inp}
def assistant(mid: str, use: dict[str, int], blocks: list[dict[str, Any]], stop: str | None = "tool_use") -> dict[str, Any]:
message: dict[str, Any] = {"id": mid, "role": "assistant", "usage": use, "content": blocks}
if stop is not None:
message["stop_reason"] = stop
return {"type": "assistant", "message": message}
def user(text: str, stamp: str = "2026-10-09T10:00:00Z") -> dict[str, Any]:
return {"type": "user", "timestamp": stamp, "message": {"role": "user", "content": text}}
def write_transcript(path: Path, lines: list[dict[str, Any]], description: str | None = None) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text("".join(json.dumps(line) + "\n" for line in lines) + "not json\n", encoding="utf-8")
if description is not None:
meta = path.with_name(path.stem + ".meta.json")
meta.write_text(json.dumps({"agentType": "general-purpose", "description": description}), encoding="utf-8")
class HarvestTest(Base):
def setUp(self) -> None:
super().setUp()
self.assertEqual(self.setup_arm()[0], 0)
self.basis = self.run_dir / "arms" / "filsok"
self.transcripts = self.tmp / "transcripts"
self.subagents = self.transcripts / "session-1" / "subagents"
def task(self, qid: str) -> Path:
return self.run_dir / "tasks" / "filsok" / f"{qid}.md"
def answer(self, qid: str) -> Path:
return self.run_dir / "answers" / "filsok" / f"{qid}.json"
def good_lines(self, qid: str, stamp: str = "2026-10-09T10:00:00Z") -> list[dict[str, Any]]:
return [
user(f"Read {self.task(qid)} and do what it says.", stamp),
assistant("m1", usage(10, 100, 1000, 5), [tool("Read", file_path=str(self.task(qid)))]),
assistant("m1", usage(10, 100, 1000, 7), []),
assistant("m2", usage(3, 0, 1100, 20), [tool("Bash", command=f"cd {self.basis} && grep -rn dough .")]),
assistant("m3", usage(2, 0, 1200, 30), [tool("Write", file_path=str(self.answer(qid)), content="{}")],
"end_turn"),
]
def harvest(self) -> tuple[int, str, dict[str, Any]]:
code, out = run(["harvest", "--run", str(self.run_dir), "--name", "r1", "--arm", "filsok",
"--transcripts", str(self.transcripts)])
path = self.run_dir / "tokens" / "filsok.json"
return code, out, json.loads(path.read_text(encoding="utf-8")) if path.is_file() else {}
def breaches(self, calls: list[dict[str, Any]]) -> list[str]:
lines = self.good_lines("d01")
lines.insert(3, assistant("m9", usage(1, 0, 0, 1), calls))
write_transcript(self.subagents / "agent-a1.jsonl", lines, "S1-r1-filsok-d01")
return list(self.harvest()[2]["questions"][0]["breach_detail"])
def test_tokens_per_question_each_message_once_all_four_fields(self) -> None:
write_transcript(self.subagents / "agent-a1.jsonl", self.good_lines("d01"), "S1-r1-filsok-d01")
write_transcript(self.subagents / "agent-a2.jsonl", self.good_lines("d02"), "S1-r1-filsok-d02")
code, out, doc = self.harvest()
self.assertEqual(code, 0, out)
self.assertEqual(doc["schema"], "okf-wiki/answer-tokens/1")
self.assertEqual(doc["arm"], "filsok")
row = doc["questions"][0]
self.assertEqual(row["id"], "d01")
self.assertEqual((row["input_tokens"], row["cache_creation_input_tokens"], row["cache_read_input_tokens"],
row["output_tokens"]), (15, 100, 3300, 57))
self.assertEqual(row["isolation_breaches"], 0)
self.assertEqual([r["id"] for r in doc["questions"]], ["d01", "d02"])
path = self.run_dir / "tokens" / "filsok.json"
self.assertEqual(path.read_bytes(), dump(doc))
def test_isolation_breaches_are_counted(self) -> None:
lines = self.good_lines("d01")
lines.insert(3, assistant("m9", usage(1, 0, 0, 1), [
tool("Read", file_path="/etc/hosts"),
tool("Bash", command="cat /etc/passwd"),
tool("Grep", pattern="dough", path=str(self.run_dir / "arms" / "bundle")),
tool("Glob", pattern="*.md"),
tool("Read", file_path=str(self.basis / "kilde" / "docs-bread-oven.md")),
tool("Bash", command=f"cd {self.basis} && grep -rn crust ../.."),
tool("WebFetch", url="https://example.invalid/"),
tool("Write", file_path=str(self.basis / "notes.md"), content="x"),
]))
write_transcript(self.subagents / "agent-a1.jsonl", lines, "S1-r1-filsok-d01")
write_transcript(self.subagents / "agent-a2.jsonl", self.good_lines("d02"), "S1-r1-filsok-d02")
code, out, doc = self.harvest()
row = doc["questions"][0]
self.assertEqual(row["isolation_breaches"], 7, row)
self.assertEqual(len(row["breach_detail"]), 7)
self.assertIn("breaches: 7", out)
def test_echo_of_literal_words_in_a_chain_into_the_basis_is_not_a_breach(self) -> None:
lines = self.good_lines("d01")
lines.insert(3, assistant("m9", usage(1, 0, 0, 1), [
tool("Bash", command=f"cd {self.basis} && grep -rn dough kilde && echo --- && grep -rn crust kilde"),
tool("Bash", command=f"cd {self.basis} && grep -rn dough kilde | head -5; echo ----; ls kilde"),
tool("Bash", command=f'cd {self.basis} && echo "next file" && sed -n 1,5p kilde/docs-bread-oven.md'),
]))
write_transcript(self.subagents / "agent-a1.jsonl", lines, "S1-r1-filsok-d01")
code, out, doc = self.harvest()
self.assertEqual(doc["questions"][0]["isolation_breaches"], 0, doc["questions"][0]["breach_detail"])
self.assertNotIn("echo", self.task("d01").read_text(encoding="utf-8"))
def test_echo_outside_the_basis_or_with_an_expansion_is_still_a_breach(self) -> None:
lines = self.good_lines("d01")
lines.insert(3, assistant("m9", usage(1, 0, 0, 1), [
tool("Bash", command=f"cd {self.basis} && echo /outside/*"),
tool("Bash", command=f"cd {self.basis} && echo ~/*"),
tool("Bash", command=f"cd {self.basis} && echo ../../*"),
tool("Bash", command=f"cd {self.basis} && echo x > kilde/notes.md"),
tool("Bash", command=f"cd {self.basis} && echo $HOME"),
tool("Bash", command=f"cd {self.basis} && echo `id`"),
tool("Bash", command="echo --- && grep -rn dough kilde"),
]))
write_transcript(self.subagents / "agent-a1.jsonl", lines, "S1-r1-filsok-d01")
code, out, doc = self.harvest()
self.assertEqual(doc["questions"][0]["isolation_breaches"], 7, doc["questions"][0]["breach_detail"])
def test_stderr_sent_away_right_before_a_separator_is_not_a_breach(self) -> None:
b = self.basis
self.assertEqual(self.breaches([
tool("Bash", command=f"cd {b} && grep -rn dough kilde 2>/dev/null; ls kilde"),
tool("Bash", command=f"cd {b} && grep -rn dough kilde 2>/dev/null| head -3"),
tool("Bash", command=f"cd {b} && grep -rn dough kilde 2>/dev/null&& ls kilde"),
tool("Bash", command=f"cd {b} && grep -rn dough kilde 2>/dev/null|| ls kilde"),
]), [])
def test_other_redirections_before_a_separator_are_still_breaches(self) -> None:
b = self.basis
found = self.breaches([
tool("Bash", command=f"cd {b} && grep -rn dough kilde > kilde/out.md; ls kilde"),
tool("Bash", command=f"cd {b} && grep -rn dough kilde >/dev/null; ls kilde"),
tool("Bash", command=f"cd {b} && grep -rn dough kilde 2>&1; ls kilde"),
tool("Bash", command=f"cd {b} && grep -rn dough kilde 2>/outside/err; ls kilde"),
tool("Bash", command=f"cd {b} && grep -rn dough kilde 2>/dev/nullx; ls kilde"),
])
self.assertEqual(len(found), 5, found)
def test_a_search_pattern_or_sed_script_starting_with_a_slash_is_not_a_path(self) -> None:
b, oven = self.basis, "kilde/docs-bread-oven.md"
self.assertEqual(self.breaches([
tool("Bash", command=f"cd {b} && grep -rn /oven kilde"),
tool("Bash", command=f'cd {b} && grep -rlni "/oven" kilde | head -3'),
tool("Bash", command=f"cd {b} && egrep -n -e /oven -e /crust {oven}"),
tool("Bash", command=f"cd {b} && fgrep -A 3 -m 2 /oven {oven}"),
tool("Bash", command=f"cd {b} && grep -rn -- /oven kilde"),
tool("Bash", command=f"cd {b} && rg --regexp=/oven kilde"),
tool("Bash", command=f"cd {b} && rg -n --regexp /oven -g '*.md' kilde"),
tool("Bash", command=f"cd {b} && sed -n '/^oven:/p' {oven}"),
tool("Bash", command=f"cd {b} && sed -n -e '/^oven:/,/^crust:/p' {oven}"),
]), [])
def test_a_file_path_after_the_pattern_or_in_a_file_option_is_still_a_breach(self) -> None:
b, oven = self.basis, "kilde/docs-bread-oven.md"
found = self.breaches([
tool("Bash", command=f"cd {b} && grep -rn oven /outside/notes.md"),
tool("Bash", command=f"cd {b} && grep -e oven /outside/notes.md"),
tool("Bash", command=f"cd {b} && grep -e /oven /outside/notes.md"),
tool("Bash", command=f"cd {b} && grep -f /outside/words kilde"),
tool("Bash", command=f"cd {b} && grep -rnf /outside/words kilde"),
tool("Bash", command=f"cd {b} && grep --file=/outside/words kilde"),
tool("Bash", command=f"cd {b} && rg -e oven -- /outside/notes.md"),
tool("Bash", command=f"cd {b} && sed -f /outside/script {oven}"),
tool("Bash", command=f"cd {b} && sed -n 1p /outside/notes.md"),
tool("Bash", command=f"cd {b} && sed -n -e 1p -e 2p /outside/notes.md"),
tool("Bash", command=f"cd {b} && grep -rn oven ~/notes"),
tool("Bash", command=f"cd {b} && grep -rn oven ../.."),
])
self.assertEqual(len(found), 12, found)
def test_a_dollar_or_backtick_inside_single_quotes_is_not_an_expansion(self) -> None:
b, oven = self.basis, "kilde/docs-bread-oven.md"
self.assertEqual(self.breaches([
tool("Bash", command=f"cd {b} && sed -n '1,$p' {oven}"),
tool("Bash", command=f"cd {b} && grep -n '`oven`' {oven}"),
tool("Bash", command=f"cd {b} && grep -n '\"$HOME\"' {oven}"),
tool("Bash", command=f"cd {b} && grep -n \"it's\" {oven} | grep -c 'a$(b)'"),
]), [])
def test_a_dollar_or_backtick_outside_single_quotes_is_still_an_expansion(self) -> None:
b, oven = self.basis, "kilde/docs-bread-oven.md"
found = self.breaches([
tool("Bash", command=f'cd {b} && sed -n "1,$p" {oven}'),
tool("Bash", command=f'cd {b} && grep -n "\\`oven\\`" {oven}'),
tool("Bash", command=f"cd {b} && grep -n $HOME {oven}"),
tool("Bash", command=f"cd {b} && grep -n 'a'$HOME'b' {oven}"),
tool("Bash", command=f"cd {b} && grep -n \"'$HOME'\" {oven}"),
tool("Bash", command=f"cd {b} && grep -n `id` {oven}"),
tool("Bash", command=f"cd {b} && grep -n \\'$HOME {oven}"),
])
self.assertEqual(found, ["Bash with an expansion"] * 7, found)
def test_reading_the_own_answer_file_is_not_a_breach(self) -> None:
b, own = self.basis, self.answer("d01")
self.assertEqual(self.breaches([
tool("Read", file_path=str(own)),
tool("Bash", command=f"cd {own.parent} && jq -r '.citations|length' {own.name}"),
tool("Bash", command=f"cd {own.parent}/ && jq -r .id {own}"),
tool("Bash", command=f"jq -e '.citations|length' {own}"),
tool("Bash", command=f"cd {b} && jq . {own} && grep -rn dough kilde"),
tool("Bash", command=f"cd {b} && grep -c quote {own}"),
]), [])
def test_another_file_beside_the_answer_or_another_command_is_still_a_breach(self) -> None:
b, own, other = self.basis, self.answer("d01"), self.answer("d02")
found = self.breaches([
tool("Read", file_path=str(other)),
tool("Bash", command=f"cd {own.parent} && jq . {other.name}"),
tool("Bash", command=f"cd {own.parent} && ls"),
tool("Bash", command=f"cd {own.parent} && jq . {own.name} && cat {other.name}"),
tool("Bash", command=f"cd {own.parent} && jq . *.json"),
tool("Bash", command=f"cd {own.parent} && jq -f filter.jq {own.name}"),
tool("Bash", command=f"cd {own.parent} && jq . ../bundle/d01.json"),
tool("Bash", command=f"jq . {other}"),
tool("Bash", command=f"jq . {own.name}"),
tool("Bash", command=f"jq --slurpfile x {other} . {own}"),
tool("Bash", command=f"jq . {own} > {own}.bak"),
tool("Bash", command=f'cd {b} && python3 -c "import json;json.load(open(\'{own}\'))"'),
tool("Bash", command=f"python3 -m json.tool {own}"),
tool("Bash", command=f"cd {b} && jq . kilde/docs-bread-oven.md"),
tool("Bash", command=f"cd {self.run_dir / 'answers'} && jq . filsok/{own.name}"),
])
self.assertEqual(len(found), 15, found)
def test_other_arms_runs_and_unrelated_agents_are_ignored(self) -> None:
write_transcript(self.subagents / "agent-a1.jsonl", self.good_lines("d01"), "S1-r1-filsok-d01")
write_transcript(self.subagents / "agent-a2.jsonl", self.good_lines("d02"), "S1-r1-filsok-d02")
write_transcript(self.subagents / "agent-b1.jsonl", self.good_lines("d01"), "S1-r1-bundle-d01")
write_transcript(self.subagents / "agent-b2.jsonl", self.good_lines("d01"), "S1-r2-filsok-d01")
write_transcript(self.subagents / "agent-c1.jsonl", self.good_lines("d01"), "V1-r1-read-c001")
code, out, doc = self.harvest()
self.assertEqual(code, 0, out)
self.assertEqual([r["runs"] for r in doc["questions"]], [1, 1])
def test_a_retried_question_sums_its_runs(self) -> None:
write_transcript(self.subagents / "agent-a1.jsonl", self.good_lines("d01"), "S1-r1-filsok-d01")
write_transcript(self.subagents / "agent-a3.jsonl", self.good_lines("d01", "2026-10-09T11:00:00Z"),
"S1-r1-filsok-d01")
write_transcript(self.subagents / "agent-a2.jsonl", self.good_lines("d02"), "S1-r1-filsok-d02")
code, out, doc = self.harvest()
row = doc["questions"][0]
self.assertEqual((row["runs"], row["input_tokens"], row["output_tokens"]), (2, 30, 114))
def test_a_transcript_without_a_description_is_matched_by_its_task_path(self) -> None:
write_transcript(self.subagents / "agent-a1.jsonl", self.good_lines("d01"), "S1-r1-filsok-d01")
write_transcript(self.transcripts / "workflow-x" / "agent-w1.jsonl", self.good_lines("d02"))
code, out, doc = self.harvest()
self.assertEqual(code, 0, out)
self.assertEqual([r["id"] for r in doc["questions"]], ["d01", "d02"])
def test_a_task_without_a_transcript_is_listed_and_exits_1(self) -> None:
write_transcript(self.subagents / "agent-a1.jsonl", self.good_lines("d01"), "S1-r1-filsok-d01")
code, out, doc = self.harvest()
self.assertEqual(code, 1)
self.assertIn("d02", out)
self.assertEqual([r["id"] for r in doc["questions"]], ["d01"])
def test_an_output_never_recorded_is_derived_from_the_next_input_as_a_lower_bound(self) -> None:
lines = [
user(f"Read {self.task('d01')} and do what it says."),
assistant("m1", usage(1, 0, 1000, 2), [tool("Read", file_path=str(self.task("d01")))], None),
assistant("m2", usage(1, 5000, 1000, 10), [], "end_turn"),
]
write_transcript(self.subagents / "agent-a1.jsonl", lines, "S1-r1-filsok-d01")
write_transcript(self.subagents / "agent-a2.jsonl", self.good_lines("d02"), "S1-r1-filsok-d02")
_, _, doc = self.harvest()
row = doc["questions"][0]
self.assertEqual(row["output_tokens"], 12)
self.assertEqual(row["output_tokens_derived"], 4998)
def result(qid: str, group: str, correct: bool, paths: int = 2) -> dict[str, Any]:
return {"id": qid, "group": group, "correct": correct, "keys_held": 2 if correct else 0, "keys": 2,
"paths_cited": paths if correct else 0, "paths_needed": paths}
def results_doc(arm: str, rows: list[dict[str, Any]], set_sha: str = "a" * 64) -> dict[str, Any]:
return {"schema": "okf-wiki/answer-results/1", "arm": arm, "set_sha256": set_sha, "results": rows}
# Ten questions: both right 2, only filsok 1, only bundle 5, neither 2.
PATTERN = [ # (id, group, paths, filsok, bundle)
("d01", "docs-2-3", 2, True, True), ("d02", "docs-2-3", 3, True, True), ("d03", "docs-2-3", 2, True, False),
("d04", "docs-4", 4, False, True), ("d05", "docs-4", 5, False, True), ("d06", "docs-4", 4, False, True),
("d07", "history", 2, False, True), ("d08", "history", 4, False, True), ("d09", "docs-4", 4, False, False),
("d10", "history", 3, False, False),
]
class BashBreachTest(unittest.TestCase):
"""Review findings 8-11 (order 39): option values, sed scripts, comments and newlines in the breach parser."""
basis = Path("/w/arms/filsok")
def breach(self, rest: str) -> str | None:
return answers.bash_breach(f"cd {self.basis} && {rest}", self.basis)
def assertClean(self, rest: str) -> None:
self.assertIsNone(self.breach(rest), rest)
def assertBreach(self, rest: str) -> None:
self.assertIsNotNone(self.breach(rest), rest)
# finding 9: grep -C and long options with a separate value
def test_grep_context_and_long_option_values_are_not_the_search_word(self) -> None:
for rest in ('grep -C 2 "/kilde/docs-x.md" wiki/index.md',
'grep -rn --include "*.md" "/kilde/x" wiki',
'grep -rn --exclude-dir node_modules /kilde/x wiki',
'grep --after-context 3 /kilde/x wiki/index.md',
'grep --max-count 1 -e /kilde/x wiki/index.md',
"rg --glob '*.md' /kilde/x wiki",
"rg -n --max-depth 2 /kilde/x wiki"):
with self.subTest(rest=rest):
self.assertClean(rest)
def test_grep_context_with_a_separate_value_stays_strict(self) -> None:
# BSD grep's --context takes its value only after '=': `--context 1 /x f` makes 1 the word and /x a file.
self.assertBreach("grep --context 1 /outside/notes.md wiki")
self.assertClean("grep --context=1 /kilde/x wiki")
# finding 8: options whose value is a file are checked as paths
def test_a_file_option_value_outside_the_basis_is_a_breach(self) -> None:
for rest in ("grep --exclude-from /outside/list -rn dough kilde",
"grep --exclude-from=/outside/list -rn dough kilde",
"rg --ignore-file /outside/x dough kilde",
"grep -rn --include /outside/x dough kilde",
"rg --pre cat dough kilde",
"rg --pre=cat dough kilde"):
with self.subTest(rest=rest):
self.assertBreach(rest)
# finding 11 and 8: sed scripts that read, write or run, and clustered in-place flags
def test_sed_scripts_that_write_read_outside_or_run_are_breaches(self) -> None:
for rest in ("sed -n '/x/w /tmp/out' wiki/index.md",
"sed '1e touch /tmp/p' wiki/index.md",
"sed -n '1r /outside/notes.md' wiki/index.md",
"sed -n '1R /outside/notes.md' wiki/index.md",
"sed -n 'W out.txt' wiki/index.md",
"sed 's/a/b/w out.txt' wiki/index.md",
"sed 's/a/b/e' wiki/index.md",
"sed -n -e 1p -e '2w out' wiki/index.md",
"sed -n --expression='2w out' wiki/index.md",
"sed -ne'2w out' wiki/index.md",
"sed -n '1{p;w out\n}' wiki/index.md",
"sed -n '1r ../../x' wiki/index.md",
"sed -ni s/a/b/ wiki/index.md",
"sed -Ei s/a/b/ wiki/index.md",
"sed -I '' s/a/b/ wiki/index.md",
"sed -n '1q;}' wiki/index.md"):
with self.subTest(rest=rest):
self.assertBreach(rest)
def test_read_only_sed_scripts_stay_clean(self) -> None:
for rest in ("sed -n '/^oven:/p' kilde/a.md",
"sed -n '/write/,/^## /p' kilde/a.md",
"sed -n '/^## Reading/,/^## /{p;}' kilde/a.md",
"sed -n '10,20p;25q' kilde/a.md",
"sed -n '$=' kilde/a.md",
"sed 's/where/w/g' kilde/a.md",
"sed -E 's#/kilde/(.*)#\\1#' kilde/a.md",
"sed -n '\\,/kilde/,p' kilde/a.md",
"sed -n '/x/I p' kilde/a.md",
"sed -n '1r kilde/b.md' kilde/a.md",
"sed -n -e '/oven/!d' -e p kilde/a.md",
"sed -n 'y/abc/xyz/;p' kilde/a.md"):
with self.subTest(rest=rest):
self.assertClean(rest)
# finding 10: '#' inside a word and a newline between commands
def test_a_hash_inside_a_word_does_not_hide_the_rest_of_the_line(self) -> None:
self.assertEqual("Bash command not allowed: python3", self.breach("grep -c a#b kilde && python3 -c 'print(1)'"))
def test_a_newline_separates_commands(self) -> None:
self.assertEqual("Bash command not allowed: python3", self.breach("grep -c a kilde\npython3 -c 'print(1)'"))
self.assertEqual("Bash command not allowed: python3", self.breach("grep -c a kilde &&\npython3 x"))
self.assertClean("grep -c a kilde\nls kilde\n")
self.assertClean("grep -rn a kilde |\nhead -3")
self.assertClean('grep -c "a\nb" kilde')
class ScoreTest(unittest.TestCase):
def setUp(self) -> None:
self._tmp = tempfile.TemporaryDirectory()
self.tmp = Path(self._tmp.name)
self.a = self.write("filsok", [result(i, g, f, p) for i, g, p, f, _ in PATTERN])
self.b = self.write("bundle", [result(i, g, b, p) for i, g, p, _, b in PATTERN])
def tearDown(self) -> None:
self._tmp.cleanup()
def write(self, arm: str, rows: list[dict[str, Any]], name: str = "", set_sha: str = "a" * 64) -> Path:
path = self.tmp / (name or f"{arm}.json")
path.write_bytes(dump(results_doc(arm, rows, set_sha)))
return path
def score(self, *extra: str) -> tuple[int, str]:
return run(["score", "--results", str(self.a), "--results", str(self.b), *extra])
def test_per_arm_and_group_and_by_paths(self) -> None:
code, out = self.score()
self.assertEqual(code, 0, out)
self.assertIn("filsok: docs-2-3 3 of 3, docs-4 0 of 4, history 0 of 3, all 3 of 10", out)
self.assertIn("bundle: docs-2-3 2 of 3, docs-4 3 of 4, history 2 of 3, all 7 of 10", out)
self.assertIn("filsok by paths: 2-3 3 of 5, 4+ 0 of 5", out)
self.assertIn("bundle by paths: 2-3 3 of 5, 4+ 4 of 5", out)
def test_pairs_mcnemar_difference_and_smallest_visible_difference(self) -> None:
out_path = self.tmp / "score.json"
code, out = self.score("--out", str(out_path))
self.assertEqual(code, 0, out)
self.assertIn("pairs: both 2, only filsok 1, only bundle 5, neither 2", out)
self.assertIn("exact McNemar p = 0.219", out)
doc = json.loads(out_path.read_text(encoding="utf-8"))
self.assertEqual(doc["schema"], "okf-wiki/answer-score/1")
self.assertEqual(doc["pairs"]["all"], {"both": 2, "only filsok": 1, "only bundle": 5, "neither": 2})
self.assertAlmostEqual(doc["mcnemar_p"], 0.21875)
low, high = answers.paired_difference_ci(2, 5, 1, 2)
self.assertAlmostEqual(doc["difference"]["point"], 0.4)
self.assertAlmostEqual(doc["difference"]["low"], low)
self.assertAlmostEqual(doc["difference"]["high"], high)
self.assertAlmostEqual(doc["smallest_visible"]["discordant_share"], 0.6)
self.assertEqual(doc["smallest_visible"]["difference"], answers.min_detectable(10, 0.6))
self.assertIn("bundle - filsok = 0.400, 95 % CI", out)
self.assertIn("smallest difference visible at 80 % power", out)
self.assertEqual(out_path.read_bytes(), dump(doc))
def test_no_line_per_question(self) -> None:
_, out = self.score()
for qid, *_ in PATTERN:
self.assertNotIn(qid, out)
def test_tokens_per_question_are_reported_beside(self) -> None:
tok = {"schema": "okf-wiki/answer-tokens/1", "arm": "filsok", "questions": [
{"id": "d01", "input_tokens": 100, "cache_creation_input_tokens": 200, "cache_read_input_tokens": 600,
"output_tokens": 50, "output_tokens_derived": 50, "isolation_breaches": 1},
{"id": "d02", "input_tokens": 100, "cache_creation_input_tokens": 200, "cache_read_input_tokens": 600,
"output_tokens": 100, "output_tokens_derived": 0, "isolation_breaches": 0}]}
path = self.tmp / "tok-filsok.json"
path.write_bytes(dump(tok))
code, out = self.score("--tokens", str(path))
self.assertEqual(code, 0, out)
self.assertIn("filsok tokens: 2 questions harvested, 1000 per question, isolation breaches 1", out)
def test_two_result_files_for_different_sets_cannot_run(self) -> None:
self.b = self.write("bundle", [result(i, g, b, p) for i, g, p, _, b in PATTERN], set_sha="b" * 64)
self.assertEqual(self.score()[0], 1)
def test_two_result_files_with_different_questions_cannot_run(self) -> None:
self.b = self.write("bundle", [result(i, g, b, p) for i, g, p, _, b in PATTERN[:-1]])
self.assertEqual(self.score()[0], 1)
def test_the_arms_are_filsok_and_bundle(self) -> None:
self.b = self.write("server", [result(i, g, b, p) for i, g, p, _, b in PATTERN])
self.assertEqual(self.score()[0], 1)
def test_a_missing_result_file_cannot_run(self) -> None:
self.b = self.tmp / "none.json"
self.assertEqual(self.score()[0], 1)
class StatisticsTest(unittest.TestCase):
def test_exact_mcnemar_against_known_values(self) -> None:
self.assertAlmostEqual(answers.mcnemar_exact(0, 6), 0.03125)
self.assertAlmostEqual(answers.mcnemar_exact(6, 0), 0.03125)
self.assertAlmostEqual(answers.mcnemar_exact(1, 9), 0.021484375)
self.assertAlmostEqual(answers.mcnemar_exact(2, 0), 0.5)
self.assertEqual(answers.mcnemar_exact(0, 0), 1.0)
self.assertEqual(answers.mcnemar_exact(5, 5), 1.0)
def test_paired_interval_against_newcombe_table_iii_method_10(self) -> None:
for cells, (low, high) in NEWCOMBE_TABLE_III:
got = answers.paired_difference_ci(*cells)
self.assertAlmostEqual(got[0], low, delta=1e-4, msg=str(cells))
self.assertAlmostEqual(got[1], high, delta=1e-4, msg=str(cells))
def test_paired_interval_is_antisymmetric_in_the_discordant_cells(self) -> None:
low, high = answers.paired_difference_ci(20, 12, 2, 16)
back = answers.paired_difference_ci(20, 2, 12, 16)
self.assertAlmostEqual(back[0], -high)
self.assertAlmostEqual(back[1], -low)
def test_smallest_visible_difference_against_the_plans_sample_sizes(self) -> None:
# PLAN.md § Styrke: 37-57 questions see 20 points, 155-234 see 10 points, at 20-30 % discordant pairs.
for n, share, diff in ((37, 0.2, 0.2), (57, 0.3, 0.2), (155, 0.2, 0.1), (234, 0.3, 0.1)):
visible = answers.min_detectable(n, share)
assert visible is not None
self.assertLessEqual(visible, diff, (n, share))
fewer = answers.min_detectable(n - 1, share)
self.assertTrue(fewer is None or fewer > diff, (n - 1, share, fewer))
def test_no_discordant_pairs_has_no_smallest_difference(self) -> None:
self.assertIsNone(answers.min_detectable(60, 0.0))
def test_too_few_pairs_for_any_difference_up_to_the_discordant_share(self) -> None:
# n = 10, psi = 0.6: even d = psi needs (z_a sqrt(.6) + z_b sqrt(.6 - .36))^2 / .36 = 10.35 pairs.
self.assertIsNone(answers.min_detectable(10, 0.6))
visible = answers.min_detectable(11, 0.6)
assert visible is not None
self.assertLessEqual(visible, 0.6)
if __name__ == "__main__":
unittest.main()

View file

@ -0,0 +1,790 @@
"""Tests for gates/answerset.py: the question set for criterion 4 on collection 1 (contract K5 point 1).
check, inside, split, gate and stub on synthetic K1 roots about bread and gardens. No corpus text is used
(CLAUDE.md rule 3).
"""
from __future__ import annotations
import contextlib
import copy
import hashlib
import io
import json
import random
import sys
import tempfile
import unittest
from pathlib import Path
from typing import Any
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
import answerset # noqa: E402
REPO_ROOT = Path(__file__).resolve().parents[2]
RISING = "kilde/docs-bread-rising.md"
OVEN = "kilde/docs-bread-oven.md"
SOIL = "kilde/docs-garden-soil.md"
WATER = "kilde/docs-garden-water.md"
FROST = "kilde/docs-garden-frost.md"
RELEASE = "kilde/release-v1.2.0.md"
TEXTS: dict[str, tuple[str, bool]] = {
RISING: ("# Rising\n\nThe dough rises for 90 minutes at room temperature.\n"
"Use the `--slow-proof` flag for a cold rise overnight.\n", True),
OVEN: ("# Oven\n\nBake the loaf at 230 degrees with the lid on.\nThe crust does not form without steam.\n", True),
SOIL: ("# Soil\n\nGarden beds need compost every spring.\nSet `SOIL_MODE=loam` for heavy clay.\n", True),
WATER: ("# Water\n\nWater the tomatoes twice a week in July.\nThe drip line is called hose-9.\n", True),
FROST: ("# Frost\n\nCover the seedlings when frost is forecast below 2 degrees.\n", True),
RELEASE: ("# 1.2.0\n\n- Added the `--slow-proof` flag for cold rises.\n- The oven timer no longer resets on restart.\n",
False),
}
PART_PATHS = (RISING, OVEN, SOIL, WATER)
SPAN = {
RISING: "The dough rises for 90 minutes at room temperature.",
OVEN: "Bake the loaf at 230 degrees with the lid on.",
SOIL: "Set `SOIL_MODE=loam` for heavy clay.",
WATER: "The drip line is called hose-9.",
FROST: "Cover the seedlings when frost is forecast below 2 degrees.",
RELEASE: "Added the `--slow-proof` flag for cold rises.",
}
KEY = {RISING: "90 minutes", OVEN: "230 degrees", SOIL: "SOIL_MODE=loam", WATER: "hose-9", FROST: "seedlings",
RELEASE: "--slow-proof"}
def dump(data: Any) -> bytes:
return (json.dumps(data, indent=1, sort_keys=True, ensure_ascii=False) + "\n").encode("utf-8")
def sha(data: bytes) -> str:
return hashlib.sha256(data).hexdigest()
def write_root(root: Path, paths: tuple[str, ...] | None = None, frozen_at: str = "2026-10-08T12:00:00Z",
subset_of: dict[str, str] | None = None) -> Path:
"""A K1 root: kilde/ files and source-manifest.json, written as contract K1 says."""
sources = []
for path in sorted(paths if paths is not None else TEXTS):
text, main = TEXTS[path]
data = text.encode("utf-8")
target = root / path
target.parent.mkdir(parents=True, exist_ok=True)
target.write_bytes(data)
kind = "release-notes" if path.startswith("kilde/release-") else "docs"
entry: dict[str, Any] = {"bytes": len(data), "kind": kind, "lines": len(text.splitlines()),
"main_topic": main, "path": path, "sha256": sha(data),
"url": "https://example.invalid/" + path}
if kind == "release-notes":
entry["version"] = "1.2.0"
sources.append(entry)
manifest = {"collection": "bread-and-garden", "frozen_at": frozen_at, "schema": "okf-wiki/source-manifest/1",
"sources": sources, "subset_of": subset_of}
root.mkdir(parents=True, exist_ok=True)
(root / "source-manifest.json").write_bytes(dump(manifest))
return root
def root_ref(root: Path) -> dict[str, str]:
data = (root / "source-manifest.json").read_bytes()
return {"frozen_at": json.loads(data)["frozen_at"], "manifest_sha256": sha(data)}
def question(qid: str, paths: tuple[str, ...], group: str) -> dict[str, Any]:
return {
"id": qid,
"question": "How do the synthetic notes about bread and gardens fit together?",
"group": group,
"evidence": [{"path": p, "span": SPAN[p]} for p in paths],
"answer_keys": [{"path": p, "text": KEY[p]} for p in paths],
}
def q23(qid: str = "q01") -> dict[str, Any]:
return question(qid, (RISING, OVEN), "docs-2-3")
def q4(qid: str = "q02") -> dict[str, Any]:
return question(qid, (RISING, OVEN, SOIL, WATER), "docs-4")
def qhist(qid: str = "q03") -> dict[str, Any]:
return question(qid, (RISING, RELEASE), "history")
def make_set(root: Path, questions: list[dict[str, Any]]) -> dict[str, Any]:
return {"schema": "okf-wiki/answer-set/1", "sources": root_ref(root), "questions": questions}
def run(argv: list[str]) -> tuple[int, str]:
out, err = io.StringIO(), io.StringIO()
with contextlib.redirect_stdout(out), contextlib.redirect_stderr(err):
code = answerset.main(argv)
return code, out.getvalue() + err.getvalue()
class Base(unittest.TestCase):
def setUp(self) -> None:
self._tmp = tempfile.TemporaryDirectory()
self.tmp = Path(self._tmp.name)
self.root = write_root(self.tmp / "copy")
def tearDown(self) -> None:
self._tmp.cleanup()
def write_set(self, questions: list[dict[str, Any]], name: str = "set.json", root: Path | None = None) -> Path:
path = self.tmp / name
path.write_bytes(dump(make_set(root or self.root, questions)))
return path
def check(self, questions: list[dict[str, Any]], root: Path | None = None) -> tuple[int, str]:
return run(["check", "--set", str(self.write_set(questions, root=root)), "--sources", str(root or self.root)])
class CheckTest(Base):
def test_a_good_set_passes_with_the_groups_counted(self) -> None:
code, out = self.check([q23(), q4(), qhist()])
self.assertEqual(code, 0, out)
self.assertIn("questions: 3", out)
self.assertIn("groups: docs-2-3 1, docs-4 1, history 1", out)
self.assertIn("spans verbatim: 8 of 8", out)
self.assertIn("questions with >= 2 distinct paths: 3 of 3", out)
self.assertIn("questions whose group matches K1 and the path count: 3 of 3", out)
self.assertIn("questions with a key in every path's spans: 3 of 3", out)
self.assertIn("answer keys that are wording: 0 of 8", out)
self.assertTrue(out.rstrip().endswith("PASS"), out)
def test_an_empty_set_fails(self) -> None:
code, out = self.check([])
self.assertEqual(code, 1)
self.assertIn("FAIL", out)
def test_a_span_that_does_not_stand_fails(self) -> None:
q = q23()
q["evidence"][0]["span"] = "The dough rises for 80 minutes at room temperature."
code, out = self.check([q, q4()])
self.assertEqual(code, 1)
self.assertIn("spans verbatim: 5 of 6", out)
self.assertIn("FAIL", out)
def test_a_span_must_stand_byte_for_byte(self) -> None:
q = q23()
q["evidence"][0]["span"] = "The dough rises for 90 minutes at room temperature."
code, out = self.check([q])
self.assertEqual(code, 1)
self.assertIn("spans verbatim: 1 of 2", out)
def test_a_question_with_one_path_fails(self) -> None:
q = question("q09", (RISING,), "docs-2-3")
code, out = self.check([q23(), q])
self.assertEqual(code, 1)
self.assertIn("questions with >= 2 distinct paths: 1 of 2", out)
def test_two_spans_in_one_file_are_one_path(self) -> None:
q = q23()
q["evidence"] = [{"path": RISING, "span": SPAN[RISING]},
{"path": RISING, "span": "Use the `--slow-proof` flag for a cold rise overnight."}]
q["answer_keys"] = [{"path": RISING, "text": "90 minutes"}]
code, out = self.check([q])
self.assertEqual(code, 1)
self.assertIn("questions with >= 2 distinct paths: 0 of 1", out)
def test_history_is_set_by_a_source_without_a_main_topic(self) -> None:
q = qhist()
q["group"] = "docs-2-3"
code, out = self.check([q])
self.assertEqual(code, 1)
self.assertIn("questions whose group matches K1 and the path count: 0 of 1", out)
def test_four_paths_are_docs_4(self) -> None:
q = q4()
q["group"] = "docs-2-3"
code, out = self.check([q])
self.assertEqual(code, 1)
self.assertIn("questions whose group matches K1 and the path count: 0 of 1", out)
def test_three_paths_are_docs_2_3(self) -> None:
q = question("q05", (RISING, OVEN, SOIL), "docs-4")
self.assertEqual(self.check([q])[0], 1)
q["group"] = "docs-2-3"
self.assertEqual(self.check([q])[0], 0)
def test_an_unknown_group_fails(self) -> None:
q = q23()
q["group"] = "lookup"
self.assertEqual(self.check([q])[0], 1)
def test_a_wording_key_fails(self) -> None:
q = q23()
q["answer_keys"][0]["text"] = "rises for 90 minutes"
code, out = self.check([q])
self.assertEqual(code, 1)
self.assertIn("answer keys that are wording: 1 of 2", out)
def test_a_negated_key_must_stand_negated(self) -> None:
q = q23()
q["answer_keys"][0]["negated"] = True
code, out = self.check([q])
self.assertEqual(code, 1)
self.assertIn("negated keys negated in their span: 0 of 1", out)
def test_a_negated_key_that_stands_negated_passes(self) -> None:
q = q23()
q["evidence"][1]["span"] = "The crust does not form without steam."
q["answer_keys"][1] = {"path": OVEN, "text": "form", "negated": True}
code, out = self.check([q])
self.assertEqual(code, 0, out)
self.assertIn("negated keys negated in their span: 1 of 1", out)
def test_a_path_without_a_key_in_its_span_fails(self) -> None:
q = q23()
q["answer_keys"] = [{"path": RISING, "text": "90 minutes"}]
code, out = self.check([q])
self.assertEqual(code, 1)
self.assertIn("questions with a key in every path's spans: 0 of 1", out)
def test_a_key_that_stands_only_in_another_path_fails(self) -> None:
q = q23()
q["answer_keys"][1] = {"path": OVEN, "text": "90 minutes"}
code, out = self.check([q])
self.assertEqual(code, 1)
self.assertIn("questions with a key in every path's spans: 0 of 1", out)
def test_a_path_that_k1_does_not_list_fails(self) -> None:
q = q23()
q["evidence"][1]["path"] = "kilde/docs-bread-crumb.md"
q["answer_keys"][1]["path"] = "kilde/docs-bread-crumb.md"
code, out = self.check([q])
self.assertEqual(code, 1)
self.assertIn("error:", out)
def test_a_file_that_differs_from_k1_fails(self) -> None:
(self.root / OVEN).write_text(TEXTS[OVEN][0] + "Extra line.\n", encoding="utf-8")
code, out = self.check([q23()])
self.assertEqual(code, 1)
self.assertIn("sha256", out)
def test_the_set_must_name_the_root_it_was_made_from(self) -> None:
other = write_root(self.tmp / "other", frozen_at="2026-10-09T12:00:00Z")
path = self.write_set([q23()], root=other)
code, out = run(["check", "--set", str(path), "--sources", str(self.root)])
self.assertEqual(code, 1)
self.assertIn("manifest_sha256", out)
def test_a_root_without_k1_cannot_run(self) -> None:
path = self.write_set([q23()])
(self.root / "source-manifest.json").unlink()
code, out = run(["check", "--set", str(path), "--sources", str(self.root)])
self.assertEqual(code, 1)
self.assertIn("FAIL", out)
def test_a_wrong_schema_cannot_run(self) -> None:
data = make_set(self.root, [q23()])
data["schema"] = "okf-wiki/multidoc-set/1"
path = self.tmp / "wrong.json"
path.write_bytes(dump(data))
self.assertEqual(run(["check", "--set", str(path), "--sources", str(self.root)])[0], 1)
def test_duplicate_ids_fail(self) -> None:
self.assertEqual(self.check([q23("q01"), q4("q01")])[0], 1)
class InsideTest(Base):
def setUp(self) -> None:
super().setUp()
ref = root_ref(self.root)
self.part = write_root(self.tmp / "part", PART_PATHS, frozen_at="2026-10-08T19:00:00Z", subset_of=ref)
def test_counts_the_questions_with_every_span_in_the_part_root(self) -> None:
path = self.write_set([q23(), q4(), qhist()])
code, out = run(["inside", "--set", str(path), "--part-root", str(self.part)])
self.assertEqual(code, 0, out)
self.assertIn("questions with every span in the part root: 2 of 3", out)
def test_require_all_fails_unless_every_question_is_inside(self) -> None:
path = self.write_set([q23(), qhist()])
code, out = run(["inside", "--set", str(path), "--part-root", str(self.part), "--require-all"])
self.assertEqual(code, 1)
self.assertIn("1 of 2", out)
inside = self.write_set([q23(), q4()], name="inside.json", root=self.part)
code, out = run(["inside", "--set", str(inside), "--part-root", str(self.part), "--require-all"])
self.assertEqual(code, 0, out)
self.assertIn("2 of 2", out)
def test_a_span_that_is_not_in_the_part_file_is_outside(self) -> None:
(self.part / OVEN).write_text("# Oven\n\nSomething else entirely.\n", encoding="utf-8")
path = self.write_set([q23()])
code, out = run(["inside", "--set", str(path), "--part-root", str(self.part)])
self.assertIn("0 of 1", out)
class SplitAndGateBase(Base):
"""Eight questions (4 docs-2-3, 2 docs-4, 2 history), a part of two, split with seed 7, 4 held out."""
def setUp(self) -> None:
super().setUp()
self.part_root = write_root(self.tmp / "part", PART_PATHS, frozen_at="2026-10-08T19:00:00Z",
subset_of=root_ref(self.root))
questions = ([q23(f"q{i:02d}") for i in (1, 2, 3, 4)] + [q4(f"q{i:02d}") for i in (5, 6)]
+ [qhist(f"q{i:02d}") for i in (7, 8)])
for n, q in enumerate(questions):
q["question"] = f"Synthetic question number {n} about bread and gardens?"
self.all = self.write_set(questions, name="all.json")
self.part = self.write_set([q23("p01"), q4("p02")], name="part.json", root=self.part_root)
self.out = self.tmp / "out"
def split(self, held: Path, seed: int = 7, out: Path | None = None) -> tuple[int, str]:
out = out or self.out
return run(["split", "--set", str(self.all), "--sources", str(self.root), "--seed", str(seed),
"--held-out-count", "4", "--set-out", str(out / "set.json"), "--set-label", "set.json",
"--held-out", str(held), "--part", str(self.part), "--part-sources", str(self.part_root),
"--part-label", "part.json", "--manifest", str(out / "manifest.json")])
class SplitTest(SplitAndGateBase):
def test_split_is_stratified_by_group_and_renumbers_both_parts(self) -> None:
held = self.tmp / "outside" / "held-out.json"
code, out = self.split(held)
self.assertEqual(code, 0, out)
dev = json.loads((self.out / "set.json").read_text(encoding="utf-8"))
kept = json.loads(held.read_text(encoding="utf-8"))
self.assertEqual([q["id"] for q in dev["questions"]], ["d01", "d02", "d03", "d04"])
self.assertEqual([q["id"] for q in kept["questions"]], ["h01", "h02", "h03", "h04"])
groups = sorted(q["group"] for q in kept["questions"])
self.assertEqual(groups, ["docs-2-3", "docs-2-3", "docs-4", "history"])
self.assertEqual(dev["sources"], kept["sources"])
self.assertEqual(dev["schema"], "okf-wiki/answer-set/1")
def test_the_manifest_holds_seed_counts_and_sha256_only(self) -> None:
held = self.tmp / "outside" / "held-out.json"
self.assertEqual(self.split(held)[0], 0)
manifest = json.loads((self.out / "manifest.json").read_text(encoding="utf-8"))
self.assertEqual(manifest["schema"], "okf-wiki/answer-set-manifest/1")
self.assertEqual(manifest["seed"], 7)
self.assertEqual(manifest["set"]["sha256"], sha((self.out / "set.json").read_bytes()))
self.assertEqual(manifest["set"]["file"], "set.json")
self.assertEqual(manifest["held_out"]["sha256"], sha(held.read_bytes()))
self.assertEqual(manifest["part"]["sha256"], sha(self.part.read_bytes()))
self.assertEqual(manifest["part"]["file"], "part.json")
self.assertEqual(manifest["groups"]["held_out"], {"docs-2-3": 2, "docs-4": 1, "history": 1})
self.assertEqual(manifest["groups"]["set"], {"docs-2-3": 2, "docs-4": 1, "history": 1})
self.assertEqual(manifest["groups"]["part"], {"docs-2-3": 1, "docs-4": 1, "history": 0})
self.assertEqual(manifest["sources"]["copy"], root_ref(self.root))
self.assertEqual(manifest["sources"]["part"], root_ref(self.part_root))
text = (self.out / "manifest.json").read_text(encoding="utf-8")
self.assertNotIn("bread", text)
self.assertNotIn(SPAN[RISING], text)
def test_two_runs_with_one_seed_are_byte_identical(self) -> None:
first, second = self.tmp / "a" / "held.json", self.tmp / "b" / "held.json"
self.assertEqual(self.split(first, out=self.tmp / "o1")[0], 0)
self.assertEqual(self.split(second, out=self.tmp / "o2")[0], 0)
self.assertEqual(first.read_bytes(), second.read_bytes())
self.assertEqual((self.tmp / "o1" / "set.json").read_bytes(), (self.tmp / "o2" / "set.json").read_bytes())
def test_split_refuses_to_overwrite_the_held_out_file(self) -> None:
held = self.tmp / "outside" / "held-out.json"
self.assertEqual(self.split(held)[0], 0)
before = held.read_bytes()
code, out = self.split(held, seed=8, out=self.tmp / "again")
self.assertEqual(code, 1)
self.assertIn("exists", out)
self.assertEqual(held.read_bytes(), before)
def test_split_refuses_a_held_out_path_in_the_repository(self) -> None:
held = REPO_ROOT / "gates-local" / "answerset" / "test-held-out-never-written.json"
code, out = self.split(held)
self.assertEqual(code, 1)
self.assertIn("inside the repository", out)
self.assertFalse(held.exists())
def test_split_refuses_a_set_that_fails_the_check(self) -> None:
data = json.loads(self.all.read_text(encoding="utf-8"))
data["questions"][0]["evidence"][0]["span"] = "No such sentence stands in any synthetic file."
self.all.write_bytes(dump(data))
held = self.tmp / "outside" / "held-out.json"
self.assertEqual(self.split(held)[0], 1)
self.assertFalse(held.exists())
def test_split_refuses_a_part_with_a_span_outside_its_root(self) -> None:
self.part.write_bytes(dump(make_set(self.part_root, [q23("p01"), qhist("p02")])))
held = self.tmp / "outside" / "held-out.json"
self.assertEqual(self.split(held)[0], 1)
self.assertFalse(held.exists())
def fasit(q: dict[str, Any]) -> dict[str, Any]:
text = " ".join(("not " if k.get("negated") else "") + k["text"] + "." for k in q["answer_keys"])
return {"id": q["id"], "answer": text, "citations": [{"path": e["path"], "quote": e["span"]} for e in q["evidence"]]}
class GateBase(SplitAndGateBase):
def setUp(self) -> None:
super().setUp()
self.held = self.tmp / "outside" / "held-out.json"
code, out = self.split(self.held)
self.assertEqual(code, 0, out)
self.set = self.out / "set.json"
self.manifest = self.out / "manifest.json"
self.questions: list[dict[str, Any]] = json.loads(self.set.read_text(encoding="utf-8"))["questions"]
def answers(self, answers: list[dict[str, Any]], arm: str = "bundle", name: str = "answers.json") -> Path:
path = self.tmp / name
path.write_bytes(dump({"schema": "okf-wiki/answers/1", "arm": arm, "answers": answers}))
return path
def gate(self, answers: Path, arm: str = "bundle", set_path: Path | None = None, root: Path | None = None,
out: Path | None = None) -> tuple[int, str]:
argv = ["gate", "--set", str(set_path or self.set), "--manifest", str(self.manifest),
"--sources", str(root or self.root), "--arm", arm, "--answers", str(answers)]
return run(argv + (["--out", str(out)] if out else []))
class GateTest(GateBase):
def test_a_missing_answers_file_scores_0_red_exit_1(self) -> None:
code, out = self.gate(self.tmp / "none.json")
self.assertEqual(code, 1)
self.assertIn("correct answer and source: 0 of 4", out)
self.assertIn("fail-closed", out)
self.assertIn("RED", out)
def test_the_fasit_scores_n_of_n_green(self) -> None:
code, out = self.gate(self.answers([fasit(q) for q in self.questions]))
self.assertEqual(code, 0, out)
self.assertIn("correct answer and source: 4 of 4", out)
self.assertIn("GREEN", out)
def test_right_answers_without_citations_score_0(self) -> None:
answers = [dict(fasit(q), citations=[]) for q in self.questions]
code, out = self.gate(self.answers(answers))
self.assertEqual(code, 0, out)
self.assertIn("correct answer: 4 of 4", out)
self.assertIn("correct answer and source: 0 of 4", out)
self.assertIn("RED", out)
def test_a_citation_of_only_some_needed_paths_is_wrong(self) -> None:
answers = [fasit(q) for q in self.questions]
answers[0]["citations"] = answers[0]["citations"][:1]
code, out = self.gate(self.answers(answers))
self.assertIn("correct answer and source: 3 of 4", out)
def test_a_quote_counts_with_whitespace_collapsed_and_a_leading_slash(self) -> None:
answers = [fasit(q) for q in self.questions]
for c in answers[0]["citations"]:
c["path"] = "/" + c["path"]
c["quote"] = " " + c["quote"].replace(" ", "\n ") + " "
self.assertIn("correct answer and source: 4 of 4", self.gate(self.answers(answers))[1])
def test_a_quote_that_does_not_stand_is_wrong(self) -> None:
answers = [fasit(q) for q in self.questions]
answers[0]["citations"][0]["quote"] = "The dough rises for 80 minutes."
self.assertIn("correct answer and source: 3 of 4", self.gate(self.answers(answers))[1])
def test_a_missing_answer_counts_as_wrong(self) -> None:
code, out = self.gate(self.answers([fasit(q) for q in self.questions[1:]]))
self.assertEqual(code, 1)
self.assertIn("missing answers: 1", out)
self.assertIn("correct answer and source: 3 of 4", out)
def test_answers_for_another_arm_cannot_run(self) -> None:
path = self.answers([fasit(q) for q in self.questions], arm="filsok")
self.assertEqual(self.gate(path, arm="bundle")[0], 1)
def test_an_unknown_arm_cannot_run(self) -> None:
path = self.answers([fasit(q) for q in self.questions], arm="server")
self.assertEqual(self.gate(path, arm="server")[0], 1)
def test_a_set_whose_sha256_is_not_in_the_manifest_cannot_run(self) -> None:
changed = self.tmp / "changed.json"
changed.write_bytes(self.set.read_bytes() + b"\n")
code, out = self.gate(self.answers([fasit(q) for q in self.questions]), set_path=changed)
self.assertEqual(code, 1)
self.assertIn("sha256", out)
def test_a_root_whose_manifest_differs_cannot_run(self) -> None:
other = write_root(self.tmp / "other", frozen_at="2026-10-09T12:00:00Z")
code, out = self.gate(self.answers([fasit(q) for q in self.questions]), root=other)
self.assertEqual(code, 1)
self.assertIn("sha256", out)
def test_the_part_is_scored_against_the_part_root_only(self) -> None:
part_q = json.loads(self.part.read_text(encoding="utf-8"))["questions"]
answers = self.answers([fasit(q) for q in part_q])
code, out = self.gate(answers, set_path=self.part, root=self.part_root)
self.assertEqual(code, 0, out)
self.assertIn("set: part, 2 questions", out)
self.assertEqual(self.gate(answers, set_path=self.part, root=self.root)[0], 1)
def test_a_negated_key_needs_the_negation_in_the_answer(self) -> None:
data = json.loads(self.all.read_text(encoding="utf-8"))
for q in data["questions"]:
if q["group"] == "docs-2-3":
q["evidence"][1]["span"] = "The crust does not form without steam."
q["answer_keys"][1] = {"path": OVEN, "text": "form", "negated": True}
self.all.write_bytes(dump(data))
held = self.tmp / "outside2" / "held.json"
out_dir = self.tmp / "neg"
self.assertEqual(self.split(held, out=out_dir)[0], 0)
self.manifest = out_dir / "manifest.json"
questions = json.loads((out_dir / "set.json").read_text(encoding="utf-8"))["questions"]
answers = [fasit(x) for x in questions]
good = self.gate(self.answers(answers), set_path=out_dir / "set.json")[1]
self.assertIn("correct answer and source: 4 of 4", good)
for a, x in zip(answers, questions):
a["answer"] = " ".join(k["text"] + "." for k in x["answer_keys"])
plain = self.gate(self.answers(answers), set_path=out_dir / "set.json")[1]
self.assertEqual(sum(1 for x in questions if x["group"] == "docs-2-3"), 2)
self.assertIn("correct answer and source: 2 of 4", plain)
def test_out_writes_the_k5_results_sorted_and_byte_identical(self) -> None:
answers = [fasit(q) for q in self.questions]
answers[1]["citations"] = answers[1]["citations"][:1]
answers[2]["answer"] = "Nothing useful."
first, second = self.tmp / "r1.json", self.tmp / "r2.json"
self.gate(self.answers(list(reversed(answers))), out=first)
self.gate(self.answers(answers), out=second)
self.assertEqual(first.read_bytes(), second.read_bytes())
doc = json.loads(first.read_text(encoding="utf-8"))
self.assertEqual(doc["schema"], "okf-wiki/answer-results/1")
self.assertEqual(doc["arm"], "bundle")
self.assertEqual(doc["set_sha256"], sha(self.set.read_bytes()))
ids = [r["id"] for r in doc["results"]]
self.assertEqual(ids, sorted(ids))
by_id = {r["id"]: r for r in doc["results"]}
q1, q2 = self.questions[1], self.questions[2]
r1, r2 = by_id[q1["id"]], by_id[q2["id"]]
self.assertEqual(set(r1), {"id", "group", "correct", "keys_held", "keys", "paths_cited", "paths_needed"})
self.assertEqual((r1["correct"], r1["paths_cited"]), (False, 1))
self.assertEqual(r1["paths_needed"], len({e["path"] for e in q1["evidence"]}))
self.assertEqual((r2["correct"], r2["keys_held"], r2["keys"]), (False, 0, len(q2["answer_keys"])))
self.assertEqual(r1["group"], q1["group"])
self.assertEqual(first.read_bytes(), dump(doc))
class StubTest(GateBase):
def test_stub_fasit_scores_n_of_n_and_stub_without_citations_0(self) -> None:
fas, bare = self.tmp / "fasit.json", self.tmp / "bare.json"
self.assertEqual(run(["stub", "--set", str(self.set), "--arm", "bundle", "--kind", "fasit",
"--out", str(fas)])[0], 0)
self.assertEqual(run(["stub", "--set", str(self.set), "--arm", "bundle", "--kind", "no-citations",
"--out", str(bare)])[0], 0)
self.assertIn("correct answer and source: 4 of 4", self.gate(fas)[1])
code, out = self.gate(bare)
self.assertEqual(code, 0)
self.assertIn("correct answer: 4 of 4", out)
self.assertIn("correct answer and source: 0 of 4", out)
def test_stub_answers_are_k5_answers(self) -> None:
fas = self.tmp / "fasit.json"
run(["stub", "--set", str(self.set), "--arm", "filsok", "--kind", "fasit", "--out", str(fas)])
doc = json.loads(fas.read_text(encoding="utf-8"))
self.assertEqual((doc["schema"], doc["arm"]), ("okf-wiki/answers/1", "filsok"))
self.assertEqual(set(doc["answers"][0]["citations"][0]), {"path", "quote"})
HARD_PATHS = (RISING, OVEN, SOIL, WATER, FROST)
def q3(qid: str) -> dict[str, Any]:
return question(qid, (RISING, OVEN, FROST), "docs-2-3")
def hard_questions(three: int, four: int, two: int = 0) -> list[dict[str, Any]]:
"""`three` questions with 3 paths, `four` with 4 and `two` with 2, each with its own wording, ids k01.."""
questions = [q3("") for _ in range(three)] + [q4("") for _ in range(four)] + [q23("") for _ in range(two)]
for n, q in enumerate(questions, 1):
q["id"] = f"k{n:02d}"
q["question"] = f"Synthetic hard question number {n} about bread and gardens?"
return questions
class HardSetBase(Base):
"""The harder set (order 21): every question needs at least 3 documents and at least 20 of 30 need 4 or more.
Its root is a K1 subset root of the copy, as the first part is."""
def setUp(self) -> None:
super().setUp()
self.hard_root = write_root(self.tmp / "hard-root", HARD_PATHS, frozen_at="2026-10-08T19:00:00Z",
subset_of=root_ref(self.root))
self.out = self.tmp / "frozen"
def hard_set(self, questions: list[dict[str, Any]], name: str = "drafts.json") -> Path:
return self.write_set(questions, name=name, root=self.hard_root)
def check_hard(self, path: Path, min_paths: int = 3, min_many: int = 20) -> tuple[int, str]:
return run(["check", "--set", str(path), "--sources", str(self.hard_root), "--min-paths", str(min_paths),
"--min-many", str(min_many)])
def freeze(self, path: Path, seed: int = 11, out: Path | None = None, count: int = 30,
min_many: int = 20) -> tuple[int, str]:
out = out or self.out
return run(["freeze", "--set", str(path), "--sources", str(self.hard_root), "--seed", str(seed),
"--count", str(count), "--min-paths", "3", "--min-many", str(min_many), "--prefix", "x",
"--set-out", str(out / "hard.json"), "--set-label", "hard.json",
"--manifest", str(out / "hard.manifest.json")])
class HardCheckTest(HardSetBase):
def test_ten_with_three_and_twenty_with_four_pass_the_thresholds(self) -> None:
code, out = self.check_hard(self.hard_set(hard_questions(10, 20)))
self.assertEqual(code, 0, out)
self.assertIn("questions with >= 3 distinct paths: 30 of 30", out)
self.assertIn("questions with >= 4 distinct paths: 20 of 30 (required >= 20)", out)
self.assertIn("groups: docs-2-3 10, docs-4 20, history 0", out)
self.assertIn("PASS", out)
def test_a_question_with_two_paths_fails_min_paths_3(self) -> None:
path = self.hard_set(hard_questions(9, 20, two=1))
code, out = self.check_hard(path)
self.assertEqual(code, 1)
self.assertIn("questions with >= 3 distinct paths: 29 of 30", out)
self.assertIn("FAIL", out)
code, out = run(["check", "--set", str(path), "--sources", str(self.hard_root)])
self.assertEqual(code, 0, out)
self.assertIn("questions with >= 2 distinct paths: 30 of 30", out)
def test_nineteen_of_thirty_with_four_fail_min_many_20(self) -> None:
code, out = self.check_hard(self.hard_set(hard_questions(11, 19)))
self.assertEqual(code, 1)
self.assertIn("questions with >= 4 distinct paths: 19 of 30 (required >= 20)", out)
self.assertIn("FAIL", out)
def test_the_default_check_prints_no_many_line(self) -> None:
code, out = run(["check", "--set", str(self.hard_set(hard_questions(10, 20))), "--sources",
str(self.hard_root)])
self.assertEqual(code, 0, out)
self.assertNotIn("required >=", out)
class FreezeTest(HardSetBase):
def test_freeze_writes_the_set_and_a_manifest_with_seed_counts_sha256_and_roots_only(self) -> None:
code, out = self.freeze(self.hard_set(hard_questions(10, 20)))
self.assertEqual(code, 0, out)
frozen = (self.out / "hard.json").read_bytes()
manifest = json.loads((self.out / "hard.manifest.json").read_text(encoding="utf-8"))
self.assertEqual(manifest["schema"], "okf-wiki/answer-set-manifest/1")
self.assertEqual(manifest["seed"], 11)
self.assertEqual(manifest["rules"], {"min_paths": 3, "many_paths": 4, "min_many": 20})
self.assertEqual(manifest["part"], {"file": "hard.json", "sha256": sha(frozen), "questions": 30})
self.assertEqual(manifest["groups"], {"part": {"docs-2-3": 10, "docs-4": 20, "history": 0}})
self.assertEqual(manifest["sources"], {"part": root_ref(self.hard_root), "copy": root_ref(self.root)})
self.assertNotIn("set", manifest)
self.assertNotIn("held_out", manifest)
text = (self.out / "hard.manifest.json").read_text(encoding="utf-8")
self.assertNotIn("bread", text)
self.assertNotIn(SPAN[RISING], text)
data = json.loads(frozen)
self.assertEqual(data["sources"], root_ref(self.hard_root))
self.assertEqual(data["schema"], "okf-wiki/answer-set/1")
def test_freeze_renumbers_in_a_seeded_order_and_is_byte_identical(self) -> None:
drafts = self.hard_set(hard_questions(10, 20))
self.assertEqual(self.freeze(drafts, out=self.tmp / "o1")[0], 0)
self.assertEqual(self.freeze(drafts, out=self.tmp / "o2")[0], 0)
first = (self.tmp / "o1" / "hard.json").read_bytes()
self.assertEqual(first, (self.tmp / "o2" / "hard.json").read_bytes())
questions = json.loads(first)["questions"]
self.assertEqual([q["id"] for q in questions], [f"x{n:02d}" for n in range(1, 31)])
order = sorted(f"k{n:02d}" for n in range(1, 31))
random.Random(11).shuffle(order)
wording = {f"k{n:02d}": f"Synthetic hard question number {n} about bread and gardens?" for n in range(1, 31)}
self.assertEqual([q["question"] for q in questions], [wording[i] for i in order])
self.assertEqual(self.freeze(drafts, seed=12, out=self.tmp / "o3")[0], 0)
other = json.loads((self.tmp / "o3" / "hard.json").read_text(encoding="utf-8"))["questions"]
self.assertNotEqual([q["question"] for q in other], [q["question"] for q in questions])
self.assertEqual(sorted(q["question"] for q in other), sorted(q["question"] for q in questions))
def test_freeze_refuses_a_set_that_fails_the_thresholds(self) -> None:
code, out = self.freeze(self.hard_set(hard_questions(11, 19)))
self.assertEqual(code, 1)
self.assertIn("19 of 30", out)
self.assertFalse((self.out / "hard.json").exists())
self.assertFalse((self.out / "hard.manifest.json").exists())
def test_freeze_refuses_a_set_with_another_count(self) -> None:
code, out = self.freeze(self.hard_set(hard_questions(9, 20)))
self.assertEqual(code, 1)
self.assertIn("29", out)
self.assertFalse((self.out / "hard.json").exists())
def test_freeze_refuses_a_set_that_fails_the_check(self) -> None:
questions = hard_questions(10, 20)
questions[0]["evidence"][0]["span"] = "No such sentence stands in any synthetic file."
self.assertEqual(self.freeze(self.hard_set(questions))[0], 1)
self.assertFalse((self.out / "hard.manifest.json").exists())
def test_freeze_never_overwrites_a_frozen_set_or_manifest(self) -> None:
drafts = self.hard_set(hard_questions(10, 20))
self.assertEqual(self.freeze(drafts)[0], 0)
before = ((self.out / "hard.json").read_bytes(), (self.out / "hard.manifest.json").read_bytes())
code, out = self.freeze(drafts, seed=12)
self.assertEqual(code, 1)
self.assertIn("exists", out)
self.assertEqual(before, ((self.out / "hard.json").read_bytes(), (self.out / "hard.manifest.json").read_bytes()))
(self.out / "hard.manifest.json").unlink()
code, out = self.freeze(drafts, seed=12)
self.assertEqual(code, 1)
self.assertIn("exists", out)
self.assertEqual((self.out / "hard.json").read_bytes(), before[0])
class HardGateTest(HardSetBase):
def setUp(self) -> None:
super().setUp()
code, out = self.freeze(self.hard_set(hard_questions(10, 20)))
self.assertEqual(code, 0, out)
self.set = self.out / "hard.json"
self.manifest = self.out / "hard.manifest.json"
def gate(self, answers: Path, root: Path | None = None) -> tuple[int, str]:
return run(["gate", "--set", str(self.set), "--manifest", str(self.manifest), "--sources",
str(root or self.hard_root), "--arm", "filsok", "--answers", str(answers)])
def stub(self, kind: str) -> Path:
path = self.tmp / f"{kind}.json"
self.assertEqual(run(["stub", "--set", str(self.set), "--arm", "filsok", "--kind", kind,
"--out", str(path)])[0], 0)
return path
def test_the_fasit_is_green_and_no_citations_red_on_the_frozen_set(self) -> None:
code, out = self.gate(self.stub("fasit"))
self.assertEqual(code, 0, out)
self.assertIn("set: part, 30 questions", out)
self.assertIn("correct answer and source: 30 of 30", out)
self.assertIn("GREEN", out)
code, out = self.gate(self.stub("no-citations"))
self.assertEqual(code, 0, out)
self.assertIn("correct answer: 30 of 30", out)
self.assertIn("correct answer and source: 0 of 30", out)
self.assertIn("RED", out)
def test_no_answers_score_0_of_30_red_exit_1(self) -> None:
code, out = self.gate(self.tmp / "none.json")
self.assertEqual(code, 1)
self.assertIn("correct answer and source: 0 of 30", out)
self.assertIn("RED", out)
def test_the_frozen_set_is_judged_against_its_own_root_only(self) -> None:
code, out = self.gate(self.stub("fasit"), root=self.root)
self.assertEqual(code, 1)
self.assertIn("sha256", out)
class DumpTest(unittest.TestCase):
def test_dump_is_the_k5_form(self) -> None:
data = {"b": [1, {"z": "æ", "a": None}], "a": True}
self.assertEqual(answerset.dump(data), dump(data))
def test_group_of_follows_k1_and_the_path_count(self) -> None:
main = {RISING: True, OVEN: True, SOIL: True, WATER: True, RELEASE: False}
self.assertEqual(answerset.group_of([RISING, OVEN], main), "docs-2-3")
self.assertEqual(answerset.group_of([RISING, OVEN, SOIL], main), "docs-2-3")
self.assertEqual(answerset.group_of([RISING, OVEN, SOIL, WATER], main), "docs-4")
self.assertEqual(answerset.group_of([RISING, RELEASE], main), "history")
self.assertEqual(answerset.group_of([RISING, OVEN, SOIL, RELEASE], main), "history")
if __name__ == "__main__":
unittest.main()

View file

@ -0,0 +1,938 @@
"""Tests for gates/calibration.py: the reader-calibration set checker, the seeded build with its call layout (K3), the
stubs and the verdict gate.
Every fixture is synthetic text written here (widgets and garden beds). No corpus text is used (CLAUDE.md rule 3).
"""
from __future__ import annotations
import contextlib
import hashlib
import io
import json
import re
import sys
import tempfile
import unittest
from collections import Counter
from pathlib import Path
from typing import Any
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
import calibration # noqa: E402
REPO_ROOT = Path(__file__).resolve().parents[2]
SOURCES = {
"alpha.md": "# Alpha\n\nThe widget timeout is 45 seconds by default.\nRaising it above 90 seconds disables retries.\n",
"beta.md": "# Beta\n\nBeta adds the --frobnicate flag for batch runs.\nThe flag is ignored on interactive runs.\n",
"gamma.md": "# Gamma\n\nGamma stores its cache under the .gamma-cache directory.\nThe cache is cleared on every upgrade.\n",
"north.md": "# North beds\n\n" + "".join(f"Bed N{i} gets {i + 2} litres of water every morning.\n" for i in range(1, 31)),
"south.md": "# South beds\n\n" + "".join(f"Bed S{i} is weeded every {i + 3} days in summer.\n" for i in range(1, 31)),
}
ALPHA = "The widget timeout is 45 seconds by default."
ALPHA2 = "Raising it above 90 seconds disables retries."
BETA = "Beta adds the --frobnicate flag for batch runs."
BETA2 = "The flag is ignored on interactive runs."
GAMMA = "Gamma stores its cache under the .gamma-cache directory."
GAMMA2 = "The cache is cleared on every upgrade."
UNSUPPORTED, SUPPORTED, FILLER = "unsupported", "supported", "filler-supported"
CALL_ID = re.compile(r"\bk\d{3,}\b")
CLAIM_ID = re.compile(r"\bc\d{3,}\b")
def ex(name: str, text: str) -> dict[str, Any]:
return {"file": name, "text": text}
def draft_pairs() -> list[dict[str, Any]]:
"""Six unsupported pairs (one per kind) and six supported pairs (two per kind): a valid small draft."""
u = [
("overgeneralisation", "Every Alpha setting defaults to 45 seconds.", [ex("alpha.md", ALPHA)]),
("wrong-number", "The widget timeout is 30 seconds by default.", [ex("alpha.md", ALPHA)]),
("swapped-cause", "Retries being disabled is what raises the timeout above 90 seconds.", [ex("alpha.md", ALPHA2)]),
("not-mentioned", "Beta writes a summary file after each batch run.", [ex("beta.md", BETA)]),
("reversed-polarity", "The --frobnicate flag also applies to interactive runs.", [ex("beta.md", BETA2)]),
(
"conflated-sources",
"Beta stores its cache under the .gamma-cache directory.",
[ex("beta.md", BETA), ex("gamma.md", GAMMA)],
),
]
s = [
("paraphrase", "By default, the widget waits 45 seconds before timing out.", [ex("alpha.md", ALPHA)]),
("paraphrase", "Upgrading clears the Gamma cache.", [ex("gamma.md", GAMMA2)]),
("inference", "A widget left at its default timeout still retries.", [ex("alpha.md", ALPHA + "\n" + ALPHA2)]),
("inference", "Batch runs are the ones where --frobnicate has an effect.", [ex("beta.md", BETA + "\n" + BETA2)]),
(
"synthesis",
"An upgrade empties the directory Gamma uses for its cache.",
[ex("gamma.md", GAMMA), ex("gamma.md", GAMMA2)],
),
(
"synthesis",
"Both a 45-second default timeout and a --frobnicate flag for batch runs exist.",
[ex("alpha.md", ALPHA), ex("beta.md", BETA)],
),
]
pairs: list[dict[str, Any]] = []
for n, (kind, claim, excerpts) in enumerate(u, 1):
pairs.append({"id": f"u{n:02d}", "label": UNSUPPORTED, "kind": kind, "claim": claim, "excerpts": excerpts, "why": "planted"})
for n, (kind, claim, excerpts) in enumerate(s, 1):
pairs.append({"id": f"s{n:02d}", "label": SUPPORTED, "kind": kind, "claim": claim, "excerpts": excerpts, "why": "follows"})
return pairs
def north(i: int) -> str:
return f"Bed N{i} gets {i + 2} litres of water every morning."
def south(i: int) -> str:
return f"Bed S{i} is weeded every {i + 3} days in summer."
def garden_draft(n_unsupported: int = 20, n_supported: int = 6, extra_fillers: int = 4) -> list[dict[str, Any]]:
"""More unsupported than supported pairs, each unsupported with its supported twin (filler-supported), plus a few
fillers without a twin. Every kind of both counted labels is present."""
pairs: list[dict[str, Any]] = []
for i in range(1, n_unsupported + 1):
kind = calibration.UNSUPPORTED_KINDS[(i - 1) % len(calibration.UNSUPPORTED_KINDS)]
excerpts = [ex("north.md", north(i))]
claim = {
"overgeneralisation": f"Every north bed, N{i} included, gets {i + 2} litres of water each morning.",
"wrong-number": f"Bed N{i} gets {i + 50} litres of water each morning.",
"swapped-cause": f"Watering bed N{i} each morning is what makes it need {i + 2} litres.",
"not-mentioned": f"Bed N{i} gets {i + 2} litres of water each morning and is mulched in spring.",
"reversed-polarity": f"Bed N{i} never gets {i + 2} litres of water in the morning.",
"conflated-sources": f"Bed N{i} is weeded every {i + 3} days in summer.",
}[kind]
twin, twin_kind = f"Each morning, bed N{i} receives {i + 2} litres of water.", "paraphrase"
if kind == "conflated-sources":
excerpts = [ex("north.md", north(i)), ex("south.md", south(i))]
twin = f"Bed N{i} gets {i + 2} litres each morning, and bed S{i} is weeded every {i + 3} days in summer."
twin_kind = "synthesis"
pairs.append({"id": f"u{i:02d}", "label": UNSUPPORTED, "kind": kind, "claim": claim, "excerpts": excerpts, "why": "planted"})
pairs.append({"id": f"t{i:02d}", "label": FILLER, "kind": twin_kind, "twin_of": f"u{i:02d}", "claim": twin,
"excerpts": [dict(e) for e in excerpts], "why": "the supported reading"})
for n in range(1, n_supported + 1):
j = 20 + n
kind = calibration.SUPPORTED_KINDS[(n - 1) % len(calibration.SUPPORTED_KINDS)]
claim, excerpts = {
"paraphrase": (f"In summer, bed S{j} has its weeds pulled every {j + 3} days.", [ex("south.md", south(j))]),
"inference": (f"Bed S{j} is weeded more often than once a season in summer.", [ex("south.md", south(j))]),
"synthesis": (
f"Bed N{j} is watered each morning and bed S{j} is weeded every {j + 3} days in summer.",
[ex("north.md", north(j)), ex("south.md", south(j))],
),
}[kind]
pairs.append({"id": f"s{n:02d}", "label": SUPPORTED, "kind": kind, "claim": claim, "excerpts": excerpts, "why": "follows"})
for n in range(1, extra_fillers + 1):
j = 26 + n
pairs.append({"id": f"f{n:02d}", "label": FILLER, "kind": "paraphrase",
"claim": f"Each morning, bed N{j} receives {j + 2} litres of water.",
"excerpts": [ex("north.md", north(j))], "why": "follows"})
return pairs
def sha256_file(path: Path) -> str:
return hashlib.sha256(path.read_bytes()).hexdigest()
class Workspace:
"""A temporary sources directory and helpers to write drafts, verdicts and build outputs."""
def __init__(self) -> None:
self._tmp = tempfile.TemporaryDirectory()
self.root = Path(self._tmp.name)
self.sources = self.root / "sources"
self.sources.mkdir()
for name, text in SOURCES.items():
(self.sources / name).write_text(text, encoding="utf-8")
def close(self) -> None:
self._tmp.cleanup()
def write_json(self, name: str, data: Any) -> Path:
path = self.root / name
path.write_text(json.dumps(data), encoding="utf-8")
return path
def write_draft(self, pairs: list[dict[str, Any]], name: str = "draft.json") -> Path:
return self.write_json(name, {"schema": calibration.DRAFT_SCHEMA, "pairs": pairs})
def paths(self) -> dict[str, Path]:
return {
"set": self.root / "out" / "pairs.json",
"key": self.root / "out" / "key.json",
"calls": self.root / "out" / "calls.json",
"manifest": self.root / "calibration.manifest.json",
}
def build(
self, pairs: list[dict[str, Any]] | None = None, seed: int = 7, lone: int = 10, dense: int = 5
) -> tuple[int, str]:
draft = self.write_draft(garden_draft() if pairs is None else pairs)
p = self.paths()
return run(
"build", "--draft", str(draft), "--sources", str(self.sources), "--seed", str(seed),
"--lone", str(lone), "--dense", str(dense),
"--set-out", str(p["set"]), "--key-out", str(p["key"]), "--calls-out", str(p["calls"]),
"--manifest", str(p["manifest"]),
)
def frozen_args(self) -> list[str]:
p = self.paths()
return ["--set", str(p["set"]), "--key", str(p["key"]), "--calls", str(p["calls"]),
"--manifest", str(p["manifest"]), "--sources", str(self.sources)]
def gate(self, verdicts: Path | None) -> tuple[int, str]:
return run("gate", *self.frozen_args(), "--verdicts", str(verdicts or self.root / "no-such-verdicts.json"))
def check(self) -> tuple[int, str]:
return run("check", *self.frozen_args())
def load(self, which: str) -> Any:
return json.loads(self.paths()[which].read_text(encoding="utf-8"))
def labels(self) -> dict[str, str]:
return {k["id"]: k["label"] for k in self.load("key")["labels"]}
def positions(self) -> dict[str, int]:
return {i: n for c in self.load("calls")["calls"] for n, i in enumerate(c["ids"], 1)}
def rewrite_calls(self, layout: dict[str, Any]) -> None:
"""Write a changed layout and make the manifest's sha256 follow it, so only the layout rule under test fails."""
p = self.paths()
p["calls"].write_text(json.dumps(layout, indent=1, sort_keys=True) + "\n", encoding="utf-8")
manifest = self.load("manifest")
manifest["calls"]["sha256"] = sha256_file(p["calls"])
p["manifest"].write_text(json.dumps(manifest, indent=1) + "\n", encoding="utf-8")
def run(*argv: str) -> tuple[int, str]:
out, err = io.StringIO(), io.StringIO()
with contextlib.redirect_stdout(out), contextlib.redirect_stderr(err):
try:
code = calibration.main(list(argv))
except SystemExit as exc: # argparse refuses an unknown option
code = exc.code if isinstance(exc.code, int) else 2
return code, out.getvalue() + err.getvalue()
class Base(unittest.TestCase):
def setUp(self) -> None:
self.ws = Workspace()
def tearDown(self) -> None:
self.ws.close()
class CheckDraftTest(Base):
def check(self, pairs: list[dict[str, Any]]) -> calibration.CheckReport:
return calibration.check_pairs(pairs, self.ws.sources)
def test_valid_draft_passes_and_counts_kinds(self) -> None:
rep = self.check(draft_pairs())
self.assertTrue(rep.passed, rep.lines())
self.assertEqual(rep.unsupported, 6)
self.assertEqual(rep.supported, 6)
self.assertEqual(rep.excerpts_verbatim, rep.excerpts)
self.assertEqual(rep.by_kind["wrong-number"], 1)
self.assertEqual(rep.by_kind["synthesis"], 2)
def test_excerpt_not_verbatim_fails(self) -> None:
pairs = draft_pairs()
pairs[0]["excerpts"] = [ex("alpha.md", "The widget timeout is 46 seconds by default.")]
rep = self.check(pairs)
self.assertFalse(rep.passed)
self.assertEqual(rep.excerpts_verbatim, rep.excerpts - 1)
def test_missing_source_file_fails(self) -> None:
pairs = draft_pairs()
pairs[0]["excerpts"] = [ex("nowhere.md", ALPHA)]
self.assertFalse(self.check(pairs).passed)
def test_path_outside_sources_fails(self) -> None:
pairs = draft_pairs()
pairs[0]["excerpts"] = [ex("../alpha.md", ALPHA)]
self.assertFalse(self.check(pairs).passed)
def test_unbalanced_labels_pass_the_draft_check(self) -> None:
# The counts are the manifest's business now (order 3, point 1), not a balance rule.
rep = self.check(garden_draft())
self.assertTrue(rep.passed, rep.lines())
self.assertEqual((rep.unsupported, rep.supported, rep.filler), (20, 6, 24))
def test_every_unsupported_kind_must_be_present(self) -> None:
pairs = draft_pairs()
pairs[1]["kind"] = "overgeneralisation"
pairs[1]["claim"] = "Every widget setting is 45 seconds long."
rep = self.check(pairs)
self.assertFalse(rep.passed)
self.assertTrue(any("wrong-number" in e for e in rep.errors), rep.errors)
def test_every_supported_kind_must_be_present(self) -> None:
pairs = draft_pairs()
for p in pairs:
if p["kind"] == "inference":
p["kind"] = "paraphrase"
rep = self.check(pairs)
self.assertFalse(rep.passed)
self.assertTrue(any("inference" in e for e in rep.errors), rep.errors)
def test_unknown_kind_or_label_fails(self) -> None:
pairs = draft_pairs()
pairs[0]["kind"] = "vibes"
self.assertFalse(self.check(pairs).passed)
pairs = draft_pairs()
pairs[0]["label"] = "maybe"
self.assertFalse(self.check(pairs).passed)
def test_kind_must_match_label(self) -> None:
pairs = draft_pairs()
pairs[0]["kind"] = "paraphrase"
self.assertFalse(self.check(pairs).passed)
def test_filler_takes_a_supported_kind(self) -> None:
pairs = garden_draft()
twin = next(p for p in pairs if p["label"] == FILLER)
twin["kind"] = "wrong-number"
self.assertFalse(self.check(pairs).passed)
def test_filler_may_not_be_a_verbatim_copy(self) -> None:
pairs = garden_draft()
twin = next(p for p in pairs if p["label"] == FILLER and p["kind"] == "paraphrase")
twin["claim"] = twin["excerpts"][0]["text"]
rep = self.check(pairs)
self.assertFalse(rep.passed)
self.assertTrue(any(twin["id"] in e and "verbatim" in e for e in rep.errors), rep.errors)
def test_twin_of_must_name_an_unsupported_pair(self) -> None:
for target in ("s01", "nowhere"):
pairs = garden_draft()
next(p for p in pairs if p["label"] == FILLER)["twin_of"] = target
rep = self.check(pairs)
self.assertFalse(rep.passed, target)
self.assertTrue(any("twin_of" in e for e in rep.errors), rep.errors)
def test_wrong_number_claim_needs_a_number_absent_from_the_excerpts(self) -> None:
pairs = draft_pairs()
pairs[1]["claim"] = "The widget timeout is 45 seconds, which is short."
rep = self.check(pairs)
self.assertFalse(rep.passed)
self.assertTrue(any("u02" in e and "number" in e for e in rep.errors), rep.errors)
def test_conflated_sources_needs_two_files(self) -> None:
pairs = draft_pairs()
pairs[5]["excerpts"] = [ex("beta.md", BETA), ex("beta.md", BETA2)]
self.assertFalse(self.check(pairs).passed)
def test_synthesis_needs_two_excerpts(self) -> None:
pairs = draft_pairs()
pairs[10]["excerpts"] = [ex("gamma.md", GAMMA)]
self.assertFalse(self.check(pairs).passed)
def test_supported_claim_may_not_be_a_verbatim_copy(self) -> None:
pairs = draft_pairs()
pairs[6]["claim"] = "the widget timeout is 45 seconds by default"
rep = self.check(pairs)
self.assertFalse(rep.passed)
self.assertTrue(any("s01" in e and "verbatim" in e for e in rep.errors), rep.errors)
def test_duplicate_ids_fail(self) -> None:
pairs = draft_pairs()
pairs[1]["id"] = "u01"
self.assertFalse(self.check(pairs).passed)
def test_too_many_excerpts_fail(self) -> None:
pairs = draft_pairs()
pairs[0]["excerpts"] = [ex("alpha.md", ALPHA), ex("beta.md", BETA), ex("gamma.md", GAMMA)]
self.assertFalse(self.check(pairs).passed)
def test_empty_draft_fails(self) -> None:
self.assertFalse(self.check([]).passed)
def test_check_command_on_draft(self) -> None:
draft = self.ws.write_draft(draft_pairs())
code, out = run("check", "--draft", str(draft), "--sources", str(self.ws.sources))
self.assertEqual(code, 0, out)
self.assertIn("excerpts verbatim: 15 of 15", out)
self.assertIn("PASS", out)
def test_check_command_missing_sources_is_a_fail(self) -> None:
draft = self.ws.write_draft(draft_pairs())
code, out = run("check", "--draft", str(draft), "--sources", str(self.ws.root / "absent"))
self.assertEqual(code, 1)
self.assertIn("FAIL", out)
class BuildTest(Base):
def built(self, **kwargs: Any) -> None:
code, out = self.ws.build(**kwargs)
self.assertEqual(code, 0, out)
def test_build_writes_set_key_layout_and_manifest(self) -> None:
self.built()
pset, key, layout, manifest = (self.ws.load(w) for w in ("set", "key", "calls", "manifest"))
p = self.ws.paths()
self.assertEqual(pset["schema"], calibration.SET_SCHEMA)
ids = [q["id"] for q in pset["pairs"]]
self.assertEqual(ids, [f"c{n:03d}" for n in range(1, 121)])
for q in pset["pairs"]:
self.assertEqual(set(q), {"id", "claim", "excerpts"})
for e in q["excerpts"]:
self.assertEqual(set(e), {"file", "line", "text"})
self.assertNotIn("unsupported", json.dumps(pset))
self.assertNotIn("filler", json.dumps(pset))
self.assertEqual(sorted(k["id"] for k in key["labels"]), ids)
self.assertEqual(layout["schema"], "okf-wiki/reader-calls/1")
self.assertEqual(layout["call_size_max"], 10)
self.assertEqual(layout["set_sha256"], sha256_file(p["set"]))
self.assertEqual(len(layout["calls"]), 12)
self.assertEqual([c["call"] for c in layout["calls"]], [f"k{n:03d}" for n in range(1, 13)])
self.assertEqual(manifest["seed"], 7)
self.assertEqual(manifest["set"]["sha256"], sha256_file(p["set"]))
self.assertEqual(manifest["key"]["sha256"], sha256_file(p["key"]))
self.assertEqual(manifest["calls"]["sha256"], sha256_file(p["calls"]))
self.assertEqual(manifest["calls"]["calls"], 12)
counts = manifest["counts"]
self.assertEqual((counts["unsupported"], counts["supported"], counts["filler-supported"]), (20, 6, 94))
self.assertEqual(counts["filler_distinct"], 24)
self.assertEqual(sum(v["unsupported"] for v in counts["by_position"].values()), 20)
self.assertEqual(manifest["thresholds"], {"caught_min": 20, "accepted_min": 6})
self.assertEqual(manifest["sources"]["files"], len(SOURCES))
self.assertEqual(manifest["history"], [])
self.assertNotIn("claim", json.dumps(manifest))
def test_every_claim_stands_in_exactly_one_call_of_at_most_ten(self) -> None:
self.built()
placed = [i for c in self.ws.load("calls")["calls"] for i in c["ids"]]
self.assertEqual(sorted(placed), sorted(self.ws.labels()))
self.assertEqual(len(placed), len(set(placed)))
self.assertTrue(all(len(c["ids"]) <= 10 for c in self.ws.load("calls")["calls"]))
def test_lone_unsupported_stand_at_every_position(self) -> None:
self.built()
labels, pos = self.ws.labels(), self.ws.positions()
alone = []
for c in self.ws.load("calls")["calls"]:
u = [i for i in c["ids"] if labels[i] == UNSUPPORTED]
if len(u) == 1:
alone.append(pos[u[0]])
self.assertGreaterEqual(len(alone), 10)
self.assertEqual(set(alone), set(range(1, 11)))
self.assertEqual(self.ws.load("manifest")["calls"]["unsupported_alone"], len(alone))
def test_each_counted_supported_pair_stands_once(self) -> None:
self.built()
drafts = Counter(k["draft_id"] for k in self.ws.load("key")["labels"] if k["label"] == SUPPORTED)
self.assertEqual(drafts, Counter({f"s{n:02d}": 1 for n in range(1, 7)}))
def test_a_twin_never_shares_a_call_with_its_unsupported_pair_and_no_filler_stands_twice_in_a_call(self) -> None:
self.built()
draft_of = {k["id"]: k["draft_id"] for k in self.ws.load("key")["labels"]}
for c in self.ws.load("calls")["calls"]:
members = [draft_of[i] for i in c["ids"]]
self.assertEqual(len(members), len(set(members)), c["call"])
for m in members:
if m.startswith("t"):
self.assertNotIn("u" + m[1:], members, c["call"])
def test_same_seed_gives_the_same_bytes_and_another_seed_another_layout(self) -> None:
self.built(seed=7)
first = {w: self.ws.paths()[w].read_bytes() for w in ("set", "key", "calls")}
other = Workspace()
try:
self.assertEqual(other.build(seed=7)[0], 0)
self.assertEqual(first, {w: other.paths()[w].read_bytes() for w in ("set", "key", "calls")})
finally:
other.close()
other = Workspace()
try:
self.assertEqual(other.build(seed=8)[0], 0)
self.assertNotEqual(first["calls"], other.paths()["calls"].read_bytes())
self.assertNotEqual(first["set"], other.paths()["set"].read_bytes())
finally:
other.close()
def test_order_does_not_follow_the_labels(self) -> None:
self.built()
labels = self.ws.labels()
order = [labels[q["id"]] for q in self.ws.load("set")["pairs"]]
self.assertNotEqual(order, sorted(order))
self.assertNotEqual(order, sorted(order, reverse=True))
def test_build_moves_the_old_manifest_into_history(self) -> None:
old = {"schema": "okf-wiki/calibration-manifest/1", "seed": 1, "history": [{"schema": "older"}]}
self.ws.paths()["manifest"].write_text(json.dumps(old), encoding="utf-8")
self.built()
history = self.ws.load("manifest")["history"]
self.assertEqual(history, [{"schema": "older"}, {"schema": "okf-wiki/calibration-manifest/1", "seed": 1}])
def test_build_refuses_a_failing_draft(self) -> None:
pairs = garden_draft()
pairs[0]["excerpts"] = [ex("north.md", "not in the file at all, not anywhere")]
code, out = self.ws.build(pairs)
self.assertEqual(code, 1)
self.assertFalse(self.ws.paths()["set"].exists())
def test_build_refuses_when_a_position_would_have_no_unsupported_claim(self) -> None:
code, out = self.ws.build(garden_draft(n_unsupported=9), lone=9)
self.assertEqual(code, 1, out)
self.assertIn("position 10", out)
self.assertFalse(any(self.ws.paths()[w].exists() for w in ("set", "key", "calls", "manifest")))
def test_build_refuses_slots_it_cannot_fill(self) -> None:
pairs = [p for p in garden_draft() if p["label"] != FILLER]
code, out = self.ws.build(pairs)
self.assertEqual(code, 1, out)
self.assertIn("filler", out)
self.assertFalse(self.ws.paths()["calls"].exists())
def test_build_refuses_impossible_parameters(self) -> None:
for lone, dense in ((21, 5), (10, 11), (10, 0), (-1, 5)):
code, out = self.ws.build(lone=lone, dense=dense)
self.assertEqual(code, 1, (lone, dense, out))
self.assertFalse(self.ws.paths()["set"].exists())
def test_build_never_overwrites_a_frozen_set(self) -> None:
self.built()
code, out = self.ws.build()
self.assertEqual(code, 1)
self.assertIn("frozen", out)
self.ws.paths()["set"].unlink()
self.ws.paths()["key"].unlink()
code, out = self.ws.build()
self.assertEqual(code, 1)
self.assertIn("frozen", out)
class GateBase(Base):
"""A built set, its key, and helpers to write verdicts and stubs for the gate."""
def setUp(self) -> None:
super().setUp()
code, out = self.ws.build()
self.assertEqual(code, 0, out)
self.key = self.ws.labels()
def verdicts(self, mapping: dict[str, str], reader: str = "test", models: Any = ("claude-test-1",),
name: str = "verdicts.json") -> Path:
doc: dict[str, Any] = {"reader": reader, "verdicts": [{"id": i, "verdict": v} for i, v in mapping.items()]}
if models is not None:
doc["models"] = list(models) if isinstance(models, tuple) else models
return self.ws.write_json(name, doc)
def perfect(self) -> dict[str, str]:
return {i: (UNSUPPORTED if lab == UNSUPPORTED else SUPPORTED) for i, lab in self.key.items()}
def stub(self, *args: str) -> Path:
path = self.ws.root / "stub.json"
code, out = run("stub", "--set", str(self.ws.paths()["set"]), "--out", str(path), *args)
self.assertEqual(code, 0, out)
return path
def per_position(self) -> dict[int, int]:
pos = self.ws.positions()
return dict(Counter(pos[i] for i, lab in self.key.items() if lab == UNSUPPORTED))
class GateTest(GateBase):
def test_no_reader_scores_zero_and_is_red(self) -> None:
code, out = self.ws.gate(None)
self.assertEqual(code, 1)
self.assertIn("unsupported caught 0 of 20 · supported accepted 0 of 6", out)
self.assertIn("fail-closed", out)
self.assertIn("RED", out)
def test_the_key_as_judge_is_green(self) -> None:
code, out = self.ws.gate(self.stub("--key", str(self.ws.paths()["key"])))
self.assertEqual(code, 0, out)
self.assertIn("unsupported caught 20 of 20 · supported accepted 6 of 6", out)
self.assertIn("filler accepted 94 of 94", out)
self.assertIn("GREEN", out)
def test_reject_all_stub_is_red(self) -> None:
code, out = self.ws.gate(self.stub("--verdict", UNSUPPORTED))
self.assertEqual(code, 1)
self.assertIn("unsupported caught 20 of 20 · supported accepted 0 of 6", out)
self.assertIn("RED", out)
def test_accept_all_stub_is_red(self) -> None:
code, out = self.ws.gate(self.stub("--verdict", SUPPORTED))
self.assertEqual(code, 1)
self.assertIn("unsupported caught 0 of 20 · supported accepted 6 of 6", out)
self.assertIn("RED", out)
def test_a_stub_mild_at_one_position_is_red_and_the_position_is_named(self) -> None:
stub = self.stub("--key", str(self.ws.paths()["key"]), "--calls", str(self.ws.paths()["calls"]), "--accept-at", "7")
code, out = self.ws.gate(stub)
self.assertEqual(code, 1, out)
n7 = self.per_position()[7]
self.assertIn(f"7 0 of {n7}", out)
self.assertIn(f"unsupported caught {20 - n7} of 20", out)
not_met = [line for line in out.splitlines() if line.startswith("not met:")]
self.assertEqual(len(not_met), 1, out)
self.assertIn("position 7", not_met[0])
self.assertNotIn("position 6", not_met[0])
self.assertIn("RED", out)
def test_caught_per_position_is_written_and_no_line_per_call_or_pair(self) -> None:
code, out = self.ws.gate(self.verdicts(self.perfect()))
self.assertEqual(code, 0, out)
counts = self.per_position()
line = next(line for line in out.splitlines() if line.startswith("caught by position: "))
self.assertEqual(line, "caught by position: " + ", ".join(f"{p} {counts[p]} of {counts[p]}" for p in range(1, 11)))
self.assertIsNone(CALL_ID.search(out), out)
self.assertIsNone(CLAIM_ID.search(out), out)
def test_the_gate_writes_no_per_pair_results(self) -> None:
code, out = run("gate", *self.ws.frozen_args(), "--verdicts", str(self.verdicts(self.perfect())),
"--out", str(self.ws.root / "results.json"))
self.assertNotEqual(code, 0)
self.assertFalse((self.ws.root / "results.json").exists())
def test_reports_caught_per_kind(self) -> None:
code, out = self.ws.gate(self.verdicts(self.perfect()))
self.assertIn("caught by kind: overgeneralisation 4 of 4", out)
self.assertIn("conflated-sources 3 of 3", out)
def test_filler_never_counts_in_the_thresholds(self) -> None:
v = self.perfect()
for i, lab in self.key.items():
if lab == FILLER:
v[i] = UNSUPPORTED
code, out = self.ws.gate(self.verdicts(v))
self.assertEqual(code, 0, out)
self.assertIn("filler accepted 0 of 94", out)
self.assertIn("GREEN", out)
def test_a_missing_filler_verdict_is_red(self) -> None:
v = self.perfect()
del v[next(i for i, lab in self.key.items() if lab == FILLER)]
code, out = self.ws.gate(self.verdicts(v))
self.assertEqual(code, 1, out)
self.assertIn("missing verdicts: 1", out)
def test_one_missed_unsupported_is_red(self) -> None:
v = self.perfect()
v[next(i for i, lab in self.key.items() if lab == UNSUPPORTED)] = SUPPORTED
code, out = self.ws.gate(self.verdicts(v))
self.assertEqual(code, 1)
self.assertIn("unsupported caught 19 of 20", out)
def test_missing_verdict_counts_as_wrong(self) -> None:
v = self.perfect()
gone_u = next(i for i, lab in self.key.items() if lab == UNSUPPORTED)
gone_s = next(i for i, lab in self.key.items() if lab == SUPPORTED)
del v[gone_u], v[gone_s]
code, out = self.ws.gate(self.verdicts(v))
self.assertEqual(code, 1)
self.assertIn("missing verdicts: 2", out)
self.assertIn("unsupported caught 19 of 20 · supported accepted 5 of 6", out)
def test_invalid_or_duplicate_verdict_counts_as_missing(self) -> None:
v = self.perfect()
first = next(iter(v))
path = self.ws.write_json(
"verdicts.json",
{
"reader": "test",
"verdicts": [{"id": i, "verdict": val} for i, val in v.items()]
+ [{"id": first, "verdict": SUPPORTED}, {"id": "c999", "verdict": SUPPORTED}],
},
)
code, out = self.ws.gate(path)
self.assertEqual(code, 1)
self.assertIn("missing verdicts: 1", out)
for bad in ("probably", FILLER):
code, out = self.ws.gate(self.verdicts({**v, first: bad}))
self.assertEqual(code, 1, bad)
self.assertIn("missing verdicts: 1", out)
def test_accepted_threshold_is_read_from_the_manifest(self) -> None:
manifest = self.ws.load("manifest")
manifest["thresholds"]["accepted_min"] = 5
self.ws.paths()["manifest"].write_text(json.dumps(manifest), encoding="utf-8")
v = self.perfect()
v[next(i for i, lab in self.key.items() if lab == SUPPORTED)] = UNSUPPORTED
code, out = self.ws.gate(self.verdicts(v))
self.assertEqual(code, 0, out)
self.assertIn("supported accepted 5 of 6", out)
def test_gate_refuses_a_changed_set_key_or_layout(self) -> None:
for which in ("set", "key", "calls"):
path = self.ws.paths()[which]
before = path.read_text(encoding="utf-8")
path.write_text(before + " ", encoding="utf-8")
code, out = self.ws.gate(self.verdicts(self.perfect()))
self.assertEqual(code, 1, which)
self.assertIn("manifest", out)
path.write_text(before, encoding="utf-8")
def test_gate_refuses_other_sources(self) -> None:
(self.ws.sources / "alpha.md").write_text("changed\n", encoding="utf-8")
code, out = self.ws.gate(self.verdicts(self.perfect()))
self.assertEqual(code, 1)
self.assertIn("tree_sha256", out)
class GateModelTest(GateBase):
"""PLAN § Rammer 23: the gate is bound to the model the reader ran on. It names the model it scored, is GREEN only
for verdicts of exactly one model, and records a GREEN run in the file --passed names; collect reads that file."""
def passed(self) -> Path:
return self.ws.root / "passed.json"
def gate_passed(self, verdicts: Path) -> tuple[int, str]:
return run("gate", *self.ws.frozen_args(), "--verdicts", str(verdicts), "--passed", str(self.passed()))
def test_the_gate_names_the_model_it_scored(self) -> None:
code, out = self.ws.gate(self.verdicts(self.perfect(), models=["claude-test-2"]))
self.assertEqual(code, 0, out)
self.assertIn("model: claude-test-2\n", out)
self.assertIn("GREEN", out)
def test_verdicts_that_name_no_model_are_red(self) -> None:
for models in (None, [], "claude-test-2", [3]):
with self.subTest(models=models):
code, out = self.ws.gate(self.verdicts(self.perfect(), models=models))
self.assertEqual(code, 1, out)
self.assertIn("model: (none)\n", out)
self.assertIn("the verdicts name no model", out)
self.assertNotIn("GREEN", out)
self.assertIn("RED", out)
def test_verdicts_of_two_models_are_red(self) -> None:
code, out = self.ws.gate(self.verdicts(self.perfect(), models=["claude-test-1", "claude-test-2"]))
self.assertEqual(code, 1, out)
self.assertIn("model: claude-test-1, claude-test-2\n", out)
self.assertIn("the verdicts name 2 models", out)
self.assertIn("RED", out)
def test_a_stub_names_the_model_stub_and_its_meaning_is_kept(self) -> None:
stub = self.stub("--key", str(self.ws.paths()["key"]))
self.assertEqual(json.loads(stub.read_text(encoding="utf-8"))["models"], ["stub"])
code, out = self.ws.gate(stub)
self.assertEqual(code, 0, out)
self.assertIn("model: stub\n", out)
code, out = self.ws.gate(self.stub("--verdict", SUPPORTED))
self.assertEqual(code, 1, out)
self.assertIn("model: stub\n", out)
def test_a_green_run_is_recorded_with_its_hashes(self) -> None:
verdicts = self.verdicts(self.perfect(), reader="r7", models=["claude-test-2"])
code, out = self.gate_passed(verdicts)
self.assertEqual(code, 0, out)
p = self.ws.paths()
record = json.loads(self.passed().read_text(encoding="utf-8"))
self.assertEqual(record, {"schema": "okf-wiki/calibration-passed/1", "passed": [{
"model": "claude-test-2", "set_sha256": sha256_file(p["set"]), "key_sha256": sha256_file(p["key"]),
"verdicts_sha256": sha256_file(verdicts), "reader": "r7"}]})
self.assertIn(str(self.passed()), out)
def test_the_same_green_run_is_recorded_once_and_the_file_has_no_time(self) -> None:
verdicts = self.verdicts(self.perfect(), models=["claude-test-2"])
self.assertEqual(self.gate_passed(verdicts)[0], 0)
first = self.passed().read_bytes()
self.assertEqual(self.gate_passed(verdicts)[0], 0)
self.assertEqual(self.passed().read_bytes(), first)
self.assertEqual(len(json.loads(first)["passed"]), 1)
def test_another_green_run_is_added_after_the_first(self) -> None:
self.assertEqual(self.gate_passed(self.verdicts(self.perfect(), models=["claude-test-2"]))[0], 0)
self.assertEqual(self.gate_passed(self.verdicts(self.perfect(), models=["claude-test-3"],
name="v3.json"))[0], 0)
record = json.loads(self.passed().read_text(encoding="utf-8"))
self.assertEqual([e["model"] for e in record["passed"]], ["claude-test-2", "claude-test-3"])
def test_a_red_run_never_writes_the_record(self) -> None:
v = self.perfect()
v[next(i for i, lab in self.key.items() if lab == UNSUPPORTED)] = SUPPORTED
code, out = self.gate_passed(self.verdicts(v))
self.assertEqual(code, 1, out)
self.assertFalse(self.passed().exists())
code, out = self.gate_passed(self.verdicts(self.perfect(), models=None))
self.assertEqual(code, 1, out)
self.assertFalse(self.passed().exists())
self.assertEqual(self.gate_passed(self.verdicts(self.perfect()))[0], 0)
before = self.passed().read_bytes()
self.assertEqual(self.gate_passed(self.verdicts(v, models=["claude-test-9"]))[0], 1)
self.assertEqual(self.passed().read_bytes(), before)
def test_a_record_that_cannot_be_read_is_a_fail_and_is_left_as_it_was(self) -> None:
for bad in ("{", json.dumps({"schema": "okf-wiki/other/1", "passed": []}),
json.dumps({"schema": "okf-wiki/calibration-passed/1", "passed": {}})):
with self.subTest(bad=bad):
self.passed().write_text(bad, encoding="utf-8")
code, out = self.gate_passed(self.verdicts(self.perfect()))
self.assertEqual(code, 1, out)
self.assertIn("FAIL", out)
self.assertEqual(self.passed().read_text(encoding="utf-8"), bad)
class LayoutRuleTest(Base):
"""A layout that breaks K3 point 1 is refused by the gate and by check, even when the manifest follows it."""
def setUp(self) -> None:
super().setUp()
code, out = self.ws.build()
self.assertEqual(code, 0, out)
self.layout = self.ws.load("calls")
self.key = self.ws.labels()
self.green = self.ws.write_json(
"verdicts.json",
{"reader": "key", "models": ["claude-test-1"],
"verdicts": [{"id": i, "verdict": UNSUPPORTED if lab == UNSUPPORTED else SUPPORTED}
for i, lab in self.key.items()]},
)
def refused(self, needle: str) -> None:
self.ws.rewrite_calls(self.layout)
code, out = self.ws.gate(self.green)
self.assertEqual(code, 1, out)
self.assertIn(needle, out)
self.assertNotIn("GREEN", out)
code, out = self.ws.check()
self.assertEqual(code, 1, out)
self.assertIn(needle, out)
def test_the_unchanged_layout_passes(self) -> None:
self.ws.rewrite_calls(self.layout)
self.assertEqual(self.ws.gate(self.green)[0], 0)
self.assertEqual(self.ws.check()[0], 0)
def test_an_id_given_twice_is_refused(self) -> None:
self.layout["calls"].append({"call": "k999", "ids": [self.layout["calls"][1]["ids"][0]]})
self.refused("more than one place")
def test_an_id_missing_from_every_call_is_refused(self) -> None:
self.layout["calls"][-1]["ids"].pop()
self.refused("in no call")
def test_an_id_not_in_the_set_is_refused(self) -> None:
self.layout["calls"][-1]["ids"][-1] = "c999"
self.refused("not in the set")
def test_a_wrong_set_sha256_is_refused(self) -> None:
self.layout["set_sha256"] = "0" * 64
self.refused("set_sha256")
def test_a_call_over_ten_is_refused(self) -> None:
moved = self.layout["calls"][1]["ids"].pop()
self.layout["calls"][0]["ids"].append(moved)
self.refused("more than 10")
def test_a_call_size_max_over_ten_is_refused(self) -> None:
self.layout["call_size_max"] = 11
self.refused("call_size_max")
def test_a_position_without_unsupported_is_refused(self) -> None:
for c in self.layout["calls"]:
ids = c["ids"]
if self.key[ids[2]] == UNSUPPORTED:
swap = next(n for n, i in enumerate(ids) if self.key[i] != UNSUPPORTED and n != 2)
ids[2], ids[swap] = ids[swap], ids[2]
self.refused("position 3 has no unsupported claim")
class CheckBuiltSetTest(Base):
def setUp(self) -> None:
super().setUp()
code, out = self.ws.build()
self.assertEqual(code, 0, out)
def excerpts(self) -> int:
return sum(len(q["excerpts"]) for q in self.ws.load("set")["pairs"])
def test_check_passes_on_the_built_set_and_states_the_layout(self) -> None:
code, out = self.ws.check()
self.assertEqual(code, 0, out)
n = self.excerpts()
self.assertIn(f"excerpt lines correct: {n} of {n}", out)
self.assertIn("pairs: 120 (20 unsupported, 6 supported, 94 filler-supported)", out)
self.assertIn("unsupported per call: 1 in 10 calls, 5 in 2 calls", out)
self.assertIn("PASS", out)
def test_more_unsupported_than_supported_passes_when_the_manifest_says_so(self) -> None:
self.assertEqual(self.ws.check()[0], 0)
manifest = self.ws.load("manifest")
manifest["counts"]["unsupported"] = 6
self.ws.paths()["manifest"].write_text(json.dumps(manifest), encoding="utf-8")
code, out = self.ws.check()
self.assertEqual(code, 1, out)
self.assertIn("manifest", out)
def test_check_catches_a_wrong_line(self) -> None:
data = self.ws.load("set")
data["pairs"][0]["excerpts"][0]["line"] += 1
self.ws.paths()["set"].write_text(json.dumps(data), encoding="utf-8")
code, out = self.ws.check()
self.assertEqual(code, 1)
n = self.excerpts()
self.assertIn(f"excerpt lines correct: {n - 1} of {n}", out)
def test_check_of_a_built_set_needs_the_layout_and_the_manifest(self) -> None:
p = self.ws.paths()
code, out = run("check", "--set", str(p["set"]), "--key", str(p["key"]), "--sources", str(self.ws.sources))
self.assertEqual(code, 1)
self.assertIn("FAIL", out)
class StubTest(Base):
def setUp(self) -> None:
super().setUp()
code, out = self.ws.build()
self.assertEqual(code, 0, out)
def stub(self, *args: str) -> tuple[int, str, dict[str, str]]:
path = self.ws.root / "stub.json"
code, out = run("stub", "--set", str(self.ws.paths()["set"]), "--out", str(path), *args)
verdicts = {}
if path.exists():
verdicts = {v["id"]: v["verdict"] for v in json.loads(path.read_text(encoding="utf-8"))["verdicts"]}
path.unlink()
return code, out, verdicts
def test_the_same_verdict_for_every_claim(self) -> None:
code, out, v = self.stub("--verdict", UNSUPPORTED)
self.assertEqual(code, 0, out)
self.assertEqual(set(v.values()), {UNSUPPORTED})
self.assertEqual(len(v), 120)
def test_the_key_gives_supported_for_every_filler(self) -> None:
code, out, v = self.stub("--key", str(self.ws.paths()["key"]))
self.assertEqual(code, 0, out)
labels = self.ws.labels()
self.assertEqual(v, {i: UNSUPPORTED if lab == UNSUPPORTED else SUPPORTED for i, lab in labels.items()})
def test_accept_at_changes_only_that_position(self) -> None:
code, out, v = self.stub("--verdict", UNSUPPORTED, "--calls", str(self.ws.paths()["calls"]), "--accept-at", "4")
self.assertEqual(code, 0, out)
pos = self.ws.positions()
self.assertEqual({i for i, val in v.items() if val == SUPPORTED}, {i for i, n in pos.items() if n == 4})
def test_accept_at_needs_the_layout_and_a_position_from_one_to_ten(self) -> None:
self.assertEqual(self.stub("--verdict", UNSUPPORTED, "--accept-at", "4")[0], 1)
for bad in ("0", "11"):
code, out, v = self.stub("--verdict", UNSUPPORTED, "--calls", str(self.ws.paths()["calls"]), "--accept-at", bad)
self.assertEqual(code, 1, bad)
def test_verdict_and_key_exclude_each_other(self) -> None:
self.assertNotEqual(self.stub("--verdict", UNSUPPORTED, "--key", str(self.ws.paths()["key"]))[0], 0)
self.assertNotEqual(self.stub()[0], 0)
class NoModelCallTest(unittest.TestCase):
def test_tool_imports_no_network_or_model_client(self) -> None:
text = (REPO_ROOT / "gates" / "calibration.py").read_text(encoding="utf-8")
for banned in ("anthropic", "openai", "urllib", "http.client", "requests", "subprocess", "socket"):
self.assertNotIn(f"import {banned}", text)
self.assertNotIn(f"from {banned}", text)
if __name__ == "__main__":
unittest.main()

View file

@ -0,0 +1,345 @@
"""Tests for gates/filetype_gate.py, the fail-closed runner of the reader's file-type gate (E4).
Every fixture here is synthetic and neutral text. The real pairs live in gates-local/ and outside the repository.
Run: python3 -m unittest discover -s gates/tests
"""
from __future__ import annotations
import io
import json
import sys
import tempfile
import textwrap
import unittest
import zipfile
from contextlib import redirect_stdout
from pathlib import Path
from typing import Any
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
import filetype_gate # noqa: E402
PY = sys.executable
STUBS = Path(__file__).resolve().parents[1] / "stubs"
# Answers every query from the pairs file given as argv[3]: the known-positive.
PERFECT = """
import json, sys
file, queries, pairs = sys.argv[1], sys.argv[2], sys.argv[3]
key = {p["id"]: p for p in json.load(open(pairs))["pairs"]}
answers = [{"id": q["id"], "found": [{"locator": key[q["id"]]["locator"], "hidden": key[q["id"]]["hidden"]}]}
for q in json.load(open(queries))["queries"]]
print(json.dumps({"answers": answers}))
"""
# Like PERFECT, but every answer also claims a second place.
TWO_PLACES = PERFECT.replace(
'[{"locator": key[q["id"]]["locator"], "hidden": key[q["id"]]["hidden"]}]',
'[{"locator": key[q["id"]]["locator"], "hidden": key[q["id"]]["hidden"]}, {"locator": {"line": 99}, "hidden": False}]')
# Like PERFECT, but says hidden=false for everything (flattens hidden content into ordinary text).
FLATTEN = PERFECT.replace('"hidden": key[q["id"]]["hidden"]', '"hidden": False')
# Like PERFECT, but leaves out the first query of every file.
DROP_FIRST = PERFECT.replace("for q in json.load(open(queries))[\"queries\"]]",
"for q in json.load(open(queries))[\"queries\"][1:]]")
CRASH = """
raise RuntimeError("boom")
"""
NOT_JSON = """
print("answers: many")
"""
NO_ANSWERS_KEY = """
import json
print(json.dumps({"found": []}))
"""
SLEEPER = """
import time
time.sleep(10)
"""
# Records what the reader was shown, then answers nothing.
SPY = """
import json, sys, pathlib
q = json.load(open(sys.argv[2]))
out = pathlib.Path(sys.argv[3])
with out.open("a") as fh:
fh.write(json.dumps({"file": sys.argv[1], "queries": q}) + "\\n")
print(json.dumps({"answers": []}))
"""
# Tries to damage the file it was given.
VANDAL = """
import json, sys, pathlib
pathlib.Path(sys.argv[1]).write_text("vandalised")
print(json.dumps({"answers": []}))
"""
NOTES_MD = "# Kettles\n\nA kettle boils water for tea.\nThe spout pours the water out slowly.\n"
NOTES_TXT = "Line one of the plain file.\nThe handle stays cool while the water heats.\n"
class GateFixture:
"""A synthetic manifest, pairs file, sources root and cases root in a temporary directory."""
def __init__(self, root: Path) -> None:
self.root = root
self.sources = root / "sources"
self.cases = root / "cases"
(self.sources / "docs").mkdir(parents=True)
(self.cases / "txt").mkdir(parents=True)
(self.sources / "docs" / "kettle.md").write_text(NOTES_MD)
(self.cases / "txt" / "handle.txt").write_text(NOTES_TXT)
self.pairs_data: dict[str, Any] = {
"schema": filetype_gate.PAIRS_SCHEMA,
"set": "working",
"pairs": [
{"id": "t01", "type": "md", "root": "sources", "path": "docs/kettle.md", "group": "filetype",
"hidden_class": None, "quote": "Kettles", "locator": {"line": 1}, "hidden": False},
{"id": "t02", "type": "md", "root": "sources", "path": "docs/kettle.md", "group": "filetype",
"hidden_class": None, "quote": "spout pours the water", "locator": {"line": 4}, "hidden": False},
{"id": "t03", "type": "txt", "root": "cases", "path": "txt/handle.txt", "group": "filetype",
"hidden_class": None, "quote": "handle stays cool", "locator": {"line": 2}, "hidden": False},
{"id": "t04", "type": "txt", "root": "cases", "path": "txt/handle.txt", "group": "hidden",
"hidden_class": "demo-hidden", "quote": "Line one", "locator": {"line": 1}, "hidden": True},
],
}
self.write_pairs()
self.write_manifest()
def write_pairs(self) -> None:
self.pairs = self.root / "pairs.json"
self.pairs.write_text(json.dumps(self.pairs_data))
def write_manifest(self) -> None:
self.manifest = self.root / "manifest.json"
self.manifest.write_text(json.dumps({
"schema": filetype_gate.MANIFEST_SCHEMA,
"sets": {"working": {"file": "pairs.json", "sha256": filetype_gate.sha256_file(self.pairs),
"pairs": len(self.pairs_data["pairs"])}},
"sources": [
{"root": "sources", "path": "docs/kettle.md",
"sha256": filetype_gate.sha256_file(self.sources / "docs" / "kettle.md")},
{"root": "cases", "path": "txt/handle.txt",
"sha256": filetype_gate.sha256_file(self.cases / "txt" / "handle.txt")},
],
}))
def script(self, name: str, body: str) -> Path:
p = self.root / f"{name}.py"
p.write_text(textwrap.dedent(body))
return p
def run(self, command: list[str], timeout: float = 5.0) -> tuple[int, str]:
out = io.StringIO()
with redirect_stdout(out):
code = filetype_gate.main([
"--manifest", str(self.manifest), "--pairs", str(self.pairs), "--sources", str(self.sources),
"--cases", str(self.cases), "--timeout", str(timeout), "--", *command,
])
return code, out.getvalue()
def perfect(self, body: str = PERFECT) -> list[str]:
return [PY, str(self.script("reader", body)), "{file}", "{queries}", str(self.pairs)]
class FiletypeGateTest(unittest.TestCase):
def setUp(self) -> None:
self._tmp = tempfile.TemporaryDirectory()
self.fx = GateFixture(Path(self._tmp.name))
def tearDown(self) -> None:
self._tmp.cleanup()
# --- the three outcomes the order names -----------------------------------------------------------
def test_missing_reader_locates_nothing_and_says_so(self) -> None:
code, out = self.fx.run([str(self.fx.root / "no-such-reader"), "{file}", "{queries}"])
self.assertEqual(code, 1)
self.assertIn("located 0 of 4", out)
self.assertIn("reader errors 2 of 2 files", out)
self.assertIn("reader not found", out)
self.assertIn("verdict: RED", out)
def test_known_positive_turns_the_gate_green(self) -> None:
code, out = self.fx.run(self.fx.perfect())
self.assertEqual(code, 0, out)
self.assertIn("located 4 of 4", out)
self.assertIn("md: 2 of 2", out)
self.assertIn("txt: 1 of 1", out)
self.assertIn("demo-hidden: 1 of 1", out)
self.assertIn("reader errors 0 of 2 files", out)
self.assertIn("file types with every pair located: 2 of 2", out)
self.assertIn("verdict: GREEN", out)
def test_the_tracked_fasit_stub_is_the_known_positive(self) -> None:
code, out = self.fx.run([PY, str(STUBS / "fasit_reader.py"), "{file}", "{queries}", str(self.fx.pairs)])
self.assertEqual(code, 0, out)
self.assertIn("located 4 of 4", out)
self.assertIn("verdict: GREEN", out)
def test_first_locator_stub_only_gets_pairs_on_the_first_line(self) -> None:
code, out = self.fx.run([PY, str(STUBS / "first_locator.py"), "{file}", "{queries}"])
self.assertEqual(code, 1)
# t01 is on line 1 and visible: located. t04 is on line 1 but hidden: the stub says visible, so not located.
self.assertIn("located 1 of 4", out)
self.assertIn("md: 1 of 2", out)
self.assertIn("demo-hidden: 0 of 1", out)
self.assertIn("file types with every pair located: 0 of 2", out)
self.assertIn("verdict: RED", out)
# --- what counts as located ------------------------------------------------------------------------
def test_a_second_claimed_place_is_not_located(self) -> None:
_, out = self.fx.run(self.fx.perfect(TWO_PLACES))
self.assertIn("located 0 of 4", out)
self.assertIn("t02", out)
def test_flattening_hidden_content_fails_the_hidden_class_only(self) -> None:
_, out = self.fx.run(self.fx.perfect(FLATTEN))
self.assertIn("located 3 of 4", out)
self.assertIn("demo-hidden: 0 of 1", out)
self.assertIn("hidden flag wrong", out)
def test_a_missing_answer_is_not_located(self) -> None:
_, out = self.fx.run(self.fx.perfect(DROP_FIRST))
self.assertIn("located 2 of 4", out)
self.assertIn("no answer", out)
def test_the_reader_sees_only_ids_and_quotes_and_a_copy_of_the_file(self) -> None:
log = self.fx.root / "spy.log"
self.fx.run([PY, str(self.fx.script("spy", SPY)), "{file}", "{queries}", str(log)])
seen = [json.loads(line) for line in log.read_text().splitlines()]
self.assertEqual(len(seen), 2)
for entry in seen:
for q in entry["queries"]["queries"]:
self.assertEqual(set(q), {"id", "quote"})
self.assertNotIn(str(self.fx.sources), entry["file"])
self.assertNotIn(str(self.fx.cases), entry["file"])
self.assertEqual({Path(e["file"]).name for e in seen}, {"kettle.md", "handle.txt"})
def test_a_reader_that_writes_to_its_file_cannot_damage_the_source(self) -> None:
self.fx.run([PY, str(self.fx.script("vandal", VANDAL)), "{file}", "{queries}"])
self.assertEqual((self.fx.sources / "docs" / "kettle.md").read_text(), NOTES_MD)
# --- fail-closed: anything but a well-formed answer is an error, never a location -------------------
def test_crashing_reader_is_an_error(self) -> None:
code, out = self.fx.run([PY, str(self.fx.script("crash", CRASH)), "{file}", "{queries}"])
self.assertEqual(code, 1)
self.assertIn("located 0 of 4", out)
self.assertIn("reader errors 2 of 2 files", out)
def test_output_that_is_not_json_is_an_error(self) -> None:
_, out = self.fx.run([PY, str(self.fx.script("notjson", NOT_JSON)), "{file}", "{queries}"])
self.assertIn("not valid JSON", out)
self.assertIn("located 0 of 4", out)
def test_output_without_answers_is_an_error(self) -> None:
_, out = self.fx.run([PY, str(self.fx.script("noans", NO_ANSWERS_KEY)), "{file}", "{queries}"])
self.assertIn('lacks an "answers" list', out)
def test_timeout_is_an_error(self) -> None:
_, out = self.fx.run([PY, str(self.fx.script("sleep", SLEEPER)), "{file}", "{queries}"], timeout=0.5)
self.assertIn("timed out", out)
self.assertIn("located 0 of 4", out)
# --- the gate refuses to measure anything but the frozen set ----------------------------------------
def test_a_changed_source_stops_the_gate(self) -> None:
(self.fx.sources / "docs" / "kettle.md").write_text(NOTES_MD + "extra\n")
code, out = self.fx.run(self.fx.perfect())
self.assertEqual(code, 2)
self.assertIn("GATE CANNOT RUN", out)
self.assertIn("sha256 mismatch", out)
def test_a_changed_pairs_file_stops_the_gate(self) -> None:
self.fx.pairs_data["pairs"][1]["locator"] = {"line": 3}
self.fx.write_pairs()
code, out = self.fx.run(self.fx.perfect())
self.assertEqual(code, 2)
self.assertIn("pairs file", out)
def test_a_pair_on_a_file_the_manifest_does_not_list_stops_the_gate(self) -> None:
self.fx.pairs_data["pairs"][0]["path"] = "docs/other.md"
self.fx.write_pairs()
self.fx.write_manifest()
code, out = self.fx.run(self.fx.perfect())
self.assertEqual(code, 2)
self.assertIn("not in the manifest", out)
def test_no_command_is_a_usage_error(self) -> None:
out = io.StringIO()
with redirect_stdout(out):
code = filetype_gate.main(["--manifest", str(self.fx.manifest), "--pairs", str(self.fx.pairs),
"--sources", str(self.fx.sources), "--cases", str(self.fx.cases)])
self.assertEqual(code, 2)
self.assertIn("usage error", out.getvalue())
class LocatorTest(unittest.TestCase):
def test_equal_locators_match(self) -> None:
self.assertTrue(filetype_gate.same_locator("pdf", {"page": 3}, {"page": 3}))
self.assertTrue(filetype_gate.same_locator("docx", {"paragraph": 12}, {"paragraph": 12}))
self.assertTrue(filetype_gate.same_locator("md", {"line": 7}, {"line": 7}))
def test_a_boolean_is_not_a_number(self) -> None:
self.assertFalse(filetype_gate.same_locator("pdf", {"page": 1}, {"page": True}))
def test_cell_references_ignore_case_but_sheet_names_do_not(self) -> None:
self.assertTrue(filetype_gate.same_locator("xlsx", {"sheet": "T1", "cell": "B7"}, {"sheet": "T1", "cell": "b7"}))
self.assertFalse(filetype_gate.same_locator("xlsx", {"sheet": "T1", "cell": "B7"}, {"sheet": "t1", "cell": "B7"}))
def test_slide_notes_default_to_false(self) -> None:
self.assertTrue(filetype_gate.same_locator("pptx", {"slide": 2, "notes": False}, {"slide": 2}))
self.assertFalse(filetype_gate.same_locator("pptx", {"slide": 2, "notes": True}, {"slide": 2}))
def test_extra_or_missing_keys_do_not_match(self) -> None:
self.assertFalse(filetype_gate.same_locator("md", {"line": 7}, {"line": 7, "column": 3}))
self.assertFalse(filetype_gate.same_locator("pdf", {"page": 3}, {}))
self.assertFalse(filetype_gate.same_locator("pdf", {"page": 3}, "page 3"))
class FirstLocatorStubTest(unittest.TestCase):
"""The stub that always answers the first page, paragraph, cell, slide or line."""
def answer(self, path: Path) -> dict[str, object]:
import subprocess
q = path.parent / "q.json"
q.write_text(json.dumps({"queries": [{"id": "a", "quote": "anything"}]}))
proc = subprocess.run([PY, str(STUBS / "first_locator.py"), str(path), str(q)],
capture_output=True, text=True, check=True)
found: dict[str, object] = json.loads(proc.stdout)["answers"][0]["found"][0]
return found
def test_first_locator_per_type(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
d = Path(tmp)
(d / "a.md").write_text("x\n")
self.assertEqual(self.answer(d / "a.md"), {"locator": {"line": 1}, "hidden": False})
(d / "a.pdf").write_bytes(b"%PDF-1.4\n")
self.assertEqual(self.answer(d / "a.pdf"), {"locator": {"page": 1}, "hidden": False})
(d / "a.pptx").write_bytes(b"")
self.assertEqual(self.answer(d / "a.pptx"), {"locator": {"slide": 1, "notes": False}, "hidden": False})
(d / "a.docx").write_bytes(b"")
self.assertEqual(self.answer(d / "a.docx"), {"locator": {"paragraph": 1}, "hidden": False})
def test_first_cell_is_a1_on_the_first_sheet_in_workbook_order(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
p = Path(tmp) / "a.xlsx"
with zipfile.ZipFile(p, "w") as z:
z.writestr("xl/workbook.xml",
'<workbook xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main">'
'<sheets><sheet name="Summary &amp; notes" sheetId="2"/><sheet name="Data" sheetId="1"/>'
'</sheets></workbook>')
self.assertEqual(self.answer(p), {"locator": {"sheet": "Summary & notes", "cell": "A1"}, "hidden": False})
if __name__ == "__main__":
unittest.main()

108
gates/tests/test_keyform.py Normal file
View file

@ -0,0 +1,108 @@
"""Tests for gates/keyform.py: counts answer keys that are wording rather than a fact (E2c).
Every fixture is synthetic text written here. No corpus text is used (CLAUDE.md rule 3).
"""
from __future__ import annotations
import contextlib
import io
import json
import sys
import tempfile
import unittest
from pathlib import Path
from typing import Any
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
import keyform # noqa: E402
class WordingTests(unittest.TestCase):
def test_single_words_and_literals_are_facts(self) -> None:
for key in ("lantern", "/north", "3.1.4", "GATE_TOLL", "--toll-mode dawn", "copper-coins"):
self.assertFalse(keyform.is_wording(key), key)
def test_a_name_of_several_words_without_function_words_is_a_fact(self) -> None:
for key in ("Toll mode", "north gate keeper", "Bridge (upper)"):
self.assertFalse(keyform.is_wording(key), key)
def test_a_conjunction_makes_a_key_wording(self) -> None:
self.assertTrue(keyform.is_wording("/north or /south"))
self.assertTrue(keyform.is_wording("dawn and dusk"))
def test_a_pronoun_or_article_makes_a_key_wording(self) -> None:
self.assertTrue(keyform.is_wording("pay your toll"))
self.assertTrue(keyform.is_wording("The toll rises"))
self.assertTrue(keyform.is_wording("a lantern maker"))
def test_prepositions_auxiliaries_negations_and_particles_make_a_key_wording(self) -> None:
for key in ("toll for carters", "gate is closed", "can't open", "no toll", "only at dawn", "still open"):
self.assertTrue(keyform.is_wording(key), key)
def test_function_words_inside_backticks_are_code_not_wording(self) -> None:
self.assertFalse(keyform.is_wording("`open --if the gate` flag"))
self.assertTrue(keyform.is_wording("`toll` for carters"))
def md_set(keys: list[str]) -> dict[str, Any]:
return {"schema": "okf-wiki/multidoc-set/1", "questions": [
{"id": "q01", "question": "?", "evidence": [], "answer_keys": [{"file": "a.md", "text": k} for k in keys]}]}
def wd_set() -> dict[str, Any]:
return {"schema": "okf-wiki/wholedoc-set/1", "questions": [
{"id": "w01", "type": "whole", "question": "?", "file": "a.md", "evidence": [],
"answer_keys": [{"text": "dawn"}, {"text": "pay your toll"}, {"text": "copper coins"}]},
{"id": "w02", "type": "count", "question": "?", "file": "a.md", "evidence": [], "count": 6, "item_pattern": "x"}]}
class CountTests(unittest.TestCase):
def test_counts_every_key_of_either_schema_and_lists_the_wording_ones(self) -> None:
rep = keyform.count_keys(md_set(["dawn", "/north or /south", "Toll mode"])["questions"])
self.assertEqual((rep.wording, rep.keys), (1, 3))
self.assertEqual(rep.listed, ["q01: /north or /south"])
rep = keyform.count_keys(wd_set()["questions"])
self.assertEqual((rep.wording, rep.keys), (1, 3))
class CliTests(unittest.TestCase):
def run_cli(self, *sets: dict[str, Any]) -> tuple[int, str]:
with tempfile.TemporaryDirectory() as tmp:
args: list[str] = []
for n, data in enumerate(sets):
p = Path(tmp) / f"set{n}.json"
p.write_text(json.dumps(data), encoding="utf-8")
args += ["--set", str(p)]
out = io.StringIO()
with contextlib.redirect_stdout(out), contextlib.redirect_stderr(io.StringIO()):
code = keyform.main(args)
return code, out.getvalue()
def test_wording_keys_fail_with_x_of_n_over_all_sets(self) -> None:
code, out = self.run_cli(md_set(["dawn", "/north or /south"]), wd_set())
self.assertEqual(code, 1)
self.assertIn("answer keys that are wording: 2 of 5", out)
self.assertIn("FAIL", out)
def test_no_wording_keys_pass(self) -> None:
code, out = self.run_cli(md_set(["dawn", "/north", "/south"]))
self.assertEqual(code, 0)
self.assertIn("answer keys that are wording: 0 of 3", out)
self.assertIn("PASS", out)
def test_no_keys_at_all_is_a_fail_not_a_pass(self) -> None:
code, out = self.run_cli(md_set([]))
self.assertEqual(code, 1)
def test_unreadable_set_cannot_run(self) -> None:
err = io.StringIO()
with contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(err):
code = keyform.main(["--set", "/nonexistent/set.json"])
self.assertEqual(code, 1)
self.assertIn("FAIL", err.getvalue())
if __name__ == "__main__":
unittest.main()

View file

@ -0,0 +1,541 @@
"""Tests for gates/multidoc.py: the multi-document set checker, the held-out split and the answer gate.
Every fixture is synthetic text written here. No corpus text is used (CLAUDE.md rule 3).
"""
from __future__ import annotations
import contextlib
import hashlib
import io
import json
import sys
import tempfile
import unittest
from pathlib import Path
from typing import Any
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
import multidoc # noqa: E402
REPO_ROOT = Path(__file__).resolve().parents[2]
SOURCES = {
"alpha.md": "# Alpha\n\nThe widget timeout is 45 seconds by default.\nSet `WIDGET_MODE=fast` to skip checks.\n",
"beta.md": "# Beta\n\nBeta adds the --frobnicate flag for batch runs.\nIt was introduced in the spring build.\n",
"gamma.md": "# Gamma\n\nGamma stores its cache under the .gamma-cache directory.\n",
"delta.md": "# Delta\n\nDelta rejects requests larger than 12 megabytes with error E_TOO_BIG.\n",
"epsilon.md": "# Epsilon\n\nEpsilon logs every call to the audit trail named ledger-7.\n",
}
def good_question(qid: str = "x01") -> dict[str, Any]:
return {
"id": qid,
"question": "How do the four synthetic tools fit together?",
"evidence": [
{"file": "alpha.md", "span": "The widget timeout is 45 seconds by default."},
{"file": "beta.md", "span": "Beta adds the --frobnicate flag for batch runs."},
{"file": "gamma.md", "span": "Gamma stores its cache under the .gamma-cache directory."},
{"file": "delta.md", "span": "rejects requests larger than 12 megabytes with error E_TOO_BIG"},
],
"answer_keys": [
{"file": "alpha.md", "text": "45 seconds"},
{"file": "beta.md", "text": "--frobnicate"},
{"file": "gamma.md", "text": ".gamma-cache"},
{"file": "delta.md", "text": "E_TOO_BIG"},
],
}
def perfect_answer(q: dict[str, Any]) -> dict[str, Any]:
keys = " ".join(k["text"] for k in q["answer_keys"])
return {
"id": q["id"],
"answer": "The answer mentions " + keys + ".",
"citations": [{"file": e["file"], "quote": e["span"]} for e in q["evidence"]],
}
class Workspace:
"""A temporary sources directory and helpers to write sets, manifests and answers."""
def __init__(self) -> None:
self._tmp = tempfile.TemporaryDirectory()
self.root = Path(self._tmp.name)
self.sources = self.root / "sources"
self.sources.mkdir()
for name, text in SOURCES.items():
(self.sources / name).write_text(text, encoding="utf-8")
def close(self) -> None:
self._tmp.cleanup()
def write_set(self, questions: list[dict[str, Any]], name: str = "set.json") -> Path:
path = self.root / name
path.write_text(json.dumps({"schema": multidoc.SET_SCHEMA, "questions": questions}), encoding="utf-8")
return path
def write_json(self, name: str, data: Any) -> Path:
path = self.root / name
path.write_text(json.dumps(data), encoding="utf-8")
return path
def run(argv: list[str]) -> tuple[int, str]:
out = io.StringIO()
err = io.StringIO()
with contextlib.redirect_stdout(out), contextlib.redirect_stderr(err):
code = multidoc.main(argv)
return code, out.getvalue() + err.getvalue()
def sha256_file(path: Path) -> str:
return hashlib.sha256(path.read_bytes()).hexdigest()
class CheckTests(unittest.TestCase):
def setUp(self) -> None:
self.ws = Workspace()
def tearDown(self) -> None:
self.ws.close()
def check(self, questions: list[dict[str, Any]]) -> tuple[int, str]:
path = self.ws.write_set(questions)
return run(["check", "--set", str(path), "--sources", str(self.ws.sources)])
def test_good_set_passes_with_counts_from_the_set(self) -> None:
code, out = self.check([good_question("x01"), good_question("x02")])
self.assertEqual(code, 0, out)
self.assertIn("spans verbatim: 8 of 8", out)
self.assertIn("questions with every span verbatim: 2 of 2", out)
self.assertIn("questions with >= 4 distinct files: 2 of 2", out)
self.assertIn("questions with a key per file: 2 of 2", out)
self.assertIn("PASS", out)
def test_span_one_character_off_fails(self) -> None:
q = good_question()
q["evidence"][0]["span"] = "The widget timeout is 46 seconds by default."
code, out = self.check([q])
self.assertEqual(code, 1)
self.assertIn("spans verbatim: 3 of 4", out)
self.assertIn("questions with every span verbatim: 0 of 1", out)
self.assertIn("FAIL", out)
def test_span_match_is_exact_not_whitespace_normalised(self) -> None:
q = good_question()
q["evidence"][0]["span"] = "The widget timeout is 45 seconds by default."
code, out = self.check([q])
self.assertEqual(code, 1)
self.assertIn("spans verbatim: 3 of 4", out)
def test_three_distinct_files_fail_the_minimum(self) -> None:
q = good_question()
q["evidence"][3] = {"file": "alpha.md", "span": "Set `WIDGET_MODE=fast` to skip checks."}
q["answer_keys"][3] = {"file": "alpha.md", "text": "WIDGET_MODE=fast"}
code, out = self.check([q])
self.assertEqual(code, 1)
self.assertIn("questions with >= 4 distinct files: 0 of 1", out)
def test_file_missing_from_sources_fails(self) -> None:
q = good_question()
q["evidence"][0]["file"] = "missing.md"
q["answer_keys"][0]["file"] = "missing.md"
code, out = self.check([q])
self.assertEqual(code, 1)
self.assertIn("spans verbatim: 3 of 4", out)
def test_path_outside_sources_is_an_error(self) -> None:
for bad in ("../alpha.md", "sub/alpha.md", "/etc/hosts"):
q = good_question()
q["evidence"][0]["file"] = bad
q["answer_keys"][0]["file"] = bad
code, out = self.check([q])
self.assertEqual(code, 1, bad)
self.assertIn("not a plain file name", out)
def test_key_outside_its_files_spans_fails(self) -> None:
q = good_question()
q["answer_keys"][0]["text"] = "WIDGET_MODE" # in alpha.md, but not in the cited span
code, out = self.check([q])
self.assertEqual(code, 1)
self.assertIn("questions with a key per file: 0 of 1", out)
def test_key_match_ignores_case(self) -> None:
q = good_question()
q["answer_keys"][3]["text"] = "e_too_big"
code, out = self.check([q])
self.assertEqual(code, 0, out)
def test_evidence_file_without_a_key_fails(self) -> None:
q = good_question()
del q["answer_keys"][2]
code, out = self.check([q])
self.assertEqual(code, 1)
self.assertIn("questions with a key per file: 0 of 1", out)
def test_duplicate_ids_fail(self) -> None:
code, out = self.check([good_question("x01"), good_question("x01")])
self.assertEqual(code, 1)
self.assertIn("duplicate id", out)
def test_empty_set_is_a_fail_not_a_pass(self) -> None:
code, out = self.check([])
self.assertEqual(code, 1)
self.assertIn("FAIL", out)
def test_a_wording_key_fails_even_inside_its_span(self) -> None:
q = good_question()
q["answer_keys"][0] = {"file": "alpha.md", "text": "is 45 seconds by default"}
code, out = self.check([q])
self.assertEqual(code, 1)
self.assertIn("questions with a key per file: 1 of 1", out)
self.assertIn("answer keys that are wording: 1 of 4", out)
def test_good_set_has_no_wording_keys(self) -> None:
code, out = self.check([good_question()])
self.assertIn("answer keys that are wording: 0 of 4", out)
def test_short_span_and_short_key_fail(self) -> None:
q = good_question()
q["evidence"][0]["span"] = "45 seconds"
code, out = self.check([q])
self.assertEqual(code, 1)
self.assertIn("span length", out)
q = good_question()
q["answer_keys"][0]["text"] = "se"
code, out = self.check([q])
self.assertEqual(code, 1)
self.assertIn("key length", out)
def test_missing_sources_directory_cannot_run(self) -> None:
path = self.ws.write_set([good_question()])
code, out = run(["check", "--set", str(path), "--sources", str(self.ws.root / "nope")])
self.assertEqual(code, 1)
self.assertIn("FAIL", out)
def test_unreadable_set_cannot_run(self) -> None:
path = self.ws.root / "broken.json"
path.write_text("{not json", encoding="utf-8")
code, out = run(["check", "--set", str(path), "--sources", str(self.ws.sources)])
self.assertEqual(code, 1)
self.assertIn("FAIL", out)
def test_a_negated_key_must_stand_negated_in_its_span(self) -> None:
q = good_question()
q["answer_keys"][0]["negated"] = True
code, out = self.check([q])
self.assertEqual(code, 1)
self.assertIn("negated keys negated in their span: 0 of 1", out)
def test_a_negated_key_negated_in_its_span_passes(self) -> None:
(self.ws.sources / "alpha.md").write_text("The widget is not synced back to the hub.\n", encoding="utf-8")
q = good_question()
q["evidence"][0]["span"] = "The widget is not synced back to the hub."
q["answer_keys"][0] = {"file": "alpha.md", "text": "synced", "negated": True}
code, out = self.check([q])
self.assertEqual(code, 0, out)
self.assertIn("negated keys negated in their span: 1 of 1", out)
def test_negated_must_be_true_or_absent(self) -> None:
q = good_question()
q["answer_keys"][0]["negated"] = "yes"
code, out = self.check([q])
self.assertEqual(code, 1)
self.assertIn("negated", out)
class NegatedKeyTests(unittest.TestCase):
KEY = {"file": "a.md", "text": "synced", "negated": True}
def test_a_negation_within_six_words_before_the_key(self) -> None:
for text in ("Changes are not synced back.", "It isn't synced.", "It isn\u2019t synced.",
"Changes are no longer synced.", "It is never, ever synced.", "Not one two three four five synced.",
"Some are synced; others are not synced."):
self.assertTrue(multidoc.key_holds(self.KEY, text), text)
def test_no_negation_or_one_out_of_reach_fails(self) -> None:
for text in ("Changes are synced back.", "It is not late. Changes are synced.", "Not late; synced.",
"Not one two three four five six synced.", "Synced, but not back.", "Not here:\nsynced."):
self.assertFalse(multidoc.key_holds(self.KEY, text), text)
def test_every_negation_in_the_list(self) -> None:
for neg in ("not", "no", "never", "none", "neither", "nor", "without", "cannot", "can't", "doesn't", "won't"):
self.assertTrue(multidoc.key_holds(self.KEY, f"It {neg} synced."), neg)
def test_an_unmarked_key_holds_with_or_without_a_negation(self) -> None:
key = {"file": "a.md", "text": "synced"}
self.assertTrue(multidoc.key_holds(key, "Changes are synced back."))
self.assertTrue(multidoc.key_holds(key, "Changes are not synced back."))
def test_the_gate_scores_a_negated_key_by_its_negation(self) -> None:
ws = Workspace()
try:
(ws.sources / "alpha.md").write_text("The widget is not synced back to the hub.\n", encoding="utf-8")
q = good_question()
q["evidence"][0]["span"] = "The widget is not synced back to the hub."
q["answer_keys"][0] = {"file": "alpha.md", "text": "synced", "negated": True}
cache = multidoc.SourceCache(ws.sources)
right = perfect_answer(q)
right["answer"] = "The widget is not synced; --frobnicate, .gamma-cache, E_TOO_BIG."
wrong = dict(right, answer="The widget is synced; --frobnicate, .gamma-cache, E_TOO_BIG.")
self.assertTrue(multidoc.score_one(q, right, cache).answer_ok)
self.assertFalse(multidoc.score_one(q, wrong, cache).answer_ok)
finally:
ws.close()
class SplitTests(unittest.TestCase):
def setUp(self) -> None:
self.ws = Workspace()
self.questions = [good_question(f"x{i:02d}") for i in range(1, 11)]
for i, q in enumerate(self.questions):
q["question"] = f"Synthetic question number {i + 1}?"
self.all_path = self.ws.write_set(self.questions, "all.json")
def tearDown(self) -> None:
self.ws.close()
def split(self, seed: int, tag: str, held_dir: Path | None = None) -> tuple[int, str, Path, Path, Path]:
dev = self.ws.root / f"dev-{tag}.json"
held = (held_dir or self.ws.root / f"held-{tag}") / "held.json"
manifest = self.ws.root / f"manifest-{tag}.json"
code, out = run([
"split", "--set", str(self.all_path), "--sources", str(self.ws.sources), "--seed", str(seed),
"--held-out-count", "3", "--dev-out", str(dev), "--held-out", str(held), "--manifest", str(manifest),
])
return code, out, dev, held, manifest
def load(self, path: Path) -> list[dict[str, Any]]:
data: dict[str, Any] = json.loads(path.read_text(encoding="utf-8"))
questions: list[dict[str, Any]] = data["questions"]
return questions
def test_split_counts_renumbers_and_partitions(self) -> None:
code, out, dev, held, _ = self.split(7, "a")
self.assertEqual(code, 0, out)
dev_q, held_q = self.load(dev), self.load(held)
self.assertEqual((len(dev_q), len(held_q)), (7, 3))
self.assertEqual([q["id"] for q in dev_q], [f"d{i:02d}" for i in range(1, 8)])
self.assertEqual([q["id"] for q in held_q], [f"h{i:02d}" for i in range(1, 4)])
texts = sorted(q["question"] for q in dev_q + held_q)
self.assertEqual(texts, sorted(q["question"] for q in self.questions))
def test_same_seed_same_draw_other_seed_other_draw(self) -> None:
_, _, _, held_a, _ = self.split(7, "a")
_, _, _, held_b, _ = self.split(7, "b")
self.assertEqual(held_a.read_bytes(), held_b.read_bytes())
draws = set()
for seed in range(1, 6):
_, _, _, held, _ = self.split(seed, f"s{seed}")
draws.add(tuple(q["question"] for q in self.load(held)))
self.assertGreater(len(draws), 1)
def test_manifest_carries_hashes_and_nothing_about_the_held_out_questions(self) -> None:
code, out, dev, held, manifest = self.split(7, "a")
self.assertEqual(code, 0, out)
data = json.loads(manifest.read_text(encoding="utf-8"))
self.assertEqual(data["seed"], 7)
self.assertEqual(data["dev"]["sha256"], sha256_file(dev))
self.assertEqual(data["dev"]["questions"], 7)
self.assertEqual(data["held_out"]["sha256"], sha256_file(held))
self.assertEqual(data["held_out"]["questions"], 3)
self.assertEqual(data["sources"]["files"], len(SOURCES))
self.assertEqual(data["sources"]["tree_sha256"], multidoc.tree_sha256(self.ws.sources))
manifest_text = manifest.read_text(encoding="utf-8")
for q in self.load(held):
self.assertNotIn(q["question"], manifest_text)
self.assertNotIn(q["question"], out)
self.assertNotIn(str(held.parent), manifest_text)
def test_refuses_held_out_inside_the_repository(self) -> None:
code, out, _, _, _ = self.split(7, "in", held_dir=REPO_ROOT / "gates-local" / "should-not-exist")
self.assertEqual(code, 1)
self.assertIn("inside the repository", out)
self.assertFalse((REPO_ROOT / "gates-local" / "should-not-exist").exists())
def test_refuses_to_overwrite_a_frozen_held_out_file(self) -> None:
code, out, _, _, _ = self.split(7, "a")
self.assertEqual(code, 0, out)
code, out, _, _, _ = self.split(7, "a")
self.assertEqual(code, 1)
self.assertIn("already exists", out)
def test_refuses_a_set_that_fails_the_check(self) -> None:
self.questions[0]["evidence"][0]["span"] = "This sentence is not in any source file."
self.all_path = self.ws.write_set(self.questions, "all.json")
code, out, dev, held, _ = self.split(7, "bad")
self.assertEqual(code, 1)
self.assertFalse(dev.exists())
self.assertFalse(held.exists())
class GateTests(unittest.TestCase):
def setUp(self) -> None:
self.ws = Workspace()
self.questions = [good_question("d01"), good_question("d02")]
self.set_path = self.ws.write_set(self.questions, "dev.json")
self.manifest = self.ws.write_json("manifest.json", {
"schema": multidoc.MANIFEST_SCHEMA,
"seed": 1,
"dev": {"sha256": sha256_file(self.set_path), "questions": 2},
"held_out": {"sha256": "0" * 64, "questions": 1},
"sources": {"files": len(SOURCES), "tree_sha256": multidoc.tree_sha256(self.ws.sources)},
})
def tearDown(self) -> None:
self.ws.close()
def gate(self, answers: Path, arm: str = "collection") -> tuple[int, str]:
return run([
"gate", "--set", str(self.set_path), "--manifest", str(self.manifest), "--sources", str(self.ws.sources),
"--arm", arm, "--answers", str(answers),
])
def answers(self, entries: list[dict[str, Any]], arm: str = "collection") -> Path:
return self.ws.write_json("answers.json", {"arm": arm, "answers": entries})
def test_missing_arm_is_fail_closed_at_zero(self) -> None:
code, out = self.gate(self.ws.root / "no-such-arm.json")
self.assertEqual(code, 1)
self.assertIn("correct answer and source: 0 of 2", out)
self.assertIn("fail-closed", out)
def test_perfect_answers_score_all(self) -> None:
code, out = self.gate(self.answers([perfect_answer(q) for q in self.questions]))
self.assertEqual(code, 0, out)
self.assertIn("correct answer and source: 2 of 2", out)
def test_answer_missing_a_key_is_wrong(self) -> None:
a = perfect_answer(self.questions[0])
a["answer"] = a["answer"].replace("E_TOO_BIG", "a big error")
code, out = self.gate(self.answers([a, perfect_answer(self.questions[1])]))
self.assertEqual(code, 0, out)
self.assertIn("correct answer: 1 of 2", out)
self.assertIn("correct answer and source: 1 of 2", out)
def test_answer_key_match_ignores_case_backticks_and_whitespace(self) -> None:
a = perfect_answer(self.questions[0])
a["answer"] = "Use `--FROBNICATE`, wait 45\n seconds, look in .gamma-cache, expect e_too_big."
code, out = self.gate(self.answers([a, perfect_answer(self.questions[1])]))
self.assertIn("correct answer: 2 of 2", out)
def test_citation_not_verbatim_is_wrong_source(self) -> None:
a = perfect_answer(self.questions[0])
a["citations"][1]["quote"] = "Beta adds a flag that nobody documented."
code, out = self.gate(self.answers([a, perfect_answer(self.questions[1])]))
self.assertIn("correct source: 1 of 2", out)
self.assertIn("correct answer and source: 1 of 2", out)
def test_citation_whitespace_is_normalised(self) -> None:
a = perfect_answer(self.questions[0])
a["citations"][0]["quote"] = "The widget timeout is\n45 seconds by default."
code, out = self.gate(self.answers([a, perfect_answer(self.questions[1])]))
self.assertIn("correct source: 2 of 2", out)
def test_citations_missing_one_file_is_wrong_source(self) -> None:
a = perfect_answer(self.questions[0])
del a["citations"][2]
code, out = self.gate(self.answers([a, perfect_answer(self.questions[1])]))
self.assertIn("correct source: 1 of 2", out)
def test_citation_outside_sources_counts_for_nothing(self) -> None:
a = perfect_answer(self.questions[0])
a["citations"][0]["file"] = "../sources/alpha.md"
code, out = self.gate(self.answers([a, perfect_answer(self.questions[1])]))
self.assertIn("correct source: 1 of 2", out)
def test_missing_entry_counts_wrong_and_fails(self) -> None:
code, out = self.gate(self.answers([perfect_answer(self.questions[0])]))
self.assertEqual(code, 1)
self.assertIn("correct answer and source: 1 of 2", out)
self.assertIn("missing answers: 1", out)
def test_set_that_differs_from_the_manifest_is_refused(self) -> None:
self.set_path.write_text(self.set_path.read_text(encoding="utf-8") + "\n", encoding="utf-8")
code, out = self.gate(self.answers([perfect_answer(q) for q in self.questions]))
self.assertEqual(code, 1)
self.assertIn("sha256", out)
def test_changed_sources_are_refused(self) -> None:
(self.ws.sources / "alpha.md").write_text("changed\n", encoding="utf-8")
code, out = self.gate(self.answers([perfect_answer(q) for q in self.questions]))
self.assertEqual(code, 1)
self.assertIn("sources", out)
def test_arm_name_mismatch_is_refused(self) -> None:
code, out = self.gate(self.answers([perfect_answer(q) for q in self.questions], arm="filesearch"))
self.assertEqual(code, 1)
self.assertIn("arm", out)
def test_per_question_results_are_written_for_pairing(self) -> None:
a = perfect_answer(self.questions[0])
answers = self.answers([a])
out_path = self.ws.root / "results.json"
run([
"gate", "--set", str(self.set_path), "--manifest", str(self.manifest), "--sources", str(self.ws.sources),
"--arm", "collection", "--answers", str(answers), "--out", str(out_path),
])
results = json.loads(out_path.read_text(encoding="utf-8"))
self.assertEqual(results["arm"], "collection")
self.assertEqual([r["id"] for r in results["results"]], ["d01", "d02"])
self.assertEqual([r["correct"] for r in results["results"]], [True, False])
class NumberKeyTests(unittest.TestCase):
"""A key that is a number matches as a number: digits or its English word, never inside a longer number."""
def test_a_number_key_matches_digits_or_word_but_not_inside_a_longer_number(self) -> None:
for text in ("It retries 5 times.", "RETRIES=5", "five attempts", "(5)"):
self.assertTrue(multidoc.key_in("5", text), text)
for text in ("15 attempts", "version 2.1.5", "v5", "50", "5.2 seconds", "fifty"):
self.assertFalse(multidoc.key_in("5", text), text)
def test_a_number_word_key_matches_the_digits(self) -> None:
self.assertTrue(multidoc.key_in("three", "fails 3 times in a row"))
self.assertTrue(multidoc.key_in("Three", "three times"))
self.assertFalse(multidoc.key_in("three", "33 times"))
def test_a_percentage_key_matches_the_percentage(self) -> None:
for text in ("at 10% of the window", "at 10 % of it", "10 percent"):
self.assertTrue(multidoc.key_in("10%", text), text)
for text in ("110%", "10 of them", "2.10%"):
self.assertFalse(multidoc.key_in("10%", text), text)
def test_other_keys_still_match_as_normalised_text(self) -> None:
self.assertTrue(multidoc.key_in("model version", "Pin their **model versions**."))
self.assertTrue(multidoc.key_in("2.1.9", "since 2.1.92"))
def test_check_accepts_a_short_number_key_standing_alone_in_its_span(self) -> None:
ws = Workspace()
try:
q = good_question()
q["answer_keys"][3] = {"file": "delta.md", "text": "12"}
code, out = run(["check", "--set", str(ws.write_set([q])), "--sources", str(ws.sources)])
self.assertEqual(code, 0, out)
q["answer_keys"][3] = {"file": "delta.md", "text": "2"}
code, out = run(["check", "--set", str(ws.write_set([q])), "--sources", str(ws.sources)])
self.assertEqual(code, 1)
self.assertIn("questions with a key per file: 0 of 1", out)
finally:
ws.close()
def test_the_gate_scores_a_number_key_as_a_number(self) -> None:
ws = Workspace()
try:
q = good_question()
q["answer_keys"][3] = {"file": "delta.md", "text": "12"}
cache = multidoc.SourceCache(ws.sources)
right = perfect_answer(q)
self.assertTrue(multidoc.score_one(q, right, cache).answer_ok)
wrong = dict(right, answer=right["answer"].replace(" 12", " 112"))
self.assertFalse(multidoc.score_one(q, wrong, cache).answer_ok)
finally:
ws.close()
if __name__ == "__main__":
unittest.main()

View file

@ -0,0 +1,376 @@
"""Tests for gates/mutant_gate.py, the fail-closed runner of the auditor's mutant gate.
Every fixture here is synthetic and neutral text. The real mutants live in gates-local/ (CLAUDE.md rule 3).
Run: python3 -m unittest discover -s gates/tests
"""
from __future__ import annotations
import io
import json
import re
import sys
import tempfile
import textwrap
import unittest
from contextlib import redirect_stdout
from pathlib import Path
from typing import Any
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
import mutant_gate # noqa: E402
from okfwiki.audit import GROUP_OF # noqa: E402
PY = sys.executable
# Reads {bundle}/EXPECT: if it names a class, report exactly that class; otherwise report clean.
PERFECT = """
import json, sys, pathlib
p = pathlib.Path(sys.argv[1]) / "EXPECT"
cls = p.read_text().strip() if p.exists() else ""
if cls:
print(json.dumps({"findings": [{"check": cls}]})); sys.exit(1)
print(json.dumps({"findings": []})); sys.exit(0)
"""
REJECT_ALL = """
import json, sys
print(json.dumps({"findings": [{"check": "reject-all"}]})); sys.exit(1)
"""
CRASH = """
raise RuntimeError("boom")
"""
WRONG_CLASS = """
import json, sys, pathlib
p = pathlib.Path(sys.argv[1]) / "EXPECT"
if p.exists():
print(json.dumps({"findings": [{"check": "broken-link"}]})); sys.exit(1)
print(json.dumps({"findings": []})); sys.exit(0)
"""
NOT_JSON = """
import sys
print("findings: lots"); sys.exit(1)
"""
EXIT0_WITH_FINDINGS = """
import json
print(json.dumps({"findings": [{"check": "x"}]}))
"""
EXIT1_NO_FINDINGS = """
import json, sys
print(json.dumps({"findings": []})); sys.exit(1)
"""
SLEEPER = """
import time
time.sleep(10)
"""
VANDAL = """
import json, sys, pathlib
for f in pathlib.Path(sys.argv[1]).iterdir():
f.write_text("vandalised")
print(json.dumps({"findings": []})); sys.exit(0)
"""
SOURCES_ECHO = """
import json, sys, pathlib
ok = (pathlib.Path(sys.argv[2]) / "src" / "a.txt").read_text() == "alpha\\n"
print(json.dumps({"findings": [] if ok else [{"check": "sources-missing"}]}))
sys.exit(0 if ok else 1)
"""
CASES: dict[str, tuple[str, str | None, dict[str, str]]] = {
"baseline": ("known-positive", None, {"page.md": "---\ntype: Note\n---\nPlain text.\n"}),
"m01-alpha": ("mutant", "class-alpha", {"page.md": "---\ntype: Note\n---\nA.\n", "EXPECT": "class-alpha\n"}),
"m02-beta": ("mutant", "class-beta", {"page.md": "---\ntype: Note\n---\nB.\n", "EXPECT": "class-beta\n"}),
"m03-beta": ("mutant", "class-beta", {"page.md": "---\ntype: Note\n---\nC.\n", "EXPECT": "class-beta\n"}),
}
class GateFixture:
"""A synthetic manifest, case tree and sources tree in a temporary directory."""
def __init__(self, root: Path) -> None:
self.root = root
self.cases = root / "cases"
self.sources = root / "sources"
(self.sources / "src").mkdir(parents=True)
(self.sources / "src" / "a.txt").write_text("alpha\n")
entries = []
for name, (kind, cls, files) in CASES.items():
d = self.cases / name
d.mkdir(parents=True)
for rel, content in files.items():
(d / rel).write_text(content)
entries.append({"name": name, "kind": kind, "class": cls, "mutation": "synthetic",
"files": mutant_gate.hash_tree(d)})
self.manifest = root / "manifest.json"
self.manifest.write_text(json.dumps({
"schema": 1,
"sources": [{"path": "src/a.txt", "sha256": mutant_gate.sha256_file(self.sources / "src" / "a.txt")}],
"cases": entries,
}))
def script(self, name: str, body: str) -> Path:
p = self.root / f"{name}.py"
p.write_text(textwrap.dedent(body))
return p
def run(self, command: list[str], timeout: float = 5.0) -> tuple[int, str]:
out = io.StringIO()
with redirect_stdout(out):
code = mutant_gate.main([
"--manifest", str(self.manifest), "--cases", str(self.cases),
"--sources", str(self.sources), "--timeout", str(timeout), "--", *command,
])
return code, out.getvalue()
class MutantGateTest(unittest.TestCase):
def setUp(self) -> None:
self._tmp = tempfile.TemporaryDirectory()
self.fx = GateFixture(Path(self._tmp.name))
def tearDown(self) -> None:
self._tmp.cleanup()
# --- the three outcomes the order names ---------------------------------------------------------
def test_missing_auditor_catches_nothing_and_says_so(self) -> None:
code, out = self.fx.run([str(self.fx.root / "no-such-auditor"), "{bundle}"])
self.assertEqual(code, 1)
self.assertIn("caught 0 of 3", out)
self.assertIn("errors 4 of 4", out)
self.assertIn("auditor not found", out)
self.assertIn("verdict: RED", out)
def test_reject_all_catches_everything_but_raises_a_false_alarm(self) -> None:
code, out = self.fx.run([PY, str(self.fx.script("reject", REJECT_ALL)), "{bundle}"])
self.assertEqual(code, 1)
self.assertIn("caught 3 of 3", out)
self.assertIn("false alarms 1 of 1", out)
self.assertIn("class matched 0 of 3", out)
self.assertIn("verdict: RED", out)
def test_perfect_auditor_turns_the_gate_green(self) -> None:
code, out = self.fx.run([PY, str(self.fx.script("perfect", PERFECT)), "{bundle}"])
self.assertEqual(code, 0, out)
self.assertIn("caught 3 of 3", out)
self.assertIn("class matched 3 of 3", out)
self.assertIn("false alarms 0 of 1", out)
self.assertIn("errors 0 of 4", out)
self.assertIn("verdict: GREEN", out)
# --- fail-closed: anything but a well-formed verdict is an error, never a catch --------------------
def test_crashing_auditor_is_an_error_not_a_catch(self) -> None:
code, out = self.fx.run([PY, str(self.fx.script("crash", CRASH)), "{bundle}"])
self.assertEqual(code, 1)
self.assertIn("caught 0 of 3", out)
self.assertIn("errors 4 of 4", out)
def test_output_that_is_not_json_is_an_error(self) -> None:
_, out = self.fx.run([PY, str(self.fx.script("notjson", NOT_JSON)), "{bundle}"])
self.assertIn("caught 0 of 3", out)
self.assertIn("not valid JSON", out)
def test_exit_status_must_agree_with_findings(self) -> None:
_, out0 = self.fx.run([PY, str(self.fx.script("e0", EXIT0_WITH_FINDINGS)), "{bundle}"])
self.assertIn("caught 0 of 3", out0)
self.assertIn("errors 4 of 4", out0)
_, out1 = self.fx.run([PY, str(self.fx.script("e1", EXIT1_NO_FINDINGS)), "{bundle}"])
self.assertIn("caught 0 of 3", out1)
self.assertIn("errors 4 of 4", out1)
def test_timeout_is_an_error(self) -> None:
_, out = self.fx.run([PY, str(self.fx.script("sleep", SLEEPER)), "{bundle}"], timeout=0.5)
self.assertIn("caught 0 of 3", out)
self.assertIn("timed out", out)
def test_alarm_for_the_wrong_class_is_caught_but_not_class_matched(self) -> None:
code, out = self.fx.run([PY, str(self.fx.script("wrong", WRONG_CLASS)), "{bundle}"])
self.assertEqual(code, 1)
self.assertIn("caught 3 of 3", out)
self.assertIn("class matched 0 of 3", out)
self.assertIn("verdict: RED", out)
# --- reporting -------------------------------------------------------------------------------------
def test_report_lists_what_slipped_through_and_n_per_class(self) -> None:
_, out = self.fx.run([str(self.fx.root / "no-such-auditor"), "{bundle}"])
self.assertIn("slipped through: m01-alpha, m02-beta, m03-beta", out)
self.assertIn("class-alpha: 1", out)
self.assertIn("class-beta: 2", out)
def test_sources_placeholder_is_substituted(self) -> None:
code, out = self.fx.run([PY, str(self.fx.script("src", SOURCES_ECHO)), "{bundle}", "{sources}"])
self.assertIn("false alarms 0 of 1", out)
self.assertNotIn("sources-missing", out)
self.assertEqual(code, 1) # this auditor never catches a mutant
# --- the fixtures themselves are verified before anything runs --------------------------------------
def test_auditor_cannot_alter_the_fixtures(self) -> None:
self.fx.run([PY, str(self.fx.script("vandal", VANDAL)), "{bundle}"])
code, _ = self.fx.run([PY, str(self.fx.script("perfect", PERFECT)), "{bundle}"])
self.assertEqual(code, 0)
def test_changed_case_file_stops_the_gate(self) -> None:
(self.fx.cases / "m01-alpha" / "page.md").write_text("changed\n")
code, out = self.fx.run([PY, str(self.fx.script("perfect", PERFECT)), "{bundle}"])
self.assertEqual(code, 2)
self.assertIn("sha256 mismatch", out)
def test_extra_or_missing_case_file_stops_the_gate(self) -> None:
(self.fx.cases / "m02-beta" / "extra.md").write_text("x\n")
code, out = self.fx.run([PY, str(self.fx.script("perfect", PERFECT)), "{bundle}"])
self.assertEqual(code, 2)
self.assertIn("unexpected file", out)
(self.fx.cases / "m02-beta" / "extra.md").unlink()
(self.fx.cases / "m03-beta" / "page.md").unlink()
code, out = self.fx.run([PY, str(self.fx.script("perfect", PERFECT)), "{bundle}"])
self.assertEqual(code, 2)
self.assertIn("missing file", out)
def test_changed_source_stops_the_gate(self) -> None:
(self.fx.sources / "src" / "a.txt").write_text("beta\n")
code, out = self.fx.run([PY, str(self.fx.script("perfect", PERFECT)), "{bundle}"])
self.assertEqual(code, 2)
self.assertIn("source", out)
def test_manifest_needs_exactly_one_known_positive(self) -> None:
data = json.loads(self.fx.manifest.read_text())
data["cases"] = [c for c in data["cases"] if c["kind"] != "known-positive"]
self.fx.manifest.write_text(json.dumps(data))
code, out = self.fx.run([PY, str(self.fx.script("perfect", PERFECT)), "{bundle}"])
self.assertEqual(code, 2)
self.assertIn("known-positive", out)
def test_missing_command_is_a_usage_error(self) -> None:
out = io.StringIO()
with redirect_stdout(out):
code = mutant_gate.main(["--manifest", str(self.fx.manifest), "--cases", str(self.fx.cases),
"--sources", str(self.fx.sources)])
self.assertEqual(code, 2)
class TrackedManifestTest(unittest.TestCase):
"""The tracked manifest must keep a mutant for every check the L1 row in PLAN.md promises (E1b)."""
LINK_CLASSES = ("broken-link", "orphan-page", "index-coverage", "link-misplaced")
# Wave 1 order 4: criterion 6 under the operator's rule of 06.10.2026 (choice 2), contract K2 in
# docs/bolge1/0-felles.local.md § 4. Every main-topic source has one main page (x-main-for), and every other
# concept page has a link from a main page. The auditor emits none of these four classes yet; Step 7 adds them.
K2_CLASSES = ("main-page-missing", "main-page-duplicate", "main-page-invalid", "no-main-page-link")
# One mutant per case of order 4, point 3; the key is the case, the value the class its mutant must carry.
K2_CASES = {
"main-page-missing: the main page loses its mark for one source": "main-page-missing",
"main-page-duplicate: a non-main page marks the same source": "main-page-duplicate",
"main-page-invalid: the value is not a list": "main-page-invalid",
"main-page-invalid: a value that is not among the page's own sources[].resource": "main-page-invalid",
"main-page-invalid: a release-note path, in K1 but not main_topic": "main-page-invalid",
"no-main-page-link: linked only from the other non-main page": "no-main-page-link",
"no-main-page-link: linked only from index.md": "no-main-page-link",
"no-main-page-link: linked only from log.md": "no-main-page-link",
"no-main-page-link: linked only from itself": "no-main-page-link",
"link-misplaced: the main page's only link to a non-main page sits in code": "link-misplaced",
"link-misplaced: the main page's only link to a non-main page sits in a heading": "link-misplaced",
"link-misplaced: the main page's only link to a non-main page sits in a footnote definition": "link-misplaced",
# K2 point 2, last sentence: the role follows the valid values. The gate counts a mutant matched when its own
# class is among the findings, so each case is one whose own class a wrong reading of the role cannot give.
"main-page-duplicate: an invalid value does not take the main role from a page's valid values": "main-page-duplicate",
"no-main-page-link: a page whose every value is invalid is not a main page": "no-main-page-link",
}
RELEASE = re.compile(r"^kilde/release-v\d+\.\d+\.\d+\.md$")
def path(self) -> Path:
return Path(__file__).resolve().parents[1] / "mutants.manifest.json"
def manifest(self) -> mutant_gate.Manifest:
return mutant_gate.load_manifest(self.path())
def k2(self) -> dict[str, Any]:
k2 = json.loads(self.path().read_text(encoding="utf-8"))["provenance"].get("k2")
self.assertIsInstance(k2, dict, "provenance.k2 is missing")
return k2 # type: ignore[no-any-return]
def test_every_link_class_has_a_mutant(self) -> None:
classes = {m.cls for m in self.manifest().mutants}
for cls in self.LINK_CLASSES + self.K2_CLASSES:
self.assertIn(cls, classes)
def test_every_k2_case_has_its_own_mutant(self) -> None:
manifest = self.manifest()
self.assertEqual(len(manifest.mutants), 37)
self.assertEqual(manifest.known_positive.name, "baseline")
by_name = {m.name: m for m in manifest.mutants}
coverage = self.k2()["coverage"]
named = []
for case, cls in self.K2_CASES.items():
self.assertIn(case, coverage)
name = coverage[case]
self.assertIn(name, by_name, case)
self.assertEqual(by_name[name].cls, cls, case)
named.append(name)
self.assertEqual(len(named), len(set(named)), "two cases share a mutant")
def test_the_inbound_rule_of_04_10_has_no_mutant_left(self) -> None:
# link-rule (an inbound link from any other concept page) goes in Step 7; m24, m28 and m29 now carry K2.
self.assertEqual([m.name for m in self.manifest().mutants if m.cls == "link-rule"], [])
def test_every_mutant_class_is_a_check_the_auditor_emits_or_k2_names(self) -> None:
expected = set(GROUP_OF) | set(self.K2_CLASSES)
for m in self.manifest().mutants:
self.assertIn(m.cls, expected, m.name)
def test_the_sources_hold_the_k1_fixture_and_a_release_note(self) -> None:
k2 = self.k2()
self.assertEqual(k2["sources_root"], "gates-local/mutants-kilder")
self.assertEqual(k2["k1_file"], "source-manifest.json")
self.assertEqual(len(k2["main_topic"]), 3)
self.assertTrue(all(p.startswith("kilde/docs-en-") for p in k2["main_topic"]), k2["main_topic"])
self.assertTrue(any(self.RELEASE.match(p) for p in k2["not_main_topic"]), k2["not_main_topic"])
sources = self.manifest().sources
self.assertEqual(set(sources), {k2["k1_file"], *k2["main_topic"], *k2["not_main_topic"]})
self.assertEqual(k2["k1_sha256"], sources[k2["k1_file"]], "the K1 file the gate checks is the one described")
def test_every_case_documents_its_expected_extras(self) -> None:
cases = json.loads(self.path().read_text(encoding="utf-8"))["cases"]
known = set(GROUP_OF) | set(self.K2_CLASSES)
for c in cases:
extra = c.get("expected_extra")
self.assertIsInstance(extra, list, c["name"])
self.assertLessEqual(set(extra), known - {"link-rule", c["class"]}, c["name"])
def test_the_known_positive_meets_k2(self) -> None:
k2 = self.k2()
kp = k2["known_positive"]
files = self.manifest().known_positive.files
concepts = {f for f in files if f.endswith(".md") and f.rsplit("/", 1)[-1] not in ("index.md", "log.md")}
main = kp["x-main-for"]
self.assertEqual(len(main), 1, "one main page for the three documentation sources")
(values,) = main.values()
self.assertEqual(sorted(values), sorted("/" + p for p in k2["main_topic"]))
non_main = kp["non_main_pages"]
self.assertGreaterEqual(len(non_main), 2)
self.assertIn("bypasspermissions-modus.md", non_main)
self.assertEqual(set(main) | set(non_main), concepts)
self.assertFalse(set(main) & set(non_main))
for page in non_main:
self.assertTrue(kp["links_from_main_pages"][page], page)
self.assertLessEqual(set(kp["links_from_main_pages"][page]), set(main), page)
for page in main:
self.assertEqual(kp["concept_links_into_main_pages"][page], [], "a main page needs no inbound link")
self.assertTrue(kp["cites_release_notes"])
self.assertLessEqual(set(kp["cites_release_notes"]), set(non_main))
if __name__ == "__main__":
unittest.main()

View file

@ -0,0 +1,136 @@
"""Tests for gates/polarity.py: counts polarity keys that accept an answer of the opposite polarity (E2d).
Every fixture is synthetic text written here. No corpus text is used (CLAUDE.md rule 3).
"""
from __future__ import annotations
import contextlib
import io
import json
import sys
import tempfile
import unittest
from pathlib import Path
from typing import Any
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
import polarity # noqa: E402
from multidoc import RunError # noqa: E402
def md_question(qid: str, span: str, keys: list[dict[str, Any]]) -> dict[str, Any]:
return {"id": qid, "question": "?", "evidence": [{"file": "a.md", "span": "Unrelated words about the ferry."},
{"file": "b.md", "span": span}],
"answer_keys": [dict(k, file="b.md") for k in keys]}
def run(argv: list[str]) -> tuple[int, str]:
out, err = io.StringIO(), io.StringIO()
with contextlib.redirect_stdout(out), contextlib.redirect_stderr(err):
code = polarity.main(argv)
return code, out.getvalue() + err.getvalue()
class FlipTests(unittest.TestCase):
def opposite(self, sentence: str, atom: str) -> str:
found = polarity.answer_from_span(atom, [sentence])
assert found is not None
return polarity.flip(*found)
def test_a_negation_in_the_window_is_taken_out(self) -> None:
self.assertEqual(self.opposite("The lantern is not lit at dawn", "lit"), "the lantern is lit at dawn")
self.assertEqual(self.opposite("The gate isn't locked", "locked"), "the gate is locked")
self.assertEqual(self.opposite("Carters can't cross the bridge", "cross"), "carters can cross the bridge")
self.assertEqual(self.opposite("A pass is no longer required", "required"), "a pass is required")
self.assertEqual(self.opposite("It sails without a pilot", "pilot"), "it sails with a pilot")
def test_a_negation_far_before_the_atom_is_outside_the_window(self) -> None:
self.assertEqual(self.opposite("Never mind: the one two three four five six keeper rings", "rings"),
"never mind: the one two three four five six keeper not rings")
def test_without_a_negation_not_is_put_in_front_of_the_atom(self) -> None:
self.assertEqual(self.opposite("The keeper still rings the bell", "rings"), "the keeper still not rings the bell")
def test_the_sentence_with_the_negated_atom_is_chosen(self) -> None:
found = polarity.answer_from_span("lit", ["The lantern is lit at dusk. At dawn it is not lit."])
self.assertEqual(found, ("at dawn it is not lit", len("at dawn it is not ")))
class CountTests(unittest.TestCase):
def test_an_unmarked_negation_key_accepts_the_opposite_answer(self) -> None:
q = md_question("q01", "The ferry is not staffed at night.", [{"text": "staffed"}])
rep = polarity.count_polarity([q], [{"id": "q01", "atoms": ["staffed"]}])
self.assertEqual((rep.keys, rep.accept_span, rep.accept_opposite), (1, 1, 1))
self.assertFalse(rep.passed)
def test_a_negated_key_refuses_the_opposite_answer(self) -> None:
q = md_question("q01", "The ferry is not staffed at night.", [{"text": "staffed", "negated": True}])
rep = polarity.count_polarity([q], [{"id": "q01", "atoms": ["staffed"]}])
self.assertEqual((rep.keys, rep.accept_span, rep.accept_opposite), (1, 1, 0))
self.assertTrue(rep.passed)
def test_a_key_of_several_atoms_counts_once(self) -> None:
q = md_question("q01", "Only the north gate opens at dawn.",
[{"text": "north gate"}, {"text": "dawn"}])
rep = polarity.count_polarity([q], [{"id": "q01", "atoms": ["north gate", "dawn"]}])
self.assertEqual((rep.keys, rep.accept_span, rep.accept_opposite), (1, 1, 1))
def test_an_atom_that_is_not_a_key_cannot_run(self) -> None:
q = md_question("q01", "The ferry is not staffed at night.", [{"text": "staffed"}])
with self.assertRaises(RunError):
polarity.count_polarity([q], [{"id": "q01", "atoms": ["night"]}])
def test_an_atom_outside_its_spans_cannot_run(self) -> None:
q = md_question("q01", "The ferry is not staffed at night.", [{"text": "pilot"}])
with self.assertRaises(RunError):
polarity.count_polarity([q], [{"id": "q01", "atoms": ["pilot"]}])
def test_a_whole_document_key_uses_every_span_of_the_file(self) -> None:
q = {"id": "w01", "type": "whole", "question": "?", "file": "a.md",
"evidence": [{"span": "The bell rings at noon."}, {"span": "The keeper never rings it at night."}],
"answer_keys": [{"text": "rings"}]}
rep = polarity.count_polarity([q], [{"id": "w01", "atoms": ["rings"]}])
self.assertEqual(rep.listed[0].split("| ")[1], "the keeper rings it at night")
class CliTests(unittest.TestCase):
def setUp(self) -> None:
self._tmp = tempfile.TemporaryDirectory()
self.root = Path(self._tmp.name)
def tearDown(self) -> None:
self._tmp.cleanup()
def write(self, name: str, data: Any) -> str:
(self.root / name).write_text(json.dumps(data), encoding="utf-8")
return str(self.root / name)
def test_counts_and_fails_while_a_key_accepts_the_opposite(self) -> None:
s = self.write("set.json", {"schema": "okf-wiki/multidoc-set/1", "questions": [
md_question("q01", "The ferry is not staffed at night.", [{"text": "staffed"}])]})
k = self.write("keys.json", {"schema": polarity.KEYS_SCHEMA, "keys": [{"id": "q01", "atoms": ["staffed"]}]})
code, out = run(["--set", s, "--keys", k])
self.assertEqual(code, 1, out)
self.assertIn("polarity keys that accept the answer from their span: 1 of 1", out)
self.assertIn("polarity keys that accept the opposite answer: 1 of 1", out)
self.assertIn("FAIL", out)
def test_an_empty_list_is_a_fail_not_a_pass(self) -> None:
s = self.write("set.json", {"schema": "okf-wiki/multidoc-set/1", "questions": []})
k = self.write("keys.json", {"schema": polarity.KEYS_SCHEMA, "keys": []})
code, out = run(["--set", s, "--keys", k])
self.assertEqual(code, 1)
self.assertIn("0 of 0", out)
def test_a_list_without_its_schema_cannot_run(self) -> None:
s = self.write("set.json", {"schema": "okf-wiki/multidoc-set/1", "questions": []})
k = self.write("keys.json", {"keys": []})
code, out = run(["--set", s, "--keys", k])
self.assertEqual(code, 1)
self.assertIn("schema", out)
if __name__ == "__main__":
unittest.main()

View file

@ -0,0 +1,182 @@
"""Tests for gates/quote_rule.py: rule E against rule L, the answer key and the two new reject rules (V1 Step 4).
Every fixture is synthetic gardening text (CLAUDE.md rule 3). The P1 fragments are measured by the CLI, outside the tests.
Run: python3 -m unittest discover -s gates/tests -p test_quote_rule.py
"""
from __future__ import annotations
import contextlib
import importlib
import io
import json
import sys
import tempfile
import unittest
from pathlib import Path
from typing import Any
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
import quote_rule # noqa: E402
REPO_ROOT = Path(__file__).resolve().parents[2]
def frag(text: str, quote: str | None = None, concept: str = "Basil", fid: str = "basil",
status: str = "existing") -> dict[str, Any]:
f: dict[str, Any] = {"concept": concept, "status": status, "id": fid, "text": text,
"anchor": {"file": "docs-en-basil.md", "start": 3, "end": 3}}
if quote is not None:
f["quote"] = quote
return f
INDEX: dict[str, Any] = {"entries": [
{"id": "basil", "title": "Basil", "aliases": ["Basil", "Sweet basil"]},
{"id": "soil-mix", "title": "Soil mix", "aliases": ["Soil mix"]},
]}
class RulesTest(unittest.TestCase):
def test_e_rejects_a_quoteless_prose_fragment_and_l_accepts_it(self) -> None:
f = frag("Basil likes a warm and sheltered spot.")
self.assertTrue(quote_rule.rule_e(f))
self.assertFalse(quote_rule.rule_l(f))
def test_both_reject_a_quoteless_narrow_list_word(self) -> None:
f = frag("Basil needs six hours of sun.")
self.assertTrue(quote_rule.rule_e(f))
self.assertTrue(quote_rule.rule_l(f))
def test_both_reject_a_quoteless_digit_or_backticked_name(self) -> None:
for text in ("Basil needs 6 hours of sun.", "Pinch the top `leaf` pairs."):
self.assertTrue(quote_rule.rule_e(frag(text)), text)
self.assertTrue(quote_rule.rule_l(frag(text)), text)
def test_l_lets_several_through(self) -> None:
f = frag("Sow several seeds in each pot.")
self.assertFalse(quote_rule.rule_l(f))
self.assertTrue(quote_rule.number_words(f)) # the answer key catches it
def test_a_quoted_fragment_passes_both(self) -> None:
f = frag("Basil needs six hours of sun.", quote="six hours of sun")
self.assertFalse(quote_rule.rule_e(f))
self.assertFalse(quote_rule.rule_l(f))
def test_a_blank_quote_counts_as_no_quote(self) -> None:
self.assertTrue(quote_rule.rule_e(frag("Basil likes warmth.", quote=" ")))
def test_number_words_are_whole_words(self) -> None:
self.assertFalse(quote_rule.number_words(frag("Often the seedlings wilt."))) # "ten" inside "often"
self.assertEqual(quote_rule.number_words(frag("Water Twice a week, at least once.")), ["once", "twice"])
def test_the_narrow_list_is_a_strict_subset_of_the_broad_list(self) -> None:
self.assertLess(set(quote_rule.NARROW), set(quote_rule.BROAD))
for w in ("zero", "twenty", "hundred", "thousand", "million"):
self.assertIn(w, quote_rule.NARROW)
for w in ("thirty", "ninety", "dozen", "once", "twice", "first", "tenth", "half", "double", "pair", "couple",
"several", "single", "dozens"):
self.assertIn(w, quote_rule.BROAD)
self.assertNotIn(w, quote_rule.NARROW)
class ChooseTest(unittest.TestCase):
def test_l_only_when_nothing_slips_and_e_rejects_five_percent_more(self) -> None:
self.assertEqual(quote_rule.choose(e_rejects=133, l_rejects=100, l_slips=0, n=650), "L")
self.assertEqual(quote_rule.choose(e_rejects=132, l_rejects=100, l_slips=0, n=650), "E")
self.assertEqual(quote_rule.choose(e_rejects=133, l_rejects=100, l_slips=1, n=650), "E")
def test_the_boundary_is_33_of_650(self) -> None:
self.assertEqual(quote_rule.five_percent(650), 33)
class RejectRulesTest(unittest.TestCase):
def test_short_quote(self) -> None:
self.assertTrue(quote_rule.short_quote(frag("Basil needs sun.", quote="needs sun")))
self.assertFalse(quote_rule.short_quote(frag("Basil needs sun.", quote="needs full sun")))
self.assertFalse(quote_rule.short_quote(frag("Basil needs sun."))) # no quote is rule E/L's business
def test_id_mismatch(self) -> None:
self.assertFalse(quote_rule.id_mismatch(frag("x", concept="Basil"), INDEX))
self.assertFalse(quote_rule.id_mismatch(frag("x", concept="sweet basil"), INDEX)) # an alias
self.assertFalse(quote_rule.id_mismatch(frag("x", concept="The Soil mix", fid="soil-mix"), INDEX)) # norm
self.assertTrue(quote_rule.id_mismatch(frag("x", concept="Pruning", fid="basil"), INDEX))
self.assertTrue(quote_rule.id_mismatch(frag("x", concept="Basil", fid="no-such-id"), INDEX))
self.assertFalse(quote_rule.id_mismatch(frag("x", concept="Pruning", status="new"), INDEX))
def test_id_mismatch_accepts_a_jaccard_match(self) -> None:
index = {"entries": [{"id": "pot-size", "title": "Pot size for basil", "aliases": []}]}
self.assertFalse(quote_rule.id_mismatch(frag("x", concept="Pot size for herbs", fid="pot-size"), index))
class TrialConstantsTest(unittest.TestCase):
"""The copies of the trial's constants and norm stay equal to it (read at run time, so mypy does not follow)."""
def test_copies_equal_the_trial(self) -> None:
sys.path.insert(0, str(REPO_ROOT))
sys.path.insert(0, str(REPO_ROOT / "evals"))
trial = importlib.import_module("p1.kpack")
self.assertEqual(quote_rule.NEEDS_QUOTE.pattern, getattr(trial, "NEEDS_QUOTE").pattern)
self.assertEqual(quote_rule.MERGE_JACCARD, getattr(trial, "MERGE_JACCARD"))
for name in ("The Soil mixes", "`leaf` pairs", "Pot size / basil", "glass", "Sweet basil."):
self.assertEqual(quote_rule.norm(name), getattr(trial, "norm")(name), name)
class MeasureCliTest(unittest.TestCase):
def setUp(self) -> None:
self._tmp = tempfile.TemporaryDirectory()
self.root = Path(self._tmp.name)
self.frags = self.root / "K-fragments"
self.frags.mkdir()
self.index = self.root / "index.json"
self.index.write_text(json.dumps(INDEX))
def tearDown(self) -> None:
self._tmp.cleanup()
def write(self, *fragments: dict[str, Any]) -> None:
(self.frags / "pack-1.json").write_text(json.dumps({"fragments": list(fragments)}))
def run_cli(self) -> tuple[int, dict[str, Any] | None, str]:
out = io.StringIO()
with contextlib.redirect_stdout(out):
code = quote_rule.main(["measure", str(self.frags), str(self.index)])
try:
return code, json.loads(out.getvalue()), out.getvalue()
except json.JSONDecodeError:
return code, None, out.getvalue()
def test_counts_and_choice(self) -> None:
self.write(frag("Basil likes warmth."), frag("Sow several seeds."), frag("Basil needs six hours of sun."),
frag("Basil needs sun.", quote="Basil needs warm soil"), frag("x", concept="Pruning"))
code, out, raw = self.run_cli()
self.assertEqual(code, 0, raw)
assert out is not None
self.assertEqual(out["fragments"], 5)
self.assertEqual(out["e_rejects"], 4)
self.assertEqual(out["l_rejects"], 1)
self.assertEqual(out["key_hits"], 2)
self.assertEqual(out["l_slips"], 1)
self.assertEqual(out["e_slips"], 0)
self.assertEqual(out["id_mismatches"], 1)
self.assertEqual(out["short_quotes"], 0)
self.assertEqual(out["choice"], "E")
self.assertEqual(out["key_hits_per_file"], {"pack-1.json": 2})
self.assertNotIn("Basil", raw) # counts only, never fragment text
def test_no_known_positive_fails(self) -> None:
self.write(frag("Basil likes warmth."), frag("Basil needs sun.", quote="Basil needs warm soil"))
code, out, raw = self.run_cli()
self.assertEqual(code, 1, raw)
assert out is not None
self.assertEqual(out["key_hits"], 0)
self.assertIsNone(out["choice"])
def test_no_fragments_cannot_run(self) -> None:
code, _, _ = self.run_cli()
self.assertEqual(code, 2)
if __name__ == "__main__":
unittest.main()

View file

@ -0,0 +1,332 @@
"""Tests for gates/repair_gate.py, the fail-closed judge of the repair command (order 34, contracts K18 and K19).
Order 34 A3: the three stubs in gates/repair/stubs/ (one that does nothing, one that refuses everything, one that gives
every page a new generated.at) each turn the gate RED; a manifest whose fixtures have changed stops the gate (exit 2);
and a missing command, or a missing module, is RED. Beside them, the parts of the gate a command cannot be run against
yet (the repair command does not exist when this is written) are tried with handmade outputs: the base the gate builds,
the keys it computes, the checks of task and linkset, and a refusal that must name its code and write nothing. Every
fixture is synthetic text about beekeeping (CLAUDE.md rule 3).
Run: python3 -m unittest discover -s gates/tests -p 'test_repair_gate.py'
"""
from __future__ import annotations
import io
import json
import shutil
import sys
import tempfile
import unittest
from contextlib import redirect_stdout
from pathlib import Path
from typing import Any
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
import repair_gate as gate # noqa: E402
from okfwiki import audit, claims, kassemble, klink, kpack # noqa: E402
GATES = Path(__file__).resolve().parents[1]
MANIFEST = GATES / "repair.manifest.json"
STUBS = GATES / "repair" / "stubs"
PY = sys.executable
# The kinds of case order 34 A1 asks for, by the names the manifest gives them.
REQUIRED = (
"replace-one-with-one", "replace-one-with-two", "two-sentence-fragment", "merged-fragment", "release-note-fragment",
"context-kept", "context-other", "remove-one", "remove-last-on-page", "remove-last-on-main-page",
"remove-last-from-source-on-main-page", "shared-footnote", "equal-sentences-first-removed", "repair-makes-equal",
"link-anchor-stays", "link-anchor-gone", "unlink", "relink", "linkset-two-candidates", "two-repairs-two-pages",
"refuse-quote-not-verbatim", "refuse-quote-other-source", "refuse-unknown-key", "refuse-stale-bundle",
"refuse-duplicate-answer", "refuse-stale-list", "refuse-relink-missing-target", "refuse-unlink-wrong-at",
"refuse-unlink-wrong-anchor", "identity",
)
def run_gate(*command: str, manifest: Path = MANIFEST) -> tuple[int, str]:
out = io.StringIO()
with redirect_stdout(out):
code = gate.main(["--manifest", str(manifest), "--timeout", "60", "--", *command])
return code, out.getvalue()
def case(name: str) -> dict[str, Any]:
cases: list[dict[str, Any]] = gate.load_manifest(MANIFEST)["cases"]
return next(c for c in cases if c["name"] == name)
class StubsTest(unittest.TestCase):
"""A3: each stub is RED, and no case passes with it."""
def assert_red(self, stub: str) -> str:
code, out = run_gate(PY, str(STUBS / stub))
self.assertEqual(code, 1, out)
self.assertIn("verdict: RED", out)
n = len(gate.load_manifest(MANIFEST)["cases"])
self.assertIn(f"passed 0 of {n}", out)
return out
def test_a_command_that_does_nothing_is_red(self) -> None:
out = self.assert_red("noop.py")
self.assertIn("pages changed: [], the case says ['honey-harvest.md']", out)
def test_a_command_that_refuses_everything_is_red(self) -> None:
out = self.assert_red("refuse_all.py")
self.assertIn("refusal codes ['x'], the case says ['unknown-key']", out)
self.assertIn("1 refusal line(s) name no claim or record", out)
def test_a_command_that_touches_every_page_is_red(self) -> None:
out = self.assert_red("touch_generated.py")
lines = out.splitlines()
at = next(i for i, line in enumerate(lines) if line.startswith(" identity "))
identity = "\n".join(line for line in lines[at + 1:at + 4] if line.startswith(" "))
self.assertIn("pages changed: ['brood.md', 'frames.md', 'hive.md'", identity)
class CannotRunTest(unittest.TestCase):
def setUp(self) -> None:
self._tmp = tempfile.TemporaryDirectory()
self.tmp = Path(self._tmp.name)
shutil.copyfile(MANIFEST, self.tmp / MANIFEST.name)
shutil.copytree(GATES / "repair", self.tmp / "repair", ignore=shutil.ignore_patterns("__pycache__"))
self.manifest = self.tmp / MANIFEST.name
def tearDown(self) -> None:
self._tmp.cleanup()
def test_a_changed_fixture_stops_the_gate(self) -> None:
p = self.tmp / "repair" / "kilder" / "kilde" / "docs-en-hive.md"
p.chmod(0o644)
p.write_text(p.read_text(encoding="utf-8").replace("warm and calm", "cold and windy"), encoding="utf-8")
code, out = run_gate(PY, str(STUBS / "noop.py"), manifest=self.manifest)
self.assertEqual(code, 2, out)
self.assertIn("GATE CANNOT RUN", out)
self.assertIn("sha256 mismatch in kilder/kilde/docs-en-hive.md", out)
def test_an_extra_or_a_missing_fixture_stops_the_gate(self) -> None:
(self.tmp / "repair" / "extra.json").write_text("{}", encoding="utf-8")
(self.tmp / "repair" / "agent.md").unlink()
code, out = run_gate(PY, str(STUBS / "noop.py"), manifest=self.manifest)
self.assertEqual(code, 2, out)
self.assertIn("unexpected file extra.json", out)
self.assertIn("missing file agent.md", out)
def test_a_changed_case_is_no_other_manifest_than_the_one_reported(self) -> None:
doc = json.loads(self.manifest.read_text(encoding="utf-8"))
doc["cases"][0]["steps"][0]["exit"] = 2
self.manifest.write_text(json.dumps(doc), encoding="utf-8")
code, out = run_gate(PY, str(STUBS / "noop.py"), manifest=self.manifest)
self.assertEqual(code, 2, out)
self.assertIn("a step needs argv and exit 0 or 1", out)
def test_a_refusal_code_the_manifest_does_not_define_stops_the_gate(self) -> None:
doc = json.loads(self.manifest.read_text(encoding="utf-8"))
del doc["refusal_codes"]["unknown-key"]
self.manifest.write_text(json.dumps(doc), encoding="utf-8")
code, out = run_gate(PY, str(STUBS / "noop.py"), manifest=self.manifest)
self.assertEqual(code, 2, out)
self.assertIn("refusal codes not in the manifest", out)
def test_no_command_is_a_usage_error(self) -> None:
code, out = run_gate(manifest=self.manifest)
self.assertEqual(code, 2)
self.assertIn("usage error", out)
def test_a_missing_command_is_red(self) -> None:
code, out = run_gate(str(self.tmp / "no-such-command"))
self.assertEqual(code, 1, out)
self.assertIn("command not found", out)
self.assertIn("verdict: RED", out)
def test_a_missing_module_is_red(self) -> None:
code, out = run_gate(PY, "-m", "okfwiki.no_such_module_here")
self.assertEqual(code, 1, out)
self.assertIn("No module named okfwiki.no_such_module_here", out)
self.assertIn("passed 0 of", out)
class BaseTest(unittest.TestCase):
"""The collection the gate builds, and the checks a command cannot be run against yet."""
_tmp: tempfile.TemporaryDirectory[str]
base: gate.Base
manifest: dict[str, Any]
@classmethod
def setUpClass(cls) -> None:
cls._tmp = tempfile.TemporaryDirectory()
cls.manifest = gate.load_manifest(MANIFEST)
gate.verify_fixtures(cls.manifest, GATES / "repair")
cls.base = gate.build_base(cls.manifest, GATES / "repair", Path(cls._tmp.name))
@classmethod
def tearDownClass(cls) -> None:
cls._tmp.cleanup()
def setUp(self) -> None:
self._case_tmp = tempfile.TemporaryDirectory()
self.tmp = Path(self._case_tmp.name)
def tearDown(self) -> None:
self._case_tmp.cleanup()
def test_the_base_is_clean_and_has_the_pages_the_cases_are_written_for(self) -> None:
pages = sorted(gate.concept_pages(self.base.wiki))
self.assertEqual(pages, ["brood.md", "frames.md", "hive.md", "honey-harvest.md", "queen-cells.md", "smoker.md",
"swarming.md", "water.md"])
self.assertEqual(audit.audit(self.base.wiki, self.base.sources), [])
found = claims.extract(self.base.wiki, self.base.sources)
self.assertEqual(claims.criterion_1(found)["total"], (20, 20))
self.assertEqual(len(json.loads(self.base.links.read_text(encoding="utf-8"))["links"]), 5)
def test_every_case_names_pages_and_sentences_of_the_base_or_of_its_own_answers(self) -> None:
pages = set(gate.concept_pages(self.base.wiki))
ids = {c.id for c in self.base.found.claims}
for c in self.manifest["cases"]:
for p in c.get("pages_changed", []) + c.get("pages_gone", []):
self.assertIn(p, pages, c["name"])
for x in c.get("claims_before", []):
self.assertIn(gate.claim_id(x["page"], x["text"], x.get("n", 0)), ids, c["name"])
for x in c.get("links_removed", []):
self.assertIn(gate.link_tuple(x), gate.link_set(json.loads(self.base.links.read_text())["links"]))
def test_the_cases_cover_what_order_34_a1_names(self) -> None:
names = {c["name"] for c in self.manifest["cases"]}
self.assertEqual(set(REQUIRED) - names, set())
def test_a_sentence_key_is_klinks_key_on_the_built_page(self) -> None:
page = (self.base.wiki / "water.md").read_text(encoding="utf-8")
keys = [s.key for s in klink.page_sentences(page)]
self.assertEqual(keys, [gate.sentence_key("Bees need a water source near the hive.", 0),
gate.sentence_key("Bees need a water source near the hive.", 1)])
def test_a_fragment_key_is_the_footnote_on_its_page_and_two_concepts_share_one(self) -> None:
key = {"concept_id": "frames", "file": "docs-en-hive.md", "start": 8, "end": 10,
"quote": "holds wax comb where bees store honey and raise brood", "provenance": "documented"}
label = gate.fragment_label(self.base.sources, key)
for page in ("frames.md", "brood.md"):
self.assertIn(f"[^{label}]", (self.base.wiki / page).read_text(encoding="utf-8"))
records = json.loads((self.base.work / "K-work" / "accepted.json").read_text(encoding="utf-8"))["fragments"]
self.assertEqual(sorted(r["concept_id"] for r in records if kassemble.label(r) == label), ["brood", "frames"])
def test_a_rebuild_without_change_measures_nothing(self) -> None:
c = gate.fresh_copy(self.base, self.tmp, "same")
code, out = gate.quiet(kassemble.main, ["build", "--work", c.places["work"], "--sources", c.places["sources"],
"--out", c.places["wiki"], "--links", c.places["links"], "--replace",
"--at", self.manifest["build_at"]])
self.assertEqual(code, 0, out)
m = gate.measure(c.path("wiki_old"), c.path("wiki"), c.path("sources"))
self.assertEqual((m.changed, sum(m.claims_gone.values()), sum(m.claims_came.values())), ([], 0, 0))
def test_a_list_is_rendered_in_k17s_schema_with_the_hash_of_the_claim_list(self) -> None:
doc = gate.render_list({"claims": [{"page": "frames.md", "text": "Frames can be lifted out one by one."}],
"links": [{"page": "hive.md", "sentence": "Light the smoker before you lift the lid.",
"anchor": "smoker", "target": "smoker.md"}]}, self.base)
self.assertEqual(doc["schema"], "okf-wiki/repair-list/1")
self.assertEqual(doc["claims_sha256"], gate.sha256_file(self.base.claims))
self.assertEqual(set(doc), {"schema", "claims_sha256", "verdicts_sha256", "claims", "pages", "pairs", "links"})
row = doc["claims"][0]
self.assertEqual((row["page"], row["provenance"], row["kind"], row["sources"]),
("frames.md", "documented", "docs", [{"file": "kilde/docs-en-hive.md", "line": 8}]))
self.assertEqual((doc["links"][0]["at"], doc["links"][0]["origin"]), (10, "o33-kandidat"))
stale = gate.render_list({"claims": [], "stale": True}, self.base)
self.assertNotEqual(stale["claims_sha256"], doc["claims_sha256"])
def test_answers_are_keyed_by_concept_and_footnote(self) -> None:
doc = gate.render_answers(case("refuse-unknown-key")["steps"][0]["answers"]
+ case("remove-one")["steps"][0]["answers"], self.base.sources)
self.assertEqual(doc["schema"], "okf-wiki/repair-answers/1")
self.assertEqual(doc["answers"][0], {"concept_id": "hive", "footnote": "s0000000000", "action": "remove"})
self.assertEqual(doc["answers"][1]["concept_id"], "brood")
self.assertIn(f"[^{doc['answers'][1]['footnote']}]", (self.base.wiki / "brood.md").read_text(encoding="utf-8"))
def fake(self, body: str) -> list[str]:
script = self.tmp / f"fake-{len(list(self.tmp.iterdir()))}.py"
script.write_text(body, encoding="utf-8")
return [PY, str(script)]
def run_one(self, name: str, command: list[str]) -> list[str]:
where = Path(tempfile.mkdtemp(dir=self.tmp))
return gate.run_case(case(name), self.base, command, GATES / "repair", where, self.manifest["build_at"], 60)
def test_a_refusal_passes_only_with_its_code_and_nothing_written(self) -> None:
right = self.fake("import sys\nprint('refused: unknown-key hive s0000000000')\nsys.exit(1)\n")
self.assertEqual(self.run_one("refuse-unknown-key", right), [])
wrong_code = self.fake("import sys\nprint('refused: stale-bundle hive s0000000000')\nsys.exit(1)\n")
problems = self.run_one("refuse-unknown-key", wrong_code)
self.assertTrue(any("refusal codes ['stale-bundle']" in p for p in problems))
no_record = self.fake("import sys\nprint('refused: unknown-key')\nsys.exit(1)\n")
self.assertEqual(self.run_one("refuse-unknown-key", no_record),
["step 1: 1 refusal line(s) name no claim or record"])
writes = self.fake("import sys, pathlib\nargs = sys.argv[1:]\n"
"w = pathlib.Path(args[args.index('--work') + 1])\n"
"(w / 'K-work' / 'repairs.json').write_text('{}')\n"
"print('refused: unknown-key hive s0000000000')\nsys.exit(1)\n")
self.assertTrue(any("refused but changed work" in p for p in self.run_one("refuse-unknown-key", writes)))
crash = self.fake("raise RuntimeError('a crash exits 1 too')\n")
self.assertTrue(any("refusal codes []" in p for p in self.run_one("refuse-unknown-key", crash)))
def test_a_link_command_is_judged_by_the_links_file_and_the_built_page(self) -> None:
unlink = self.fake(
"import json, sys\nargs = sys.argv[1:]\np = args[args.index('--links') + 1]\nd = json.load(open(p))\n"
"d['links'] = [r for r in d['links'] if r['anchor'] != 'smoker']\nd['accepted'] = len(d['links'])\n"
"json.dump(d, open(p, 'w'))\n")
self.assertEqual(self.run_one("unlink", unlink), [])
self.assertTrue(self.run_one("relink", unlink))
def task_output(self, out: Path, parts: dict[str, list[str]], answers: dict[str, str]) -> None:
for name, texts in parts.items():
paths = []
for i, t in enumerate(texts, 1):
p = out / f"{name}-part-{i}.md"
p.write_text(t, encoding="utf-8")
paths.append(str(p))
(out / f"{name}.parts.json").write_text(json.dumps({"parts": paths, "answer": answers[name]}),
encoding="utf-8")
def test_task_output_is_judged_by_where_each_fragment_stands_and_what_its_task_holds(self) -> None:
want = case("task")
c = gate.fresh_copy(self.base, self.tmp, "task")
labels = [gate.fragment_label(self.base.sources, t["key"]) for t in want["task"]]
full = ["\n".join([lab] + t["must_contain"]) for lab, t in zip(labels, want["task"])]
self.task_output(c.path("out"), {"t1": [full[0], full[1]], "t2": [full[2]]}, {"t1": "a1.json", "t2": "a2.json"})
self.assertEqual(gate.check_task(want, c, self.base), [])
short = gate.fresh_copy(self.base, self.tmp, "task-short")
cut = full[1].replace("A queen cell looks like a peanut shell.", "")
self.task_output(short.path("out"), {"t1": [full[0], cut], "t2": [full[2]]}, {"t1": "a1.json", "t2": "a2.json"})
self.assertEqual(gate.check_task(want, short, self.base),
[f"the task with {labels[1]} lacks 'A queen cell looks like a peanut shell.'"])
twice = gate.fresh_copy(self.base, self.tmp, "task-twice")
self.task_output(twice.path("out"), {"t1": [full[0], full[1]], "t2": [full[2], labels[0]]},
{"t1": "a.json", "t2": "a.json"})
problems = gate.check_task(want, twice, self.base)
self.assertIn("two tasks share an answer path", problems)
self.assertTrue(any("stands in 2 tasks" in p for p in problems))
big = gate.fresh_copy(self.base, self.tmp, "task-big")
self.task_output(big.path("out"), {"t1": ["\n".join(full) + "x" * kpack.PART_CHARS]}, {"t1": "a.json"})
self.assertTrue(any("over 60000" in p for p in gate.check_task(want, big, self.base)))
def test_linkset_output_is_judged_by_klink_task_and_the_current_targets(self) -> None:
want = case("linkset-two-candidates")
c = gate.fresh_copy(self.base, self.tmp, "linkset")
out = c.path("out")
keys = [(x["page"], gate.sentence_key(x["sentence"], x.get("n", 0))) for x in want["linkset"]]
def group(n: int, page: str, key: str, ids: list[str]) -> None:
(out / f"group-{n}.sentences.json").write_text(json.dumps(
{"schema": klink.SENTENCES_SCHEMA, "pages": {page: [key]}}), encoding="utf-8")
(out / f"group-{n}.catalogue").write_text("\n".join(ids) + "\n", encoding="utf-8")
group(1, *keys[0], ["smoker", "hive"])
group(2, *keys[1], ["hive", "honey-harvest"])
self.assertEqual(gate.check_linkset(want, c), [])
(out / "group-2.catalogue").write_text("honey-harvest\n", encoding="utf-8")
self.assertEqual(gate.check_linkset(want, c),
[f"the catalogue for {keys[1]} lacks its current target hive.md"])
(out / "group-2.sentences.json").unlink()
self.assertTrue(gate.check_linkset(want, c)[0].startswith("sentences:"))
if __name__ == "__main__":
unittest.main()

View file

@ -0,0 +1,552 @@
"""Tests for gates/wholedoc.py: the whole-document and counting group (E2b), its cut, checker, split and gate.
Every fixture is synthetic text written here. No corpus text is used (CLAUDE.md rule 3).
"""
from __future__ import annotations
import contextlib
import hashlib
import io
import json
import sys
import tempfile
import unittest
from pathlib import Path
from typing import Any
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "stubs"))
import multidoc # noqa: E402
import wholedoc # noqa: E402
import wholedoc_arm # noqa: E402
REPO_ROOT = Path(__file__).resolve().parents[2]
def long_doc() -> str:
"""A 200-line synthetic document: facts at the top, the middle and the bottom, and six tagged items."""
lines = [f"Filler line {i:03d} about nothing in particular." for i in range(200)]
lines[5] = "The north gate opens at dawn for the carters."
lines[100] = "The middle bridge carries a toll of copper coins."
lines[190] = "The south wall was rebuilt after the flood year."
for n, i in enumerate((10, 40, 70, 110, 150, 180), 1):
lines[i] = f"- Item: the lantern maker number {chr(64 + n)} works here."
return "\n".join(lines) + "\n"
SOURCES = {
"long.md": long_doc(),
"short.md": "# Short\n\nOnly three lines here, far too short for a whole-document question.\n",
"a.md": "alpha text\n", "b.md": "beta text\n", "c.md": "gamma text\n", "d.md": "delta text\n",
}
def whole_question(qid: str = "x01") -> dict[str, Any]:
return {
"id": qid,
"type": "whole",
"question": "What does the town record at its north gate, middle bridge and south wall?",
"file": "long.md",
"evidence": [
{"span": "The north gate opens at dawn for the carters."},
{"span": "The middle bridge carries a toll of copper coins."},
{"span": "The south wall was rebuilt after the flood year."},
],
"answer_keys": [{"text": "dawn"}, {"text": "copper coins"}, {"text": "flood year"}],
}
def count_question(qid: str = "x02") -> dict[str, Any]:
return {
"id": qid,
"type": "count",
"question": "How many lantern makers does the document list?",
"file": "long.md",
"evidence": [{"span": f"- Item: the lantern maker number {chr(64 + n)} works here."} for n in range(1, 7)],
"item_pattern": "(?m)^- Item: the lantern maker",
"count": 6,
}
def e2_set(questions: list[list[str]]) -> dict[str, Any]:
"""An E2-style dev set: one question per file list, ids q01.. in the given order."""
return {
"schema": "okf-wiki/multidoc-set/1",
"questions": [
{"id": f"q{n:02d}", "question": "?", "evidence": [{"file": f, "span": "x" * 20} for f in files],
"answer_keys": []}
for n, files in enumerate(questions, 1)
],
}
class Workspace:
def __init__(self) -> None:
self._tmp = tempfile.TemporaryDirectory()
self.root = Path(self._tmp.name)
self.sources = self.root / "sources"
self.sources.mkdir()
for name, text in SOURCES.items():
(self.sources / name).write_text(text, encoding="utf-8")
self.e2 = self.write_json("e2.json", e2_set([["long.md", "a.md", "b.md", "c.md"], ["short.md", "d.md"]]))
def close(self) -> None:
self._tmp.cleanup()
def write_set(self, questions: list[dict[str, Any]], name: str = "set.json") -> Path:
return self.write_json(name, {"schema": wholedoc.SET_SCHEMA, "questions": questions})
def write_json(self, name: str, data: Any) -> Path:
path = self.root / name
path.write_text(json.dumps(data), encoding="utf-8")
return path
def cut_manifest(self, name: str = "manifest.json", max_files: int = 20) -> Path:
path = self.root / name
code, out = run(["cut", "--e2-set", str(self.e2), "--sources", str(self.sources), "--max-files",
str(max_files), "--manifest", str(path)])
assert code == 0, out
return path
def run(argv: list[str]) -> tuple[int, str]:
out, err = io.StringIO(), io.StringIO()
with contextlib.redirect_stdout(out), contextlib.redirect_stderr(err):
code = wholedoc.main(argv)
return code, out.getvalue() + err.getvalue()
def sha256_file(path: Path) -> str:
return hashlib.sha256(path.read_bytes()).hexdigest()
class CutTests(unittest.TestCase):
def setUp(self) -> None:
self.ws = Workspace()
def tearDown(self) -> None:
self.ws.close()
def test_greedy_fewest_new_files_ties_by_id(self) -> None:
questions = [
{"id": "q02", "evidence": [{"file": f} for f in ("a", "b", "c", "d")]},
{"id": "q01", "evidence": [{"file": f} for f in ("a", "b", "e", "f")]},
{"id": "q03", "evidence": [{"file": f} for f in ("a", "b", "c", "g")]},
{"id": "q04", "evidence": [{"file": f} for f in ("h", "i", "j", "k")]},
]
chosen, files = wholedoc.compute_cut(questions, 7)
self.assertEqual(chosen, ["q01", "q02", "q03"])
self.assertEqual(files, sorted({"a", "b", "c", "d", "e", "f", "g"}))
def test_cut_stops_before_exceeding_the_limit(self) -> None:
questions = [{"id": "q01", "evidence": [{"file": "a"}]}, {"id": "q02", "evidence": [{"file": f} for f in "bc"]}]
chosen, files = wholedoc.compute_cut(questions, 2)
self.assertEqual((chosen, files), (["q01"], ["a"]))
def test_cut_writes_thresholds_before_any_question(self) -> None:
manifest = json.loads(self.ws.cut_manifest().read_text(encoding="utf-8"))
self.assertEqual(manifest["schema"], wholedoc.MANIFEST_SCHEMA)
self.assertEqual(manifest["thresholds"], wholedoc.thresholds())
self.assertEqual(manifest["cut"]["files"], 6)
self.assertEqual(manifest["cut"]["questions"], 2)
self.assertEqual(manifest["cut"]["e2_set_sha256"], sha256_file(self.ws.e2))
self.assertIsNone(manifest["dev"])
self.assertIsNone(manifest["held_out"])
self.assertEqual(manifest["sources"]["tree_sha256"], multidoc.tree_sha256(self.ws.sources))
def test_cut_refuses_to_overwrite_a_manifest(self) -> None:
path = self.ws.cut_manifest()
code, out = run(["cut", "--e2-set", str(self.ws.e2), "--sources", str(self.ws.sources), "--manifest", str(path)])
self.assertEqual(code, 1)
self.assertIn("exists", out)
class CheckTests(unittest.TestCase):
def setUp(self) -> None:
self.ws = Workspace()
self.manifest = self.ws.cut_manifest()
def tearDown(self) -> None:
self.ws.close()
def check(self, questions: list[dict[str, Any]], e2: Path | None = None) -> tuple[int, str]:
path = self.ws.write_set(questions)
return run(["check", "--set", str(path), "--sources", str(self.ws.sources), "--manifest", str(self.manifest),
"--e2-set", str(e2 or self.ws.e2)])
def test_good_group_passes_with_counts_from_the_set(self) -> None:
code, out = self.check([whole_question("x01"), count_question("x02")])
self.assertEqual(code, 0, out)
self.assertIn("questions: 2 (whole 1, count 1)", out)
self.assertIn("spans verbatim and unique: 9 of 9", out)
self.assertIn("questions whose file is in the cut: 2 of 2", out)
self.assertIn("whole: spread over >= 3 parts, first to last third: 1 of 1", out)
self.assertIn("count: count = spans = pattern matches >= 5: 1 of 1", out)
self.assertIn("count: answer not verbatim in the file: 1 of 1", out)
self.assertIn("not answerable from one span: 2 of 2", out)
self.assertIn("PASS", out)
def test_a_wording_key_fails_the_check(self) -> None:
q = whole_question()
q["answer_keys"][0] = {"text": "at dawn for the carters"}
code, out = self.check([q, count_question("x02")])
self.assertEqual(code, 1)
self.assertIn("answer keys that are wording: 1 of 3", out)
self.assertIn("whole: every key in a span, every span keyed: 1 of 1", out)
def test_a_negated_key_must_stand_negated_in_its_span(self) -> None:
q = whole_question()
q["answer_keys"][2]["negated"] = True
code, out = self.check([q, count_question("x02")])
self.assertEqual(code, 1)
self.assertIn("negated keys negated in their span: 0 of 1", out)
def test_a_negated_key_negated_in_its_span_passes(self) -> None:
long = self.ws.sources / "long.md"
long.write_text(long.read_text(encoding="utf-8").replace("wall was rebuilt", "wall was not rebuilt"),
encoding="utf-8")
q = whole_question()
q["evidence"][2]["span"] = "The south wall was not rebuilt after the flood year."
q["answer_keys"][2]["negated"] = True
code, out = self.check([q, count_question("x02")])
self.assertEqual(code, 0, out)
self.assertIn("negated keys negated in their span: 1 of 1", out)
def test_good_group_has_no_wording_keys(self) -> None:
code, out = self.check([whole_question("x01"), count_question("x02")])
self.assertIn("answer keys that are wording: 0 of 3", out)
def test_whole_without_the_last_third_fails(self) -> None:
q = whole_question()
q["evidence"][2] = {"span": "- Item: the lantern maker number D works here."}
q["answer_keys"][2] = {"text": "number D"}
code, out = self.check([q])
self.assertEqual(code, 1)
self.assertIn("whole: spread over >= 3 parts, first to last third: 0 of 1", out)
def test_whole_with_two_parts_fails(self) -> None:
q = whole_question()
q["evidence"][1] = {"span": "- Item: the lantern maker number A works here."}
q["answer_keys"][1] = {"text": "number A"}
code, out = self.check([q])
self.assertEqual(code, 1)
self.assertIn("first to last third: 0 of 1", out)
def test_whole_on_a_short_file_fails(self) -> None:
q = whole_question()
q["file"] = "short.md"
code, out = self.check([q])
self.assertEqual(code, 1)
self.assertIn(f"whole: file >= {wholedoc.WHOLE_MIN_LINES} lines: 0 of 1", out)
def test_whole_answerable_from_one_span_fails(self) -> None:
q = whole_question()
q["answer_keys"] = [{"text": "north gate"}, {"text": "dawn"}, {"text": "carters"}]
code, out = self.check([q])
self.assertEqual(code, 1)
self.assertIn("not answerable from one span: 0 of 1", out)
def test_whole_key_outside_its_spans_fails(self) -> None:
q = whole_question()
q["answer_keys"].append({"text": "Filler line"})
code, out = self.check([q])
self.assertEqual(code, 1)
self.assertIn("whole: every key in a span, every span keyed: 0 of 1", out)
def test_span_that_occurs_twice_fails(self) -> None:
q = whole_question()
q["evidence"][0] = {"span": "about nothing in particular."}
code, out = self.check([q])
self.assertEqual(code, 1)
self.assertIn("spans verbatim and unique: 2 of 3", out)
def test_count_that_disagrees_with_the_pattern_fails(self) -> None:
q = count_question()
q["item_pattern"] = "(?m)^- Item: the lantern maker number [A-E]"
code, out = self.check([q])
self.assertEqual(code, 1)
self.assertIn("count: count = spans = pattern matches >= 5: 0 of 1", out)
def test_count_below_the_minimum_fails(self) -> None:
q = count_question()
q["evidence"] = q["evidence"][:4]
q["item_pattern"] = "(?m)^- Item: the lantern maker number [A-D]"
q["count"] = 4
code, out = self.check([q])
self.assertEqual(code, 1)
self.assertIn("count: count = spans = pattern matches >= 5: 0 of 1", out)
def test_count_that_stands_in_the_file_fails(self) -> None:
ws_file = self.ws.sources / "long.md"
ws_file.write_text(ws_file.read_text(encoding="utf-8") + "There are six of them.\n", encoding="utf-8")
self.manifest.unlink()
self.manifest = self.ws.cut_manifest()
code, out = self.check([count_question()])
self.assertEqual(code, 1)
self.assertIn("count: answer not verbatim in the file: 0 of 1", out)
def test_number_inside_a_version_is_not_the_answer(self) -> None:
self.assertFalse(wholedoc.number_stands_in(6, "Released in 2.1.6 and v6.0, see item-6 and 16 or 60."))
self.assertTrue(wholedoc.number_stands_in(6, "It holds 6 entries."))
self.assertTrue(wholedoc.number_stands_in(6, "Up to 6."))
self.assertTrue(wholedoc.number_stands_in(6, "Six rules apply."))
def test_file_outside_the_cut_fails(self) -> None:
self.ws.e2 = self.ws.write_json("e2-small.json", e2_set([["short.md", "a.md"]]))
self.manifest.unlink()
self.manifest = self.ws.cut_manifest()
code, out = self.check([whole_question()])
self.assertEqual(code, 1)
self.assertIn("not in the cut", out)
self.assertIn("questions whose file is in the cut: 0 of 1", out)
def test_cut_that_differs_from_the_manifest_cannot_run(self) -> None:
e2 = self.ws.write_json("e2-other.json", e2_set([["long.md", "a.md", "b.md", "c.md", "d.md", "short.md"]]))
code, out = self.check([whole_question()], e2=e2)
self.assertEqual(code, 1)
self.assertIn("e2_set_sha256", out)
def test_empty_set_is_a_fail_not_a_pass(self) -> None:
code, out = self.check([])
self.assertEqual(code, 1)
self.assertIn("FAIL", out)
def test_unknown_type_fails(self) -> None:
q = whole_question()
q["type"] = "other"
code, out = self.check([q])
self.assertEqual(code, 1)
self.assertIn("type", out)
class SplitTests(unittest.TestCase):
def setUp(self) -> None:
self.ws = Workspace()
self.manifest = self.ws.cut_manifest()
questions = [whole_question(f"w{n}") for n in range(1, 5)] + [count_question(f"c{n}") for n in range(1, 5)]
self.set_path = self.ws.write_set(questions, "all.json")
self.held = Path(tempfile.mkdtemp()) / "held.json"
def tearDown(self) -> None:
self.ws.close()
def split(self, seed: int = 7, held: Path | None = None) -> tuple[int, str]:
return run(["split", "--set", str(self.set_path), "--sources", str(self.ws.sources), "--manifest",
str(self.manifest), "--e2-set", str(self.ws.e2), "--seed", str(seed), "--held-out-per-type", "1",
"--dev-out", str(self.ws.root / "dev.json"), "--dev-label", "gates-local/wholedoc/dev.json",
"--held-out", str(held or self.held)])
def test_split_is_stratified_renumbered_and_recorded(self) -> None:
code, out = self.split()
self.assertEqual(code, 0, out)
dev = json.loads((self.ws.root / "dev.json").read_text(encoding="utf-8"))["questions"]
held = json.loads(self.held.read_text(encoding="utf-8"))["questions"]
self.assertEqual([q["id"] for q in dev], [f"d{n:02d}" for n in range(1, 7)])
self.assertEqual([q["id"] for q in held], ["h01", "h02"])
self.assertEqual(sorted(q["type"] for q in held), ["count", "whole"])
manifest = json.loads(self.manifest.read_text(encoding="utf-8"))
self.assertEqual(manifest["seed"], 7)
self.assertEqual(manifest["dev"], {"file": "gates-local/wholedoc/dev.json",
"sha256": sha256_file(self.ws.root / "dev.json"),
"questions": 6, "whole": 3, "count": 3})
self.assertEqual(manifest["held_out"], {"sha256": sha256_file(self.held), "questions": 2, "whole": 1,
"count": 1})
self.assertEqual(manifest["thresholds"], wholedoc.thresholds())
def test_same_seed_same_draw(self) -> None:
self.split(seed=3)
first = self.held.read_bytes()
other = Path(tempfile.mkdtemp()) / "held.json"
self.manifest.unlink()
self.manifest = self.ws.cut_manifest()
self.split(seed=3, held=other)
self.assertEqual(first, other.read_bytes())
def test_refuses_held_out_inside_the_repository(self) -> None:
code, out = self.split(held=REPO_ROOT / "gates" / "held.json")
self.assertEqual(code, 1)
self.assertIn("inside the repository", out)
def test_refuses_to_redraw(self) -> None:
self.split()
code, out = self.split()
self.assertEqual(code, 1)
self.assertIn("already", out)
def test_refuses_when_thresholds_changed_after_the_cut(self) -> None:
manifest = json.loads(self.manifest.read_text(encoding="utf-8"))
manifest["thresholds"]["count_min"] = 2
self.manifest.write_text(json.dumps(manifest), encoding="utf-8")
code, out = self.split()
self.assertEqual(code, 1)
self.assertIn("thresholds", out)
def test_refuses_a_set_that_fails_the_check(self) -> None:
q = whole_question("w9")
q["file"] = "short.md"
self.set_path = self.ws.write_set([q, count_question("c9")], "bad.json")
code, out = self.split()
self.assertEqual(code, 1)
self.assertFalse(self.held.exists())
class GateTests(unittest.TestCase):
def setUp(self) -> None:
self.ws = Workspace()
self.manifest = self.ws.cut_manifest()
self.questions = [whole_question("d01"), count_question("d02")]
self.set_path = self.ws.write_set(self.questions, "dev.json")
manifest = json.loads(self.manifest.read_text(encoding="utf-8"))
manifest["dev"] = {"sha256": sha256_file(self.set_path), "questions": 2, "whole": 1, "count": 1}
manifest["held_out"] = {"sha256": "0" * 64, "questions": 2, "whole": 1, "count": 1}
self.manifest.write_text(json.dumps(manifest), encoding="utf-8")
def tearDown(self) -> None:
self.ws.close()
def gate(self, answers: Path, arm: str = "collection") -> tuple[int, str]:
return run(["gate", "--set", str(self.set_path), "--manifest", str(self.manifest), "--sources",
str(self.ws.sources), "--arm", arm, "--answers", str(answers)])
def answers(self, entries: list[dict[str, Any]], arm: str = "collection") -> Path:
return self.ws.write_json("answers.json", {"arm": arm, "answers": entries})
def fasit(self) -> list[dict[str, Any]]:
return [wholedoc_arm.fasit_answer(q) for q in self.questions]
def test_missing_arm_is_fail_closed_at_zero(self) -> None:
code, out = self.gate(self.ws.root / "none.json")
self.assertEqual(code, 1)
self.assertIn("correct answer and source: 0 of 2", out)
self.assertIn("fail-closed", out)
def test_fasit_scores_all_per_type(self) -> None:
code, out = self.gate(self.answers(self.fasit()))
self.assertEqual(code, 0, out)
self.assertIn("whole: correct 1 of 1", out)
self.assertIn("count: correct 1 of 1", out)
self.assertIn("correct answer and source: 2 of 2", out)
self.assertIn("GREEN", out)
def test_one_span_stub_scores_zero(self) -> None:
code, out = self.gate(self.answers([wholedoc_arm.one_span_answer(q) for q in self.questions]))
self.assertEqual(code, 0, out)
self.assertIn("correct answer and source: 0 of 2", out)
self.assertIn("RED", out)
def test_count_must_be_an_exact_integer(self) -> None:
for wrong in (5, 7, "6", 6.0, None):
a = self.fasit()
a[1]["count"] = wrong
code, out = self.gate(self.answers(a))
self.assertIn("count: correct 0 of 1", out, repr(wrong))
def test_count_true_is_not_one(self) -> None:
q = count_question("d02")
a = wholedoc_arm.fasit_answer(q)
a["count"] = True
self.assertFalse(wholedoc.score_one(q, a, multidoc.SourceCache(self.ws.sources)).answer_ok)
def test_whole_missing_a_key_is_wrong(self) -> None:
a = self.fasit()
a[0]["answer"] = a[0]["answer"].replace("flood year", "a bad year")
code, out = self.gate(self.answers(a))
self.assertIn("whole: correct 0 of 1", out)
def test_citation_must_quote_the_file(self) -> None:
a = self.fasit()
a[0]["citations"] = [{"file": "long.md", "quote": "The north gate never opens."}]
a[1]["citations"] = [{"file": "short.md", "quote": "Only three lines here"}]
code, out = self.gate(self.answers(a))
self.assertIn("correct source: 0 of 2", out)
def test_set_that_differs_from_the_manifest_is_refused(self) -> None:
self.set_path.write_text(self.set_path.read_text(encoding="utf-8") + "\n", encoding="utf-8")
code, out = self.gate(self.answers(self.fasit()))
self.assertEqual(code, 1)
self.assertIn("sha256", out)
def test_changed_sources_are_refused(self) -> None:
(self.ws.sources / "a.md").write_text("changed\n", encoding="utf-8")
code, out = self.gate(self.answers(self.fasit()))
self.assertEqual(code, 1)
self.assertIn("sources", out)
def test_manifest_before_the_split_is_refused(self) -> None:
manifest = json.loads(self.manifest.read_text(encoding="utf-8"))
manifest["dev"] = None
self.manifest.write_text(json.dumps(manifest), encoding="utf-8")
code, out = self.gate(self.answers(self.fasit()))
self.assertEqual(code, 1)
def test_arm_name_mismatch_is_refused(self) -> None:
code, out = self.gate(self.answers(self.fasit(), arm="filesearch"))
self.assertEqual(code, 1)
self.assertIn("arm", out)
def test_per_question_results_are_written_for_pairing(self) -> None:
out_path = self.ws.root / "results.json"
run(["gate", "--set", str(self.set_path), "--manifest", str(self.manifest), "--sources", str(self.ws.sources),
"--arm", "collection", "--answers", str(self.answers(self.fasit()[:1])), "--out", str(out_path)])
results = json.loads(out_path.read_text(encoding="utf-8"))
self.assertEqual([(r["id"], r["type"], r["correct"]) for r in results["results"]],
[("d01", "whole", True), ("d02", "count", False)])
class ArmStubTests(unittest.TestCase):
def test_arm_file_is_written_for_every_question(self) -> None:
ws = Workspace()
try:
set_path = ws.write_set([whole_question("d01"), count_question("d02")])
out = ws.root / "arm.json"
code = wholedoc_arm.main(["one-span", str(set_path), str(out)])
self.assertEqual(code, 0)
data = json.loads(out.read_text(encoding="utf-8"))
self.assertEqual(data["arm"], "one-span")
self.assertEqual([a["id"] for a in data["answers"]], ["d01", "d02"])
self.assertEqual(data["answers"][1]["count"], 1)
finally:
ws.close()
class NegatedKeyTests(unittest.TestCase):
def test_a_whole_question_scores_a_negated_key_by_its_negation(self) -> None:
ws = Workspace()
try:
q = whole_question()
q["answer_keys"][2]["negated"] = True
cache = multidoc.SourceCache(ws.sources)
cite = [{"file": "long.md", "quote": "The north gate opens at dawn for the carters."}]
right = {"id": q["id"], "answer": "dawn, copper coins, not after the flood year", "citations": cite}
wrong = {"id": q["id"], "answer": "dawn, copper coins, after the flood year", "citations": cite}
self.assertTrue(wholedoc.score_one(q, right, cache).answer_ok)
self.assertFalse(wholedoc.score_one(q, wrong, cache).answer_ok)
finally:
ws.close()
def test_the_fasit_states_a_negated_key_with_its_negation(self) -> None:
q = whole_question()
q["answer_keys"][2]["negated"] = True
self.assertEqual(wholedoc_arm.fasit_answer(q)["answer"], "dawn; copper coins; not flood year")
class NumberKeyTests(unittest.TestCase):
def test_a_whole_question_scores_a_number_key_as_a_number(self) -> None:
ws = Workspace()
try:
q = whole_question()
q["answer_keys"][0] = {"text": "three"}
cache = multidoc.SourceCache(ws.sources)
cite = [{"file": "long.md", "quote": "The north gate opens at dawn for the carters."}]
right = {"id": q["id"], "answer": "3 of them, copper coins, flood year", "citations": cite}
wrong = {"id": q["id"], "answer": "33 of them, copper coins, flood year", "citations": cite}
self.assertTrue(wholedoc.score_one(q, right, cache).answer_ok)
self.assertFalse(wholedoc.score_one(q, wrong, cache).answer_ok)
finally:
ws.close()
if __name__ == "__main__":
unittest.main()