okf-wiki 1.1.1
First public release, published as one snapshot. What changed between versions is in CHANGELOG.md. Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
This commit is contained in:
commit
0fd86a23a3
227 changed files with 96077 additions and 0 deletions
752
gates/tests/test_answers.py
Normal file
752
gates/tests/test_answers.py
Normal file
|
|
@ -0,0 +1,752 @@
|
|||
"""Tests for gates/answers.py: the answer harness of contract K5 (setup, collect, harvest, score).
|
||||
|
||||
Synthetic K1 roots about bread and gardens, synthetic transcripts and synthetic result files. No corpus text is used
|
||||
(CLAUDE.md rule 3), and nothing here starts a model.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
import io
|
||||
import json
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
|
||||
import answers # noqa: E402
|
||||
from test_answerset import ( # noqa: E402
|
||||
KEY, OVEN, RISING, SOIL, SPAN, TEXTS, WATER, dump, make_set, q4, q23, root_ref, sha, write_root,
|
||||
)
|
||||
|
||||
# Newcombe (1998), Statist. Med. 17:2635-2650, Table III, method 10: (e, f, g, h) -> 95 % interval for
|
||||
# (e + f)/n - (e + g)/n, as printed (four decimals).
|
||||
NEWCOMBE_TABLE_III = [
|
||||
((36, 12, 2, 0), (0.0569, 0.3404)),
|
||||
((20, 12, 2, 16), (0.0562, 0.3292)),
|
||||
((18, 12, 2, 18), (0.0562, 0.3290)),
|
||||
((35, 14, 0, 1), (0.1461, 0.4175)),
|
||||
((18, 14, 0, 18), (0.1441, 0.3963)),
|
||||
((1, 97, 1, 1), (0.8736, 0.9850)),
|
||||
((0, 30, 0, 0), (0.8395, 1.0)),
|
||||
((1, 98, 0, 1), (0.9171, 0.9916)),
|
||||
((54, 0, 0, 0), (-0.0664, 0.0664)),
|
||||
((53, 0, 0, 1), (-0.0729, 0.0729)),
|
||||
((27, 0, 0, 27), (-0.0351, 0.0351)),
|
||||
]
|
||||
|
||||
|
||||
def run(argv: list[str]) -> tuple[int, str]:
|
||||
out, err = io.StringIO(), io.StringIO()
|
||||
with contextlib.redirect_stdout(out), contextlib.redirect_stderr(err):
|
||||
try:
|
||||
code = answers.main(argv)
|
||||
except SystemExit as exc: # argparse refuses an option it does not know
|
||||
code = int(exc.code or 0)
|
||||
return code, out.getvalue() + err.getvalue()
|
||||
|
||||
|
||||
class Base(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
self._tmp = tempfile.TemporaryDirectory()
|
||||
self.tmp = Path(self._tmp.name).resolve()
|
||||
self.root = write_root(self.tmp / "copy")
|
||||
self.set = self.tmp / "set.json"
|
||||
self.set.write_bytes(dump(make_set(self.root, [q23("d01"), q4("d02")])))
|
||||
self.run_dir = self.tmp / "run"
|
||||
|
||||
def tearDown(self) -> None:
|
||||
self._tmp.cleanup()
|
||||
|
||||
def setup_arm(self, arm: str = "filsok", extra: list[str] | None = None) -> tuple[int, str]:
|
||||
return run(["setup", "--run", str(self.run_dir), "--name", "r1", "--arm", arm, "--set", str(self.set),
|
||||
"--sources", str(self.root)] + (extra or []))
|
||||
|
||||
|
||||
class SetupTest(Base):
|
||||
def test_setup_copies_the_sources_as_the_arm_basis(self) -> None:
|
||||
code, out = self.setup_arm()
|
||||
self.assertEqual(code, 0, out)
|
||||
basis = self.run_dir / "arms" / "filsok"
|
||||
for path in TEXTS:
|
||||
self.assertEqual((basis / path).read_bytes(), (self.root / path).read_bytes())
|
||||
self.assertEqual(sorted(p.relative_to(basis).as_posix() for p in basis.rglob("*") if p.is_file()),
|
||||
sorted(TEXTS))
|
||||
|
||||
def test_one_task_per_question_without_keys_spans_or_counts(self) -> None:
|
||||
code, out = self.setup_arm()
|
||||
self.assertEqual(code, 0, out)
|
||||
basis = self.run_dir / "arms" / "filsok"
|
||||
for qid in ("d01", "d02"):
|
||||
task = self.run_dir / "tasks" / "filsok" / f"{qid}.md"
|
||||
text = task.read_text(encoding="utf-8")
|
||||
self.assertIn("How do the synthetic notes about bread and gardens fit together?", text)
|
||||
self.assertIn(str(basis), text)
|
||||
self.assertIn(str(self.run_dir / "answers" / "filsok" / f"{qid}.json"), text)
|
||||
for value in list(SPAN.values()) + list(KEY.values()):
|
||||
self.assertNotIn(value, text)
|
||||
for word in ("answer_keys", "evidence", "docs-2-3", "docs-4", "history", "four documents"):
|
||||
self.assertNotIn(word, text)
|
||||
self.assertIn(f"S1-r1-filsok-{qid}\t{task}", out)
|
||||
self.assertTrue((self.run_dir / "answers" / "filsok").is_dir())
|
||||
|
||||
def test_the_task_asks_for_k1_paths_in_the_citations(self) -> None:
|
||||
self.setup_arm()
|
||||
text = (self.run_dir / "tasks" / "filsok" / "d01.md").read_text(encoding="utf-8")
|
||||
self.assertIn('"path": "kilde/', text)
|
||||
self.assertIn('"quote"', text)
|
||||
|
||||
def test_an_arm_is_set_up_once(self) -> None:
|
||||
self.assertEqual(self.setup_arm()[0], 0)
|
||||
code, out = self.setup_arm()
|
||||
self.assertEqual(code, 1)
|
||||
self.assertIn("exists", out)
|
||||
|
||||
def test_the_bundle_arm_needs_a_bundle_and_gets_the_sources_beside_it(self) -> None:
|
||||
self.assertEqual(self.setup_arm("bundle")[0], 1)
|
||||
bundle = self.tmp / "bundle"
|
||||
bundle.mkdir()
|
||||
(bundle / "index.md").write_text("# Index\n", encoding="utf-8")
|
||||
code, out = self.setup_arm("bundle", ["--bundle", str(bundle)])
|
||||
self.assertEqual(code, 0, out)
|
||||
basis = self.run_dir / "arms" / "bundle"
|
||||
self.assertEqual((basis / "wiki" / "index.md").read_text(encoding="utf-8"), "# Index\n")
|
||||
self.assertEqual((basis / "kilde" / "docs-bread-oven.md").read_bytes(),
|
||||
(self.root / "kilde" / "docs-bread-oven.md").read_bytes())
|
||||
text = (self.run_dir / "tasks" / "bundle" / "d01.md").read_text(encoding="utf-8")
|
||||
self.assertIn("wiki/index.md", text)
|
||||
self.assertNotIn("Read `BASIS/wiki/index.md` first", text)
|
||||
search, main, links = (text.index("Search `BASIS/wiki/index.md`"), text.index("Open the main page"),
|
||||
text.index("follow its links"))
|
||||
self.assertLess(search, main)
|
||||
self.assertLess(main, links)
|
||||
self.assertIn("never read whole", text)
|
||||
|
||||
def test_only_the_fixed_arms(self) -> None:
|
||||
code, out = self.setup_arm("server")
|
||||
self.assertEqual(code, 1)
|
||||
self.assertIn("filsok", out)
|
||||
|
||||
def test_the_root_must_be_the_one_the_set_names(self) -> None:
|
||||
other = write_root(self.tmp / "other", frozen_at="2026-10-09T12:00:00Z")
|
||||
code, out = run(["setup", "--run", str(self.run_dir), "--name", "r1", "--arm", "filsok",
|
||||
"--set", str(self.set), "--sources", str(other)])
|
||||
self.assertEqual(code, 1)
|
||||
self.assertIn("manifest_sha256", out)
|
||||
|
||||
def test_a_run_name_with_a_dash_or_space_is_refused(self) -> None:
|
||||
code, _ = run(["setup", "--run", str(self.run_dir), "--name", "r 1", "--arm", "filsok",
|
||||
"--set", str(self.set), "--sources", str(self.root)])
|
||||
self.assertEqual(code, 1)
|
||||
|
||||
|
||||
class PartRootTest(Base):
|
||||
"""Order 31 A3: an arm searches the whole copy for a set made from a part of it (`--part-root`)."""
|
||||
|
||||
PART = (RISING, OVEN, SOIL, WATER)
|
||||
|
||||
def make_part(self, copy: Path, change: Callable[[dict[str, Any]], None] | None = None) -> Path:
|
||||
part = write_root(self.tmp / "part", self.PART, subset_of=root_ref(copy))
|
||||
if change is not None:
|
||||
manifest = part / "source-manifest.json"
|
||||
data = json.loads(manifest.read_bytes())
|
||||
change(data)
|
||||
manifest.write_bytes(dump(data))
|
||||
self.set.write_bytes(dump(make_set(part, [q23("d01"), q4("d02")])))
|
||||
return part
|
||||
|
||||
def setup_part(self, part: Path | None, sources: Path | None = None) -> tuple[int, str]:
|
||||
extra = ["--part-root", str(part)] if part is not None else []
|
||||
return run(["setup", "--run", str(self.run_dir), "--name", "r1", "--arm", "filsok", "--set", str(self.set),
|
||||
"--sources", str(sources or self.root)] + extra)
|
||||
|
||||
def test_a_set_of_the_part_is_set_up_on_the_whole_copy(self) -> None:
|
||||
code, out = self.setup_part(self.make_part(self.root))
|
||||
self.assertEqual(code, 0, out)
|
||||
basis = self.run_dir / "arms" / "filsok"
|
||||
self.assertEqual(sorted(p.relative_to(basis).as_posix() for p in basis.rglob("*") if p.is_file()),
|
||||
sorted(TEXTS))
|
||||
|
||||
def test_a_part_that_is_not_a_subset_of_the_root_is_refused(self) -> None:
|
||||
def other(data: dict[str, Any]) -> None:
|
||||
data["subset_of"]["manifest_sha256"] = "0" * 64
|
||||
code, out = self.setup_part(self.make_part(self.root, other))
|
||||
self.assertEqual(code, 1)
|
||||
self.assertIn("subset_of", out)
|
||||
self.assertFalse((self.run_dir / "arms").exists())
|
||||
|
||||
def test_a_part_source_with_another_sha256_than_the_root_is_refused(self) -> None:
|
||||
def changed(data: dict[str, Any]) -> None:
|
||||
data["sources"][0]["sha256"] = "0" * 64
|
||||
code, out = self.setup_part(self.make_part(self.root, changed))
|
||||
self.assertEqual(code, 1)
|
||||
self.assertIn("sha256", out)
|
||||
self.assertFalse((self.run_dir / "arms").exists())
|
||||
|
||||
def test_a_part_source_missing_in_the_root_is_refused(self) -> None:
|
||||
smaller = write_root(self.tmp / "smaller", (RISING, OVEN, SOIL))
|
||||
code, out = self.setup_part(self.make_part(smaller), smaller)
|
||||
self.assertEqual(code, 1)
|
||||
self.assertIn(WATER, out)
|
||||
self.assertFalse((self.run_dir / "arms").exists())
|
||||
|
||||
def test_a_part_root_that_is_not_the_sets_root_is_refused(self) -> None:
|
||||
part = self.make_part(self.root)
|
||||
self.set.write_bytes(dump(make_set(self.root, [q23("d01"), q4("d02")])))
|
||||
code, out = self.setup_part(part)
|
||||
self.assertEqual(code, 1)
|
||||
self.assertIn("--part-root", out)
|
||||
|
||||
def test_without_part_root_a_set_of_the_part_is_still_refused_on_the_copy(self) -> None:
|
||||
self.make_part(self.root)
|
||||
code, out = self.setup_part(None)
|
||||
self.assertEqual(code, 1)
|
||||
self.assertIn("manifest_sha256", out)
|
||||
|
||||
|
||||
class CollectTest(Base):
|
||||
def test_collect_writes_the_k5_answers_file_with_the_id_from_the_file_name(self) -> None:
|
||||
self.setup_arm()
|
||||
folder = self.run_dir / "answers" / "filsok"
|
||||
(folder / "d02.json").write_text(json.dumps({"id": "zz", "answer": "B", "citations": []}), encoding="utf-8")
|
||||
(folder / "d01.json").write_text(json.dumps({"answer": "A", "citations": [{"path": "kilde/x.md",
|
||||
"quote": "q"}]}),
|
||||
encoding="utf-8")
|
||||
code, out = run(["collect", "--run", str(self.run_dir), "--arm", "filsok"])
|
||||
self.assertEqual(code, 0, out)
|
||||
path = self.run_dir / "answers" / "filsok.json"
|
||||
doc = json.loads(path.read_text(encoding="utf-8"))
|
||||
self.assertEqual(doc["schema"], "okf-wiki/answers/1")
|
||||
self.assertEqual(doc["arm"], "filsok")
|
||||
self.assertEqual([a["id"] for a in doc["answers"]], ["d01", "d02"])
|
||||
self.assertEqual(doc["answers"][0]["citations"], [{"path": "kilde/x.md", "quote": "q"}])
|
||||
self.assertEqual(path.read_bytes(), dump(doc))
|
||||
|
||||
def test_an_answer_file_that_is_not_a_json_object_is_a_problem(self) -> None:
|
||||
self.setup_arm()
|
||||
folder = self.run_dir / "answers" / "filsok"
|
||||
(folder / "d01.json").write_text("{not json", encoding="utf-8")
|
||||
(folder / "d02.json").write_text(json.dumps({"answer": "B", "citations": []}), encoding="utf-8")
|
||||
code, out = run(["collect", "--run", str(self.run_dir), "--arm", "filsok"])
|
||||
self.assertEqual(code, 1)
|
||||
self.assertIn("d01.json", out)
|
||||
doc = json.loads((self.run_dir / "answers" / "filsok.json").read_text(encoding="utf-8"))
|
||||
self.assertEqual([a["id"] for a in doc["answers"]], ["d02"])
|
||||
|
||||
|
||||
def usage(inp: int, cc: int, cr: int, out: int) -> dict[str, int]:
|
||||
return {"input_tokens": inp, "cache_creation_input_tokens": cc, "cache_read_input_tokens": cr,
|
||||
"output_tokens": out}
|
||||
|
||||
|
||||
def tool(name: str, **inp: Any) -> dict[str, Any]:
|
||||
return {"type": "tool_use", "id": f"t-{name}-{len(inp)}", "name": name, "input": inp}
|
||||
|
||||
|
||||
def assistant(mid: str, use: dict[str, int], blocks: list[dict[str, Any]], stop: str | None = "tool_use") -> dict[str, Any]:
|
||||
message: dict[str, Any] = {"id": mid, "role": "assistant", "usage": use, "content": blocks}
|
||||
if stop is not None:
|
||||
message["stop_reason"] = stop
|
||||
return {"type": "assistant", "message": message}
|
||||
|
||||
|
||||
def user(text: str, stamp: str = "2026-10-09T10:00:00Z") -> dict[str, Any]:
|
||||
return {"type": "user", "timestamp": stamp, "message": {"role": "user", "content": text}}
|
||||
|
||||
|
||||
def write_transcript(path: Path, lines: list[dict[str, Any]], description: str | None = None) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text("".join(json.dumps(line) + "\n" for line in lines) + "not json\n", encoding="utf-8")
|
||||
if description is not None:
|
||||
meta = path.with_name(path.stem + ".meta.json")
|
||||
meta.write_text(json.dumps({"agentType": "general-purpose", "description": description}), encoding="utf-8")
|
||||
|
||||
|
||||
class HarvestTest(Base):
|
||||
def setUp(self) -> None:
|
||||
super().setUp()
|
||||
self.assertEqual(self.setup_arm()[0], 0)
|
||||
self.basis = self.run_dir / "arms" / "filsok"
|
||||
self.transcripts = self.tmp / "transcripts"
|
||||
self.subagents = self.transcripts / "session-1" / "subagents"
|
||||
|
||||
def task(self, qid: str) -> Path:
|
||||
return self.run_dir / "tasks" / "filsok" / f"{qid}.md"
|
||||
|
||||
def answer(self, qid: str) -> Path:
|
||||
return self.run_dir / "answers" / "filsok" / f"{qid}.json"
|
||||
|
||||
def good_lines(self, qid: str, stamp: str = "2026-10-09T10:00:00Z") -> list[dict[str, Any]]:
|
||||
return [
|
||||
user(f"Read {self.task(qid)} and do what it says.", stamp),
|
||||
assistant("m1", usage(10, 100, 1000, 5), [tool("Read", file_path=str(self.task(qid)))]),
|
||||
assistant("m1", usage(10, 100, 1000, 7), []),
|
||||
assistant("m2", usage(3, 0, 1100, 20), [tool("Bash", command=f"cd {self.basis} && grep -rn dough .")]),
|
||||
assistant("m3", usage(2, 0, 1200, 30), [tool("Write", file_path=str(self.answer(qid)), content="{}")],
|
||||
"end_turn"),
|
||||
]
|
||||
|
||||
def harvest(self) -> tuple[int, str, dict[str, Any]]:
|
||||
code, out = run(["harvest", "--run", str(self.run_dir), "--name", "r1", "--arm", "filsok",
|
||||
"--transcripts", str(self.transcripts)])
|
||||
path = self.run_dir / "tokens" / "filsok.json"
|
||||
return code, out, json.loads(path.read_text(encoding="utf-8")) if path.is_file() else {}
|
||||
|
||||
def breaches(self, calls: list[dict[str, Any]]) -> list[str]:
|
||||
lines = self.good_lines("d01")
|
||||
lines.insert(3, assistant("m9", usage(1, 0, 0, 1), calls))
|
||||
write_transcript(self.subagents / "agent-a1.jsonl", lines, "S1-r1-filsok-d01")
|
||||
return list(self.harvest()[2]["questions"][0]["breach_detail"])
|
||||
|
||||
def test_tokens_per_question_each_message_once_all_four_fields(self) -> None:
|
||||
write_transcript(self.subagents / "agent-a1.jsonl", self.good_lines("d01"), "S1-r1-filsok-d01")
|
||||
write_transcript(self.subagents / "agent-a2.jsonl", self.good_lines("d02"), "S1-r1-filsok-d02")
|
||||
code, out, doc = self.harvest()
|
||||
self.assertEqual(code, 0, out)
|
||||
self.assertEqual(doc["schema"], "okf-wiki/answer-tokens/1")
|
||||
self.assertEqual(doc["arm"], "filsok")
|
||||
row = doc["questions"][0]
|
||||
self.assertEqual(row["id"], "d01")
|
||||
self.assertEqual((row["input_tokens"], row["cache_creation_input_tokens"], row["cache_read_input_tokens"],
|
||||
row["output_tokens"]), (15, 100, 3300, 57))
|
||||
self.assertEqual(row["isolation_breaches"], 0)
|
||||
self.assertEqual([r["id"] for r in doc["questions"]], ["d01", "d02"])
|
||||
path = self.run_dir / "tokens" / "filsok.json"
|
||||
self.assertEqual(path.read_bytes(), dump(doc))
|
||||
|
||||
def test_isolation_breaches_are_counted(self) -> None:
|
||||
lines = self.good_lines("d01")
|
||||
lines.insert(3, assistant("m9", usage(1, 0, 0, 1), [
|
||||
tool("Read", file_path="/etc/hosts"),
|
||||
tool("Bash", command="cat /etc/passwd"),
|
||||
tool("Grep", pattern="dough", path=str(self.run_dir / "arms" / "bundle")),
|
||||
tool("Glob", pattern="*.md"),
|
||||
tool("Read", file_path=str(self.basis / "kilde" / "docs-bread-oven.md")),
|
||||
tool("Bash", command=f"cd {self.basis} && grep -rn crust ../.."),
|
||||
tool("WebFetch", url="https://example.invalid/"),
|
||||
tool("Write", file_path=str(self.basis / "notes.md"), content="x"),
|
||||
]))
|
||||
write_transcript(self.subagents / "agent-a1.jsonl", lines, "S1-r1-filsok-d01")
|
||||
write_transcript(self.subagents / "agent-a2.jsonl", self.good_lines("d02"), "S1-r1-filsok-d02")
|
||||
code, out, doc = self.harvest()
|
||||
row = doc["questions"][0]
|
||||
self.assertEqual(row["isolation_breaches"], 7, row)
|
||||
self.assertEqual(len(row["breach_detail"]), 7)
|
||||
self.assertIn("breaches: 7", out)
|
||||
|
||||
def test_echo_of_literal_words_in_a_chain_into_the_basis_is_not_a_breach(self) -> None:
|
||||
lines = self.good_lines("d01")
|
||||
lines.insert(3, assistant("m9", usage(1, 0, 0, 1), [
|
||||
tool("Bash", command=f"cd {self.basis} && grep -rn dough kilde && echo --- && grep -rn crust kilde"),
|
||||
tool("Bash", command=f"cd {self.basis} && grep -rn dough kilde | head -5; echo ----; ls kilde"),
|
||||
tool("Bash", command=f'cd {self.basis} && echo "next file" && sed -n 1,5p kilde/docs-bread-oven.md'),
|
||||
]))
|
||||
write_transcript(self.subagents / "agent-a1.jsonl", lines, "S1-r1-filsok-d01")
|
||||
code, out, doc = self.harvest()
|
||||
self.assertEqual(doc["questions"][0]["isolation_breaches"], 0, doc["questions"][0]["breach_detail"])
|
||||
self.assertNotIn("echo", self.task("d01").read_text(encoding="utf-8"))
|
||||
|
||||
def test_echo_outside_the_basis_or_with_an_expansion_is_still_a_breach(self) -> None:
|
||||
lines = self.good_lines("d01")
|
||||
lines.insert(3, assistant("m9", usage(1, 0, 0, 1), [
|
||||
tool("Bash", command=f"cd {self.basis} && echo /outside/*"),
|
||||
tool("Bash", command=f"cd {self.basis} && echo ~/*"),
|
||||
tool("Bash", command=f"cd {self.basis} && echo ../../*"),
|
||||
tool("Bash", command=f"cd {self.basis} && echo x > kilde/notes.md"),
|
||||
tool("Bash", command=f"cd {self.basis} && echo $HOME"),
|
||||
tool("Bash", command=f"cd {self.basis} && echo `id`"),
|
||||
tool("Bash", command="echo --- && grep -rn dough kilde"),
|
||||
]))
|
||||
write_transcript(self.subagents / "agent-a1.jsonl", lines, "S1-r1-filsok-d01")
|
||||
code, out, doc = self.harvest()
|
||||
self.assertEqual(doc["questions"][0]["isolation_breaches"], 7, doc["questions"][0]["breach_detail"])
|
||||
|
||||
def test_stderr_sent_away_right_before_a_separator_is_not_a_breach(self) -> None:
|
||||
b = self.basis
|
||||
self.assertEqual(self.breaches([
|
||||
tool("Bash", command=f"cd {b} && grep -rn dough kilde 2>/dev/null; ls kilde"),
|
||||
tool("Bash", command=f"cd {b} && grep -rn dough kilde 2>/dev/null| head -3"),
|
||||
tool("Bash", command=f"cd {b} && grep -rn dough kilde 2>/dev/null&& ls kilde"),
|
||||
tool("Bash", command=f"cd {b} && grep -rn dough kilde 2>/dev/null|| ls kilde"),
|
||||
]), [])
|
||||
|
||||
def test_other_redirections_before_a_separator_are_still_breaches(self) -> None:
|
||||
b = self.basis
|
||||
found = self.breaches([
|
||||
tool("Bash", command=f"cd {b} && grep -rn dough kilde > kilde/out.md; ls kilde"),
|
||||
tool("Bash", command=f"cd {b} && grep -rn dough kilde >/dev/null; ls kilde"),
|
||||
tool("Bash", command=f"cd {b} && grep -rn dough kilde 2>&1; ls kilde"),
|
||||
tool("Bash", command=f"cd {b} && grep -rn dough kilde 2>/outside/err; ls kilde"),
|
||||
tool("Bash", command=f"cd {b} && grep -rn dough kilde 2>/dev/nullx; ls kilde"),
|
||||
])
|
||||
self.assertEqual(len(found), 5, found)
|
||||
|
||||
def test_a_search_pattern_or_sed_script_starting_with_a_slash_is_not_a_path(self) -> None:
|
||||
b, oven = self.basis, "kilde/docs-bread-oven.md"
|
||||
self.assertEqual(self.breaches([
|
||||
tool("Bash", command=f"cd {b} && grep -rn /oven kilde"),
|
||||
tool("Bash", command=f'cd {b} && grep -rlni "/oven" kilde | head -3'),
|
||||
tool("Bash", command=f"cd {b} && egrep -n -e /oven -e /crust {oven}"),
|
||||
tool("Bash", command=f"cd {b} && fgrep -A 3 -m 2 /oven {oven}"),
|
||||
tool("Bash", command=f"cd {b} && grep -rn -- /oven kilde"),
|
||||
tool("Bash", command=f"cd {b} && rg --regexp=/oven kilde"),
|
||||
tool("Bash", command=f"cd {b} && rg -n --regexp /oven -g '*.md' kilde"),
|
||||
tool("Bash", command=f"cd {b} && sed -n '/^oven:/p' {oven}"),
|
||||
tool("Bash", command=f"cd {b} && sed -n -e '/^oven:/,/^crust:/p' {oven}"),
|
||||
]), [])
|
||||
|
||||
def test_a_file_path_after_the_pattern_or_in_a_file_option_is_still_a_breach(self) -> None:
|
||||
b, oven = self.basis, "kilde/docs-bread-oven.md"
|
||||
found = self.breaches([
|
||||
tool("Bash", command=f"cd {b} && grep -rn oven /outside/notes.md"),
|
||||
tool("Bash", command=f"cd {b} && grep -e oven /outside/notes.md"),
|
||||
tool("Bash", command=f"cd {b} && grep -e /oven /outside/notes.md"),
|
||||
tool("Bash", command=f"cd {b} && grep -f /outside/words kilde"),
|
||||
tool("Bash", command=f"cd {b} && grep -rnf /outside/words kilde"),
|
||||
tool("Bash", command=f"cd {b} && grep --file=/outside/words kilde"),
|
||||
tool("Bash", command=f"cd {b} && rg -e oven -- /outside/notes.md"),
|
||||
tool("Bash", command=f"cd {b} && sed -f /outside/script {oven}"),
|
||||
tool("Bash", command=f"cd {b} && sed -n 1p /outside/notes.md"),
|
||||
tool("Bash", command=f"cd {b} && sed -n -e 1p -e 2p /outside/notes.md"),
|
||||
tool("Bash", command=f"cd {b} && grep -rn oven ~/notes"),
|
||||
tool("Bash", command=f"cd {b} && grep -rn oven ../.."),
|
||||
])
|
||||
self.assertEqual(len(found), 12, found)
|
||||
|
||||
def test_a_dollar_or_backtick_inside_single_quotes_is_not_an_expansion(self) -> None:
|
||||
b, oven = self.basis, "kilde/docs-bread-oven.md"
|
||||
self.assertEqual(self.breaches([
|
||||
tool("Bash", command=f"cd {b} && sed -n '1,$p' {oven}"),
|
||||
tool("Bash", command=f"cd {b} && grep -n '`oven`' {oven}"),
|
||||
tool("Bash", command=f"cd {b} && grep -n '\"$HOME\"' {oven}"),
|
||||
tool("Bash", command=f"cd {b} && grep -n \"it's\" {oven} | grep -c 'a$(b)'"),
|
||||
]), [])
|
||||
|
||||
def test_a_dollar_or_backtick_outside_single_quotes_is_still_an_expansion(self) -> None:
|
||||
b, oven = self.basis, "kilde/docs-bread-oven.md"
|
||||
found = self.breaches([
|
||||
tool("Bash", command=f'cd {b} && sed -n "1,$p" {oven}'),
|
||||
tool("Bash", command=f'cd {b} && grep -n "\\`oven\\`" {oven}'),
|
||||
tool("Bash", command=f"cd {b} && grep -n $HOME {oven}"),
|
||||
tool("Bash", command=f"cd {b} && grep -n 'a'$HOME'b' {oven}"),
|
||||
tool("Bash", command=f"cd {b} && grep -n \"'$HOME'\" {oven}"),
|
||||
tool("Bash", command=f"cd {b} && grep -n `id` {oven}"),
|
||||
tool("Bash", command=f"cd {b} && grep -n \\'$HOME {oven}"),
|
||||
])
|
||||
self.assertEqual(found, ["Bash with an expansion"] * 7, found)
|
||||
|
||||
def test_reading_the_own_answer_file_is_not_a_breach(self) -> None:
|
||||
b, own = self.basis, self.answer("d01")
|
||||
self.assertEqual(self.breaches([
|
||||
tool("Read", file_path=str(own)),
|
||||
tool("Bash", command=f"cd {own.parent} && jq -r '.citations|length' {own.name}"),
|
||||
tool("Bash", command=f"cd {own.parent}/ && jq -r .id {own}"),
|
||||
tool("Bash", command=f"jq -e '.citations|length' {own}"),
|
||||
tool("Bash", command=f"cd {b} && jq . {own} && grep -rn dough kilde"),
|
||||
tool("Bash", command=f"cd {b} && grep -c quote {own}"),
|
||||
]), [])
|
||||
|
||||
def test_another_file_beside_the_answer_or_another_command_is_still_a_breach(self) -> None:
|
||||
b, own, other = self.basis, self.answer("d01"), self.answer("d02")
|
||||
found = self.breaches([
|
||||
tool("Read", file_path=str(other)),
|
||||
tool("Bash", command=f"cd {own.parent} && jq . {other.name}"),
|
||||
tool("Bash", command=f"cd {own.parent} && ls"),
|
||||
tool("Bash", command=f"cd {own.parent} && jq . {own.name} && cat {other.name}"),
|
||||
tool("Bash", command=f"cd {own.parent} && jq . *.json"),
|
||||
tool("Bash", command=f"cd {own.parent} && jq -f filter.jq {own.name}"),
|
||||
tool("Bash", command=f"cd {own.parent} && jq . ../bundle/d01.json"),
|
||||
tool("Bash", command=f"jq . {other}"),
|
||||
tool("Bash", command=f"jq . {own.name}"),
|
||||
tool("Bash", command=f"jq --slurpfile x {other} . {own}"),
|
||||
tool("Bash", command=f"jq . {own} > {own}.bak"),
|
||||
tool("Bash", command=f'cd {b} && python3 -c "import json;json.load(open(\'{own}\'))"'),
|
||||
tool("Bash", command=f"python3 -m json.tool {own}"),
|
||||
tool("Bash", command=f"cd {b} && jq . kilde/docs-bread-oven.md"),
|
||||
tool("Bash", command=f"cd {self.run_dir / 'answers'} && jq . filsok/{own.name}"),
|
||||
])
|
||||
self.assertEqual(len(found), 15, found)
|
||||
|
||||
def test_other_arms_runs_and_unrelated_agents_are_ignored(self) -> None:
|
||||
write_transcript(self.subagents / "agent-a1.jsonl", self.good_lines("d01"), "S1-r1-filsok-d01")
|
||||
write_transcript(self.subagents / "agent-a2.jsonl", self.good_lines("d02"), "S1-r1-filsok-d02")
|
||||
write_transcript(self.subagents / "agent-b1.jsonl", self.good_lines("d01"), "S1-r1-bundle-d01")
|
||||
write_transcript(self.subagents / "agent-b2.jsonl", self.good_lines("d01"), "S1-r2-filsok-d01")
|
||||
write_transcript(self.subagents / "agent-c1.jsonl", self.good_lines("d01"), "V1-r1-read-c001")
|
||||
code, out, doc = self.harvest()
|
||||
self.assertEqual(code, 0, out)
|
||||
self.assertEqual([r["runs"] for r in doc["questions"]], [1, 1])
|
||||
|
||||
def test_a_retried_question_sums_its_runs(self) -> None:
|
||||
write_transcript(self.subagents / "agent-a1.jsonl", self.good_lines("d01"), "S1-r1-filsok-d01")
|
||||
write_transcript(self.subagents / "agent-a3.jsonl", self.good_lines("d01", "2026-10-09T11:00:00Z"),
|
||||
"S1-r1-filsok-d01")
|
||||
write_transcript(self.subagents / "agent-a2.jsonl", self.good_lines("d02"), "S1-r1-filsok-d02")
|
||||
code, out, doc = self.harvest()
|
||||
row = doc["questions"][0]
|
||||
self.assertEqual((row["runs"], row["input_tokens"], row["output_tokens"]), (2, 30, 114))
|
||||
|
||||
def test_a_transcript_without_a_description_is_matched_by_its_task_path(self) -> None:
|
||||
write_transcript(self.subagents / "agent-a1.jsonl", self.good_lines("d01"), "S1-r1-filsok-d01")
|
||||
write_transcript(self.transcripts / "workflow-x" / "agent-w1.jsonl", self.good_lines("d02"))
|
||||
code, out, doc = self.harvest()
|
||||
self.assertEqual(code, 0, out)
|
||||
self.assertEqual([r["id"] for r in doc["questions"]], ["d01", "d02"])
|
||||
|
||||
def test_a_task_without_a_transcript_is_listed_and_exits_1(self) -> None:
|
||||
write_transcript(self.subagents / "agent-a1.jsonl", self.good_lines("d01"), "S1-r1-filsok-d01")
|
||||
code, out, doc = self.harvest()
|
||||
self.assertEqual(code, 1)
|
||||
self.assertIn("d02", out)
|
||||
self.assertEqual([r["id"] for r in doc["questions"]], ["d01"])
|
||||
|
||||
def test_an_output_never_recorded_is_derived_from_the_next_input_as_a_lower_bound(self) -> None:
|
||||
lines = [
|
||||
user(f"Read {self.task('d01')} and do what it says."),
|
||||
assistant("m1", usage(1, 0, 1000, 2), [tool("Read", file_path=str(self.task("d01")))], None),
|
||||
assistant("m2", usage(1, 5000, 1000, 10), [], "end_turn"),
|
||||
]
|
||||
write_transcript(self.subagents / "agent-a1.jsonl", lines, "S1-r1-filsok-d01")
|
||||
write_transcript(self.subagents / "agent-a2.jsonl", self.good_lines("d02"), "S1-r1-filsok-d02")
|
||||
_, _, doc = self.harvest()
|
||||
row = doc["questions"][0]
|
||||
self.assertEqual(row["output_tokens"], 12)
|
||||
self.assertEqual(row["output_tokens_derived"], 4998)
|
||||
|
||||
|
||||
def result(qid: str, group: str, correct: bool, paths: int = 2) -> dict[str, Any]:
|
||||
return {"id": qid, "group": group, "correct": correct, "keys_held": 2 if correct else 0, "keys": 2,
|
||||
"paths_cited": paths if correct else 0, "paths_needed": paths}
|
||||
|
||||
|
||||
def results_doc(arm: str, rows: list[dict[str, Any]], set_sha: str = "a" * 64) -> dict[str, Any]:
|
||||
return {"schema": "okf-wiki/answer-results/1", "arm": arm, "set_sha256": set_sha, "results": rows}
|
||||
|
||||
|
||||
# Ten questions: both right 2, only filsok 1, only bundle 5, neither 2.
|
||||
PATTERN = [ # (id, group, paths, filsok, bundle)
|
||||
("d01", "docs-2-3", 2, True, True), ("d02", "docs-2-3", 3, True, True), ("d03", "docs-2-3", 2, True, False),
|
||||
("d04", "docs-4", 4, False, True), ("d05", "docs-4", 5, False, True), ("d06", "docs-4", 4, False, True),
|
||||
("d07", "history", 2, False, True), ("d08", "history", 4, False, True), ("d09", "docs-4", 4, False, False),
|
||||
("d10", "history", 3, False, False),
|
||||
]
|
||||
|
||||
|
||||
class BashBreachTest(unittest.TestCase):
|
||||
"""Review findings 8-11 (order 39): option values, sed scripts, comments and newlines in the breach parser."""
|
||||
|
||||
basis = Path("/w/arms/filsok")
|
||||
|
||||
def breach(self, rest: str) -> str | None:
|
||||
return answers.bash_breach(f"cd {self.basis} && {rest}", self.basis)
|
||||
|
||||
def assertClean(self, rest: str) -> None:
|
||||
self.assertIsNone(self.breach(rest), rest)
|
||||
|
||||
def assertBreach(self, rest: str) -> None:
|
||||
self.assertIsNotNone(self.breach(rest), rest)
|
||||
|
||||
# finding 9: grep -C and long options with a separate value
|
||||
def test_grep_context_and_long_option_values_are_not_the_search_word(self) -> None:
|
||||
for rest in ('grep -C 2 "/kilde/docs-x.md" wiki/index.md',
|
||||
'grep -rn --include "*.md" "/kilde/x" wiki',
|
||||
'grep -rn --exclude-dir node_modules /kilde/x wiki',
|
||||
'grep --after-context 3 /kilde/x wiki/index.md',
|
||||
'grep --max-count 1 -e /kilde/x wiki/index.md',
|
||||
"rg --glob '*.md' /kilde/x wiki",
|
||||
"rg -n --max-depth 2 /kilde/x wiki"):
|
||||
with self.subTest(rest=rest):
|
||||
self.assertClean(rest)
|
||||
|
||||
def test_grep_context_with_a_separate_value_stays_strict(self) -> None:
|
||||
# BSD grep's --context takes its value only after '=': `--context 1 /x f` makes 1 the word and /x a file.
|
||||
self.assertBreach("grep --context 1 /outside/notes.md wiki")
|
||||
self.assertClean("grep --context=1 /kilde/x wiki")
|
||||
|
||||
# finding 8: options whose value is a file are checked as paths
|
||||
def test_a_file_option_value_outside_the_basis_is_a_breach(self) -> None:
|
||||
for rest in ("grep --exclude-from /outside/list -rn dough kilde",
|
||||
"grep --exclude-from=/outside/list -rn dough kilde",
|
||||
"rg --ignore-file /outside/x dough kilde",
|
||||
"grep -rn --include /outside/x dough kilde",
|
||||
"rg --pre cat dough kilde",
|
||||
"rg --pre=cat dough kilde"):
|
||||
with self.subTest(rest=rest):
|
||||
self.assertBreach(rest)
|
||||
|
||||
# finding 11 and 8: sed scripts that read, write or run, and clustered in-place flags
|
||||
def test_sed_scripts_that_write_read_outside_or_run_are_breaches(self) -> None:
|
||||
for rest in ("sed -n '/x/w /tmp/out' wiki/index.md",
|
||||
"sed '1e touch /tmp/p' wiki/index.md",
|
||||
"sed -n '1r /outside/notes.md' wiki/index.md",
|
||||
"sed -n '1R /outside/notes.md' wiki/index.md",
|
||||
"sed -n 'W out.txt' wiki/index.md",
|
||||
"sed 's/a/b/w out.txt' wiki/index.md",
|
||||
"sed 's/a/b/e' wiki/index.md",
|
||||
"sed -n -e 1p -e '2w out' wiki/index.md",
|
||||
"sed -n --expression='2w out' wiki/index.md",
|
||||
"sed -ne'2w out' wiki/index.md",
|
||||
"sed -n '1{p;w out\n}' wiki/index.md",
|
||||
"sed -n '1r ../../x' wiki/index.md",
|
||||
"sed -ni s/a/b/ wiki/index.md",
|
||||
"sed -Ei s/a/b/ wiki/index.md",
|
||||
"sed -I '' s/a/b/ wiki/index.md",
|
||||
"sed -n '1q;}' wiki/index.md"):
|
||||
with self.subTest(rest=rest):
|
||||
self.assertBreach(rest)
|
||||
|
||||
def test_read_only_sed_scripts_stay_clean(self) -> None:
|
||||
for rest in ("sed -n '/^oven:/p' kilde/a.md",
|
||||
"sed -n '/write/,/^## /p' kilde/a.md",
|
||||
"sed -n '/^## Reading/,/^## /{p;}' kilde/a.md",
|
||||
"sed -n '10,20p;25q' kilde/a.md",
|
||||
"sed -n '$=' kilde/a.md",
|
||||
"sed 's/where/w/g' kilde/a.md",
|
||||
"sed -E 's#/kilde/(.*)#\\1#' kilde/a.md",
|
||||
"sed -n '\\,/kilde/,p' kilde/a.md",
|
||||
"sed -n '/x/I p' kilde/a.md",
|
||||
"sed -n '1r kilde/b.md' kilde/a.md",
|
||||
"sed -n -e '/oven/!d' -e p kilde/a.md",
|
||||
"sed -n 'y/abc/xyz/;p' kilde/a.md"):
|
||||
with self.subTest(rest=rest):
|
||||
self.assertClean(rest)
|
||||
|
||||
# finding 10: '#' inside a word and a newline between commands
|
||||
def test_a_hash_inside_a_word_does_not_hide_the_rest_of_the_line(self) -> None:
|
||||
self.assertEqual("Bash command not allowed: python3", self.breach("grep -c a#b kilde && python3 -c 'print(1)'"))
|
||||
|
||||
def test_a_newline_separates_commands(self) -> None:
|
||||
self.assertEqual("Bash command not allowed: python3", self.breach("grep -c a kilde\npython3 -c 'print(1)'"))
|
||||
self.assertEqual("Bash command not allowed: python3", self.breach("grep -c a kilde &&\npython3 x"))
|
||||
self.assertClean("grep -c a kilde\nls kilde\n")
|
||||
self.assertClean("grep -rn a kilde |\nhead -3")
|
||||
self.assertClean('grep -c "a\nb" kilde')
|
||||
|
||||
|
||||
class ScoreTest(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
self._tmp = tempfile.TemporaryDirectory()
|
||||
self.tmp = Path(self._tmp.name)
|
||||
self.a = self.write("filsok", [result(i, g, f, p) for i, g, p, f, _ in PATTERN])
|
||||
self.b = self.write("bundle", [result(i, g, b, p) for i, g, p, _, b in PATTERN])
|
||||
|
||||
def tearDown(self) -> None:
|
||||
self._tmp.cleanup()
|
||||
|
||||
def write(self, arm: str, rows: list[dict[str, Any]], name: str = "", set_sha: str = "a" * 64) -> Path:
|
||||
path = self.tmp / (name or f"{arm}.json")
|
||||
path.write_bytes(dump(results_doc(arm, rows, set_sha)))
|
||||
return path
|
||||
|
||||
def score(self, *extra: str) -> tuple[int, str]:
|
||||
return run(["score", "--results", str(self.a), "--results", str(self.b), *extra])
|
||||
|
||||
def test_per_arm_and_group_and_by_paths(self) -> None:
|
||||
code, out = self.score()
|
||||
self.assertEqual(code, 0, out)
|
||||
self.assertIn("filsok: docs-2-3 3 of 3, docs-4 0 of 4, history 0 of 3, all 3 of 10", out)
|
||||
self.assertIn("bundle: docs-2-3 2 of 3, docs-4 3 of 4, history 2 of 3, all 7 of 10", out)
|
||||
self.assertIn("filsok by paths: 2-3 3 of 5, 4+ 0 of 5", out)
|
||||
self.assertIn("bundle by paths: 2-3 3 of 5, 4+ 4 of 5", out)
|
||||
|
||||
def test_pairs_mcnemar_difference_and_smallest_visible_difference(self) -> None:
|
||||
out_path = self.tmp / "score.json"
|
||||
code, out = self.score("--out", str(out_path))
|
||||
self.assertEqual(code, 0, out)
|
||||
self.assertIn("pairs: both 2, only filsok 1, only bundle 5, neither 2", out)
|
||||
self.assertIn("exact McNemar p = 0.219", out)
|
||||
doc = json.loads(out_path.read_text(encoding="utf-8"))
|
||||
self.assertEqual(doc["schema"], "okf-wiki/answer-score/1")
|
||||
self.assertEqual(doc["pairs"]["all"], {"both": 2, "only filsok": 1, "only bundle": 5, "neither": 2})
|
||||
self.assertAlmostEqual(doc["mcnemar_p"], 0.21875)
|
||||
low, high = answers.paired_difference_ci(2, 5, 1, 2)
|
||||
self.assertAlmostEqual(doc["difference"]["point"], 0.4)
|
||||
self.assertAlmostEqual(doc["difference"]["low"], low)
|
||||
self.assertAlmostEqual(doc["difference"]["high"], high)
|
||||
self.assertAlmostEqual(doc["smallest_visible"]["discordant_share"], 0.6)
|
||||
self.assertEqual(doc["smallest_visible"]["difference"], answers.min_detectable(10, 0.6))
|
||||
self.assertIn("bundle - filsok = 0.400, 95 % CI", out)
|
||||
self.assertIn("smallest difference visible at 80 % power", out)
|
||||
self.assertEqual(out_path.read_bytes(), dump(doc))
|
||||
|
||||
def test_no_line_per_question(self) -> None:
|
||||
_, out = self.score()
|
||||
for qid, *_ in PATTERN:
|
||||
self.assertNotIn(qid, out)
|
||||
|
||||
def test_tokens_per_question_are_reported_beside(self) -> None:
|
||||
tok = {"schema": "okf-wiki/answer-tokens/1", "arm": "filsok", "questions": [
|
||||
{"id": "d01", "input_tokens": 100, "cache_creation_input_tokens": 200, "cache_read_input_tokens": 600,
|
||||
"output_tokens": 50, "output_tokens_derived": 50, "isolation_breaches": 1},
|
||||
{"id": "d02", "input_tokens": 100, "cache_creation_input_tokens": 200, "cache_read_input_tokens": 600,
|
||||
"output_tokens": 100, "output_tokens_derived": 0, "isolation_breaches": 0}]}
|
||||
path = self.tmp / "tok-filsok.json"
|
||||
path.write_bytes(dump(tok))
|
||||
code, out = self.score("--tokens", str(path))
|
||||
self.assertEqual(code, 0, out)
|
||||
self.assertIn("filsok tokens: 2 questions harvested, 1000 per question, isolation breaches 1", out)
|
||||
|
||||
def test_two_result_files_for_different_sets_cannot_run(self) -> None:
|
||||
self.b = self.write("bundle", [result(i, g, b, p) for i, g, p, _, b in PATTERN], set_sha="b" * 64)
|
||||
self.assertEqual(self.score()[0], 1)
|
||||
|
||||
def test_two_result_files_with_different_questions_cannot_run(self) -> None:
|
||||
self.b = self.write("bundle", [result(i, g, b, p) for i, g, p, _, b in PATTERN[:-1]])
|
||||
self.assertEqual(self.score()[0], 1)
|
||||
|
||||
def test_the_arms_are_filsok_and_bundle(self) -> None:
|
||||
self.b = self.write("server", [result(i, g, b, p) for i, g, p, _, b in PATTERN])
|
||||
self.assertEqual(self.score()[0], 1)
|
||||
|
||||
def test_a_missing_result_file_cannot_run(self) -> None:
|
||||
self.b = self.tmp / "none.json"
|
||||
self.assertEqual(self.score()[0], 1)
|
||||
|
||||
|
||||
class StatisticsTest(unittest.TestCase):
|
||||
def test_exact_mcnemar_against_known_values(self) -> None:
|
||||
self.assertAlmostEqual(answers.mcnemar_exact(0, 6), 0.03125)
|
||||
self.assertAlmostEqual(answers.mcnemar_exact(6, 0), 0.03125)
|
||||
self.assertAlmostEqual(answers.mcnemar_exact(1, 9), 0.021484375)
|
||||
self.assertAlmostEqual(answers.mcnemar_exact(2, 0), 0.5)
|
||||
self.assertEqual(answers.mcnemar_exact(0, 0), 1.0)
|
||||
self.assertEqual(answers.mcnemar_exact(5, 5), 1.0)
|
||||
|
||||
def test_paired_interval_against_newcombe_table_iii_method_10(self) -> None:
|
||||
for cells, (low, high) in NEWCOMBE_TABLE_III:
|
||||
got = answers.paired_difference_ci(*cells)
|
||||
self.assertAlmostEqual(got[0], low, delta=1e-4, msg=str(cells))
|
||||
self.assertAlmostEqual(got[1], high, delta=1e-4, msg=str(cells))
|
||||
|
||||
def test_paired_interval_is_antisymmetric_in_the_discordant_cells(self) -> None:
|
||||
low, high = answers.paired_difference_ci(20, 12, 2, 16)
|
||||
back = answers.paired_difference_ci(20, 2, 12, 16)
|
||||
self.assertAlmostEqual(back[0], -high)
|
||||
self.assertAlmostEqual(back[1], -low)
|
||||
|
||||
def test_smallest_visible_difference_against_the_plans_sample_sizes(self) -> None:
|
||||
# PLAN.md § Styrke: 37-57 questions see 20 points, 155-234 see 10 points, at 20-30 % discordant pairs.
|
||||
for n, share, diff in ((37, 0.2, 0.2), (57, 0.3, 0.2), (155, 0.2, 0.1), (234, 0.3, 0.1)):
|
||||
visible = answers.min_detectable(n, share)
|
||||
assert visible is not None
|
||||
self.assertLessEqual(visible, diff, (n, share))
|
||||
fewer = answers.min_detectable(n - 1, share)
|
||||
self.assertTrue(fewer is None or fewer > diff, (n - 1, share, fewer))
|
||||
|
||||
def test_no_discordant_pairs_has_no_smallest_difference(self) -> None:
|
||||
self.assertIsNone(answers.min_detectable(60, 0.0))
|
||||
|
||||
def test_too_few_pairs_for_any_difference_up_to_the_discordant_share(self) -> None:
|
||||
# n = 10, psi = 0.6: even d = psi needs (z_a sqrt(.6) + z_b sqrt(.6 - .36))^2 / .36 = 10.35 pairs.
|
||||
self.assertIsNone(answers.min_detectable(10, 0.6))
|
||||
visible = answers.min_detectable(11, 0.6)
|
||||
assert visible is not None
|
||||
self.assertLessEqual(visible, 0.6)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
790
gates/tests/test_answerset.py
Normal file
790
gates/tests/test_answerset.py
Normal file
|
|
@ -0,0 +1,790 @@
|
|||
"""Tests for gates/answerset.py: the question set for criterion 4 on collection 1 (contract K5 point 1).
|
||||
|
||||
check, inside, split, gate and stub on synthetic K1 roots about bread and gardens. No corpus text is used
|
||||
(CLAUDE.md rule 3).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
import copy
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import random
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
|
||||
import answerset # noqa: E402
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
|
||||
RISING = "kilde/docs-bread-rising.md"
|
||||
OVEN = "kilde/docs-bread-oven.md"
|
||||
SOIL = "kilde/docs-garden-soil.md"
|
||||
WATER = "kilde/docs-garden-water.md"
|
||||
FROST = "kilde/docs-garden-frost.md"
|
||||
RELEASE = "kilde/release-v1.2.0.md"
|
||||
|
||||
TEXTS: dict[str, tuple[str, bool]] = {
|
||||
RISING: ("# Rising\n\nThe dough rises for 90 minutes at room temperature.\n"
|
||||
"Use the `--slow-proof` flag for a cold rise overnight.\n", True),
|
||||
OVEN: ("# Oven\n\nBake the loaf at 230 degrees with the lid on.\nThe crust does not form without steam.\n", True),
|
||||
SOIL: ("# Soil\n\nGarden beds need compost every spring.\nSet `SOIL_MODE=loam` for heavy clay.\n", True),
|
||||
WATER: ("# Water\n\nWater the tomatoes twice a week in July.\nThe drip line is called hose-9.\n", True),
|
||||
FROST: ("# Frost\n\nCover the seedlings when frost is forecast below 2 degrees.\n", True),
|
||||
RELEASE: ("# 1.2.0\n\n- Added the `--slow-proof` flag for cold rises.\n- The oven timer no longer resets on restart.\n",
|
||||
False),
|
||||
}
|
||||
PART_PATHS = (RISING, OVEN, SOIL, WATER)
|
||||
|
||||
SPAN = {
|
||||
RISING: "The dough rises for 90 minutes at room temperature.",
|
||||
OVEN: "Bake the loaf at 230 degrees with the lid on.",
|
||||
SOIL: "Set `SOIL_MODE=loam` for heavy clay.",
|
||||
WATER: "The drip line is called hose-9.",
|
||||
FROST: "Cover the seedlings when frost is forecast below 2 degrees.",
|
||||
RELEASE: "Added the `--slow-proof` flag for cold rises.",
|
||||
}
|
||||
KEY = {RISING: "90 minutes", OVEN: "230 degrees", SOIL: "SOIL_MODE=loam", WATER: "hose-9", FROST: "seedlings",
|
||||
RELEASE: "--slow-proof"}
|
||||
|
||||
|
||||
def dump(data: Any) -> bytes:
|
||||
return (json.dumps(data, indent=1, sort_keys=True, ensure_ascii=False) + "\n").encode("utf-8")
|
||||
|
||||
|
||||
def sha(data: bytes) -> str:
|
||||
return hashlib.sha256(data).hexdigest()
|
||||
|
||||
|
||||
def write_root(root: Path, paths: tuple[str, ...] | None = None, frozen_at: str = "2026-10-08T12:00:00Z",
|
||||
subset_of: dict[str, str] | None = None) -> Path:
|
||||
"""A K1 root: kilde/ files and source-manifest.json, written as contract K1 says."""
|
||||
sources = []
|
||||
for path in sorted(paths if paths is not None else TEXTS):
|
||||
text, main = TEXTS[path]
|
||||
data = text.encode("utf-8")
|
||||
target = root / path
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
target.write_bytes(data)
|
||||
kind = "release-notes" if path.startswith("kilde/release-") else "docs"
|
||||
entry: dict[str, Any] = {"bytes": len(data), "kind": kind, "lines": len(text.splitlines()),
|
||||
"main_topic": main, "path": path, "sha256": sha(data),
|
||||
"url": "https://example.invalid/" + path}
|
||||
if kind == "release-notes":
|
||||
entry["version"] = "1.2.0"
|
||||
sources.append(entry)
|
||||
manifest = {"collection": "bread-and-garden", "frozen_at": frozen_at, "schema": "okf-wiki/source-manifest/1",
|
||||
"sources": sources, "subset_of": subset_of}
|
||||
root.mkdir(parents=True, exist_ok=True)
|
||||
(root / "source-manifest.json").write_bytes(dump(manifest))
|
||||
return root
|
||||
|
||||
|
||||
def root_ref(root: Path) -> dict[str, str]:
|
||||
data = (root / "source-manifest.json").read_bytes()
|
||||
return {"frozen_at": json.loads(data)["frozen_at"], "manifest_sha256": sha(data)}
|
||||
|
||||
|
||||
def question(qid: str, paths: tuple[str, ...], group: str) -> dict[str, Any]:
|
||||
return {
|
||||
"id": qid,
|
||||
"question": "How do the synthetic notes about bread and gardens fit together?",
|
||||
"group": group,
|
||||
"evidence": [{"path": p, "span": SPAN[p]} for p in paths],
|
||||
"answer_keys": [{"path": p, "text": KEY[p]} for p in paths],
|
||||
}
|
||||
|
||||
|
||||
def q23(qid: str = "q01") -> dict[str, Any]:
|
||||
return question(qid, (RISING, OVEN), "docs-2-3")
|
||||
|
||||
|
||||
def q4(qid: str = "q02") -> dict[str, Any]:
|
||||
return question(qid, (RISING, OVEN, SOIL, WATER), "docs-4")
|
||||
|
||||
|
||||
def qhist(qid: str = "q03") -> dict[str, Any]:
|
||||
return question(qid, (RISING, RELEASE), "history")
|
||||
|
||||
|
||||
def make_set(root: Path, questions: list[dict[str, Any]]) -> dict[str, Any]:
|
||||
return {"schema": "okf-wiki/answer-set/1", "sources": root_ref(root), "questions": questions}
|
||||
|
||||
|
||||
def run(argv: list[str]) -> tuple[int, str]:
|
||||
out, err = io.StringIO(), io.StringIO()
|
||||
with contextlib.redirect_stdout(out), contextlib.redirect_stderr(err):
|
||||
code = answerset.main(argv)
|
||||
return code, out.getvalue() + err.getvalue()
|
||||
|
||||
|
||||
class Base(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
self._tmp = tempfile.TemporaryDirectory()
|
||||
self.tmp = Path(self._tmp.name)
|
||||
self.root = write_root(self.tmp / "copy")
|
||||
|
||||
def tearDown(self) -> None:
|
||||
self._tmp.cleanup()
|
||||
|
||||
def write_set(self, questions: list[dict[str, Any]], name: str = "set.json", root: Path | None = None) -> Path:
|
||||
path = self.tmp / name
|
||||
path.write_bytes(dump(make_set(root or self.root, questions)))
|
||||
return path
|
||||
|
||||
def check(self, questions: list[dict[str, Any]], root: Path | None = None) -> tuple[int, str]:
|
||||
return run(["check", "--set", str(self.write_set(questions, root=root)), "--sources", str(root or self.root)])
|
||||
|
||||
|
||||
class CheckTest(Base):
|
||||
def test_a_good_set_passes_with_the_groups_counted(self) -> None:
|
||||
code, out = self.check([q23(), q4(), qhist()])
|
||||
self.assertEqual(code, 0, out)
|
||||
self.assertIn("questions: 3", out)
|
||||
self.assertIn("groups: docs-2-3 1, docs-4 1, history 1", out)
|
||||
self.assertIn("spans verbatim: 8 of 8", out)
|
||||
self.assertIn("questions with >= 2 distinct paths: 3 of 3", out)
|
||||
self.assertIn("questions whose group matches K1 and the path count: 3 of 3", out)
|
||||
self.assertIn("questions with a key in every path's spans: 3 of 3", out)
|
||||
self.assertIn("answer keys that are wording: 0 of 8", out)
|
||||
self.assertTrue(out.rstrip().endswith("PASS"), out)
|
||||
|
||||
def test_an_empty_set_fails(self) -> None:
|
||||
code, out = self.check([])
|
||||
self.assertEqual(code, 1)
|
||||
self.assertIn("FAIL", out)
|
||||
|
||||
def test_a_span_that_does_not_stand_fails(self) -> None:
|
||||
q = q23()
|
||||
q["evidence"][0]["span"] = "The dough rises for 80 minutes at room temperature."
|
||||
code, out = self.check([q, q4()])
|
||||
self.assertEqual(code, 1)
|
||||
self.assertIn("spans verbatim: 5 of 6", out)
|
||||
self.assertIn("FAIL", out)
|
||||
|
||||
def test_a_span_must_stand_byte_for_byte(self) -> None:
|
||||
q = q23()
|
||||
q["evidence"][0]["span"] = "The dough rises for 90 minutes at room temperature."
|
||||
code, out = self.check([q])
|
||||
self.assertEqual(code, 1)
|
||||
self.assertIn("spans verbatim: 1 of 2", out)
|
||||
|
||||
def test_a_question_with_one_path_fails(self) -> None:
|
||||
q = question("q09", (RISING,), "docs-2-3")
|
||||
code, out = self.check([q23(), q])
|
||||
self.assertEqual(code, 1)
|
||||
self.assertIn("questions with >= 2 distinct paths: 1 of 2", out)
|
||||
|
||||
def test_two_spans_in_one_file_are_one_path(self) -> None:
|
||||
q = q23()
|
||||
q["evidence"] = [{"path": RISING, "span": SPAN[RISING]},
|
||||
{"path": RISING, "span": "Use the `--slow-proof` flag for a cold rise overnight."}]
|
||||
q["answer_keys"] = [{"path": RISING, "text": "90 minutes"}]
|
||||
code, out = self.check([q])
|
||||
self.assertEqual(code, 1)
|
||||
self.assertIn("questions with >= 2 distinct paths: 0 of 1", out)
|
||||
|
||||
def test_history_is_set_by_a_source_without_a_main_topic(self) -> None:
|
||||
q = qhist()
|
||||
q["group"] = "docs-2-3"
|
||||
code, out = self.check([q])
|
||||
self.assertEqual(code, 1)
|
||||
self.assertIn("questions whose group matches K1 and the path count: 0 of 1", out)
|
||||
|
||||
def test_four_paths_are_docs_4(self) -> None:
|
||||
q = q4()
|
||||
q["group"] = "docs-2-3"
|
||||
code, out = self.check([q])
|
||||
self.assertEqual(code, 1)
|
||||
self.assertIn("questions whose group matches K1 and the path count: 0 of 1", out)
|
||||
|
||||
def test_three_paths_are_docs_2_3(self) -> None:
|
||||
q = question("q05", (RISING, OVEN, SOIL), "docs-4")
|
||||
self.assertEqual(self.check([q])[0], 1)
|
||||
q["group"] = "docs-2-3"
|
||||
self.assertEqual(self.check([q])[0], 0)
|
||||
|
||||
def test_an_unknown_group_fails(self) -> None:
|
||||
q = q23()
|
||||
q["group"] = "lookup"
|
||||
self.assertEqual(self.check([q])[0], 1)
|
||||
|
||||
def test_a_wording_key_fails(self) -> None:
|
||||
q = q23()
|
||||
q["answer_keys"][0]["text"] = "rises for 90 minutes"
|
||||
code, out = self.check([q])
|
||||
self.assertEqual(code, 1)
|
||||
self.assertIn("answer keys that are wording: 1 of 2", out)
|
||||
|
||||
def test_a_negated_key_must_stand_negated(self) -> None:
|
||||
q = q23()
|
||||
q["answer_keys"][0]["negated"] = True
|
||||
code, out = self.check([q])
|
||||
self.assertEqual(code, 1)
|
||||
self.assertIn("negated keys negated in their span: 0 of 1", out)
|
||||
|
||||
def test_a_negated_key_that_stands_negated_passes(self) -> None:
|
||||
q = q23()
|
||||
q["evidence"][1]["span"] = "The crust does not form without steam."
|
||||
q["answer_keys"][1] = {"path": OVEN, "text": "form", "negated": True}
|
||||
code, out = self.check([q])
|
||||
self.assertEqual(code, 0, out)
|
||||
self.assertIn("negated keys negated in their span: 1 of 1", out)
|
||||
|
||||
def test_a_path_without_a_key_in_its_span_fails(self) -> None:
|
||||
q = q23()
|
||||
q["answer_keys"] = [{"path": RISING, "text": "90 minutes"}]
|
||||
code, out = self.check([q])
|
||||
self.assertEqual(code, 1)
|
||||
self.assertIn("questions with a key in every path's spans: 0 of 1", out)
|
||||
|
||||
def test_a_key_that_stands_only_in_another_path_fails(self) -> None:
|
||||
q = q23()
|
||||
q["answer_keys"][1] = {"path": OVEN, "text": "90 minutes"}
|
||||
code, out = self.check([q])
|
||||
self.assertEqual(code, 1)
|
||||
self.assertIn("questions with a key in every path's spans: 0 of 1", out)
|
||||
|
||||
def test_a_path_that_k1_does_not_list_fails(self) -> None:
|
||||
q = q23()
|
||||
q["evidence"][1]["path"] = "kilde/docs-bread-crumb.md"
|
||||
q["answer_keys"][1]["path"] = "kilde/docs-bread-crumb.md"
|
||||
code, out = self.check([q])
|
||||
self.assertEqual(code, 1)
|
||||
self.assertIn("error:", out)
|
||||
|
||||
def test_a_file_that_differs_from_k1_fails(self) -> None:
|
||||
(self.root / OVEN).write_text(TEXTS[OVEN][0] + "Extra line.\n", encoding="utf-8")
|
||||
code, out = self.check([q23()])
|
||||
self.assertEqual(code, 1)
|
||||
self.assertIn("sha256", out)
|
||||
|
||||
def test_the_set_must_name_the_root_it_was_made_from(self) -> None:
|
||||
other = write_root(self.tmp / "other", frozen_at="2026-10-09T12:00:00Z")
|
||||
path = self.write_set([q23()], root=other)
|
||||
code, out = run(["check", "--set", str(path), "--sources", str(self.root)])
|
||||
self.assertEqual(code, 1)
|
||||
self.assertIn("manifest_sha256", out)
|
||||
|
||||
def test_a_root_without_k1_cannot_run(self) -> None:
|
||||
path = self.write_set([q23()])
|
||||
(self.root / "source-manifest.json").unlink()
|
||||
code, out = run(["check", "--set", str(path), "--sources", str(self.root)])
|
||||
self.assertEqual(code, 1)
|
||||
self.assertIn("FAIL", out)
|
||||
|
||||
def test_a_wrong_schema_cannot_run(self) -> None:
|
||||
data = make_set(self.root, [q23()])
|
||||
data["schema"] = "okf-wiki/multidoc-set/1"
|
||||
path = self.tmp / "wrong.json"
|
||||
path.write_bytes(dump(data))
|
||||
self.assertEqual(run(["check", "--set", str(path), "--sources", str(self.root)])[0], 1)
|
||||
|
||||
def test_duplicate_ids_fail(self) -> None:
|
||||
self.assertEqual(self.check([q23("q01"), q4("q01")])[0], 1)
|
||||
|
||||
|
||||
class InsideTest(Base):
|
||||
def setUp(self) -> None:
|
||||
super().setUp()
|
||||
ref = root_ref(self.root)
|
||||
self.part = write_root(self.tmp / "part", PART_PATHS, frozen_at="2026-10-08T19:00:00Z", subset_of=ref)
|
||||
|
||||
def test_counts_the_questions_with_every_span_in_the_part_root(self) -> None:
|
||||
path = self.write_set([q23(), q4(), qhist()])
|
||||
code, out = run(["inside", "--set", str(path), "--part-root", str(self.part)])
|
||||
self.assertEqual(code, 0, out)
|
||||
self.assertIn("questions with every span in the part root: 2 of 3", out)
|
||||
|
||||
def test_require_all_fails_unless_every_question_is_inside(self) -> None:
|
||||
path = self.write_set([q23(), qhist()])
|
||||
code, out = run(["inside", "--set", str(path), "--part-root", str(self.part), "--require-all"])
|
||||
self.assertEqual(code, 1)
|
||||
self.assertIn("1 of 2", out)
|
||||
inside = self.write_set([q23(), q4()], name="inside.json", root=self.part)
|
||||
code, out = run(["inside", "--set", str(inside), "--part-root", str(self.part), "--require-all"])
|
||||
self.assertEqual(code, 0, out)
|
||||
self.assertIn("2 of 2", out)
|
||||
|
||||
def test_a_span_that_is_not_in_the_part_file_is_outside(self) -> None:
|
||||
(self.part / OVEN).write_text("# Oven\n\nSomething else entirely.\n", encoding="utf-8")
|
||||
path = self.write_set([q23()])
|
||||
code, out = run(["inside", "--set", str(path), "--part-root", str(self.part)])
|
||||
self.assertIn("0 of 1", out)
|
||||
|
||||
|
||||
class SplitAndGateBase(Base):
|
||||
"""Eight questions (4 docs-2-3, 2 docs-4, 2 history), a part of two, split with seed 7, 4 held out."""
|
||||
|
||||
def setUp(self) -> None:
|
||||
super().setUp()
|
||||
self.part_root = write_root(self.tmp / "part", PART_PATHS, frozen_at="2026-10-08T19:00:00Z",
|
||||
subset_of=root_ref(self.root))
|
||||
questions = ([q23(f"q{i:02d}") for i in (1, 2, 3, 4)] + [q4(f"q{i:02d}") for i in (5, 6)]
|
||||
+ [qhist(f"q{i:02d}") for i in (7, 8)])
|
||||
for n, q in enumerate(questions):
|
||||
q["question"] = f"Synthetic question number {n} about bread and gardens?"
|
||||
self.all = self.write_set(questions, name="all.json")
|
||||
self.part = self.write_set([q23("p01"), q4("p02")], name="part.json", root=self.part_root)
|
||||
self.out = self.tmp / "out"
|
||||
|
||||
def split(self, held: Path, seed: int = 7, out: Path | None = None) -> tuple[int, str]:
|
||||
out = out or self.out
|
||||
return run(["split", "--set", str(self.all), "--sources", str(self.root), "--seed", str(seed),
|
||||
"--held-out-count", "4", "--set-out", str(out / "set.json"), "--set-label", "set.json",
|
||||
"--held-out", str(held), "--part", str(self.part), "--part-sources", str(self.part_root),
|
||||
"--part-label", "part.json", "--manifest", str(out / "manifest.json")])
|
||||
|
||||
|
||||
class SplitTest(SplitAndGateBase):
|
||||
def test_split_is_stratified_by_group_and_renumbers_both_parts(self) -> None:
|
||||
held = self.tmp / "outside" / "held-out.json"
|
||||
code, out = self.split(held)
|
||||
self.assertEqual(code, 0, out)
|
||||
dev = json.loads((self.out / "set.json").read_text(encoding="utf-8"))
|
||||
kept = json.loads(held.read_text(encoding="utf-8"))
|
||||
self.assertEqual([q["id"] for q in dev["questions"]], ["d01", "d02", "d03", "d04"])
|
||||
self.assertEqual([q["id"] for q in kept["questions"]], ["h01", "h02", "h03", "h04"])
|
||||
groups = sorted(q["group"] for q in kept["questions"])
|
||||
self.assertEqual(groups, ["docs-2-3", "docs-2-3", "docs-4", "history"])
|
||||
self.assertEqual(dev["sources"], kept["sources"])
|
||||
self.assertEqual(dev["schema"], "okf-wiki/answer-set/1")
|
||||
|
||||
def test_the_manifest_holds_seed_counts_and_sha256_only(self) -> None:
|
||||
held = self.tmp / "outside" / "held-out.json"
|
||||
self.assertEqual(self.split(held)[0], 0)
|
||||
manifest = json.loads((self.out / "manifest.json").read_text(encoding="utf-8"))
|
||||
self.assertEqual(manifest["schema"], "okf-wiki/answer-set-manifest/1")
|
||||
self.assertEqual(manifest["seed"], 7)
|
||||
self.assertEqual(manifest["set"]["sha256"], sha((self.out / "set.json").read_bytes()))
|
||||
self.assertEqual(manifest["set"]["file"], "set.json")
|
||||
self.assertEqual(manifest["held_out"]["sha256"], sha(held.read_bytes()))
|
||||
self.assertEqual(manifest["part"]["sha256"], sha(self.part.read_bytes()))
|
||||
self.assertEqual(manifest["part"]["file"], "part.json")
|
||||
self.assertEqual(manifest["groups"]["held_out"], {"docs-2-3": 2, "docs-4": 1, "history": 1})
|
||||
self.assertEqual(manifest["groups"]["set"], {"docs-2-3": 2, "docs-4": 1, "history": 1})
|
||||
self.assertEqual(manifest["groups"]["part"], {"docs-2-3": 1, "docs-4": 1, "history": 0})
|
||||
self.assertEqual(manifest["sources"]["copy"], root_ref(self.root))
|
||||
self.assertEqual(manifest["sources"]["part"], root_ref(self.part_root))
|
||||
text = (self.out / "manifest.json").read_text(encoding="utf-8")
|
||||
self.assertNotIn("bread", text)
|
||||
self.assertNotIn(SPAN[RISING], text)
|
||||
|
||||
def test_two_runs_with_one_seed_are_byte_identical(self) -> None:
|
||||
first, second = self.tmp / "a" / "held.json", self.tmp / "b" / "held.json"
|
||||
self.assertEqual(self.split(first, out=self.tmp / "o1")[0], 0)
|
||||
self.assertEqual(self.split(second, out=self.tmp / "o2")[0], 0)
|
||||
self.assertEqual(first.read_bytes(), second.read_bytes())
|
||||
self.assertEqual((self.tmp / "o1" / "set.json").read_bytes(), (self.tmp / "o2" / "set.json").read_bytes())
|
||||
|
||||
def test_split_refuses_to_overwrite_the_held_out_file(self) -> None:
|
||||
held = self.tmp / "outside" / "held-out.json"
|
||||
self.assertEqual(self.split(held)[0], 0)
|
||||
before = held.read_bytes()
|
||||
code, out = self.split(held, seed=8, out=self.tmp / "again")
|
||||
self.assertEqual(code, 1)
|
||||
self.assertIn("exists", out)
|
||||
self.assertEqual(held.read_bytes(), before)
|
||||
|
||||
def test_split_refuses_a_held_out_path_in_the_repository(self) -> None:
|
||||
held = REPO_ROOT / "gates-local" / "answerset" / "test-held-out-never-written.json"
|
||||
code, out = self.split(held)
|
||||
self.assertEqual(code, 1)
|
||||
self.assertIn("inside the repository", out)
|
||||
self.assertFalse(held.exists())
|
||||
|
||||
def test_split_refuses_a_set_that_fails_the_check(self) -> None:
|
||||
data = json.loads(self.all.read_text(encoding="utf-8"))
|
||||
data["questions"][0]["evidence"][0]["span"] = "No such sentence stands in any synthetic file."
|
||||
self.all.write_bytes(dump(data))
|
||||
held = self.tmp / "outside" / "held-out.json"
|
||||
self.assertEqual(self.split(held)[0], 1)
|
||||
self.assertFalse(held.exists())
|
||||
|
||||
def test_split_refuses_a_part_with_a_span_outside_its_root(self) -> None:
|
||||
self.part.write_bytes(dump(make_set(self.part_root, [q23("p01"), qhist("p02")])))
|
||||
held = self.tmp / "outside" / "held-out.json"
|
||||
self.assertEqual(self.split(held)[0], 1)
|
||||
self.assertFalse(held.exists())
|
||||
|
||||
|
||||
def fasit(q: dict[str, Any]) -> dict[str, Any]:
|
||||
text = " ".join(("not " if k.get("negated") else "") + k["text"] + "." for k in q["answer_keys"])
|
||||
return {"id": q["id"], "answer": text, "citations": [{"path": e["path"], "quote": e["span"]} for e in q["evidence"]]}
|
||||
|
||||
|
||||
class GateBase(SplitAndGateBase):
|
||||
def setUp(self) -> None:
|
||||
super().setUp()
|
||||
self.held = self.tmp / "outside" / "held-out.json"
|
||||
code, out = self.split(self.held)
|
||||
self.assertEqual(code, 0, out)
|
||||
self.set = self.out / "set.json"
|
||||
self.manifest = self.out / "manifest.json"
|
||||
self.questions: list[dict[str, Any]] = json.loads(self.set.read_text(encoding="utf-8"))["questions"]
|
||||
|
||||
def answers(self, answers: list[dict[str, Any]], arm: str = "bundle", name: str = "answers.json") -> Path:
|
||||
path = self.tmp / name
|
||||
path.write_bytes(dump({"schema": "okf-wiki/answers/1", "arm": arm, "answers": answers}))
|
||||
return path
|
||||
|
||||
def gate(self, answers: Path, arm: str = "bundle", set_path: Path | None = None, root: Path | None = None,
|
||||
out: Path | None = None) -> tuple[int, str]:
|
||||
argv = ["gate", "--set", str(set_path or self.set), "--manifest", str(self.manifest),
|
||||
"--sources", str(root or self.root), "--arm", arm, "--answers", str(answers)]
|
||||
return run(argv + (["--out", str(out)] if out else []))
|
||||
|
||||
|
||||
class GateTest(GateBase):
|
||||
def test_a_missing_answers_file_scores_0_red_exit_1(self) -> None:
|
||||
code, out = self.gate(self.tmp / "none.json")
|
||||
self.assertEqual(code, 1)
|
||||
self.assertIn("correct answer and source: 0 of 4", out)
|
||||
self.assertIn("fail-closed", out)
|
||||
self.assertIn("RED", out)
|
||||
|
||||
def test_the_fasit_scores_n_of_n_green(self) -> None:
|
||||
code, out = self.gate(self.answers([fasit(q) for q in self.questions]))
|
||||
self.assertEqual(code, 0, out)
|
||||
self.assertIn("correct answer and source: 4 of 4", out)
|
||||
self.assertIn("GREEN", out)
|
||||
|
||||
def test_right_answers_without_citations_score_0(self) -> None:
|
||||
answers = [dict(fasit(q), citations=[]) for q in self.questions]
|
||||
code, out = self.gate(self.answers(answers))
|
||||
self.assertEqual(code, 0, out)
|
||||
self.assertIn("correct answer: 4 of 4", out)
|
||||
self.assertIn("correct answer and source: 0 of 4", out)
|
||||
self.assertIn("RED", out)
|
||||
|
||||
def test_a_citation_of_only_some_needed_paths_is_wrong(self) -> None:
|
||||
answers = [fasit(q) for q in self.questions]
|
||||
answers[0]["citations"] = answers[0]["citations"][:1]
|
||||
code, out = self.gate(self.answers(answers))
|
||||
self.assertIn("correct answer and source: 3 of 4", out)
|
||||
|
||||
def test_a_quote_counts_with_whitespace_collapsed_and_a_leading_slash(self) -> None:
|
||||
answers = [fasit(q) for q in self.questions]
|
||||
for c in answers[0]["citations"]:
|
||||
c["path"] = "/" + c["path"]
|
||||
c["quote"] = " " + c["quote"].replace(" ", "\n ") + " "
|
||||
self.assertIn("correct answer and source: 4 of 4", self.gate(self.answers(answers))[1])
|
||||
|
||||
def test_a_quote_that_does_not_stand_is_wrong(self) -> None:
|
||||
answers = [fasit(q) for q in self.questions]
|
||||
answers[0]["citations"][0]["quote"] = "The dough rises for 80 minutes."
|
||||
self.assertIn("correct answer and source: 3 of 4", self.gate(self.answers(answers))[1])
|
||||
|
||||
def test_a_missing_answer_counts_as_wrong(self) -> None:
|
||||
code, out = self.gate(self.answers([fasit(q) for q in self.questions[1:]]))
|
||||
self.assertEqual(code, 1)
|
||||
self.assertIn("missing answers: 1", out)
|
||||
self.assertIn("correct answer and source: 3 of 4", out)
|
||||
|
||||
def test_answers_for_another_arm_cannot_run(self) -> None:
|
||||
path = self.answers([fasit(q) for q in self.questions], arm="filsok")
|
||||
self.assertEqual(self.gate(path, arm="bundle")[0], 1)
|
||||
|
||||
def test_an_unknown_arm_cannot_run(self) -> None:
|
||||
path = self.answers([fasit(q) for q in self.questions], arm="server")
|
||||
self.assertEqual(self.gate(path, arm="server")[0], 1)
|
||||
|
||||
def test_a_set_whose_sha256_is_not_in_the_manifest_cannot_run(self) -> None:
|
||||
changed = self.tmp / "changed.json"
|
||||
changed.write_bytes(self.set.read_bytes() + b"\n")
|
||||
code, out = self.gate(self.answers([fasit(q) for q in self.questions]), set_path=changed)
|
||||
self.assertEqual(code, 1)
|
||||
self.assertIn("sha256", out)
|
||||
|
||||
def test_a_root_whose_manifest_differs_cannot_run(self) -> None:
|
||||
other = write_root(self.tmp / "other", frozen_at="2026-10-09T12:00:00Z")
|
||||
code, out = self.gate(self.answers([fasit(q) for q in self.questions]), root=other)
|
||||
self.assertEqual(code, 1)
|
||||
self.assertIn("sha256", out)
|
||||
|
||||
def test_the_part_is_scored_against_the_part_root_only(self) -> None:
|
||||
part_q = json.loads(self.part.read_text(encoding="utf-8"))["questions"]
|
||||
answers = self.answers([fasit(q) for q in part_q])
|
||||
code, out = self.gate(answers, set_path=self.part, root=self.part_root)
|
||||
self.assertEqual(code, 0, out)
|
||||
self.assertIn("set: part, 2 questions", out)
|
||||
self.assertEqual(self.gate(answers, set_path=self.part, root=self.root)[0], 1)
|
||||
|
||||
def test_a_negated_key_needs_the_negation_in_the_answer(self) -> None:
|
||||
data = json.loads(self.all.read_text(encoding="utf-8"))
|
||||
for q in data["questions"]:
|
||||
if q["group"] == "docs-2-3":
|
||||
q["evidence"][1]["span"] = "The crust does not form without steam."
|
||||
q["answer_keys"][1] = {"path": OVEN, "text": "form", "negated": True}
|
||||
self.all.write_bytes(dump(data))
|
||||
held = self.tmp / "outside2" / "held.json"
|
||||
out_dir = self.tmp / "neg"
|
||||
self.assertEqual(self.split(held, out=out_dir)[0], 0)
|
||||
self.manifest = out_dir / "manifest.json"
|
||||
questions = json.loads((out_dir / "set.json").read_text(encoding="utf-8"))["questions"]
|
||||
answers = [fasit(x) for x in questions]
|
||||
good = self.gate(self.answers(answers), set_path=out_dir / "set.json")[1]
|
||||
self.assertIn("correct answer and source: 4 of 4", good)
|
||||
for a, x in zip(answers, questions):
|
||||
a["answer"] = " ".join(k["text"] + "." for k in x["answer_keys"])
|
||||
plain = self.gate(self.answers(answers), set_path=out_dir / "set.json")[1]
|
||||
self.assertEqual(sum(1 for x in questions if x["group"] == "docs-2-3"), 2)
|
||||
self.assertIn("correct answer and source: 2 of 4", plain)
|
||||
|
||||
def test_out_writes_the_k5_results_sorted_and_byte_identical(self) -> None:
|
||||
answers = [fasit(q) for q in self.questions]
|
||||
answers[1]["citations"] = answers[1]["citations"][:1]
|
||||
answers[2]["answer"] = "Nothing useful."
|
||||
first, second = self.tmp / "r1.json", self.tmp / "r2.json"
|
||||
self.gate(self.answers(list(reversed(answers))), out=first)
|
||||
self.gate(self.answers(answers), out=second)
|
||||
self.assertEqual(first.read_bytes(), second.read_bytes())
|
||||
doc = json.loads(first.read_text(encoding="utf-8"))
|
||||
self.assertEqual(doc["schema"], "okf-wiki/answer-results/1")
|
||||
self.assertEqual(doc["arm"], "bundle")
|
||||
self.assertEqual(doc["set_sha256"], sha(self.set.read_bytes()))
|
||||
ids = [r["id"] for r in doc["results"]]
|
||||
self.assertEqual(ids, sorted(ids))
|
||||
by_id = {r["id"]: r for r in doc["results"]}
|
||||
q1, q2 = self.questions[1], self.questions[2]
|
||||
r1, r2 = by_id[q1["id"]], by_id[q2["id"]]
|
||||
self.assertEqual(set(r1), {"id", "group", "correct", "keys_held", "keys", "paths_cited", "paths_needed"})
|
||||
self.assertEqual((r1["correct"], r1["paths_cited"]), (False, 1))
|
||||
self.assertEqual(r1["paths_needed"], len({e["path"] for e in q1["evidence"]}))
|
||||
self.assertEqual((r2["correct"], r2["keys_held"], r2["keys"]), (False, 0, len(q2["answer_keys"])))
|
||||
self.assertEqual(r1["group"], q1["group"])
|
||||
self.assertEqual(first.read_bytes(), dump(doc))
|
||||
|
||||
|
||||
class StubTest(GateBase):
|
||||
def test_stub_fasit_scores_n_of_n_and_stub_without_citations_0(self) -> None:
|
||||
fas, bare = self.tmp / "fasit.json", self.tmp / "bare.json"
|
||||
self.assertEqual(run(["stub", "--set", str(self.set), "--arm", "bundle", "--kind", "fasit",
|
||||
"--out", str(fas)])[0], 0)
|
||||
self.assertEqual(run(["stub", "--set", str(self.set), "--arm", "bundle", "--kind", "no-citations",
|
||||
"--out", str(bare)])[0], 0)
|
||||
self.assertIn("correct answer and source: 4 of 4", self.gate(fas)[1])
|
||||
code, out = self.gate(bare)
|
||||
self.assertEqual(code, 0)
|
||||
self.assertIn("correct answer: 4 of 4", out)
|
||||
self.assertIn("correct answer and source: 0 of 4", out)
|
||||
|
||||
def test_stub_answers_are_k5_answers(self) -> None:
|
||||
fas = self.tmp / "fasit.json"
|
||||
run(["stub", "--set", str(self.set), "--arm", "filsok", "--kind", "fasit", "--out", str(fas)])
|
||||
doc = json.loads(fas.read_text(encoding="utf-8"))
|
||||
self.assertEqual((doc["schema"], doc["arm"]), ("okf-wiki/answers/1", "filsok"))
|
||||
self.assertEqual(set(doc["answers"][0]["citations"][0]), {"path", "quote"})
|
||||
|
||||
|
||||
HARD_PATHS = (RISING, OVEN, SOIL, WATER, FROST)
|
||||
|
||||
|
||||
def q3(qid: str) -> dict[str, Any]:
|
||||
return question(qid, (RISING, OVEN, FROST), "docs-2-3")
|
||||
|
||||
|
||||
def hard_questions(three: int, four: int, two: int = 0) -> list[dict[str, Any]]:
|
||||
"""`three` questions with 3 paths, `four` with 4 and `two` with 2, each with its own wording, ids k01.."""
|
||||
questions = [q3("") for _ in range(three)] + [q4("") for _ in range(four)] + [q23("") for _ in range(two)]
|
||||
for n, q in enumerate(questions, 1):
|
||||
q["id"] = f"k{n:02d}"
|
||||
q["question"] = f"Synthetic hard question number {n} about bread and gardens?"
|
||||
return questions
|
||||
|
||||
|
||||
class HardSetBase(Base):
|
||||
"""The harder set (order 21): every question needs at least 3 documents and at least 20 of 30 need 4 or more.
|
||||
Its root is a K1 subset root of the copy, as the first part is."""
|
||||
|
||||
def setUp(self) -> None:
|
||||
super().setUp()
|
||||
self.hard_root = write_root(self.tmp / "hard-root", HARD_PATHS, frozen_at="2026-10-08T19:00:00Z",
|
||||
subset_of=root_ref(self.root))
|
||||
self.out = self.tmp / "frozen"
|
||||
|
||||
def hard_set(self, questions: list[dict[str, Any]], name: str = "drafts.json") -> Path:
|
||||
return self.write_set(questions, name=name, root=self.hard_root)
|
||||
|
||||
def check_hard(self, path: Path, min_paths: int = 3, min_many: int = 20) -> tuple[int, str]:
|
||||
return run(["check", "--set", str(path), "--sources", str(self.hard_root), "--min-paths", str(min_paths),
|
||||
"--min-many", str(min_many)])
|
||||
|
||||
def freeze(self, path: Path, seed: int = 11, out: Path | None = None, count: int = 30,
|
||||
min_many: int = 20) -> tuple[int, str]:
|
||||
out = out or self.out
|
||||
return run(["freeze", "--set", str(path), "--sources", str(self.hard_root), "--seed", str(seed),
|
||||
"--count", str(count), "--min-paths", "3", "--min-many", str(min_many), "--prefix", "x",
|
||||
"--set-out", str(out / "hard.json"), "--set-label", "hard.json",
|
||||
"--manifest", str(out / "hard.manifest.json")])
|
||||
|
||||
|
||||
class HardCheckTest(HardSetBase):
|
||||
def test_ten_with_three_and_twenty_with_four_pass_the_thresholds(self) -> None:
|
||||
code, out = self.check_hard(self.hard_set(hard_questions(10, 20)))
|
||||
self.assertEqual(code, 0, out)
|
||||
self.assertIn("questions with >= 3 distinct paths: 30 of 30", out)
|
||||
self.assertIn("questions with >= 4 distinct paths: 20 of 30 (required >= 20)", out)
|
||||
self.assertIn("groups: docs-2-3 10, docs-4 20, history 0", out)
|
||||
self.assertIn("PASS", out)
|
||||
|
||||
def test_a_question_with_two_paths_fails_min_paths_3(self) -> None:
|
||||
path = self.hard_set(hard_questions(9, 20, two=1))
|
||||
code, out = self.check_hard(path)
|
||||
self.assertEqual(code, 1)
|
||||
self.assertIn("questions with >= 3 distinct paths: 29 of 30", out)
|
||||
self.assertIn("FAIL", out)
|
||||
code, out = run(["check", "--set", str(path), "--sources", str(self.hard_root)])
|
||||
self.assertEqual(code, 0, out)
|
||||
self.assertIn("questions with >= 2 distinct paths: 30 of 30", out)
|
||||
|
||||
def test_nineteen_of_thirty_with_four_fail_min_many_20(self) -> None:
|
||||
code, out = self.check_hard(self.hard_set(hard_questions(11, 19)))
|
||||
self.assertEqual(code, 1)
|
||||
self.assertIn("questions with >= 4 distinct paths: 19 of 30 (required >= 20)", out)
|
||||
self.assertIn("FAIL", out)
|
||||
|
||||
def test_the_default_check_prints_no_many_line(self) -> None:
|
||||
code, out = run(["check", "--set", str(self.hard_set(hard_questions(10, 20))), "--sources",
|
||||
str(self.hard_root)])
|
||||
self.assertEqual(code, 0, out)
|
||||
self.assertNotIn("required >=", out)
|
||||
|
||||
|
||||
class FreezeTest(HardSetBase):
|
||||
def test_freeze_writes_the_set_and_a_manifest_with_seed_counts_sha256_and_roots_only(self) -> None:
|
||||
code, out = self.freeze(self.hard_set(hard_questions(10, 20)))
|
||||
self.assertEqual(code, 0, out)
|
||||
frozen = (self.out / "hard.json").read_bytes()
|
||||
manifest = json.loads((self.out / "hard.manifest.json").read_text(encoding="utf-8"))
|
||||
self.assertEqual(manifest["schema"], "okf-wiki/answer-set-manifest/1")
|
||||
self.assertEqual(manifest["seed"], 11)
|
||||
self.assertEqual(manifest["rules"], {"min_paths": 3, "many_paths": 4, "min_many": 20})
|
||||
self.assertEqual(manifest["part"], {"file": "hard.json", "sha256": sha(frozen), "questions": 30})
|
||||
self.assertEqual(manifest["groups"], {"part": {"docs-2-3": 10, "docs-4": 20, "history": 0}})
|
||||
self.assertEqual(manifest["sources"], {"part": root_ref(self.hard_root), "copy": root_ref(self.root)})
|
||||
self.assertNotIn("set", manifest)
|
||||
self.assertNotIn("held_out", manifest)
|
||||
text = (self.out / "hard.manifest.json").read_text(encoding="utf-8")
|
||||
self.assertNotIn("bread", text)
|
||||
self.assertNotIn(SPAN[RISING], text)
|
||||
data = json.loads(frozen)
|
||||
self.assertEqual(data["sources"], root_ref(self.hard_root))
|
||||
self.assertEqual(data["schema"], "okf-wiki/answer-set/1")
|
||||
|
||||
def test_freeze_renumbers_in_a_seeded_order_and_is_byte_identical(self) -> None:
|
||||
drafts = self.hard_set(hard_questions(10, 20))
|
||||
self.assertEqual(self.freeze(drafts, out=self.tmp / "o1")[0], 0)
|
||||
self.assertEqual(self.freeze(drafts, out=self.tmp / "o2")[0], 0)
|
||||
first = (self.tmp / "o1" / "hard.json").read_bytes()
|
||||
self.assertEqual(first, (self.tmp / "o2" / "hard.json").read_bytes())
|
||||
questions = json.loads(first)["questions"]
|
||||
self.assertEqual([q["id"] for q in questions], [f"x{n:02d}" for n in range(1, 31)])
|
||||
order = sorted(f"k{n:02d}" for n in range(1, 31))
|
||||
random.Random(11).shuffle(order)
|
||||
wording = {f"k{n:02d}": f"Synthetic hard question number {n} about bread and gardens?" for n in range(1, 31)}
|
||||
self.assertEqual([q["question"] for q in questions], [wording[i] for i in order])
|
||||
self.assertEqual(self.freeze(drafts, seed=12, out=self.tmp / "o3")[0], 0)
|
||||
other = json.loads((self.tmp / "o3" / "hard.json").read_text(encoding="utf-8"))["questions"]
|
||||
self.assertNotEqual([q["question"] for q in other], [q["question"] for q in questions])
|
||||
self.assertEqual(sorted(q["question"] for q in other), sorted(q["question"] for q in questions))
|
||||
|
||||
def test_freeze_refuses_a_set_that_fails_the_thresholds(self) -> None:
|
||||
code, out = self.freeze(self.hard_set(hard_questions(11, 19)))
|
||||
self.assertEqual(code, 1)
|
||||
self.assertIn("19 of 30", out)
|
||||
self.assertFalse((self.out / "hard.json").exists())
|
||||
self.assertFalse((self.out / "hard.manifest.json").exists())
|
||||
|
||||
def test_freeze_refuses_a_set_with_another_count(self) -> None:
|
||||
code, out = self.freeze(self.hard_set(hard_questions(9, 20)))
|
||||
self.assertEqual(code, 1)
|
||||
self.assertIn("29", out)
|
||||
self.assertFalse((self.out / "hard.json").exists())
|
||||
|
||||
def test_freeze_refuses_a_set_that_fails_the_check(self) -> None:
|
||||
questions = hard_questions(10, 20)
|
||||
questions[0]["evidence"][0]["span"] = "No such sentence stands in any synthetic file."
|
||||
self.assertEqual(self.freeze(self.hard_set(questions))[0], 1)
|
||||
self.assertFalse((self.out / "hard.manifest.json").exists())
|
||||
|
||||
def test_freeze_never_overwrites_a_frozen_set_or_manifest(self) -> None:
|
||||
drafts = self.hard_set(hard_questions(10, 20))
|
||||
self.assertEqual(self.freeze(drafts)[0], 0)
|
||||
before = ((self.out / "hard.json").read_bytes(), (self.out / "hard.manifest.json").read_bytes())
|
||||
code, out = self.freeze(drafts, seed=12)
|
||||
self.assertEqual(code, 1)
|
||||
self.assertIn("exists", out)
|
||||
self.assertEqual(before, ((self.out / "hard.json").read_bytes(), (self.out / "hard.manifest.json").read_bytes()))
|
||||
(self.out / "hard.manifest.json").unlink()
|
||||
code, out = self.freeze(drafts, seed=12)
|
||||
self.assertEqual(code, 1)
|
||||
self.assertIn("exists", out)
|
||||
self.assertEqual((self.out / "hard.json").read_bytes(), before[0])
|
||||
|
||||
|
||||
class HardGateTest(HardSetBase):
|
||||
def setUp(self) -> None:
|
||||
super().setUp()
|
||||
code, out = self.freeze(self.hard_set(hard_questions(10, 20)))
|
||||
self.assertEqual(code, 0, out)
|
||||
self.set = self.out / "hard.json"
|
||||
self.manifest = self.out / "hard.manifest.json"
|
||||
|
||||
def gate(self, answers: Path, root: Path | None = None) -> tuple[int, str]:
|
||||
return run(["gate", "--set", str(self.set), "--manifest", str(self.manifest), "--sources",
|
||||
str(root or self.hard_root), "--arm", "filsok", "--answers", str(answers)])
|
||||
|
||||
def stub(self, kind: str) -> Path:
|
||||
path = self.tmp / f"{kind}.json"
|
||||
self.assertEqual(run(["stub", "--set", str(self.set), "--arm", "filsok", "--kind", kind,
|
||||
"--out", str(path)])[0], 0)
|
||||
return path
|
||||
|
||||
def test_the_fasit_is_green_and_no_citations_red_on_the_frozen_set(self) -> None:
|
||||
code, out = self.gate(self.stub("fasit"))
|
||||
self.assertEqual(code, 0, out)
|
||||
self.assertIn("set: part, 30 questions", out)
|
||||
self.assertIn("correct answer and source: 30 of 30", out)
|
||||
self.assertIn("GREEN", out)
|
||||
code, out = self.gate(self.stub("no-citations"))
|
||||
self.assertEqual(code, 0, out)
|
||||
self.assertIn("correct answer: 30 of 30", out)
|
||||
self.assertIn("correct answer and source: 0 of 30", out)
|
||||
self.assertIn("RED", out)
|
||||
|
||||
def test_no_answers_score_0_of_30_red_exit_1(self) -> None:
|
||||
code, out = self.gate(self.tmp / "none.json")
|
||||
self.assertEqual(code, 1)
|
||||
self.assertIn("correct answer and source: 0 of 30", out)
|
||||
self.assertIn("RED", out)
|
||||
|
||||
def test_the_frozen_set_is_judged_against_its_own_root_only(self) -> None:
|
||||
code, out = self.gate(self.stub("fasit"), root=self.root)
|
||||
self.assertEqual(code, 1)
|
||||
self.assertIn("sha256", out)
|
||||
|
||||
|
||||
class DumpTest(unittest.TestCase):
|
||||
def test_dump_is_the_k5_form(self) -> None:
|
||||
data = {"b": [1, {"z": "æ", "a": None}], "a": True}
|
||||
self.assertEqual(answerset.dump(data), dump(data))
|
||||
|
||||
def test_group_of_follows_k1_and_the_path_count(self) -> None:
|
||||
main = {RISING: True, OVEN: True, SOIL: True, WATER: True, RELEASE: False}
|
||||
self.assertEqual(answerset.group_of([RISING, OVEN], main), "docs-2-3")
|
||||
self.assertEqual(answerset.group_of([RISING, OVEN, SOIL], main), "docs-2-3")
|
||||
self.assertEqual(answerset.group_of([RISING, OVEN, SOIL, WATER], main), "docs-4")
|
||||
self.assertEqual(answerset.group_of([RISING, RELEASE], main), "history")
|
||||
self.assertEqual(answerset.group_of([RISING, OVEN, SOIL, RELEASE], main), "history")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
938
gates/tests/test_calibration.py
Normal file
938
gates/tests/test_calibration.py
Normal file
|
|
@ -0,0 +1,938 @@
|
|||
"""Tests for gates/calibration.py: the reader-calibration set checker, the seeded build with its call layout (K3), the
|
||||
stubs and the verdict gate.
|
||||
|
||||
Every fixture is synthetic text written here (widgets and garden beds). No corpus text is used (CLAUDE.md rule 3).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
|
||||
import calibration # noqa: E402
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
|
||||
SOURCES = {
|
||||
"alpha.md": "# Alpha\n\nThe widget timeout is 45 seconds by default.\nRaising it above 90 seconds disables retries.\n",
|
||||
"beta.md": "# Beta\n\nBeta adds the --frobnicate flag for batch runs.\nThe flag is ignored on interactive runs.\n",
|
||||
"gamma.md": "# Gamma\n\nGamma stores its cache under the .gamma-cache directory.\nThe cache is cleared on every upgrade.\n",
|
||||
"north.md": "# North beds\n\n" + "".join(f"Bed N{i} gets {i + 2} litres of water every morning.\n" for i in range(1, 31)),
|
||||
"south.md": "# South beds\n\n" + "".join(f"Bed S{i} is weeded every {i + 3} days in summer.\n" for i in range(1, 31)),
|
||||
}
|
||||
|
||||
ALPHA = "The widget timeout is 45 seconds by default."
|
||||
ALPHA2 = "Raising it above 90 seconds disables retries."
|
||||
BETA = "Beta adds the --frobnicate flag for batch runs."
|
||||
BETA2 = "The flag is ignored on interactive runs."
|
||||
GAMMA = "Gamma stores its cache under the .gamma-cache directory."
|
||||
GAMMA2 = "The cache is cleared on every upgrade."
|
||||
|
||||
UNSUPPORTED, SUPPORTED, FILLER = "unsupported", "supported", "filler-supported"
|
||||
CALL_ID = re.compile(r"\bk\d{3,}\b")
|
||||
CLAIM_ID = re.compile(r"\bc\d{3,}\b")
|
||||
|
||||
|
||||
def ex(name: str, text: str) -> dict[str, Any]:
|
||||
return {"file": name, "text": text}
|
||||
|
||||
|
||||
def draft_pairs() -> list[dict[str, Any]]:
|
||||
"""Six unsupported pairs (one per kind) and six supported pairs (two per kind): a valid small draft."""
|
||||
u = [
|
||||
("overgeneralisation", "Every Alpha setting defaults to 45 seconds.", [ex("alpha.md", ALPHA)]),
|
||||
("wrong-number", "The widget timeout is 30 seconds by default.", [ex("alpha.md", ALPHA)]),
|
||||
("swapped-cause", "Retries being disabled is what raises the timeout above 90 seconds.", [ex("alpha.md", ALPHA2)]),
|
||||
("not-mentioned", "Beta writes a summary file after each batch run.", [ex("beta.md", BETA)]),
|
||||
("reversed-polarity", "The --frobnicate flag also applies to interactive runs.", [ex("beta.md", BETA2)]),
|
||||
(
|
||||
"conflated-sources",
|
||||
"Beta stores its cache under the .gamma-cache directory.",
|
||||
[ex("beta.md", BETA), ex("gamma.md", GAMMA)],
|
||||
),
|
||||
]
|
||||
s = [
|
||||
("paraphrase", "By default, the widget waits 45 seconds before timing out.", [ex("alpha.md", ALPHA)]),
|
||||
("paraphrase", "Upgrading clears the Gamma cache.", [ex("gamma.md", GAMMA2)]),
|
||||
("inference", "A widget left at its default timeout still retries.", [ex("alpha.md", ALPHA + "\n" + ALPHA2)]),
|
||||
("inference", "Batch runs are the ones where --frobnicate has an effect.", [ex("beta.md", BETA + "\n" + BETA2)]),
|
||||
(
|
||||
"synthesis",
|
||||
"An upgrade empties the directory Gamma uses for its cache.",
|
||||
[ex("gamma.md", GAMMA), ex("gamma.md", GAMMA2)],
|
||||
),
|
||||
(
|
||||
"synthesis",
|
||||
"Both a 45-second default timeout and a --frobnicate flag for batch runs exist.",
|
||||
[ex("alpha.md", ALPHA), ex("beta.md", BETA)],
|
||||
),
|
||||
]
|
||||
pairs: list[dict[str, Any]] = []
|
||||
for n, (kind, claim, excerpts) in enumerate(u, 1):
|
||||
pairs.append({"id": f"u{n:02d}", "label": UNSUPPORTED, "kind": kind, "claim": claim, "excerpts": excerpts, "why": "planted"})
|
||||
for n, (kind, claim, excerpts) in enumerate(s, 1):
|
||||
pairs.append({"id": f"s{n:02d}", "label": SUPPORTED, "kind": kind, "claim": claim, "excerpts": excerpts, "why": "follows"})
|
||||
return pairs
|
||||
|
||||
|
||||
def north(i: int) -> str:
|
||||
return f"Bed N{i} gets {i + 2} litres of water every morning."
|
||||
|
||||
|
||||
def south(i: int) -> str:
|
||||
return f"Bed S{i} is weeded every {i + 3} days in summer."
|
||||
|
||||
|
||||
def garden_draft(n_unsupported: int = 20, n_supported: int = 6, extra_fillers: int = 4) -> list[dict[str, Any]]:
|
||||
"""More unsupported than supported pairs, each unsupported with its supported twin (filler-supported), plus a few
|
||||
fillers without a twin. Every kind of both counted labels is present."""
|
||||
pairs: list[dict[str, Any]] = []
|
||||
for i in range(1, n_unsupported + 1):
|
||||
kind = calibration.UNSUPPORTED_KINDS[(i - 1) % len(calibration.UNSUPPORTED_KINDS)]
|
||||
excerpts = [ex("north.md", north(i))]
|
||||
claim = {
|
||||
"overgeneralisation": f"Every north bed, N{i} included, gets {i + 2} litres of water each morning.",
|
||||
"wrong-number": f"Bed N{i} gets {i + 50} litres of water each morning.",
|
||||
"swapped-cause": f"Watering bed N{i} each morning is what makes it need {i + 2} litres.",
|
||||
"not-mentioned": f"Bed N{i} gets {i + 2} litres of water each morning and is mulched in spring.",
|
||||
"reversed-polarity": f"Bed N{i} never gets {i + 2} litres of water in the morning.",
|
||||
"conflated-sources": f"Bed N{i} is weeded every {i + 3} days in summer.",
|
||||
}[kind]
|
||||
twin, twin_kind = f"Each morning, bed N{i} receives {i + 2} litres of water.", "paraphrase"
|
||||
if kind == "conflated-sources":
|
||||
excerpts = [ex("north.md", north(i)), ex("south.md", south(i))]
|
||||
twin = f"Bed N{i} gets {i + 2} litres each morning, and bed S{i} is weeded every {i + 3} days in summer."
|
||||
twin_kind = "synthesis"
|
||||
pairs.append({"id": f"u{i:02d}", "label": UNSUPPORTED, "kind": kind, "claim": claim, "excerpts": excerpts, "why": "planted"})
|
||||
pairs.append({"id": f"t{i:02d}", "label": FILLER, "kind": twin_kind, "twin_of": f"u{i:02d}", "claim": twin,
|
||||
"excerpts": [dict(e) for e in excerpts], "why": "the supported reading"})
|
||||
for n in range(1, n_supported + 1):
|
||||
j = 20 + n
|
||||
kind = calibration.SUPPORTED_KINDS[(n - 1) % len(calibration.SUPPORTED_KINDS)]
|
||||
claim, excerpts = {
|
||||
"paraphrase": (f"In summer, bed S{j} has its weeds pulled every {j + 3} days.", [ex("south.md", south(j))]),
|
||||
"inference": (f"Bed S{j} is weeded more often than once a season in summer.", [ex("south.md", south(j))]),
|
||||
"synthesis": (
|
||||
f"Bed N{j} is watered each morning and bed S{j} is weeded every {j + 3} days in summer.",
|
||||
[ex("north.md", north(j)), ex("south.md", south(j))],
|
||||
),
|
||||
}[kind]
|
||||
pairs.append({"id": f"s{n:02d}", "label": SUPPORTED, "kind": kind, "claim": claim, "excerpts": excerpts, "why": "follows"})
|
||||
for n in range(1, extra_fillers + 1):
|
||||
j = 26 + n
|
||||
pairs.append({"id": f"f{n:02d}", "label": FILLER, "kind": "paraphrase",
|
||||
"claim": f"Each morning, bed N{j} receives {j + 2} litres of water.",
|
||||
"excerpts": [ex("north.md", north(j))], "why": "follows"})
|
||||
return pairs
|
||||
|
||||
|
||||
def sha256_file(path: Path) -> str:
|
||||
return hashlib.sha256(path.read_bytes()).hexdigest()
|
||||
|
||||
|
||||
class Workspace:
|
||||
"""A temporary sources directory and helpers to write drafts, verdicts and build outputs."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._tmp = tempfile.TemporaryDirectory()
|
||||
self.root = Path(self._tmp.name)
|
||||
self.sources = self.root / "sources"
|
||||
self.sources.mkdir()
|
||||
for name, text in SOURCES.items():
|
||||
(self.sources / name).write_text(text, encoding="utf-8")
|
||||
|
||||
def close(self) -> None:
|
||||
self._tmp.cleanup()
|
||||
|
||||
def write_json(self, name: str, data: Any) -> Path:
|
||||
path = self.root / name
|
||||
path.write_text(json.dumps(data), encoding="utf-8")
|
||||
return path
|
||||
|
||||
def write_draft(self, pairs: list[dict[str, Any]], name: str = "draft.json") -> Path:
|
||||
return self.write_json(name, {"schema": calibration.DRAFT_SCHEMA, "pairs": pairs})
|
||||
|
||||
def paths(self) -> dict[str, Path]:
|
||||
return {
|
||||
"set": self.root / "out" / "pairs.json",
|
||||
"key": self.root / "out" / "key.json",
|
||||
"calls": self.root / "out" / "calls.json",
|
||||
"manifest": self.root / "calibration.manifest.json",
|
||||
}
|
||||
|
||||
def build(
|
||||
self, pairs: list[dict[str, Any]] | None = None, seed: int = 7, lone: int = 10, dense: int = 5
|
||||
) -> tuple[int, str]:
|
||||
draft = self.write_draft(garden_draft() if pairs is None else pairs)
|
||||
p = self.paths()
|
||||
return run(
|
||||
"build", "--draft", str(draft), "--sources", str(self.sources), "--seed", str(seed),
|
||||
"--lone", str(lone), "--dense", str(dense),
|
||||
"--set-out", str(p["set"]), "--key-out", str(p["key"]), "--calls-out", str(p["calls"]),
|
||||
"--manifest", str(p["manifest"]),
|
||||
)
|
||||
|
||||
def frozen_args(self) -> list[str]:
|
||||
p = self.paths()
|
||||
return ["--set", str(p["set"]), "--key", str(p["key"]), "--calls", str(p["calls"]),
|
||||
"--manifest", str(p["manifest"]), "--sources", str(self.sources)]
|
||||
|
||||
def gate(self, verdicts: Path | None) -> tuple[int, str]:
|
||||
return run("gate", *self.frozen_args(), "--verdicts", str(verdicts or self.root / "no-such-verdicts.json"))
|
||||
|
||||
def check(self) -> tuple[int, str]:
|
||||
return run("check", *self.frozen_args())
|
||||
|
||||
def load(self, which: str) -> Any:
|
||||
return json.loads(self.paths()[which].read_text(encoding="utf-8"))
|
||||
|
||||
def labels(self) -> dict[str, str]:
|
||||
return {k["id"]: k["label"] for k in self.load("key")["labels"]}
|
||||
|
||||
def positions(self) -> dict[str, int]:
|
||||
return {i: n for c in self.load("calls")["calls"] for n, i in enumerate(c["ids"], 1)}
|
||||
|
||||
def rewrite_calls(self, layout: dict[str, Any]) -> None:
|
||||
"""Write a changed layout and make the manifest's sha256 follow it, so only the layout rule under test fails."""
|
||||
p = self.paths()
|
||||
p["calls"].write_text(json.dumps(layout, indent=1, sort_keys=True) + "\n", encoding="utf-8")
|
||||
manifest = self.load("manifest")
|
||||
manifest["calls"]["sha256"] = sha256_file(p["calls"])
|
||||
p["manifest"].write_text(json.dumps(manifest, indent=1) + "\n", encoding="utf-8")
|
||||
|
||||
|
||||
def run(*argv: str) -> tuple[int, str]:
|
||||
out, err = io.StringIO(), io.StringIO()
|
||||
with contextlib.redirect_stdout(out), contextlib.redirect_stderr(err):
|
||||
try:
|
||||
code = calibration.main(list(argv))
|
||||
except SystemExit as exc: # argparse refuses an unknown option
|
||||
code = exc.code if isinstance(exc.code, int) else 2
|
||||
return code, out.getvalue() + err.getvalue()
|
||||
|
||||
|
||||
class Base(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
self.ws = Workspace()
|
||||
|
||||
def tearDown(self) -> None:
|
||||
self.ws.close()
|
||||
|
||||
|
||||
class CheckDraftTest(Base):
|
||||
def check(self, pairs: list[dict[str, Any]]) -> calibration.CheckReport:
|
||||
return calibration.check_pairs(pairs, self.ws.sources)
|
||||
|
||||
def test_valid_draft_passes_and_counts_kinds(self) -> None:
|
||||
rep = self.check(draft_pairs())
|
||||
self.assertTrue(rep.passed, rep.lines())
|
||||
self.assertEqual(rep.unsupported, 6)
|
||||
self.assertEqual(rep.supported, 6)
|
||||
self.assertEqual(rep.excerpts_verbatim, rep.excerpts)
|
||||
self.assertEqual(rep.by_kind["wrong-number"], 1)
|
||||
self.assertEqual(rep.by_kind["synthesis"], 2)
|
||||
|
||||
def test_excerpt_not_verbatim_fails(self) -> None:
|
||||
pairs = draft_pairs()
|
||||
pairs[0]["excerpts"] = [ex("alpha.md", "The widget timeout is 46 seconds by default.")]
|
||||
rep = self.check(pairs)
|
||||
self.assertFalse(rep.passed)
|
||||
self.assertEqual(rep.excerpts_verbatim, rep.excerpts - 1)
|
||||
|
||||
def test_missing_source_file_fails(self) -> None:
|
||||
pairs = draft_pairs()
|
||||
pairs[0]["excerpts"] = [ex("nowhere.md", ALPHA)]
|
||||
self.assertFalse(self.check(pairs).passed)
|
||||
|
||||
def test_path_outside_sources_fails(self) -> None:
|
||||
pairs = draft_pairs()
|
||||
pairs[0]["excerpts"] = [ex("../alpha.md", ALPHA)]
|
||||
self.assertFalse(self.check(pairs).passed)
|
||||
|
||||
def test_unbalanced_labels_pass_the_draft_check(self) -> None:
|
||||
# The counts are the manifest's business now (order 3, point 1), not a balance rule.
|
||||
rep = self.check(garden_draft())
|
||||
self.assertTrue(rep.passed, rep.lines())
|
||||
self.assertEqual((rep.unsupported, rep.supported, rep.filler), (20, 6, 24))
|
||||
|
||||
def test_every_unsupported_kind_must_be_present(self) -> None:
|
||||
pairs = draft_pairs()
|
||||
pairs[1]["kind"] = "overgeneralisation"
|
||||
pairs[1]["claim"] = "Every widget setting is 45 seconds long."
|
||||
rep = self.check(pairs)
|
||||
self.assertFalse(rep.passed)
|
||||
self.assertTrue(any("wrong-number" in e for e in rep.errors), rep.errors)
|
||||
|
||||
def test_every_supported_kind_must_be_present(self) -> None:
|
||||
pairs = draft_pairs()
|
||||
for p in pairs:
|
||||
if p["kind"] == "inference":
|
||||
p["kind"] = "paraphrase"
|
||||
rep = self.check(pairs)
|
||||
self.assertFalse(rep.passed)
|
||||
self.assertTrue(any("inference" in e for e in rep.errors), rep.errors)
|
||||
|
||||
def test_unknown_kind_or_label_fails(self) -> None:
|
||||
pairs = draft_pairs()
|
||||
pairs[0]["kind"] = "vibes"
|
||||
self.assertFalse(self.check(pairs).passed)
|
||||
pairs = draft_pairs()
|
||||
pairs[0]["label"] = "maybe"
|
||||
self.assertFalse(self.check(pairs).passed)
|
||||
|
||||
def test_kind_must_match_label(self) -> None:
|
||||
pairs = draft_pairs()
|
||||
pairs[0]["kind"] = "paraphrase"
|
||||
self.assertFalse(self.check(pairs).passed)
|
||||
|
||||
def test_filler_takes_a_supported_kind(self) -> None:
|
||||
pairs = garden_draft()
|
||||
twin = next(p for p in pairs if p["label"] == FILLER)
|
||||
twin["kind"] = "wrong-number"
|
||||
self.assertFalse(self.check(pairs).passed)
|
||||
|
||||
def test_filler_may_not_be_a_verbatim_copy(self) -> None:
|
||||
pairs = garden_draft()
|
||||
twin = next(p for p in pairs if p["label"] == FILLER and p["kind"] == "paraphrase")
|
||||
twin["claim"] = twin["excerpts"][0]["text"]
|
||||
rep = self.check(pairs)
|
||||
self.assertFalse(rep.passed)
|
||||
self.assertTrue(any(twin["id"] in e and "verbatim" in e for e in rep.errors), rep.errors)
|
||||
|
||||
def test_twin_of_must_name_an_unsupported_pair(self) -> None:
|
||||
for target in ("s01", "nowhere"):
|
||||
pairs = garden_draft()
|
||||
next(p for p in pairs if p["label"] == FILLER)["twin_of"] = target
|
||||
rep = self.check(pairs)
|
||||
self.assertFalse(rep.passed, target)
|
||||
self.assertTrue(any("twin_of" in e for e in rep.errors), rep.errors)
|
||||
|
||||
def test_wrong_number_claim_needs_a_number_absent_from_the_excerpts(self) -> None:
|
||||
pairs = draft_pairs()
|
||||
pairs[1]["claim"] = "The widget timeout is 45 seconds, which is short."
|
||||
rep = self.check(pairs)
|
||||
self.assertFalse(rep.passed)
|
||||
self.assertTrue(any("u02" in e and "number" in e for e in rep.errors), rep.errors)
|
||||
|
||||
def test_conflated_sources_needs_two_files(self) -> None:
|
||||
pairs = draft_pairs()
|
||||
pairs[5]["excerpts"] = [ex("beta.md", BETA), ex("beta.md", BETA2)]
|
||||
self.assertFalse(self.check(pairs).passed)
|
||||
|
||||
def test_synthesis_needs_two_excerpts(self) -> None:
|
||||
pairs = draft_pairs()
|
||||
pairs[10]["excerpts"] = [ex("gamma.md", GAMMA)]
|
||||
self.assertFalse(self.check(pairs).passed)
|
||||
|
||||
def test_supported_claim_may_not_be_a_verbatim_copy(self) -> None:
|
||||
pairs = draft_pairs()
|
||||
pairs[6]["claim"] = "the widget timeout is 45 seconds by default"
|
||||
rep = self.check(pairs)
|
||||
self.assertFalse(rep.passed)
|
||||
self.assertTrue(any("s01" in e and "verbatim" in e for e in rep.errors), rep.errors)
|
||||
|
||||
def test_duplicate_ids_fail(self) -> None:
|
||||
pairs = draft_pairs()
|
||||
pairs[1]["id"] = "u01"
|
||||
self.assertFalse(self.check(pairs).passed)
|
||||
|
||||
def test_too_many_excerpts_fail(self) -> None:
|
||||
pairs = draft_pairs()
|
||||
pairs[0]["excerpts"] = [ex("alpha.md", ALPHA), ex("beta.md", BETA), ex("gamma.md", GAMMA)]
|
||||
self.assertFalse(self.check(pairs).passed)
|
||||
|
||||
def test_empty_draft_fails(self) -> None:
|
||||
self.assertFalse(self.check([]).passed)
|
||||
|
||||
def test_check_command_on_draft(self) -> None:
|
||||
draft = self.ws.write_draft(draft_pairs())
|
||||
code, out = run("check", "--draft", str(draft), "--sources", str(self.ws.sources))
|
||||
self.assertEqual(code, 0, out)
|
||||
self.assertIn("excerpts verbatim: 15 of 15", out)
|
||||
self.assertIn("PASS", out)
|
||||
|
||||
def test_check_command_missing_sources_is_a_fail(self) -> None:
|
||||
draft = self.ws.write_draft(draft_pairs())
|
||||
code, out = run("check", "--draft", str(draft), "--sources", str(self.ws.root / "absent"))
|
||||
self.assertEqual(code, 1)
|
||||
self.assertIn("FAIL", out)
|
||||
|
||||
|
||||
class BuildTest(Base):
|
||||
def built(self, **kwargs: Any) -> None:
|
||||
code, out = self.ws.build(**kwargs)
|
||||
self.assertEqual(code, 0, out)
|
||||
|
||||
def test_build_writes_set_key_layout_and_manifest(self) -> None:
|
||||
self.built()
|
||||
pset, key, layout, manifest = (self.ws.load(w) for w in ("set", "key", "calls", "manifest"))
|
||||
p = self.ws.paths()
|
||||
self.assertEqual(pset["schema"], calibration.SET_SCHEMA)
|
||||
ids = [q["id"] for q in pset["pairs"]]
|
||||
self.assertEqual(ids, [f"c{n:03d}" for n in range(1, 121)])
|
||||
for q in pset["pairs"]:
|
||||
self.assertEqual(set(q), {"id", "claim", "excerpts"})
|
||||
for e in q["excerpts"]:
|
||||
self.assertEqual(set(e), {"file", "line", "text"})
|
||||
self.assertNotIn("unsupported", json.dumps(pset))
|
||||
self.assertNotIn("filler", json.dumps(pset))
|
||||
self.assertEqual(sorted(k["id"] for k in key["labels"]), ids)
|
||||
self.assertEqual(layout["schema"], "okf-wiki/reader-calls/1")
|
||||
self.assertEqual(layout["call_size_max"], 10)
|
||||
self.assertEqual(layout["set_sha256"], sha256_file(p["set"]))
|
||||
self.assertEqual(len(layout["calls"]), 12)
|
||||
self.assertEqual([c["call"] for c in layout["calls"]], [f"k{n:03d}" for n in range(1, 13)])
|
||||
self.assertEqual(manifest["seed"], 7)
|
||||
self.assertEqual(manifest["set"]["sha256"], sha256_file(p["set"]))
|
||||
self.assertEqual(manifest["key"]["sha256"], sha256_file(p["key"]))
|
||||
self.assertEqual(manifest["calls"]["sha256"], sha256_file(p["calls"]))
|
||||
self.assertEqual(manifest["calls"]["calls"], 12)
|
||||
counts = manifest["counts"]
|
||||
self.assertEqual((counts["unsupported"], counts["supported"], counts["filler-supported"]), (20, 6, 94))
|
||||
self.assertEqual(counts["filler_distinct"], 24)
|
||||
self.assertEqual(sum(v["unsupported"] for v in counts["by_position"].values()), 20)
|
||||
self.assertEqual(manifest["thresholds"], {"caught_min": 20, "accepted_min": 6})
|
||||
self.assertEqual(manifest["sources"]["files"], len(SOURCES))
|
||||
self.assertEqual(manifest["history"], [])
|
||||
self.assertNotIn("claim", json.dumps(manifest))
|
||||
|
||||
def test_every_claim_stands_in_exactly_one_call_of_at_most_ten(self) -> None:
|
||||
self.built()
|
||||
placed = [i for c in self.ws.load("calls")["calls"] for i in c["ids"]]
|
||||
self.assertEqual(sorted(placed), sorted(self.ws.labels()))
|
||||
self.assertEqual(len(placed), len(set(placed)))
|
||||
self.assertTrue(all(len(c["ids"]) <= 10 for c in self.ws.load("calls")["calls"]))
|
||||
|
||||
def test_lone_unsupported_stand_at_every_position(self) -> None:
|
||||
self.built()
|
||||
labels, pos = self.ws.labels(), self.ws.positions()
|
||||
alone = []
|
||||
for c in self.ws.load("calls")["calls"]:
|
||||
u = [i for i in c["ids"] if labels[i] == UNSUPPORTED]
|
||||
if len(u) == 1:
|
||||
alone.append(pos[u[0]])
|
||||
self.assertGreaterEqual(len(alone), 10)
|
||||
self.assertEqual(set(alone), set(range(1, 11)))
|
||||
self.assertEqual(self.ws.load("manifest")["calls"]["unsupported_alone"], len(alone))
|
||||
|
||||
def test_each_counted_supported_pair_stands_once(self) -> None:
|
||||
self.built()
|
||||
drafts = Counter(k["draft_id"] for k in self.ws.load("key")["labels"] if k["label"] == SUPPORTED)
|
||||
self.assertEqual(drafts, Counter({f"s{n:02d}": 1 for n in range(1, 7)}))
|
||||
|
||||
def test_a_twin_never_shares_a_call_with_its_unsupported_pair_and_no_filler_stands_twice_in_a_call(self) -> None:
|
||||
self.built()
|
||||
draft_of = {k["id"]: k["draft_id"] for k in self.ws.load("key")["labels"]}
|
||||
for c in self.ws.load("calls")["calls"]:
|
||||
members = [draft_of[i] for i in c["ids"]]
|
||||
self.assertEqual(len(members), len(set(members)), c["call"])
|
||||
for m in members:
|
||||
if m.startswith("t"):
|
||||
self.assertNotIn("u" + m[1:], members, c["call"])
|
||||
|
||||
def test_same_seed_gives_the_same_bytes_and_another_seed_another_layout(self) -> None:
|
||||
self.built(seed=7)
|
||||
first = {w: self.ws.paths()[w].read_bytes() for w in ("set", "key", "calls")}
|
||||
other = Workspace()
|
||||
try:
|
||||
self.assertEqual(other.build(seed=7)[0], 0)
|
||||
self.assertEqual(first, {w: other.paths()[w].read_bytes() for w in ("set", "key", "calls")})
|
||||
finally:
|
||||
other.close()
|
||||
other = Workspace()
|
||||
try:
|
||||
self.assertEqual(other.build(seed=8)[0], 0)
|
||||
self.assertNotEqual(first["calls"], other.paths()["calls"].read_bytes())
|
||||
self.assertNotEqual(first["set"], other.paths()["set"].read_bytes())
|
||||
finally:
|
||||
other.close()
|
||||
|
||||
def test_order_does_not_follow_the_labels(self) -> None:
|
||||
self.built()
|
||||
labels = self.ws.labels()
|
||||
order = [labels[q["id"]] for q in self.ws.load("set")["pairs"]]
|
||||
self.assertNotEqual(order, sorted(order))
|
||||
self.assertNotEqual(order, sorted(order, reverse=True))
|
||||
|
||||
def test_build_moves_the_old_manifest_into_history(self) -> None:
|
||||
old = {"schema": "okf-wiki/calibration-manifest/1", "seed": 1, "history": [{"schema": "older"}]}
|
||||
self.ws.paths()["manifest"].write_text(json.dumps(old), encoding="utf-8")
|
||||
self.built()
|
||||
history = self.ws.load("manifest")["history"]
|
||||
self.assertEqual(history, [{"schema": "older"}, {"schema": "okf-wiki/calibration-manifest/1", "seed": 1}])
|
||||
|
||||
def test_build_refuses_a_failing_draft(self) -> None:
|
||||
pairs = garden_draft()
|
||||
pairs[0]["excerpts"] = [ex("north.md", "not in the file at all, not anywhere")]
|
||||
code, out = self.ws.build(pairs)
|
||||
self.assertEqual(code, 1)
|
||||
self.assertFalse(self.ws.paths()["set"].exists())
|
||||
|
||||
def test_build_refuses_when_a_position_would_have_no_unsupported_claim(self) -> None:
|
||||
code, out = self.ws.build(garden_draft(n_unsupported=9), lone=9)
|
||||
self.assertEqual(code, 1, out)
|
||||
self.assertIn("position 10", out)
|
||||
self.assertFalse(any(self.ws.paths()[w].exists() for w in ("set", "key", "calls", "manifest")))
|
||||
|
||||
def test_build_refuses_slots_it_cannot_fill(self) -> None:
|
||||
pairs = [p for p in garden_draft() if p["label"] != FILLER]
|
||||
code, out = self.ws.build(pairs)
|
||||
self.assertEqual(code, 1, out)
|
||||
self.assertIn("filler", out)
|
||||
self.assertFalse(self.ws.paths()["calls"].exists())
|
||||
|
||||
def test_build_refuses_impossible_parameters(self) -> None:
|
||||
for lone, dense in ((21, 5), (10, 11), (10, 0), (-1, 5)):
|
||||
code, out = self.ws.build(lone=lone, dense=dense)
|
||||
self.assertEqual(code, 1, (lone, dense, out))
|
||||
self.assertFalse(self.ws.paths()["set"].exists())
|
||||
|
||||
def test_build_never_overwrites_a_frozen_set(self) -> None:
|
||||
self.built()
|
||||
code, out = self.ws.build()
|
||||
self.assertEqual(code, 1)
|
||||
self.assertIn("frozen", out)
|
||||
self.ws.paths()["set"].unlink()
|
||||
self.ws.paths()["key"].unlink()
|
||||
code, out = self.ws.build()
|
||||
self.assertEqual(code, 1)
|
||||
self.assertIn("frozen", out)
|
||||
|
||||
|
||||
class GateBase(Base):
|
||||
"""A built set, its key, and helpers to write verdicts and stubs for the gate."""
|
||||
|
||||
def setUp(self) -> None:
|
||||
super().setUp()
|
||||
code, out = self.ws.build()
|
||||
self.assertEqual(code, 0, out)
|
||||
self.key = self.ws.labels()
|
||||
|
||||
def verdicts(self, mapping: dict[str, str], reader: str = "test", models: Any = ("claude-test-1",),
|
||||
name: str = "verdicts.json") -> Path:
|
||||
doc: dict[str, Any] = {"reader": reader, "verdicts": [{"id": i, "verdict": v} for i, v in mapping.items()]}
|
||||
if models is not None:
|
||||
doc["models"] = list(models) if isinstance(models, tuple) else models
|
||||
return self.ws.write_json(name, doc)
|
||||
|
||||
def perfect(self) -> dict[str, str]:
|
||||
return {i: (UNSUPPORTED if lab == UNSUPPORTED else SUPPORTED) for i, lab in self.key.items()}
|
||||
|
||||
def stub(self, *args: str) -> Path:
|
||||
path = self.ws.root / "stub.json"
|
||||
code, out = run("stub", "--set", str(self.ws.paths()["set"]), "--out", str(path), *args)
|
||||
self.assertEqual(code, 0, out)
|
||||
return path
|
||||
|
||||
def per_position(self) -> dict[int, int]:
|
||||
pos = self.ws.positions()
|
||||
return dict(Counter(pos[i] for i, lab in self.key.items() if lab == UNSUPPORTED))
|
||||
|
||||
|
||||
class GateTest(GateBase):
|
||||
def test_no_reader_scores_zero_and_is_red(self) -> None:
|
||||
code, out = self.ws.gate(None)
|
||||
self.assertEqual(code, 1)
|
||||
self.assertIn("unsupported caught 0 of 20 · supported accepted 0 of 6", out)
|
||||
self.assertIn("fail-closed", out)
|
||||
self.assertIn("RED", out)
|
||||
|
||||
def test_the_key_as_judge_is_green(self) -> None:
|
||||
code, out = self.ws.gate(self.stub("--key", str(self.ws.paths()["key"])))
|
||||
self.assertEqual(code, 0, out)
|
||||
self.assertIn("unsupported caught 20 of 20 · supported accepted 6 of 6", out)
|
||||
self.assertIn("filler accepted 94 of 94", out)
|
||||
self.assertIn("GREEN", out)
|
||||
|
||||
def test_reject_all_stub_is_red(self) -> None:
|
||||
code, out = self.ws.gate(self.stub("--verdict", UNSUPPORTED))
|
||||
self.assertEqual(code, 1)
|
||||
self.assertIn("unsupported caught 20 of 20 · supported accepted 0 of 6", out)
|
||||
self.assertIn("RED", out)
|
||||
|
||||
def test_accept_all_stub_is_red(self) -> None:
|
||||
code, out = self.ws.gate(self.stub("--verdict", SUPPORTED))
|
||||
self.assertEqual(code, 1)
|
||||
self.assertIn("unsupported caught 0 of 20 · supported accepted 6 of 6", out)
|
||||
self.assertIn("RED", out)
|
||||
|
||||
def test_a_stub_mild_at_one_position_is_red_and_the_position_is_named(self) -> None:
|
||||
stub = self.stub("--key", str(self.ws.paths()["key"]), "--calls", str(self.ws.paths()["calls"]), "--accept-at", "7")
|
||||
code, out = self.ws.gate(stub)
|
||||
self.assertEqual(code, 1, out)
|
||||
n7 = self.per_position()[7]
|
||||
self.assertIn(f"7 0 of {n7}", out)
|
||||
self.assertIn(f"unsupported caught {20 - n7} of 20", out)
|
||||
not_met = [line for line in out.splitlines() if line.startswith("not met:")]
|
||||
self.assertEqual(len(not_met), 1, out)
|
||||
self.assertIn("position 7", not_met[0])
|
||||
self.assertNotIn("position 6", not_met[0])
|
||||
self.assertIn("RED", out)
|
||||
|
||||
def test_caught_per_position_is_written_and_no_line_per_call_or_pair(self) -> None:
|
||||
code, out = self.ws.gate(self.verdicts(self.perfect()))
|
||||
self.assertEqual(code, 0, out)
|
||||
counts = self.per_position()
|
||||
line = next(line for line in out.splitlines() if line.startswith("caught by position: "))
|
||||
self.assertEqual(line, "caught by position: " + ", ".join(f"{p} {counts[p]} of {counts[p]}" for p in range(1, 11)))
|
||||
self.assertIsNone(CALL_ID.search(out), out)
|
||||
self.assertIsNone(CLAIM_ID.search(out), out)
|
||||
|
||||
def test_the_gate_writes_no_per_pair_results(self) -> None:
|
||||
code, out = run("gate", *self.ws.frozen_args(), "--verdicts", str(self.verdicts(self.perfect())),
|
||||
"--out", str(self.ws.root / "results.json"))
|
||||
self.assertNotEqual(code, 0)
|
||||
self.assertFalse((self.ws.root / "results.json").exists())
|
||||
|
||||
def test_reports_caught_per_kind(self) -> None:
|
||||
code, out = self.ws.gate(self.verdicts(self.perfect()))
|
||||
self.assertIn("caught by kind: overgeneralisation 4 of 4", out)
|
||||
self.assertIn("conflated-sources 3 of 3", out)
|
||||
|
||||
def test_filler_never_counts_in_the_thresholds(self) -> None:
|
||||
v = self.perfect()
|
||||
for i, lab in self.key.items():
|
||||
if lab == FILLER:
|
||||
v[i] = UNSUPPORTED
|
||||
code, out = self.ws.gate(self.verdicts(v))
|
||||
self.assertEqual(code, 0, out)
|
||||
self.assertIn("filler accepted 0 of 94", out)
|
||||
self.assertIn("GREEN", out)
|
||||
|
||||
def test_a_missing_filler_verdict_is_red(self) -> None:
|
||||
v = self.perfect()
|
||||
del v[next(i for i, lab in self.key.items() if lab == FILLER)]
|
||||
code, out = self.ws.gate(self.verdicts(v))
|
||||
self.assertEqual(code, 1, out)
|
||||
self.assertIn("missing verdicts: 1", out)
|
||||
|
||||
def test_one_missed_unsupported_is_red(self) -> None:
|
||||
v = self.perfect()
|
||||
v[next(i for i, lab in self.key.items() if lab == UNSUPPORTED)] = SUPPORTED
|
||||
code, out = self.ws.gate(self.verdicts(v))
|
||||
self.assertEqual(code, 1)
|
||||
self.assertIn("unsupported caught 19 of 20", out)
|
||||
|
||||
def test_missing_verdict_counts_as_wrong(self) -> None:
|
||||
v = self.perfect()
|
||||
gone_u = next(i for i, lab in self.key.items() if lab == UNSUPPORTED)
|
||||
gone_s = next(i for i, lab in self.key.items() if lab == SUPPORTED)
|
||||
del v[gone_u], v[gone_s]
|
||||
code, out = self.ws.gate(self.verdicts(v))
|
||||
self.assertEqual(code, 1)
|
||||
self.assertIn("missing verdicts: 2", out)
|
||||
self.assertIn("unsupported caught 19 of 20 · supported accepted 5 of 6", out)
|
||||
|
||||
def test_invalid_or_duplicate_verdict_counts_as_missing(self) -> None:
|
||||
v = self.perfect()
|
||||
first = next(iter(v))
|
||||
path = self.ws.write_json(
|
||||
"verdicts.json",
|
||||
{
|
||||
"reader": "test",
|
||||
"verdicts": [{"id": i, "verdict": val} for i, val in v.items()]
|
||||
+ [{"id": first, "verdict": SUPPORTED}, {"id": "c999", "verdict": SUPPORTED}],
|
||||
},
|
||||
)
|
||||
code, out = self.ws.gate(path)
|
||||
self.assertEqual(code, 1)
|
||||
self.assertIn("missing verdicts: 1", out)
|
||||
for bad in ("probably", FILLER):
|
||||
code, out = self.ws.gate(self.verdicts({**v, first: bad}))
|
||||
self.assertEqual(code, 1, bad)
|
||||
self.assertIn("missing verdicts: 1", out)
|
||||
|
||||
def test_accepted_threshold_is_read_from_the_manifest(self) -> None:
|
||||
manifest = self.ws.load("manifest")
|
||||
manifest["thresholds"]["accepted_min"] = 5
|
||||
self.ws.paths()["manifest"].write_text(json.dumps(manifest), encoding="utf-8")
|
||||
v = self.perfect()
|
||||
v[next(i for i, lab in self.key.items() if lab == SUPPORTED)] = UNSUPPORTED
|
||||
code, out = self.ws.gate(self.verdicts(v))
|
||||
self.assertEqual(code, 0, out)
|
||||
self.assertIn("supported accepted 5 of 6", out)
|
||||
|
||||
def test_gate_refuses_a_changed_set_key_or_layout(self) -> None:
|
||||
for which in ("set", "key", "calls"):
|
||||
path = self.ws.paths()[which]
|
||||
before = path.read_text(encoding="utf-8")
|
||||
path.write_text(before + " ", encoding="utf-8")
|
||||
code, out = self.ws.gate(self.verdicts(self.perfect()))
|
||||
self.assertEqual(code, 1, which)
|
||||
self.assertIn("manifest", out)
|
||||
path.write_text(before, encoding="utf-8")
|
||||
|
||||
def test_gate_refuses_other_sources(self) -> None:
|
||||
(self.ws.sources / "alpha.md").write_text("changed\n", encoding="utf-8")
|
||||
code, out = self.ws.gate(self.verdicts(self.perfect()))
|
||||
self.assertEqual(code, 1)
|
||||
self.assertIn("tree_sha256", out)
|
||||
|
||||
|
||||
class GateModelTest(GateBase):
|
||||
"""PLAN § Rammer 23: the gate is bound to the model the reader ran on. It names the model it scored, is GREEN only
|
||||
for verdicts of exactly one model, and records a GREEN run in the file --passed names; collect reads that file."""
|
||||
|
||||
def passed(self) -> Path:
|
||||
return self.ws.root / "passed.json"
|
||||
|
||||
def gate_passed(self, verdicts: Path) -> tuple[int, str]:
|
||||
return run("gate", *self.ws.frozen_args(), "--verdicts", str(verdicts), "--passed", str(self.passed()))
|
||||
|
||||
def test_the_gate_names_the_model_it_scored(self) -> None:
|
||||
code, out = self.ws.gate(self.verdicts(self.perfect(), models=["claude-test-2"]))
|
||||
self.assertEqual(code, 0, out)
|
||||
self.assertIn("model: claude-test-2\n", out)
|
||||
self.assertIn("GREEN", out)
|
||||
|
||||
def test_verdicts_that_name_no_model_are_red(self) -> None:
|
||||
for models in (None, [], "claude-test-2", [3]):
|
||||
with self.subTest(models=models):
|
||||
code, out = self.ws.gate(self.verdicts(self.perfect(), models=models))
|
||||
self.assertEqual(code, 1, out)
|
||||
self.assertIn("model: (none)\n", out)
|
||||
self.assertIn("the verdicts name no model", out)
|
||||
self.assertNotIn("GREEN", out)
|
||||
self.assertIn("RED", out)
|
||||
|
||||
def test_verdicts_of_two_models_are_red(self) -> None:
|
||||
code, out = self.ws.gate(self.verdicts(self.perfect(), models=["claude-test-1", "claude-test-2"]))
|
||||
self.assertEqual(code, 1, out)
|
||||
self.assertIn("model: claude-test-1, claude-test-2\n", out)
|
||||
self.assertIn("the verdicts name 2 models", out)
|
||||
self.assertIn("RED", out)
|
||||
|
||||
def test_a_stub_names_the_model_stub_and_its_meaning_is_kept(self) -> None:
|
||||
stub = self.stub("--key", str(self.ws.paths()["key"]))
|
||||
self.assertEqual(json.loads(stub.read_text(encoding="utf-8"))["models"], ["stub"])
|
||||
code, out = self.ws.gate(stub)
|
||||
self.assertEqual(code, 0, out)
|
||||
self.assertIn("model: stub\n", out)
|
||||
code, out = self.ws.gate(self.stub("--verdict", SUPPORTED))
|
||||
self.assertEqual(code, 1, out)
|
||||
self.assertIn("model: stub\n", out)
|
||||
|
||||
def test_a_green_run_is_recorded_with_its_hashes(self) -> None:
|
||||
verdicts = self.verdicts(self.perfect(), reader="r7", models=["claude-test-2"])
|
||||
code, out = self.gate_passed(verdicts)
|
||||
self.assertEqual(code, 0, out)
|
||||
p = self.ws.paths()
|
||||
record = json.loads(self.passed().read_text(encoding="utf-8"))
|
||||
self.assertEqual(record, {"schema": "okf-wiki/calibration-passed/1", "passed": [{
|
||||
"model": "claude-test-2", "set_sha256": sha256_file(p["set"]), "key_sha256": sha256_file(p["key"]),
|
||||
"verdicts_sha256": sha256_file(verdicts), "reader": "r7"}]})
|
||||
self.assertIn(str(self.passed()), out)
|
||||
|
||||
def test_the_same_green_run_is_recorded_once_and_the_file_has_no_time(self) -> None:
|
||||
verdicts = self.verdicts(self.perfect(), models=["claude-test-2"])
|
||||
self.assertEqual(self.gate_passed(verdicts)[0], 0)
|
||||
first = self.passed().read_bytes()
|
||||
self.assertEqual(self.gate_passed(verdicts)[0], 0)
|
||||
self.assertEqual(self.passed().read_bytes(), first)
|
||||
self.assertEqual(len(json.loads(first)["passed"]), 1)
|
||||
|
||||
def test_another_green_run_is_added_after_the_first(self) -> None:
|
||||
self.assertEqual(self.gate_passed(self.verdicts(self.perfect(), models=["claude-test-2"]))[0], 0)
|
||||
self.assertEqual(self.gate_passed(self.verdicts(self.perfect(), models=["claude-test-3"],
|
||||
name="v3.json"))[0], 0)
|
||||
record = json.loads(self.passed().read_text(encoding="utf-8"))
|
||||
self.assertEqual([e["model"] for e in record["passed"]], ["claude-test-2", "claude-test-3"])
|
||||
|
||||
def test_a_red_run_never_writes_the_record(self) -> None:
|
||||
v = self.perfect()
|
||||
v[next(i for i, lab in self.key.items() if lab == UNSUPPORTED)] = SUPPORTED
|
||||
code, out = self.gate_passed(self.verdicts(v))
|
||||
self.assertEqual(code, 1, out)
|
||||
self.assertFalse(self.passed().exists())
|
||||
code, out = self.gate_passed(self.verdicts(self.perfect(), models=None))
|
||||
self.assertEqual(code, 1, out)
|
||||
self.assertFalse(self.passed().exists())
|
||||
self.assertEqual(self.gate_passed(self.verdicts(self.perfect()))[0], 0)
|
||||
before = self.passed().read_bytes()
|
||||
self.assertEqual(self.gate_passed(self.verdicts(v, models=["claude-test-9"]))[0], 1)
|
||||
self.assertEqual(self.passed().read_bytes(), before)
|
||||
|
||||
def test_a_record_that_cannot_be_read_is_a_fail_and_is_left_as_it_was(self) -> None:
|
||||
for bad in ("{", json.dumps({"schema": "okf-wiki/other/1", "passed": []}),
|
||||
json.dumps({"schema": "okf-wiki/calibration-passed/1", "passed": {}})):
|
||||
with self.subTest(bad=bad):
|
||||
self.passed().write_text(bad, encoding="utf-8")
|
||||
code, out = self.gate_passed(self.verdicts(self.perfect()))
|
||||
self.assertEqual(code, 1, out)
|
||||
self.assertIn("FAIL", out)
|
||||
self.assertEqual(self.passed().read_text(encoding="utf-8"), bad)
|
||||
|
||||
|
||||
class LayoutRuleTest(Base):
|
||||
"""A layout that breaks K3 point 1 is refused by the gate and by check, even when the manifest follows it."""
|
||||
|
||||
def setUp(self) -> None:
|
||||
super().setUp()
|
||||
code, out = self.ws.build()
|
||||
self.assertEqual(code, 0, out)
|
||||
self.layout = self.ws.load("calls")
|
||||
self.key = self.ws.labels()
|
||||
self.green = self.ws.write_json(
|
||||
"verdicts.json",
|
||||
{"reader": "key", "models": ["claude-test-1"],
|
||||
"verdicts": [{"id": i, "verdict": UNSUPPORTED if lab == UNSUPPORTED else SUPPORTED}
|
||||
for i, lab in self.key.items()]},
|
||||
)
|
||||
|
||||
def refused(self, needle: str) -> None:
|
||||
self.ws.rewrite_calls(self.layout)
|
||||
code, out = self.ws.gate(self.green)
|
||||
self.assertEqual(code, 1, out)
|
||||
self.assertIn(needle, out)
|
||||
self.assertNotIn("GREEN", out)
|
||||
code, out = self.ws.check()
|
||||
self.assertEqual(code, 1, out)
|
||||
self.assertIn(needle, out)
|
||||
|
||||
def test_the_unchanged_layout_passes(self) -> None:
|
||||
self.ws.rewrite_calls(self.layout)
|
||||
self.assertEqual(self.ws.gate(self.green)[0], 0)
|
||||
self.assertEqual(self.ws.check()[0], 0)
|
||||
|
||||
def test_an_id_given_twice_is_refused(self) -> None:
|
||||
self.layout["calls"].append({"call": "k999", "ids": [self.layout["calls"][1]["ids"][0]]})
|
||||
self.refused("more than one place")
|
||||
|
||||
def test_an_id_missing_from_every_call_is_refused(self) -> None:
|
||||
self.layout["calls"][-1]["ids"].pop()
|
||||
self.refused("in no call")
|
||||
|
||||
def test_an_id_not_in_the_set_is_refused(self) -> None:
|
||||
self.layout["calls"][-1]["ids"][-1] = "c999"
|
||||
self.refused("not in the set")
|
||||
|
||||
def test_a_wrong_set_sha256_is_refused(self) -> None:
|
||||
self.layout["set_sha256"] = "0" * 64
|
||||
self.refused("set_sha256")
|
||||
|
||||
def test_a_call_over_ten_is_refused(self) -> None:
|
||||
moved = self.layout["calls"][1]["ids"].pop()
|
||||
self.layout["calls"][0]["ids"].append(moved)
|
||||
self.refused("more than 10")
|
||||
|
||||
def test_a_call_size_max_over_ten_is_refused(self) -> None:
|
||||
self.layout["call_size_max"] = 11
|
||||
self.refused("call_size_max")
|
||||
|
||||
def test_a_position_without_unsupported_is_refused(self) -> None:
|
||||
for c in self.layout["calls"]:
|
||||
ids = c["ids"]
|
||||
if self.key[ids[2]] == UNSUPPORTED:
|
||||
swap = next(n for n, i in enumerate(ids) if self.key[i] != UNSUPPORTED and n != 2)
|
||||
ids[2], ids[swap] = ids[swap], ids[2]
|
||||
self.refused("position 3 has no unsupported claim")
|
||||
|
||||
|
||||
class CheckBuiltSetTest(Base):
|
||||
def setUp(self) -> None:
|
||||
super().setUp()
|
||||
code, out = self.ws.build()
|
||||
self.assertEqual(code, 0, out)
|
||||
|
||||
def excerpts(self) -> int:
|
||||
return sum(len(q["excerpts"]) for q in self.ws.load("set")["pairs"])
|
||||
|
||||
def test_check_passes_on_the_built_set_and_states_the_layout(self) -> None:
|
||||
code, out = self.ws.check()
|
||||
self.assertEqual(code, 0, out)
|
||||
n = self.excerpts()
|
||||
self.assertIn(f"excerpt lines correct: {n} of {n}", out)
|
||||
self.assertIn("pairs: 120 (20 unsupported, 6 supported, 94 filler-supported)", out)
|
||||
self.assertIn("unsupported per call: 1 in 10 calls, 5 in 2 calls", out)
|
||||
self.assertIn("PASS", out)
|
||||
|
||||
def test_more_unsupported_than_supported_passes_when_the_manifest_says_so(self) -> None:
|
||||
self.assertEqual(self.ws.check()[0], 0)
|
||||
manifest = self.ws.load("manifest")
|
||||
manifest["counts"]["unsupported"] = 6
|
||||
self.ws.paths()["manifest"].write_text(json.dumps(manifest), encoding="utf-8")
|
||||
code, out = self.ws.check()
|
||||
self.assertEqual(code, 1, out)
|
||||
self.assertIn("manifest", out)
|
||||
|
||||
def test_check_catches_a_wrong_line(self) -> None:
|
||||
data = self.ws.load("set")
|
||||
data["pairs"][0]["excerpts"][0]["line"] += 1
|
||||
self.ws.paths()["set"].write_text(json.dumps(data), encoding="utf-8")
|
||||
code, out = self.ws.check()
|
||||
self.assertEqual(code, 1)
|
||||
n = self.excerpts()
|
||||
self.assertIn(f"excerpt lines correct: {n - 1} of {n}", out)
|
||||
|
||||
def test_check_of_a_built_set_needs_the_layout_and_the_manifest(self) -> None:
|
||||
p = self.ws.paths()
|
||||
code, out = run("check", "--set", str(p["set"]), "--key", str(p["key"]), "--sources", str(self.ws.sources))
|
||||
self.assertEqual(code, 1)
|
||||
self.assertIn("FAIL", out)
|
||||
|
||||
|
||||
class StubTest(Base):
|
||||
def setUp(self) -> None:
|
||||
super().setUp()
|
||||
code, out = self.ws.build()
|
||||
self.assertEqual(code, 0, out)
|
||||
|
||||
def stub(self, *args: str) -> tuple[int, str, dict[str, str]]:
|
||||
path = self.ws.root / "stub.json"
|
||||
code, out = run("stub", "--set", str(self.ws.paths()["set"]), "--out", str(path), *args)
|
||||
verdicts = {}
|
||||
if path.exists():
|
||||
verdicts = {v["id"]: v["verdict"] for v in json.loads(path.read_text(encoding="utf-8"))["verdicts"]}
|
||||
path.unlink()
|
||||
return code, out, verdicts
|
||||
|
||||
def test_the_same_verdict_for_every_claim(self) -> None:
|
||||
code, out, v = self.stub("--verdict", UNSUPPORTED)
|
||||
self.assertEqual(code, 0, out)
|
||||
self.assertEqual(set(v.values()), {UNSUPPORTED})
|
||||
self.assertEqual(len(v), 120)
|
||||
|
||||
def test_the_key_gives_supported_for_every_filler(self) -> None:
|
||||
code, out, v = self.stub("--key", str(self.ws.paths()["key"]))
|
||||
self.assertEqual(code, 0, out)
|
||||
labels = self.ws.labels()
|
||||
self.assertEqual(v, {i: UNSUPPORTED if lab == UNSUPPORTED else SUPPORTED for i, lab in labels.items()})
|
||||
|
||||
def test_accept_at_changes_only_that_position(self) -> None:
|
||||
code, out, v = self.stub("--verdict", UNSUPPORTED, "--calls", str(self.ws.paths()["calls"]), "--accept-at", "4")
|
||||
self.assertEqual(code, 0, out)
|
||||
pos = self.ws.positions()
|
||||
self.assertEqual({i for i, val in v.items() if val == SUPPORTED}, {i for i, n in pos.items() if n == 4})
|
||||
|
||||
def test_accept_at_needs_the_layout_and_a_position_from_one_to_ten(self) -> None:
|
||||
self.assertEqual(self.stub("--verdict", UNSUPPORTED, "--accept-at", "4")[0], 1)
|
||||
for bad in ("0", "11"):
|
||||
code, out, v = self.stub("--verdict", UNSUPPORTED, "--calls", str(self.ws.paths()["calls"]), "--accept-at", bad)
|
||||
self.assertEqual(code, 1, bad)
|
||||
|
||||
def test_verdict_and_key_exclude_each_other(self) -> None:
|
||||
self.assertNotEqual(self.stub("--verdict", UNSUPPORTED, "--key", str(self.ws.paths()["key"]))[0], 0)
|
||||
self.assertNotEqual(self.stub()[0], 0)
|
||||
|
||||
|
||||
class NoModelCallTest(unittest.TestCase):
|
||||
def test_tool_imports_no_network_or_model_client(self) -> None:
|
||||
text = (REPO_ROOT / "gates" / "calibration.py").read_text(encoding="utf-8")
|
||||
for banned in ("anthropic", "openai", "urllib", "http.client", "requests", "subprocess", "socket"):
|
||||
self.assertNotIn(f"import {banned}", text)
|
||||
self.assertNotIn(f"from {banned}", text)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
345
gates/tests/test_filetype_gate.py
Normal file
345
gates/tests/test_filetype_gate.py
Normal file
|
|
@ -0,0 +1,345 @@
|
|||
"""Tests for gates/filetype_gate.py, the fail-closed runner of the reader's file-type gate (E4).
|
||||
|
||||
Every fixture here is synthetic and neutral text. The real pairs live in gates-local/ and outside the repository.
|
||||
Run: python3 -m unittest discover -s gates/tests
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import json
|
||||
import sys
|
||||
import tempfile
|
||||
import textwrap
|
||||
import unittest
|
||||
import zipfile
|
||||
from contextlib import redirect_stdout
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
|
||||
import filetype_gate # noqa: E402
|
||||
|
||||
PY = sys.executable
|
||||
STUBS = Path(__file__).resolve().parents[1] / "stubs"
|
||||
|
||||
# Answers every query from the pairs file given as argv[3]: the known-positive.
|
||||
PERFECT = """
|
||||
import json, sys
|
||||
file, queries, pairs = sys.argv[1], sys.argv[2], sys.argv[3]
|
||||
key = {p["id"]: p for p in json.load(open(pairs))["pairs"]}
|
||||
answers = [{"id": q["id"], "found": [{"locator": key[q["id"]]["locator"], "hidden": key[q["id"]]["hidden"]}]}
|
||||
for q in json.load(open(queries))["queries"]]
|
||||
print(json.dumps({"answers": answers}))
|
||||
"""
|
||||
|
||||
# Like PERFECT, but every answer also claims a second place.
|
||||
TWO_PLACES = PERFECT.replace(
|
||||
'[{"locator": key[q["id"]]["locator"], "hidden": key[q["id"]]["hidden"]}]',
|
||||
'[{"locator": key[q["id"]]["locator"], "hidden": key[q["id"]]["hidden"]}, {"locator": {"line": 99}, "hidden": False}]')
|
||||
|
||||
# Like PERFECT, but says hidden=false for everything (flattens hidden content into ordinary text).
|
||||
FLATTEN = PERFECT.replace('"hidden": key[q["id"]]["hidden"]', '"hidden": False')
|
||||
|
||||
# Like PERFECT, but leaves out the first query of every file.
|
||||
DROP_FIRST = PERFECT.replace("for q in json.load(open(queries))[\"queries\"]]",
|
||||
"for q in json.load(open(queries))[\"queries\"][1:]]")
|
||||
|
||||
CRASH = """
|
||||
raise RuntimeError("boom")
|
||||
"""
|
||||
|
||||
NOT_JSON = """
|
||||
print("answers: many")
|
||||
"""
|
||||
|
||||
NO_ANSWERS_KEY = """
|
||||
import json
|
||||
print(json.dumps({"found": []}))
|
||||
"""
|
||||
|
||||
SLEEPER = """
|
||||
import time
|
||||
time.sleep(10)
|
||||
"""
|
||||
|
||||
# Records what the reader was shown, then answers nothing.
|
||||
SPY = """
|
||||
import json, sys, pathlib
|
||||
q = json.load(open(sys.argv[2]))
|
||||
out = pathlib.Path(sys.argv[3])
|
||||
with out.open("a") as fh:
|
||||
fh.write(json.dumps({"file": sys.argv[1], "queries": q}) + "\\n")
|
||||
print(json.dumps({"answers": []}))
|
||||
"""
|
||||
|
||||
# Tries to damage the file it was given.
|
||||
VANDAL = """
|
||||
import json, sys, pathlib
|
||||
pathlib.Path(sys.argv[1]).write_text("vandalised")
|
||||
print(json.dumps({"answers": []}))
|
||||
"""
|
||||
|
||||
NOTES_MD = "# Kettles\n\nA kettle boils water for tea.\nThe spout pours the water out slowly.\n"
|
||||
NOTES_TXT = "Line one of the plain file.\nThe handle stays cool while the water heats.\n"
|
||||
|
||||
|
||||
class GateFixture:
|
||||
"""A synthetic manifest, pairs file, sources root and cases root in a temporary directory."""
|
||||
|
||||
def __init__(self, root: Path) -> None:
|
||||
self.root = root
|
||||
self.sources = root / "sources"
|
||||
self.cases = root / "cases"
|
||||
(self.sources / "docs").mkdir(parents=True)
|
||||
(self.cases / "txt").mkdir(parents=True)
|
||||
(self.sources / "docs" / "kettle.md").write_text(NOTES_MD)
|
||||
(self.cases / "txt" / "handle.txt").write_text(NOTES_TXT)
|
||||
self.pairs_data: dict[str, Any] = {
|
||||
"schema": filetype_gate.PAIRS_SCHEMA,
|
||||
"set": "working",
|
||||
"pairs": [
|
||||
{"id": "t01", "type": "md", "root": "sources", "path": "docs/kettle.md", "group": "filetype",
|
||||
"hidden_class": None, "quote": "Kettles", "locator": {"line": 1}, "hidden": False},
|
||||
{"id": "t02", "type": "md", "root": "sources", "path": "docs/kettle.md", "group": "filetype",
|
||||
"hidden_class": None, "quote": "spout pours the water", "locator": {"line": 4}, "hidden": False},
|
||||
{"id": "t03", "type": "txt", "root": "cases", "path": "txt/handle.txt", "group": "filetype",
|
||||
"hidden_class": None, "quote": "handle stays cool", "locator": {"line": 2}, "hidden": False},
|
||||
{"id": "t04", "type": "txt", "root": "cases", "path": "txt/handle.txt", "group": "hidden",
|
||||
"hidden_class": "demo-hidden", "quote": "Line one", "locator": {"line": 1}, "hidden": True},
|
||||
],
|
||||
}
|
||||
self.write_pairs()
|
||||
self.write_manifest()
|
||||
|
||||
def write_pairs(self) -> None:
|
||||
self.pairs = self.root / "pairs.json"
|
||||
self.pairs.write_text(json.dumps(self.pairs_data))
|
||||
|
||||
def write_manifest(self) -> None:
|
||||
self.manifest = self.root / "manifest.json"
|
||||
self.manifest.write_text(json.dumps({
|
||||
"schema": filetype_gate.MANIFEST_SCHEMA,
|
||||
"sets": {"working": {"file": "pairs.json", "sha256": filetype_gate.sha256_file(self.pairs),
|
||||
"pairs": len(self.pairs_data["pairs"])}},
|
||||
"sources": [
|
||||
{"root": "sources", "path": "docs/kettle.md",
|
||||
"sha256": filetype_gate.sha256_file(self.sources / "docs" / "kettle.md")},
|
||||
{"root": "cases", "path": "txt/handle.txt",
|
||||
"sha256": filetype_gate.sha256_file(self.cases / "txt" / "handle.txt")},
|
||||
],
|
||||
}))
|
||||
|
||||
def script(self, name: str, body: str) -> Path:
|
||||
p = self.root / f"{name}.py"
|
||||
p.write_text(textwrap.dedent(body))
|
||||
return p
|
||||
|
||||
def run(self, command: list[str], timeout: float = 5.0) -> tuple[int, str]:
|
||||
out = io.StringIO()
|
||||
with redirect_stdout(out):
|
||||
code = filetype_gate.main([
|
||||
"--manifest", str(self.manifest), "--pairs", str(self.pairs), "--sources", str(self.sources),
|
||||
"--cases", str(self.cases), "--timeout", str(timeout), "--", *command,
|
||||
])
|
||||
return code, out.getvalue()
|
||||
|
||||
def perfect(self, body: str = PERFECT) -> list[str]:
|
||||
return [PY, str(self.script("reader", body)), "{file}", "{queries}", str(self.pairs)]
|
||||
|
||||
|
||||
class FiletypeGateTest(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
self._tmp = tempfile.TemporaryDirectory()
|
||||
self.fx = GateFixture(Path(self._tmp.name))
|
||||
|
||||
def tearDown(self) -> None:
|
||||
self._tmp.cleanup()
|
||||
|
||||
# --- the three outcomes the order names -----------------------------------------------------------
|
||||
|
||||
def test_missing_reader_locates_nothing_and_says_so(self) -> None:
|
||||
code, out = self.fx.run([str(self.fx.root / "no-such-reader"), "{file}", "{queries}"])
|
||||
self.assertEqual(code, 1)
|
||||
self.assertIn("located 0 of 4", out)
|
||||
self.assertIn("reader errors 2 of 2 files", out)
|
||||
self.assertIn("reader not found", out)
|
||||
self.assertIn("verdict: RED", out)
|
||||
|
||||
def test_known_positive_turns_the_gate_green(self) -> None:
|
||||
code, out = self.fx.run(self.fx.perfect())
|
||||
self.assertEqual(code, 0, out)
|
||||
self.assertIn("located 4 of 4", out)
|
||||
self.assertIn("md: 2 of 2", out)
|
||||
self.assertIn("txt: 1 of 1", out)
|
||||
self.assertIn("demo-hidden: 1 of 1", out)
|
||||
self.assertIn("reader errors 0 of 2 files", out)
|
||||
self.assertIn("file types with every pair located: 2 of 2", out)
|
||||
self.assertIn("verdict: GREEN", out)
|
||||
|
||||
def test_the_tracked_fasit_stub_is_the_known_positive(self) -> None:
|
||||
code, out = self.fx.run([PY, str(STUBS / "fasit_reader.py"), "{file}", "{queries}", str(self.fx.pairs)])
|
||||
self.assertEqual(code, 0, out)
|
||||
self.assertIn("located 4 of 4", out)
|
||||
self.assertIn("verdict: GREEN", out)
|
||||
|
||||
def test_first_locator_stub_only_gets_pairs_on_the_first_line(self) -> None:
|
||||
code, out = self.fx.run([PY, str(STUBS / "first_locator.py"), "{file}", "{queries}"])
|
||||
self.assertEqual(code, 1)
|
||||
# t01 is on line 1 and visible: located. t04 is on line 1 but hidden: the stub says visible, so not located.
|
||||
self.assertIn("located 1 of 4", out)
|
||||
self.assertIn("md: 1 of 2", out)
|
||||
self.assertIn("demo-hidden: 0 of 1", out)
|
||||
self.assertIn("file types with every pair located: 0 of 2", out)
|
||||
self.assertIn("verdict: RED", out)
|
||||
|
||||
# --- what counts as located ------------------------------------------------------------------------
|
||||
|
||||
def test_a_second_claimed_place_is_not_located(self) -> None:
|
||||
_, out = self.fx.run(self.fx.perfect(TWO_PLACES))
|
||||
self.assertIn("located 0 of 4", out)
|
||||
self.assertIn("t02", out)
|
||||
|
||||
def test_flattening_hidden_content_fails_the_hidden_class_only(self) -> None:
|
||||
_, out = self.fx.run(self.fx.perfect(FLATTEN))
|
||||
self.assertIn("located 3 of 4", out)
|
||||
self.assertIn("demo-hidden: 0 of 1", out)
|
||||
self.assertIn("hidden flag wrong", out)
|
||||
|
||||
def test_a_missing_answer_is_not_located(self) -> None:
|
||||
_, out = self.fx.run(self.fx.perfect(DROP_FIRST))
|
||||
self.assertIn("located 2 of 4", out)
|
||||
self.assertIn("no answer", out)
|
||||
|
||||
def test_the_reader_sees_only_ids_and_quotes_and_a_copy_of_the_file(self) -> None:
|
||||
log = self.fx.root / "spy.log"
|
||||
self.fx.run([PY, str(self.fx.script("spy", SPY)), "{file}", "{queries}", str(log)])
|
||||
seen = [json.loads(line) for line in log.read_text().splitlines()]
|
||||
self.assertEqual(len(seen), 2)
|
||||
for entry in seen:
|
||||
for q in entry["queries"]["queries"]:
|
||||
self.assertEqual(set(q), {"id", "quote"})
|
||||
self.assertNotIn(str(self.fx.sources), entry["file"])
|
||||
self.assertNotIn(str(self.fx.cases), entry["file"])
|
||||
self.assertEqual({Path(e["file"]).name for e in seen}, {"kettle.md", "handle.txt"})
|
||||
|
||||
def test_a_reader_that_writes_to_its_file_cannot_damage_the_source(self) -> None:
|
||||
self.fx.run([PY, str(self.fx.script("vandal", VANDAL)), "{file}", "{queries}"])
|
||||
self.assertEqual((self.fx.sources / "docs" / "kettle.md").read_text(), NOTES_MD)
|
||||
|
||||
# --- fail-closed: anything but a well-formed answer is an error, never a location -------------------
|
||||
|
||||
def test_crashing_reader_is_an_error(self) -> None:
|
||||
code, out = self.fx.run([PY, str(self.fx.script("crash", CRASH)), "{file}", "{queries}"])
|
||||
self.assertEqual(code, 1)
|
||||
self.assertIn("located 0 of 4", out)
|
||||
self.assertIn("reader errors 2 of 2 files", out)
|
||||
|
||||
def test_output_that_is_not_json_is_an_error(self) -> None:
|
||||
_, out = self.fx.run([PY, str(self.fx.script("notjson", NOT_JSON)), "{file}", "{queries}"])
|
||||
self.assertIn("not valid JSON", out)
|
||||
self.assertIn("located 0 of 4", out)
|
||||
|
||||
def test_output_without_answers_is_an_error(self) -> None:
|
||||
_, out = self.fx.run([PY, str(self.fx.script("noans", NO_ANSWERS_KEY)), "{file}", "{queries}"])
|
||||
self.assertIn('lacks an "answers" list', out)
|
||||
|
||||
def test_timeout_is_an_error(self) -> None:
|
||||
_, out = self.fx.run([PY, str(self.fx.script("sleep", SLEEPER)), "{file}", "{queries}"], timeout=0.5)
|
||||
self.assertIn("timed out", out)
|
||||
self.assertIn("located 0 of 4", out)
|
||||
|
||||
# --- the gate refuses to measure anything but the frozen set ----------------------------------------
|
||||
|
||||
def test_a_changed_source_stops_the_gate(self) -> None:
|
||||
(self.fx.sources / "docs" / "kettle.md").write_text(NOTES_MD + "extra\n")
|
||||
code, out = self.fx.run(self.fx.perfect())
|
||||
self.assertEqual(code, 2)
|
||||
self.assertIn("GATE CANNOT RUN", out)
|
||||
self.assertIn("sha256 mismatch", out)
|
||||
|
||||
def test_a_changed_pairs_file_stops_the_gate(self) -> None:
|
||||
self.fx.pairs_data["pairs"][1]["locator"] = {"line": 3}
|
||||
self.fx.write_pairs()
|
||||
code, out = self.fx.run(self.fx.perfect())
|
||||
self.assertEqual(code, 2)
|
||||
self.assertIn("pairs file", out)
|
||||
|
||||
def test_a_pair_on_a_file_the_manifest_does_not_list_stops_the_gate(self) -> None:
|
||||
self.fx.pairs_data["pairs"][0]["path"] = "docs/other.md"
|
||||
self.fx.write_pairs()
|
||||
self.fx.write_manifest()
|
||||
code, out = self.fx.run(self.fx.perfect())
|
||||
self.assertEqual(code, 2)
|
||||
self.assertIn("not in the manifest", out)
|
||||
|
||||
def test_no_command_is_a_usage_error(self) -> None:
|
||||
out = io.StringIO()
|
||||
with redirect_stdout(out):
|
||||
code = filetype_gate.main(["--manifest", str(self.fx.manifest), "--pairs", str(self.fx.pairs),
|
||||
"--sources", str(self.fx.sources), "--cases", str(self.fx.cases)])
|
||||
self.assertEqual(code, 2)
|
||||
self.assertIn("usage error", out.getvalue())
|
||||
|
||||
|
||||
class LocatorTest(unittest.TestCase):
|
||||
def test_equal_locators_match(self) -> None:
|
||||
self.assertTrue(filetype_gate.same_locator("pdf", {"page": 3}, {"page": 3}))
|
||||
self.assertTrue(filetype_gate.same_locator("docx", {"paragraph": 12}, {"paragraph": 12}))
|
||||
self.assertTrue(filetype_gate.same_locator("md", {"line": 7}, {"line": 7}))
|
||||
|
||||
def test_a_boolean_is_not_a_number(self) -> None:
|
||||
self.assertFalse(filetype_gate.same_locator("pdf", {"page": 1}, {"page": True}))
|
||||
|
||||
def test_cell_references_ignore_case_but_sheet_names_do_not(self) -> None:
|
||||
self.assertTrue(filetype_gate.same_locator("xlsx", {"sheet": "T1", "cell": "B7"}, {"sheet": "T1", "cell": "b7"}))
|
||||
self.assertFalse(filetype_gate.same_locator("xlsx", {"sheet": "T1", "cell": "B7"}, {"sheet": "t1", "cell": "B7"}))
|
||||
|
||||
def test_slide_notes_default_to_false(self) -> None:
|
||||
self.assertTrue(filetype_gate.same_locator("pptx", {"slide": 2, "notes": False}, {"slide": 2}))
|
||||
self.assertFalse(filetype_gate.same_locator("pptx", {"slide": 2, "notes": True}, {"slide": 2}))
|
||||
|
||||
def test_extra_or_missing_keys_do_not_match(self) -> None:
|
||||
self.assertFalse(filetype_gate.same_locator("md", {"line": 7}, {"line": 7, "column": 3}))
|
||||
self.assertFalse(filetype_gate.same_locator("pdf", {"page": 3}, {}))
|
||||
self.assertFalse(filetype_gate.same_locator("pdf", {"page": 3}, "page 3"))
|
||||
|
||||
|
||||
class FirstLocatorStubTest(unittest.TestCase):
|
||||
"""The stub that always answers the first page, paragraph, cell, slide or line."""
|
||||
|
||||
def answer(self, path: Path) -> dict[str, object]:
|
||||
import subprocess
|
||||
q = path.parent / "q.json"
|
||||
q.write_text(json.dumps({"queries": [{"id": "a", "quote": "anything"}]}))
|
||||
proc = subprocess.run([PY, str(STUBS / "first_locator.py"), str(path), str(q)],
|
||||
capture_output=True, text=True, check=True)
|
||||
found: dict[str, object] = json.loads(proc.stdout)["answers"][0]["found"][0]
|
||||
return found
|
||||
|
||||
def test_first_locator_per_type(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
d = Path(tmp)
|
||||
(d / "a.md").write_text("x\n")
|
||||
self.assertEqual(self.answer(d / "a.md"), {"locator": {"line": 1}, "hidden": False})
|
||||
(d / "a.pdf").write_bytes(b"%PDF-1.4\n")
|
||||
self.assertEqual(self.answer(d / "a.pdf"), {"locator": {"page": 1}, "hidden": False})
|
||||
(d / "a.pptx").write_bytes(b"")
|
||||
self.assertEqual(self.answer(d / "a.pptx"), {"locator": {"slide": 1, "notes": False}, "hidden": False})
|
||||
(d / "a.docx").write_bytes(b"")
|
||||
self.assertEqual(self.answer(d / "a.docx"), {"locator": {"paragraph": 1}, "hidden": False})
|
||||
|
||||
def test_first_cell_is_a1_on_the_first_sheet_in_workbook_order(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
p = Path(tmp) / "a.xlsx"
|
||||
with zipfile.ZipFile(p, "w") as z:
|
||||
z.writestr("xl/workbook.xml",
|
||||
'<workbook xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main">'
|
||||
'<sheets><sheet name="Summary & notes" sheetId="2"/><sheet name="Data" sheetId="1"/>'
|
||||
'</sheets></workbook>')
|
||||
self.assertEqual(self.answer(p), {"locator": {"sheet": "Summary & notes", "cell": "A1"}, "hidden": False})
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
108
gates/tests/test_keyform.py
Normal file
108
gates/tests/test_keyform.py
Normal file
|
|
@ -0,0 +1,108 @@
|
|||
"""Tests for gates/keyform.py: counts answer keys that are wording rather than a fact (E2c).
|
||||
|
||||
Every fixture is synthetic text written here. No corpus text is used (CLAUDE.md rule 3).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
import io
|
||||
import json
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
|
||||
import keyform # noqa: E402
|
||||
|
||||
|
||||
class WordingTests(unittest.TestCase):
|
||||
def test_single_words_and_literals_are_facts(self) -> None:
|
||||
for key in ("lantern", "/north", "3.1.4", "GATE_TOLL", "--toll-mode dawn", "copper-coins"):
|
||||
self.assertFalse(keyform.is_wording(key), key)
|
||||
|
||||
def test_a_name_of_several_words_without_function_words_is_a_fact(self) -> None:
|
||||
for key in ("Toll mode", "north gate keeper", "Bridge (upper)"):
|
||||
self.assertFalse(keyform.is_wording(key), key)
|
||||
|
||||
def test_a_conjunction_makes_a_key_wording(self) -> None:
|
||||
self.assertTrue(keyform.is_wording("/north or /south"))
|
||||
self.assertTrue(keyform.is_wording("dawn and dusk"))
|
||||
|
||||
def test_a_pronoun_or_article_makes_a_key_wording(self) -> None:
|
||||
self.assertTrue(keyform.is_wording("pay your toll"))
|
||||
self.assertTrue(keyform.is_wording("The toll rises"))
|
||||
self.assertTrue(keyform.is_wording("a lantern maker"))
|
||||
|
||||
def test_prepositions_auxiliaries_negations_and_particles_make_a_key_wording(self) -> None:
|
||||
for key in ("toll for carters", "gate is closed", "can't open", "no toll", "only at dawn", "still open"):
|
||||
self.assertTrue(keyform.is_wording(key), key)
|
||||
|
||||
def test_function_words_inside_backticks_are_code_not_wording(self) -> None:
|
||||
self.assertFalse(keyform.is_wording("`open --if the gate` flag"))
|
||||
self.assertTrue(keyform.is_wording("`toll` for carters"))
|
||||
|
||||
|
||||
def md_set(keys: list[str]) -> dict[str, Any]:
|
||||
return {"schema": "okf-wiki/multidoc-set/1", "questions": [
|
||||
{"id": "q01", "question": "?", "evidence": [], "answer_keys": [{"file": "a.md", "text": k} for k in keys]}]}
|
||||
|
||||
|
||||
def wd_set() -> dict[str, Any]:
|
||||
return {"schema": "okf-wiki/wholedoc-set/1", "questions": [
|
||||
{"id": "w01", "type": "whole", "question": "?", "file": "a.md", "evidence": [],
|
||||
"answer_keys": [{"text": "dawn"}, {"text": "pay your toll"}, {"text": "copper coins"}]},
|
||||
{"id": "w02", "type": "count", "question": "?", "file": "a.md", "evidence": [], "count": 6, "item_pattern": "x"}]}
|
||||
|
||||
|
||||
class CountTests(unittest.TestCase):
|
||||
def test_counts_every_key_of_either_schema_and_lists_the_wording_ones(self) -> None:
|
||||
rep = keyform.count_keys(md_set(["dawn", "/north or /south", "Toll mode"])["questions"])
|
||||
self.assertEqual((rep.wording, rep.keys), (1, 3))
|
||||
self.assertEqual(rep.listed, ["q01: /north or /south"])
|
||||
rep = keyform.count_keys(wd_set()["questions"])
|
||||
self.assertEqual((rep.wording, rep.keys), (1, 3))
|
||||
|
||||
|
||||
class CliTests(unittest.TestCase):
|
||||
def run_cli(self, *sets: dict[str, Any]) -> tuple[int, str]:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
args: list[str] = []
|
||||
for n, data in enumerate(sets):
|
||||
p = Path(tmp) / f"set{n}.json"
|
||||
p.write_text(json.dumps(data), encoding="utf-8")
|
||||
args += ["--set", str(p)]
|
||||
out = io.StringIO()
|
||||
with contextlib.redirect_stdout(out), contextlib.redirect_stderr(io.StringIO()):
|
||||
code = keyform.main(args)
|
||||
return code, out.getvalue()
|
||||
|
||||
def test_wording_keys_fail_with_x_of_n_over_all_sets(self) -> None:
|
||||
code, out = self.run_cli(md_set(["dawn", "/north or /south"]), wd_set())
|
||||
self.assertEqual(code, 1)
|
||||
self.assertIn("answer keys that are wording: 2 of 5", out)
|
||||
self.assertIn("FAIL", out)
|
||||
|
||||
def test_no_wording_keys_pass(self) -> None:
|
||||
code, out = self.run_cli(md_set(["dawn", "/north", "/south"]))
|
||||
self.assertEqual(code, 0)
|
||||
self.assertIn("answer keys that are wording: 0 of 3", out)
|
||||
self.assertIn("PASS", out)
|
||||
|
||||
def test_no_keys_at_all_is_a_fail_not_a_pass(self) -> None:
|
||||
code, out = self.run_cli(md_set([]))
|
||||
self.assertEqual(code, 1)
|
||||
|
||||
def test_unreadable_set_cannot_run(self) -> None:
|
||||
err = io.StringIO()
|
||||
with contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(err):
|
||||
code = keyform.main(["--set", "/nonexistent/set.json"])
|
||||
self.assertEqual(code, 1)
|
||||
self.assertIn("FAIL", err.getvalue())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
541
gates/tests/test_multidoc.py
Normal file
541
gates/tests/test_multidoc.py
Normal file
|
|
@ -0,0 +1,541 @@
|
|||
"""Tests for gates/multidoc.py: the multi-document set checker, the held-out split and the answer gate.
|
||||
|
||||
Every fixture is synthetic text written here. No corpus text is used (CLAUDE.md rule 3).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
|
||||
import multidoc # noqa: E402
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
|
||||
SOURCES = {
|
||||
"alpha.md": "# Alpha\n\nThe widget timeout is 45 seconds by default.\nSet `WIDGET_MODE=fast` to skip checks.\n",
|
||||
"beta.md": "# Beta\n\nBeta adds the --frobnicate flag for batch runs.\nIt was introduced in the spring build.\n",
|
||||
"gamma.md": "# Gamma\n\nGamma stores its cache under the .gamma-cache directory.\n",
|
||||
"delta.md": "# Delta\n\nDelta rejects requests larger than 12 megabytes with error E_TOO_BIG.\n",
|
||||
"epsilon.md": "# Epsilon\n\nEpsilon logs every call to the audit trail named ledger-7.\n",
|
||||
}
|
||||
|
||||
|
||||
def good_question(qid: str = "x01") -> dict[str, Any]:
|
||||
return {
|
||||
"id": qid,
|
||||
"question": "How do the four synthetic tools fit together?",
|
||||
"evidence": [
|
||||
{"file": "alpha.md", "span": "The widget timeout is 45 seconds by default."},
|
||||
{"file": "beta.md", "span": "Beta adds the --frobnicate flag for batch runs."},
|
||||
{"file": "gamma.md", "span": "Gamma stores its cache under the .gamma-cache directory."},
|
||||
{"file": "delta.md", "span": "rejects requests larger than 12 megabytes with error E_TOO_BIG"},
|
||||
],
|
||||
"answer_keys": [
|
||||
{"file": "alpha.md", "text": "45 seconds"},
|
||||
{"file": "beta.md", "text": "--frobnicate"},
|
||||
{"file": "gamma.md", "text": ".gamma-cache"},
|
||||
{"file": "delta.md", "text": "E_TOO_BIG"},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def perfect_answer(q: dict[str, Any]) -> dict[str, Any]:
|
||||
keys = " ".join(k["text"] for k in q["answer_keys"])
|
||||
return {
|
||||
"id": q["id"],
|
||||
"answer": "The answer mentions " + keys + ".",
|
||||
"citations": [{"file": e["file"], "quote": e["span"]} for e in q["evidence"]],
|
||||
}
|
||||
|
||||
|
||||
class Workspace:
|
||||
"""A temporary sources directory and helpers to write sets, manifests and answers."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._tmp = tempfile.TemporaryDirectory()
|
||||
self.root = Path(self._tmp.name)
|
||||
self.sources = self.root / "sources"
|
||||
self.sources.mkdir()
|
||||
for name, text in SOURCES.items():
|
||||
(self.sources / name).write_text(text, encoding="utf-8")
|
||||
|
||||
def close(self) -> None:
|
||||
self._tmp.cleanup()
|
||||
|
||||
def write_set(self, questions: list[dict[str, Any]], name: str = "set.json") -> Path:
|
||||
path = self.root / name
|
||||
path.write_text(json.dumps({"schema": multidoc.SET_SCHEMA, "questions": questions}), encoding="utf-8")
|
||||
return path
|
||||
|
||||
def write_json(self, name: str, data: Any) -> Path:
|
||||
path = self.root / name
|
||||
path.write_text(json.dumps(data), encoding="utf-8")
|
||||
return path
|
||||
|
||||
|
||||
def run(argv: list[str]) -> tuple[int, str]:
|
||||
out = io.StringIO()
|
||||
err = io.StringIO()
|
||||
with contextlib.redirect_stdout(out), contextlib.redirect_stderr(err):
|
||||
code = multidoc.main(argv)
|
||||
return code, out.getvalue() + err.getvalue()
|
||||
|
||||
|
||||
def sha256_file(path: Path) -> str:
|
||||
return hashlib.sha256(path.read_bytes()).hexdigest()
|
||||
|
||||
|
||||
class CheckTests(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
self.ws = Workspace()
|
||||
|
||||
def tearDown(self) -> None:
|
||||
self.ws.close()
|
||||
|
||||
def check(self, questions: list[dict[str, Any]]) -> tuple[int, str]:
|
||||
path = self.ws.write_set(questions)
|
||||
return run(["check", "--set", str(path), "--sources", str(self.ws.sources)])
|
||||
|
||||
def test_good_set_passes_with_counts_from_the_set(self) -> None:
|
||||
code, out = self.check([good_question("x01"), good_question("x02")])
|
||||
self.assertEqual(code, 0, out)
|
||||
self.assertIn("spans verbatim: 8 of 8", out)
|
||||
self.assertIn("questions with every span verbatim: 2 of 2", out)
|
||||
self.assertIn("questions with >= 4 distinct files: 2 of 2", out)
|
||||
self.assertIn("questions with a key per file: 2 of 2", out)
|
||||
self.assertIn("PASS", out)
|
||||
|
||||
def test_span_one_character_off_fails(self) -> None:
|
||||
q = good_question()
|
||||
q["evidence"][0]["span"] = "The widget timeout is 46 seconds by default."
|
||||
code, out = self.check([q])
|
||||
self.assertEqual(code, 1)
|
||||
self.assertIn("spans verbatim: 3 of 4", out)
|
||||
self.assertIn("questions with every span verbatim: 0 of 1", out)
|
||||
self.assertIn("FAIL", out)
|
||||
|
||||
def test_span_match_is_exact_not_whitespace_normalised(self) -> None:
|
||||
q = good_question()
|
||||
q["evidence"][0]["span"] = "The widget timeout is 45 seconds by default."
|
||||
code, out = self.check([q])
|
||||
self.assertEqual(code, 1)
|
||||
self.assertIn("spans verbatim: 3 of 4", out)
|
||||
|
||||
def test_three_distinct_files_fail_the_minimum(self) -> None:
|
||||
q = good_question()
|
||||
q["evidence"][3] = {"file": "alpha.md", "span": "Set `WIDGET_MODE=fast` to skip checks."}
|
||||
q["answer_keys"][3] = {"file": "alpha.md", "text": "WIDGET_MODE=fast"}
|
||||
code, out = self.check([q])
|
||||
self.assertEqual(code, 1)
|
||||
self.assertIn("questions with >= 4 distinct files: 0 of 1", out)
|
||||
|
||||
def test_file_missing_from_sources_fails(self) -> None:
|
||||
q = good_question()
|
||||
q["evidence"][0]["file"] = "missing.md"
|
||||
q["answer_keys"][0]["file"] = "missing.md"
|
||||
code, out = self.check([q])
|
||||
self.assertEqual(code, 1)
|
||||
self.assertIn("spans verbatim: 3 of 4", out)
|
||||
|
||||
def test_path_outside_sources_is_an_error(self) -> None:
|
||||
for bad in ("../alpha.md", "sub/alpha.md", "/etc/hosts"):
|
||||
q = good_question()
|
||||
q["evidence"][0]["file"] = bad
|
||||
q["answer_keys"][0]["file"] = bad
|
||||
code, out = self.check([q])
|
||||
self.assertEqual(code, 1, bad)
|
||||
self.assertIn("not a plain file name", out)
|
||||
|
||||
def test_key_outside_its_files_spans_fails(self) -> None:
|
||||
q = good_question()
|
||||
q["answer_keys"][0]["text"] = "WIDGET_MODE" # in alpha.md, but not in the cited span
|
||||
code, out = self.check([q])
|
||||
self.assertEqual(code, 1)
|
||||
self.assertIn("questions with a key per file: 0 of 1", out)
|
||||
|
||||
def test_key_match_ignores_case(self) -> None:
|
||||
q = good_question()
|
||||
q["answer_keys"][3]["text"] = "e_too_big"
|
||||
code, out = self.check([q])
|
||||
self.assertEqual(code, 0, out)
|
||||
|
||||
def test_evidence_file_without_a_key_fails(self) -> None:
|
||||
q = good_question()
|
||||
del q["answer_keys"][2]
|
||||
code, out = self.check([q])
|
||||
self.assertEqual(code, 1)
|
||||
self.assertIn("questions with a key per file: 0 of 1", out)
|
||||
|
||||
def test_duplicate_ids_fail(self) -> None:
|
||||
code, out = self.check([good_question("x01"), good_question("x01")])
|
||||
self.assertEqual(code, 1)
|
||||
self.assertIn("duplicate id", out)
|
||||
|
||||
def test_empty_set_is_a_fail_not_a_pass(self) -> None:
|
||||
code, out = self.check([])
|
||||
self.assertEqual(code, 1)
|
||||
self.assertIn("FAIL", out)
|
||||
|
||||
def test_a_wording_key_fails_even_inside_its_span(self) -> None:
|
||||
q = good_question()
|
||||
q["answer_keys"][0] = {"file": "alpha.md", "text": "is 45 seconds by default"}
|
||||
code, out = self.check([q])
|
||||
self.assertEqual(code, 1)
|
||||
self.assertIn("questions with a key per file: 1 of 1", out)
|
||||
self.assertIn("answer keys that are wording: 1 of 4", out)
|
||||
|
||||
def test_good_set_has_no_wording_keys(self) -> None:
|
||||
code, out = self.check([good_question()])
|
||||
self.assertIn("answer keys that are wording: 0 of 4", out)
|
||||
|
||||
def test_short_span_and_short_key_fail(self) -> None:
|
||||
q = good_question()
|
||||
q["evidence"][0]["span"] = "45 seconds"
|
||||
code, out = self.check([q])
|
||||
self.assertEqual(code, 1)
|
||||
self.assertIn("span length", out)
|
||||
q = good_question()
|
||||
q["answer_keys"][0]["text"] = "se"
|
||||
code, out = self.check([q])
|
||||
self.assertEqual(code, 1)
|
||||
self.assertIn("key length", out)
|
||||
|
||||
def test_missing_sources_directory_cannot_run(self) -> None:
|
||||
path = self.ws.write_set([good_question()])
|
||||
code, out = run(["check", "--set", str(path), "--sources", str(self.ws.root / "nope")])
|
||||
self.assertEqual(code, 1)
|
||||
self.assertIn("FAIL", out)
|
||||
|
||||
def test_unreadable_set_cannot_run(self) -> None:
|
||||
path = self.ws.root / "broken.json"
|
||||
path.write_text("{not json", encoding="utf-8")
|
||||
code, out = run(["check", "--set", str(path), "--sources", str(self.ws.sources)])
|
||||
self.assertEqual(code, 1)
|
||||
self.assertIn("FAIL", out)
|
||||
|
||||
|
||||
def test_a_negated_key_must_stand_negated_in_its_span(self) -> None:
|
||||
q = good_question()
|
||||
q["answer_keys"][0]["negated"] = True
|
||||
code, out = self.check([q])
|
||||
self.assertEqual(code, 1)
|
||||
self.assertIn("negated keys negated in their span: 0 of 1", out)
|
||||
|
||||
def test_a_negated_key_negated_in_its_span_passes(self) -> None:
|
||||
(self.ws.sources / "alpha.md").write_text("The widget is not synced back to the hub.\n", encoding="utf-8")
|
||||
q = good_question()
|
||||
q["evidence"][0]["span"] = "The widget is not synced back to the hub."
|
||||
q["answer_keys"][0] = {"file": "alpha.md", "text": "synced", "negated": True}
|
||||
code, out = self.check([q])
|
||||
self.assertEqual(code, 0, out)
|
||||
self.assertIn("negated keys negated in their span: 1 of 1", out)
|
||||
|
||||
def test_negated_must_be_true_or_absent(self) -> None:
|
||||
q = good_question()
|
||||
q["answer_keys"][0]["negated"] = "yes"
|
||||
code, out = self.check([q])
|
||||
self.assertEqual(code, 1)
|
||||
self.assertIn("negated", out)
|
||||
|
||||
|
||||
class NegatedKeyTests(unittest.TestCase):
|
||||
KEY = {"file": "a.md", "text": "synced", "negated": True}
|
||||
|
||||
def test_a_negation_within_six_words_before_the_key(self) -> None:
|
||||
for text in ("Changes are not synced back.", "It isn't synced.", "It isn\u2019t synced.",
|
||||
"Changes are no longer synced.", "It is never, ever synced.", "Not one two three four five synced.",
|
||||
"Some are synced; others are not synced."):
|
||||
self.assertTrue(multidoc.key_holds(self.KEY, text), text)
|
||||
|
||||
def test_no_negation_or_one_out_of_reach_fails(self) -> None:
|
||||
for text in ("Changes are synced back.", "It is not late. Changes are synced.", "Not late; synced.",
|
||||
"Not one two three four five six synced.", "Synced, but not back.", "Not here:\nsynced."):
|
||||
self.assertFalse(multidoc.key_holds(self.KEY, text), text)
|
||||
|
||||
def test_every_negation_in_the_list(self) -> None:
|
||||
for neg in ("not", "no", "never", "none", "neither", "nor", "without", "cannot", "can't", "doesn't", "won't"):
|
||||
self.assertTrue(multidoc.key_holds(self.KEY, f"It {neg} synced."), neg)
|
||||
|
||||
def test_an_unmarked_key_holds_with_or_without_a_negation(self) -> None:
|
||||
key = {"file": "a.md", "text": "synced"}
|
||||
self.assertTrue(multidoc.key_holds(key, "Changes are synced back."))
|
||||
self.assertTrue(multidoc.key_holds(key, "Changes are not synced back."))
|
||||
|
||||
def test_the_gate_scores_a_negated_key_by_its_negation(self) -> None:
|
||||
ws = Workspace()
|
||||
try:
|
||||
(ws.sources / "alpha.md").write_text("The widget is not synced back to the hub.\n", encoding="utf-8")
|
||||
q = good_question()
|
||||
q["evidence"][0]["span"] = "The widget is not synced back to the hub."
|
||||
q["answer_keys"][0] = {"file": "alpha.md", "text": "synced", "negated": True}
|
||||
cache = multidoc.SourceCache(ws.sources)
|
||||
right = perfect_answer(q)
|
||||
right["answer"] = "The widget is not synced; --frobnicate, .gamma-cache, E_TOO_BIG."
|
||||
wrong = dict(right, answer="The widget is synced; --frobnicate, .gamma-cache, E_TOO_BIG.")
|
||||
self.assertTrue(multidoc.score_one(q, right, cache).answer_ok)
|
||||
self.assertFalse(multidoc.score_one(q, wrong, cache).answer_ok)
|
||||
finally:
|
||||
ws.close()
|
||||
|
||||
|
||||
class SplitTests(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
self.ws = Workspace()
|
||||
self.questions = [good_question(f"x{i:02d}") for i in range(1, 11)]
|
||||
for i, q in enumerate(self.questions):
|
||||
q["question"] = f"Synthetic question number {i + 1}?"
|
||||
self.all_path = self.ws.write_set(self.questions, "all.json")
|
||||
|
||||
def tearDown(self) -> None:
|
||||
self.ws.close()
|
||||
|
||||
def split(self, seed: int, tag: str, held_dir: Path | None = None) -> tuple[int, str, Path, Path, Path]:
|
||||
dev = self.ws.root / f"dev-{tag}.json"
|
||||
held = (held_dir or self.ws.root / f"held-{tag}") / "held.json"
|
||||
manifest = self.ws.root / f"manifest-{tag}.json"
|
||||
code, out = run([
|
||||
"split", "--set", str(self.all_path), "--sources", str(self.ws.sources), "--seed", str(seed),
|
||||
"--held-out-count", "3", "--dev-out", str(dev), "--held-out", str(held), "--manifest", str(manifest),
|
||||
])
|
||||
return code, out, dev, held, manifest
|
||||
|
||||
def load(self, path: Path) -> list[dict[str, Any]]:
|
||||
data: dict[str, Any] = json.loads(path.read_text(encoding="utf-8"))
|
||||
questions: list[dict[str, Any]] = data["questions"]
|
||||
return questions
|
||||
|
||||
def test_split_counts_renumbers_and_partitions(self) -> None:
|
||||
code, out, dev, held, _ = self.split(7, "a")
|
||||
self.assertEqual(code, 0, out)
|
||||
dev_q, held_q = self.load(dev), self.load(held)
|
||||
self.assertEqual((len(dev_q), len(held_q)), (7, 3))
|
||||
self.assertEqual([q["id"] for q in dev_q], [f"d{i:02d}" for i in range(1, 8)])
|
||||
self.assertEqual([q["id"] for q in held_q], [f"h{i:02d}" for i in range(1, 4)])
|
||||
texts = sorted(q["question"] for q in dev_q + held_q)
|
||||
self.assertEqual(texts, sorted(q["question"] for q in self.questions))
|
||||
|
||||
def test_same_seed_same_draw_other_seed_other_draw(self) -> None:
|
||||
_, _, _, held_a, _ = self.split(7, "a")
|
||||
_, _, _, held_b, _ = self.split(7, "b")
|
||||
self.assertEqual(held_a.read_bytes(), held_b.read_bytes())
|
||||
draws = set()
|
||||
for seed in range(1, 6):
|
||||
_, _, _, held, _ = self.split(seed, f"s{seed}")
|
||||
draws.add(tuple(q["question"] for q in self.load(held)))
|
||||
self.assertGreater(len(draws), 1)
|
||||
|
||||
def test_manifest_carries_hashes_and_nothing_about_the_held_out_questions(self) -> None:
|
||||
code, out, dev, held, manifest = self.split(7, "a")
|
||||
self.assertEqual(code, 0, out)
|
||||
data = json.loads(manifest.read_text(encoding="utf-8"))
|
||||
self.assertEqual(data["seed"], 7)
|
||||
self.assertEqual(data["dev"]["sha256"], sha256_file(dev))
|
||||
self.assertEqual(data["dev"]["questions"], 7)
|
||||
self.assertEqual(data["held_out"]["sha256"], sha256_file(held))
|
||||
self.assertEqual(data["held_out"]["questions"], 3)
|
||||
self.assertEqual(data["sources"]["files"], len(SOURCES))
|
||||
self.assertEqual(data["sources"]["tree_sha256"], multidoc.tree_sha256(self.ws.sources))
|
||||
manifest_text = manifest.read_text(encoding="utf-8")
|
||||
for q in self.load(held):
|
||||
self.assertNotIn(q["question"], manifest_text)
|
||||
self.assertNotIn(q["question"], out)
|
||||
self.assertNotIn(str(held.parent), manifest_text)
|
||||
|
||||
def test_refuses_held_out_inside_the_repository(self) -> None:
|
||||
code, out, _, _, _ = self.split(7, "in", held_dir=REPO_ROOT / "gates-local" / "should-not-exist")
|
||||
self.assertEqual(code, 1)
|
||||
self.assertIn("inside the repository", out)
|
||||
self.assertFalse((REPO_ROOT / "gates-local" / "should-not-exist").exists())
|
||||
|
||||
def test_refuses_to_overwrite_a_frozen_held_out_file(self) -> None:
|
||||
code, out, _, _, _ = self.split(7, "a")
|
||||
self.assertEqual(code, 0, out)
|
||||
code, out, _, _, _ = self.split(7, "a")
|
||||
self.assertEqual(code, 1)
|
||||
self.assertIn("already exists", out)
|
||||
|
||||
def test_refuses_a_set_that_fails_the_check(self) -> None:
|
||||
self.questions[0]["evidence"][0]["span"] = "This sentence is not in any source file."
|
||||
self.all_path = self.ws.write_set(self.questions, "all.json")
|
||||
code, out, dev, held, _ = self.split(7, "bad")
|
||||
self.assertEqual(code, 1)
|
||||
self.assertFalse(dev.exists())
|
||||
self.assertFalse(held.exists())
|
||||
|
||||
|
||||
class GateTests(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
self.ws = Workspace()
|
||||
self.questions = [good_question("d01"), good_question("d02")]
|
||||
self.set_path = self.ws.write_set(self.questions, "dev.json")
|
||||
self.manifest = self.ws.write_json("manifest.json", {
|
||||
"schema": multidoc.MANIFEST_SCHEMA,
|
||||
"seed": 1,
|
||||
"dev": {"sha256": sha256_file(self.set_path), "questions": 2},
|
||||
"held_out": {"sha256": "0" * 64, "questions": 1},
|
||||
"sources": {"files": len(SOURCES), "tree_sha256": multidoc.tree_sha256(self.ws.sources)},
|
||||
})
|
||||
|
||||
def tearDown(self) -> None:
|
||||
self.ws.close()
|
||||
|
||||
def gate(self, answers: Path, arm: str = "collection") -> tuple[int, str]:
|
||||
return run([
|
||||
"gate", "--set", str(self.set_path), "--manifest", str(self.manifest), "--sources", str(self.ws.sources),
|
||||
"--arm", arm, "--answers", str(answers),
|
||||
])
|
||||
|
||||
def answers(self, entries: list[dict[str, Any]], arm: str = "collection") -> Path:
|
||||
return self.ws.write_json("answers.json", {"arm": arm, "answers": entries})
|
||||
|
||||
def test_missing_arm_is_fail_closed_at_zero(self) -> None:
|
||||
code, out = self.gate(self.ws.root / "no-such-arm.json")
|
||||
self.assertEqual(code, 1)
|
||||
self.assertIn("correct answer and source: 0 of 2", out)
|
||||
self.assertIn("fail-closed", out)
|
||||
|
||||
def test_perfect_answers_score_all(self) -> None:
|
||||
code, out = self.gate(self.answers([perfect_answer(q) for q in self.questions]))
|
||||
self.assertEqual(code, 0, out)
|
||||
self.assertIn("correct answer and source: 2 of 2", out)
|
||||
|
||||
def test_answer_missing_a_key_is_wrong(self) -> None:
|
||||
a = perfect_answer(self.questions[0])
|
||||
a["answer"] = a["answer"].replace("E_TOO_BIG", "a big error")
|
||||
code, out = self.gate(self.answers([a, perfect_answer(self.questions[1])]))
|
||||
self.assertEqual(code, 0, out)
|
||||
self.assertIn("correct answer: 1 of 2", out)
|
||||
self.assertIn("correct answer and source: 1 of 2", out)
|
||||
|
||||
def test_answer_key_match_ignores_case_backticks_and_whitespace(self) -> None:
|
||||
a = perfect_answer(self.questions[0])
|
||||
a["answer"] = "Use `--FROBNICATE`, wait 45\n seconds, look in .gamma-cache, expect e_too_big."
|
||||
code, out = self.gate(self.answers([a, perfect_answer(self.questions[1])]))
|
||||
self.assertIn("correct answer: 2 of 2", out)
|
||||
|
||||
def test_citation_not_verbatim_is_wrong_source(self) -> None:
|
||||
a = perfect_answer(self.questions[0])
|
||||
a["citations"][1]["quote"] = "Beta adds a flag that nobody documented."
|
||||
code, out = self.gate(self.answers([a, perfect_answer(self.questions[1])]))
|
||||
self.assertIn("correct source: 1 of 2", out)
|
||||
self.assertIn("correct answer and source: 1 of 2", out)
|
||||
|
||||
def test_citation_whitespace_is_normalised(self) -> None:
|
||||
a = perfect_answer(self.questions[0])
|
||||
a["citations"][0]["quote"] = "The widget timeout is\n45 seconds by default."
|
||||
code, out = self.gate(self.answers([a, perfect_answer(self.questions[1])]))
|
||||
self.assertIn("correct source: 2 of 2", out)
|
||||
|
||||
def test_citations_missing_one_file_is_wrong_source(self) -> None:
|
||||
a = perfect_answer(self.questions[0])
|
||||
del a["citations"][2]
|
||||
code, out = self.gate(self.answers([a, perfect_answer(self.questions[1])]))
|
||||
self.assertIn("correct source: 1 of 2", out)
|
||||
|
||||
def test_citation_outside_sources_counts_for_nothing(self) -> None:
|
||||
a = perfect_answer(self.questions[0])
|
||||
a["citations"][0]["file"] = "../sources/alpha.md"
|
||||
code, out = self.gate(self.answers([a, perfect_answer(self.questions[1])]))
|
||||
self.assertIn("correct source: 1 of 2", out)
|
||||
|
||||
def test_missing_entry_counts_wrong_and_fails(self) -> None:
|
||||
code, out = self.gate(self.answers([perfect_answer(self.questions[0])]))
|
||||
self.assertEqual(code, 1)
|
||||
self.assertIn("correct answer and source: 1 of 2", out)
|
||||
self.assertIn("missing answers: 1", out)
|
||||
|
||||
def test_set_that_differs_from_the_manifest_is_refused(self) -> None:
|
||||
self.set_path.write_text(self.set_path.read_text(encoding="utf-8") + "\n", encoding="utf-8")
|
||||
code, out = self.gate(self.answers([perfect_answer(q) for q in self.questions]))
|
||||
self.assertEqual(code, 1)
|
||||
self.assertIn("sha256", out)
|
||||
|
||||
def test_changed_sources_are_refused(self) -> None:
|
||||
(self.ws.sources / "alpha.md").write_text("changed\n", encoding="utf-8")
|
||||
code, out = self.gate(self.answers([perfect_answer(q) for q in self.questions]))
|
||||
self.assertEqual(code, 1)
|
||||
self.assertIn("sources", out)
|
||||
|
||||
def test_arm_name_mismatch_is_refused(self) -> None:
|
||||
code, out = self.gate(self.answers([perfect_answer(q) for q in self.questions], arm="filesearch"))
|
||||
self.assertEqual(code, 1)
|
||||
self.assertIn("arm", out)
|
||||
|
||||
def test_per_question_results_are_written_for_pairing(self) -> None:
|
||||
a = perfect_answer(self.questions[0])
|
||||
answers = self.answers([a])
|
||||
out_path = self.ws.root / "results.json"
|
||||
run([
|
||||
"gate", "--set", str(self.set_path), "--manifest", str(self.manifest), "--sources", str(self.ws.sources),
|
||||
"--arm", "collection", "--answers", str(answers), "--out", str(out_path),
|
||||
])
|
||||
results = json.loads(out_path.read_text(encoding="utf-8"))
|
||||
self.assertEqual(results["arm"], "collection")
|
||||
self.assertEqual([r["id"] for r in results["results"]], ["d01", "d02"])
|
||||
self.assertEqual([r["correct"] for r in results["results"]], [True, False])
|
||||
|
||||
|
||||
class NumberKeyTests(unittest.TestCase):
|
||||
"""A key that is a number matches as a number: digits or its English word, never inside a longer number."""
|
||||
|
||||
def test_a_number_key_matches_digits_or_word_but_not_inside_a_longer_number(self) -> None:
|
||||
for text in ("It retries 5 times.", "RETRIES=5", "five attempts", "(5)"):
|
||||
self.assertTrue(multidoc.key_in("5", text), text)
|
||||
for text in ("15 attempts", "version 2.1.5", "v5", "50", "5.2 seconds", "fifty"):
|
||||
self.assertFalse(multidoc.key_in("5", text), text)
|
||||
|
||||
def test_a_number_word_key_matches_the_digits(self) -> None:
|
||||
self.assertTrue(multidoc.key_in("three", "fails 3 times in a row"))
|
||||
self.assertTrue(multidoc.key_in("Three", "three times"))
|
||||
self.assertFalse(multidoc.key_in("three", "33 times"))
|
||||
|
||||
def test_a_percentage_key_matches_the_percentage(self) -> None:
|
||||
for text in ("at 10% of the window", "at 10 % of it", "10 percent"):
|
||||
self.assertTrue(multidoc.key_in("10%", text), text)
|
||||
for text in ("110%", "10 of them", "2.10%"):
|
||||
self.assertFalse(multidoc.key_in("10%", text), text)
|
||||
|
||||
def test_other_keys_still_match_as_normalised_text(self) -> None:
|
||||
self.assertTrue(multidoc.key_in("model version", "Pin their **model versions**."))
|
||||
self.assertTrue(multidoc.key_in("2.1.9", "since 2.1.92"))
|
||||
|
||||
def test_check_accepts_a_short_number_key_standing_alone_in_its_span(self) -> None:
|
||||
ws = Workspace()
|
||||
try:
|
||||
q = good_question()
|
||||
q["answer_keys"][3] = {"file": "delta.md", "text": "12"}
|
||||
code, out = run(["check", "--set", str(ws.write_set([q])), "--sources", str(ws.sources)])
|
||||
self.assertEqual(code, 0, out)
|
||||
q["answer_keys"][3] = {"file": "delta.md", "text": "2"}
|
||||
code, out = run(["check", "--set", str(ws.write_set([q])), "--sources", str(ws.sources)])
|
||||
self.assertEqual(code, 1)
|
||||
self.assertIn("questions with a key per file: 0 of 1", out)
|
||||
finally:
|
||||
ws.close()
|
||||
|
||||
def test_the_gate_scores_a_number_key_as_a_number(self) -> None:
|
||||
ws = Workspace()
|
||||
try:
|
||||
q = good_question()
|
||||
q["answer_keys"][3] = {"file": "delta.md", "text": "12"}
|
||||
cache = multidoc.SourceCache(ws.sources)
|
||||
right = perfect_answer(q)
|
||||
self.assertTrue(multidoc.score_one(q, right, cache).answer_ok)
|
||||
wrong = dict(right, answer=right["answer"].replace(" 12", " 112"))
|
||||
self.assertFalse(multidoc.score_one(q, wrong, cache).answer_ok)
|
||||
finally:
|
||||
ws.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
376
gates/tests/test_mutant_gate.py
Normal file
376
gates/tests/test_mutant_gate.py
Normal file
|
|
@ -0,0 +1,376 @@
|
|||
"""Tests for gates/mutant_gate.py, the fail-closed runner of the auditor's mutant gate.
|
||||
|
||||
Every fixture here is synthetic and neutral text. The real mutants live in gates-local/ (CLAUDE.md rule 3).
|
||||
Run: python3 -m unittest discover -s gates/tests
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
import tempfile
|
||||
import textwrap
|
||||
import unittest
|
||||
from contextlib import redirect_stdout
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
|
||||
|
||||
import mutant_gate # noqa: E402
|
||||
from okfwiki.audit import GROUP_OF # noqa: E402
|
||||
|
||||
PY = sys.executable
|
||||
|
||||
# Reads {bundle}/EXPECT: if it names a class, report exactly that class; otherwise report clean.
|
||||
PERFECT = """
|
||||
import json, sys, pathlib
|
||||
p = pathlib.Path(sys.argv[1]) / "EXPECT"
|
||||
cls = p.read_text().strip() if p.exists() else ""
|
||||
if cls:
|
||||
print(json.dumps({"findings": [{"check": cls}]})); sys.exit(1)
|
||||
print(json.dumps({"findings": []})); sys.exit(0)
|
||||
"""
|
||||
|
||||
REJECT_ALL = """
|
||||
import json, sys
|
||||
print(json.dumps({"findings": [{"check": "reject-all"}]})); sys.exit(1)
|
||||
"""
|
||||
|
||||
CRASH = """
|
||||
raise RuntimeError("boom")
|
||||
"""
|
||||
|
||||
WRONG_CLASS = """
|
||||
import json, sys, pathlib
|
||||
p = pathlib.Path(sys.argv[1]) / "EXPECT"
|
||||
if p.exists():
|
||||
print(json.dumps({"findings": [{"check": "broken-link"}]})); sys.exit(1)
|
||||
print(json.dumps({"findings": []})); sys.exit(0)
|
||||
"""
|
||||
|
||||
NOT_JSON = """
|
||||
import sys
|
||||
print("findings: lots"); sys.exit(1)
|
||||
"""
|
||||
|
||||
EXIT0_WITH_FINDINGS = """
|
||||
import json
|
||||
print(json.dumps({"findings": [{"check": "x"}]}))
|
||||
"""
|
||||
|
||||
EXIT1_NO_FINDINGS = """
|
||||
import json, sys
|
||||
print(json.dumps({"findings": []})); sys.exit(1)
|
||||
"""
|
||||
|
||||
SLEEPER = """
|
||||
import time
|
||||
time.sleep(10)
|
||||
"""
|
||||
|
||||
VANDAL = """
|
||||
import json, sys, pathlib
|
||||
for f in pathlib.Path(sys.argv[1]).iterdir():
|
||||
f.write_text("vandalised")
|
||||
print(json.dumps({"findings": []})); sys.exit(0)
|
||||
"""
|
||||
|
||||
SOURCES_ECHO = """
|
||||
import json, sys, pathlib
|
||||
ok = (pathlib.Path(sys.argv[2]) / "src" / "a.txt").read_text() == "alpha\\n"
|
||||
print(json.dumps({"findings": [] if ok else [{"check": "sources-missing"}]}))
|
||||
sys.exit(0 if ok else 1)
|
||||
"""
|
||||
|
||||
CASES: dict[str, tuple[str, str | None, dict[str, str]]] = {
|
||||
"baseline": ("known-positive", None, {"page.md": "---\ntype: Note\n---\nPlain text.\n"}),
|
||||
"m01-alpha": ("mutant", "class-alpha", {"page.md": "---\ntype: Note\n---\nA.\n", "EXPECT": "class-alpha\n"}),
|
||||
"m02-beta": ("mutant", "class-beta", {"page.md": "---\ntype: Note\n---\nB.\n", "EXPECT": "class-beta\n"}),
|
||||
"m03-beta": ("mutant", "class-beta", {"page.md": "---\ntype: Note\n---\nC.\n", "EXPECT": "class-beta\n"}),
|
||||
}
|
||||
|
||||
|
||||
class GateFixture:
|
||||
"""A synthetic manifest, case tree and sources tree in a temporary directory."""
|
||||
|
||||
def __init__(self, root: Path) -> None:
|
||||
self.root = root
|
||||
self.cases = root / "cases"
|
||||
self.sources = root / "sources"
|
||||
(self.sources / "src").mkdir(parents=True)
|
||||
(self.sources / "src" / "a.txt").write_text("alpha\n")
|
||||
entries = []
|
||||
for name, (kind, cls, files) in CASES.items():
|
||||
d = self.cases / name
|
||||
d.mkdir(parents=True)
|
||||
for rel, content in files.items():
|
||||
(d / rel).write_text(content)
|
||||
entries.append({"name": name, "kind": kind, "class": cls, "mutation": "synthetic",
|
||||
"files": mutant_gate.hash_tree(d)})
|
||||
self.manifest = root / "manifest.json"
|
||||
self.manifest.write_text(json.dumps({
|
||||
"schema": 1,
|
||||
"sources": [{"path": "src/a.txt", "sha256": mutant_gate.sha256_file(self.sources / "src" / "a.txt")}],
|
||||
"cases": entries,
|
||||
}))
|
||||
|
||||
def script(self, name: str, body: str) -> Path:
|
||||
p = self.root / f"{name}.py"
|
||||
p.write_text(textwrap.dedent(body))
|
||||
return p
|
||||
|
||||
def run(self, command: list[str], timeout: float = 5.0) -> tuple[int, str]:
|
||||
out = io.StringIO()
|
||||
with redirect_stdout(out):
|
||||
code = mutant_gate.main([
|
||||
"--manifest", str(self.manifest), "--cases", str(self.cases),
|
||||
"--sources", str(self.sources), "--timeout", str(timeout), "--", *command,
|
||||
])
|
||||
return code, out.getvalue()
|
||||
|
||||
|
||||
class MutantGateTest(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
self._tmp = tempfile.TemporaryDirectory()
|
||||
self.fx = GateFixture(Path(self._tmp.name))
|
||||
|
||||
def tearDown(self) -> None:
|
||||
self._tmp.cleanup()
|
||||
|
||||
# --- the three outcomes the order names ---------------------------------------------------------
|
||||
|
||||
def test_missing_auditor_catches_nothing_and_says_so(self) -> None:
|
||||
code, out = self.fx.run([str(self.fx.root / "no-such-auditor"), "{bundle}"])
|
||||
self.assertEqual(code, 1)
|
||||
self.assertIn("caught 0 of 3", out)
|
||||
self.assertIn("errors 4 of 4", out)
|
||||
self.assertIn("auditor not found", out)
|
||||
self.assertIn("verdict: RED", out)
|
||||
|
||||
def test_reject_all_catches_everything_but_raises_a_false_alarm(self) -> None:
|
||||
code, out = self.fx.run([PY, str(self.fx.script("reject", REJECT_ALL)), "{bundle}"])
|
||||
self.assertEqual(code, 1)
|
||||
self.assertIn("caught 3 of 3", out)
|
||||
self.assertIn("false alarms 1 of 1", out)
|
||||
self.assertIn("class matched 0 of 3", out)
|
||||
self.assertIn("verdict: RED", out)
|
||||
|
||||
def test_perfect_auditor_turns_the_gate_green(self) -> None:
|
||||
code, out = self.fx.run([PY, str(self.fx.script("perfect", PERFECT)), "{bundle}"])
|
||||
self.assertEqual(code, 0, out)
|
||||
self.assertIn("caught 3 of 3", out)
|
||||
self.assertIn("class matched 3 of 3", out)
|
||||
self.assertIn("false alarms 0 of 1", out)
|
||||
self.assertIn("errors 0 of 4", out)
|
||||
self.assertIn("verdict: GREEN", out)
|
||||
|
||||
# --- fail-closed: anything but a well-formed verdict is an error, never a catch --------------------
|
||||
|
||||
def test_crashing_auditor_is_an_error_not_a_catch(self) -> None:
|
||||
code, out = self.fx.run([PY, str(self.fx.script("crash", CRASH)), "{bundle}"])
|
||||
self.assertEqual(code, 1)
|
||||
self.assertIn("caught 0 of 3", out)
|
||||
self.assertIn("errors 4 of 4", out)
|
||||
|
||||
def test_output_that_is_not_json_is_an_error(self) -> None:
|
||||
_, out = self.fx.run([PY, str(self.fx.script("notjson", NOT_JSON)), "{bundle}"])
|
||||
self.assertIn("caught 0 of 3", out)
|
||||
self.assertIn("not valid JSON", out)
|
||||
|
||||
def test_exit_status_must_agree_with_findings(self) -> None:
|
||||
_, out0 = self.fx.run([PY, str(self.fx.script("e0", EXIT0_WITH_FINDINGS)), "{bundle}"])
|
||||
self.assertIn("caught 0 of 3", out0)
|
||||
self.assertIn("errors 4 of 4", out0)
|
||||
_, out1 = self.fx.run([PY, str(self.fx.script("e1", EXIT1_NO_FINDINGS)), "{bundle}"])
|
||||
self.assertIn("caught 0 of 3", out1)
|
||||
self.assertIn("errors 4 of 4", out1)
|
||||
|
||||
def test_timeout_is_an_error(self) -> None:
|
||||
_, out = self.fx.run([PY, str(self.fx.script("sleep", SLEEPER)), "{bundle}"], timeout=0.5)
|
||||
self.assertIn("caught 0 of 3", out)
|
||||
self.assertIn("timed out", out)
|
||||
|
||||
def test_alarm_for_the_wrong_class_is_caught_but_not_class_matched(self) -> None:
|
||||
code, out = self.fx.run([PY, str(self.fx.script("wrong", WRONG_CLASS)), "{bundle}"])
|
||||
self.assertEqual(code, 1)
|
||||
self.assertIn("caught 3 of 3", out)
|
||||
self.assertIn("class matched 0 of 3", out)
|
||||
self.assertIn("verdict: RED", out)
|
||||
|
||||
# --- reporting -------------------------------------------------------------------------------------
|
||||
|
||||
def test_report_lists_what_slipped_through_and_n_per_class(self) -> None:
|
||||
_, out = self.fx.run([str(self.fx.root / "no-such-auditor"), "{bundle}"])
|
||||
self.assertIn("slipped through: m01-alpha, m02-beta, m03-beta", out)
|
||||
self.assertIn("class-alpha: 1", out)
|
||||
self.assertIn("class-beta: 2", out)
|
||||
|
||||
def test_sources_placeholder_is_substituted(self) -> None:
|
||||
code, out = self.fx.run([PY, str(self.fx.script("src", SOURCES_ECHO)), "{bundle}", "{sources}"])
|
||||
self.assertIn("false alarms 0 of 1", out)
|
||||
self.assertNotIn("sources-missing", out)
|
||||
self.assertEqual(code, 1) # this auditor never catches a mutant
|
||||
|
||||
# --- the fixtures themselves are verified before anything runs --------------------------------------
|
||||
|
||||
def test_auditor_cannot_alter_the_fixtures(self) -> None:
|
||||
self.fx.run([PY, str(self.fx.script("vandal", VANDAL)), "{bundle}"])
|
||||
code, _ = self.fx.run([PY, str(self.fx.script("perfect", PERFECT)), "{bundle}"])
|
||||
self.assertEqual(code, 0)
|
||||
|
||||
def test_changed_case_file_stops_the_gate(self) -> None:
|
||||
(self.fx.cases / "m01-alpha" / "page.md").write_text("changed\n")
|
||||
code, out = self.fx.run([PY, str(self.fx.script("perfect", PERFECT)), "{bundle}"])
|
||||
self.assertEqual(code, 2)
|
||||
self.assertIn("sha256 mismatch", out)
|
||||
|
||||
def test_extra_or_missing_case_file_stops_the_gate(self) -> None:
|
||||
(self.fx.cases / "m02-beta" / "extra.md").write_text("x\n")
|
||||
code, out = self.fx.run([PY, str(self.fx.script("perfect", PERFECT)), "{bundle}"])
|
||||
self.assertEqual(code, 2)
|
||||
self.assertIn("unexpected file", out)
|
||||
(self.fx.cases / "m02-beta" / "extra.md").unlink()
|
||||
(self.fx.cases / "m03-beta" / "page.md").unlink()
|
||||
code, out = self.fx.run([PY, str(self.fx.script("perfect", PERFECT)), "{bundle}"])
|
||||
self.assertEqual(code, 2)
|
||||
self.assertIn("missing file", out)
|
||||
|
||||
def test_changed_source_stops_the_gate(self) -> None:
|
||||
(self.fx.sources / "src" / "a.txt").write_text("beta\n")
|
||||
code, out = self.fx.run([PY, str(self.fx.script("perfect", PERFECT)), "{bundle}"])
|
||||
self.assertEqual(code, 2)
|
||||
self.assertIn("source", out)
|
||||
|
||||
def test_manifest_needs_exactly_one_known_positive(self) -> None:
|
||||
data = json.loads(self.fx.manifest.read_text())
|
||||
data["cases"] = [c for c in data["cases"] if c["kind"] != "known-positive"]
|
||||
self.fx.manifest.write_text(json.dumps(data))
|
||||
code, out = self.fx.run([PY, str(self.fx.script("perfect", PERFECT)), "{bundle}"])
|
||||
self.assertEqual(code, 2)
|
||||
self.assertIn("known-positive", out)
|
||||
|
||||
def test_missing_command_is_a_usage_error(self) -> None:
|
||||
out = io.StringIO()
|
||||
with redirect_stdout(out):
|
||||
code = mutant_gate.main(["--manifest", str(self.fx.manifest), "--cases", str(self.fx.cases),
|
||||
"--sources", str(self.fx.sources)])
|
||||
self.assertEqual(code, 2)
|
||||
|
||||
|
||||
class TrackedManifestTest(unittest.TestCase):
|
||||
"""The tracked manifest must keep a mutant for every check the L1 row in PLAN.md promises (E1b)."""
|
||||
|
||||
LINK_CLASSES = ("broken-link", "orphan-page", "index-coverage", "link-misplaced")
|
||||
# Wave 1 order 4: criterion 6 under the operator's rule of 06.10.2026 (choice 2), contract K2 in
|
||||
# docs/bolge1/0-felles.local.md § 4. Every main-topic source has one main page (x-main-for), and every other
|
||||
# concept page has a link from a main page. The auditor emits none of these four classes yet; Step 7 adds them.
|
||||
K2_CLASSES = ("main-page-missing", "main-page-duplicate", "main-page-invalid", "no-main-page-link")
|
||||
# One mutant per case of order 4, point 3; the key is the case, the value the class its mutant must carry.
|
||||
K2_CASES = {
|
||||
"main-page-missing: the main page loses its mark for one source": "main-page-missing",
|
||||
"main-page-duplicate: a non-main page marks the same source": "main-page-duplicate",
|
||||
"main-page-invalid: the value is not a list": "main-page-invalid",
|
||||
"main-page-invalid: a value that is not among the page's own sources[].resource": "main-page-invalid",
|
||||
"main-page-invalid: a release-note path, in K1 but not main_topic": "main-page-invalid",
|
||||
"no-main-page-link: linked only from the other non-main page": "no-main-page-link",
|
||||
"no-main-page-link: linked only from index.md": "no-main-page-link",
|
||||
"no-main-page-link: linked only from log.md": "no-main-page-link",
|
||||
"no-main-page-link: linked only from itself": "no-main-page-link",
|
||||
"link-misplaced: the main page's only link to a non-main page sits in code": "link-misplaced",
|
||||
"link-misplaced: the main page's only link to a non-main page sits in a heading": "link-misplaced",
|
||||
"link-misplaced: the main page's only link to a non-main page sits in a footnote definition": "link-misplaced",
|
||||
# K2 point 2, last sentence: the role follows the valid values. The gate counts a mutant matched when its own
|
||||
# class is among the findings, so each case is one whose own class a wrong reading of the role cannot give.
|
||||
"main-page-duplicate: an invalid value does not take the main role from a page's valid values": "main-page-duplicate",
|
||||
"no-main-page-link: a page whose every value is invalid is not a main page": "no-main-page-link",
|
||||
}
|
||||
RELEASE = re.compile(r"^kilde/release-v\d+\.\d+\.\d+\.md$")
|
||||
|
||||
def path(self) -> Path:
|
||||
return Path(__file__).resolve().parents[1] / "mutants.manifest.json"
|
||||
|
||||
def manifest(self) -> mutant_gate.Manifest:
|
||||
return mutant_gate.load_manifest(self.path())
|
||||
|
||||
def k2(self) -> dict[str, Any]:
|
||||
k2 = json.loads(self.path().read_text(encoding="utf-8"))["provenance"].get("k2")
|
||||
self.assertIsInstance(k2, dict, "provenance.k2 is missing")
|
||||
return k2 # type: ignore[no-any-return]
|
||||
|
||||
def test_every_link_class_has_a_mutant(self) -> None:
|
||||
classes = {m.cls for m in self.manifest().mutants}
|
||||
for cls in self.LINK_CLASSES + self.K2_CLASSES:
|
||||
self.assertIn(cls, classes)
|
||||
|
||||
def test_every_k2_case_has_its_own_mutant(self) -> None:
|
||||
manifest = self.manifest()
|
||||
self.assertEqual(len(manifest.mutants), 37)
|
||||
self.assertEqual(manifest.known_positive.name, "baseline")
|
||||
by_name = {m.name: m for m in manifest.mutants}
|
||||
coverage = self.k2()["coverage"]
|
||||
named = []
|
||||
for case, cls in self.K2_CASES.items():
|
||||
self.assertIn(case, coverage)
|
||||
name = coverage[case]
|
||||
self.assertIn(name, by_name, case)
|
||||
self.assertEqual(by_name[name].cls, cls, case)
|
||||
named.append(name)
|
||||
self.assertEqual(len(named), len(set(named)), "two cases share a mutant")
|
||||
|
||||
def test_the_inbound_rule_of_04_10_has_no_mutant_left(self) -> None:
|
||||
# link-rule (an inbound link from any other concept page) goes in Step 7; m24, m28 and m29 now carry K2.
|
||||
self.assertEqual([m.name for m in self.manifest().mutants if m.cls == "link-rule"], [])
|
||||
|
||||
def test_every_mutant_class_is_a_check_the_auditor_emits_or_k2_names(self) -> None:
|
||||
expected = set(GROUP_OF) | set(self.K2_CLASSES)
|
||||
for m in self.manifest().mutants:
|
||||
self.assertIn(m.cls, expected, m.name)
|
||||
|
||||
def test_the_sources_hold_the_k1_fixture_and_a_release_note(self) -> None:
|
||||
k2 = self.k2()
|
||||
self.assertEqual(k2["sources_root"], "gates-local/mutants-kilder")
|
||||
self.assertEqual(k2["k1_file"], "source-manifest.json")
|
||||
self.assertEqual(len(k2["main_topic"]), 3)
|
||||
self.assertTrue(all(p.startswith("kilde/docs-en-") for p in k2["main_topic"]), k2["main_topic"])
|
||||
self.assertTrue(any(self.RELEASE.match(p) for p in k2["not_main_topic"]), k2["not_main_topic"])
|
||||
sources = self.manifest().sources
|
||||
self.assertEqual(set(sources), {k2["k1_file"], *k2["main_topic"], *k2["not_main_topic"]})
|
||||
self.assertEqual(k2["k1_sha256"], sources[k2["k1_file"]], "the K1 file the gate checks is the one described")
|
||||
|
||||
def test_every_case_documents_its_expected_extras(self) -> None:
|
||||
cases = json.loads(self.path().read_text(encoding="utf-8"))["cases"]
|
||||
known = set(GROUP_OF) | set(self.K2_CLASSES)
|
||||
for c in cases:
|
||||
extra = c.get("expected_extra")
|
||||
self.assertIsInstance(extra, list, c["name"])
|
||||
self.assertLessEqual(set(extra), known - {"link-rule", c["class"]}, c["name"])
|
||||
|
||||
def test_the_known_positive_meets_k2(self) -> None:
|
||||
k2 = self.k2()
|
||||
kp = k2["known_positive"]
|
||||
files = self.manifest().known_positive.files
|
||||
concepts = {f for f in files if f.endswith(".md") and f.rsplit("/", 1)[-1] not in ("index.md", "log.md")}
|
||||
main = kp["x-main-for"]
|
||||
self.assertEqual(len(main), 1, "one main page for the three documentation sources")
|
||||
(values,) = main.values()
|
||||
self.assertEqual(sorted(values), sorted("/" + p for p in k2["main_topic"]))
|
||||
non_main = kp["non_main_pages"]
|
||||
self.assertGreaterEqual(len(non_main), 2)
|
||||
self.assertIn("bypasspermissions-modus.md", non_main)
|
||||
self.assertEqual(set(main) | set(non_main), concepts)
|
||||
self.assertFalse(set(main) & set(non_main))
|
||||
for page in non_main:
|
||||
self.assertTrue(kp["links_from_main_pages"][page], page)
|
||||
self.assertLessEqual(set(kp["links_from_main_pages"][page]), set(main), page)
|
||||
for page in main:
|
||||
self.assertEqual(kp["concept_links_into_main_pages"][page], [], "a main page needs no inbound link")
|
||||
self.assertTrue(kp["cites_release_notes"])
|
||||
self.assertLessEqual(set(kp["cites_release_notes"]), set(non_main))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
136
gates/tests/test_polarity.py
Normal file
136
gates/tests/test_polarity.py
Normal file
|
|
@ -0,0 +1,136 @@
|
|||
"""Tests for gates/polarity.py: counts polarity keys that accept an answer of the opposite polarity (E2d).
|
||||
|
||||
Every fixture is synthetic text written here. No corpus text is used (CLAUDE.md rule 3).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
import io
|
||||
import json
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
|
||||
import polarity # noqa: E402
|
||||
from multidoc import RunError # noqa: E402
|
||||
|
||||
|
||||
def md_question(qid: str, span: str, keys: list[dict[str, Any]]) -> dict[str, Any]:
|
||||
return {"id": qid, "question": "?", "evidence": [{"file": "a.md", "span": "Unrelated words about the ferry."},
|
||||
{"file": "b.md", "span": span}],
|
||||
"answer_keys": [dict(k, file="b.md") for k in keys]}
|
||||
|
||||
|
||||
def run(argv: list[str]) -> tuple[int, str]:
|
||||
out, err = io.StringIO(), io.StringIO()
|
||||
with contextlib.redirect_stdout(out), contextlib.redirect_stderr(err):
|
||||
code = polarity.main(argv)
|
||||
return code, out.getvalue() + err.getvalue()
|
||||
|
||||
|
||||
class FlipTests(unittest.TestCase):
|
||||
def opposite(self, sentence: str, atom: str) -> str:
|
||||
found = polarity.answer_from_span(atom, [sentence])
|
||||
assert found is not None
|
||||
return polarity.flip(*found)
|
||||
|
||||
def test_a_negation_in_the_window_is_taken_out(self) -> None:
|
||||
self.assertEqual(self.opposite("The lantern is not lit at dawn", "lit"), "the lantern is lit at dawn")
|
||||
self.assertEqual(self.opposite("The gate isn't locked", "locked"), "the gate is locked")
|
||||
self.assertEqual(self.opposite("Carters can't cross the bridge", "cross"), "carters can cross the bridge")
|
||||
self.assertEqual(self.opposite("A pass is no longer required", "required"), "a pass is required")
|
||||
self.assertEqual(self.opposite("It sails without a pilot", "pilot"), "it sails with a pilot")
|
||||
|
||||
def test_a_negation_far_before_the_atom_is_outside_the_window(self) -> None:
|
||||
self.assertEqual(self.opposite("Never mind: the one two three four five six keeper rings", "rings"),
|
||||
"never mind: the one two three four five six keeper not rings")
|
||||
|
||||
def test_without_a_negation_not_is_put_in_front_of_the_atom(self) -> None:
|
||||
self.assertEqual(self.opposite("The keeper still rings the bell", "rings"), "the keeper still not rings the bell")
|
||||
|
||||
def test_the_sentence_with_the_negated_atom_is_chosen(self) -> None:
|
||||
found = polarity.answer_from_span("lit", ["The lantern is lit at dusk. At dawn it is not lit."])
|
||||
self.assertEqual(found, ("at dawn it is not lit", len("at dawn it is not ")))
|
||||
|
||||
|
||||
class CountTests(unittest.TestCase):
|
||||
def test_an_unmarked_negation_key_accepts_the_opposite_answer(self) -> None:
|
||||
q = md_question("q01", "The ferry is not staffed at night.", [{"text": "staffed"}])
|
||||
rep = polarity.count_polarity([q], [{"id": "q01", "atoms": ["staffed"]}])
|
||||
self.assertEqual((rep.keys, rep.accept_span, rep.accept_opposite), (1, 1, 1))
|
||||
self.assertFalse(rep.passed)
|
||||
|
||||
def test_a_negated_key_refuses_the_opposite_answer(self) -> None:
|
||||
q = md_question("q01", "The ferry is not staffed at night.", [{"text": "staffed", "negated": True}])
|
||||
rep = polarity.count_polarity([q], [{"id": "q01", "atoms": ["staffed"]}])
|
||||
self.assertEqual((rep.keys, rep.accept_span, rep.accept_opposite), (1, 1, 0))
|
||||
self.assertTrue(rep.passed)
|
||||
|
||||
def test_a_key_of_several_atoms_counts_once(self) -> None:
|
||||
q = md_question("q01", "Only the north gate opens at dawn.",
|
||||
[{"text": "north gate"}, {"text": "dawn"}])
|
||||
rep = polarity.count_polarity([q], [{"id": "q01", "atoms": ["north gate", "dawn"]}])
|
||||
self.assertEqual((rep.keys, rep.accept_span, rep.accept_opposite), (1, 1, 1))
|
||||
|
||||
def test_an_atom_that_is_not_a_key_cannot_run(self) -> None:
|
||||
q = md_question("q01", "The ferry is not staffed at night.", [{"text": "staffed"}])
|
||||
with self.assertRaises(RunError):
|
||||
polarity.count_polarity([q], [{"id": "q01", "atoms": ["night"]}])
|
||||
|
||||
def test_an_atom_outside_its_spans_cannot_run(self) -> None:
|
||||
q = md_question("q01", "The ferry is not staffed at night.", [{"text": "pilot"}])
|
||||
with self.assertRaises(RunError):
|
||||
polarity.count_polarity([q], [{"id": "q01", "atoms": ["pilot"]}])
|
||||
|
||||
def test_a_whole_document_key_uses_every_span_of_the_file(self) -> None:
|
||||
q = {"id": "w01", "type": "whole", "question": "?", "file": "a.md",
|
||||
"evidence": [{"span": "The bell rings at noon."}, {"span": "The keeper never rings it at night."}],
|
||||
"answer_keys": [{"text": "rings"}]}
|
||||
rep = polarity.count_polarity([q], [{"id": "w01", "atoms": ["rings"]}])
|
||||
self.assertEqual(rep.listed[0].split("| ")[1], "the keeper rings it at night")
|
||||
|
||||
|
||||
class CliTests(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
self._tmp = tempfile.TemporaryDirectory()
|
||||
self.root = Path(self._tmp.name)
|
||||
|
||||
def tearDown(self) -> None:
|
||||
self._tmp.cleanup()
|
||||
|
||||
def write(self, name: str, data: Any) -> str:
|
||||
(self.root / name).write_text(json.dumps(data), encoding="utf-8")
|
||||
return str(self.root / name)
|
||||
|
||||
def test_counts_and_fails_while_a_key_accepts_the_opposite(self) -> None:
|
||||
s = self.write("set.json", {"schema": "okf-wiki/multidoc-set/1", "questions": [
|
||||
md_question("q01", "The ferry is not staffed at night.", [{"text": "staffed"}])]})
|
||||
k = self.write("keys.json", {"schema": polarity.KEYS_SCHEMA, "keys": [{"id": "q01", "atoms": ["staffed"]}]})
|
||||
code, out = run(["--set", s, "--keys", k])
|
||||
self.assertEqual(code, 1, out)
|
||||
self.assertIn("polarity keys that accept the answer from their span: 1 of 1", out)
|
||||
self.assertIn("polarity keys that accept the opposite answer: 1 of 1", out)
|
||||
self.assertIn("FAIL", out)
|
||||
|
||||
def test_an_empty_list_is_a_fail_not_a_pass(self) -> None:
|
||||
s = self.write("set.json", {"schema": "okf-wiki/multidoc-set/1", "questions": []})
|
||||
k = self.write("keys.json", {"schema": polarity.KEYS_SCHEMA, "keys": []})
|
||||
code, out = run(["--set", s, "--keys", k])
|
||||
self.assertEqual(code, 1)
|
||||
self.assertIn("0 of 0", out)
|
||||
|
||||
def test_a_list_without_its_schema_cannot_run(self) -> None:
|
||||
s = self.write("set.json", {"schema": "okf-wiki/multidoc-set/1", "questions": []})
|
||||
k = self.write("keys.json", {"keys": []})
|
||||
code, out = run(["--set", s, "--keys", k])
|
||||
self.assertEqual(code, 1)
|
||||
self.assertIn("schema", out)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
182
gates/tests/test_quote_rule.py
Normal file
182
gates/tests/test_quote_rule.py
Normal file
|
|
@ -0,0 +1,182 @@
|
|||
"""Tests for gates/quote_rule.py: rule E against rule L, the answer key and the two new reject rules (V1 Step 4).
|
||||
|
||||
Every fixture is synthetic gardening text (CLAUDE.md rule 3). The P1 fragments are measured by the CLI, outside the tests.
|
||||
Run: python3 -m unittest discover -s gates/tests -p test_quote_rule.py
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
import importlib
|
||||
import io
|
||||
import json
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
|
||||
import quote_rule # noqa: E402
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
|
||||
|
||||
def frag(text: str, quote: str | None = None, concept: str = "Basil", fid: str = "basil",
|
||||
status: str = "existing") -> dict[str, Any]:
|
||||
f: dict[str, Any] = {"concept": concept, "status": status, "id": fid, "text": text,
|
||||
"anchor": {"file": "docs-en-basil.md", "start": 3, "end": 3}}
|
||||
if quote is not None:
|
||||
f["quote"] = quote
|
||||
return f
|
||||
|
||||
|
||||
INDEX: dict[str, Any] = {"entries": [
|
||||
{"id": "basil", "title": "Basil", "aliases": ["Basil", "Sweet basil"]},
|
||||
{"id": "soil-mix", "title": "Soil mix", "aliases": ["Soil mix"]},
|
||||
]}
|
||||
|
||||
|
||||
class RulesTest(unittest.TestCase):
|
||||
def test_e_rejects_a_quoteless_prose_fragment_and_l_accepts_it(self) -> None:
|
||||
f = frag("Basil likes a warm and sheltered spot.")
|
||||
self.assertTrue(quote_rule.rule_e(f))
|
||||
self.assertFalse(quote_rule.rule_l(f))
|
||||
|
||||
def test_both_reject_a_quoteless_narrow_list_word(self) -> None:
|
||||
f = frag("Basil needs six hours of sun.")
|
||||
self.assertTrue(quote_rule.rule_e(f))
|
||||
self.assertTrue(quote_rule.rule_l(f))
|
||||
|
||||
def test_both_reject_a_quoteless_digit_or_backticked_name(self) -> None:
|
||||
for text in ("Basil needs 6 hours of sun.", "Pinch the top `leaf` pairs."):
|
||||
self.assertTrue(quote_rule.rule_e(frag(text)), text)
|
||||
self.assertTrue(quote_rule.rule_l(frag(text)), text)
|
||||
|
||||
def test_l_lets_several_through(self) -> None:
|
||||
f = frag("Sow several seeds in each pot.")
|
||||
self.assertFalse(quote_rule.rule_l(f))
|
||||
self.assertTrue(quote_rule.number_words(f)) # the answer key catches it
|
||||
|
||||
def test_a_quoted_fragment_passes_both(self) -> None:
|
||||
f = frag("Basil needs six hours of sun.", quote="six hours of sun")
|
||||
self.assertFalse(quote_rule.rule_e(f))
|
||||
self.assertFalse(quote_rule.rule_l(f))
|
||||
|
||||
def test_a_blank_quote_counts_as_no_quote(self) -> None:
|
||||
self.assertTrue(quote_rule.rule_e(frag("Basil likes warmth.", quote=" ")))
|
||||
|
||||
def test_number_words_are_whole_words(self) -> None:
|
||||
self.assertFalse(quote_rule.number_words(frag("Often the seedlings wilt."))) # "ten" inside "often"
|
||||
self.assertEqual(quote_rule.number_words(frag("Water Twice a week, at least once.")), ["once", "twice"])
|
||||
|
||||
def test_the_narrow_list_is_a_strict_subset_of_the_broad_list(self) -> None:
|
||||
self.assertLess(set(quote_rule.NARROW), set(quote_rule.BROAD))
|
||||
for w in ("zero", "twenty", "hundred", "thousand", "million"):
|
||||
self.assertIn(w, quote_rule.NARROW)
|
||||
for w in ("thirty", "ninety", "dozen", "once", "twice", "first", "tenth", "half", "double", "pair", "couple",
|
||||
"several", "single", "dozens"):
|
||||
self.assertIn(w, quote_rule.BROAD)
|
||||
self.assertNotIn(w, quote_rule.NARROW)
|
||||
|
||||
|
||||
class ChooseTest(unittest.TestCase):
|
||||
def test_l_only_when_nothing_slips_and_e_rejects_five_percent_more(self) -> None:
|
||||
self.assertEqual(quote_rule.choose(e_rejects=133, l_rejects=100, l_slips=0, n=650), "L")
|
||||
self.assertEqual(quote_rule.choose(e_rejects=132, l_rejects=100, l_slips=0, n=650), "E")
|
||||
self.assertEqual(quote_rule.choose(e_rejects=133, l_rejects=100, l_slips=1, n=650), "E")
|
||||
|
||||
def test_the_boundary_is_33_of_650(self) -> None:
|
||||
self.assertEqual(quote_rule.five_percent(650), 33)
|
||||
|
||||
|
||||
class RejectRulesTest(unittest.TestCase):
|
||||
def test_short_quote(self) -> None:
|
||||
self.assertTrue(quote_rule.short_quote(frag("Basil needs sun.", quote="needs sun")))
|
||||
self.assertFalse(quote_rule.short_quote(frag("Basil needs sun.", quote="needs full sun")))
|
||||
self.assertFalse(quote_rule.short_quote(frag("Basil needs sun."))) # no quote is rule E/L's business
|
||||
|
||||
def test_id_mismatch(self) -> None:
|
||||
self.assertFalse(quote_rule.id_mismatch(frag("x", concept="Basil"), INDEX))
|
||||
self.assertFalse(quote_rule.id_mismatch(frag("x", concept="sweet basil"), INDEX)) # an alias
|
||||
self.assertFalse(quote_rule.id_mismatch(frag("x", concept="The Soil mix", fid="soil-mix"), INDEX)) # norm
|
||||
self.assertTrue(quote_rule.id_mismatch(frag("x", concept="Pruning", fid="basil"), INDEX))
|
||||
self.assertTrue(quote_rule.id_mismatch(frag("x", concept="Basil", fid="no-such-id"), INDEX))
|
||||
self.assertFalse(quote_rule.id_mismatch(frag("x", concept="Pruning", status="new"), INDEX))
|
||||
|
||||
def test_id_mismatch_accepts_a_jaccard_match(self) -> None:
|
||||
index = {"entries": [{"id": "pot-size", "title": "Pot size for basil", "aliases": []}]}
|
||||
self.assertFalse(quote_rule.id_mismatch(frag("x", concept="Pot size for herbs", fid="pot-size"), index))
|
||||
|
||||
|
||||
class TrialConstantsTest(unittest.TestCase):
|
||||
"""The copies of the trial's constants and norm stay equal to it (read at run time, so mypy does not follow)."""
|
||||
|
||||
def test_copies_equal_the_trial(self) -> None:
|
||||
sys.path.insert(0, str(REPO_ROOT))
|
||||
sys.path.insert(0, str(REPO_ROOT / "evals"))
|
||||
trial = importlib.import_module("p1.kpack")
|
||||
self.assertEqual(quote_rule.NEEDS_QUOTE.pattern, getattr(trial, "NEEDS_QUOTE").pattern)
|
||||
self.assertEqual(quote_rule.MERGE_JACCARD, getattr(trial, "MERGE_JACCARD"))
|
||||
for name in ("The Soil mixes", "`leaf` pairs", "Pot size / basil", "glass", "Sweet basil."):
|
||||
self.assertEqual(quote_rule.norm(name), getattr(trial, "norm")(name), name)
|
||||
|
||||
|
||||
class MeasureCliTest(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
self._tmp = tempfile.TemporaryDirectory()
|
||||
self.root = Path(self._tmp.name)
|
||||
self.frags = self.root / "K-fragments"
|
||||
self.frags.mkdir()
|
||||
self.index = self.root / "index.json"
|
||||
self.index.write_text(json.dumps(INDEX))
|
||||
|
||||
def tearDown(self) -> None:
|
||||
self._tmp.cleanup()
|
||||
|
||||
def write(self, *fragments: dict[str, Any]) -> None:
|
||||
(self.frags / "pack-1.json").write_text(json.dumps({"fragments": list(fragments)}))
|
||||
|
||||
def run_cli(self) -> tuple[int, dict[str, Any] | None, str]:
|
||||
out = io.StringIO()
|
||||
with contextlib.redirect_stdout(out):
|
||||
code = quote_rule.main(["measure", str(self.frags), str(self.index)])
|
||||
try:
|
||||
return code, json.loads(out.getvalue()), out.getvalue()
|
||||
except json.JSONDecodeError:
|
||||
return code, None, out.getvalue()
|
||||
|
||||
def test_counts_and_choice(self) -> None:
|
||||
self.write(frag("Basil likes warmth."), frag("Sow several seeds."), frag("Basil needs six hours of sun."),
|
||||
frag("Basil needs sun.", quote="Basil needs warm soil"), frag("x", concept="Pruning"))
|
||||
code, out, raw = self.run_cli()
|
||||
self.assertEqual(code, 0, raw)
|
||||
assert out is not None
|
||||
self.assertEqual(out["fragments"], 5)
|
||||
self.assertEqual(out["e_rejects"], 4)
|
||||
self.assertEqual(out["l_rejects"], 1)
|
||||
self.assertEqual(out["key_hits"], 2)
|
||||
self.assertEqual(out["l_slips"], 1)
|
||||
self.assertEqual(out["e_slips"], 0)
|
||||
self.assertEqual(out["id_mismatches"], 1)
|
||||
self.assertEqual(out["short_quotes"], 0)
|
||||
self.assertEqual(out["choice"], "E")
|
||||
self.assertEqual(out["key_hits_per_file"], {"pack-1.json": 2})
|
||||
self.assertNotIn("Basil", raw) # counts only, never fragment text
|
||||
|
||||
def test_no_known_positive_fails(self) -> None:
|
||||
self.write(frag("Basil likes warmth."), frag("Basil needs sun.", quote="Basil needs warm soil"))
|
||||
code, out, raw = self.run_cli()
|
||||
self.assertEqual(code, 1, raw)
|
||||
assert out is not None
|
||||
self.assertEqual(out["key_hits"], 0)
|
||||
self.assertIsNone(out["choice"])
|
||||
|
||||
def test_no_fragments_cannot_run(self) -> None:
|
||||
code, _, _ = self.run_cli()
|
||||
self.assertEqual(code, 2)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
332
gates/tests/test_repair_gate.py
Normal file
332
gates/tests/test_repair_gate.py
Normal file
|
|
@ -0,0 +1,332 @@
|
|||
"""Tests for gates/repair_gate.py, the fail-closed judge of the repair command (order 34, contracts K18 and K19).
|
||||
|
||||
Order 34 A3: the three stubs in gates/repair/stubs/ (one that does nothing, one that refuses everything, one that gives
|
||||
every page a new generated.at) each turn the gate RED; a manifest whose fixtures have changed stops the gate (exit 2);
|
||||
and a missing command, or a missing module, is RED. Beside them, the parts of the gate a command cannot be run against
|
||||
yet (the repair command does not exist when this is written) are tried with handmade outputs: the base the gate builds,
|
||||
the keys it computes, the checks of task and linkset, and a refusal that must name its code and write nothing. Every
|
||||
fixture is synthetic text about beekeeping (CLAUDE.md rule 3).
|
||||
Run: python3 -m unittest discover -s gates/tests -p 'test_repair_gate.py'
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import json
|
||||
import shutil
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from contextlib import redirect_stdout
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
|
||||
|
||||
import repair_gate as gate # noqa: E402
|
||||
from okfwiki import audit, claims, kassemble, klink, kpack # noqa: E402
|
||||
|
||||
GATES = Path(__file__).resolve().parents[1]
|
||||
MANIFEST = GATES / "repair.manifest.json"
|
||||
STUBS = GATES / "repair" / "stubs"
|
||||
PY = sys.executable
|
||||
|
||||
# The kinds of case order 34 A1 asks for, by the names the manifest gives them.
|
||||
REQUIRED = (
|
||||
"replace-one-with-one", "replace-one-with-two", "two-sentence-fragment", "merged-fragment", "release-note-fragment",
|
||||
"context-kept", "context-other", "remove-one", "remove-last-on-page", "remove-last-on-main-page",
|
||||
"remove-last-from-source-on-main-page", "shared-footnote", "equal-sentences-first-removed", "repair-makes-equal",
|
||||
"link-anchor-stays", "link-anchor-gone", "unlink", "relink", "linkset-two-candidates", "two-repairs-two-pages",
|
||||
"refuse-quote-not-verbatim", "refuse-quote-other-source", "refuse-unknown-key", "refuse-stale-bundle",
|
||||
"refuse-duplicate-answer", "refuse-stale-list", "refuse-relink-missing-target", "refuse-unlink-wrong-at",
|
||||
"refuse-unlink-wrong-anchor", "identity",
|
||||
)
|
||||
|
||||
|
||||
def run_gate(*command: str, manifest: Path = MANIFEST) -> tuple[int, str]:
|
||||
out = io.StringIO()
|
||||
with redirect_stdout(out):
|
||||
code = gate.main(["--manifest", str(manifest), "--timeout", "60", "--", *command])
|
||||
return code, out.getvalue()
|
||||
|
||||
|
||||
def case(name: str) -> dict[str, Any]:
|
||||
cases: list[dict[str, Any]] = gate.load_manifest(MANIFEST)["cases"]
|
||||
return next(c for c in cases if c["name"] == name)
|
||||
|
||||
|
||||
class StubsTest(unittest.TestCase):
|
||||
"""A3: each stub is RED, and no case passes with it."""
|
||||
|
||||
def assert_red(self, stub: str) -> str:
|
||||
code, out = run_gate(PY, str(STUBS / stub))
|
||||
self.assertEqual(code, 1, out)
|
||||
self.assertIn("verdict: RED", out)
|
||||
n = len(gate.load_manifest(MANIFEST)["cases"])
|
||||
self.assertIn(f"passed 0 of {n}", out)
|
||||
return out
|
||||
|
||||
def test_a_command_that_does_nothing_is_red(self) -> None:
|
||||
out = self.assert_red("noop.py")
|
||||
self.assertIn("pages changed: [], the case says ['honey-harvest.md']", out)
|
||||
|
||||
def test_a_command_that_refuses_everything_is_red(self) -> None:
|
||||
out = self.assert_red("refuse_all.py")
|
||||
self.assertIn("refusal codes ['x'], the case says ['unknown-key']", out)
|
||||
self.assertIn("1 refusal line(s) name no claim or record", out)
|
||||
|
||||
def test_a_command_that_touches_every_page_is_red(self) -> None:
|
||||
out = self.assert_red("touch_generated.py")
|
||||
lines = out.splitlines()
|
||||
at = next(i for i, line in enumerate(lines) if line.startswith(" identity "))
|
||||
identity = "\n".join(line for line in lines[at + 1:at + 4] if line.startswith(" "))
|
||||
self.assertIn("pages changed: ['brood.md', 'frames.md', 'hive.md'", identity)
|
||||
|
||||
|
||||
class CannotRunTest(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
self._tmp = tempfile.TemporaryDirectory()
|
||||
self.tmp = Path(self._tmp.name)
|
||||
shutil.copyfile(MANIFEST, self.tmp / MANIFEST.name)
|
||||
shutil.copytree(GATES / "repair", self.tmp / "repair", ignore=shutil.ignore_patterns("__pycache__"))
|
||||
self.manifest = self.tmp / MANIFEST.name
|
||||
|
||||
def tearDown(self) -> None:
|
||||
self._tmp.cleanup()
|
||||
|
||||
def test_a_changed_fixture_stops_the_gate(self) -> None:
|
||||
p = self.tmp / "repair" / "kilder" / "kilde" / "docs-en-hive.md"
|
||||
p.chmod(0o644)
|
||||
p.write_text(p.read_text(encoding="utf-8").replace("warm and calm", "cold and windy"), encoding="utf-8")
|
||||
code, out = run_gate(PY, str(STUBS / "noop.py"), manifest=self.manifest)
|
||||
self.assertEqual(code, 2, out)
|
||||
self.assertIn("GATE CANNOT RUN", out)
|
||||
self.assertIn("sha256 mismatch in kilder/kilde/docs-en-hive.md", out)
|
||||
|
||||
def test_an_extra_or_a_missing_fixture_stops_the_gate(self) -> None:
|
||||
(self.tmp / "repair" / "extra.json").write_text("{}", encoding="utf-8")
|
||||
(self.tmp / "repair" / "agent.md").unlink()
|
||||
code, out = run_gate(PY, str(STUBS / "noop.py"), manifest=self.manifest)
|
||||
self.assertEqual(code, 2, out)
|
||||
self.assertIn("unexpected file extra.json", out)
|
||||
self.assertIn("missing file agent.md", out)
|
||||
|
||||
def test_a_changed_case_is_no_other_manifest_than_the_one_reported(self) -> None:
|
||||
doc = json.loads(self.manifest.read_text(encoding="utf-8"))
|
||||
doc["cases"][0]["steps"][0]["exit"] = 2
|
||||
self.manifest.write_text(json.dumps(doc), encoding="utf-8")
|
||||
code, out = run_gate(PY, str(STUBS / "noop.py"), manifest=self.manifest)
|
||||
self.assertEqual(code, 2, out)
|
||||
self.assertIn("a step needs argv and exit 0 or 1", out)
|
||||
|
||||
def test_a_refusal_code_the_manifest_does_not_define_stops_the_gate(self) -> None:
|
||||
doc = json.loads(self.manifest.read_text(encoding="utf-8"))
|
||||
del doc["refusal_codes"]["unknown-key"]
|
||||
self.manifest.write_text(json.dumps(doc), encoding="utf-8")
|
||||
code, out = run_gate(PY, str(STUBS / "noop.py"), manifest=self.manifest)
|
||||
self.assertEqual(code, 2, out)
|
||||
self.assertIn("refusal codes not in the manifest", out)
|
||||
|
||||
def test_no_command_is_a_usage_error(self) -> None:
|
||||
code, out = run_gate(manifest=self.manifest)
|
||||
self.assertEqual(code, 2)
|
||||
self.assertIn("usage error", out)
|
||||
|
||||
def test_a_missing_command_is_red(self) -> None:
|
||||
code, out = run_gate(str(self.tmp / "no-such-command"))
|
||||
self.assertEqual(code, 1, out)
|
||||
self.assertIn("command not found", out)
|
||||
self.assertIn("verdict: RED", out)
|
||||
|
||||
def test_a_missing_module_is_red(self) -> None:
|
||||
code, out = run_gate(PY, "-m", "okfwiki.no_such_module_here")
|
||||
self.assertEqual(code, 1, out)
|
||||
self.assertIn("No module named okfwiki.no_such_module_here", out)
|
||||
self.assertIn("passed 0 of", out)
|
||||
|
||||
|
||||
class BaseTest(unittest.TestCase):
|
||||
"""The collection the gate builds, and the checks a command cannot be run against yet."""
|
||||
|
||||
_tmp: tempfile.TemporaryDirectory[str]
|
||||
base: gate.Base
|
||||
manifest: dict[str, Any]
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls) -> None:
|
||||
cls._tmp = tempfile.TemporaryDirectory()
|
||||
cls.manifest = gate.load_manifest(MANIFEST)
|
||||
gate.verify_fixtures(cls.manifest, GATES / "repair")
|
||||
cls.base = gate.build_base(cls.manifest, GATES / "repair", Path(cls._tmp.name))
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls) -> None:
|
||||
cls._tmp.cleanup()
|
||||
|
||||
def setUp(self) -> None:
|
||||
self._case_tmp = tempfile.TemporaryDirectory()
|
||||
self.tmp = Path(self._case_tmp.name)
|
||||
|
||||
def tearDown(self) -> None:
|
||||
self._case_tmp.cleanup()
|
||||
|
||||
def test_the_base_is_clean_and_has_the_pages_the_cases_are_written_for(self) -> None:
|
||||
pages = sorted(gate.concept_pages(self.base.wiki))
|
||||
self.assertEqual(pages, ["brood.md", "frames.md", "hive.md", "honey-harvest.md", "queen-cells.md", "smoker.md",
|
||||
"swarming.md", "water.md"])
|
||||
self.assertEqual(audit.audit(self.base.wiki, self.base.sources), [])
|
||||
found = claims.extract(self.base.wiki, self.base.sources)
|
||||
self.assertEqual(claims.criterion_1(found)["total"], (20, 20))
|
||||
self.assertEqual(len(json.loads(self.base.links.read_text(encoding="utf-8"))["links"]), 5)
|
||||
|
||||
def test_every_case_names_pages_and_sentences_of_the_base_or_of_its_own_answers(self) -> None:
|
||||
pages = set(gate.concept_pages(self.base.wiki))
|
||||
ids = {c.id for c in self.base.found.claims}
|
||||
for c in self.manifest["cases"]:
|
||||
for p in c.get("pages_changed", []) + c.get("pages_gone", []):
|
||||
self.assertIn(p, pages, c["name"])
|
||||
for x in c.get("claims_before", []):
|
||||
self.assertIn(gate.claim_id(x["page"], x["text"], x.get("n", 0)), ids, c["name"])
|
||||
for x in c.get("links_removed", []):
|
||||
self.assertIn(gate.link_tuple(x), gate.link_set(json.loads(self.base.links.read_text())["links"]))
|
||||
|
||||
def test_the_cases_cover_what_order_34_a1_names(self) -> None:
|
||||
names = {c["name"] for c in self.manifest["cases"]}
|
||||
self.assertEqual(set(REQUIRED) - names, set())
|
||||
|
||||
def test_a_sentence_key_is_klinks_key_on_the_built_page(self) -> None:
|
||||
page = (self.base.wiki / "water.md").read_text(encoding="utf-8")
|
||||
keys = [s.key for s in klink.page_sentences(page)]
|
||||
self.assertEqual(keys, [gate.sentence_key("Bees need a water source near the hive.", 0),
|
||||
gate.sentence_key("Bees need a water source near the hive.", 1)])
|
||||
|
||||
def test_a_fragment_key_is_the_footnote_on_its_page_and_two_concepts_share_one(self) -> None:
|
||||
key = {"concept_id": "frames", "file": "docs-en-hive.md", "start": 8, "end": 10,
|
||||
"quote": "holds wax comb where bees store honey and raise brood", "provenance": "documented"}
|
||||
label = gate.fragment_label(self.base.sources, key)
|
||||
for page in ("frames.md", "brood.md"):
|
||||
self.assertIn(f"[^{label}]", (self.base.wiki / page).read_text(encoding="utf-8"))
|
||||
records = json.loads((self.base.work / "K-work" / "accepted.json").read_text(encoding="utf-8"))["fragments"]
|
||||
self.assertEqual(sorted(r["concept_id"] for r in records if kassemble.label(r) == label), ["brood", "frames"])
|
||||
|
||||
def test_a_rebuild_without_change_measures_nothing(self) -> None:
|
||||
c = gate.fresh_copy(self.base, self.tmp, "same")
|
||||
code, out = gate.quiet(kassemble.main, ["build", "--work", c.places["work"], "--sources", c.places["sources"],
|
||||
"--out", c.places["wiki"], "--links", c.places["links"], "--replace",
|
||||
"--at", self.manifest["build_at"]])
|
||||
self.assertEqual(code, 0, out)
|
||||
m = gate.measure(c.path("wiki_old"), c.path("wiki"), c.path("sources"))
|
||||
self.assertEqual((m.changed, sum(m.claims_gone.values()), sum(m.claims_came.values())), ([], 0, 0))
|
||||
|
||||
def test_a_list_is_rendered_in_k17s_schema_with_the_hash_of_the_claim_list(self) -> None:
|
||||
doc = gate.render_list({"claims": [{"page": "frames.md", "text": "Frames can be lifted out one by one."}],
|
||||
"links": [{"page": "hive.md", "sentence": "Light the smoker before you lift the lid.",
|
||||
"anchor": "smoker", "target": "smoker.md"}]}, self.base)
|
||||
self.assertEqual(doc["schema"], "okf-wiki/repair-list/1")
|
||||
self.assertEqual(doc["claims_sha256"], gate.sha256_file(self.base.claims))
|
||||
self.assertEqual(set(doc), {"schema", "claims_sha256", "verdicts_sha256", "claims", "pages", "pairs", "links"})
|
||||
row = doc["claims"][0]
|
||||
self.assertEqual((row["page"], row["provenance"], row["kind"], row["sources"]),
|
||||
("frames.md", "documented", "docs", [{"file": "kilde/docs-en-hive.md", "line": 8}]))
|
||||
self.assertEqual((doc["links"][0]["at"], doc["links"][0]["origin"]), (10, "o33-kandidat"))
|
||||
stale = gate.render_list({"claims": [], "stale": True}, self.base)
|
||||
self.assertNotEqual(stale["claims_sha256"], doc["claims_sha256"])
|
||||
|
||||
def test_answers_are_keyed_by_concept_and_footnote(self) -> None:
|
||||
doc = gate.render_answers(case("refuse-unknown-key")["steps"][0]["answers"]
|
||||
+ case("remove-one")["steps"][0]["answers"], self.base.sources)
|
||||
self.assertEqual(doc["schema"], "okf-wiki/repair-answers/1")
|
||||
self.assertEqual(doc["answers"][0], {"concept_id": "hive", "footnote": "s0000000000", "action": "remove"})
|
||||
self.assertEqual(doc["answers"][1]["concept_id"], "brood")
|
||||
self.assertIn(f"[^{doc['answers'][1]['footnote']}]", (self.base.wiki / "brood.md").read_text(encoding="utf-8"))
|
||||
|
||||
def fake(self, body: str) -> list[str]:
|
||||
script = self.tmp / f"fake-{len(list(self.tmp.iterdir()))}.py"
|
||||
script.write_text(body, encoding="utf-8")
|
||||
return [PY, str(script)]
|
||||
|
||||
def run_one(self, name: str, command: list[str]) -> list[str]:
|
||||
where = Path(tempfile.mkdtemp(dir=self.tmp))
|
||||
return gate.run_case(case(name), self.base, command, GATES / "repair", where, self.manifest["build_at"], 60)
|
||||
|
||||
def test_a_refusal_passes_only_with_its_code_and_nothing_written(self) -> None:
|
||||
right = self.fake("import sys\nprint('refused: unknown-key hive s0000000000')\nsys.exit(1)\n")
|
||||
self.assertEqual(self.run_one("refuse-unknown-key", right), [])
|
||||
wrong_code = self.fake("import sys\nprint('refused: stale-bundle hive s0000000000')\nsys.exit(1)\n")
|
||||
problems = self.run_one("refuse-unknown-key", wrong_code)
|
||||
self.assertTrue(any("refusal codes ['stale-bundle']" in p for p in problems))
|
||||
no_record = self.fake("import sys\nprint('refused: unknown-key')\nsys.exit(1)\n")
|
||||
self.assertEqual(self.run_one("refuse-unknown-key", no_record),
|
||||
["step 1: 1 refusal line(s) name no claim or record"])
|
||||
writes = self.fake("import sys, pathlib\nargs = sys.argv[1:]\n"
|
||||
"w = pathlib.Path(args[args.index('--work') + 1])\n"
|
||||
"(w / 'K-work' / 'repairs.json').write_text('{}')\n"
|
||||
"print('refused: unknown-key hive s0000000000')\nsys.exit(1)\n")
|
||||
self.assertTrue(any("refused but changed work" in p for p in self.run_one("refuse-unknown-key", writes)))
|
||||
crash = self.fake("raise RuntimeError('a crash exits 1 too')\n")
|
||||
self.assertTrue(any("refusal codes []" in p for p in self.run_one("refuse-unknown-key", crash)))
|
||||
|
||||
def test_a_link_command_is_judged_by_the_links_file_and_the_built_page(self) -> None:
|
||||
unlink = self.fake(
|
||||
"import json, sys\nargs = sys.argv[1:]\np = args[args.index('--links') + 1]\nd = json.load(open(p))\n"
|
||||
"d['links'] = [r for r in d['links'] if r['anchor'] != 'smoker']\nd['accepted'] = len(d['links'])\n"
|
||||
"json.dump(d, open(p, 'w'))\n")
|
||||
self.assertEqual(self.run_one("unlink", unlink), [])
|
||||
self.assertTrue(self.run_one("relink", unlink))
|
||||
|
||||
def task_output(self, out: Path, parts: dict[str, list[str]], answers: dict[str, str]) -> None:
|
||||
for name, texts in parts.items():
|
||||
paths = []
|
||||
for i, t in enumerate(texts, 1):
|
||||
p = out / f"{name}-part-{i}.md"
|
||||
p.write_text(t, encoding="utf-8")
|
||||
paths.append(str(p))
|
||||
(out / f"{name}.parts.json").write_text(json.dumps({"parts": paths, "answer": answers[name]}),
|
||||
encoding="utf-8")
|
||||
|
||||
def test_task_output_is_judged_by_where_each_fragment_stands_and_what_its_task_holds(self) -> None:
|
||||
want = case("task")
|
||||
c = gate.fresh_copy(self.base, self.tmp, "task")
|
||||
labels = [gate.fragment_label(self.base.sources, t["key"]) for t in want["task"]]
|
||||
full = ["\n".join([lab] + t["must_contain"]) for lab, t in zip(labels, want["task"])]
|
||||
self.task_output(c.path("out"), {"t1": [full[0], full[1]], "t2": [full[2]]}, {"t1": "a1.json", "t2": "a2.json"})
|
||||
self.assertEqual(gate.check_task(want, c, self.base), [])
|
||||
short = gate.fresh_copy(self.base, self.tmp, "task-short")
|
||||
cut = full[1].replace("A queen cell looks like a peanut shell.", "")
|
||||
self.task_output(short.path("out"), {"t1": [full[0], cut], "t2": [full[2]]}, {"t1": "a1.json", "t2": "a2.json"})
|
||||
self.assertEqual(gate.check_task(want, short, self.base),
|
||||
[f"the task with {labels[1]} lacks 'A queen cell looks like a peanut shell.'"])
|
||||
twice = gate.fresh_copy(self.base, self.tmp, "task-twice")
|
||||
self.task_output(twice.path("out"), {"t1": [full[0], full[1]], "t2": [full[2], labels[0]]},
|
||||
{"t1": "a.json", "t2": "a.json"})
|
||||
problems = gate.check_task(want, twice, self.base)
|
||||
self.assertIn("two tasks share an answer path", problems)
|
||||
self.assertTrue(any("stands in 2 tasks" in p for p in problems))
|
||||
big = gate.fresh_copy(self.base, self.tmp, "task-big")
|
||||
self.task_output(big.path("out"), {"t1": ["\n".join(full) + "x" * kpack.PART_CHARS]}, {"t1": "a.json"})
|
||||
self.assertTrue(any("over 60000" in p for p in gate.check_task(want, big, self.base)))
|
||||
|
||||
def test_linkset_output_is_judged_by_klink_task_and_the_current_targets(self) -> None:
|
||||
want = case("linkset-two-candidates")
|
||||
c = gate.fresh_copy(self.base, self.tmp, "linkset")
|
||||
out = c.path("out")
|
||||
keys = [(x["page"], gate.sentence_key(x["sentence"], x.get("n", 0))) for x in want["linkset"]]
|
||||
|
||||
def group(n: int, page: str, key: str, ids: list[str]) -> None:
|
||||
(out / f"group-{n}.sentences.json").write_text(json.dumps(
|
||||
{"schema": klink.SENTENCES_SCHEMA, "pages": {page: [key]}}), encoding="utf-8")
|
||||
(out / f"group-{n}.catalogue").write_text("\n".join(ids) + "\n", encoding="utf-8")
|
||||
|
||||
group(1, *keys[0], ["smoker", "hive"])
|
||||
group(2, *keys[1], ["hive", "honey-harvest"])
|
||||
self.assertEqual(gate.check_linkset(want, c), [])
|
||||
(out / "group-2.catalogue").write_text("honey-harvest\n", encoding="utf-8")
|
||||
self.assertEqual(gate.check_linkset(want, c),
|
||||
[f"the catalogue for {keys[1]} lacks its current target hive.md"])
|
||||
(out / "group-2.sentences.json").unlink()
|
||||
self.assertTrue(gate.check_linkset(want, c)[0].startswith("sentences:"))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
552
gates/tests/test_wholedoc.py
Normal file
552
gates/tests/test_wholedoc.py
Normal file
|
|
@ -0,0 +1,552 @@
|
|||
"""Tests for gates/wholedoc.py: the whole-document and counting group (E2b), its cut, checker, split and gate.
|
||||
|
||||
Every fixture is synthetic text written here. No corpus text is used (CLAUDE.md rule 3).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "stubs"))
|
||||
|
||||
import multidoc # noqa: E402
|
||||
import wholedoc # noqa: E402
|
||||
import wholedoc_arm # noqa: E402
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
|
||||
|
||||
def long_doc() -> str:
|
||||
"""A 200-line synthetic document: facts at the top, the middle and the bottom, and six tagged items."""
|
||||
lines = [f"Filler line {i:03d} about nothing in particular." for i in range(200)]
|
||||
lines[5] = "The north gate opens at dawn for the carters."
|
||||
lines[100] = "The middle bridge carries a toll of copper coins."
|
||||
lines[190] = "The south wall was rebuilt after the flood year."
|
||||
for n, i in enumerate((10, 40, 70, 110, 150, 180), 1):
|
||||
lines[i] = f"- Item: the lantern maker number {chr(64 + n)} works here."
|
||||
return "\n".join(lines) + "\n"
|
||||
|
||||
|
||||
SOURCES = {
|
||||
"long.md": long_doc(),
|
||||
"short.md": "# Short\n\nOnly three lines here, far too short for a whole-document question.\n",
|
||||
"a.md": "alpha text\n", "b.md": "beta text\n", "c.md": "gamma text\n", "d.md": "delta text\n",
|
||||
}
|
||||
|
||||
|
||||
def whole_question(qid: str = "x01") -> dict[str, Any]:
|
||||
return {
|
||||
"id": qid,
|
||||
"type": "whole",
|
||||
"question": "What does the town record at its north gate, middle bridge and south wall?",
|
||||
"file": "long.md",
|
||||
"evidence": [
|
||||
{"span": "The north gate opens at dawn for the carters."},
|
||||
{"span": "The middle bridge carries a toll of copper coins."},
|
||||
{"span": "The south wall was rebuilt after the flood year."},
|
||||
],
|
||||
"answer_keys": [{"text": "dawn"}, {"text": "copper coins"}, {"text": "flood year"}],
|
||||
}
|
||||
|
||||
|
||||
def count_question(qid: str = "x02") -> dict[str, Any]:
|
||||
return {
|
||||
"id": qid,
|
||||
"type": "count",
|
||||
"question": "How many lantern makers does the document list?",
|
||||
"file": "long.md",
|
||||
"evidence": [{"span": f"- Item: the lantern maker number {chr(64 + n)} works here."} for n in range(1, 7)],
|
||||
"item_pattern": "(?m)^- Item: the lantern maker",
|
||||
"count": 6,
|
||||
}
|
||||
|
||||
|
||||
def e2_set(questions: list[list[str]]) -> dict[str, Any]:
|
||||
"""An E2-style dev set: one question per file list, ids q01.. in the given order."""
|
||||
return {
|
||||
"schema": "okf-wiki/multidoc-set/1",
|
||||
"questions": [
|
||||
{"id": f"q{n:02d}", "question": "?", "evidence": [{"file": f, "span": "x" * 20} for f in files],
|
||||
"answer_keys": []}
|
||||
for n, files in enumerate(questions, 1)
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
class Workspace:
|
||||
def __init__(self) -> None:
|
||||
self._tmp = tempfile.TemporaryDirectory()
|
||||
self.root = Path(self._tmp.name)
|
||||
self.sources = self.root / "sources"
|
||||
self.sources.mkdir()
|
||||
for name, text in SOURCES.items():
|
||||
(self.sources / name).write_text(text, encoding="utf-8")
|
||||
self.e2 = self.write_json("e2.json", e2_set([["long.md", "a.md", "b.md", "c.md"], ["short.md", "d.md"]]))
|
||||
|
||||
def close(self) -> None:
|
||||
self._tmp.cleanup()
|
||||
|
||||
def write_set(self, questions: list[dict[str, Any]], name: str = "set.json") -> Path:
|
||||
return self.write_json(name, {"schema": wholedoc.SET_SCHEMA, "questions": questions})
|
||||
|
||||
def write_json(self, name: str, data: Any) -> Path:
|
||||
path = self.root / name
|
||||
path.write_text(json.dumps(data), encoding="utf-8")
|
||||
return path
|
||||
|
||||
def cut_manifest(self, name: str = "manifest.json", max_files: int = 20) -> Path:
|
||||
path = self.root / name
|
||||
code, out = run(["cut", "--e2-set", str(self.e2), "--sources", str(self.sources), "--max-files",
|
||||
str(max_files), "--manifest", str(path)])
|
||||
assert code == 0, out
|
||||
return path
|
||||
|
||||
|
||||
def run(argv: list[str]) -> tuple[int, str]:
|
||||
out, err = io.StringIO(), io.StringIO()
|
||||
with contextlib.redirect_stdout(out), contextlib.redirect_stderr(err):
|
||||
code = wholedoc.main(argv)
|
||||
return code, out.getvalue() + err.getvalue()
|
||||
|
||||
|
||||
def sha256_file(path: Path) -> str:
|
||||
return hashlib.sha256(path.read_bytes()).hexdigest()
|
||||
|
||||
|
||||
class CutTests(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
self.ws = Workspace()
|
||||
|
||||
def tearDown(self) -> None:
|
||||
self.ws.close()
|
||||
|
||||
def test_greedy_fewest_new_files_ties_by_id(self) -> None:
|
||||
questions = [
|
||||
{"id": "q02", "evidence": [{"file": f} for f in ("a", "b", "c", "d")]},
|
||||
{"id": "q01", "evidence": [{"file": f} for f in ("a", "b", "e", "f")]},
|
||||
{"id": "q03", "evidence": [{"file": f} for f in ("a", "b", "c", "g")]},
|
||||
{"id": "q04", "evidence": [{"file": f} for f in ("h", "i", "j", "k")]},
|
||||
]
|
||||
chosen, files = wholedoc.compute_cut(questions, 7)
|
||||
self.assertEqual(chosen, ["q01", "q02", "q03"])
|
||||
self.assertEqual(files, sorted({"a", "b", "c", "d", "e", "f", "g"}))
|
||||
|
||||
def test_cut_stops_before_exceeding_the_limit(self) -> None:
|
||||
questions = [{"id": "q01", "evidence": [{"file": "a"}]}, {"id": "q02", "evidence": [{"file": f} for f in "bc"]}]
|
||||
chosen, files = wholedoc.compute_cut(questions, 2)
|
||||
self.assertEqual((chosen, files), (["q01"], ["a"]))
|
||||
|
||||
def test_cut_writes_thresholds_before_any_question(self) -> None:
|
||||
manifest = json.loads(self.ws.cut_manifest().read_text(encoding="utf-8"))
|
||||
self.assertEqual(manifest["schema"], wholedoc.MANIFEST_SCHEMA)
|
||||
self.assertEqual(manifest["thresholds"], wholedoc.thresholds())
|
||||
self.assertEqual(manifest["cut"]["files"], 6)
|
||||
self.assertEqual(manifest["cut"]["questions"], 2)
|
||||
self.assertEqual(manifest["cut"]["e2_set_sha256"], sha256_file(self.ws.e2))
|
||||
self.assertIsNone(manifest["dev"])
|
||||
self.assertIsNone(manifest["held_out"])
|
||||
self.assertEqual(manifest["sources"]["tree_sha256"], multidoc.tree_sha256(self.ws.sources))
|
||||
|
||||
def test_cut_refuses_to_overwrite_a_manifest(self) -> None:
|
||||
path = self.ws.cut_manifest()
|
||||
code, out = run(["cut", "--e2-set", str(self.ws.e2), "--sources", str(self.ws.sources), "--manifest", str(path)])
|
||||
self.assertEqual(code, 1)
|
||||
self.assertIn("exists", out)
|
||||
|
||||
|
||||
class CheckTests(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
self.ws = Workspace()
|
||||
self.manifest = self.ws.cut_manifest()
|
||||
|
||||
def tearDown(self) -> None:
|
||||
self.ws.close()
|
||||
|
||||
def check(self, questions: list[dict[str, Any]], e2: Path | None = None) -> tuple[int, str]:
|
||||
path = self.ws.write_set(questions)
|
||||
return run(["check", "--set", str(path), "--sources", str(self.ws.sources), "--manifest", str(self.manifest),
|
||||
"--e2-set", str(e2 or self.ws.e2)])
|
||||
|
||||
def test_good_group_passes_with_counts_from_the_set(self) -> None:
|
||||
code, out = self.check([whole_question("x01"), count_question("x02")])
|
||||
self.assertEqual(code, 0, out)
|
||||
self.assertIn("questions: 2 (whole 1, count 1)", out)
|
||||
self.assertIn("spans verbatim and unique: 9 of 9", out)
|
||||
self.assertIn("questions whose file is in the cut: 2 of 2", out)
|
||||
self.assertIn("whole: spread over >= 3 parts, first to last third: 1 of 1", out)
|
||||
self.assertIn("count: count = spans = pattern matches >= 5: 1 of 1", out)
|
||||
self.assertIn("count: answer not verbatim in the file: 1 of 1", out)
|
||||
self.assertIn("not answerable from one span: 2 of 2", out)
|
||||
self.assertIn("PASS", out)
|
||||
|
||||
def test_a_wording_key_fails_the_check(self) -> None:
|
||||
q = whole_question()
|
||||
q["answer_keys"][0] = {"text": "at dawn for the carters"}
|
||||
code, out = self.check([q, count_question("x02")])
|
||||
self.assertEqual(code, 1)
|
||||
self.assertIn("answer keys that are wording: 1 of 3", out)
|
||||
self.assertIn("whole: every key in a span, every span keyed: 1 of 1", out)
|
||||
|
||||
def test_a_negated_key_must_stand_negated_in_its_span(self) -> None:
|
||||
q = whole_question()
|
||||
q["answer_keys"][2]["negated"] = True
|
||||
code, out = self.check([q, count_question("x02")])
|
||||
self.assertEqual(code, 1)
|
||||
self.assertIn("negated keys negated in their span: 0 of 1", out)
|
||||
|
||||
def test_a_negated_key_negated_in_its_span_passes(self) -> None:
|
||||
long = self.ws.sources / "long.md"
|
||||
long.write_text(long.read_text(encoding="utf-8").replace("wall was rebuilt", "wall was not rebuilt"),
|
||||
encoding="utf-8")
|
||||
q = whole_question()
|
||||
q["evidence"][2]["span"] = "The south wall was not rebuilt after the flood year."
|
||||
q["answer_keys"][2]["negated"] = True
|
||||
code, out = self.check([q, count_question("x02")])
|
||||
self.assertEqual(code, 0, out)
|
||||
self.assertIn("negated keys negated in their span: 1 of 1", out)
|
||||
|
||||
def test_good_group_has_no_wording_keys(self) -> None:
|
||||
code, out = self.check([whole_question("x01"), count_question("x02")])
|
||||
self.assertIn("answer keys that are wording: 0 of 3", out)
|
||||
|
||||
def test_whole_without_the_last_third_fails(self) -> None:
|
||||
q = whole_question()
|
||||
q["evidence"][2] = {"span": "- Item: the lantern maker number D works here."}
|
||||
q["answer_keys"][2] = {"text": "number D"}
|
||||
code, out = self.check([q])
|
||||
self.assertEqual(code, 1)
|
||||
self.assertIn("whole: spread over >= 3 parts, first to last third: 0 of 1", out)
|
||||
|
||||
def test_whole_with_two_parts_fails(self) -> None:
|
||||
q = whole_question()
|
||||
q["evidence"][1] = {"span": "- Item: the lantern maker number A works here."}
|
||||
q["answer_keys"][1] = {"text": "number A"}
|
||||
code, out = self.check([q])
|
||||
self.assertEqual(code, 1)
|
||||
self.assertIn("first to last third: 0 of 1", out)
|
||||
|
||||
def test_whole_on_a_short_file_fails(self) -> None:
|
||||
q = whole_question()
|
||||
q["file"] = "short.md"
|
||||
code, out = self.check([q])
|
||||
self.assertEqual(code, 1)
|
||||
self.assertIn(f"whole: file >= {wholedoc.WHOLE_MIN_LINES} lines: 0 of 1", out)
|
||||
|
||||
def test_whole_answerable_from_one_span_fails(self) -> None:
|
||||
q = whole_question()
|
||||
q["answer_keys"] = [{"text": "north gate"}, {"text": "dawn"}, {"text": "carters"}]
|
||||
code, out = self.check([q])
|
||||
self.assertEqual(code, 1)
|
||||
self.assertIn("not answerable from one span: 0 of 1", out)
|
||||
|
||||
def test_whole_key_outside_its_spans_fails(self) -> None:
|
||||
q = whole_question()
|
||||
q["answer_keys"].append({"text": "Filler line"})
|
||||
code, out = self.check([q])
|
||||
self.assertEqual(code, 1)
|
||||
self.assertIn("whole: every key in a span, every span keyed: 0 of 1", out)
|
||||
|
||||
def test_span_that_occurs_twice_fails(self) -> None:
|
||||
q = whole_question()
|
||||
q["evidence"][0] = {"span": "about nothing in particular."}
|
||||
code, out = self.check([q])
|
||||
self.assertEqual(code, 1)
|
||||
self.assertIn("spans verbatim and unique: 2 of 3", out)
|
||||
|
||||
def test_count_that_disagrees_with_the_pattern_fails(self) -> None:
|
||||
q = count_question()
|
||||
q["item_pattern"] = "(?m)^- Item: the lantern maker number [A-E]"
|
||||
code, out = self.check([q])
|
||||
self.assertEqual(code, 1)
|
||||
self.assertIn("count: count = spans = pattern matches >= 5: 0 of 1", out)
|
||||
|
||||
def test_count_below_the_minimum_fails(self) -> None:
|
||||
q = count_question()
|
||||
q["evidence"] = q["evidence"][:4]
|
||||
q["item_pattern"] = "(?m)^- Item: the lantern maker number [A-D]"
|
||||
q["count"] = 4
|
||||
code, out = self.check([q])
|
||||
self.assertEqual(code, 1)
|
||||
self.assertIn("count: count = spans = pattern matches >= 5: 0 of 1", out)
|
||||
|
||||
def test_count_that_stands_in_the_file_fails(self) -> None:
|
||||
ws_file = self.ws.sources / "long.md"
|
||||
ws_file.write_text(ws_file.read_text(encoding="utf-8") + "There are six of them.\n", encoding="utf-8")
|
||||
self.manifest.unlink()
|
||||
self.manifest = self.ws.cut_manifest()
|
||||
code, out = self.check([count_question()])
|
||||
self.assertEqual(code, 1)
|
||||
self.assertIn("count: answer not verbatim in the file: 0 of 1", out)
|
||||
|
||||
def test_number_inside_a_version_is_not_the_answer(self) -> None:
|
||||
self.assertFalse(wholedoc.number_stands_in(6, "Released in 2.1.6 and v6.0, see item-6 and 16 or 60."))
|
||||
self.assertTrue(wholedoc.number_stands_in(6, "It holds 6 entries."))
|
||||
self.assertTrue(wholedoc.number_stands_in(6, "Up to 6."))
|
||||
self.assertTrue(wholedoc.number_stands_in(6, "Six rules apply."))
|
||||
|
||||
def test_file_outside_the_cut_fails(self) -> None:
|
||||
self.ws.e2 = self.ws.write_json("e2-small.json", e2_set([["short.md", "a.md"]]))
|
||||
self.manifest.unlink()
|
||||
self.manifest = self.ws.cut_manifest()
|
||||
code, out = self.check([whole_question()])
|
||||
self.assertEqual(code, 1)
|
||||
self.assertIn("not in the cut", out)
|
||||
self.assertIn("questions whose file is in the cut: 0 of 1", out)
|
||||
|
||||
def test_cut_that_differs_from_the_manifest_cannot_run(self) -> None:
|
||||
e2 = self.ws.write_json("e2-other.json", e2_set([["long.md", "a.md", "b.md", "c.md", "d.md", "short.md"]]))
|
||||
code, out = self.check([whole_question()], e2=e2)
|
||||
self.assertEqual(code, 1)
|
||||
self.assertIn("e2_set_sha256", out)
|
||||
|
||||
def test_empty_set_is_a_fail_not_a_pass(self) -> None:
|
||||
code, out = self.check([])
|
||||
self.assertEqual(code, 1)
|
||||
self.assertIn("FAIL", out)
|
||||
|
||||
def test_unknown_type_fails(self) -> None:
|
||||
q = whole_question()
|
||||
q["type"] = "other"
|
||||
code, out = self.check([q])
|
||||
self.assertEqual(code, 1)
|
||||
self.assertIn("type", out)
|
||||
|
||||
|
||||
class SplitTests(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
self.ws = Workspace()
|
||||
self.manifest = self.ws.cut_manifest()
|
||||
questions = [whole_question(f"w{n}") for n in range(1, 5)] + [count_question(f"c{n}") for n in range(1, 5)]
|
||||
self.set_path = self.ws.write_set(questions, "all.json")
|
||||
self.held = Path(tempfile.mkdtemp()) / "held.json"
|
||||
|
||||
def tearDown(self) -> None:
|
||||
self.ws.close()
|
||||
|
||||
def split(self, seed: int = 7, held: Path | None = None) -> tuple[int, str]:
|
||||
return run(["split", "--set", str(self.set_path), "--sources", str(self.ws.sources), "--manifest",
|
||||
str(self.manifest), "--e2-set", str(self.ws.e2), "--seed", str(seed), "--held-out-per-type", "1",
|
||||
"--dev-out", str(self.ws.root / "dev.json"), "--dev-label", "gates-local/wholedoc/dev.json",
|
||||
"--held-out", str(held or self.held)])
|
||||
|
||||
def test_split_is_stratified_renumbered_and_recorded(self) -> None:
|
||||
code, out = self.split()
|
||||
self.assertEqual(code, 0, out)
|
||||
dev = json.loads((self.ws.root / "dev.json").read_text(encoding="utf-8"))["questions"]
|
||||
held = json.loads(self.held.read_text(encoding="utf-8"))["questions"]
|
||||
self.assertEqual([q["id"] for q in dev], [f"d{n:02d}" for n in range(1, 7)])
|
||||
self.assertEqual([q["id"] for q in held], ["h01", "h02"])
|
||||
self.assertEqual(sorted(q["type"] for q in held), ["count", "whole"])
|
||||
manifest = json.loads(self.manifest.read_text(encoding="utf-8"))
|
||||
self.assertEqual(manifest["seed"], 7)
|
||||
self.assertEqual(manifest["dev"], {"file": "gates-local/wholedoc/dev.json",
|
||||
"sha256": sha256_file(self.ws.root / "dev.json"),
|
||||
"questions": 6, "whole": 3, "count": 3})
|
||||
self.assertEqual(manifest["held_out"], {"sha256": sha256_file(self.held), "questions": 2, "whole": 1,
|
||||
"count": 1})
|
||||
self.assertEqual(manifest["thresholds"], wholedoc.thresholds())
|
||||
|
||||
def test_same_seed_same_draw(self) -> None:
|
||||
self.split(seed=3)
|
||||
first = self.held.read_bytes()
|
||||
other = Path(tempfile.mkdtemp()) / "held.json"
|
||||
self.manifest.unlink()
|
||||
self.manifest = self.ws.cut_manifest()
|
||||
self.split(seed=3, held=other)
|
||||
self.assertEqual(first, other.read_bytes())
|
||||
|
||||
def test_refuses_held_out_inside_the_repository(self) -> None:
|
||||
code, out = self.split(held=REPO_ROOT / "gates" / "held.json")
|
||||
self.assertEqual(code, 1)
|
||||
self.assertIn("inside the repository", out)
|
||||
|
||||
def test_refuses_to_redraw(self) -> None:
|
||||
self.split()
|
||||
code, out = self.split()
|
||||
self.assertEqual(code, 1)
|
||||
self.assertIn("already", out)
|
||||
|
||||
def test_refuses_when_thresholds_changed_after_the_cut(self) -> None:
|
||||
manifest = json.loads(self.manifest.read_text(encoding="utf-8"))
|
||||
manifest["thresholds"]["count_min"] = 2
|
||||
self.manifest.write_text(json.dumps(manifest), encoding="utf-8")
|
||||
code, out = self.split()
|
||||
self.assertEqual(code, 1)
|
||||
self.assertIn("thresholds", out)
|
||||
|
||||
def test_refuses_a_set_that_fails_the_check(self) -> None:
|
||||
q = whole_question("w9")
|
||||
q["file"] = "short.md"
|
||||
self.set_path = self.ws.write_set([q, count_question("c9")], "bad.json")
|
||||
code, out = self.split()
|
||||
self.assertEqual(code, 1)
|
||||
self.assertFalse(self.held.exists())
|
||||
|
||||
|
||||
class GateTests(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
self.ws = Workspace()
|
||||
self.manifest = self.ws.cut_manifest()
|
||||
self.questions = [whole_question("d01"), count_question("d02")]
|
||||
self.set_path = self.ws.write_set(self.questions, "dev.json")
|
||||
manifest = json.loads(self.manifest.read_text(encoding="utf-8"))
|
||||
manifest["dev"] = {"sha256": sha256_file(self.set_path), "questions": 2, "whole": 1, "count": 1}
|
||||
manifest["held_out"] = {"sha256": "0" * 64, "questions": 2, "whole": 1, "count": 1}
|
||||
self.manifest.write_text(json.dumps(manifest), encoding="utf-8")
|
||||
|
||||
def tearDown(self) -> None:
|
||||
self.ws.close()
|
||||
|
||||
def gate(self, answers: Path, arm: str = "collection") -> tuple[int, str]:
|
||||
return run(["gate", "--set", str(self.set_path), "--manifest", str(self.manifest), "--sources",
|
||||
str(self.ws.sources), "--arm", arm, "--answers", str(answers)])
|
||||
|
||||
def answers(self, entries: list[dict[str, Any]], arm: str = "collection") -> Path:
|
||||
return self.ws.write_json("answers.json", {"arm": arm, "answers": entries})
|
||||
|
||||
def fasit(self) -> list[dict[str, Any]]:
|
||||
return [wholedoc_arm.fasit_answer(q) for q in self.questions]
|
||||
|
||||
def test_missing_arm_is_fail_closed_at_zero(self) -> None:
|
||||
code, out = self.gate(self.ws.root / "none.json")
|
||||
self.assertEqual(code, 1)
|
||||
self.assertIn("correct answer and source: 0 of 2", out)
|
||||
self.assertIn("fail-closed", out)
|
||||
|
||||
def test_fasit_scores_all_per_type(self) -> None:
|
||||
code, out = self.gate(self.answers(self.fasit()))
|
||||
self.assertEqual(code, 0, out)
|
||||
self.assertIn("whole: correct 1 of 1", out)
|
||||
self.assertIn("count: correct 1 of 1", out)
|
||||
self.assertIn("correct answer and source: 2 of 2", out)
|
||||
self.assertIn("GREEN", out)
|
||||
|
||||
def test_one_span_stub_scores_zero(self) -> None:
|
||||
code, out = self.gate(self.answers([wholedoc_arm.one_span_answer(q) for q in self.questions]))
|
||||
self.assertEqual(code, 0, out)
|
||||
self.assertIn("correct answer and source: 0 of 2", out)
|
||||
self.assertIn("RED", out)
|
||||
|
||||
def test_count_must_be_an_exact_integer(self) -> None:
|
||||
for wrong in (5, 7, "6", 6.0, None):
|
||||
a = self.fasit()
|
||||
a[1]["count"] = wrong
|
||||
code, out = self.gate(self.answers(a))
|
||||
self.assertIn("count: correct 0 of 1", out, repr(wrong))
|
||||
|
||||
def test_count_true_is_not_one(self) -> None:
|
||||
q = count_question("d02")
|
||||
a = wholedoc_arm.fasit_answer(q)
|
||||
a["count"] = True
|
||||
self.assertFalse(wholedoc.score_one(q, a, multidoc.SourceCache(self.ws.sources)).answer_ok)
|
||||
|
||||
def test_whole_missing_a_key_is_wrong(self) -> None:
|
||||
a = self.fasit()
|
||||
a[0]["answer"] = a[0]["answer"].replace("flood year", "a bad year")
|
||||
code, out = self.gate(self.answers(a))
|
||||
self.assertIn("whole: correct 0 of 1", out)
|
||||
|
||||
def test_citation_must_quote_the_file(self) -> None:
|
||||
a = self.fasit()
|
||||
a[0]["citations"] = [{"file": "long.md", "quote": "The north gate never opens."}]
|
||||
a[1]["citations"] = [{"file": "short.md", "quote": "Only three lines here"}]
|
||||
code, out = self.gate(self.answers(a))
|
||||
self.assertIn("correct source: 0 of 2", out)
|
||||
|
||||
def test_set_that_differs_from_the_manifest_is_refused(self) -> None:
|
||||
self.set_path.write_text(self.set_path.read_text(encoding="utf-8") + "\n", encoding="utf-8")
|
||||
code, out = self.gate(self.answers(self.fasit()))
|
||||
self.assertEqual(code, 1)
|
||||
self.assertIn("sha256", out)
|
||||
|
||||
def test_changed_sources_are_refused(self) -> None:
|
||||
(self.ws.sources / "a.md").write_text("changed\n", encoding="utf-8")
|
||||
code, out = self.gate(self.answers(self.fasit()))
|
||||
self.assertEqual(code, 1)
|
||||
self.assertIn("sources", out)
|
||||
|
||||
def test_manifest_before_the_split_is_refused(self) -> None:
|
||||
manifest = json.loads(self.manifest.read_text(encoding="utf-8"))
|
||||
manifest["dev"] = None
|
||||
self.manifest.write_text(json.dumps(manifest), encoding="utf-8")
|
||||
code, out = self.gate(self.answers(self.fasit()))
|
||||
self.assertEqual(code, 1)
|
||||
|
||||
def test_arm_name_mismatch_is_refused(self) -> None:
|
||||
code, out = self.gate(self.answers(self.fasit(), arm="filesearch"))
|
||||
self.assertEqual(code, 1)
|
||||
self.assertIn("arm", out)
|
||||
|
||||
def test_per_question_results_are_written_for_pairing(self) -> None:
|
||||
out_path = self.ws.root / "results.json"
|
||||
run(["gate", "--set", str(self.set_path), "--manifest", str(self.manifest), "--sources", str(self.ws.sources),
|
||||
"--arm", "collection", "--answers", str(self.answers(self.fasit()[:1])), "--out", str(out_path)])
|
||||
results = json.loads(out_path.read_text(encoding="utf-8"))
|
||||
self.assertEqual([(r["id"], r["type"], r["correct"]) for r in results["results"]],
|
||||
[("d01", "whole", True), ("d02", "count", False)])
|
||||
|
||||
|
||||
class ArmStubTests(unittest.TestCase):
|
||||
def test_arm_file_is_written_for_every_question(self) -> None:
|
||||
ws = Workspace()
|
||||
try:
|
||||
set_path = ws.write_set([whole_question("d01"), count_question("d02")])
|
||||
out = ws.root / "arm.json"
|
||||
code = wholedoc_arm.main(["one-span", str(set_path), str(out)])
|
||||
self.assertEqual(code, 0)
|
||||
data = json.loads(out.read_text(encoding="utf-8"))
|
||||
self.assertEqual(data["arm"], "one-span")
|
||||
self.assertEqual([a["id"] for a in data["answers"]], ["d01", "d02"])
|
||||
self.assertEqual(data["answers"][1]["count"], 1)
|
||||
finally:
|
||||
ws.close()
|
||||
|
||||
|
||||
class NegatedKeyTests(unittest.TestCase):
|
||||
def test_a_whole_question_scores_a_negated_key_by_its_negation(self) -> None:
|
||||
ws = Workspace()
|
||||
try:
|
||||
q = whole_question()
|
||||
q["answer_keys"][2]["negated"] = True
|
||||
cache = multidoc.SourceCache(ws.sources)
|
||||
cite = [{"file": "long.md", "quote": "The north gate opens at dawn for the carters."}]
|
||||
right = {"id": q["id"], "answer": "dawn, copper coins, not after the flood year", "citations": cite}
|
||||
wrong = {"id": q["id"], "answer": "dawn, copper coins, after the flood year", "citations": cite}
|
||||
self.assertTrue(wholedoc.score_one(q, right, cache).answer_ok)
|
||||
self.assertFalse(wholedoc.score_one(q, wrong, cache).answer_ok)
|
||||
finally:
|
||||
ws.close()
|
||||
|
||||
def test_the_fasit_states_a_negated_key_with_its_negation(self) -> None:
|
||||
q = whole_question()
|
||||
q["answer_keys"][2]["negated"] = True
|
||||
self.assertEqual(wholedoc_arm.fasit_answer(q)["answer"], "dawn; copper coins; not flood year")
|
||||
|
||||
|
||||
class NumberKeyTests(unittest.TestCase):
|
||||
def test_a_whole_question_scores_a_number_key_as_a_number(self) -> None:
|
||||
ws = Workspace()
|
||||
try:
|
||||
q = whole_question()
|
||||
q["answer_keys"][0] = {"text": "three"}
|
||||
cache = multidoc.SourceCache(ws.sources)
|
||||
cite = [{"file": "long.md", "quote": "The north gate opens at dawn for the carters."}]
|
||||
right = {"id": q["id"], "answer": "3 of them, copper coins, flood year", "citations": cite}
|
||||
wrong = {"id": q["id"], "answer": "33 of them, copper coins, flood year", "citations": cite}
|
||||
self.assertTrue(wholedoc.score_one(q, right, cache).answer_ok)
|
||||
self.assertFalse(wholedoc.score_one(q, wrong, cache).answer_ok)
|
||||
finally:
|
||||
ws.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Loading…
Add table
Add a link
Reference in a new issue