"""Tests for gates/multidoc.py: the multi-document set checker, the held-out split and the answer gate. Every fixture is synthetic text written here. No corpus text is used (CLAUDE.md rule 3). """ from __future__ import annotations import contextlib import hashlib import io import json import sys import tempfile import unittest from pathlib import Path from typing import Any sys.path.insert(0, str(Path(__file__).resolve().parents[1])) import multidoc # noqa: E402 REPO_ROOT = Path(__file__).resolve().parents[2] SOURCES = { "alpha.md": "# Alpha\n\nThe widget timeout is 45 seconds by default.\nSet `WIDGET_MODE=fast` to skip checks.\n", "beta.md": "# Beta\n\nBeta adds the --frobnicate flag for batch runs.\nIt was introduced in the spring build.\n", "gamma.md": "# Gamma\n\nGamma stores its cache under the .gamma-cache directory.\n", "delta.md": "# Delta\n\nDelta rejects requests larger than 12 megabytes with error E_TOO_BIG.\n", "epsilon.md": "# Epsilon\n\nEpsilon logs every call to the audit trail named ledger-7.\n", } def good_question(qid: str = "x01") -> dict[str, Any]: return { "id": qid, "question": "How do the four synthetic tools fit together?", "evidence": [ {"file": "alpha.md", "span": "The widget timeout is 45 seconds by default."}, {"file": "beta.md", "span": "Beta adds the --frobnicate flag for batch runs."}, {"file": "gamma.md", "span": "Gamma stores its cache under the .gamma-cache directory."}, {"file": "delta.md", "span": "rejects requests larger than 12 megabytes with error E_TOO_BIG"}, ], "answer_keys": [ {"file": "alpha.md", "text": "45 seconds"}, {"file": "beta.md", "text": "--frobnicate"}, {"file": "gamma.md", "text": ".gamma-cache"}, {"file": "delta.md", "text": "E_TOO_BIG"}, ], } def perfect_answer(q: dict[str, Any]) -> dict[str, Any]: keys = " ".join(k["text"] for k in q["answer_keys"]) return { "id": q["id"], "answer": "The answer mentions " + keys + ".", "citations": [{"file": e["file"], "quote": e["span"]} for e in q["evidence"]], } class Workspace: """A temporary sources directory and helpers to write sets, manifests and answers.""" def __init__(self) -> None: self._tmp = tempfile.TemporaryDirectory() self.root = Path(self._tmp.name) self.sources = self.root / "sources" self.sources.mkdir() for name, text in SOURCES.items(): (self.sources / name).write_text(text, encoding="utf-8") def close(self) -> None: self._tmp.cleanup() def write_set(self, questions: list[dict[str, Any]], name: str = "set.json") -> Path: path = self.root / name path.write_text(json.dumps({"schema": multidoc.SET_SCHEMA, "questions": questions}), encoding="utf-8") return path def write_json(self, name: str, data: Any) -> Path: path = self.root / name path.write_text(json.dumps(data), encoding="utf-8") return path def run(argv: list[str]) -> tuple[int, str]: out = io.StringIO() err = io.StringIO() with contextlib.redirect_stdout(out), contextlib.redirect_stderr(err): code = multidoc.main(argv) return code, out.getvalue() + err.getvalue() def sha256_file(path: Path) -> str: return hashlib.sha256(path.read_bytes()).hexdigest() class CheckTests(unittest.TestCase): def setUp(self) -> None: self.ws = Workspace() def tearDown(self) -> None: self.ws.close() def check(self, questions: list[dict[str, Any]]) -> tuple[int, str]: path = self.ws.write_set(questions) return run(["check", "--set", str(path), "--sources", str(self.ws.sources)]) def test_good_set_passes_with_counts_from_the_set(self) -> None: code, out = self.check([good_question("x01"), good_question("x02")]) self.assertEqual(code, 0, out) self.assertIn("spans verbatim: 8 of 8", out) self.assertIn("questions with every span verbatim: 2 of 2", out) self.assertIn("questions with >= 4 distinct files: 2 of 2", out) self.assertIn("questions with a key per file: 2 of 2", out) self.assertIn("PASS", out) def test_span_one_character_off_fails(self) -> None: q = good_question() q["evidence"][0]["span"] = "The widget timeout is 46 seconds by default." code, out = self.check([q]) self.assertEqual(code, 1) self.assertIn("spans verbatim: 3 of 4", out) self.assertIn("questions with every span verbatim: 0 of 1", out) self.assertIn("FAIL", out) def test_span_match_is_exact_not_whitespace_normalised(self) -> None: q = good_question() q["evidence"][0]["span"] = "The widget timeout is 45 seconds by default." code, out = self.check([q]) self.assertEqual(code, 1) self.assertIn("spans verbatim: 3 of 4", out) def test_three_distinct_files_fail_the_minimum(self) -> None: q = good_question() q["evidence"][3] = {"file": "alpha.md", "span": "Set `WIDGET_MODE=fast` to skip checks."} q["answer_keys"][3] = {"file": "alpha.md", "text": "WIDGET_MODE=fast"} code, out = self.check([q]) self.assertEqual(code, 1) self.assertIn("questions with >= 4 distinct files: 0 of 1", out) def test_file_missing_from_sources_fails(self) -> None: q = good_question() q["evidence"][0]["file"] = "missing.md" q["answer_keys"][0]["file"] = "missing.md" code, out = self.check([q]) self.assertEqual(code, 1) self.assertIn("spans verbatim: 3 of 4", out) def test_path_outside_sources_is_an_error(self) -> None: for bad in ("../alpha.md", "sub/alpha.md", "/etc/hosts"): q = good_question() q["evidence"][0]["file"] = bad q["answer_keys"][0]["file"] = bad code, out = self.check([q]) self.assertEqual(code, 1, bad) self.assertIn("not a plain file name", out) def test_key_outside_its_files_spans_fails(self) -> None: q = good_question() q["answer_keys"][0]["text"] = "WIDGET_MODE" # in alpha.md, but not in the cited span code, out = self.check([q]) self.assertEqual(code, 1) self.assertIn("questions with a key per file: 0 of 1", out) def test_key_match_ignores_case(self) -> None: q = good_question() q["answer_keys"][3]["text"] = "e_too_big" code, out = self.check([q]) self.assertEqual(code, 0, out) def test_evidence_file_without_a_key_fails(self) -> None: q = good_question() del q["answer_keys"][2] code, out = self.check([q]) self.assertEqual(code, 1) self.assertIn("questions with a key per file: 0 of 1", out) def test_duplicate_ids_fail(self) -> None: code, out = self.check([good_question("x01"), good_question("x01")]) self.assertEqual(code, 1) self.assertIn("duplicate id", out) def test_empty_set_is_a_fail_not_a_pass(self) -> None: code, out = self.check([]) self.assertEqual(code, 1) self.assertIn("FAIL", out) def test_a_wording_key_fails_even_inside_its_span(self) -> None: q = good_question() q["answer_keys"][0] = {"file": "alpha.md", "text": "is 45 seconds by default"} code, out = self.check([q]) self.assertEqual(code, 1) self.assertIn("questions with a key per file: 1 of 1", out) self.assertIn("answer keys that are wording: 1 of 4", out) def test_good_set_has_no_wording_keys(self) -> None: code, out = self.check([good_question()]) self.assertIn("answer keys that are wording: 0 of 4", out) def test_short_span_and_short_key_fail(self) -> None: q = good_question() q["evidence"][0]["span"] = "45 seconds" code, out = self.check([q]) self.assertEqual(code, 1) self.assertIn("span length", out) q = good_question() q["answer_keys"][0]["text"] = "se" code, out = self.check([q]) self.assertEqual(code, 1) self.assertIn("key length", out) def test_missing_sources_directory_cannot_run(self) -> None: path = self.ws.write_set([good_question()]) code, out = run(["check", "--set", str(path), "--sources", str(self.ws.root / "nope")]) self.assertEqual(code, 1) self.assertIn("FAIL", out) def test_unreadable_set_cannot_run(self) -> None: path = self.ws.root / "broken.json" path.write_text("{not json", encoding="utf-8") code, out = run(["check", "--set", str(path), "--sources", str(self.ws.sources)]) self.assertEqual(code, 1) self.assertIn("FAIL", out) def test_a_negated_key_must_stand_negated_in_its_span(self) -> None: q = good_question() q["answer_keys"][0]["negated"] = True code, out = self.check([q]) self.assertEqual(code, 1) self.assertIn("negated keys negated in their span: 0 of 1", out) def test_a_negated_key_negated_in_its_span_passes(self) -> None: (self.ws.sources / "alpha.md").write_text("The widget is not synced back to the hub.\n", encoding="utf-8") q = good_question() q["evidence"][0]["span"] = "The widget is not synced back to the hub." q["answer_keys"][0] = {"file": "alpha.md", "text": "synced", "negated": True} code, out = self.check([q]) self.assertEqual(code, 0, out) self.assertIn("negated keys negated in their span: 1 of 1", out) def test_negated_must_be_true_or_absent(self) -> None: q = good_question() q["answer_keys"][0]["negated"] = "yes" code, out = self.check([q]) self.assertEqual(code, 1) self.assertIn("negated", out) class NegatedKeyTests(unittest.TestCase): KEY = {"file": "a.md", "text": "synced", "negated": True} def test_a_negation_within_six_words_before_the_key(self) -> None: for text in ("Changes are not synced back.", "It isn't synced.", "It isn\u2019t synced.", "Changes are no longer synced.", "It is never, ever synced.", "Not one two three four five synced.", "Some are synced; others are not synced."): self.assertTrue(multidoc.key_holds(self.KEY, text), text) def test_no_negation_or_one_out_of_reach_fails(self) -> None: for text in ("Changes are synced back.", "It is not late. Changes are synced.", "Not late; synced.", "Not one two three four five six synced.", "Synced, but not back.", "Not here:\nsynced."): self.assertFalse(multidoc.key_holds(self.KEY, text), text) def test_every_negation_in_the_list(self) -> None: for neg in ("not", "no", "never", "none", "neither", "nor", "without", "cannot", "can't", "doesn't", "won't"): self.assertTrue(multidoc.key_holds(self.KEY, f"It {neg} synced."), neg) def test_an_unmarked_key_holds_with_or_without_a_negation(self) -> None: key = {"file": "a.md", "text": "synced"} self.assertTrue(multidoc.key_holds(key, "Changes are synced back.")) self.assertTrue(multidoc.key_holds(key, "Changes are not synced back.")) def test_the_gate_scores_a_negated_key_by_its_negation(self) -> None: ws = Workspace() try: (ws.sources / "alpha.md").write_text("The widget is not synced back to the hub.\n", encoding="utf-8") q = good_question() q["evidence"][0]["span"] = "The widget is not synced back to the hub." q["answer_keys"][0] = {"file": "alpha.md", "text": "synced", "negated": True} cache = multidoc.SourceCache(ws.sources) right = perfect_answer(q) right["answer"] = "The widget is not synced; --frobnicate, .gamma-cache, E_TOO_BIG." wrong = dict(right, answer="The widget is synced; --frobnicate, .gamma-cache, E_TOO_BIG.") self.assertTrue(multidoc.score_one(q, right, cache).answer_ok) self.assertFalse(multidoc.score_one(q, wrong, cache).answer_ok) finally: ws.close() class SplitTests(unittest.TestCase): def setUp(self) -> None: self.ws = Workspace() self.questions = [good_question(f"x{i:02d}") for i in range(1, 11)] for i, q in enumerate(self.questions): q["question"] = f"Synthetic question number {i + 1}?" self.all_path = self.ws.write_set(self.questions, "all.json") def tearDown(self) -> None: self.ws.close() def split(self, seed: int, tag: str, held_dir: Path | None = None) -> tuple[int, str, Path, Path, Path]: dev = self.ws.root / f"dev-{tag}.json" held = (held_dir or self.ws.root / f"held-{tag}") / "held.json" manifest = self.ws.root / f"manifest-{tag}.json" code, out = run([ "split", "--set", str(self.all_path), "--sources", str(self.ws.sources), "--seed", str(seed), "--held-out-count", "3", "--dev-out", str(dev), "--held-out", str(held), "--manifest", str(manifest), ]) return code, out, dev, held, manifest def load(self, path: Path) -> list[dict[str, Any]]: data: dict[str, Any] = json.loads(path.read_text(encoding="utf-8")) questions: list[dict[str, Any]] = data["questions"] return questions def test_split_counts_renumbers_and_partitions(self) -> None: code, out, dev, held, _ = self.split(7, "a") self.assertEqual(code, 0, out) dev_q, held_q = self.load(dev), self.load(held) self.assertEqual((len(dev_q), len(held_q)), (7, 3)) self.assertEqual([q["id"] for q in dev_q], [f"d{i:02d}" for i in range(1, 8)]) self.assertEqual([q["id"] for q in held_q], [f"h{i:02d}" for i in range(1, 4)]) texts = sorted(q["question"] for q in dev_q + held_q) self.assertEqual(texts, sorted(q["question"] for q in self.questions)) def test_same_seed_same_draw_other_seed_other_draw(self) -> None: _, _, _, held_a, _ = self.split(7, "a") _, _, _, held_b, _ = self.split(7, "b") self.assertEqual(held_a.read_bytes(), held_b.read_bytes()) draws = set() for seed in range(1, 6): _, _, _, held, _ = self.split(seed, f"s{seed}") draws.add(tuple(q["question"] for q in self.load(held))) self.assertGreater(len(draws), 1) def test_manifest_carries_hashes_and_nothing_about_the_held_out_questions(self) -> None: code, out, dev, held, manifest = self.split(7, "a") self.assertEqual(code, 0, out) data = json.loads(manifest.read_text(encoding="utf-8")) self.assertEqual(data["seed"], 7) self.assertEqual(data["dev"]["sha256"], sha256_file(dev)) self.assertEqual(data["dev"]["questions"], 7) self.assertEqual(data["held_out"]["sha256"], sha256_file(held)) self.assertEqual(data["held_out"]["questions"], 3) self.assertEqual(data["sources"]["files"], len(SOURCES)) self.assertEqual(data["sources"]["tree_sha256"], multidoc.tree_sha256(self.ws.sources)) manifest_text = manifest.read_text(encoding="utf-8") for q in self.load(held): self.assertNotIn(q["question"], manifest_text) self.assertNotIn(q["question"], out) self.assertNotIn(str(held.parent), manifest_text) def test_refuses_held_out_inside_the_repository(self) -> None: code, out, _, _, _ = self.split(7, "in", held_dir=REPO_ROOT / "gates-local" / "should-not-exist") self.assertEqual(code, 1) self.assertIn("inside the repository", out) self.assertFalse((REPO_ROOT / "gates-local" / "should-not-exist").exists()) def test_refuses_to_overwrite_a_frozen_held_out_file(self) -> None: code, out, _, _, _ = self.split(7, "a") self.assertEqual(code, 0, out) code, out, _, _, _ = self.split(7, "a") self.assertEqual(code, 1) self.assertIn("already exists", out) def test_refuses_a_set_that_fails_the_check(self) -> None: self.questions[0]["evidence"][0]["span"] = "This sentence is not in any source file." self.all_path = self.ws.write_set(self.questions, "all.json") code, out, dev, held, _ = self.split(7, "bad") self.assertEqual(code, 1) self.assertFalse(dev.exists()) self.assertFalse(held.exists()) class GateTests(unittest.TestCase): def setUp(self) -> None: self.ws = Workspace() self.questions = [good_question("d01"), good_question("d02")] self.set_path = self.ws.write_set(self.questions, "dev.json") self.manifest = self.ws.write_json("manifest.json", { "schema": multidoc.MANIFEST_SCHEMA, "seed": 1, "dev": {"sha256": sha256_file(self.set_path), "questions": 2}, "held_out": {"sha256": "0" * 64, "questions": 1}, "sources": {"files": len(SOURCES), "tree_sha256": multidoc.tree_sha256(self.ws.sources)}, }) def tearDown(self) -> None: self.ws.close() def gate(self, answers: Path, arm: str = "collection") -> tuple[int, str]: return run([ "gate", "--set", str(self.set_path), "--manifest", str(self.manifest), "--sources", str(self.ws.sources), "--arm", arm, "--answers", str(answers), ]) def answers(self, entries: list[dict[str, Any]], arm: str = "collection") -> Path: return self.ws.write_json("answers.json", {"arm": arm, "answers": entries}) def test_missing_arm_is_fail_closed_at_zero(self) -> None: code, out = self.gate(self.ws.root / "no-such-arm.json") self.assertEqual(code, 1) self.assertIn("correct answer and source: 0 of 2", out) self.assertIn("fail-closed", out) def test_perfect_answers_score_all(self) -> None: code, out = self.gate(self.answers([perfect_answer(q) for q in self.questions])) self.assertEqual(code, 0, out) self.assertIn("correct answer and source: 2 of 2", out) def test_answer_missing_a_key_is_wrong(self) -> None: a = perfect_answer(self.questions[0]) a["answer"] = a["answer"].replace("E_TOO_BIG", "a big error") code, out = self.gate(self.answers([a, perfect_answer(self.questions[1])])) self.assertEqual(code, 0, out) self.assertIn("correct answer: 1 of 2", out) self.assertIn("correct answer and source: 1 of 2", out) def test_answer_key_match_ignores_case_backticks_and_whitespace(self) -> None: a = perfect_answer(self.questions[0]) a["answer"] = "Use `--FROBNICATE`, wait 45\n seconds, look in .gamma-cache, expect e_too_big." code, out = self.gate(self.answers([a, perfect_answer(self.questions[1])])) self.assertIn("correct answer: 2 of 2", out) def test_citation_not_verbatim_is_wrong_source(self) -> None: a = perfect_answer(self.questions[0]) a["citations"][1]["quote"] = "Beta adds a flag that nobody documented." code, out = self.gate(self.answers([a, perfect_answer(self.questions[1])])) self.assertIn("correct source: 1 of 2", out) self.assertIn("correct answer and source: 1 of 2", out) def test_citation_whitespace_is_normalised(self) -> None: a = perfect_answer(self.questions[0]) a["citations"][0]["quote"] = "The widget timeout is\n45 seconds by default." code, out = self.gate(self.answers([a, perfect_answer(self.questions[1])])) self.assertIn("correct source: 2 of 2", out) def test_citations_missing_one_file_is_wrong_source(self) -> None: a = perfect_answer(self.questions[0]) del a["citations"][2] code, out = self.gate(self.answers([a, perfect_answer(self.questions[1])])) self.assertIn("correct source: 1 of 2", out) def test_citation_outside_sources_counts_for_nothing(self) -> None: a = perfect_answer(self.questions[0]) a["citations"][0]["file"] = "../sources/alpha.md" code, out = self.gate(self.answers([a, perfect_answer(self.questions[1])])) self.assertIn("correct source: 1 of 2", out) def test_missing_entry_counts_wrong_and_fails(self) -> None: code, out = self.gate(self.answers([perfect_answer(self.questions[0])])) self.assertEqual(code, 1) self.assertIn("correct answer and source: 1 of 2", out) self.assertIn("missing answers: 1", out) def test_set_that_differs_from_the_manifest_is_refused(self) -> None: self.set_path.write_text(self.set_path.read_text(encoding="utf-8") + "\n", encoding="utf-8") code, out = self.gate(self.answers([perfect_answer(q) for q in self.questions])) self.assertEqual(code, 1) self.assertIn("sha256", out) def test_changed_sources_are_refused(self) -> None: (self.ws.sources / "alpha.md").write_text("changed\n", encoding="utf-8") code, out = self.gate(self.answers([perfect_answer(q) for q in self.questions])) self.assertEqual(code, 1) self.assertIn("sources", out) def test_arm_name_mismatch_is_refused(self) -> None: code, out = self.gate(self.answers([perfect_answer(q) for q in self.questions], arm="filesearch")) self.assertEqual(code, 1) self.assertIn("arm", out) def test_per_question_results_are_written_for_pairing(self) -> None: a = perfect_answer(self.questions[0]) answers = self.answers([a]) out_path = self.ws.root / "results.json" run([ "gate", "--set", str(self.set_path), "--manifest", str(self.manifest), "--sources", str(self.ws.sources), "--arm", "collection", "--answers", str(answers), "--out", str(out_path), ]) results = json.loads(out_path.read_text(encoding="utf-8")) self.assertEqual(results["arm"], "collection") self.assertEqual([r["id"] for r in results["results"]], ["d01", "d02"]) self.assertEqual([r["correct"] for r in results["results"]], [True, False]) class NumberKeyTests(unittest.TestCase): """A key that is a number matches as a number: digits or its English word, never inside a longer number.""" def test_a_number_key_matches_digits_or_word_but_not_inside_a_longer_number(self) -> None: for text in ("It retries 5 times.", "RETRIES=5", "five attempts", "(5)"): self.assertTrue(multidoc.key_in("5", text), text) for text in ("15 attempts", "version 2.1.5", "v5", "50", "5.2 seconds", "fifty"): self.assertFalse(multidoc.key_in("5", text), text) def test_a_number_word_key_matches_the_digits(self) -> None: self.assertTrue(multidoc.key_in("three", "fails 3 times in a row")) self.assertTrue(multidoc.key_in("Three", "three times")) self.assertFalse(multidoc.key_in("three", "33 times")) def test_a_percentage_key_matches_the_percentage(self) -> None: for text in ("at 10% of the window", "at 10 % of it", "10 percent"): self.assertTrue(multidoc.key_in("10%", text), text) for text in ("110%", "10 of them", "2.10%"): self.assertFalse(multidoc.key_in("10%", text), text) def test_other_keys_still_match_as_normalised_text(self) -> None: self.assertTrue(multidoc.key_in("model version", "Pin their **model versions**.")) self.assertTrue(multidoc.key_in("2.1.9", "since 2.1.92")) def test_check_accepts_a_short_number_key_standing_alone_in_its_span(self) -> None: ws = Workspace() try: q = good_question() q["answer_keys"][3] = {"file": "delta.md", "text": "12"} code, out = run(["check", "--set", str(ws.write_set([q])), "--sources", str(ws.sources)]) self.assertEqual(code, 0, out) q["answer_keys"][3] = {"file": "delta.md", "text": "2"} code, out = run(["check", "--set", str(ws.write_set([q])), "--sources", str(ws.sources)]) self.assertEqual(code, 1) self.assertIn("questions with a key per file: 0 of 1", out) finally: ws.close() def test_the_gate_scores_a_number_key_as_a_number(self) -> None: ws = Workspace() try: q = good_question() q["answer_keys"][3] = {"file": "delta.md", "text": "12"} cache = multidoc.SourceCache(ws.sources) right = perfect_answer(q) self.assertTrue(multidoc.score_one(q, right, cache).answer_ok) wrong = dict(right, answer=right["answer"].replace(" 12", " 112")) self.assertFalse(multidoc.score_one(q, wrong, cache).answer_ok) finally: ws.close() if __name__ == "__main__": unittest.main()