"""Tests for gates/wholedoc.py: the whole-document and counting group (E2b), its cut, checker, split and gate. Every fixture is synthetic text written here. No corpus text is used (CLAUDE.md rule 3). """ from __future__ import annotations import contextlib import hashlib import io import json import sys import tempfile import unittest from pathlib import Path from typing import Any sys.path.insert(0, str(Path(__file__).resolve().parents[1])) sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "stubs")) import multidoc # noqa: E402 import wholedoc # noqa: E402 import wholedoc_arm # noqa: E402 REPO_ROOT = Path(__file__).resolve().parents[2] def long_doc() -> str: """A 200-line synthetic document: facts at the top, the middle and the bottom, and six tagged items.""" lines = [f"Filler line {i:03d} about nothing in particular." for i in range(200)] lines[5] = "The north gate opens at dawn for the carters." lines[100] = "The middle bridge carries a toll of copper coins." lines[190] = "The south wall was rebuilt after the flood year." for n, i in enumerate((10, 40, 70, 110, 150, 180), 1): lines[i] = f"- Item: the lantern maker number {chr(64 + n)} works here." return "\n".join(lines) + "\n" SOURCES = { "long.md": long_doc(), "short.md": "# Short\n\nOnly three lines here, far too short for a whole-document question.\n", "a.md": "alpha text\n", "b.md": "beta text\n", "c.md": "gamma text\n", "d.md": "delta text\n", } def whole_question(qid: str = "x01") -> dict[str, Any]: return { "id": qid, "type": "whole", "question": "What does the town record at its north gate, middle bridge and south wall?", "file": "long.md", "evidence": [ {"span": "The north gate opens at dawn for the carters."}, {"span": "The middle bridge carries a toll of copper coins."}, {"span": "The south wall was rebuilt after the flood year."}, ], "answer_keys": [{"text": "dawn"}, {"text": "copper coins"}, {"text": "flood year"}], } def count_question(qid: str = "x02") -> dict[str, Any]: return { "id": qid, "type": "count", "question": "How many lantern makers does the document list?", "file": "long.md", "evidence": [{"span": f"- Item: the lantern maker number {chr(64 + n)} works here."} for n in range(1, 7)], "item_pattern": "(?m)^- Item: the lantern maker", "count": 6, } def e2_set(questions: list[list[str]]) -> dict[str, Any]: """An E2-style dev set: one question per file list, ids q01.. in the given order.""" return { "schema": "okf-wiki/multidoc-set/1", "questions": [ {"id": f"q{n:02d}", "question": "?", "evidence": [{"file": f, "span": "x" * 20} for f in files], "answer_keys": []} for n, files in enumerate(questions, 1) ], } class Workspace: def __init__(self) -> None: self._tmp = tempfile.TemporaryDirectory() self.root = Path(self._tmp.name) self.sources = self.root / "sources" self.sources.mkdir() for name, text in SOURCES.items(): (self.sources / name).write_text(text, encoding="utf-8") self.e2 = self.write_json("e2.json", e2_set([["long.md", "a.md", "b.md", "c.md"], ["short.md", "d.md"]])) def close(self) -> None: self._tmp.cleanup() def write_set(self, questions: list[dict[str, Any]], name: str = "set.json") -> Path: return self.write_json(name, {"schema": wholedoc.SET_SCHEMA, "questions": questions}) def write_json(self, name: str, data: Any) -> Path: path = self.root / name path.write_text(json.dumps(data), encoding="utf-8") return path def cut_manifest(self, name: str = "manifest.json", max_files: int = 20) -> Path: path = self.root / name code, out = run(["cut", "--e2-set", str(self.e2), "--sources", str(self.sources), "--max-files", str(max_files), "--manifest", str(path)]) assert code == 0, out return path def run(argv: list[str]) -> tuple[int, str]: out, err = io.StringIO(), io.StringIO() with contextlib.redirect_stdout(out), contextlib.redirect_stderr(err): code = wholedoc.main(argv) return code, out.getvalue() + err.getvalue() def sha256_file(path: Path) -> str: return hashlib.sha256(path.read_bytes()).hexdigest() class CutTests(unittest.TestCase): def setUp(self) -> None: self.ws = Workspace() def tearDown(self) -> None: self.ws.close() def test_greedy_fewest_new_files_ties_by_id(self) -> None: questions = [ {"id": "q02", "evidence": [{"file": f} for f in ("a", "b", "c", "d")]}, {"id": "q01", "evidence": [{"file": f} for f in ("a", "b", "e", "f")]}, {"id": "q03", "evidence": [{"file": f} for f in ("a", "b", "c", "g")]}, {"id": "q04", "evidence": [{"file": f} for f in ("h", "i", "j", "k")]}, ] chosen, files = wholedoc.compute_cut(questions, 7) self.assertEqual(chosen, ["q01", "q02", "q03"]) self.assertEqual(files, sorted({"a", "b", "c", "d", "e", "f", "g"})) def test_cut_stops_before_exceeding_the_limit(self) -> None: questions = [{"id": "q01", "evidence": [{"file": "a"}]}, {"id": "q02", "evidence": [{"file": f} for f in "bc"]}] chosen, files = wholedoc.compute_cut(questions, 2) self.assertEqual((chosen, files), (["q01"], ["a"])) def test_cut_writes_thresholds_before_any_question(self) -> None: manifest = json.loads(self.ws.cut_manifest().read_text(encoding="utf-8")) self.assertEqual(manifest["schema"], wholedoc.MANIFEST_SCHEMA) self.assertEqual(manifest["thresholds"], wholedoc.thresholds()) self.assertEqual(manifest["cut"]["files"], 6) self.assertEqual(manifest["cut"]["questions"], 2) self.assertEqual(manifest["cut"]["e2_set_sha256"], sha256_file(self.ws.e2)) self.assertIsNone(manifest["dev"]) self.assertIsNone(manifest["held_out"]) self.assertEqual(manifest["sources"]["tree_sha256"], multidoc.tree_sha256(self.ws.sources)) def test_cut_refuses_to_overwrite_a_manifest(self) -> None: path = self.ws.cut_manifest() code, out = run(["cut", "--e2-set", str(self.ws.e2), "--sources", str(self.ws.sources), "--manifest", str(path)]) self.assertEqual(code, 1) self.assertIn("exists", out) class CheckTests(unittest.TestCase): def setUp(self) -> None: self.ws = Workspace() self.manifest = self.ws.cut_manifest() def tearDown(self) -> None: self.ws.close() def check(self, questions: list[dict[str, Any]], e2: Path | None = None) -> tuple[int, str]: path = self.ws.write_set(questions) return run(["check", "--set", str(path), "--sources", str(self.ws.sources), "--manifest", str(self.manifest), "--e2-set", str(e2 or self.ws.e2)]) def test_good_group_passes_with_counts_from_the_set(self) -> None: code, out = self.check([whole_question("x01"), count_question("x02")]) self.assertEqual(code, 0, out) self.assertIn("questions: 2 (whole 1, count 1)", out) self.assertIn("spans verbatim and unique: 9 of 9", out) self.assertIn("questions whose file is in the cut: 2 of 2", out) self.assertIn("whole: spread over >= 3 parts, first to last third: 1 of 1", out) self.assertIn("count: count = spans = pattern matches >= 5: 1 of 1", out) self.assertIn("count: answer not verbatim in the file: 1 of 1", out) self.assertIn("not answerable from one span: 2 of 2", out) self.assertIn("PASS", out) def test_a_wording_key_fails_the_check(self) -> None: q = whole_question() q["answer_keys"][0] = {"text": "at dawn for the carters"} code, out = self.check([q, count_question("x02")]) self.assertEqual(code, 1) self.assertIn("answer keys that are wording: 1 of 3", out) self.assertIn("whole: every key in a span, every span keyed: 1 of 1", out) def test_a_negated_key_must_stand_negated_in_its_span(self) -> None: q = whole_question() q["answer_keys"][2]["negated"] = True code, out = self.check([q, count_question("x02")]) self.assertEqual(code, 1) self.assertIn("negated keys negated in their span: 0 of 1", out) def test_a_negated_key_negated_in_its_span_passes(self) -> None: long = self.ws.sources / "long.md" long.write_text(long.read_text(encoding="utf-8").replace("wall was rebuilt", "wall was not rebuilt"), encoding="utf-8") q = whole_question() q["evidence"][2]["span"] = "The south wall was not rebuilt after the flood year." q["answer_keys"][2]["negated"] = True code, out = self.check([q, count_question("x02")]) self.assertEqual(code, 0, out) self.assertIn("negated keys negated in their span: 1 of 1", out) def test_good_group_has_no_wording_keys(self) -> None: code, out = self.check([whole_question("x01"), count_question("x02")]) self.assertIn("answer keys that are wording: 0 of 3", out) def test_whole_without_the_last_third_fails(self) -> None: q = whole_question() q["evidence"][2] = {"span": "- Item: the lantern maker number D works here."} q["answer_keys"][2] = {"text": "number D"} code, out = self.check([q]) self.assertEqual(code, 1) self.assertIn("whole: spread over >= 3 parts, first to last third: 0 of 1", out) def test_whole_with_two_parts_fails(self) -> None: q = whole_question() q["evidence"][1] = {"span": "- Item: the lantern maker number A works here."} q["answer_keys"][1] = {"text": "number A"} code, out = self.check([q]) self.assertEqual(code, 1) self.assertIn("first to last third: 0 of 1", out) def test_whole_on_a_short_file_fails(self) -> None: q = whole_question() q["file"] = "short.md" code, out = self.check([q]) self.assertEqual(code, 1) self.assertIn(f"whole: file >= {wholedoc.WHOLE_MIN_LINES} lines: 0 of 1", out) def test_whole_answerable_from_one_span_fails(self) -> None: q = whole_question() q["answer_keys"] = [{"text": "north gate"}, {"text": "dawn"}, {"text": "carters"}] code, out = self.check([q]) self.assertEqual(code, 1) self.assertIn("not answerable from one span: 0 of 1", out) def test_whole_key_outside_its_spans_fails(self) -> None: q = whole_question() q["answer_keys"].append({"text": "Filler line"}) code, out = self.check([q]) self.assertEqual(code, 1) self.assertIn("whole: every key in a span, every span keyed: 0 of 1", out) def test_span_that_occurs_twice_fails(self) -> None: q = whole_question() q["evidence"][0] = {"span": "about nothing in particular."} code, out = self.check([q]) self.assertEqual(code, 1) self.assertIn("spans verbatim and unique: 2 of 3", out) def test_count_that_disagrees_with_the_pattern_fails(self) -> None: q = count_question() q["item_pattern"] = "(?m)^- Item: the lantern maker number [A-E]" code, out = self.check([q]) self.assertEqual(code, 1) self.assertIn("count: count = spans = pattern matches >= 5: 0 of 1", out) def test_count_below_the_minimum_fails(self) -> None: q = count_question() q["evidence"] = q["evidence"][:4] q["item_pattern"] = "(?m)^- Item: the lantern maker number [A-D]" q["count"] = 4 code, out = self.check([q]) self.assertEqual(code, 1) self.assertIn("count: count = spans = pattern matches >= 5: 0 of 1", out) def test_count_that_stands_in_the_file_fails(self) -> None: ws_file = self.ws.sources / "long.md" ws_file.write_text(ws_file.read_text(encoding="utf-8") + "There are six of them.\n", encoding="utf-8") self.manifest.unlink() self.manifest = self.ws.cut_manifest() code, out = self.check([count_question()]) self.assertEqual(code, 1) self.assertIn("count: answer not verbatim in the file: 0 of 1", out) def test_number_inside_a_version_is_not_the_answer(self) -> None: self.assertFalse(wholedoc.number_stands_in(6, "Released in 2.1.6 and v6.0, see item-6 and 16 or 60.")) self.assertTrue(wholedoc.number_stands_in(6, "It holds 6 entries.")) self.assertTrue(wholedoc.number_stands_in(6, "Up to 6.")) self.assertTrue(wholedoc.number_stands_in(6, "Six rules apply.")) def test_file_outside_the_cut_fails(self) -> None: self.ws.e2 = self.ws.write_json("e2-small.json", e2_set([["short.md", "a.md"]])) self.manifest.unlink() self.manifest = self.ws.cut_manifest() code, out = self.check([whole_question()]) self.assertEqual(code, 1) self.assertIn("not in the cut", out) self.assertIn("questions whose file is in the cut: 0 of 1", out) def test_cut_that_differs_from_the_manifest_cannot_run(self) -> None: e2 = self.ws.write_json("e2-other.json", e2_set([["long.md", "a.md", "b.md", "c.md", "d.md", "short.md"]])) code, out = self.check([whole_question()], e2=e2) self.assertEqual(code, 1) self.assertIn("e2_set_sha256", out) def test_empty_set_is_a_fail_not_a_pass(self) -> None: code, out = self.check([]) self.assertEqual(code, 1) self.assertIn("FAIL", out) def test_unknown_type_fails(self) -> None: q = whole_question() q["type"] = "other" code, out = self.check([q]) self.assertEqual(code, 1) self.assertIn("type", out) class SplitTests(unittest.TestCase): def setUp(self) -> None: self.ws = Workspace() self.manifest = self.ws.cut_manifest() questions = [whole_question(f"w{n}") for n in range(1, 5)] + [count_question(f"c{n}") for n in range(1, 5)] self.set_path = self.ws.write_set(questions, "all.json") self.held = Path(tempfile.mkdtemp()) / "held.json" def tearDown(self) -> None: self.ws.close() def split(self, seed: int = 7, held: Path | None = None) -> tuple[int, str]: return run(["split", "--set", str(self.set_path), "--sources", str(self.ws.sources), "--manifest", str(self.manifest), "--e2-set", str(self.ws.e2), "--seed", str(seed), "--held-out-per-type", "1", "--dev-out", str(self.ws.root / "dev.json"), "--dev-label", "gates-local/wholedoc/dev.json", "--held-out", str(held or self.held)]) def test_split_is_stratified_renumbered_and_recorded(self) -> None: code, out = self.split() self.assertEqual(code, 0, out) dev = json.loads((self.ws.root / "dev.json").read_text(encoding="utf-8"))["questions"] held = json.loads(self.held.read_text(encoding="utf-8"))["questions"] self.assertEqual([q["id"] for q in dev], [f"d{n:02d}" for n in range(1, 7)]) self.assertEqual([q["id"] for q in held], ["h01", "h02"]) self.assertEqual(sorted(q["type"] for q in held), ["count", "whole"]) manifest = json.loads(self.manifest.read_text(encoding="utf-8")) self.assertEqual(manifest["seed"], 7) self.assertEqual(manifest["dev"], {"file": "gates-local/wholedoc/dev.json", "sha256": sha256_file(self.ws.root / "dev.json"), "questions": 6, "whole": 3, "count": 3}) self.assertEqual(manifest["held_out"], {"sha256": sha256_file(self.held), "questions": 2, "whole": 1, "count": 1}) self.assertEqual(manifest["thresholds"], wholedoc.thresholds()) def test_same_seed_same_draw(self) -> None: self.split(seed=3) first = self.held.read_bytes() other = Path(tempfile.mkdtemp()) / "held.json" self.manifest.unlink() self.manifest = self.ws.cut_manifest() self.split(seed=3, held=other) self.assertEqual(first, other.read_bytes()) def test_refuses_held_out_inside_the_repository(self) -> None: code, out = self.split(held=REPO_ROOT / "gates" / "held.json") self.assertEqual(code, 1) self.assertIn("inside the repository", out) def test_refuses_to_redraw(self) -> None: self.split() code, out = self.split() self.assertEqual(code, 1) self.assertIn("already", out) def test_refuses_when_thresholds_changed_after_the_cut(self) -> None: manifest = json.loads(self.manifest.read_text(encoding="utf-8")) manifest["thresholds"]["count_min"] = 2 self.manifest.write_text(json.dumps(manifest), encoding="utf-8") code, out = self.split() self.assertEqual(code, 1) self.assertIn("thresholds", out) def test_refuses_a_set_that_fails_the_check(self) -> None: q = whole_question("w9") q["file"] = "short.md" self.set_path = self.ws.write_set([q, count_question("c9")], "bad.json") code, out = self.split() self.assertEqual(code, 1) self.assertFalse(self.held.exists()) class GateTests(unittest.TestCase): def setUp(self) -> None: self.ws = Workspace() self.manifest = self.ws.cut_manifest() self.questions = [whole_question("d01"), count_question("d02")] self.set_path = self.ws.write_set(self.questions, "dev.json") manifest = json.loads(self.manifest.read_text(encoding="utf-8")) manifest["dev"] = {"sha256": sha256_file(self.set_path), "questions": 2, "whole": 1, "count": 1} manifest["held_out"] = {"sha256": "0" * 64, "questions": 2, "whole": 1, "count": 1} self.manifest.write_text(json.dumps(manifest), encoding="utf-8") def tearDown(self) -> None: self.ws.close() def gate(self, answers: Path, arm: str = "collection") -> tuple[int, str]: return run(["gate", "--set", str(self.set_path), "--manifest", str(self.manifest), "--sources", str(self.ws.sources), "--arm", arm, "--answers", str(answers)]) def answers(self, entries: list[dict[str, Any]], arm: str = "collection") -> Path: return self.ws.write_json("answers.json", {"arm": arm, "answers": entries}) def fasit(self) -> list[dict[str, Any]]: return [wholedoc_arm.fasit_answer(q) for q in self.questions] def test_missing_arm_is_fail_closed_at_zero(self) -> None: code, out = self.gate(self.ws.root / "none.json") self.assertEqual(code, 1) self.assertIn("correct answer and source: 0 of 2", out) self.assertIn("fail-closed", out) def test_fasit_scores_all_per_type(self) -> None: code, out = self.gate(self.answers(self.fasit())) self.assertEqual(code, 0, out) self.assertIn("whole: correct 1 of 1", out) self.assertIn("count: correct 1 of 1", out) self.assertIn("correct answer and source: 2 of 2", out) self.assertIn("GREEN", out) def test_one_span_stub_scores_zero(self) -> None: code, out = self.gate(self.answers([wholedoc_arm.one_span_answer(q) for q in self.questions])) self.assertEqual(code, 0, out) self.assertIn("correct answer and source: 0 of 2", out) self.assertIn("RED", out) def test_count_must_be_an_exact_integer(self) -> None: for wrong in (5, 7, "6", 6.0, None): a = self.fasit() a[1]["count"] = wrong code, out = self.gate(self.answers(a)) self.assertIn("count: correct 0 of 1", out, repr(wrong)) def test_count_true_is_not_one(self) -> None: q = count_question("d02") a = wholedoc_arm.fasit_answer(q) a["count"] = True self.assertFalse(wholedoc.score_one(q, a, multidoc.SourceCache(self.ws.sources)).answer_ok) def test_whole_missing_a_key_is_wrong(self) -> None: a = self.fasit() a[0]["answer"] = a[0]["answer"].replace("flood year", "a bad year") code, out = self.gate(self.answers(a)) self.assertIn("whole: correct 0 of 1", out) def test_citation_must_quote_the_file(self) -> None: a = self.fasit() a[0]["citations"] = [{"file": "long.md", "quote": "The north gate never opens."}] a[1]["citations"] = [{"file": "short.md", "quote": "Only three lines here"}] code, out = self.gate(self.answers(a)) self.assertIn("correct source: 0 of 2", out) def test_set_that_differs_from_the_manifest_is_refused(self) -> None: self.set_path.write_text(self.set_path.read_text(encoding="utf-8") + "\n", encoding="utf-8") code, out = self.gate(self.answers(self.fasit())) self.assertEqual(code, 1) self.assertIn("sha256", out) def test_changed_sources_are_refused(self) -> None: (self.ws.sources / "a.md").write_text("changed\n", encoding="utf-8") code, out = self.gate(self.answers(self.fasit())) self.assertEqual(code, 1) self.assertIn("sources", out) def test_manifest_before_the_split_is_refused(self) -> None: manifest = json.loads(self.manifest.read_text(encoding="utf-8")) manifest["dev"] = None self.manifest.write_text(json.dumps(manifest), encoding="utf-8") code, out = self.gate(self.answers(self.fasit())) self.assertEqual(code, 1) def test_arm_name_mismatch_is_refused(self) -> None: code, out = self.gate(self.answers(self.fasit(), arm="filesearch")) self.assertEqual(code, 1) self.assertIn("arm", out) def test_per_question_results_are_written_for_pairing(self) -> None: out_path = self.ws.root / "results.json" run(["gate", "--set", str(self.set_path), "--manifest", str(self.manifest), "--sources", str(self.ws.sources), "--arm", "collection", "--answers", str(self.answers(self.fasit()[:1])), "--out", str(out_path)]) results = json.loads(out_path.read_text(encoding="utf-8")) self.assertEqual([(r["id"], r["type"], r["correct"]) for r in results["results"]], [("d01", "whole", True), ("d02", "count", False)]) class ArmStubTests(unittest.TestCase): def test_arm_file_is_written_for_every_question(self) -> None: ws = Workspace() try: set_path = ws.write_set([whole_question("d01"), count_question("d02")]) out = ws.root / "arm.json" code = wholedoc_arm.main(["one-span", str(set_path), str(out)]) self.assertEqual(code, 0) data = json.loads(out.read_text(encoding="utf-8")) self.assertEqual(data["arm"], "one-span") self.assertEqual([a["id"] for a in data["answers"]], ["d01", "d02"]) self.assertEqual(data["answers"][1]["count"], 1) finally: ws.close() class NegatedKeyTests(unittest.TestCase): def test_a_whole_question_scores_a_negated_key_by_its_negation(self) -> None: ws = Workspace() try: q = whole_question() q["answer_keys"][2]["negated"] = True cache = multidoc.SourceCache(ws.sources) cite = [{"file": "long.md", "quote": "The north gate opens at dawn for the carters."}] right = {"id": q["id"], "answer": "dawn, copper coins, not after the flood year", "citations": cite} wrong = {"id": q["id"], "answer": "dawn, copper coins, after the flood year", "citations": cite} self.assertTrue(wholedoc.score_one(q, right, cache).answer_ok) self.assertFalse(wholedoc.score_one(q, wrong, cache).answer_ok) finally: ws.close() def test_the_fasit_states_a_negated_key_with_its_negation(self) -> None: q = whole_question() q["answer_keys"][2]["negated"] = True self.assertEqual(wholedoc_arm.fasit_answer(q)["answer"], "dawn; copper coins; not flood year") class NumberKeyTests(unittest.TestCase): def test_a_whole_question_scores_a_number_key_as_a_number(self) -> None: ws = Workspace() try: q = whole_question() q["answer_keys"][0] = {"text": "three"} cache = multidoc.SourceCache(ws.sources) cite = [{"file": "long.md", "quote": "The north gate opens at dawn for the carters."}] right = {"id": q["id"], "answer": "3 of them, copper coins, flood year", "citations": cite} wrong = {"id": q["id"], "answer": "33 of them, copper coins, flood year", "citations": cite} self.assertTrue(wholedoc.score_one(q, right, cache).answer_ok) self.assertFalse(wholedoc.score_one(q, wrong, cache).answer_ok) finally: ws.close() if __name__ == "__main__": unittest.main()