okf-wiki/gates/tests/test_wholedoc.py
Kjell Tore Guttormsen 0fd86a23a3
okf-wiki 1.1.1
First public release, published as one snapshot. What changed between
versions is in CHANGELOG.md.

Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
2026-11-14 21:03:20 +01:00

552 lines
24 KiB
Python

"""Tests for gates/wholedoc.py: the whole-document and counting group (E2b), its cut, checker, split and gate.
Every fixture is synthetic text written here. No corpus text is used (CLAUDE.md rule 3).
"""
from __future__ import annotations
import contextlib
import hashlib
import io
import json
import sys
import tempfile
import unittest
from pathlib import Path
from typing import Any
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "stubs"))
import multidoc # noqa: E402
import wholedoc # noqa: E402
import wholedoc_arm # noqa: E402
REPO_ROOT = Path(__file__).resolve().parents[2]
def long_doc() -> str:
"""A 200-line synthetic document: facts at the top, the middle and the bottom, and six tagged items."""
lines = [f"Filler line {i:03d} about nothing in particular." for i in range(200)]
lines[5] = "The north gate opens at dawn for the carters."
lines[100] = "The middle bridge carries a toll of copper coins."
lines[190] = "The south wall was rebuilt after the flood year."
for n, i in enumerate((10, 40, 70, 110, 150, 180), 1):
lines[i] = f"- Item: the lantern maker number {chr(64 + n)} works here."
return "\n".join(lines) + "\n"
SOURCES = {
"long.md": long_doc(),
"short.md": "# Short\n\nOnly three lines here, far too short for a whole-document question.\n",
"a.md": "alpha text\n", "b.md": "beta text\n", "c.md": "gamma text\n", "d.md": "delta text\n",
}
def whole_question(qid: str = "x01") -> dict[str, Any]:
return {
"id": qid,
"type": "whole",
"question": "What does the town record at its north gate, middle bridge and south wall?",
"file": "long.md",
"evidence": [
{"span": "The north gate opens at dawn for the carters."},
{"span": "The middle bridge carries a toll of copper coins."},
{"span": "The south wall was rebuilt after the flood year."},
],
"answer_keys": [{"text": "dawn"}, {"text": "copper coins"}, {"text": "flood year"}],
}
def count_question(qid: str = "x02") -> dict[str, Any]:
return {
"id": qid,
"type": "count",
"question": "How many lantern makers does the document list?",
"file": "long.md",
"evidence": [{"span": f"- Item: the lantern maker number {chr(64 + n)} works here."} for n in range(1, 7)],
"item_pattern": "(?m)^- Item: the lantern maker",
"count": 6,
}
def e2_set(questions: list[list[str]]) -> dict[str, Any]:
"""An E2-style dev set: one question per file list, ids q01.. in the given order."""
return {
"schema": "okf-wiki/multidoc-set/1",
"questions": [
{"id": f"q{n:02d}", "question": "?", "evidence": [{"file": f, "span": "x" * 20} for f in files],
"answer_keys": []}
for n, files in enumerate(questions, 1)
],
}
class Workspace:
def __init__(self) -> None:
self._tmp = tempfile.TemporaryDirectory()
self.root = Path(self._tmp.name)
self.sources = self.root / "sources"
self.sources.mkdir()
for name, text in SOURCES.items():
(self.sources / name).write_text(text, encoding="utf-8")
self.e2 = self.write_json("e2.json", e2_set([["long.md", "a.md", "b.md", "c.md"], ["short.md", "d.md"]]))
def close(self) -> None:
self._tmp.cleanup()
def write_set(self, questions: list[dict[str, Any]], name: str = "set.json") -> Path:
return self.write_json(name, {"schema": wholedoc.SET_SCHEMA, "questions": questions})
def write_json(self, name: str, data: Any) -> Path:
path = self.root / name
path.write_text(json.dumps(data), encoding="utf-8")
return path
def cut_manifest(self, name: str = "manifest.json", max_files: int = 20) -> Path:
path = self.root / name
code, out = run(["cut", "--e2-set", str(self.e2), "--sources", str(self.sources), "--max-files",
str(max_files), "--manifest", str(path)])
assert code == 0, out
return path
def run(argv: list[str]) -> tuple[int, str]:
out, err = io.StringIO(), io.StringIO()
with contextlib.redirect_stdout(out), contextlib.redirect_stderr(err):
code = wholedoc.main(argv)
return code, out.getvalue() + err.getvalue()
def sha256_file(path: Path) -> str:
return hashlib.sha256(path.read_bytes()).hexdigest()
class CutTests(unittest.TestCase):
def setUp(self) -> None:
self.ws = Workspace()
def tearDown(self) -> None:
self.ws.close()
def test_greedy_fewest_new_files_ties_by_id(self) -> None:
questions = [
{"id": "q02", "evidence": [{"file": f} for f in ("a", "b", "c", "d")]},
{"id": "q01", "evidence": [{"file": f} for f in ("a", "b", "e", "f")]},
{"id": "q03", "evidence": [{"file": f} for f in ("a", "b", "c", "g")]},
{"id": "q04", "evidence": [{"file": f} for f in ("h", "i", "j", "k")]},
]
chosen, files = wholedoc.compute_cut(questions, 7)
self.assertEqual(chosen, ["q01", "q02", "q03"])
self.assertEqual(files, sorted({"a", "b", "c", "d", "e", "f", "g"}))
def test_cut_stops_before_exceeding_the_limit(self) -> None:
questions = [{"id": "q01", "evidence": [{"file": "a"}]}, {"id": "q02", "evidence": [{"file": f} for f in "bc"]}]
chosen, files = wholedoc.compute_cut(questions, 2)
self.assertEqual((chosen, files), (["q01"], ["a"]))
def test_cut_writes_thresholds_before_any_question(self) -> None:
manifest = json.loads(self.ws.cut_manifest().read_text(encoding="utf-8"))
self.assertEqual(manifest["schema"], wholedoc.MANIFEST_SCHEMA)
self.assertEqual(manifest["thresholds"], wholedoc.thresholds())
self.assertEqual(manifest["cut"]["files"], 6)
self.assertEqual(manifest["cut"]["questions"], 2)
self.assertEqual(manifest["cut"]["e2_set_sha256"], sha256_file(self.ws.e2))
self.assertIsNone(manifest["dev"])
self.assertIsNone(manifest["held_out"])
self.assertEqual(manifest["sources"]["tree_sha256"], multidoc.tree_sha256(self.ws.sources))
def test_cut_refuses_to_overwrite_a_manifest(self) -> None:
path = self.ws.cut_manifest()
code, out = run(["cut", "--e2-set", str(self.ws.e2), "--sources", str(self.ws.sources), "--manifest", str(path)])
self.assertEqual(code, 1)
self.assertIn("exists", out)
class CheckTests(unittest.TestCase):
def setUp(self) -> None:
self.ws = Workspace()
self.manifest = self.ws.cut_manifest()
def tearDown(self) -> None:
self.ws.close()
def check(self, questions: list[dict[str, Any]], e2: Path | None = None) -> tuple[int, str]:
path = self.ws.write_set(questions)
return run(["check", "--set", str(path), "--sources", str(self.ws.sources), "--manifest", str(self.manifest),
"--e2-set", str(e2 or self.ws.e2)])
def test_good_group_passes_with_counts_from_the_set(self) -> None:
code, out = self.check([whole_question("x01"), count_question("x02")])
self.assertEqual(code, 0, out)
self.assertIn("questions: 2 (whole 1, count 1)", out)
self.assertIn("spans verbatim and unique: 9 of 9", out)
self.assertIn("questions whose file is in the cut: 2 of 2", out)
self.assertIn("whole: spread over >= 3 parts, first to last third: 1 of 1", out)
self.assertIn("count: count = spans = pattern matches >= 5: 1 of 1", out)
self.assertIn("count: answer not verbatim in the file: 1 of 1", out)
self.assertIn("not answerable from one span: 2 of 2", out)
self.assertIn("PASS", out)
def test_a_wording_key_fails_the_check(self) -> None:
q = whole_question()
q["answer_keys"][0] = {"text": "at dawn for the carters"}
code, out = self.check([q, count_question("x02")])
self.assertEqual(code, 1)
self.assertIn("answer keys that are wording: 1 of 3", out)
self.assertIn("whole: every key in a span, every span keyed: 1 of 1", out)
def test_a_negated_key_must_stand_negated_in_its_span(self) -> None:
q = whole_question()
q["answer_keys"][2]["negated"] = True
code, out = self.check([q, count_question("x02")])
self.assertEqual(code, 1)
self.assertIn("negated keys negated in their span: 0 of 1", out)
def test_a_negated_key_negated_in_its_span_passes(self) -> None:
long = self.ws.sources / "long.md"
long.write_text(long.read_text(encoding="utf-8").replace("wall was rebuilt", "wall was not rebuilt"),
encoding="utf-8")
q = whole_question()
q["evidence"][2]["span"] = "The south wall was not rebuilt after the flood year."
q["answer_keys"][2]["negated"] = True
code, out = self.check([q, count_question("x02")])
self.assertEqual(code, 0, out)
self.assertIn("negated keys negated in their span: 1 of 1", out)
def test_good_group_has_no_wording_keys(self) -> None:
code, out = self.check([whole_question("x01"), count_question("x02")])
self.assertIn("answer keys that are wording: 0 of 3", out)
def test_whole_without_the_last_third_fails(self) -> None:
q = whole_question()
q["evidence"][2] = {"span": "- Item: the lantern maker number D works here."}
q["answer_keys"][2] = {"text": "number D"}
code, out = self.check([q])
self.assertEqual(code, 1)
self.assertIn("whole: spread over >= 3 parts, first to last third: 0 of 1", out)
def test_whole_with_two_parts_fails(self) -> None:
q = whole_question()
q["evidence"][1] = {"span": "- Item: the lantern maker number A works here."}
q["answer_keys"][1] = {"text": "number A"}
code, out = self.check([q])
self.assertEqual(code, 1)
self.assertIn("first to last third: 0 of 1", out)
def test_whole_on_a_short_file_fails(self) -> None:
q = whole_question()
q["file"] = "short.md"
code, out = self.check([q])
self.assertEqual(code, 1)
self.assertIn(f"whole: file >= {wholedoc.WHOLE_MIN_LINES} lines: 0 of 1", out)
def test_whole_answerable_from_one_span_fails(self) -> None:
q = whole_question()
q["answer_keys"] = [{"text": "north gate"}, {"text": "dawn"}, {"text": "carters"}]
code, out = self.check([q])
self.assertEqual(code, 1)
self.assertIn("not answerable from one span: 0 of 1", out)
def test_whole_key_outside_its_spans_fails(self) -> None:
q = whole_question()
q["answer_keys"].append({"text": "Filler line"})
code, out = self.check([q])
self.assertEqual(code, 1)
self.assertIn("whole: every key in a span, every span keyed: 0 of 1", out)
def test_span_that_occurs_twice_fails(self) -> None:
q = whole_question()
q["evidence"][0] = {"span": "about nothing in particular."}
code, out = self.check([q])
self.assertEqual(code, 1)
self.assertIn("spans verbatim and unique: 2 of 3", out)
def test_count_that_disagrees_with_the_pattern_fails(self) -> None:
q = count_question()
q["item_pattern"] = "(?m)^- Item: the lantern maker number [A-E]"
code, out = self.check([q])
self.assertEqual(code, 1)
self.assertIn("count: count = spans = pattern matches >= 5: 0 of 1", out)
def test_count_below_the_minimum_fails(self) -> None:
q = count_question()
q["evidence"] = q["evidence"][:4]
q["item_pattern"] = "(?m)^- Item: the lantern maker number [A-D]"
q["count"] = 4
code, out = self.check([q])
self.assertEqual(code, 1)
self.assertIn("count: count = spans = pattern matches >= 5: 0 of 1", out)
def test_count_that_stands_in_the_file_fails(self) -> None:
ws_file = self.ws.sources / "long.md"
ws_file.write_text(ws_file.read_text(encoding="utf-8") + "There are six of them.\n", encoding="utf-8")
self.manifest.unlink()
self.manifest = self.ws.cut_manifest()
code, out = self.check([count_question()])
self.assertEqual(code, 1)
self.assertIn("count: answer not verbatim in the file: 0 of 1", out)
def test_number_inside_a_version_is_not_the_answer(self) -> None:
self.assertFalse(wholedoc.number_stands_in(6, "Released in 2.1.6 and v6.0, see item-6 and 16 or 60."))
self.assertTrue(wholedoc.number_stands_in(6, "It holds 6 entries."))
self.assertTrue(wholedoc.number_stands_in(6, "Up to 6."))
self.assertTrue(wholedoc.number_stands_in(6, "Six rules apply."))
def test_file_outside_the_cut_fails(self) -> None:
self.ws.e2 = self.ws.write_json("e2-small.json", e2_set([["short.md", "a.md"]]))
self.manifest.unlink()
self.manifest = self.ws.cut_manifest()
code, out = self.check([whole_question()])
self.assertEqual(code, 1)
self.assertIn("not in the cut", out)
self.assertIn("questions whose file is in the cut: 0 of 1", out)
def test_cut_that_differs_from_the_manifest_cannot_run(self) -> None:
e2 = self.ws.write_json("e2-other.json", e2_set([["long.md", "a.md", "b.md", "c.md", "d.md", "short.md"]]))
code, out = self.check([whole_question()], e2=e2)
self.assertEqual(code, 1)
self.assertIn("e2_set_sha256", out)
def test_empty_set_is_a_fail_not_a_pass(self) -> None:
code, out = self.check([])
self.assertEqual(code, 1)
self.assertIn("FAIL", out)
def test_unknown_type_fails(self) -> None:
q = whole_question()
q["type"] = "other"
code, out = self.check([q])
self.assertEqual(code, 1)
self.assertIn("type", out)
class SplitTests(unittest.TestCase):
def setUp(self) -> None:
self.ws = Workspace()
self.manifest = self.ws.cut_manifest()
questions = [whole_question(f"w{n}") for n in range(1, 5)] + [count_question(f"c{n}") for n in range(1, 5)]
self.set_path = self.ws.write_set(questions, "all.json")
self.held = Path(tempfile.mkdtemp()) / "held.json"
def tearDown(self) -> None:
self.ws.close()
def split(self, seed: int = 7, held: Path | None = None) -> tuple[int, str]:
return run(["split", "--set", str(self.set_path), "--sources", str(self.ws.sources), "--manifest",
str(self.manifest), "--e2-set", str(self.ws.e2), "--seed", str(seed), "--held-out-per-type", "1",
"--dev-out", str(self.ws.root / "dev.json"), "--dev-label", "gates-local/wholedoc/dev.json",
"--held-out", str(held or self.held)])
def test_split_is_stratified_renumbered_and_recorded(self) -> None:
code, out = self.split()
self.assertEqual(code, 0, out)
dev = json.loads((self.ws.root / "dev.json").read_text(encoding="utf-8"))["questions"]
held = json.loads(self.held.read_text(encoding="utf-8"))["questions"]
self.assertEqual([q["id"] for q in dev], [f"d{n:02d}" for n in range(1, 7)])
self.assertEqual([q["id"] for q in held], ["h01", "h02"])
self.assertEqual(sorted(q["type"] for q in held), ["count", "whole"])
manifest = json.loads(self.manifest.read_text(encoding="utf-8"))
self.assertEqual(manifest["seed"], 7)
self.assertEqual(manifest["dev"], {"file": "gates-local/wholedoc/dev.json",
"sha256": sha256_file(self.ws.root / "dev.json"),
"questions": 6, "whole": 3, "count": 3})
self.assertEqual(manifest["held_out"], {"sha256": sha256_file(self.held), "questions": 2, "whole": 1,
"count": 1})
self.assertEqual(manifest["thresholds"], wholedoc.thresholds())
def test_same_seed_same_draw(self) -> None:
self.split(seed=3)
first = self.held.read_bytes()
other = Path(tempfile.mkdtemp()) / "held.json"
self.manifest.unlink()
self.manifest = self.ws.cut_manifest()
self.split(seed=3, held=other)
self.assertEqual(first, other.read_bytes())
def test_refuses_held_out_inside_the_repository(self) -> None:
code, out = self.split(held=REPO_ROOT / "gates" / "held.json")
self.assertEqual(code, 1)
self.assertIn("inside the repository", out)
def test_refuses_to_redraw(self) -> None:
self.split()
code, out = self.split()
self.assertEqual(code, 1)
self.assertIn("already", out)
def test_refuses_when_thresholds_changed_after_the_cut(self) -> None:
manifest = json.loads(self.manifest.read_text(encoding="utf-8"))
manifest["thresholds"]["count_min"] = 2
self.manifest.write_text(json.dumps(manifest), encoding="utf-8")
code, out = self.split()
self.assertEqual(code, 1)
self.assertIn("thresholds", out)
def test_refuses_a_set_that_fails_the_check(self) -> None:
q = whole_question("w9")
q["file"] = "short.md"
self.set_path = self.ws.write_set([q, count_question("c9")], "bad.json")
code, out = self.split()
self.assertEqual(code, 1)
self.assertFalse(self.held.exists())
class GateTests(unittest.TestCase):
def setUp(self) -> None:
self.ws = Workspace()
self.manifest = self.ws.cut_manifest()
self.questions = [whole_question("d01"), count_question("d02")]
self.set_path = self.ws.write_set(self.questions, "dev.json")
manifest = json.loads(self.manifest.read_text(encoding="utf-8"))
manifest["dev"] = {"sha256": sha256_file(self.set_path), "questions": 2, "whole": 1, "count": 1}
manifest["held_out"] = {"sha256": "0" * 64, "questions": 2, "whole": 1, "count": 1}
self.manifest.write_text(json.dumps(manifest), encoding="utf-8")
def tearDown(self) -> None:
self.ws.close()
def gate(self, answers: Path, arm: str = "collection") -> tuple[int, str]:
return run(["gate", "--set", str(self.set_path), "--manifest", str(self.manifest), "--sources",
str(self.ws.sources), "--arm", arm, "--answers", str(answers)])
def answers(self, entries: list[dict[str, Any]], arm: str = "collection") -> Path:
return self.ws.write_json("answers.json", {"arm": arm, "answers": entries})
def fasit(self) -> list[dict[str, Any]]:
return [wholedoc_arm.fasit_answer(q) for q in self.questions]
def test_missing_arm_is_fail_closed_at_zero(self) -> None:
code, out = self.gate(self.ws.root / "none.json")
self.assertEqual(code, 1)
self.assertIn("correct answer and source: 0 of 2", out)
self.assertIn("fail-closed", out)
def test_fasit_scores_all_per_type(self) -> None:
code, out = self.gate(self.answers(self.fasit()))
self.assertEqual(code, 0, out)
self.assertIn("whole: correct 1 of 1", out)
self.assertIn("count: correct 1 of 1", out)
self.assertIn("correct answer and source: 2 of 2", out)
self.assertIn("GREEN", out)
def test_one_span_stub_scores_zero(self) -> None:
code, out = self.gate(self.answers([wholedoc_arm.one_span_answer(q) for q in self.questions]))
self.assertEqual(code, 0, out)
self.assertIn("correct answer and source: 0 of 2", out)
self.assertIn("RED", out)
def test_count_must_be_an_exact_integer(self) -> None:
for wrong in (5, 7, "6", 6.0, None):
a = self.fasit()
a[1]["count"] = wrong
code, out = self.gate(self.answers(a))
self.assertIn("count: correct 0 of 1", out, repr(wrong))
def test_count_true_is_not_one(self) -> None:
q = count_question("d02")
a = wholedoc_arm.fasit_answer(q)
a["count"] = True
self.assertFalse(wholedoc.score_one(q, a, multidoc.SourceCache(self.ws.sources)).answer_ok)
def test_whole_missing_a_key_is_wrong(self) -> None:
a = self.fasit()
a[0]["answer"] = a[0]["answer"].replace("flood year", "a bad year")
code, out = self.gate(self.answers(a))
self.assertIn("whole: correct 0 of 1", out)
def test_citation_must_quote_the_file(self) -> None:
a = self.fasit()
a[0]["citations"] = [{"file": "long.md", "quote": "The north gate never opens."}]
a[1]["citations"] = [{"file": "short.md", "quote": "Only three lines here"}]
code, out = self.gate(self.answers(a))
self.assertIn("correct source: 0 of 2", out)
def test_set_that_differs_from_the_manifest_is_refused(self) -> None:
self.set_path.write_text(self.set_path.read_text(encoding="utf-8") + "\n", encoding="utf-8")
code, out = self.gate(self.answers(self.fasit()))
self.assertEqual(code, 1)
self.assertIn("sha256", out)
def test_changed_sources_are_refused(self) -> None:
(self.ws.sources / "a.md").write_text("changed\n", encoding="utf-8")
code, out = self.gate(self.answers(self.fasit()))
self.assertEqual(code, 1)
self.assertIn("sources", out)
def test_manifest_before_the_split_is_refused(self) -> None:
manifest = json.loads(self.manifest.read_text(encoding="utf-8"))
manifest["dev"] = None
self.manifest.write_text(json.dumps(manifest), encoding="utf-8")
code, out = self.gate(self.answers(self.fasit()))
self.assertEqual(code, 1)
def test_arm_name_mismatch_is_refused(self) -> None:
code, out = self.gate(self.answers(self.fasit(), arm="filesearch"))
self.assertEqual(code, 1)
self.assertIn("arm", out)
def test_per_question_results_are_written_for_pairing(self) -> None:
out_path = self.ws.root / "results.json"
run(["gate", "--set", str(self.set_path), "--manifest", str(self.manifest), "--sources", str(self.ws.sources),
"--arm", "collection", "--answers", str(self.answers(self.fasit()[:1])), "--out", str(out_path)])
results = json.loads(out_path.read_text(encoding="utf-8"))
self.assertEqual([(r["id"], r["type"], r["correct"]) for r in results["results"]],
[("d01", "whole", True), ("d02", "count", False)])
class ArmStubTests(unittest.TestCase):
def test_arm_file_is_written_for_every_question(self) -> None:
ws = Workspace()
try:
set_path = ws.write_set([whole_question("d01"), count_question("d02")])
out = ws.root / "arm.json"
code = wholedoc_arm.main(["one-span", str(set_path), str(out)])
self.assertEqual(code, 0)
data = json.loads(out.read_text(encoding="utf-8"))
self.assertEqual(data["arm"], "one-span")
self.assertEqual([a["id"] for a in data["answers"]], ["d01", "d02"])
self.assertEqual(data["answers"][1]["count"], 1)
finally:
ws.close()
class NegatedKeyTests(unittest.TestCase):
def test_a_whole_question_scores_a_negated_key_by_its_negation(self) -> None:
ws = Workspace()
try:
q = whole_question()
q["answer_keys"][2]["negated"] = True
cache = multidoc.SourceCache(ws.sources)
cite = [{"file": "long.md", "quote": "The north gate opens at dawn for the carters."}]
right = {"id": q["id"], "answer": "dawn, copper coins, not after the flood year", "citations": cite}
wrong = {"id": q["id"], "answer": "dawn, copper coins, after the flood year", "citations": cite}
self.assertTrue(wholedoc.score_one(q, right, cache).answer_ok)
self.assertFalse(wholedoc.score_one(q, wrong, cache).answer_ok)
finally:
ws.close()
def test_the_fasit_states_a_negated_key_with_its_negation(self) -> None:
q = whole_question()
q["answer_keys"][2]["negated"] = True
self.assertEqual(wholedoc_arm.fasit_answer(q)["answer"], "dawn; copper coins; not flood year")
class NumberKeyTests(unittest.TestCase):
def test_a_whole_question_scores_a_number_key_as_a_number(self) -> None:
ws = Workspace()
try:
q = whole_question()
q["answer_keys"][0] = {"text": "three"}
cache = multidoc.SourceCache(ws.sources)
cite = [{"file": "long.md", "quote": "The north gate opens at dawn for the carters."}]
right = {"id": q["id"], "answer": "3 of them, copper coins, flood year", "citations": cite}
wrong = {"id": q["id"], "answer": "33 of them, copper coins, flood year", "citations": cite}
self.assertTrue(wholedoc.score_one(q, right, cache).answer_ok)
self.assertFalse(wholedoc.score_one(q, wrong, cache).answer_ok)
finally:
ws.close()
if __name__ == "__main__":
unittest.main()