First public release, published as one snapshot. What changed between versions is in CHANGELOG.md. Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
552 lines
24 KiB
Python
552 lines
24 KiB
Python
"""Tests for gates/wholedoc.py: the whole-document and counting group (E2b), its cut, checker, split and gate.
|
|
|
|
Every fixture is synthetic text written here. No corpus text is used (CLAUDE.md rule 3).
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import contextlib
|
|
import hashlib
|
|
import io
|
|
import json
|
|
import sys
|
|
import tempfile
|
|
import unittest
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "stubs"))
|
|
|
|
import multidoc # noqa: E402
|
|
import wholedoc # noqa: E402
|
|
import wholedoc_arm # noqa: E402
|
|
|
|
REPO_ROOT = Path(__file__).resolve().parents[2]
|
|
|
|
|
|
def long_doc() -> str:
|
|
"""A 200-line synthetic document: facts at the top, the middle and the bottom, and six tagged items."""
|
|
lines = [f"Filler line {i:03d} about nothing in particular." for i in range(200)]
|
|
lines[5] = "The north gate opens at dawn for the carters."
|
|
lines[100] = "The middle bridge carries a toll of copper coins."
|
|
lines[190] = "The south wall was rebuilt after the flood year."
|
|
for n, i in enumerate((10, 40, 70, 110, 150, 180), 1):
|
|
lines[i] = f"- Item: the lantern maker number {chr(64 + n)} works here."
|
|
return "\n".join(lines) + "\n"
|
|
|
|
|
|
SOURCES = {
|
|
"long.md": long_doc(),
|
|
"short.md": "# Short\n\nOnly three lines here, far too short for a whole-document question.\n",
|
|
"a.md": "alpha text\n", "b.md": "beta text\n", "c.md": "gamma text\n", "d.md": "delta text\n",
|
|
}
|
|
|
|
|
|
def whole_question(qid: str = "x01") -> dict[str, Any]:
|
|
return {
|
|
"id": qid,
|
|
"type": "whole",
|
|
"question": "What does the town record at its north gate, middle bridge and south wall?",
|
|
"file": "long.md",
|
|
"evidence": [
|
|
{"span": "The north gate opens at dawn for the carters."},
|
|
{"span": "The middle bridge carries a toll of copper coins."},
|
|
{"span": "The south wall was rebuilt after the flood year."},
|
|
],
|
|
"answer_keys": [{"text": "dawn"}, {"text": "copper coins"}, {"text": "flood year"}],
|
|
}
|
|
|
|
|
|
def count_question(qid: str = "x02") -> dict[str, Any]:
|
|
return {
|
|
"id": qid,
|
|
"type": "count",
|
|
"question": "How many lantern makers does the document list?",
|
|
"file": "long.md",
|
|
"evidence": [{"span": f"- Item: the lantern maker number {chr(64 + n)} works here."} for n in range(1, 7)],
|
|
"item_pattern": "(?m)^- Item: the lantern maker",
|
|
"count": 6,
|
|
}
|
|
|
|
|
|
def e2_set(questions: list[list[str]]) -> dict[str, Any]:
|
|
"""An E2-style dev set: one question per file list, ids q01.. in the given order."""
|
|
return {
|
|
"schema": "okf-wiki/multidoc-set/1",
|
|
"questions": [
|
|
{"id": f"q{n:02d}", "question": "?", "evidence": [{"file": f, "span": "x" * 20} for f in files],
|
|
"answer_keys": []}
|
|
for n, files in enumerate(questions, 1)
|
|
],
|
|
}
|
|
|
|
|
|
class Workspace:
|
|
def __init__(self) -> None:
|
|
self._tmp = tempfile.TemporaryDirectory()
|
|
self.root = Path(self._tmp.name)
|
|
self.sources = self.root / "sources"
|
|
self.sources.mkdir()
|
|
for name, text in SOURCES.items():
|
|
(self.sources / name).write_text(text, encoding="utf-8")
|
|
self.e2 = self.write_json("e2.json", e2_set([["long.md", "a.md", "b.md", "c.md"], ["short.md", "d.md"]]))
|
|
|
|
def close(self) -> None:
|
|
self._tmp.cleanup()
|
|
|
|
def write_set(self, questions: list[dict[str, Any]], name: str = "set.json") -> Path:
|
|
return self.write_json(name, {"schema": wholedoc.SET_SCHEMA, "questions": questions})
|
|
|
|
def write_json(self, name: str, data: Any) -> Path:
|
|
path = self.root / name
|
|
path.write_text(json.dumps(data), encoding="utf-8")
|
|
return path
|
|
|
|
def cut_manifest(self, name: str = "manifest.json", max_files: int = 20) -> Path:
|
|
path = self.root / name
|
|
code, out = run(["cut", "--e2-set", str(self.e2), "--sources", str(self.sources), "--max-files",
|
|
str(max_files), "--manifest", str(path)])
|
|
assert code == 0, out
|
|
return path
|
|
|
|
|
|
def run(argv: list[str]) -> tuple[int, str]:
|
|
out, err = io.StringIO(), io.StringIO()
|
|
with contextlib.redirect_stdout(out), contextlib.redirect_stderr(err):
|
|
code = wholedoc.main(argv)
|
|
return code, out.getvalue() + err.getvalue()
|
|
|
|
|
|
def sha256_file(path: Path) -> str:
|
|
return hashlib.sha256(path.read_bytes()).hexdigest()
|
|
|
|
|
|
class CutTests(unittest.TestCase):
|
|
def setUp(self) -> None:
|
|
self.ws = Workspace()
|
|
|
|
def tearDown(self) -> None:
|
|
self.ws.close()
|
|
|
|
def test_greedy_fewest_new_files_ties_by_id(self) -> None:
|
|
questions = [
|
|
{"id": "q02", "evidence": [{"file": f} for f in ("a", "b", "c", "d")]},
|
|
{"id": "q01", "evidence": [{"file": f} for f in ("a", "b", "e", "f")]},
|
|
{"id": "q03", "evidence": [{"file": f} for f in ("a", "b", "c", "g")]},
|
|
{"id": "q04", "evidence": [{"file": f} for f in ("h", "i", "j", "k")]},
|
|
]
|
|
chosen, files = wholedoc.compute_cut(questions, 7)
|
|
self.assertEqual(chosen, ["q01", "q02", "q03"])
|
|
self.assertEqual(files, sorted({"a", "b", "c", "d", "e", "f", "g"}))
|
|
|
|
def test_cut_stops_before_exceeding_the_limit(self) -> None:
|
|
questions = [{"id": "q01", "evidence": [{"file": "a"}]}, {"id": "q02", "evidence": [{"file": f} for f in "bc"]}]
|
|
chosen, files = wholedoc.compute_cut(questions, 2)
|
|
self.assertEqual((chosen, files), (["q01"], ["a"]))
|
|
|
|
def test_cut_writes_thresholds_before_any_question(self) -> None:
|
|
manifest = json.loads(self.ws.cut_manifest().read_text(encoding="utf-8"))
|
|
self.assertEqual(manifest["schema"], wholedoc.MANIFEST_SCHEMA)
|
|
self.assertEqual(manifest["thresholds"], wholedoc.thresholds())
|
|
self.assertEqual(manifest["cut"]["files"], 6)
|
|
self.assertEqual(manifest["cut"]["questions"], 2)
|
|
self.assertEqual(manifest["cut"]["e2_set_sha256"], sha256_file(self.ws.e2))
|
|
self.assertIsNone(manifest["dev"])
|
|
self.assertIsNone(manifest["held_out"])
|
|
self.assertEqual(manifest["sources"]["tree_sha256"], multidoc.tree_sha256(self.ws.sources))
|
|
|
|
def test_cut_refuses_to_overwrite_a_manifest(self) -> None:
|
|
path = self.ws.cut_manifest()
|
|
code, out = run(["cut", "--e2-set", str(self.ws.e2), "--sources", str(self.ws.sources), "--manifest", str(path)])
|
|
self.assertEqual(code, 1)
|
|
self.assertIn("exists", out)
|
|
|
|
|
|
class CheckTests(unittest.TestCase):
|
|
def setUp(self) -> None:
|
|
self.ws = Workspace()
|
|
self.manifest = self.ws.cut_manifest()
|
|
|
|
def tearDown(self) -> None:
|
|
self.ws.close()
|
|
|
|
def check(self, questions: list[dict[str, Any]], e2: Path | None = None) -> tuple[int, str]:
|
|
path = self.ws.write_set(questions)
|
|
return run(["check", "--set", str(path), "--sources", str(self.ws.sources), "--manifest", str(self.manifest),
|
|
"--e2-set", str(e2 or self.ws.e2)])
|
|
|
|
def test_good_group_passes_with_counts_from_the_set(self) -> None:
|
|
code, out = self.check([whole_question("x01"), count_question("x02")])
|
|
self.assertEqual(code, 0, out)
|
|
self.assertIn("questions: 2 (whole 1, count 1)", out)
|
|
self.assertIn("spans verbatim and unique: 9 of 9", out)
|
|
self.assertIn("questions whose file is in the cut: 2 of 2", out)
|
|
self.assertIn("whole: spread over >= 3 parts, first to last third: 1 of 1", out)
|
|
self.assertIn("count: count = spans = pattern matches >= 5: 1 of 1", out)
|
|
self.assertIn("count: answer not verbatim in the file: 1 of 1", out)
|
|
self.assertIn("not answerable from one span: 2 of 2", out)
|
|
self.assertIn("PASS", out)
|
|
|
|
def test_a_wording_key_fails_the_check(self) -> None:
|
|
q = whole_question()
|
|
q["answer_keys"][0] = {"text": "at dawn for the carters"}
|
|
code, out = self.check([q, count_question("x02")])
|
|
self.assertEqual(code, 1)
|
|
self.assertIn("answer keys that are wording: 1 of 3", out)
|
|
self.assertIn("whole: every key in a span, every span keyed: 1 of 1", out)
|
|
|
|
def test_a_negated_key_must_stand_negated_in_its_span(self) -> None:
|
|
q = whole_question()
|
|
q["answer_keys"][2]["negated"] = True
|
|
code, out = self.check([q, count_question("x02")])
|
|
self.assertEqual(code, 1)
|
|
self.assertIn("negated keys negated in their span: 0 of 1", out)
|
|
|
|
def test_a_negated_key_negated_in_its_span_passes(self) -> None:
|
|
long = self.ws.sources / "long.md"
|
|
long.write_text(long.read_text(encoding="utf-8").replace("wall was rebuilt", "wall was not rebuilt"),
|
|
encoding="utf-8")
|
|
q = whole_question()
|
|
q["evidence"][2]["span"] = "The south wall was not rebuilt after the flood year."
|
|
q["answer_keys"][2]["negated"] = True
|
|
code, out = self.check([q, count_question("x02")])
|
|
self.assertEqual(code, 0, out)
|
|
self.assertIn("negated keys negated in their span: 1 of 1", out)
|
|
|
|
def test_good_group_has_no_wording_keys(self) -> None:
|
|
code, out = self.check([whole_question("x01"), count_question("x02")])
|
|
self.assertIn("answer keys that are wording: 0 of 3", out)
|
|
|
|
def test_whole_without_the_last_third_fails(self) -> None:
|
|
q = whole_question()
|
|
q["evidence"][2] = {"span": "- Item: the lantern maker number D works here."}
|
|
q["answer_keys"][2] = {"text": "number D"}
|
|
code, out = self.check([q])
|
|
self.assertEqual(code, 1)
|
|
self.assertIn("whole: spread over >= 3 parts, first to last third: 0 of 1", out)
|
|
|
|
def test_whole_with_two_parts_fails(self) -> None:
|
|
q = whole_question()
|
|
q["evidence"][1] = {"span": "- Item: the lantern maker number A works here."}
|
|
q["answer_keys"][1] = {"text": "number A"}
|
|
code, out = self.check([q])
|
|
self.assertEqual(code, 1)
|
|
self.assertIn("first to last third: 0 of 1", out)
|
|
|
|
def test_whole_on_a_short_file_fails(self) -> None:
|
|
q = whole_question()
|
|
q["file"] = "short.md"
|
|
code, out = self.check([q])
|
|
self.assertEqual(code, 1)
|
|
self.assertIn(f"whole: file >= {wholedoc.WHOLE_MIN_LINES} lines: 0 of 1", out)
|
|
|
|
def test_whole_answerable_from_one_span_fails(self) -> None:
|
|
q = whole_question()
|
|
q["answer_keys"] = [{"text": "north gate"}, {"text": "dawn"}, {"text": "carters"}]
|
|
code, out = self.check([q])
|
|
self.assertEqual(code, 1)
|
|
self.assertIn("not answerable from one span: 0 of 1", out)
|
|
|
|
def test_whole_key_outside_its_spans_fails(self) -> None:
|
|
q = whole_question()
|
|
q["answer_keys"].append({"text": "Filler line"})
|
|
code, out = self.check([q])
|
|
self.assertEqual(code, 1)
|
|
self.assertIn("whole: every key in a span, every span keyed: 0 of 1", out)
|
|
|
|
def test_span_that_occurs_twice_fails(self) -> None:
|
|
q = whole_question()
|
|
q["evidence"][0] = {"span": "about nothing in particular."}
|
|
code, out = self.check([q])
|
|
self.assertEqual(code, 1)
|
|
self.assertIn("spans verbatim and unique: 2 of 3", out)
|
|
|
|
def test_count_that_disagrees_with_the_pattern_fails(self) -> None:
|
|
q = count_question()
|
|
q["item_pattern"] = "(?m)^- Item: the lantern maker number [A-E]"
|
|
code, out = self.check([q])
|
|
self.assertEqual(code, 1)
|
|
self.assertIn("count: count = spans = pattern matches >= 5: 0 of 1", out)
|
|
|
|
def test_count_below_the_minimum_fails(self) -> None:
|
|
q = count_question()
|
|
q["evidence"] = q["evidence"][:4]
|
|
q["item_pattern"] = "(?m)^- Item: the lantern maker number [A-D]"
|
|
q["count"] = 4
|
|
code, out = self.check([q])
|
|
self.assertEqual(code, 1)
|
|
self.assertIn("count: count = spans = pattern matches >= 5: 0 of 1", out)
|
|
|
|
def test_count_that_stands_in_the_file_fails(self) -> None:
|
|
ws_file = self.ws.sources / "long.md"
|
|
ws_file.write_text(ws_file.read_text(encoding="utf-8") + "There are six of them.\n", encoding="utf-8")
|
|
self.manifest.unlink()
|
|
self.manifest = self.ws.cut_manifest()
|
|
code, out = self.check([count_question()])
|
|
self.assertEqual(code, 1)
|
|
self.assertIn("count: answer not verbatim in the file: 0 of 1", out)
|
|
|
|
def test_number_inside_a_version_is_not_the_answer(self) -> None:
|
|
self.assertFalse(wholedoc.number_stands_in(6, "Released in 2.1.6 and v6.0, see item-6 and 16 or 60."))
|
|
self.assertTrue(wholedoc.number_stands_in(6, "It holds 6 entries."))
|
|
self.assertTrue(wholedoc.number_stands_in(6, "Up to 6."))
|
|
self.assertTrue(wholedoc.number_stands_in(6, "Six rules apply."))
|
|
|
|
def test_file_outside_the_cut_fails(self) -> None:
|
|
self.ws.e2 = self.ws.write_json("e2-small.json", e2_set([["short.md", "a.md"]]))
|
|
self.manifest.unlink()
|
|
self.manifest = self.ws.cut_manifest()
|
|
code, out = self.check([whole_question()])
|
|
self.assertEqual(code, 1)
|
|
self.assertIn("not in the cut", out)
|
|
self.assertIn("questions whose file is in the cut: 0 of 1", out)
|
|
|
|
def test_cut_that_differs_from_the_manifest_cannot_run(self) -> None:
|
|
e2 = self.ws.write_json("e2-other.json", e2_set([["long.md", "a.md", "b.md", "c.md", "d.md", "short.md"]]))
|
|
code, out = self.check([whole_question()], e2=e2)
|
|
self.assertEqual(code, 1)
|
|
self.assertIn("e2_set_sha256", out)
|
|
|
|
def test_empty_set_is_a_fail_not_a_pass(self) -> None:
|
|
code, out = self.check([])
|
|
self.assertEqual(code, 1)
|
|
self.assertIn("FAIL", out)
|
|
|
|
def test_unknown_type_fails(self) -> None:
|
|
q = whole_question()
|
|
q["type"] = "other"
|
|
code, out = self.check([q])
|
|
self.assertEqual(code, 1)
|
|
self.assertIn("type", out)
|
|
|
|
|
|
class SplitTests(unittest.TestCase):
|
|
def setUp(self) -> None:
|
|
self.ws = Workspace()
|
|
self.manifest = self.ws.cut_manifest()
|
|
questions = [whole_question(f"w{n}") for n in range(1, 5)] + [count_question(f"c{n}") for n in range(1, 5)]
|
|
self.set_path = self.ws.write_set(questions, "all.json")
|
|
self.held = Path(tempfile.mkdtemp()) / "held.json"
|
|
|
|
def tearDown(self) -> None:
|
|
self.ws.close()
|
|
|
|
def split(self, seed: int = 7, held: Path | None = None) -> tuple[int, str]:
|
|
return run(["split", "--set", str(self.set_path), "--sources", str(self.ws.sources), "--manifest",
|
|
str(self.manifest), "--e2-set", str(self.ws.e2), "--seed", str(seed), "--held-out-per-type", "1",
|
|
"--dev-out", str(self.ws.root / "dev.json"), "--dev-label", "gates-local/wholedoc/dev.json",
|
|
"--held-out", str(held or self.held)])
|
|
|
|
def test_split_is_stratified_renumbered_and_recorded(self) -> None:
|
|
code, out = self.split()
|
|
self.assertEqual(code, 0, out)
|
|
dev = json.loads((self.ws.root / "dev.json").read_text(encoding="utf-8"))["questions"]
|
|
held = json.loads(self.held.read_text(encoding="utf-8"))["questions"]
|
|
self.assertEqual([q["id"] for q in dev], [f"d{n:02d}" for n in range(1, 7)])
|
|
self.assertEqual([q["id"] for q in held], ["h01", "h02"])
|
|
self.assertEqual(sorted(q["type"] for q in held), ["count", "whole"])
|
|
manifest = json.loads(self.manifest.read_text(encoding="utf-8"))
|
|
self.assertEqual(manifest["seed"], 7)
|
|
self.assertEqual(manifest["dev"], {"file": "gates-local/wholedoc/dev.json",
|
|
"sha256": sha256_file(self.ws.root / "dev.json"),
|
|
"questions": 6, "whole": 3, "count": 3})
|
|
self.assertEqual(manifest["held_out"], {"sha256": sha256_file(self.held), "questions": 2, "whole": 1,
|
|
"count": 1})
|
|
self.assertEqual(manifest["thresholds"], wholedoc.thresholds())
|
|
|
|
def test_same_seed_same_draw(self) -> None:
|
|
self.split(seed=3)
|
|
first = self.held.read_bytes()
|
|
other = Path(tempfile.mkdtemp()) / "held.json"
|
|
self.manifest.unlink()
|
|
self.manifest = self.ws.cut_manifest()
|
|
self.split(seed=3, held=other)
|
|
self.assertEqual(first, other.read_bytes())
|
|
|
|
def test_refuses_held_out_inside_the_repository(self) -> None:
|
|
code, out = self.split(held=REPO_ROOT / "gates" / "held.json")
|
|
self.assertEqual(code, 1)
|
|
self.assertIn("inside the repository", out)
|
|
|
|
def test_refuses_to_redraw(self) -> None:
|
|
self.split()
|
|
code, out = self.split()
|
|
self.assertEqual(code, 1)
|
|
self.assertIn("already", out)
|
|
|
|
def test_refuses_when_thresholds_changed_after_the_cut(self) -> None:
|
|
manifest = json.loads(self.manifest.read_text(encoding="utf-8"))
|
|
manifest["thresholds"]["count_min"] = 2
|
|
self.manifest.write_text(json.dumps(manifest), encoding="utf-8")
|
|
code, out = self.split()
|
|
self.assertEqual(code, 1)
|
|
self.assertIn("thresholds", out)
|
|
|
|
def test_refuses_a_set_that_fails_the_check(self) -> None:
|
|
q = whole_question("w9")
|
|
q["file"] = "short.md"
|
|
self.set_path = self.ws.write_set([q, count_question("c9")], "bad.json")
|
|
code, out = self.split()
|
|
self.assertEqual(code, 1)
|
|
self.assertFalse(self.held.exists())
|
|
|
|
|
|
class GateTests(unittest.TestCase):
|
|
def setUp(self) -> None:
|
|
self.ws = Workspace()
|
|
self.manifest = self.ws.cut_manifest()
|
|
self.questions = [whole_question("d01"), count_question("d02")]
|
|
self.set_path = self.ws.write_set(self.questions, "dev.json")
|
|
manifest = json.loads(self.manifest.read_text(encoding="utf-8"))
|
|
manifest["dev"] = {"sha256": sha256_file(self.set_path), "questions": 2, "whole": 1, "count": 1}
|
|
manifest["held_out"] = {"sha256": "0" * 64, "questions": 2, "whole": 1, "count": 1}
|
|
self.manifest.write_text(json.dumps(manifest), encoding="utf-8")
|
|
|
|
def tearDown(self) -> None:
|
|
self.ws.close()
|
|
|
|
def gate(self, answers: Path, arm: str = "collection") -> tuple[int, str]:
|
|
return run(["gate", "--set", str(self.set_path), "--manifest", str(self.manifest), "--sources",
|
|
str(self.ws.sources), "--arm", arm, "--answers", str(answers)])
|
|
|
|
def answers(self, entries: list[dict[str, Any]], arm: str = "collection") -> Path:
|
|
return self.ws.write_json("answers.json", {"arm": arm, "answers": entries})
|
|
|
|
def fasit(self) -> list[dict[str, Any]]:
|
|
return [wholedoc_arm.fasit_answer(q) for q in self.questions]
|
|
|
|
def test_missing_arm_is_fail_closed_at_zero(self) -> None:
|
|
code, out = self.gate(self.ws.root / "none.json")
|
|
self.assertEqual(code, 1)
|
|
self.assertIn("correct answer and source: 0 of 2", out)
|
|
self.assertIn("fail-closed", out)
|
|
|
|
def test_fasit_scores_all_per_type(self) -> None:
|
|
code, out = self.gate(self.answers(self.fasit()))
|
|
self.assertEqual(code, 0, out)
|
|
self.assertIn("whole: correct 1 of 1", out)
|
|
self.assertIn("count: correct 1 of 1", out)
|
|
self.assertIn("correct answer and source: 2 of 2", out)
|
|
self.assertIn("GREEN", out)
|
|
|
|
def test_one_span_stub_scores_zero(self) -> None:
|
|
code, out = self.gate(self.answers([wholedoc_arm.one_span_answer(q) for q in self.questions]))
|
|
self.assertEqual(code, 0, out)
|
|
self.assertIn("correct answer and source: 0 of 2", out)
|
|
self.assertIn("RED", out)
|
|
|
|
def test_count_must_be_an_exact_integer(self) -> None:
|
|
for wrong in (5, 7, "6", 6.0, None):
|
|
a = self.fasit()
|
|
a[1]["count"] = wrong
|
|
code, out = self.gate(self.answers(a))
|
|
self.assertIn("count: correct 0 of 1", out, repr(wrong))
|
|
|
|
def test_count_true_is_not_one(self) -> None:
|
|
q = count_question("d02")
|
|
a = wholedoc_arm.fasit_answer(q)
|
|
a["count"] = True
|
|
self.assertFalse(wholedoc.score_one(q, a, multidoc.SourceCache(self.ws.sources)).answer_ok)
|
|
|
|
def test_whole_missing_a_key_is_wrong(self) -> None:
|
|
a = self.fasit()
|
|
a[0]["answer"] = a[0]["answer"].replace("flood year", "a bad year")
|
|
code, out = self.gate(self.answers(a))
|
|
self.assertIn("whole: correct 0 of 1", out)
|
|
|
|
def test_citation_must_quote_the_file(self) -> None:
|
|
a = self.fasit()
|
|
a[0]["citations"] = [{"file": "long.md", "quote": "The north gate never opens."}]
|
|
a[1]["citations"] = [{"file": "short.md", "quote": "Only three lines here"}]
|
|
code, out = self.gate(self.answers(a))
|
|
self.assertIn("correct source: 0 of 2", out)
|
|
|
|
def test_set_that_differs_from_the_manifest_is_refused(self) -> None:
|
|
self.set_path.write_text(self.set_path.read_text(encoding="utf-8") + "\n", encoding="utf-8")
|
|
code, out = self.gate(self.answers(self.fasit()))
|
|
self.assertEqual(code, 1)
|
|
self.assertIn("sha256", out)
|
|
|
|
def test_changed_sources_are_refused(self) -> None:
|
|
(self.ws.sources / "a.md").write_text("changed\n", encoding="utf-8")
|
|
code, out = self.gate(self.answers(self.fasit()))
|
|
self.assertEqual(code, 1)
|
|
self.assertIn("sources", out)
|
|
|
|
def test_manifest_before_the_split_is_refused(self) -> None:
|
|
manifest = json.loads(self.manifest.read_text(encoding="utf-8"))
|
|
manifest["dev"] = None
|
|
self.manifest.write_text(json.dumps(manifest), encoding="utf-8")
|
|
code, out = self.gate(self.answers(self.fasit()))
|
|
self.assertEqual(code, 1)
|
|
|
|
def test_arm_name_mismatch_is_refused(self) -> None:
|
|
code, out = self.gate(self.answers(self.fasit(), arm="filesearch"))
|
|
self.assertEqual(code, 1)
|
|
self.assertIn("arm", out)
|
|
|
|
def test_per_question_results_are_written_for_pairing(self) -> None:
|
|
out_path = self.ws.root / "results.json"
|
|
run(["gate", "--set", str(self.set_path), "--manifest", str(self.manifest), "--sources", str(self.ws.sources),
|
|
"--arm", "collection", "--answers", str(self.answers(self.fasit()[:1])), "--out", str(out_path)])
|
|
results = json.loads(out_path.read_text(encoding="utf-8"))
|
|
self.assertEqual([(r["id"], r["type"], r["correct"]) for r in results["results"]],
|
|
[("d01", "whole", True), ("d02", "count", False)])
|
|
|
|
|
|
class ArmStubTests(unittest.TestCase):
|
|
def test_arm_file_is_written_for_every_question(self) -> None:
|
|
ws = Workspace()
|
|
try:
|
|
set_path = ws.write_set([whole_question("d01"), count_question("d02")])
|
|
out = ws.root / "arm.json"
|
|
code = wholedoc_arm.main(["one-span", str(set_path), str(out)])
|
|
self.assertEqual(code, 0)
|
|
data = json.loads(out.read_text(encoding="utf-8"))
|
|
self.assertEqual(data["arm"], "one-span")
|
|
self.assertEqual([a["id"] for a in data["answers"]], ["d01", "d02"])
|
|
self.assertEqual(data["answers"][1]["count"], 1)
|
|
finally:
|
|
ws.close()
|
|
|
|
|
|
class NegatedKeyTests(unittest.TestCase):
|
|
def test_a_whole_question_scores_a_negated_key_by_its_negation(self) -> None:
|
|
ws = Workspace()
|
|
try:
|
|
q = whole_question()
|
|
q["answer_keys"][2]["negated"] = True
|
|
cache = multidoc.SourceCache(ws.sources)
|
|
cite = [{"file": "long.md", "quote": "The north gate opens at dawn for the carters."}]
|
|
right = {"id": q["id"], "answer": "dawn, copper coins, not after the flood year", "citations": cite}
|
|
wrong = {"id": q["id"], "answer": "dawn, copper coins, after the flood year", "citations": cite}
|
|
self.assertTrue(wholedoc.score_one(q, right, cache).answer_ok)
|
|
self.assertFalse(wholedoc.score_one(q, wrong, cache).answer_ok)
|
|
finally:
|
|
ws.close()
|
|
|
|
def test_the_fasit_states_a_negated_key_with_its_negation(self) -> None:
|
|
q = whole_question()
|
|
q["answer_keys"][2]["negated"] = True
|
|
self.assertEqual(wholedoc_arm.fasit_answer(q)["answer"], "dawn; copper coins; not flood year")
|
|
|
|
|
|
class NumberKeyTests(unittest.TestCase):
|
|
def test_a_whole_question_scores_a_number_key_as_a_number(self) -> None:
|
|
ws = Workspace()
|
|
try:
|
|
q = whole_question()
|
|
q["answer_keys"][0] = {"text": "three"}
|
|
cache = multidoc.SourceCache(ws.sources)
|
|
cite = [{"file": "long.md", "quote": "The north gate opens at dawn for the carters."}]
|
|
right = {"id": q["id"], "answer": "3 of them, copper coins, flood year", "citations": cite}
|
|
wrong = {"id": q["id"], "answer": "33 of them, copper coins, flood year", "citations": cite}
|
|
self.assertTrue(wholedoc.score_one(q, right, cache).answer_ok)
|
|
self.assertFalse(wholedoc.score_one(q, wrong, cache).answer_ok)
|
|
finally:
|
|
ws.close()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|