First public release, published as one snapshot. What changed between versions is in CHANGELOG.md. Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
541 lines
24 KiB
Python
541 lines
24 KiB
Python
"""Tests for gates/multidoc.py: the multi-document set checker, the held-out split and the answer gate.
|
|
|
|
Every fixture is synthetic text written here. No corpus text is used (CLAUDE.md rule 3).
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import contextlib
|
|
import hashlib
|
|
import io
|
|
import json
|
|
import sys
|
|
import tempfile
|
|
import unittest
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
|
|
|
import multidoc # noqa: E402
|
|
|
|
REPO_ROOT = Path(__file__).resolve().parents[2]
|
|
|
|
SOURCES = {
|
|
"alpha.md": "# Alpha\n\nThe widget timeout is 45 seconds by default.\nSet `WIDGET_MODE=fast` to skip checks.\n",
|
|
"beta.md": "# Beta\n\nBeta adds the --frobnicate flag for batch runs.\nIt was introduced in the spring build.\n",
|
|
"gamma.md": "# Gamma\n\nGamma stores its cache under the .gamma-cache directory.\n",
|
|
"delta.md": "# Delta\n\nDelta rejects requests larger than 12 megabytes with error E_TOO_BIG.\n",
|
|
"epsilon.md": "# Epsilon\n\nEpsilon logs every call to the audit trail named ledger-7.\n",
|
|
}
|
|
|
|
|
|
def good_question(qid: str = "x01") -> dict[str, Any]:
|
|
return {
|
|
"id": qid,
|
|
"question": "How do the four synthetic tools fit together?",
|
|
"evidence": [
|
|
{"file": "alpha.md", "span": "The widget timeout is 45 seconds by default."},
|
|
{"file": "beta.md", "span": "Beta adds the --frobnicate flag for batch runs."},
|
|
{"file": "gamma.md", "span": "Gamma stores its cache under the .gamma-cache directory."},
|
|
{"file": "delta.md", "span": "rejects requests larger than 12 megabytes with error E_TOO_BIG"},
|
|
],
|
|
"answer_keys": [
|
|
{"file": "alpha.md", "text": "45 seconds"},
|
|
{"file": "beta.md", "text": "--frobnicate"},
|
|
{"file": "gamma.md", "text": ".gamma-cache"},
|
|
{"file": "delta.md", "text": "E_TOO_BIG"},
|
|
],
|
|
}
|
|
|
|
|
|
def perfect_answer(q: dict[str, Any]) -> dict[str, Any]:
|
|
keys = " ".join(k["text"] for k in q["answer_keys"])
|
|
return {
|
|
"id": q["id"],
|
|
"answer": "The answer mentions " + keys + ".",
|
|
"citations": [{"file": e["file"], "quote": e["span"]} for e in q["evidence"]],
|
|
}
|
|
|
|
|
|
class Workspace:
|
|
"""A temporary sources directory and helpers to write sets, manifests and answers."""
|
|
|
|
def __init__(self) -> None:
|
|
self._tmp = tempfile.TemporaryDirectory()
|
|
self.root = Path(self._tmp.name)
|
|
self.sources = self.root / "sources"
|
|
self.sources.mkdir()
|
|
for name, text in SOURCES.items():
|
|
(self.sources / name).write_text(text, encoding="utf-8")
|
|
|
|
def close(self) -> None:
|
|
self._tmp.cleanup()
|
|
|
|
def write_set(self, questions: list[dict[str, Any]], name: str = "set.json") -> Path:
|
|
path = self.root / name
|
|
path.write_text(json.dumps({"schema": multidoc.SET_SCHEMA, "questions": questions}), encoding="utf-8")
|
|
return path
|
|
|
|
def write_json(self, name: str, data: Any) -> Path:
|
|
path = self.root / name
|
|
path.write_text(json.dumps(data), encoding="utf-8")
|
|
return path
|
|
|
|
|
|
def run(argv: list[str]) -> tuple[int, str]:
|
|
out = io.StringIO()
|
|
err = io.StringIO()
|
|
with contextlib.redirect_stdout(out), contextlib.redirect_stderr(err):
|
|
code = multidoc.main(argv)
|
|
return code, out.getvalue() + err.getvalue()
|
|
|
|
|
|
def sha256_file(path: Path) -> str:
|
|
return hashlib.sha256(path.read_bytes()).hexdigest()
|
|
|
|
|
|
class CheckTests(unittest.TestCase):
|
|
def setUp(self) -> None:
|
|
self.ws = Workspace()
|
|
|
|
def tearDown(self) -> None:
|
|
self.ws.close()
|
|
|
|
def check(self, questions: list[dict[str, Any]]) -> tuple[int, str]:
|
|
path = self.ws.write_set(questions)
|
|
return run(["check", "--set", str(path), "--sources", str(self.ws.sources)])
|
|
|
|
def test_good_set_passes_with_counts_from_the_set(self) -> None:
|
|
code, out = self.check([good_question("x01"), good_question("x02")])
|
|
self.assertEqual(code, 0, out)
|
|
self.assertIn("spans verbatim: 8 of 8", out)
|
|
self.assertIn("questions with every span verbatim: 2 of 2", out)
|
|
self.assertIn("questions with >= 4 distinct files: 2 of 2", out)
|
|
self.assertIn("questions with a key per file: 2 of 2", out)
|
|
self.assertIn("PASS", out)
|
|
|
|
def test_span_one_character_off_fails(self) -> None:
|
|
q = good_question()
|
|
q["evidence"][0]["span"] = "The widget timeout is 46 seconds by default."
|
|
code, out = self.check([q])
|
|
self.assertEqual(code, 1)
|
|
self.assertIn("spans verbatim: 3 of 4", out)
|
|
self.assertIn("questions with every span verbatim: 0 of 1", out)
|
|
self.assertIn("FAIL", out)
|
|
|
|
def test_span_match_is_exact_not_whitespace_normalised(self) -> None:
|
|
q = good_question()
|
|
q["evidence"][0]["span"] = "The widget timeout is 45 seconds by default."
|
|
code, out = self.check([q])
|
|
self.assertEqual(code, 1)
|
|
self.assertIn("spans verbatim: 3 of 4", out)
|
|
|
|
def test_three_distinct_files_fail_the_minimum(self) -> None:
|
|
q = good_question()
|
|
q["evidence"][3] = {"file": "alpha.md", "span": "Set `WIDGET_MODE=fast` to skip checks."}
|
|
q["answer_keys"][3] = {"file": "alpha.md", "text": "WIDGET_MODE=fast"}
|
|
code, out = self.check([q])
|
|
self.assertEqual(code, 1)
|
|
self.assertIn("questions with >= 4 distinct files: 0 of 1", out)
|
|
|
|
def test_file_missing_from_sources_fails(self) -> None:
|
|
q = good_question()
|
|
q["evidence"][0]["file"] = "missing.md"
|
|
q["answer_keys"][0]["file"] = "missing.md"
|
|
code, out = self.check([q])
|
|
self.assertEqual(code, 1)
|
|
self.assertIn("spans verbatim: 3 of 4", out)
|
|
|
|
def test_path_outside_sources_is_an_error(self) -> None:
|
|
for bad in ("../alpha.md", "sub/alpha.md", "/etc/hosts"):
|
|
q = good_question()
|
|
q["evidence"][0]["file"] = bad
|
|
q["answer_keys"][0]["file"] = bad
|
|
code, out = self.check([q])
|
|
self.assertEqual(code, 1, bad)
|
|
self.assertIn("not a plain file name", out)
|
|
|
|
def test_key_outside_its_files_spans_fails(self) -> None:
|
|
q = good_question()
|
|
q["answer_keys"][0]["text"] = "WIDGET_MODE" # in alpha.md, but not in the cited span
|
|
code, out = self.check([q])
|
|
self.assertEqual(code, 1)
|
|
self.assertIn("questions with a key per file: 0 of 1", out)
|
|
|
|
def test_key_match_ignores_case(self) -> None:
|
|
q = good_question()
|
|
q["answer_keys"][3]["text"] = "e_too_big"
|
|
code, out = self.check([q])
|
|
self.assertEqual(code, 0, out)
|
|
|
|
def test_evidence_file_without_a_key_fails(self) -> None:
|
|
q = good_question()
|
|
del q["answer_keys"][2]
|
|
code, out = self.check([q])
|
|
self.assertEqual(code, 1)
|
|
self.assertIn("questions with a key per file: 0 of 1", out)
|
|
|
|
def test_duplicate_ids_fail(self) -> None:
|
|
code, out = self.check([good_question("x01"), good_question("x01")])
|
|
self.assertEqual(code, 1)
|
|
self.assertIn("duplicate id", out)
|
|
|
|
def test_empty_set_is_a_fail_not_a_pass(self) -> None:
|
|
code, out = self.check([])
|
|
self.assertEqual(code, 1)
|
|
self.assertIn("FAIL", out)
|
|
|
|
def test_a_wording_key_fails_even_inside_its_span(self) -> None:
|
|
q = good_question()
|
|
q["answer_keys"][0] = {"file": "alpha.md", "text": "is 45 seconds by default"}
|
|
code, out = self.check([q])
|
|
self.assertEqual(code, 1)
|
|
self.assertIn("questions with a key per file: 1 of 1", out)
|
|
self.assertIn("answer keys that are wording: 1 of 4", out)
|
|
|
|
def test_good_set_has_no_wording_keys(self) -> None:
|
|
code, out = self.check([good_question()])
|
|
self.assertIn("answer keys that are wording: 0 of 4", out)
|
|
|
|
def test_short_span_and_short_key_fail(self) -> None:
|
|
q = good_question()
|
|
q["evidence"][0]["span"] = "45 seconds"
|
|
code, out = self.check([q])
|
|
self.assertEqual(code, 1)
|
|
self.assertIn("span length", out)
|
|
q = good_question()
|
|
q["answer_keys"][0]["text"] = "se"
|
|
code, out = self.check([q])
|
|
self.assertEqual(code, 1)
|
|
self.assertIn("key length", out)
|
|
|
|
def test_missing_sources_directory_cannot_run(self) -> None:
|
|
path = self.ws.write_set([good_question()])
|
|
code, out = run(["check", "--set", str(path), "--sources", str(self.ws.root / "nope")])
|
|
self.assertEqual(code, 1)
|
|
self.assertIn("FAIL", out)
|
|
|
|
def test_unreadable_set_cannot_run(self) -> None:
|
|
path = self.ws.root / "broken.json"
|
|
path.write_text("{not json", encoding="utf-8")
|
|
code, out = run(["check", "--set", str(path), "--sources", str(self.ws.sources)])
|
|
self.assertEqual(code, 1)
|
|
self.assertIn("FAIL", out)
|
|
|
|
|
|
def test_a_negated_key_must_stand_negated_in_its_span(self) -> None:
|
|
q = good_question()
|
|
q["answer_keys"][0]["negated"] = True
|
|
code, out = self.check([q])
|
|
self.assertEqual(code, 1)
|
|
self.assertIn("negated keys negated in their span: 0 of 1", out)
|
|
|
|
def test_a_negated_key_negated_in_its_span_passes(self) -> None:
|
|
(self.ws.sources / "alpha.md").write_text("The widget is not synced back to the hub.\n", encoding="utf-8")
|
|
q = good_question()
|
|
q["evidence"][0]["span"] = "The widget is not synced back to the hub."
|
|
q["answer_keys"][0] = {"file": "alpha.md", "text": "synced", "negated": True}
|
|
code, out = self.check([q])
|
|
self.assertEqual(code, 0, out)
|
|
self.assertIn("negated keys negated in their span: 1 of 1", out)
|
|
|
|
def test_negated_must_be_true_or_absent(self) -> None:
|
|
q = good_question()
|
|
q["answer_keys"][0]["negated"] = "yes"
|
|
code, out = self.check([q])
|
|
self.assertEqual(code, 1)
|
|
self.assertIn("negated", out)
|
|
|
|
|
|
class NegatedKeyTests(unittest.TestCase):
|
|
KEY = {"file": "a.md", "text": "synced", "negated": True}
|
|
|
|
def test_a_negation_within_six_words_before_the_key(self) -> None:
|
|
for text in ("Changes are not synced back.", "It isn't synced.", "It isn\u2019t synced.",
|
|
"Changes are no longer synced.", "It is never, ever synced.", "Not one two three four five synced.",
|
|
"Some are synced; others are not synced."):
|
|
self.assertTrue(multidoc.key_holds(self.KEY, text), text)
|
|
|
|
def test_no_negation_or_one_out_of_reach_fails(self) -> None:
|
|
for text in ("Changes are synced back.", "It is not late. Changes are synced.", "Not late; synced.",
|
|
"Not one two three four five six synced.", "Synced, but not back.", "Not here:\nsynced."):
|
|
self.assertFalse(multidoc.key_holds(self.KEY, text), text)
|
|
|
|
def test_every_negation_in_the_list(self) -> None:
|
|
for neg in ("not", "no", "never", "none", "neither", "nor", "without", "cannot", "can't", "doesn't", "won't"):
|
|
self.assertTrue(multidoc.key_holds(self.KEY, f"It {neg} synced."), neg)
|
|
|
|
def test_an_unmarked_key_holds_with_or_without_a_negation(self) -> None:
|
|
key = {"file": "a.md", "text": "synced"}
|
|
self.assertTrue(multidoc.key_holds(key, "Changes are synced back."))
|
|
self.assertTrue(multidoc.key_holds(key, "Changes are not synced back."))
|
|
|
|
def test_the_gate_scores_a_negated_key_by_its_negation(self) -> None:
|
|
ws = Workspace()
|
|
try:
|
|
(ws.sources / "alpha.md").write_text("The widget is not synced back to the hub.\n", encoding="utf-8")
|
|
q = good_question()
|
|
q["evidence"][0]["span"] = "The widget is not synced back to the hub."
|
|
q["answer_keys"][0] = {"file": "alpha.md", "text": "synced", "negated": True}
|
|
cache = multidoc.SourceCache(ws.sources)
|
|
right = perfect_answer(q)
|
|
right["answer"] = "The widget is not synced; --frobnicate, .gamma-cache, E_TOO_BIG."
|
|
wrong = dict(right, answer="The widget is synced; --frobnicate, .gamma-cache, E_TOO_BIG.")
|
|
self.assertTrue(multidoc.score_one(q, right, cache).answer_ok)
|
|
self.assertFalse(multidoc.score_one(q, wrong, cache).answer_ok)
|
|
finally:
|
|
ws.close()
|
|
|
|
|
|
class SplitTests(unittest.TestCase):
|
|
def setUp(self) -> None:
|
|
self.ws = Workspace()
|
|
self.questions = [good_question(f"x{i:02d}") for i in range(1, 11)]
|
|
for i, q in enumerate(self.questions):
|
|
q["question"] = f"Synthetic question number {i + 1}?"
|
|
self.all_path = self.ws.write_set(self.questions, "all.json")
|
|
|
|
def tearDown(self) -> None:
|
|
self.ws.close()
|
|
|
|
def split(self, seed: int, tag: str, held_dir: Path | None = None) -> tuple[int, str, Path, Path, Path]:
|
|
dev = self.ws.root / f"dev-{tag}.json"
|
|
held = (held_dir or self.ws.root / f"held-{tag}") / "held.json"
|
|
manifest = self.ws.root / f"manifest-{tag}.json"
|
|
code, out = run([
|
|
"split", "--set", str(self.all_path), "--sources", str(self.ws.sources), "--seed", str(seed),
|
|
"--held-out-count", "3", "--dev-out", str(dev), "--held-out", str(held), "--manifest", str(manifest),
|
|
])
|
|
return code, out, dev, held, manifest
|
|
|
|
def load(self, path: Path) -> list[dict[str, Any]]:
|
|
data: dict[str, Any] = json.loads(path.read_text(encoding="utf-8"))
|
|
questions: list[dict[str, Any]] = data["questions"]
|
|
return questions
|
|
|
|
def test_split_counts_renumbers_and_partitions(self) -> None:
|
|
code, out, dev, held, _ = self.split(7, "a")
|
|
self.assertEqual(code, 0, out)
|
|
dev_q, held_q = self.load(dev), self.load(held)
|
|
self.assertEqual((len(dev_q), len(held_q)), (7, 3))
|
|
self.assertEqual([q["id"] for q in dev_q], [f"d{i:02d}" for i in range(1, 8)])
|
|
self.assertEqual([q["id"] for q in held_q], [f"h{i:02d}" for i in range(1, 4)])
|
|
texts = sorted(q["question"] for q in dev_q + held_q)
|
|
self.assertEqual(texts, sorted(q["question"] for q in self.questions))
|
|
|
|
def test_same_seed_same_draw_other_seed_other_draw(self) -> None:
|
|
_, _, _, held_a, _ = self.split(7, "a")
|
|
_, _, _, held_b, _ = self.split(7, "b")
|
|
self.assertEqual(held_a.read_bytes(), held_b.read_bytes())
|
|
draws = set()
|
|
for seed in range(1, 6):
|
|
_, _, _, held, _ = self.split(seed, f"s{seed}")
|
|
draws.add(tuple(q["question"] for q in self.load(held)))
|
|
self.assertGreater(len(draws), 1)
|
|
|
|
def test_manifest_carries_hashes_and_nothing_about_the_held_out_questions(self) -> None:
|
|
code, out, dev, held, manifest = self.split(7, "a")
|
|
self.assertEqual(code, 0, out)
|
|
data = json.loads(manifest.read_text(encoding="utf-8"))
|
|
self.assertEqual(data["seed"], 7)
|
|
self.assertEqual(data["dev"]["sha256"], sha256_file(dev))
|
|
self.assertEqual(data["dev"]["questions"], 7)
|
|
self.assertEqual(data["held_out"]["sha256"], sha256_file(held))
|
|
self.assertEqual(data["held_out"]["questions"], 3)
|
|
self.assertEqual(data["sources"]["files"], len(SOURCES))
|
|
self.assertEqual(data["sources"]["tree_sha256"], multidoc.tree_sha256(self.ws.sources))
|
|
manifest_text = manifest.read_text(encoding="utf-8")
|
|
for q in self.load(held):
|
|
self.assertNotIn(q["question"], manifest_text)
|
|
self.assertNotIn(q["question"], out)
|
|
self.assertNotIn(str(held.parent), manifest_text)
|
|
|
|
def test_refuses_held_out_inside_the_repository(self) -> None:
|
|
code, out, _, _, _ = self.split(7, "in", held_dir=REPO_ROOT / "gates-local" / "should-not-exist")
|
|
self.assertEqual(code, 1)
|
|
self.assertIn("inside the repository", out)
|
|
self.assertFalse((REPO_ROOT / "gates-local" / "should-not-exist").exists())
|
|
|
|
def test_refuses_to_overwrite_a_frozen_held_out_file(self) -> None:
|
|
code, out, _, _, _ = self.split(7, "a")
|
|
self.assertEqual(code, 0, out)
|
|
code, out, _, _, _ = self.split(7, "a")
|
|
self.assertEqual(code, 1)
|
|
self.assertIn("already exists", out)
|
|
|
|
def test_refuses_a_set_that_fails_the_check(self) -> None:
|
|
self.questions[0]["evidence"][0]["span"] = "This sentence is not in any source file."
|
|
self.all_path = self.ws.write_set(self.questions, "all.json")
|
|
code, out, dev, held, _ = self.split(7, "bad")
|
|
self.assertEqual(code, 1)
|
|
self.assertFalse(dev.exists())
|
|
self.assertFalse(held.exists())
|
|
|
|
|
|
class GateTests(unittest.TestCase):
|
|
def setUp(self) -> None:
|
|
self.ws = Workspace()
|
|
self.questions = [good_question("d01"), good_question("d02")]
|
|
self.set_path = self.ws.write_set(self.questions, "dev.json")
|
|
self.manifest = self.ws.write_json("manifest.json", {
|
|
"schema": multidoc.MANIFEST_SCHEMA,
|
|
"seed": 1,
|
|
"dev": {"sha256": sha256_file(self.set_path), "questions": 2},
|
|
"held_out": {"sha256": "0" * 64, "questions": 1},
|
|
"sources": {"files": len(SOURCES), "tree_sha256": multidoc.tree_sha256(self.ws.sources)},
|
|
})
|
|
|
|
def tearDown(self) -> None:
|
|
self.ws.close()
|
|
|
|
def gate(self, answers: Path, arm: str = "collection") -> tuple[int, str]:
|
|
return run([
|
|
"gate", "--set", str(self.set_path), "--manifest", str(self.manifest), "--sources", str(self.ws.sources),
|
|
"--arm", arm, "--answers", str(answers),
|
|
])
|
|
|
|
def answers(self, entries: list[dict[str, Any]], arm: str = "collection") -> Path:
|
|
return self.ws.write_json("answers.json", {"arm": arm, "answers": entries})
|
|
|
|
def test_missing_arm_is_fail_closed_at_zero(self) -> None:
|
|
code, out = self.gate(self.ws.root / "no-such-arm.json")
|
|
self.assertEqual(code, 1)
|
|
self.assertIn("correct answer and source: 0 of 2", out)
|
|
self.assertIn("fail-closed", out)
|
|
|
|
def test_perfect_answers_score_all(self) -> None:
|
|
code, out = self.gate(self.answers([perfect_answer(q) for q in self.questions]))
|
|
self.assertEqual(code, 0, out)
|
|
self.assertIn("correct answer and source: 2 of 2", out)
|
|
|
|
def test_answer_missing_a_key_is_wrong(self) -> None:
|
|
a = perfect_answer(self.questions[0])
|
|
a["answer"] = a["answer"].replace("E_TOO_BIG", "a big error")
|
|
code, out = self.gate(self.answers([a, perfect_answer(self.questions[1])]))
|
|
self.assertEqual(code, 0, out)
|
|
self.assertIn("correct answer: 1 of 2", out)
|
|
self.assertIn("correct answer and source: 1 of 2", out)
|
|
|
|
def test_answer_key_match_ignores_case_backticks_and_whitespace(self) -> None:
|
|
a = perfect_answer(self.questions[0])
|
|
a["answer"] = "Use `--FROBNICATE`, wait 45\n seconds, look in .gamma-cache, expect e_too_big."
|
|
code, out = self.gate(self.answers([a, perfect_answer(self.questions[1])]))
|
|
self.assertIn("correct answer: 2 of 2", out)
|
|
|
|
def test_citation_not_verbatim_is_wrong_source(self) -> None:
|
|
a = perfect_answer(self.questions[0])
|
|
a["citations"][1]["quote"] = "Beta adds a flag that nobody documented."
|
|
code, out = self.gate(self.answers([a, perfect_answer(self.questions[1])]))
|
|
self.assertIn("correct source: 1 of 2", out)
|
|
self.assertIn("correct answer and source: 1 of 2", out)
|
|
|
|
def test_citation_whitespace_is_normalised(self) -> None:
|
|
a = perfect_answer(self.questions[0])
|
|
a["citations"][0]["quote"] = "The widget timeout is\n45 seconds by default."
|
|
code, out = self.gate(self.answers([a, perfect_answer(self.questions[1])]))
|
|
self.assertIn("correct source: 2 of 2", out)
|
|
|
|
def test_citations_missing_one_file_is_wrong_source(self) -> None:
|
|
a = perfect_answer(self.questions[0])
|
|
del a["citations"][2]
|
|
code, out = self.gate(self.answers([a, perfect_answer(self.questions[1])]))
|
|
self.assertIn("correct source: 1 of 2", out)
|
|
|
|
def test_citation_outside_sources_counts_for_nothing(self) -> None:
|
|
a = perfect_answer(self.questions[0])
|
|
a["citations"][0]["file"] = "../sources/alpha.md"
|
|
code, out = self.gate(self.answers([a, perfect_answer(self.questions[1])]))
|
|
self.assertIn("correct source: 1 of 2", out)
|
|
|
|
def test_missing_entry_counts_wrong_and_fails(self) -> None:
|
|
code, out = self.gate(self.answers([perfect_answer(self.questions[0])]))
|
|
self.assertEqual(code, 1)
|
|
self.assertIn("correct answer and source: 1 of 2", out)
|
|
self.assertIn("missing answers: 1", out)
|
|
|
|
def test_set_that_differs_from_the_manifest_is_refused(self) -> None:
|
|
self.set_path.write_text(self.set_path.read_text(encoding="utf-8") + "\n", encoding="utf-8")
|
|
code, out = self.gate(self.answers([perfect_answer(q) for q in self.questions]))
|
|
self.assertEqual(code, 1)
|
|
self.assertIn("sha256", out)
|
|
|
|
def test_changed_sources_are_refused(self) -> None:
|
|
(self.ws.sources / "alpha.md").write_text("changed\n", encoding="utf-8")
|
|
code, out = self.gate(self.answers([perfect_answer(q) for q in self.questions]))
|
|
self.assertEqual(code, 1)
|
|
self.assertIn("sources", out)
|
|
|
|
def test_arm_name_mismatch_is_refused(self) -> None:
|
|
code, out = self.gate(self.answers([perfect_answer(q) for q in self.questions], arm="filesearch"))
|
|
self.assertEqual(code, 1)
|
|
self.assertIn("arm", out)
|
|
|
|
def test_per_question_results_are_written_for_pairing(self) -> None:
|
|
a = perfect_answer(self.questions[0])
|
|
answers = self.answers([a])
|
|
out_path = self.ws.root / "results.json"
|
|
run([
|
|
"gate", "--set", str(self.set_path), "--manifest", str(self.manifest), "--sources", str(self.ws.sources),
|
|
"--arm", "collection", "--answers", str(answers), "--out", str(out_path),
|
|
])
|
|
results = json.loads(out_path.read_text(encoding="utf-8"))
|
|
self.assertEqual(results["arm"], "collection")
|
|
self.assertEqual([r["id"] for r in results["results"]], ["d01", "d02"])
|
|
self.assertEqual([r["correct"] for r in results["results"]], [True, False])
|
|
|
|
|
|
class NumberKeyTests(unittest.TestCase):
|
|
"""A key that is a number matches as a number: digits or its English word, never inside a longer number."""
|
|
|
|
def test_a_number_key_matches_digits_or_word_but_not_inside_a_longer_number(self) -> None:
|
|
for text in ("It retries 5 times.", "RETRIES=5", "five attempts", "(5)"):
|
|
self.assertTrue(multidoc.key_in("5", text), text)
|
|
for text in ("15 attempts", "version 2.1.5", "v5", "50", "5.2 seconds", "fifty"):
|
|
self.assertFalse(multidoc.key_in("5", text), text)
|
|
|
|
def test_a_number_word_key_matches_the_digits(self) -> None:
|
|
self.assertTrue(multidoc.key_in("three", "fails 3 times in a row"))
|
|
self.assertTrue(multidoc.key_in("Three", "three times"))
|
|
self.assertFalse(multidoc.key_in("three", "33 times"))
|
|
|
|
def test_a_percentage_key_matches_the_percentage(self) -> None:
|
|
for text in ("at 10% of the window", "at 10 % of it", "10 percent"):
|
|
self.assertTrue(multidoc.key_in("10%", text), text)
|
|
for text in ("110%", "10 of them", "2.10%"):
|
|
self.assertFalse(multidoc.key_in("10%", text), text)
|
|
|
|
def test_other_keys_still_match_as_normalised_text(self) -> None:
|
|
self.assertTrue(multidoc.key_in("model version", "Pin their **model versions**."))
|
|
self.assertTrue(multidoc.key_in("2.1.9", "since 2.1.92"))
|
|
|
|
def test_check_accepts_a_short_number_key_standing_alone_in_its_span(self) -> None:
|
|
ws = Workspace()
|
|
try:
|
|
q = good_question()
|
|
q["answer_keys"][3] = {"file": "delta.md", "text": "12"}
|
|
code, out = run(["check", "--set", str(ws.write_set([q])), "--sources", str(ws.sources)])
|
|
self.assertEqual(code, 0, out)
|
|
q["answer_keys"][3] = {"file": "delta.md", "text": "2"}
|
|
code, out = run(["check", "--set", str(ws.write_set([q])), "--sources", str(ws.sources)])
|
|
self.assertEqual(code, 1)
|
|
self.assertIn("questions with a key per file: 0 of 1", out)
|
|
finally:
|
|
ws.close()
|
|
|
|
def test_the_gate_scores_a_number_key_as_a_number(self) -> None:
|
|
ws = Workspace()
|
|
try:
|
|
q = good_question()
|
|
q["answer_keys"][3] = {"file": "delta.md", "text": "12"}
|
|
cache = multidoc.SourceCache(ws.sources)
|
|
right = perfect_answer(q)
|
|
self.assertTrue(multidoc.score_one(q, right, cache).answer_ok)
|
|
wrong = dict(right, answer=right["answer"].replace(" 12", " 112"))
|
|
self.assertFalse(multidoc.score_one(q, wrong, cache).answer_ok)
|
|
finally:
|
|
ws.close()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|