First public release, published as one snapshot. What changed between versions is in CHANGELOG.md. Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
108 lines
4.5 KiB
Python
108 lines
4.5 KiB
Python
"""Tests for gates/keyform.py: counts answer keys that are wording rather than a fact (E2c).
|
|
|
|
Every fixture is synthetic text written here. No corpus text is used (CLAUDE.md rule 3).
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import contextlib
|
|
import io
|
|
import json
|
|
import sys
|
|
import tempfile
|
|
import unittest
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
|
|
|
import keyform # noqa: E402
|
|
|
|
|
|
class WordingTests(unittest.TestCase):
|
|
def test_single_words_and_literals_are_facts(self) -> None:
|
|
for key in ("lantern", "/north", "3.1.4", "GATE_TOLL", "--toll-mode dawn", "copper-coins"):
|
|
self.assertFalse(keyform.is_wording(key), key)
|
|
|
|
def test_a_name_of_several_words_without_function_words_is_a_fact(self) -> None:
|
|
for key in ("Toll mode", "north gate keeper", "Bridge (upper)"):
|
|
self.assertFalse(keyform.is_wording(key), key)
|
|
|
|
def test_a_conjunction_makes_a_key_wording(self) -> None:
|
|
self.assertTrue(keyform.is_wording("/north or /south"))
|
|
self.assertTrue(keyform.is_wording("dawn and dusk"))
|
|
|
|
def test_a_pronoun_or_article_makes_a_key_wording(self) -> None:
|
|
self.assertTrue(keyform.is_wording("pay your toll"))
|
|
self.assertTrue(keyform.is_wording("The toll rises"))
|
|
self.assertTrue(keyform.is_wording("a lantern maker"))
|
|
|
|
def test_prepositions_auxiliaries_negations_and_particles_make_a_key_wording(self) -> None:
|
|
for key in ("toll for carters", "gate is closed", "can't open", "no toll", "only at dawn", "still open"):
|
|
self.assertTrue(keyform.is_wording(key), key)
|
|
|
|
def test_function_words_inside_backticks_are_code_not_wording(self) -> None:
|
|
self.assertFalse(keyform.is_wording("`open --if the gate` flag"))
|
|
self.assertTrue(keyform.is_wording("`toll` for carters"))
|
|
|
|
|
|
def md_set(keys: list[str]) -> dict[str, Any]:
|
|
return {"schema": "okf-wiki/multidoc-set/1", "questions": [
|
|
{"id": "q01", "question": "?", "evidence": [], "answer_keys": [{"file": "a.md", "text": k} for k in keys]}]}
|
|
|
|
|
|
def wd_set() -> dict[str, Any]:
|
|
return {"schema": "okf-wiki/wholedoc-set/1", "questions": [
|
|
{"id": "w01", "type": "whole", "question": "?", "file": "a.md", "evidence": [],
|
|
"answer_keys": [{"text": "dawn"}, {"text": "pay your toll"}, {"text": "copper coins"}]},
|
|
{"id": "w02", "type": "count", "question": "?", "file": "a.md", "evidence": [], "count": 6, "item_pattern": "x"}]}
|
|
|
|
|
|
class CountTests(unittest.TestCase):
|
|
def test_counts_every_key_of_either_schema_and_lists_the_wording_ones(self) -> None:
|
|
rep = keyform.count_keys(md_set(["dawn", "/north or /south", "Toll mode"])["questions"])
|
|
self.assertEqual((rep.wording, rep.keys), (1, 3))
|
|
self.assertEqual(rep.listed, ["q01: /north or /south"])
|
|
rep = keyform.count_keys(wd_set()["questions"])
|
|
self.assertEqual((rep.wording, rep.keys), (1, 3))
|
|
|
|
|
|
class CliTests(unittest.TestCase):
|
|
def run_cli(self, *sets: dict[str, Any]) -> tuple[int, str]:
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
args: list[str] = []
|
|
for n, data in enumerate(sets):
|
|
p = Path(tmp) / f"set{n}.json"
|
|
p.write_text(json.dumps(data), encoding="utf-8")
|
|
args += ["--set", str(p)]
|
|
out = io.StringIO()
|
|
with contextlib.redirect_stdout(out), contextlib.redirect_stderr(io.StringIO()):
|
|
code = keyform.main(args)
|
|
return code, out.getvalue()
|
|
|
|
def test_wording_keys_fail_with_x_of_n_over_all_sets(self) -> None:
|
|
code, out = self.run_cli(md_set(["dawn", "/north or /south"]), wd_set())
|
|
self.assertEqual(code, 1)
|
|
self.assertIn("answer keys that are wording: 2 of 5", out)
|
|
self.assertIn("FAIL", out)
|
|
|
|
def test_no_wording_keys_pass(self) -> None:
|
|
code, out = self.run_cli(md_set(["dawn", "/north", "/south"]))
|
|
self.assertEqual(code, 0)
|
|
self.assertIn("answer keys that are wording: 0 of 3", out)
|
|
self.assertIn("PASS", out)
|
|
|
|
def test_no_keys_at_all_is_a_fail_not_a_pass(self) -> None:
|
|
code, out = self.run_cli(md_set([]))
|
|
self.assertEqual(code, 1)
|
|
|
|
def test_unreadable_set_cannot_run(self) -> None:
|
|
err = io.StringIO()
|
|
with contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(err):
|
|
code = keyform.main(["--set", "/nonexistent/set.json"])
|
|
self.assertEqual(code, 1)
|
|
self.assertIn("FAIL", err.getvalue())
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|