First public release, published as one snapshot. What changed between versions is in CHANGELOG.md. Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
136 lines
6.5 KiB
Python
136 lines
6.5 KiB
Python
"""Tests for gates/polarity.py: counts polarity keys that accept an answer of the opposite polarity (E2d).
|
|
|
|
Every fixture is synthetic text written here. No corpus text is used (CLAUDE.md rule 3).
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import contextlib
|
|
import io
|
|
import json
|
|
import sys
|
|
import tempfile
|
|
import unittest
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
|
|
|
import polarity # noqa: E402
|
|
from multidoc import RunError # noqa: E402
|
|
|
|
|
|
def md_question(qid: str, span: str, keys: list[dict[str, Any]]) -> dict[str, Any]:
|
|
return {"id": qid, "question": "?", "evidence": [{"file": "a.md", "span": "Unrelated words about the ferry."},
|
|
{"file": "b.md", "span": span}],
|
|
"answer_keys": [dict(k, file="b.md") for k in keys]}
|
|
|
|
|
|
def run(argv: list[str]) -> tuple[int, str]:
|
|
out, err = io.StringIO(), io.StringIO()
|
|
with contextlib.redirect_stdout(out), contextlib.redirect_stderr(err):
|
|
code = polarity.main(argv)
|
|
return code, out.getvalue() + err.getvalue()
|
|
|
|
|
|
class FlipTests(unittest.TestCase):
|
|
def opposite(self, sentence: str, atom: str) -> str:
|
|
found = polarity.answer_from_span(atom, [sentence])
|
|
assert found is not None
|
|
return polarity.flip(*found)
|
|
|
|
def test_a_negation_in_the_window_is_taken_out(self) -> None:
|
|
self.assertEqual(self.opposite("The lantern is not lit at dawn", "lit"), "the lantern is lit at dawn")
|
|
self.assertEqual(self.opposite("The gate isn't locked", "locked"), "the gate is locked")
|
|
self.assertEqual(self.opposite("Carters can't cross the bridge", "cross"), "carters can cross the bridge")
|
|
self.assertEqual(self.opposite("A pass is no longer required", "required"), "a pass is required")
|
|
self.assertEqual(self.opposite("It sails without a pilot", "pilot"), "it sails with a pilot")
|
|
|
|
def test_a_negation_far_before_the_atom_is_outside_the_window(self) -> None:
|
|
self.assertEqual(self.opposite("Never mind: the one two three four five six keeper rings", "rings"),
|
|
"never mind: the one two three four five six keeper not rings")
|
|
|
|
def test_without_a_negation_not_is_put_in_front_of_the_atom(self) -> None:
|
|
self.assertEqual(self.opposite("The keeper still rings the bell", "rings"), "the keeper still not rings the bell")
|
|
|
|
def test_the_sentence_with_the_negated_atom_is_chosen(self) -> None:
|
|
found = polarity.answer_from_span("lit", ["The lantern is lit at dusk. At dawn it is not lit."])
|
|
self.assertEqual(found, ("at dawn it is not lit", len("at dawn it is not ")))
|
|
|
|
|
|
class CountTests(unittest.TestCase):
|
|
def test_an_unmarked_negation_key_accepts_the_opposite_answer(self) -> None:
|
|
q = md_question("q01", "The ferry is not staffed at night.", [{"text": "staffed"}])
|
|
rep = polarity.count_polarity([q], [{"id": "q01", "atoms": ["staffed"]}])
|
|
self.assertEqual((rep.keys, rep.accept_span, rep.accept_opposite), (1, 1, 1))
|
|
self.assertFalse(rep.passed)
|
|
|
|
def test_a_negated_key_refuses_the_opposite_answer(self) -> None:
|
|
q = md_question("q01", "The ferry is not staffed at night.", [{"text": "staffed", "negated": True}])
|
|
rep = polarity.count_polarity([q], [{"id": "q01", "atoms": ["staffed"]}])
|
|
self.assertEqual((rep.keys, rep.accept_span, rep.accept_opposite), (1, 1, 0))
|
|
self.assertTrue(rep.passed)
|
|
|
|
def test_a_key_of_several_atoms_counts_once(self) -> None:
|
|
q = md_question("q01", "Only the north gate opens at dawn.",
|
|
[{"text": "north gate"}, {"text": "dawn"}])
|
|
rep = polarity.count_polarity([q], [{"id": "q01", "atoms": ["north gate", "dawn"]}])
|
|
self.assertEqual((rep.keys, rep.accept_span, rep.accept_opposite), (1, 1, 1))
|
|
|
|
def test_an_atom_that_is_not_a_key_cannot_run(self) -> None:
|
|
q = md_question("q01", "The ferry is not staffed at night.", [{"text": "staffed"}])
|
|
with self.assertRaises(RunError):
|
|
polarity.count_polarity([q], [{"id": "q01", "atoms": ["night"]}])
|
|
|
|
def test_an_atom_outside_its_spans_cannot_run(self) -> None:
|
|
q = md_question("q01", "The ferry is not staffed at night.", [{"text": "pilot"}])
|
|
with self.assertRaises(RunError):
|
|
polarity.count_polarity([q], [{"id": "q01", "atoms": ["pilot"]}])
|
|
|
|
def test_a_whole_document_key_uses_every_span_of_the_file(self) -> None:
|
|
q = {"id": "w01", "type": "whole", "question": "?", "file": "a.md",
|
|
"evidence": [{"span": "The bell rings at noon."}, {"span": "The keeper never rings it at night."}],
|
|
"answer_keys": [{"text": "rings"}]}
|
|
rep = polarity.count_polarity([q], [{"id": "w01", "atoms": ["rings"]}])
|
|
self.assertEqual(rep.listed[0].split("| ")[1], "the keeper rings it at night")
|
|
|
|
|
|
class CliTests(unittest.TestCase):
|
|
def setUp(self) -> None:
|
|
self._tmp = tempfile.TemporaryDirectory()
|
|
self.root = Path(self._tmp.name)
|
|
|
|
def tearDown(self) -> None:
|
|
self._tmp.cleanup()
|
|
|
|
def write(self, name: str, data: Any) -> str:
|
|
(self.root / name).write_text(json.dumps(data), encoding="utf-8")
|
|
return str(self.root / name)
|
|
|
|
def test_counts_and_fails_while_a_key_accepts_the_opposite(self) -> None:
|
|
s = self.write("set.json", {"schema": "okf-wiki/multidoc-set/1", "questions": [
|
|
md_question("q01", "The ferry is not staffed at night.", [{"text": "staffed"}])]})
|
|
k = self.write("keys.json", {"schema": polarity.KEYS_SCHEMA, "keys": [{"id": "q01", "atoms": ["staffed"]}]})
|
|
code, out = run(["--set", s, "--keys", k])
|
|
self.assertEqual(code, 1, out)
|
|
self.assertIn("polarity keys that accept the answer from their span: 1 of 1", out)
|
|
self.assertIn("polarity keys that accept the opposite answer: 1 of 1", out)
|
|
self.assertIn("FAIL", out)
|
|
|
|
def test_an_empty_list_is_a_fail_not_a_pass(self) -> None:
|
|
s = self.write("set.json", {"schema": "okf-wiki/multidoc-set/1", "questions": []})
|
|
k = self.write("keys.json", {"schema": polarity.KEYS_SCHEMA, "keys": []})
|
|
code, out = run(["--set", s, "--keys", k])
|
|
self.assertEqual(code, 1)
|
|
self.assertIn("0 of 0", out)
|
|
|
|
def test_a_list_without_its_schema_cannot_run(self) -> None:
|
|
s = self.write("set.json", {"schema": "okf-wiki/multidoc-set/1", "questions": []})
|
|
k = self.write("keys.json", {"keys": []})
|
|
code, out = run(["--set", s, "--keys", k])
|
|
self.assertEqual(code, 1)
|
|
self.assertIn("schema", out)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|