First public release, published as one snapshot. What changed between versions is in CHANGELOG.md. Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
182 lines
8.2 KiB
Python
182 lines
8.2 KiB
Python
"""Tests for gates/quote_rule.py: rule E against rule L, the answer key and the two new reject rules (V1 Step 4).
|
|
|
|
Every fixture is synthetic gardening text (CLAUDE.md rule 3). The P1 fragments are measured by the CLI, outside the tests.
|
|
Run: python3 -m unittest discover -s gates/tests -p test_quote_rule.py
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import contextlib
|
|
import importlib
|
|
import io
|
|
import json
|
|
import sys
|
|
import tempfile
|
|
import unittest
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
|
|
|
import quote_rule # noqa: E402
|
|
|
|
REPO_ROOT = Path(__file__).resolve().parents[2]
|
|
|
|
|
|
def frag(text: str, quote: str | None = None, concept: str = "Basil", fid: str = "basil",
|
|
status: str = "existing") -> dict[str, Any]:
|
|
f: dict[str, Any] = {"concept": concept, "status": status, "id": fid, "text": text,
|
|
"anchor": {"file": "docs-en-basil.md", "start": 3, "end": 3}}
|
|
if quote is not None:
|
|
f["quote"] = quote
|
|
return f
|
|
|
|
|
|
INDEX: dict[str, Any] = {"entries": [
|
|
{"id": "basil", "title": "Basil", "aliases": ["Basil", "Sweet basil"]},
|
|
{"id": "soil-mix", "title": "Soil mix", "aliases": ["Soil mix"]},
|
|
]}
|
|
|
|
|
|
class RulesTest(unittest.TestCase):
|
|
def test_e_rejects_a_quoteless_prose_fragment_and_l_accepts_it(self) -> None:
|
|
f = frag("Basil likes a warm and sheltered spot.")
|
|
self.assertTrue(quote_rule.rule_e(f))
|
|
self.assertFalse(quote_rule.rule_l(f))
|
|
|
|
def test_both_reject_a_quoteless_narrow_list_word(self) -> None:
|
|
f = frag("Basil needs six hours of sun.")
|
|
self.assertTrue(quote_rule.rule_e(f))
|
|
self.assertTrue(quote_rule.rule_l(f))
|
|
|
|
def test_both_reject_a_quoteless_digit_or_backticked_name(self) -> None:
|
|
for text in ("Basil needs 6 hours of sun.", "Pinch the top `leaf` pairs."):
|
|
self.assertTrue(quote_rule.rule_e(frag(text)), text)
|
|
self.assertTrue(quote_rule.rule_l(frag(text)), text)
|
|
|
|
def test_l_lets_several_through(self) -> None:
|
|
f = frag("Sow several seeds in each pot.")
|
|
self.assertFalse(quote_rule.rule_l(f))
|
|
self.assertTrue(quote_rule.number_words(f)) # the answer key catches it
|
|
|
|
def test_a_quoted_fragment_passes_both(self) -> None:
|
|
f = frag("Basil needs six hours of sun.", quote="six hours of sun")
|
|
self.assertFalse(quote_rule.rule_e(f))
|
|
self.assertFalse(quote_rule.rule_l(f))
|
|
|
|
def test_a_blank_quote_counts_as_no_quote(self) -> None:
|
|
self.assertTrue(quote_rule.rule_e(frag("Basil likes warmth.", quote=" ")))
|
|
|
|
def test_number_words_are_whole_words(self) -> None:
|
|
self.assertFalse(quote_rule.number_words(frag("Often the seedlings wilt."))) # "ten" inside "often"
|
|
self.assertEqual(quote_rule.number_words(frag("Water Twice a week, at least once.")), ["once", "twice"])
|
|
|
|
def test_the_narrow_list_is_a_strict_subset_of_the_broad_list(self) -> None:
|
|
self.assertLess(set(quote_rule.NARROW), set(quote_rule.BROAD))
|
|
for w in ("zero", "twenty", "hundred", "thousand", "million"):
|
|
self.assertIn(w, quote_rule.NARROW)
|
|
for w in ("thirty", "ninety", "dozen", "once", "twice", "first", "tenth", "half", "double", "pair", "couple",
|
|
"several", "single", "dozens"):
|
|
self.assertIn(w, quote_rule.BROAD)
|
|
self.assertNotIn(w, quote_rule.NARROW)
|
|
|
|
|
|
class ChooseTest(unittest.TestCase):
|
|
def test_l_only_when_nothing_slips_and_e_rejects_five_percent_more(self) -> None:
|
|
self.assertEqual(quote_rule.choose(e_rejects=133, l_rejects=100, l_slips=0, n=650), "L")
|
|
self.assertEqual(quote_rule.choose(e_rejects=132, l_rejects=100, l_slips=0, n=650), "E")
|
|
self.assertEqual(quote_rule.choose(e_rejects=133, l_rejects=100, l_slips=1, n=650), "E")
|
|
|
|
def test_the_boundary_is_33_of_650(self) -> None:
|
|
self.assertEqual(quote_rule.five_percent(650), 33)
|
|
|
|
|
|
class RejectRulesTest(unittest.TestCase):
|
|
def test_short_quote(self) -> None:
|
|
self.assertTrue(quote_rule.short_quote(frag("Basil needs sun.", quote="needs sun")))
|
|
self.assertFalse(quote_rule.short_quote(frag("Basil needs sun.", quote="needs full sun")))
|
|
self.assertFalse(quote_rule.short_quote(frag("Basil needs sun."))) # no quote is rule E/L's business
|
|
|
|
def test_id_mismatch(self) -> None:
|
|
self.assertFalse(quote_rule.id_mismatch(frag("x", concept="Basil"), INDEX))
|
|
self.assertFalse(quote_rule.id_mismatch(frag("x", concept="sweet basil"), INDEX)) # an alias
|
|
self.assertFalse(quote_rule.id_mismatch(frag("x", concept="The Soil mix", fid="soil-mix"), INDEX)) # norm
|
|
self.assertTrue(quote_rule.id_mismatch(frag("x", concept="Pruning", fid="basil"), INDEX))
|
|
self.assertTrue(quote_rule.id_mismatch(frag("x", concept="Basil", fid="no-such-id"), INDEX))
|
|
self.assertFalse(quote_rule.id_mismatch(frag("x", concept="Pruning", status="new"), INDEX))
|
|
|
|
def test_id_mismatch_accepts_a_jaccard_match(self) -> None:
|
|
index = {"entries": [{"id": "pot-size", "title": "Pot size for basil", "aliases": []}]}
|
|
self.assertFalse(quote_rule.id_mismatch(frag("x", concept="Pot size for herbs", fid="pot-size"), index))
|
|
|
|
|
|
class TrialConstantsTest(unittest.TestCase):
|
|
"""The copies of the trial's constants and norm stay equal to it (read at run time, so mypy does not follow)."""
|
|
|
|
def test_copies_equal_the_trial(self) -> None:
|
|
sys.path.insert(0, str(REPO_ROOT))
|
|
sys.path.insert(0, str(REPO_ROOT / "evals"))
|
|
trial = importlib.import_module("p1.kpack")
|
|
self.assertEqual(quote_rule.NEEDS_QUOTE.pattern, getattr(trial, "NEEDS_QUOTE").pattern)
|
|
self.assertEqual(quote_rule.MERGE_JACCARD, getattr(trial, "MERGE_JACCARD"))
|
|
for name in ("The Soil mixes", "`leaf` pairs", "Pot size / basil", "glass", "Sweet basil."):
|
|
self.assertEqual(quote_rule.norm(name), getattr(trial, "norm")(name), name)
|
|
|
|
|
|
class MeasureCliTest(unittest.TestCase):
|
|
def setUp(self) -> None:
|
|
self._tmp = tempfile.TemporaryDirectory()
|
|
self.root = Path(self._tmp.name)
|
|
self.frags = self.root / "K-fragments"
|
|
self.frags.mkdir()
|
|
self.index = self.root / "index.json"
|
|
self.index.write_text(json.dumps(INDEX))
|
|
|
|
def tearDown(self) -> None:
|
|
self._tmp.cleanup()
|
|
|
|
def write(self, *fragments: dict[str, Any]) -> None:
|
|
(self.frags / "pack-1.json").write_text(json.dumps({"fragments": list(fragments)}))
|
|
|
|
def run_cli(self) -> tuple[int, dict[str, Any] | None, str]:
|
|
out = io.StringIO()
|
|
with contextlib.redirect_stdout(out):
|
|
code = quote_rule.main(["measure", str(self.frags), str(self.index)])
|
|
try:
|
|
return code, json.loads(out.getvalue()), out.getvalue()
|
|
except json.JSONDecodeError:
|
|
return code, None, out.getvalue()
|
|
|
|
def test_counts_and_choice(self) -> None:
|
|
self.write(frag("Basil likes warmth."), frag("Sow several seeds."), frag("Basil needs six hours of sun."),
|
|
frag("Basil needs sun.", quote="Basil needs warm soil"), frag("x", concept="Pruning"))
|
|
code, out, raw = self.run_cli()
|
|
self.assertEqual(code, 0, raw)
|
|
assert out is not None
|
|
self.assertEqual(out["fragments"], 5)
|
|
self.assertEqual(out["e_rejects"], 4)
|
|
self.assertEqual(out["l_rejects"], 1)
|
|
self.assertEqual(out["key_hits"], 2)
|
|
self.assertEqual(out["l_slips"], 1)
|
|
self.assertEqual(out["e_slips"], 0)
|
|
self.assertEqual(out["id_mismatches"], 1)
|
|
self.assertEqual(out["short_quotes"], 0)
|
|
self.assertEqual(out["choice"], "E")
|
|
self.assertEqual(out["key_hits_per_file"], {"pack-1.json": 2})
|
|
self.assertNotIn("Basil", raw) # counts only, never fragment text
|
|
|
|
def test_no_known_positive_fails(self) -> None:
|
|
self.write(frag("Basil likes warmth."), frag("Basil needs sun.", quote="Basil needs warm soil"))
|
|
code, out, raw = self.run_cli()
|
|
self.assertEqual(code, 1, raw)
|
|
assert out is not None
|
|
self.assertEqual(out["key_hits"], 0)
|
|
self.assertIsNone(out["choice"])
|
|
|
|
def test_no_fragments_cannot_run(self) -> None:
|
|
code, _, _ = self.run_cli()
|
|
self.assertEqual(code, 2)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|