okf-wiki/gates/tests/test_quote_rule.py
Kjell Tore Guttormsen 0fd86a23a3
okf-wiki 1.1.1
First public release, published as one snapshot. What changed between
versions is in CHANGELOG.md.

Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
2026-11-14 21:03:20 +01:00

182 lines
8.2 KiB
Python

"""Tests for gates/quote_rule.py: rule E against rule L, the answer key and the two new reject rules (V1 Step 4).
Every fixture is synthetic gardening text (CLAUDE.md rule 3). The P1 fragments are measured by the CLI, outside the tests.
Run: python3 -m unittest discover -s gates/tests -p test_quote_rule.py
"""
from __future__ import annotations
import contextlib
import importlib
import io
import json
import sys
import tempfile
import unittest
from pathlib import Path
from typing import Any
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
import quote_rule # noqa: E402
REPO_ROOT = Path(__file__).resolve().parents[2]
def frag(text: str, quote: str | None = None, concept: str = "Basil", fid: str = "basil",
status: str = "existing") -> dict[str, Any]:
f: dict[str, Any] = {"concept": concept, "status": status, "id": fid, "text": text,
"anchor": {"file": "docs-en-basil.md", "start": 3, "end": 3}}
if quote is not None:
f["quote"] = quote
return f
INDEX: dict[str, Any] = {"entries": [
{"id": "basil", "title": "Basil", "aliases": ["Basil", "Sweet basil"]},
{"id": "soil-mix", "title": "Soil mix", "aliases": ["Soil mix"]},
]}
class RulesTest(unittest.TestCase):
def test_e_rejects_a_quoteless_prose_fragment_and_l_accepts_it(self) -> None:
f = frag("Basil likes a warm and sheltered spot.")
self.assertTrue(quote_rule.rule_e(f))
self.assertFalse(quote_rule.rule_l(f))
def test_both_reject_a_quoteless_narrow_list_word(self) -> None:
f = frag("Basil needs six hours of sun.")
self.assertTrue(quote_rule.rule_e(f))
self.assertTrue(quote_rule.rule_l(f))
def test_both_reject_a_quoteless_digit_or_backticked_name(self) -> None:
for text in ("Basil needs 6 hours of sun.", "Pinch the top `leaf` pairs."):
self.assertTrue(quote_rule.rule_e(frag(text)), text)
self.assertTrue(quote_rule.rule_l(frag(text)), text)
def test_l_lets_several_through(self) -> None:
f = frag("Sow several seeds in each pot.")
self.assertFalse(quote_rule.rule_l(f))
self.assertTrue(quote_rule.number_words(f)) # the answer key catches it
def test_a_quoted_fragment_passes_both(self) -> None:
f = frag("Basil needs six hours of sun.", quote="six hours of sun")
self.assertFalse(quote_rule.rule_e(f))
self.assertFalse(quote_rule.rule_l(f))
def test_a_blank_quote_counts_as_no_quote(self) -> None:
self.assertTrue(quote_rule.rule_e(frag("Basil likes warmth.", quote=" ")))
def test_number_words_are_whole_words(self) -> None:
self.assertFalse(quote_rule.number_words(frag("Often the seedlings wilt."))) # "ten" inside "often"
self.assertEqual(quote_rule.number_words(frag("Water Twice a week, at least once.")), ["once", "twice"])
def test_the_narrow_list_is_a_strict_subset_of_the_broad_list(self) -> None:
self.assertLess(set(quote_rule.NARROW), set(quote_rule.BROAD))
for w in ("zero", "twenty", "hundred", "thousand", "million"):
self.assertIn(w, quote_rule.NARROW)
for w in ("thirty", "ninety", "dozen", "once", "twice", "first", "tenth", "half", "double", "pair", "couple",
"several", "single", "dozens"):
self.assertIn(w, quote_rule.BROAD)
self.assertNotIn(w, quote_rule.NARROW)
class ChooseTest(unittest.TestCase):
def test_l_only_when_nothing_slips_and_e_rejects_five_percent_more(self) -> None:
self.assertEqual(quote_rule.choose(e_rejects=133, l_rejects=100, l_slips=0, n=650), "L")
self.assertEqual(quote_rule.choose(e_rejects=132, l_rejects=100, l_slips=0, n=650), "E")
self.assertEqual(quote_rule.choose(e_rejects=133, l_rejects=100, l_slips=1, n=650), "E")
def test_the_boundary_is_33_of_650(self) -> None:
self.assertEqual(quote_rule.five_percent(650), 33)
class RejectRulesTest(unittest.TestCase):
def test_short_quote(self) -> None:
self.assertTrue(quote_rule.short_quote(frag("Basil needs sun.", quote="needs sun")))
self.assertFalse(quote_rule.short_quote(frag("Basil needs sun.", quote="needs full sun")))
self.assertFalse(quote_rule.short_quote(frag("Basil needs sun."))) # no quote is rule E/L's business
def test_id_mismatch(self) -> None:
self.assertFalse(quote_rule.id_mismatch(frag("x", concept="Basil"), INDEX))
self.assertFalse(quote_rule.id_mismatch(frag("x", concept="sweet basil"), INDEX)) # an alias
self.assertFalse(quote_rule.id_mismatch(frag("x", concept="The Soil mix", fid="soil-mix"), INDEX)) # norm
self.assertTrue(quote_rule.id_mismatch(frag("x", concept="Pruning", fid="basil"), INDEX))
self.assertTrue(quote_rule.id_mismatch(frag("x", concept="Basil", fid="no-such-id"), INDEX))
self.assertFalse(quote_rule.id_mismatch(frag("x", concept="Pruning", status="new"), INDEX))
def test_id_mismatch_accepts_a_jaccard_match(self) -> None:
index = {"entries": [{"id": "pot-size", "title": "Pot size for basil", "aliases": []}]}
self.assertFalse(quote_rule.id_mismatch(frag("x", concept="Pot size for herbs", fid="pot-size"), index))
class TrialConstantsTest(unittest.TestCase):
"""The copies of the trial's constants and norm stay equal to it (read at run time, so mypy does not follow)."""
def test_copies_equal_the_trial(self) -> None:
sys.path.insert(0, str(REPO_ROOT))
sys.path.insert(0, str(REPO_ROOT / "evals"))
trial = importlib.import_module("p1.kpack")
self.assertEqual(quote_rule.NEEDS_QUOTE.pattern, getattr(trial, "NEEDS_QUOTE").pattern)
self.assertEqual(quote_rule.MERGE_JACCARD, getattr(trial, "MERGE_JACCARD"))
for name in ("The Soil mixes", "`leaf` pairs", "Pot size / basil", "glass", "Sweet basil."):
self.assertEqual(quote_rule.norm(name), getattr(trial, "norm")(name), name)
class MeasureCliTest(unittest.TestCase):
def setUp(self) -> None:
self._tmp = tempfile.TemporaryDirectory()
self.root = Path(self._tmp.name)
self.frags = self.root / "K-fragments"
self.frags.mkdir()
self.index = self.root / "index.json"
self.index.write_text(json.dumps(INDEX))
def tearDown(self) -> None:
self._tmp.cleanup()
def write(self, *fragments: dict[str, Any]) -> None:
(self.frags / "pack-1.json").write_text(json.dumps({"fragments": list(fragments)}))
def run_cli(self) -> tuple[int, dict[str, Any] | None, str]:
out = io.StringIO()
with contextlib.redirect_stdout(out):
code = quote_rule.main(["measure", str(self.frags), str(self.index)])
try:
return code, json.loads(out.getvalue()), out.getvalue()
except json.JSONDecodeError:
return code, None, out.getvalue()
def test_counts_and_choice(self) -> None:
self.write(frag("Basil likes warmth."), frag("Sow several seeds."), frag("Basil needs six hours of sun."),
frag("Basil needs sun.", quote="Basil needs warm soil"), frag("x", concept="Pruning"))
code, out, raw = self.run_cli()
self.assertEqual(code, 0, raw)
assert out is not None
self.assertEqual(out["fragments"], 5)
self.assertEqual(out["e_rejects"], 4)
self.assertEqual(out["l_rejects"], 1)
self.assertEqual(out["key_hits"], 2)
self.assertEqual(out["l_slips"], 1)
self.assertEqual(out["e_slips"], 0)
self.assertEqual(out["id_mismatches"], 1)
self.assertEqual(out["short_quotes"], 0)
self.assertEqual(out["choice"], "E")
self.assertEqual(out["key_hits_per_file"], {"pack-1.json": 2})
self.assertNotIn("Basil", raw) # counts only, never fragment text
def test_no_known_positive_fails(self) -> None:
self.write(frag("Basil likes warmth."), frag("Basil needs sun.", quote="Basil needs warm soil"))
code, out, raw = self.run_cli()
self.assertEqual(code, 1, raw)
assert out is not None
self.assertEqual(out["key_hits"], 0)
self.assertIsNone(out["choice"])
def test_no_fragments_cannot_run(self) -> None:
code, _, _ = self.run_cli()
self.assertEqual(code, 2)
if __name__ == "__main__":
unittest.main()