okf-wiki/gates/quote_rule.py
Kjell Tore Guttormsen 0fd86a23a3
okf-wiki 1.1.1
First public release, published as one snapshot. What changed between
versions is in CHANGELOG.md.

Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
2026-11-14 21:03:20 +01:00

216 lines
8.8 KiB
Python

"""The quote rule for K's fragments, measured without a model (V1 Step 4, brief SC3, R5).
P1's compiler wrote numbers as words to slip past a rule that asked for a quote only when the text held a digit or a
backticked name. Two rules close that path, and this module measures both on P1's fragments:
- Rule E: every fragment carries a quote.
- Rule L: a fragment carries a quote when its text holds a digit, a backticked name, or a number word from the NARROW
list (zero to twenty, hundred, thousand, million).
The answer key is made apart from rule L: a quoteless fragment whose text holds a word from the BROAD list (brief SC3).
Its denominator is the number of hits. The key must find at least one fragment (the known-positive: P1 saw the compiler
do this), or the list fails and the run exits 1.
E is chosen unless L lets 0 key fragments through AND E rejects at least 5 % of the fragments more than L does (33 of
650); then L. E is the simplest rule and closes the path outright; L pays only if E forces noticeably more rewrites.
Two new reject rules are measured next to it, each adopted in Step 6 only if it rejects at most 5 % of the fragments:
- short quote: a quote of fewer than 3 words;
- id mismatch: an "existing" fragment whose index entry's title and aliases match its concept name neither by
normalised form nor by Jaccard >= MERGE_JACCARD (or whose id is not in the index).
NEEDS_QUOTE, MERGE_JACCARD and norm are copies of the trial's (evals/p1/kpack.py), kept equal by a test that reads the
trial at run time. They are copied, not imported, so that `mypy --strict okfwiki gates tests` never follows trial code.
measure FRAGMENTS_DIR INDEX
Every fragment in FRAGMENTS_DIR/*.json ({"fragments": [...]}) against both rules and both reject rules.
Prints one JSON object of counts only (never fragment text): the counts, the answer-key hits per file
(to find the known-positive pack) and the choice.
Exit 0 = measured and a rule chosen, 1 = the answer key found no fragment (no choice), 2 = could not run.
"""
from __future__ import annotations
import argparse
import json
import math
import re
import sys
from pathlib import Path
from typing import Any
NEEDS_QUOTE = re.compile(r"\d|`")
MERGE_JACCARD = 0.6
NARROW = ("zero", "one", "two", "three", "four", "five", "six", "seven", "eight", "nine", "ten", "eleven", "twelve",
"thirteen", "fourteen", "fifteen", "sixteen", "seventeen", "eighteen", "nineteen", "twenty",
"hundred", "thousand", "million")
BROAD = NARROW + ("thirty", "forty", "fifty", "sixty", "seventy", "eighty", "ninety",
"dozen", "dozens", "once", "twice",
"first", "second", "third", "fourth", "fifth", "sixth", "seventh", "eighth", "ninth", "tenth",
"half", "double", "pair", "couple", "several", "single")
SHORT_QUOTE_WORDS = 3
def _word_pattern(words: tuple[str, ...]) -> re.Pattern[str]:
return re.compile(r"(?<![\w-])(" + "|".join(sorted(words, key=len, reverse=True)) + r")(?![\w-])", re.IGNORECASE)
NARROW_WORD = _word_pattern(NARROW)
BROAD_WORD = _word_pattern(BROAD)
def norm(name: str) -> str:
text = name.replace("`", "").lower()
text = re.sub(r"[^\w\s/.\-]", " ", text)
words = text.split()
if words and words[0] == "the":
words = words[1:]
words = [w[:-1] if len(w) > 3 and w.endswith("s") and not w.endswith("ss") else w for w in words]
return " ".join(w.strip(".") or w for w in words)
def _text(fragment: dict[str, Any]) -> str:
return str(fragment.get("text", ""))
def has_quote(fragment: dict[str, Any]) -> bool:
q = fragment.get("quote")
return isinstance(q, str) and bool(q.strip())
def rule_e(fragment: dict[str, Any]) -> bool:
"""True when rule E rejects the fragment: it has no quote."""
return not has_quote(fragment)
def rule_l(fragment: dict[str, Any]) -> bool:
"""True when rule L rejects the fragment: no quote, and a digit, a backticked name or a NARROW number word."""
text = _text(fragment)
return not has_quote(fragment) and bool(NEEDS_QUOTE.search(text) or NARROW_WORD.search(text))
def number_words(fragment: dict[str, Any]) -> list[str]:
"""The answer key: the BROAD-list words in a quoteless fragment's text, lower-cased and sorted ([] when quoted)."""
if has_quote(fragment):
return []
return sorted({m.group(1).lower() for m in BROAD_WORD.finditer(_text(fragment))})
def short_quote(fragment: dict[str, Any], min_words: int = SHORT_QUOTE_WORDS) -> bool:
return has_quote(fragment) and len(str(fragment["quote"]).split()) < min_words
def _jaccard(a: str, b: str) -> float:
x, y = set(a.split()), set(b.split())
return len(x & y) / len(x | y) if x | y else 0.0
def id_mismatch(fragment: dict[str, Any], index: dict[str, Any]) -> bool:
if fragment.get("status") != "existing":
return False
entry = next((e for e in index.get("entries", []) if e.get("id") == fragment.get("id")), None)
if entry is None:
return True
names = {norm(str(n)) for n in [entry.get("title", ""), *entry.get("aliases", [])]} - {""}
concept = norm(str(fragment.get("concept", "")))
return concept not in names and not any(_jaccard(concept, n) >= MERGE_JACCARD for n in names)
def five_percent(n: int) -> int:
return math.ceil(0.05 * n)
def choose(e_rejects: int, l_rejects: int, l_slips: int, n: int) -> str:
return "L" if l_slips == 0 and e_rejects - l_rejects >= five_percent(n) else "E"
class RunError(Exception):
"""The measurement cannot run as asked."""
def load_fragments(directory: Path) -> list[tuple[str, dict[str, Any]]]:
"""(file name, fragment) for every fragment in directory/*.json."""
files = sorted(directory.glob("*.json"))
out: list[tuple[str, dict[str, Any]]] = []
for f in files:
try:
data = json.loads(f.read_text(encoding="utf-8"))
except (OSError, json.JSONDecodeError) as exc:
raise RunError(f"cannot read {f}: {exc}") from exc
items = data.get("fragments") if isinstance(data, dict) else None
if not isinstance(items, list):
raise RunError(f"{f} has no fragments list")
out.extend((f.name, x) for x in items if isinstance(x, dict))
if not out:
raise RunError(f"no fragments under {directory}")
return out
def measure(located: list[tuple[str, dict[str, Any]]], index: dict[str, Any]) -> dict[str, Any]:
fragments = [f for _, f in located]
n = len(fragments)
per_file: dict[str, int] = {}
for name, f in located:
if number_words(f):
per_file[name] = per_file.get(name, 0) + 1
key = [f for f in fragments if number_words(f)]
e_rejects = sum(rule_e(f) for f in fragments)
l_rejects = sum(rule_l(f) for f in fragments)
short = sum(short_quote(f) for f in fragments)
mismatch = sum(id_mismatch(f, index) for f in fragments)
words: dict[str, int] = {}
for f in key:
for w in number_words(f):
words[w] = words.get(w, 0) + 1
result: dict[str, Any] = {
"schema": "okf-wiki/quote-rule/1",
"fragments": n,
"quoteless": e_rejects,
"e_rejects": e_rejects,
"l_rejects": l_rejects,
"key_hits": len(key),
"key_hits_narrow": sum(bool(NARROW_WORD.search(_text(f))) for f in key),
"key_words": dict(sorted(words.items())),
"key_hits_per_file": dict(sorted(per_file.items())),
"e_slips": sum(not rule_e(f) for f in key),
"l_slips": sum(not rule_l(f) for f in key),
"five_percent": five_percent(n),
"short_quotes": short,
"short_quote_adopt": short * 20 <= n,
"id_mismatches": mismatch,
"id_mismatch_adopt": mismatch * 20 <= n,
"choice": None,
}
if key:
result["choice"] = choose(e_rejects, l_rejects, result["l_slips"], n)
return result
def build_parser() -> argparse.ArgumentParser:
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
sub = p.add_subparsers(dest="command", required=True)
m = sub.add_parser("measure", help="measure rule E, rule L and the reject rules on a fragments directory")
m.add_argument("fragments", type=Path)
m.add_argument("index", type=Path)
return p
def main(argv: list[str] | None = None) -> int:
args = build_parser().parse_args(argv)
try:
fragments = load_fragments(args.fragments)
try:
index = json.loads(args.index.read_text(encoding="utf-8"))
except (OSError, json.JSONDecodeError) as exc:
raise RunError(f"cannot read {args.index}: {exc}") from exc
except RunError as exc:
print(f"cannot run: {exc}", file=sys.stderr)
return 2
result = measure(fragments, index if isinstance(index, dict) else {})
print(json.dumps(result, indent=1))
return 0 if result["choice"] else 1
if __name__ == "__main__":
sys.exit(main())