"""The quote rule for K's fragments, measured without a model (V1 Step 4, brief SC3, R5). P1's compiler wrote numbers as words to slip past a rule that asked for a quote only when the text held a digit or a backticked name. Two rules close that path, and this module measures both on P1's fragments: - Rule E: every fragment carries a quote. - Rule L: a fragment carries a quote when its text holds a digit, a backticked name, or a number word from the NARROW list (zero to twenty, hundred, thousand, million). The answer key is made apart from rule L: a quoteless fragment whose text holds a word from the BROAD list (brief SC3). Its denominator is the number of hits. The key must find at least one fragment (the known-positive: P1 saw the compiler do this), or the list fails and the run exits 1. E is chosen unless L lets 0 key fragments through AND E rejects at least 5 % of the fragments more than L does (33 of 650); then L. E is the simplest rule and closes the path outright; L pays only if E forces noticeably more rewrites. Two new reject rules are measured next to it, each adopted in Step 6 only if it rejects at most 5 % of the fragments: - short quote: a quote of fewer than 3 words; - id mismatch: an "existing" fragment whose index entry's title and aliases match its concept name neither by normalised form nor by Jaccard >= MERGE_JACCARD (or whose id is not in the index). NEEDS_QUOTE, MERGE_JACCARD and norm are copies of the trial's (evals/p1/kpack.py), kept equal by a test that reads the trial at run time. They are copied, not imported, so that `mypy --strict okfwiki gates tests` never follows trial code. measure FRAGMENTS_DIR INDEX Every fragment in FRAGMENTS_DIR/*.json ({"fragments": [...]}) against both rules and both reject rules. Prints one JSON object of counts only (never fragment text): the counts, the answer-key hits per file (to find the known-positive pack) and the choice. Exit 0 = measured and a rule chosen, 1 = the answer key found no fragment (no choice), 2 = could not run. """ from __future__ import annotations import argparse import json import math import re import sys from pathlib import Path from typing import Any NEEDS_QUOTE = re.compile(r"\d|`") MERGE_JACCARD = 0.6 NARROW = ("zero", "one", "two", "three", "four", "five", "six", "seven", "eight", "nine", "ten", "eleven", "twelve", "thirteen", "fourteen", "fifteen", "sixteen", "seventeen", "eighteen", "nineteen", "twenty", "hundred", "thousand", "million") BROAD = NARROW + ("thirty", "forty", "fifty", "sixty", "seventy", "eighty", "ninety", "dozen", "dozens", "once", "twice", "first", "second", "third", "fourth", "fifth", "sixth", "seventh", "eighth", "ninth", "tenth", "half", "double", "pair", "couple", "several", "single") SHORT_QUOTE_WORDS = 3 def _word_pattern(words: tuple[str, ...]) -> re.Pattern[str]: return re.compile(r"(? str: text = name.replace("`", "").lower() text = re.sub(r"[^\w\s/.\-]", " ", text) words = text.split() if words and words[0] == "the": words = words[1:] words = [w[:-1] if len(w) > 3 and w.endswith("s") and not w.endswith("ss") else w for w in words] return " ".join(w.strip(".") or w for w in words) def _text(fragment: dict[str, Any]) -> str: return str(fragment.get("text", "")) def has_quote(fragment: dict[str, Any]) -> bool: q = fragment.get("quote") return isinstance(q, str) and bool(q.strip()) def rule_e(fragment: dict[str, Any]) -> bool: """True when rule E rejects the fragment: it has no quote.""" return not has_quote(fragment) def rule_l(fragment: dict[str, Any]) -> bool: """True when rule L rejects the fragment: no quote, and a digit, a backticked name or a NARROW number word.""" text = _text(fragment) return not has_quote(fragment) and bool(NEEDS_QUOTE.search(text) or NARROW_WORD.search(text)) def number_words(fragment: dict[str, Any]) -> list[str]: """The answer key: the BROAD-list words in a quoteless fragment's text, lower-cased and sorted ([] when quoted).""" if has_quote(fragment): return [] return sorted({m.group(1).lower() for m in BROAD_WORD.finditer(_text(fragment))}) def short_quote(fragment: dict[str, Any], min_words: int = SHORT_QUOTE_WORDS) -> bool: return has_quote(fragment) and len(str(fragment["quote"]).split()) < min_words def _jaccard(a: str, b: str) -> float: x, y = set(a.split()), set(b.split()) return len(x & y) / len(x | y) if x | y else 0.0 def id_mismatch(fragment: dict[str, Any], index: dict[str, Any]) -> bool: if fragment.get("status") != "existing": return False entry = next((e for e in index.get("entries", []) if e.get("id") == fragment.get("id")), None) if entry is None: return True names = {norm(str(n)) for n in [entry.get("title", ""), *entry.get("aliases", [])]} - {""} concept = norm(str(fragment.get("concept", ""))) return concept not in names and not any(_jaccard(concept, n) >= MERGE_JACCARD for n in names) def five_percent(n: int) -> int: return math.ceil(0.05 * n) def choose(e_rejects: int, l_rejects: int, l_slips: int, n: int) -> str: return "L" if l_slips == 0 and e_rejects - l_rejects >= five_percent(n) else "E" class RunError(Exception): """The measurement cannot run as asked.""" def load_fragments(directory: Path) -> list[tuple[str, dict[str, Any]]]: """(file name, fragment) for every fragment in directory/*.json.""" files = sorted(directory.glob("*.json")) out: list[tuple[str, dict[str, Any]]] = [] for f in files: try: data = json.loads(f.read_text(encoding="utf-8")) except (OSError, json.JSONDecodeError) as exc: raise RunError(f"cannot read {f}: {exc}") from exc items = data.get("fragments") if isinstance(data, dict) else None if not isinstance(items, list): raise RunError(f"{f} has no fragments list") out.extend((f.name, x) for x in items if isinstance(x, dict)) if not out: raise RunError(f"no fragments under {directory}") return out def measure(located: list[tuple[str, dict[str, Any]]], index: dict[str, Any]) -> dict[str, Any]: fragments = [f for _, f in located] n = len(fragments) per_file: dict[str, int] = {} for name, f in located: if number_words(f): per_file[name] = per_file.get(name, 0) + 1 key = [f for f in fragments if number_words(f)] e_rejects = sum(rule_e(f) for f in fragments) l_rejects = sum(rule_l(f) for f in fragments) short = sum(short_quote(f) for f in fragments) mismatch = sum(id_mismatch(f, index) for f in fragments) words: dict[str, int] = {} for f in key: for w in number_words(f): words[w] = words.get(w, 0) + 1 result: dict[str, Any] = { "schema": "okf-wiki/quote-rule/1", "fragments": n, "quoteless": e_rejects, "e_rejects": e_rejects, "l_rejects": l_rejects, "key_hits": len(key), "key_hits_narrow": sum(bool(NARROW_WORD.search(_text(f))) for f in key), "key_words": dict(sorted(words.items())), "key_hits_per_file": dict(sorted(per_file.items())), "e_slips": sum(not rule_e(f) for f in key), "l_slips": sum(not rule_l(f) for f in key), "five_percent": five_percent(n), "short_quotes": short, "short_quote_adopt": short * 20 <= n, "id_mismatches": mismatch, "id_mismatch_adopt": mismatch * 20 <= n, "choice": None, } if key: result["choice"] = choose(e_rejects, l_rejects, result["l_slips"], n) return result def build_parser() -> argparse.ArgumentParser: p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) sub = p.add_subparsers(dest="command", required=True) m = sub.add_parser("measure", help="measure rule E, rule L and the reject rules on a fragments directory") m.add_argument("fragments", type=Path) m.add_argument("index", type=Path) return p def main(argv: list[str] | None = None) -> int: args = build_parser().parse_args(argv) try: fragments = load_fragments(args.fragments) try: index = json.loads(args.index.read_text(encoding="utf-8")) except (OSError, json.JSONDecodeError) as exc: raise RunError(f"cannot read {args.index}: {exc}") from exc except RunError as exc: print(f"cannot run: {exc}", file=sys.stderr) return 2 result = measure(fragments, index if isinstance(index, dict) else {}) print(json.dumps(result, indent=1)) return 0 if result["choice"] else 1 if __name__ == "__main__": sys.exit(main())