"""Answer-key form (E2c): counts answer keys that are wording rather than a fact. No model call. An answer key is the smallest fact an answer must hold: a name, a command, a flag, a setting, a number, a version or one word that carries the meaning. A key that joins several words with a function word (an article, a pronoun or determiner, a conjunction, a preposition, an auxiliary or modal verb, a negation, or a focusing particle) is wording: a correct answer can say the same thing in other words and miss the key. Text inside backticks is code, not wording. python3 gates/keyform.py --set [--set ...] [--list] Prints "answer keys that are wording: X of N" over every key in the sets (either schema). Exit 0 = PASS (X = 0 and N > 0); 1 = FAIL or could not run. """ from __future__ import annotations import argparse import json import re import sys from dataclasses import dataclass, field from pathlib import Path from typing import Any FUNCTION_WORDS = frozenset(""" a an the i me my you your he him his she her it its we us our they them their this that these those who whom whose which what each every all any some and or but nor so if when while because than as whether then to of in on at by for from with without within into onto over under about after before between through beyond via per up down out off against during until is are was were be been being am can could may might will would shall should must do does did has have had isn't aren't wasn't weren't don't doesn't didn't can't cannot won't wouldn't shouldn't hasn't haven't not no never only also still just """.split()) def is_wording(text: str) -> bool: """True when the key has two or more words and a word outside backticks is a function word.""" if len(text.split()) < 2: return False outside = re.sub(r"`[^`]*`", " ", text) words = (w.strip(".,;:!?()[]{}\"'").casefold() for w in outside.split()) return any(w in FUNCTION_WORDS for w in words) @dataclass class KeyReport: keys: int = 0 wording: int = 0 listed: list[str] = field(default_factory=list) def count_keys(questions: list[dict[str, Any]]) -> KeyReport: rep = KeyReport() for q in questions: for k in q.get("answer_keys", []) if isinstance(q, dict) else []: text = k.get("text") if isinstance(k, dict) else None rep.keys += 1 if isinstance(text, str) and is_wording(text): rep.wording += 1 rep.listed.append(f"{q.get('id')}: {text}") return rep def main(argv: list[str] | None = None) -> int: p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) p.add_argument("--set", action="append", required=True) p.add_argument("--list", action="store_true", help="list every wording key") args = p.parse_args(argv) total = KeyReport() for path in args.set: try: data = json.loads(Path(path).read_text(encoding="utf-8")) except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc: print(f"FAIL: cannot read set {path}: {exc}", file=sys.stderr) return 1 if not isinstance(data, dict) or not isinstance(data.get("questions"), list): print(f"FAIL: {path} is not a set", file=sys.stderr) return 1 rep = count_keys(data["questions"]) total.keys += rep.keys total.wording += rep.wording total.listed += [f"{Path(path).name} {line}" for line in rep.listed] if args.list: print("\n".join(total.listed)) print(f"answer keys that are wording: {total.wording} of {total.keys}") passed = total.keys > 0 and total.wording == 0 print("PASS" if passed else "FAIL") return 0 if passed else 1 if __name__ == "__main__": sys.exit(main())