okf-wiki/gates/keyform.py
Kjell Tore Guttormsen 0fd86a23a3
okf-wiki 1.1.1
First public release, published as one snapshot. What changed between
versions is in CHANGELOG.md.

Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
2026-11-14 21:03:20 +01:00

93 lines
3.8 KiB
Python

"""Answer-key form (E2c): counts answer keys that are wording rather than a fact. No model call.
An answer key is the smallest fact an answer must hold: a name, a command, a flag, a setting, a number, a version or
one word that carries the meaning. A key that joins several words with a function word (an article, a pronoun or
determiner, a conjunction, a preposition, an auxiliary or modal verb, a negation, or a focusing particle) is wording:
a correct answer can say the same thing in other words and miss the key. Text inside backticks is code, not wording.
python3 gates/keyform.py --set <set.json> [--set <set.json> ...] [--list]
Prints "answer keys that are wording: X of N" over every key in the sets (either schema). Exit 0 = PASS (X = 0 and
N > 0); 1 = FAIL or could not run.
"""
from __future__ import annotations
import argparse
import json
import re
import sys
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any
FUNCTION_WORDS = frozenset("""
a an the
i me my you your he him his she her it its we us our they them their this that these those who whom whose which what
each every all any some
and or but nor so if when while because than as whether then
to of in on at by for from with without within into onto over under about after before between through beyond via
per up down out off against during until
is are was were be been being am can could may might will would shall should must do does did has have had
isn't aren't wasn't weren't don't doesn't didn't can't cannot won't wouldn't shouldn't hasn't haven't
not no never only also still just
""".split())
def is_wording(text: str) -> bool:
"""True when the key has two or more words and a word outside backticks is a function word."""
if len(text.split()) < 2:
return False
outside = re.sub(r"`[^`]*`", " ", text)
words = (w.strip(".,;:!?()[]{}\"'").casefold() for w in outside.split())
return any(w in FUNCTION_WORDS for w in words)
@dataclass
class KeyReport:
keys: int = 0
wording: int = 0
listed: list[str] = field(default_factory=list)
def count_keys(questions: list[dict[str, Any]]) -> KeyReport:
rep = KeyReport()
for q in questions:
for k in q.get("answer_keys", []) if isinstance(q, dict) else []:
text = k.get("text") if isinstance(k, dict) else None
rep.keys += 1
if isinstance(text, str) and is_wording(text):
rep.wording += 1
rep.listed.append(f"{q.get('id')}: {text}")
return rep
def main(argv: list[str] | None = None) -> int:
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
p.add_argument("--set", action="append", required=True)
p.add_argument("--list", action="store_true", help="list every wording key")
args = p.parse_args(argv)
total = KeyReport()
for path in args.set:
try:
data = json.loads(Path(path).read_text(encoding="utf-8"))
except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc:
print(f"FAIL: cannot read set {path}: {exc}", file=sys.stderr)
return 1
if not isinstance(data, dict) or not isinstance(data.get("questions"), list):
print(f"FAIL: {path} is not a set", file=sys.stderr)
return 1
rep = count_keys(data["questions"])
total.keys += rep.keys
total.wording += rep.wording
total.listed += [f"{Path(path).name} {line}" for line in rep.listed]
if args.list:
print("\n".join(total.listed))
print(f"answer keys that are wording: {total.wording} of {total.keys}")
passed = total.keys > 0 and total.wording == 0
print("PASS" if passed else "FAIL")
return 0 if passed else 1
if __name__ == "__main__":
sys.exit(main())