First public release, published as one snapshot. What changed between versions is in CHANGELOG.md. Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
93 lines
3.8 KiB
Python
93 lines
3.8 KiB
Python
"""Answer-key form (E2c): counts answer keys that are wording rather than a fact. No model call.
|
|
|
|
An answer key is the smallest fact an answer must hold: a name, a command, a flag, a setting, a number, a version or
|
|
one word that carries the meaning. A key that joins several words with a function word (an article, a pronoun or
|
|
determiner, a conjunction, a preposition, an auxiliary or modal verb, a negation, or a focusing particle) is wording:
|
|
a correct answer can say the same thing in other words and miss the key. Text inside backticks is code, not wording.
|
|
|
|
python3 gates/keyform.py --set <set.json> [--set <set.json> ...] [--list]
|
|
|
|
Prints "answer keys that are wording: X of N" over every key in the sets (either schema). Exit 0 = PASS (X = 0 and
|
|
N > 0); 1 = FAIL or could not run.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import re
|
|
import sys
|
|
from dataclasses import dataclass, field
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
FUNCTION_WORDS = frozenset("""
|
|
a an the
|
|
i me my you your he him his she her it its we us our they them their this that these those who whom whose which what
|
|
each every all any some
|
|
and or but nor so if when while because than as whether then
|
|
to of in on at by for from with without within into onto over under about after before between through beyond via
|
|
per up down out off against during until
|
|
is are was were be been being am can could may might will would shall should must do does did has have had
|
|
isn't aren't wasn't weren't don't doesn't didn't can't cannot won't wouldn't shouldn't hasn't haven't
|
|
not no never only also still just
|
|
""".split())
|
|
|
|
|
|
def is_wording(text: str) -> bool:
|
|
"""True when the key has two or more words and a word outside backticks is a function word."""
|
|
if len(text.split()) < 2:
|
|
return False
|
|
outside = re.sub(r"`[^`]*`", " ", text)
|
|
words = (w.strip(".,;:!?()[]{}\"'").casefold() for w in outside.split())
|
|
return any(w in FUNCTION_WORDS for w in words)
|
|
|
|
|
|
@dataclass
|
|
class KeyReport:
|
|
keys: int = 0
|
|
wording: int = 0
|
|
listed: list[str] = field(default_factory=list)
|
|
|
|
|
|
def count_keys(questions: list[dict[str, Any]]) -> KeyReport:
|
|
rep = KeyReport()
|
|
for q in questions:
|
|
for k in q.get("answer_keys", []) if isinstance(q, dict) else []:
|
|
text = k.get("text") if isinstance(k, dict) else None
|
|
rep.keys += 1
|
|
if isinstance(text, str) and is_wording(text):
|
|
rep.wording += 1
|
|
rep.listed.append(f"{q.get('id')}: {text}")
|
|
return rep
|
|
|
|
|
|
def main(argv: list[str] | None = None) -> int:
|
|
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
p.add_argument("--set", action="append", required=True)
|
|
p.add_argument("--list", action="store_true", help="list every wording key")
|
|
args = p.parse_args(argv)
|
|
total = KeyReport()
|
|
for path in args.set:
|
|
try:
|
|
data = json.loads(Path(path).read_text(encoding="utf-8"))
|
|
except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc:
|
|
print(f"FAIL: cannot read set {path}: {exc}", file=sys.stderr)
|
|
return 1
|
|
if not isinstance(data, dict) or not isinstance(data.get("questions"), list):
|
|
print(f"FAIL: {path} is not a set", file=sys.stderr)
|
|
return 1
|
|
rep = count_keys(data["questions"])
|
|
total.keys += rep.keys
|
|
total.wording += rep.wording
|
|
total.listed += [f"{Path(path).name} {line}" for line in rep.listed]
|
|
if args.list:
|
|
print("\n".join(total.listed))
|
|
print(f"answer keys that are wording: {total.wording} of {total.keys}")
|
|
passed = total.keys > 0 and total.wording == 0
|
|
print("PASS" if passed else "FAIL")
|
|
return 0 if passed else 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|