okf-wiki 1.1.1
First public release, published as one snapshot. What changed between versions is in CHANGELOG.md. Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
This commit is contained in:
commit
0fd86a23a3
227 changed files with 96077 additions and 0 deletions
93
gates/keyform.py
Normal file
93
gates/keyform.py
Normal file
|
|
@ -0,0 +1,93 @@
|
|||
"""Answer-key form (E2c): counts answer keys that are wording rather than a fact. No model call.
|
||||
|
||||
An answer key is the smallest fact an answer must hold: a name, a command, a flag, a setting, a number, a version or
|
||||
one word that carries the meaning. A key that joins several words with a function word (an article, a pronoun or
|
||||
determiner, a conjunction, a preposition, an auxiliary or modal verb, a negation, or a focusing particle) is wording:
|
||||
a correct answer can say the same thing in other words and miss the key. Text inside backticks is code, not wording.
|
||||
|
||||
python3 gates/keyform.py --set <set.json> [--set <set.json> ...] [--list]
|
||||
|
||||
Prints "answer keys that are wording: X of N" over every key in the sets (either schema). Exit 0 = PASS (X = 0 and
|
||||
N > 0); 1 = FAIL or could not run.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
FUNCTION_WORDS = frozenset("""
|
||||
a an the
|
||||
i me my you your he him his she her it its we us our they them their this that these those who whom whose which what
|
||||
each every all any some
|
||||
and or but nor so if when while because than as whether then
|
||||
to of in on at by for from with without within into onto over under about after before between through beyond via
|
||||
per up down out off against during until
|
||||
is are was were be been being am can could may might will would shall should must do does did has have had
|
||||
isn't aren't wasn't weren't don't doesn't didn't can't cannot won't wouldn't shouldn't hasn't haven't
|
||||
not no never only also still just
|
||||
""".split())
|
||||
|
||||
|
||||
def is_wording(text: str) -> bool:
|
||||
"""True when the key has two or more words and a word outside backticks is a function word."""
|
||||
if len(text.split()) < 2:
|
||||
return False
|
||||
outside = re.sub(r"`[^`]*`", " ", text)
|
||||
words = (w.strip(".,;:!?()[]{}\"'").casefold() for w in outside.split())
|
||||
return any(w in FUNCTION_WORDS for w in words)
|
||||
|
||||
|
||||
@dataclass
|
||||
class KeyReport:
|
||||
keys: int = 0
|
||||
wording: int = 0
|
||||
listed: list[str] = field(default_factory=list)
|
||||
|
||||
|
||||
def count_keys(questions: list[dict[str, Any]]) -> KeyReport:
|
||||
rep = KeyReport()
|
||||
for q in questions:
|
||||
for k in q.get("answer_keys", []) if isinstance(q, dict) else []:
|
||||
text = k.get("text") if isinstance(k, dict) else None
|
||||
rep.keys += 1
|
||||
if isinstance(text, str) and is_wording(text):
|
||||
rep.wording += 1
|
||||
rep.listed.append(f"{q.get('id')}: {text}")
|
||||
return rep
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
p.add_argument("--set", action="append", required=True)
|
||||
p.add_argument("--list", action="store_true", help="list every wording key")
|
||||
args = p.parse_args(argv)
|
||||
total = KeyReport()
|
||||
for path in args.set:
|
||||
try:
|
||||
data = json.loads(Path(path).read_text(encoding="utf-8"))
|
||||
except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc:
|
||||
print(f"FAIL: cannot read set {path}: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
if not isinstance(data, dict) or not isinstance(data.get("questions"), list):
|
||||
print(f"FAIL: {path} is not a set", file=sys.stderr)
|
||||
return 1
|
||||
rep = count_keys(data["questions"])
|
||||
total.keys += rep.keys
|
||||
total.wording += rep.wording
|
||||
total.listed += [f"{Path(path).name} {line}" for line in rep.listed]
|
||||
if args.list:
|
||||
print("\n".join(total.listed))
|
||||
print(f"answer keys that are wording: {total.wording} of {total.keys}")
|
||||
passed = total.keys > 0 and total.wording == 0
|
||||
print("PASS" if passed else "FAIL")
|
||||
return 0 if passed else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Loading…
Add table
Add a link
Reference in a new issue