First public release, published as one snapshot. What changed between versions is in CHANGELOG.md. Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
170 lines
7.4 KiB
Python
170 lines
7.4 KiB
Python
"""Polarity keys (E2d): counts the polarity keys that accept an answer of the opposite polarity. No model call.
|
||
|
||
A polarity key is an answer key whose fact is carried by a negation or a restriction in its source ("not synced
|
||
back" became the key `synced`). Which keys are polarity keys is given per set part as a list:
|
||
|
||
{"schema": "okf-wiki/polarity-keys/1", "keys": [{"id": "<question id>", "atoms": ["<key text>", ...]}]}
|
||
|
||
For each listed key the tool builds two answers mechanically from the key's own spans (the spans of the key's file;
|
||
in a whole-document set, every span). The answer from the span is the first sentence where the first atom has a
|
||
negation within NEGATION_WINDOW words before it, or else the first sentence that holds it. The opposite answer is
|
||
that sentence with its polarity flipped: the negations within the window are taken out ("isn't" -> "is", "can't"
|
||
-> "can", "without" -> "with", "no longer" and every other negation dropped), or, when there is none, "not" is put
|
||
in front of the atom. A key accepts an answer when the gates' own key_holds holds for every atom.
|
||
|
||
python3 gates/polarity.py --set <set.json> --keys <keys.json> [--set <set.json> --keys <keys.json> ...] [--list]
|
||
|
||
Prints "polarity keys that accept the answer from their span: Y of N" and "polarity keys that accept the opposite
|
||
answer: X of N". Exit 0 = PASS (N > 0, Y = N and X = 0); 1 = FAIL or could not run.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import sys
|
||
from dataclasses import dataclass, field
|
||
from pathlib import Path
|
||
from typing import Any
|
||
|
||
from multidoc import (
|
||
NEGATION_WINDOW,
|
||
RunError,
|
||
as_list,
|
||
is_negation,
|
||
key_holds,
|
||
key_positions,
|
||
load_json,
|
||
negated_at,
|
||
normalise_answer,
|
||
sentences,
|
||
)
|
||
|
||
KEYS_SCHEMA = "okf-wiki/polarity-keys/1"
|
||
|
||
|
||
def unnegate(word: str) -> str:
|
||
"""The word without its negation: "isn't" -> "is", "can't" and "cannot" -> "can", "won't" -> "will",
|
||
"without" -> "with"; any other negation -> "" (dropped)."""
|
||
w = word.replace("’", "'")
|
||
special = {"cannot": "can", "can't": "can", "won't": "will", "shan't": "shall", "without": "with"}
|
||
if w in special:
|
||
return special[w]
|
||
return w[: -len("n't")] if w.endswith("n't") else ""
|
||
|
||
|
||
def flip(norm: str, pos: int) -> str:
|
||
"""The normalised sentence with the polarity of the atom at pos flipped (see the module text)."""
|
||
before = norm[:pos].split()
|
||
cut = max(0, len(before) - NEGATION_WINDOW)
|
||
head, window = before[:cut], before[cut:]
|
||
if not any(is_negation(w) for w in window):
|
||
return norm[:pos] + "not " + norm[pos:]
|
||
kept: list[str] = []
|
||
dropped_no = False
|
||
for w in window:
|
||
if dropped_no and w == "longer":
|
||
dropped_no = False
|
||
continue
|
||
dropped_no = False
|
||
if is_negation(w):
|
||
base = unnegate(w)
|
||
dropped_no = w == "no"
|
||
if base:
|
||
kept.append(base)
|
||
else:
|
||
kept.append(w)
|
||
return " ".join(head + kept + [norm[pos:]])
|
||
|
||
|
||
def answer_from_span(atom: str, spans: list[str]) -> tuple[str, int] | None:
|
||
"""(normalised sentence, atom offset): the first sentence with the atom negated in the window, else the first
|
||
sentence that holds the atom; None when no span holds it."""
|
||
found: list[tuple[str, int]] = []
|
||
for span in spans:
|
||
for s in sentences(span):
|
||
norm = normalise_answer(s)
|
||
found += [(norm, p) for p in key_positions(atom, norm)]
|
||
negated = [f for f in found if negated_at(*f)]
|
||
chosen = negated or found
|
||
return chosen[0] if chosen else None
|
||
|
||
|
||
@dataclass
|
||
class PolarityReport:
|
||
keys: int = 0
|
||
accept_span: int = 0
|
||
accept_opposite: int = 0
|
||
listed: list[str] = field(default_factory=list)
|
||
|
||
@property
|
||
def passed(self) -> bool:
|
||
return self.keys > 0 and self.accept_span == self.keys and self.accept_opposite == 0
|
||
|
||
|
||
def count_polarity(questions: list[dict[str, Any]], entries: list[Any], label: str = "") -> PolarityReport:
|
||
rep = PolarityReport()
|
||
by_id = {q.get("id"): q for q in questions if isinstance(q, dict)}
|
||
for e in entries:
|
||
qid, atoms = (e.get("id"), e.get("atoms")) if isinstance(e, dict) else (None, None)
|
||
q = by_id.get(qid)
|
||
if q is None or not isinstance(atoms, list) or not atoms:
|
||
raise RunError(f"{label}: listed key {e!r} names no question or no atoms")
|
||
keys: list[dict[str, Any]] = []
|
||
for atom in atoms:
|
||
match = [k for k in as_list(q.get("answer_keys")) if isinstance(k, dict) and k.get("text") == atom]
|
||
if len(match) != 1:
|
||
raise RunError(f"{label} {qid}: atom {atom!r} is not exactly one answer key of the question")
|
||
keys.append(match[0])
|
||
name = keys[0].get("file", q.get("file"))
|
||
spans = [ev["span"] for ev in as_list(q.get("evidence"))
|
||
if isinstance(ev, dict) and isinstance(ev.get("span"), str) and ev.get("file", q.get("file")) == name]
|
||
found = answer_from_span(atoms[0], spans)
|
||
if found is None:
|
||
raise RunError(f"{label} {qid}: atom {atoms[0]!r} stands in none of its spans")
|
||
norm, pos = found
|
||
opposite = flip(norm, pos)
|
||
on_span = all(key_holds(k, norm) for k in keys)
|
||
on_opposite = all(key_holds(k, opposite) for k in keys)
|
||
rep.keys += 1
|
||
rep.accept_span += on_span
|
||
rep.accept_opposite += on_opposite
|
||
rep.listed.append(f"{label} {qid} {atoms}: span {'accepted' if on_span else 'REFUSED'}, "
|
||
f"opposite {'ACCEPTED' if on_opposite else 'refused'} | {opposite}")
|
||
return rep
|
||
|
||
|
||
def main(argv: list[str] | None = None) -> int:
|
||
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||
p.add_argument("--set", action="append", required=True)
|
||
p.add_argument("--keys", action="append", required=True, help="the polarity keys of the set given at that place")
|
||
p.add_argument("--list", action="store_true", help="list every key with its opposite answer")
|
||
args = p.parse_args(argv)
|
||
total = PolarityReport()
|
||
try:
|
||
if len(args.set) != len(args.keys):
|
||
raise RunError("give one --keys per --set")
|
||
for set_path, keys_path in zip(args.set, args.keys):
|
||
data = load_json(Path(set_path), "set")
|
||
listing = load_json(Path(keys_path), "polarity keys")
|
||
if not isinstance(data, dict) or not isinstance(data.get("questions"), list):
|
||
raise RunError(f"{set_path} is not a set")
|
||
if not isinstance(listing, dict) or listing.get("schema") != KEYS_SCHEMA:
|
||
raise RunError(f"{keys_path} is not a polarity key list with schema {KEYS_SCHEMA}")
|
||
rep = count_polarity(data["questions"], as_list(listing.get("keys")), Path(set_path).name)
|
||
total.keys += rep.keys
|
||
total.accept_span += rep.accept_span
|
||
total.accept_opposite += rep.accept_opposite
|
||
total.listed += rep.listed
|
||
except RunError as exc:
|
||
print(f"FAIL: {exc}", file=sys.stderr)
|
||
return 1
|
||
if args.list:
|
||
print("\n".join(total.listed))
|
||
print(f"polarity keys that accept the answer from their span: {total.accept_span} of {total.keys}")
|
||
print(f"polarity keys that accept the opposite answer: {total.accept_opposite} of {total.keys}")
|
||
print("PASS" if total.passed else "FAIL")
|
||
return 0 if total.passed else 1
|
||
|
||
|
||
if __name__ == "__main__":
|
||
sys.exit(main())
|