okf-wiki/gates/polarity.py
Kjell Tore Guttormsen 0fd86a23a3
okf-wiki 1.1.1
First public release, published as one snapshot. What changed between
versions is in CHANGELOG.md.

Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
2026-11-14 21:03:20 +01:00

170 lines
7.4 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

"""Polarity keys (E2d): counts the polarity keys that accept an answer of the opposite polarity. No model call.
A polarity key is an answer key whose fact is carried by a negation or a restriction in its source ("not synced
back" became the key `synced`). Which keys are polarity keys is given per set part as a list:
{"schema": "okf-wiki/polarity-keys/1", "keys": [{"id": "<question id>", "atoms": ["<key text>", ...]}]}
For each listed key the tool builds two answers mechanically from the key's own spans (the spans of the key's file;
in a whole-document set, every span). The answer from the span is the first sentence where the first atom has a
negation within NEGATION_WINDOW words before it, or else the first sentence that holds it. The opposite answer is
that sentence with its polarity flipped: the negations within the window are taken out ("isn't" -> "is", "can't"
-> "can", "without" -> "with", "no longer" and every other negation dropped), or, when there is none, "not" is put
in front of the atom. A key accepts an answer when the gates' own key_holds holds for every atom.
python3 gates/polarity.py --set <set.json> --keys <keys.json> [--set <set.json> --keys <keys.json> ...] [--list]
Prints "polarity keys that accept the answer from their span: Y of N" and "polarity keys that accept the opposite
answer: X of N". Exit 0 = PASS (N > 0, Y = N and X = 0); 1 = FAIL or could not run.
"""
from __future__ import annotations
import argparse
import sys
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any
from multidoc import (
NEGATION_WINDOW,
RunError,
as_list,
is_negation,
key_holds,
key_positions,
load_json,
negated_at,
normalise_answer,
sentences,
)
KEYS_SCHEMA = "okf-wiki/polarity-keys/1"
def unnegate(word: str) -> str:
"""The word without its negation: "isn't" -> "is", "can't" and "cannot" -> "can", "won't" -> "will",
"without" -> "with"; any other negation -> "" (dropped)."""
w = word.replace("’", "'")
special = {"cannot": "can", "can't": "can", "won't": "will", "shan't": "shall", "without": "with"}
if w in special:
return special[w]
return w[: -len("n't")] if w.endswith("n't") else ""
def flip(norm: str, pos: int) -> str:
"""The normalised sentence with the polarity of the atom at pos flipped (see the module text)."""
before = norm[:pos].split()
cut = max(0, len(before) - NEGATION_WINDOW)
head, window = before[:cut], before[cut:]
if not any(is_negation(w) for w in window):
return norm[:pos] + "not " + norm[pos:]
kept: list[str] = []
dropped_no = False
for w in window:
if dropped_no and w == "longer":
dropped_no = False
continue
dropped_no = False
if is_negation(w):
base = unnegate(w)
dropped_no = w == "no"
if base:
kept.append(base)
else:
kept.append(w)
return " ".join(head + kept + [norm[pos:]])
def answer_from_span(atom: str, spans: list[str]) -> tuple[str, int] | None:
"""(normalised sentence, atom offset): the first sentence with the atom negated in the window, else the first
sentence that holds the atom; None when no span holds it."""
found: list[tuple[str, int]] = []
for span in spans:
for s in sentences(span):
norm = normalise_answer(s)
found += [(norm, p) for p in key_positions(atom, norm)]
negated = [f for f in found if negated_at(*f)]
chosen = negated or found
return chosen[0] if chosen else None
@dataclass
class PolarityReport:
keys: int = 0
accept_span: int = 0
accept_opposite: int = 0
listed: list[str] = field(default_factory=list)
@property
def passed(self) -> bool:
return self.keys > 0 and self.accept_span == self.keys and self.accept_opposite == 0
def count_polarity(questions: list[dict[str, Any]], entries: list[Any], label: str = "") -> PolarityReport:
rep = PolarityReport()
by_id = {q.get("id"): q for q in questions if isinstance(q, dict)}
for e in entries:
qid, atoms = (e.get("id"), e.get("atoms")) if isinstance(e, dict) else (None, None)
q = by_id.get(qid)
if q is None or not isinstance(atoms, list) or not atoms:
raise RunError(f"{label}: listed key {e!r} names no question or no atoms")
keys: list[dict[str, Any]] = []
for atom in atoms:
match = [k for k in as_list(q.get("answer_keys")) if isinstance(k, dict) and k.get("text") == atom]
if len(match) != 1:
raise RunError(f"{label} {qid}: atom {atom!r} is not exactly one answer key of the question")
keys.append(match[0])
name = keys[0].get("file", q.get("file"))
spans = [ev["span"] for ev in as_list(q.get("evidence"))
if isinstance(ev, dict) and isinstance(ev.get("span"), str) and ev.get("file", q.get("file")) == name]
found = answer_from_span(atoms[0], spans)
if found is None:
raise RunError(f"{label} {qid}: atom {atoms[0]!r} stands in none of its spans")
norm, pos = found
opposite = flip(norm, pos)
on_span = all(key_holds(k, norm) for k in keys)
on_opposite = all(key_holds(k, opposite) for k in keys)
rep.keys += 1
rep.accept_span += on_span
rep.accept_opposite += on_opposite
rep.listed.append(f"{label} {qid} {atoms}: span {'accepted' if on_span else 'REFUSED'}, "
f"opposite {'ACCEPTED' if on_opposite else 'refused'} | {opposite}")
return rep
def main(argv: list[str] | None = None) -> int:
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
p.add_argument("--set", action="append", required=True)
p.add_argument("--keys", action="append", required=True, help="the polarity keys of the set given at that place")
p.add_argument("--list", action="store_true", help="list every key with its opposite answer")
args = p.parse_args(argv)
total = PolarityReport()
try:
if len(args.set) != len(args.keys):
raise RunError("give one --keys per --set")
for set_path, keys_path in zip(args.set, args.keys):
data = load_json(Path(set_path), "set")
listing = load_json(Path(keys_path), "polarity keys")
if not isinstance(data, dict) or not isinstance(data.get("questions"), list):
raise RunError(f"{set_path} is not a set")
if not isinstance(listing, dict) or listing.get("schema") != KEYS_SCHEMA:
raise RunError(f"{keys_path} is not a polarity key list with schema {KEYS_SCHEMA}")
rep = count_polarity(data["questions"], as_list(listing.get("keys")), Path(set_path).name)
total.keys += rep.keys
total.accept_span += rep.accept_span
total.accept_opposite += rep.accept_opposite
total.listed += rep.listed
except RunError as exc:
print(f"FAIL: {exc}", file=sys.stderr)
return 1
if args.list:
print("\n".join(total.listed))
print(f"polarity keys that accept the answer from their span: {total.accept_span} of {total.keys}")
print(f"polarity keys that accept the opposite answer: {total.accept_opposite} of {total.keys}")
print("PASS" if total.passed else "FAIL")
return 0 if total.passed else 1
if __name__ == "__main__":
sys.exit(main())