First public release, published as one snapshot. What changed between versions is in CHANGELOG.md. Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
52 lines
2.1 KiB
Python
52 lines
2.1 KiB
Python
"""Two answer arms for the whole-document gate (E2b), both written from the set itself. Neither reads a source file.
|
|
|
|
one-span answers every question from its first evidence span alone (a count question answers 1). The check
|
|
guarantees that no question can be answered from one span, so this arm must score 0 of N: RED.
|
|
fasit the known-positive: every answer key, the exact count, and every span as a citation. A GREEN gate
|
|
with it shows that the set, the manifest and the gate's scoring agree; it says nothing about any arm.
|
|
|
|
Usage: python3 gates/stubs/wholedoc_arm.py <one-span|fasit> <set.json> <answers-out.json>
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import sys
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
|
|
def one_span_answer(q: dict[str, Any]) -> dict[str, Any]:
|
|
span = q["evidence"][0]["span"]
|
|
answer: dict[str, Any] = {"id": q["id"], "answer": span, "citations": [{"file": q["file"], "quote": span}]}
|
|
if q["type"] == "count":
|
|
answer["count"] = 1
|
|
return answer
|
|
|
|
|
|
def fasit_answer(q: dict[str, Any]) -> dict[str, Any]:
|
|
citations = [{"file": q["file"], "quote": e["span"]} for e in q["evidence"]]
|
|
if q["type"] == "count":
|
|
return {"id": q["id"], "answer": str(q["count"]), "count": q["count"], "citations": citations}
|
|
keys = [("not " if k.get("negated") is True else "") + k["text"] for k in q["answer_keys"]]
|
|
return {"id": q["id"], "answer": "; ".join(keys), "citations": citations}
|
|
|
|
|
|
ARMS = {"one-span": one_span_answer, "fasit": fasit_answer}
|
|
|
|
|
|
def main(argv: list[str] | None = None) -> int:
|
|
args = sys.argv[1:] if argv is None else argv
|
|
if len(args) != 3 or args[0] not in ARMS:
|
|
print(__doc__, file=sys.stderr)
|
|
return 1
|
|
arm, set_path, out = args
|
|
questions = json.loads(Path(set_path).read_text(encoding="utf-8"))["questions"]
|
|
answers = [ARMS[arm](q) for q in questions]
|
|
Path(out).write_text(json.dumps({"arm": arm, "answers": answers}, ensure_ascii=False, indent=1) + "\n",
|
|
encoding="utf-8")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|