okf-wiki 1.1.1
First public release, published as one snapshot. What changed between versions is in CHANGELOG.md. Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
This commit is contained in:
commit
0fd86a23a3
227 changed files with 96077 additions and 0 deletions
26
gates/stubs/fasit_reader.py
Normal file
26
gates/stubs/fasit_reader.py
Normal file
|
|
@ -0,0 +1,26 @@
|
|||
"""Known-positive reader for the file-type gate: answers every query from the frozen pairs file (the fasit itself).
|
||||
|
||||
It reads no source file. A GREEN gate with it shows that the pairs, the manifest and the gate's scoring agree;
|
||||
it says nothing about any real reader.
|
||||
|
||||
Usage: python3 gates/stubs/fasit_reader.py {file} {queries} <pairs.json>
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def main() -> int:
|
||||
queries = json.loads(Path(sys.argv[2]).read_text(encoding="utf-8"))
|
||||
key = {p["id"]: p for p in json.loads(Path(sys.argv[3]).read_text(encoding="utf-8"))["pairs"]}
|
||||
answers = [{"id": q["id"], "found": [{"locator": key[q["id"]]["locator"], "hidden": key[q["id"]]["hidden"]}]}
|
||||
for q in queries["queries"] if q["id"] in key]
|
||||
print(json.dumps({"answers": answers}))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
50
gates/stubs/first_locator.py
Normal file
50
gates/stubs/first_locator.py
Normal file
|
|
@ -0,0 +1,50 @@
|
|||
"""Reader stub for the file-type gate: always answers the first place in the file, and never hidden.
|
||||
|
||||
First page (pdf), first paragraph (docx), cell A1 on the first sheet in workbook order (xlsx), first slide outside
|
||||
the notes (pptx), first line (md, txt, html). It shows how much of the gate a reader passes without reading.
|
||||
|
||||
Usage: python3 gates/stubs/first_locator.py {file} {queries}
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from xml.etree import ElementTree
|
||||
|
||||
MAIN_NS = "{http://schemas.openxmlformats.org/spreadsheetml/2006/main}"
|
||||
|
||||
|
||||
def first_sheet(path: Path) -> str:
|
||||
with zipfile.ZipFile(path) as z:
|
||||
root = ElementTree.fromstring(z.read("xl/workbook.xml"))
|
||||
sheet = root.find(f"{MAIN_NS}sheets/{MAIN_NS}sheet")
|
||||
return "" if sheet is None else sheet.get("name", "")
|
||||
|
||||
|
||||
def first_locator(path: Path) -> dict[str, Any]:
|
||||
ext = path.suffix.lower()
|
||||
if ext == ".pdf":
|
||||
return {"page": 1}
|
||||
if ext == ".docx":
|
||||
return {"paragraph": 1}
|
||||
if ext == ".xlsx":
|
||||
return {"sheet": first_sheet(path), "cell": "A1"}
|
||||
if ext == ".pptx":
|
||||
return {"slide": 1, "notes": False}
|
||||
return {"line": 1}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
path, queries = Path(sys.argv[1]), json.loads(Path(sys.argv[2]).read_text(encoding="utf-8"))
|
||||
loc = first_locator(path)
|
||||
answers = [{"id": q["id"], "found": [{"locator": loc, "hidden": False}]} for q in queries["queries"]]
|
||||
print(json.dumps({"answers": answers}))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
9
gates/stubs/reject_all.py
Normal file
9
gates/stubs/reject_all.py
Normal file
|
|
@ -0,0 +1,9 @@
|
|||
"""Trivial auditor stub that rejects every bundle. It shows the mutant gate cannot be passed by rejecting everything:
|
||||
it catches every mutant and raises a false alarm on the known-positive.
|
||||
"""
|
||||
|
||||
import json
|
||||
import sys
|
||||
|
||||
print(json.dumps({"findings": [{"check": "reject-all"}]}))
|
||||
sys.exit(1)
|
||||
52
gates/stubs/wholedoc_arm.py
Normal file
52
gates/stubs/wholedoc_arm.py
Normal file
|
|
@ -0,0 +1,52 @@
|
|||
"""Two answer arms for the whole-document gate (E2b), both written from the set itself. Neither reads a source file.
|
||||
|
||||
one-span answers every question from its first evidence span alone (a count question answers 1). The check
|
||||
guarantees that no question can be answered from one span, so this arm must score 0 of N: RED.
|
||||
fasit the known-positive: every answer key, the exact count, and every span as a citation. A GREEN gate
|
||||
with it shows that the set, the manifest and the gate's scoring agree; it says nothing about any arm.
|
||||
|
||||
Usage: python3 gates/stubs/wholedoc_arm.py <one-span|fasit> <set.json> <answers-out.json>
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
def one_span_answer(q: dict[str, Any]) -> dict[str, Any]:
|
||||
span = q["evidence"][0]["span"]
|
||||
answer: dict[str, Any] = {"id": q["id"], "answer": span, "citations": [{"file": q["file"], "quote": span}]}
|
||||
if q["type"] == "count":
|
||||
answer["count"] = 1
|
||||
return answer
|
||||
|
||||
|
||||
def fasit_answer(q: dict[str, Any]) -> dict[str, Any]:
|
||||
citations = [{"file": q["file"], "quote": e["span"]} for e in q["evidence"]]
|
||||
if q["type"] == "count":
|
||||
return {"id": q["id"], "answer": str(q["count"]), "count": q["count"], "citations": citations}
|
||||
keys = [("not " if k.get("negated") is True else "") + k["text"] for k in q["answer_keys"]]
|
||||
return {"id": q["id"], "answer": "; ".join(keys), "citations": citations}
|
||||
|
||||
|
||||
ARMS = {"one-span": one_span_answer, "fasit": fasit_answer}
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
args = sys.argv[1:] if argv is None else argv
|
||||
if len(args) != 3 or args[0] not in ARMS:
|
||||
print(__doc__, file=sys.stderr)
|
||||
return 1
|
||||
arm, set_path, out = args
|
||||
questions = json.loads(Path(set_path).read_text(encoding="utf-8"))["questions"]
|
||||
answers = [ARMS[arm](q) for q in questions]
|
||||
Path(out).write_text(json.dumps({"arm": arm, "answers": answers}, ensure_ascii=False, indent=1) + "\n",
|
||||
encoding="utf-8")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Loading…
Add table
Add a link
Reference in a new issue