First public release, published as one snapshot. What changed between versions is in CHANGELOG.md. Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
492 lines
21 KiB
Python
492 lines
21 KiB
Python
"""The multi-document set for PLAN.md criterion 4: checker, held-out split and answer gate. No model call.
|
||
|
||
check every answer span stands verbatim in its source file, every question needs >= 4 distinct files,
|
||
and every file carries an answer key that stands inside that file's spans. The counts printed
|
||
here are the denominators; they come from the set, not from whoever wrote it.
|
||
split draws the held-out questions with a fixed seed, writes them OUTSIDE the repository, renumbers
|
||
both parts, and writes a manifest with sha256 only (nothing about the held-out questions).
|
||
gate scores one arm's answers: a question counts when the answer holds every answer key AND the
|
||
citations quote, verbatim, every file the answer key needs. A missing arm scores 0 (fail-closed).
|
||
|
||
Exit 0 = PASS / scored completely; 1 = FAIL or could not run. A check that cannot run is a FAIL.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import hashlib
|
||
import json
|
||
import random
|
||
import re
|
||
import sys
|
||
from dataclasses import dataclass, field
|
||
from pathlib import Path
|
||
from typing import Any
|
||
|
||
from keyform import count_keys
|
||
|
||
SET_SCHEMA = "okf-wiki/multidoc-set/1"
|
||
MANIFEST_SCHEMA = "okf-wiki/multidoc-manifest/1"
|
||
ANSWERS_SCHEMA_NOTE = '{"arm": "<name>", "answers": [{"id", "answer", "citations": [{"file", "quote"}]}]}'
|
||
MIN_FILES = 4
|
||
SPAN_MIN, SPAN_MAX = 20, 400
|
||
KEY_MIN, KEY_MAX = 3, 60
|
||
NUMBER_WORDS = ("zero one two three four five six seven eight nine ten eleven twelve thirteen fourteen fifteen "
|
||
"sixteen seventeen eighteen nineteen twenty").split()
|
||
NEGATION_WORDS = frozenset("not no never none neither nor without cannot".split())
|
||
NEGATION_WINDOW = 6
|
||
REPO_ROOT = Path(__file__).resolve().parents[1]
|
||
|
||
|
||
class RunError(Exception):
|
||
"""The command cannot run; reported as FAIL with exit 1."""
|
||
|
||
|
||
# ---------------------------------------------------------------------------------------------- helpers
|
||
|
||
|
||
def sha256_bytes(data: bytes) -> str:
|
||
return hashlib.sha256(data).hexdigest()
|
||
|
||
|
||
def tree_sha256(sources: Path) -> str:
|
||
"""One hash over the flat sources directory: sorted "name NUL sha256 LF" lines."""
|
||
lines = []
|
||
for p in sorted(sources.iterdir(), key=lambda p: p.name):
|
||
if p.is_file():
|
||
lines.append(f"{p.name}\0{sha256_bytes(p.read_bytes())}\n")
|
||
return sha256_bytes("".join(lines).encode("utf-8"))
|
||
|
||
|
||
def count_files(sources: Path) -> int:
|
||
return sum(1 for p in sources.iterdir() if p.is_file())
|
||
|
||
|
||
def normalise_answer(text: str) -> str:
|
||
"""Case-folded, without backticks and asterisks, whitespace collapsed. Used for answer keys only."""
|
||
return re.sub(r"\s+", " ", text.replace("`", "").replace("*", "")).strip().casefold()
|
||
|
||
|
||
def number_key(key: str) -> tuple[int, bool] | None:
|
||
"""(n, is_percentage) when the key is a whole number, a percentage or an English number word; otherwise None."""
|
||
text = normalise_answer(key)
|
||
m = re.fullmatch(r"(\d+) ?(%?)", text)
|
||
if m:
|
||
return int(m.group(1)), m.group(2) == "%"
|
||
return (NUMBER_WORDS.index(text), False) if text in NUMBER_WORDS else None
|
||
|
||
|
||
def key_positions(key: str, norm: str) -> list[int]:
|
||
"""Start offsets of the answer key in normalised text. A number key matches as a number: its digits or its
|
||
English word, never inside a longer number, version or identifier. Any other key matches as normalised text."""
|
||
num = number_key(key)
|
||
if num is None:
|
||
needle = normalise_answer(key)
|
||
return [m.start() for m in re.finditer(re.escape(needle), norm)] if needle else []
|
||
n, percent = num
|
||
if percent:
|
||
return [m.start() for m in re.finditer(rf"(?<![\w.]){n} ?(?:%|percent\b)", norm)]
|
||
forms = [str(n)] + ([NUMBER_WORDS[n]] if n < len(NUMBER_WORDS) else [])
|
||
return [m.start() for m in re.finditer(rf"(?<![\w.])(?:{'|'.join(forms)})(?![\w%]|\.\d)", norm)]
|
||
|
||
|
||
def key_in(key: str, text: str) -> bool:
|
||
"""True when the answer key stands in the text (see key_positions)."""
|
||
return bool(key_positions(key, normalise_answer(text)))
|
||
|
||
|
||
def is_negation(word: str) -> bool:
|
||
"""True when the word is a negation: one of NEGATION_WORDS or a word ending in n't."""
|
||
w = word.strip(".,;:!?()[]{}\"'").replace("’", "'").casefold()
|
||
return w in NEGATION_WORDS or w.endswith("n't")
|
||
|
||
|
||
def sentences(text: str) -> list[str]:
|
||
"""The text split at line breaks and at a full stop, question mark, exclamation mark or semicolon ending a word."""
|
||
return [s for s in re.split(r"\n|[.!?;](?=\s|$)", text) if s.strip()]
|
||
|
||
|
||
def negated_at(norm: str, pos: int) -> bool:
|
||
"""True when one of the NEGATION_WINDOW words before offset pos in a normalised sentence is a negation."""
|
||
return any(is_negation(w) for w in norm[:pos].split()[-NEGATION_WINDOW:])
|
||
|
||
|
||
def negated_in(key: str, text: str) -> bool:
|
||
"""True when, within one sentence of the text, the key stands with a negation at most NEGATION_WINDOW words
|
||
before it."""
|
||
for s in sentences(text):
|
||
norm = normalise_answer(s)
|
||
if any(negated_at(norm, pos) for pos in key_positions(key, norm)):
|
||
return True
|
||
return False
|
||
|
||
|
||
def key_holds(key: dict[str, Any], text: str) -> bool:
|
||
"""The gates' one test of an answer key against an answer. A key marked "negated": true holds only where it
|
||
stands negated (negated_in); any other key holds where it stands (key_in), with or without a negation."""
|
||
if key.get("negated") is True:
|
||
return negated_in(key["text"], text)
|
||
return key_in(key["text"], text)
|
||
|
||
|
||
def normalise_quote(text: str) -> str:
|
||
"""Whitespace collapsed, otherwise exact. Used for an arm's citations."""
|
||
return re.sub(r"\s+", " ", text).strip()
|
||
|
||
|
||
def as_list(value: object) -> list[Any]:
|
||
return value if isinstance(value, list) else []
|
||
|
||
|
||
def is_plain_name(name: object) -> bool:
|
||
return isinstance(name, str) and name != "" and "/" not in name and "\\" not in name and name not in (".", "..")
|
||
|
||
|
||
def load_json(path: Path, what: str) -> Any:
|
||
try:
|
||
return json.loads(path.read_text(encoding="utf-8"))
|
||
except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc:
|
||
raise RunError(f"cannot read {what} {path}: {exc}") from exc
|
||
|
||
|
||
def load_questions(path: Path) -> list[dict[str, Any]]:
|
||
data = load_json(path, "set")
|
||
if not isinstance(data, dict) or data.get("schema") != SET_SCHEMA or not isinstance(data.get("questions"), list):
|
||
raise RunError(f"{path} is not a set with schema {SET_SCHEMA}")
|
||
questions: list[dict[str, Any]] = data["questions"]
|
||
return questions
|
||
|
||
|
||
def require_dir(path: Path, what: str) -> None:
|
||
if not path.is_dir():
|
||
raise RunError(f"{what} {path} is not a directory")
|
||
|
||
|
||
class SourceCache:
|
||
def __init__(self, root: Path) -> None:
|
||
self.root = root
|
||
self._text: dict[str, str | None] = {}
|
||
|
||
def text(self, name: str) -> str | None:
|
||
"""File text, or None when the name is not a plain file directly in the sources directory."""
|
||
if not is_plain_name(name):
|
||
return None
|
||
if name not in self._text:
|
||
p = self.root / name
|
||
try:
|
||
self._text[name] = p.read_text(encoding="utf-8") if p.is_file() else None
|
||
except (OSError, UnicodeDecodeError):
|
||
self._text[name] = None
|
||
return self._text[name]
|
||
|
||
|
||
# ---------------------------------------------------------------------------------------------- check
|
||
|
||
|
||
@dataclass
|
||
class CheckReport:
|
||
questions: int = 0
|
||
spans: int = 0
|
||
spans_verbatim: int = 0
|
||
all_spans_verbatim: int = 0
|
||
min_files: int = 0
|
||
key_per_file: int = 0
|
||
keys: int = 0
|
||
wording_keys: int = 0
|
||
negated_keys: int = 0
|
||
negated_ok: int = 0
|
||
errors: list[str] = field(default_factory=list)
|
||
files_per_question: dict[str, int] = field(default_factory=dict)
|
||
|
||
@property
|
||
def passed(self) -> bool:
|
||
n = self.questions
|
||
return (
|
||
n > 0
|
||
and not self.errors
|
||
and self.spans_verbatim == self.spans
|
||
and self.all_spans_verbatim == n
|
||
and self.min_files == n
|
||
and self.key_per_file == n
|
||
and self.wording_keys == 0
|
||
and self.negated_ok == self.negated_keys
|
||
)
|
||
|
||
def lines(self) -> list[str]:
|
||
n = self.questions
|
||
out = [f"error: {e}" for e in self.errors]
|
||
out += [
|
||
f"questions: {n}",
|
||
f"spans verbatim: {self.spans_verbatim} of {self.spans}",
|
||
f"questions with every span verbatim: {self.all_spans_verbatim} of {n}",
|
||
f"questions with >= {MIN_FILES} distinct files: {self.min_files} of {n}",
|
||
f"questions with a key per file: {self.key_per_file} of {n}",
|
||
f"answer keys that are wording: {self.wording_keys} of {self.keys}",
|
||
f"negated keys negated in their span: {self.negated_ok} of {self.negated_keys}",
|
||
"PASS" if self.passed else "FAIL",
|
||
]
|
||
return out
|
||
|
||
|
||
def check_questions(questions: list[dict[str, Any]], sources: Path) -> CheckReport:
|
||
require_dir(sources, "sources")
|
||
cache = SourceCache(sources)
|
||
rep = CheckReport(questions=len(questions))
|
||
form = count_keys(questions)
|
||
rep.keys, rep.wording_keys = form.keys, form.wording
|
||
seen: set[str] = set()
|
||
for q in questions:
|
||
qid = q.get("id") if isinstance(q, dict) else None
|
||
if not isinstance(qid, str) or not qid:
|
||
rep.errors.append("question without an id")
|
||
continue
|
||
if qid in seen:
|
||
rep.errors.append(f"duplicate id {qid}")
|
||
seen.add(qid)
|
||
if not isinstance(q.get("question"), str) or not q["question"].strip():
|
||
rep.errors.append(f"{qid}: empty question")
|
||
evidence = as_list(q.get("evidence"))
|
||
keys = as_list(q.get("answer_keys"))
|
||
spans_by_file: dict[str, list[str]] = {}
|
||
verbatim = 0
|
||
for e in evidence:
|
||
name, span = (e.get("file"), e.get("span")) if isinstance(e, dict) else (None, None)
|
||
rep.spans += 1
|
||
if not isinstance(name, str) or not is_plain_name(name):
|
||
rep.errors.append(f"{qid}: evidence file {name!r} is not a plain file name")
|
||
continue
|
||
if not isinstance(span, str):
|
||
rep.errors.append(f"{qid}: evidence span in {name} is not text")
|
||
continue
|
||
if not SPAN_MIN <= len(span) <= SPAN_MAX:
|
||
rep.errors.append(f"{qid}: span length {len(span)} in {name} outside {SPAN_MIN}-{SPAN_MAX}")
|
||
text = cache.text(name)
|
||
if text is not None and span in text:
|
||
verbatim += 1
|
||
spans_by_file.setdefault(name, []).append(span)
|
||
rep.spans_verbatim += verbatim
|
||
if evidence and verbatim == len(evidence):
|
||
rep.all_spans_verbatim += 1
|
||
files = set(spans_by_file)
|
||
rep.files_per_question[qid] = len(files)
|
||
if len(files) >= MIN_FILES:
|
||
rep.min_files += 1
|
||
keyed: set[str] = set()
|
||
keys_ok = bool(keys)
|
||
for k in keys:
|
||
name, key = (k.get("file"), k.get("text")) if isinstance(k, dict) else (None, None)
|
||
if not isinstance(key, str) or not isinstance(name, str):
|
||
keys_ok = False
|
||
rep.errors.append(f"{qid}: malformed answer key")
|
||
continue
|
||
if number_key(key) is None and not KEY_MIN <= len(key.strip()) <= KEY_MAX:
|
||
rep.errors.append(f"{qid}: key length {len(key.strip())} outside {KEY_MIN}-{KEY_MAX}")
|
||
if any(key_in(key, s) for s in spans_by_file.get(name, [])):
|
||
keyed.add(name)
|
||
else:
|
||
keys_ok = False
|
||
if "negated" in k:
|
||
if k["negated"] is not True:
|
||
rep.errors.append(f"{qid}: negated must be true or absent")
|
||
continue
|
||
rep.negated_keys += 1
|
||
rep.negated_ok += any(negated_in(key, s) for s in spans_by_file.get(name, []))
|
||
if keys_ok and files and keyed == files:
|
||
rep.key_per_file += 1
|
||
return rep
|
||
|
||
|
||
def cmd_check(args: argparse.Namespace) -> int:
|
||
rep = check_questions(load_questions(Path(args.set)), Path(args.sources))
|
||
if args.per_question:
|
||
for qid, n in rep.files_per_question.items():
|
||
print(f"{qid}\t{n} files")
|
||
print("\n".join(rep.lines()))
|
||
return 0 if rep.passed else 1
|
||
|
||
|
||
# ---------------------------------------------------------------------------------------------- split
|
||
|
||
|
||
def inside_repo(path: Path) -> bool:
|
||
try:
|
||
path.resolve().relative_to(REPO_ROOT)
|
||
return True
|
||
except ValueError:
|
||
return False
|
||
|
||
|
||
def dump_set(questions: list[dict[str, Any]]) -> bytes:
|
||
return (json.dumps({"schema": SET_SCHEMA, "questions": questions}, ensure_ascii=False, indent=1) + "\n").encode("utf-8")
|
||
|
||
|
||
def cmd_split(args: argparse.Namespace) -> int:
|
||
sources, held_path = Path(args.sources), Path(args.held_out)
|
||
dev_path, manifest_path = Path(args.dev_out), Path(args.manifest)
|
||
if inside_repo(held_path):
|
||
raise RunError(f"held-out path {held_path} is inside the repository; it must live outside it")
|
||
if held_path.exists():
|
||
raise RunError(f"held-out file {held_path} already exists; a frozen draw is never redrawn")
|
||
questions = load_questions(Path(args.set))
|
||
rep = check_questions(questions, sources)
|
||
if not rep.passed:
|
||
print("\n".join(rep.lines()))
|
||
raise RunError("the set fails the check; nothing was split")
|
||
n_held = args.held_out_count
|
||
if not 0 < n_held < len(questions):
|
||
raise RunError(f"cannot hold out {n_held} of {len(questions)}")
|
||
ids = sorted(q["id"] for q in questions)
|
||
held_ids = set(random.Random(args.seed).sample(ids, n_held))
|
||
by_id = {q["id"]: q for q in questions}
|
||
dev = [dict(by_id[i], id=f"d{n:02d}") for n, i in enumerate((i for i in ids if i not in held_ids), 1)]
|
||
held = [dict(by_id[i], id=f"h{n:02d}") for n, i in enumerate(sorted(held_ids), 1)]
|
||
dev_bytes, held_bytes = dump_set(dev), dump_set(held)
|
||
held_path.parent.mkdir(parents=True, exist_ok=True)
|
||
held_path.write_bytes(held_bytes)
|
||
dev_path.parent.mkdir(parents=True, exist_ok=True)
|
||
dev_path.write_bytes(dev_bytes)
|
||
manifest = {
|
||
"schema": MANIFEST_SCHEMA,
|
||
"seed": args.seed,
|
||
"draw": "random.Random(seed).sample(sorted(ids), held_out.questions); both parts renumbered d../h..",
|
||
"dev": {"file": args.dev_label or dev_path.name, "sha256": sha256_bytes(dev_bytes), "questions": len(dev)},
|
||
"held_out": {"sha256": sha256_bytes(held_bytes), "questions": len(held)},
|
||
"sources": {"files": count_files(sources), "tree_sha256": tree_sha256(sources)},
|
||
}
|
||
manifest_path.parent.mkdir(parents=True, exist_ok=True)
|
||
manifest_path.write_text(json.dumps(manifest, indent=1) + "\n", encoding="utf-8")
|
||
print(f"seed: {args.seed}")
|
||
print(f"dev: {len(dev)} questions, sha256 {manifest['dev']['sha256']}")
|
||
print(f"held out: {len(held)} questions, sha256 {manifest['held_out']['sha256']}")
|
||
print(f"manifest: {manifest_path}")
|
||
return 0
|
||
|
||
|
||
# ---------------------------------------------------------------------------------------------- gate
|
||
|
||
|
||
@dataclass
|
||
class GateResult:
|
||
qid: str
|
||
answered: bool
|
||
answer_ok: bool
|
||
source_ok: bool
|
||
|
||
@property
|
||
def correct(self) -> bool:
|
||
return self.answer_ok and self.source_ok
|
||
|
||
|
||
def score_one(q: dict[str, Any], a: dict[str, Any] | None, cache: SourceCache) -> GateResult:
|
||
if a is None:
|
||
return GateResult(q["id"], False, False, False)
|
||
answer = a.get("answer", "") if isinstance(a.get("answer"), str) else ""
|
||
answer_ok = all(key_holds(k, answer) for k in q["answer_keys"])
|
||
supported: set[str] = set()
|
||
citations = as_list(a.get("citations"))
|
||
for c in citations:
|
||
name, quote = (c.get("file"), c.get("quote")) if isinstance(c, dict) else (None, None)
|
||
if not isinstance(name, str) or not isinstance(quote, str) or not quote.strip():
|
||
continue
|
||
text = cache.text(name)
|
||
if text is not None and normalise_quote(quote) in normalise_quote(text):
|
||
supported.add(name)
|
||
needed = {e["file"] for e in q["evidence"]}
|
||
return GateResult(q["id"], True, answer_ok, needed <= supported)
|
||
|
||
|
||
def cmd_gate(args: argparse.Namespace) -> int:
|
||
set_path, sources, manifest_path = Path(args.set), Path(args.sources), Path(args.manifest)
|
||
require_dir(sources, "sources")
|
||
manifest = load_json(manifest_path, "manifest")
|
||
if not isinstance(manifest, dict) or manifest.get("schema") != MANIFEST_SCHEMA:
|
||
raise RunError(f"{manifest_path} is not a manifest with schema {MANIFEST_SCHEMA}")
|
||
set_hash = sha256_bytes(set_path.read_bytes()) if set_path.is_file() else ""
|
||
known = {manifest.get("dev", {}).get("sha256"): "dev", manifest.get("held_out", {}).get("sha256"): "held_out"}
|
||
part = known.get(set_hash)
|
||
if part is None:
|
||
raise RunError(f"set {set_path} sha256 {set_hash or '(missing)'} matches neither part of the manifest")
|
||
if tree_sha256(sources) != manifest.get("sources", {}).get("tree_sha256"):
|
||
raise RunError(f"sources {sources} differ from the manifest's tree_sha256; the set is frozen against them")
|
||
questions = load_questions(set_path)
|
||
n = len(questions)
|
||
cache = SourceCache(sources)
|
||
answers: dict[str, dict[str, Any]] = {}
|
||
fail_closed = ""
|
||
answers_path = Path(args.answers)
|
||
if not answers_path.is_file():
|
||
fail_closed = f"no answers for arm {args.arm} at {answers_path}"
|
||
else:
|
||
data = load_json(answers_path, "answers")
|
||
if not isinstance(data, dict) or not isinstance(data.get("answers"), list):
|
||
raise RunError(f"{answers_path} is not an answers file {ANSWERS_SCHEMA_NOTE}")
|
||
if data.get("arm") != args.arm:
|
||
raise RunError(f"answers file is for arm {data.get('arm')!r}, not {args.arm!r}")
|
||
for a in data["answers"]:
|
||
if isinstance(a, dict) and isinstance(a.get("id"), str):
|
||
answers[a["id"]] = a
|
||
results = [score_one(q, answers.get(q["id"]), cache) for q in questions]
|
||
missing = sum(1 for r in results if not r.answered)
|
||
print(f"set: {part}, {n} questions; arm: {args.arm}")
|
||
if fail_closed:
|
||
print(f"fail-closed: {fail_closed}")
|
||
print(f"missing answers: {missing}")
|
||
print(f"correct answer: {sum(r.answer_ok for r in results)} of {n}")
|
||
print(f"correct source: {sum(r.source_ok for r in results)} of {n}")
|
||
print(f"correct answer and source: {sum(r.correct for r in results)} of {n}")
|
||
if args.out:
|
||
out = {
|
||
"arm": args.arm,
|
||
"set_sha256": set_hash,
|
||
"results": [
|
||
{"id": r.qid, "answered": r.answered, "answer_ok": r.answer_ok, "source_ok": r.source_ok, "correct": r.correct}
|
||
for r in results
|
||
],
|
||
}
|
||
Path(args.out).write_text(json.dumps(out, indent=1) + "\n", encoding="utf-8")
|
||
scored = not fail_closed and missing == 0
|
||
print("SCORED" if scored else "UNSCORED (fail-closed)" if fail_closed else "INCOMPLETE")
|
||
return 0 if scored else 1
|
||
|
||
|
||
# ---------------------------------------------------------------------------------------------- main
|
||
|
||
|
||
def build_parser() -> argparse.ArgumentParser:
|
||
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||
sub = p.add_subparsers(dest="command", required=True)
|
||
c = sub.add_parser("check", help="check spans, file counts and answer keys")
|
||
c.add_argument("--set", required=True)
|
||
c.add_argument("--sources", required=True)
|
||
c.add_argument("--per-question", action="store_true", help="also list distinct files per question")
|
||
s = sub.add_parser("split", help="draw the held-out questions with a fixed seed")
|
||
s.add_argument("--set", required=True)
|
||
s.add_argument("--sources", required=True)
|
||
s.add_argument("--seed", type=int, required=True)
|
||
s.add_argument("--held-out-count", type=int, default=20)
|
||
s.add_argument("--dev-out", required=True)
|
||
s.add_argument("--dev-label", default="", help="path of the dev set as written in the manifest")
|
||
s.add_argument("--held-out", required=True, help="file path OUTSIDE the repository")
|
||
s.add_argument("--manifest", required=True)
|
||
g = sub.add_parser("gate", help="score one arm's answers against the answer key")
|
||
g.add_argument("--set", required=True)
|
||
g.add_argument("--manifest", required=True)
|
||
g.add_argument("--sources", required=True)
|
||
g.add_argument("--arm", required=True)
|
||
g.add_argument("--answers", required=True, help=ANSWERS_SCHEMA_NOTE)
|
||
g.add_argument("--out", default="", help="write per-question results (for paired tests)")
|
||
return p
|
||
|
||
|
||
def main(argv: list[str] | None = None) -> int:
|
||
args = build_parser().parse_args(argv)
|
||
commands = {"check": cmd_check, "split": cmd_split, "gate": cmd_gate}
|
||
try:
|
||
return commands[args.command](args)
|
||
except RunError as exc:
|
||
print(f"FAIL: {exc}", file=sys.stderr)
|
||
return 1
|
||
|
||
|
||
if __name__ == "__main__":
|
||
sys.exit(main())
|