"""The multi-document set for PLAN.md criterion 4: checker, held-out split and answer gate. No model call. check every answer span stands verbatim in its source file, every question needs >= 4 distinct files, and every file carries an answer key that stands inside that file's spans. The counts printed here are the denominators; they come from the set, not from whoever wrote it. split draws the held-out questions with a fixed seed, writes them OUTSIDE the repository, renumbers both parts, and writes a manifest with sha256 only (nothing about the held-out questions). gate scores one arm's answers: a question counts when the answer holds every answer key AND the citations quote, verbatim, every file the answer key needs. A missing arm scores 0 (fail-closed). Exit 0 = PASS / scored completely; 1 = FAIL or could not run. A check that cannot run is a FAIL. """ from __future__ import annotations import argparse import hashlib import json import random import re import sys from dataclasses import dataclass, field from pathlib import Path from typing import Any from keyform import count_keys SET_SCHEMA = "okf-wiki/multidoc-set/1" MANIFEST_SCHEMA = "okf-wiki/multidoc-manifest/1" ANSWERS_SCHEMA_NOTE = '{"arm": "", "answers": [{"id", "answer", "citations": [{"file", "quote"}]}]}' MIN_FILES = 4 SPAN_MIN, SPAN_MAX = 20, 400 KEY_MIN, KEY_MAX = 3, 60 NUMBER_WORDS = ("zero one two three four five six seven eight nine ten eleven twelve thirteen fourteen fifteen " "sixteen seventeen eighteen nineteen twenty").split() NEGATION_WORDS = frozenset("not no never none neither nor without cannot".split()) NEGATION_WINDOW = 6 REPO_ROOT = Path(__file__).resolve().parents[1] class RunError(Exception): """The command cannot run; reported as FAIL with exit 1.""" # ---------------------------------------------------------------------------------------------- helpers def sha256_bytes(data: bytes) -> str: return hashlib.sha256(data).hexdigest() def tree_sha256(sources: Path) -> str: """One hash over the flat sources directory: sorted "name NUL sha256 LF" lines.""" lines = [] for p in sorted(sources.iterdir(), key=lambda p: p.name): if p.is_file(): lines.append(f"{p.name}\0{sha256_bytes(p.read_bytes())}\n") return sha256_bytes("".join(lines).encode("utf-8")) def count_files(sources: Path) -> int: return sum(1 for p in sources.iterdir() if p.is_file()) def normalise_answer(text: str) -> str: """Case-folded, without backticks and asterisks, whitespace collapsed. Used for answer keys only.""" return re.sub(r"\s+", " ", text.replace("`", "").replace("*", "")).strip().casefold() def number_key(key: str) -> tuple[int, bool] | None: """(n, is_percentage) when the key is a whole number, a percentage or an English number word; otherwise None.""" text = normalise_answer(key) m = re.fullmatch(r"(\d+) ?(%?)", text) if m: return int(m.group(1)), m.group(2) == "%" return (NUMBER_WORDS.index(text), False) if text in NUMBER_WORDS else None def key_positions(key: str, norm: str) -> list[int]: """Start offsets of the answer key in normalised text. A number key matches as a number: its digits or its English word, never inside a longer number, version or identifier. Any other key matches as normalised text.""" num = number_key(key) if num is None: needle = normalise_answer(key) return [m.start() for m in re.finditer(re.escape(needle), norm)] if needle else [] n, percent = num if percent: return [m.start() for m in re.finditer(rf"(? bool: """True when the answer key stands in the text (see key_positions).""" return bool(key_positions(key, normalise_answer(text))) def is_negation(word: str) -> bool: """True when the word is a negation: one of NEGATION_WORDS or a word ending in n't.""" w = word.strip(".,;:!?()[]{}\"'").replace("’", "'").casefold() return w in NEGATION_WORDS or w.endswith("n't") def sentences(text: str) -> list[str]: """The text split at line breaks and at a full stop, question mark, exclamation mark or semicolon ending a word.""" return [s for s in re.split(r"\n|[.!?;](?=\s|$)", text) if s.strip()] def negated_at(norm: str, pos: int) -> bool: """True when one of the NEGATION_WINDOW words before offset pos in a normalised sentence is a negation.""" return any(is_negation(w) for w in norm[:pos].split()[-NEGATION_WINDOW:]) def negated_in(key: str, text: str) -> bool: """True when, within one sentence of the text, the key stands with a negation at most NEGATION_WINDOW words before it.""" for s in sentences(text): norm = normalise_answer(s) if any(negated_at(norm, pos) for pos in key_positions(key, norm)): return True return False def key_holds(key: dict[str, Any], text: str) -> bool: """The gates' one test of an answer key against an answer. A key marked "negated": true holds only where it stands negated (negated_in); any other key holds where it stands (key_in), with or without a negation.""" if key.get("negated") is True: return negated_in(key["text"], text) return key_in(key["text"], text) def normalise_quote(text: str) -> str: """Whitespace collapsed, otherwise exact. Used for an arm's citations.""" return re.sub(r"\s+", " ", text).strip() def as_list(value: object) -> list[Any]: return value if isinstance(value, list) else [] def is_plain_name(name: object) -> bool: return isinstance(name, str) and name != "" and "/" not in name and "\\" not in name and name not in (".", "..") def load_json(path: Path, what: str) -> Any: try: return json.loads(path.read_text(encoding="utf-8")) except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc: raise RunError(f"cannot read {what} {path}: {exc}") from exc def load_questions(path: Path) -> list[dict[str, Any]]: data = load_json(path, "set") if not isinstance(data, dict) or data.get("schema") != SET_SCHEMA or not isinstance(data.get("questions"), list): raise RunError(f"{path} is not a set with schema {SET_SCHEMA}") questions: list[dict[str, Any]] = data["questions"] return questions def require_dir(path: Path, what: str) -> None: if not path.is_dir(): raise RunError(f"{what} {path} is not a directory") class SourceCache: def __init__(self, root: Path) -> None: self.root = root self._text: dict[str, str | None] = {} def text(self, name: str) -> str | None: """File text, or None when the name is not a plain file directly in the sources directory.""" if not is_plain_name(name): return None if name not in self._text: p = self.root / name try: self._text[name] = p.read_text(encoding="utf-8") if p.is_file() else None except (OSError, UnicodeDecodeError): self._text[name] = None return self._text[name] # ---------------------------------------------------------------------------------------------- check @dataclass class CheckReport: questions: int = 0 spans: int = 0 spans_verbatim: int = 0 all_spans_verbatim: int = 0 min_files: int = 0 key_per_file: int = 0 keys: int = 0 wording_keys: int = 0 negated_keys: int = 0 negated_ok: int = 0 errors: list[str] = field(default_factory=list) files_per_question: dict[str, int] = field(default_factory=dict) @property def passed(self) -> bool: n = self.questions return ( n > 0 and not self.errors and self.spans_verbatim == self.spans and self.all_spans_verbatim == n and self.min_files == n and self.key_per_file == n and self.wording_keys == 0 and self.negated_ok == self.negated_keys ) def lines(self) -> list[str]: n = self.questions out = [f"error: {e}" for e in self.errors] out += [ f"questions: {n}", f"spans verbatim: {self.spans_verbatim} of {self.spans}", f"questions with every span verbatim: {self.all_spans_verbatim} of {n}", f"questions with >= {MIN_FILES} distinct files: {self.min_files} of {n}", f"questions with a key per file: {self.key_per_file} of {n}", f"answer keys that are wording: {self.wording_keys} of {self.keys}", f"negated keys negated in their span: {self.negated_ok} of {self.negated_keys}", "PASS" if self.passed else "FAIL", ] return out def check_questions(questions: list[dict[str, Any]], sources: Path) -> CheckReport: require_dir(sources, "sources") cache = SourceCache(sources) rep = CheckReport(questions=len(questions)) form = count_keys(questions) rep.keys, rep.wording_keys = form.keys, form.wording seen: set[str] = set() for q in questions: qid = q.get("id") if isinstance(q, dict) else None if not isinstance(qid, str) or not qid: rep.errors.append("question without an id") continue if qid in seen: rep.errors.append(f"duplicate id {qid}") seen.add(qid) if not isinstance(q.get("question"), str) or not q["question"].strip(): rep.errors.append(f"{qid}: empty question") evidence = as_list(q.get("evidence")) keys = as_list(q.get("answer_keys")) spans_by_file: dict[str, list[str]] = {} verbatim = 0 for e in evidence: name, span = (e.get("file"), e.get("span")) if isinstance(e, dict) else (None, None) rep.spans += 1 if not isinstance(name, str) or not is_plain_name(name): rep.errors.append(f"{qid}: evidence file {name!r} is not a plain file name") continue if not isinstance(span, str): rep.errors.append(f"{qid}: evidence span in {name} is not text") continue if not SPAN_MIN <= len(span) <= SPAN_MAX: rep.errors.append(f"{qid}: span length {len(span)} in {name} outside {SPAN_MIN}-{SPAN_MAX}") text = cache.text(name) if text is not None and span in text: verbatim += 1 spans_by_file.setdefault(name, []).append(span) rep.spans_verbatim += verbatim if evidence and verbatim == len(evidence): rep.all_spans_verbatim += 1 files = set(spans_by_file) rep.files_per_question[qid] = len(files) if len(files) >= MIN_FILES: rep.min_files += 1 keyed: set[str] = set() keys_ok = bool(keys) for k in keys: name, key = (k.get("file"), k.get("text")) if isinstance(k, dict) else (None, None) if not isinstance(key, str) or not isinstance(name, str): keys_ok = False rep.errors.append(f"{qid}: malformed answer key") continue if number_key(key) is None and not KEY_MIN <= len(key.strip()) <= KEY_MAX: rep.errors.append(f"{qid}: key length {len(key.strip())} outside {KEY_MIN}-{KEY_MAX}") if any(key_in(key, s) for s in spans_by_file.get(name, [])): keyed.add(name) else: keys_ok = False if "negated" in k: if k["negated"] is not True: rep.errors.append(f"{qid}: negated must be true or absent") continue rep.negated_keys += 1 rep.negated_ok += any(negated_in(key, s) for s in spans_by_file.get(name, [])) if keys_ok and files and keyed == files: rep.key_per_file += 1 return rep def cmd_check(args: argparse.Namespace) -> int: rep = check_questions(load_questions(Path(args.set)), Path(args.sources)) if args.per_question: for qid, n in rep.files_per_question.items(): print(f"{qid}\t{n} files") print("\n".join(rep.lines())) return 0 if rep.passed else 1 # ---------------------------------------------------------------------------------------------- split def inside_repo(path: Path) -> bool: try: path.resolve().relative_to(REPO_ROOT) return True except ValueError: return False def dump_set(questions: list[dict[str, Any]]) -> bytes: return (json.dumps({"schema": SET_SCHEMA, "questions": questions}, ensure_ascii=False, indent=1) + "\n").encode("utf-8") def cmd_split(args: argparse.Namespace) -> int: sources, held_path = Path(args.sources), Path(args.held_out) dev_path, manifest_path = Path(args.dev_out), Path(args.manifest) if inside_repo(held_path): raise RunError(f"held-out path {held_path} is inside the repository; it must live outside it") if held_path.exists(): raise RunError(f"held-out file {held_path} already exists; a frozen draw is never redrawn") questions = load_questions(Path(args.set)) rep = check_questions(questions, sources) if not rep.passed: print("\n".join(rep.lines())) raise RunError("the set fails the check; nothing was split") n_held = args.held_out_count if not 0 < n_held < len(questions): raise RunError(f"cannot hold out {n_held} of {len(questions)}") ids = sorted(q["id"] for q in questions) held_ids = set(random.Random(args.seed).sample(ids, n_held)) by_id = {q["id"]: q for q in questions} dev = [dict(by_id[i], id=f"d{n:02d}") for n, i in enumerate((i for i in ids if i not in held_ids), 1)] held = [dict(by_id[i], id=f"h{n:02d}") for n, i in enumerate(sorted(held_ids), 1)] dev_bytes, held_bytes = dump_set(dev), dump_set(held) held_path.parent.mkdir(parents=True, exist_ok=True) held_path.write_bytes(held_bytes) dev_path.parent.mkdir(parents=True, exist_ok=True) dev_path.write_bytes(dev_bytes) manifest = { "schema": MANIFEST_SCHEMA, "seed": args.seed, "draw": "random.Random(seed).sample(sorted(ids), held_out.questions); both parts renumbered d../h..", "dev": {"file": args.dev_label or dev_path.name, "sha256": sha256_bytes(dev_bytes), "questions": len(dev)}, "held_out": {"sha256": sha256_bytes(held_bytes), "questions": len(held)}, "sources": {"files": count_files(sources), "tree_sha256": tree_sha256(sources)}, } manifest_path.parent.mkdir(parents=True, exist_ok=True) manifest_path.write_text(json.dumps(manifest, indent=1) + "\n", encoding="utf-8") print(f"seed: {args.seed}") print(f"dev: {len(dev)} questions, sha256 {manifest['dev']['sha256']}") print(f"held out: {len(held)} questions, sha256 {manifest['held_out']['sha256']}") print(f"manifest: {manifest_path}") return 0 # ---------------------------------------------------------------------------------------------- gate @dataclass class GateResult: qid: str answered: bool answer_ok: bool source_ok: bool @property def correct(self) -> bool: return self.answer_ok and self.source_ok def score_one(q: dict[str, Any], a: dict[str, Any] | None, cache: SourceCache) -> GateResult: if a is None: return GateResult(q["id"], False, False, False) answer = a.get("answer", "") if isinstance(a.get("answer"), str) else "" answer_ok = all(key_holds(k, answer) for k in q["answer_keys"]) supported: set[str] = set() citations = as_list(a.get("citations")) for c in citations: name, quote = (c.get("file"), c.get("quote")) if isinstance(c, dict) else (None, None) if not isinstance(name, str) or not isinstance(quote, str) or not quote.strip(): continue text = cache.text(name) if text is not None and normalise_quote(quote) in normalise_quote(text): supported.add(name) needed = {e["file"] for e in q["evidence"]} return GateResult(q["id"], True, answer_ok, needed <= supported) def cmd_gate(args: argparse.Namespace) -> int: set_path, sources, manifest_path = Path(args.set), Path(args.sources), Path(args.manifest) require_dir(sources, "sources") manifest = load_json(manifest_path, "manifest") if not isinstance(manifest, dict) or manifest.get("schema") != MANIFEST_SCHEMA: raise RunError(f"{manifest_path} is not a manifest with schema {MANIFEST_SCHEMA}") set_hash = sha256_bytes(set_path.read_bytes()) if set_path.is_file() else "" known = {manifest.get("dev", {}).get("sha256"): "dev", manifest.get("held_out", {}).get("sha256"): "held_out"} part = known.get(set_hash) if part is None: raise RunError(f"set {set_path} sha256 {set_hash or '(missing)'} matches neither part of the manifest") if tree_sha256(sources) != manifest.get("sources", {}).get("tree_sha256"): raise RunError(f"sources {sources} differ from the manifest's tree_sha256; the set is frozen against them") questions = load_questions(set_path) n = len(questions) cache = SourceCache(sources) answers: dict[str, dict[str, Any]] = {} fail_closed = "" answers_path = Path(args.answers) if not answers_path.is_file(): fail_closed = f"no answers for arm {args.arm} at {answers_path}" else: data = load_json(answers_path, "answers") if not isinstance(data, dict) or not isinstance(data.get("answers"), list): raise RunError(f"{answers_path} is not an answers file {ANSWERS_SCHEMA_NOTE}") if data.get("arm") != args.arm: raise RunError(f"answers file is for arm {data.get('arm')!r}, not {args.arm!r}") for a in data["answers"]: if isinstance(a, dict) and isinstance(a.get("id"), str): answers[a["id"]] = a results = [score_one(q, answers.get(q["id"]), cache) for q in questions] missing = sum(1 for r in results if not r.answered) print(f"set: {part}, {n} questions; arm: {args.arm}") if fail_closed: print(f"fail-closed: {fail_closed}") print(f"missing answers: {missing}") print(f"correct answer: {sum(r.answer_ok for r in results)} of {n}") print(f"correct source: {sum(r.source_ok for r in results)} of {n}") print(f"correct answer and source: {sum(r.correct for r in results)} of {n}") if args.out: out = { "arm": args.arm, "set_sha256": set_hash, "results": [ {"id": r.qid, "answered": r.answered, "answer_ok": r.answer_ok, "source_ok": r.source_ok, "correct": r.correct} for r in results ], } Path(args.out).write_text(json.dumps(out, indent=1) + "\n", encoding="utf-8") scored = not fail_closed and missing == 0 print("SCORED" if scored else "UNSCORED (fail-closed)" if fail_closed else "INCOMPLETE") return 0 if scored else 1 # ---------------------------------------------------------------------------------------------- main def build_parser() -> argparse.ArgumentParser: p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) sub = p.add_subparsers(dest="command", required=True) c = sub.add_parser("check", help="check spans, file counts and answer keys") c.add_argument("--set", required=True) c.add_argument("--sources", required=True) c.add_argument("--per-question", action="store_true", help="also list distinct files per question") s = sub.add_parser("split", help="draw the held-out questions with a fixed seed") s.add_argument("--set", required=True) s.add_argument("--sources", required=True) s.add_argument("--seed", type=int, required=True) s.add_argument("--held-out-count", type=int, default=20) s.add_argument("--dev-out", required=True) s.add_argument("--dev-label", default="", help="path of the dev set as written in the manifest") s.add_argument("--held-out", required=True, help="file path OUTSIDE the repository") s.add_argument("--manifest", required=True) g = sub.add_parser("gate", help="score one arm's answers against the answer key") g.add_argument("--set", required=True) g.add_argument("--manifest", required=True) g.add_argument("--sources", required=True) g.add_argument("--arm", required=True) g.add_argument("--answers", required=True, help=ANSWERS_SCHEMA_NOTE) g.add_argument("--out", default="", help="write per-question results (for paired tests)") return p def main(argv: list[str] | None = None) -> int: args = build_parser().parse_args(argv) commands = {"check": cmd_check, "split": cmd_split, "gate": cmd_gate} try: return commands[args.command](args) except RunError as exc: print(f"FAIL: {exc}", file=sys.stderr) return 1 if __name__ == "__main__": sys.exit(main())