"""The whole-document and counting group (E2b): cut, checker, held-out split and answer gate. No model call. The E2 set asks only multi-document questions with short spans. This group asks the other class: questions whose answer needs one long file read as a whole, or a count over many places in one file. It is reported on its own. cut computes the source cut from the E2 dev set (sorted by id, repeatedly the question needing the fewest NEW files, while the union stays within --max-files) and writes the manifest with the thresholds, BEFORE any question is written. The thresholds are fixed from that moment. check the properties of every question, measured in the source files: spans verbatim and unique, the file in the cut; whole: spans from >= 3 separate parts covering the first to the last third; count: the count equals the spans and the item pattern's matches, is >= the minimum and does not stand in the file. split draws the held-out questions per type with a fixed seed, writes them OUTSIDE the repository, renumbers both parts and records sha256 values and counts in the manifest. gate scores one arm. whole: the answer holds every answer key; count: the answer's `count` is the exact integer. Both: a citation quotes the file verbatim. A missing arm scores 0 (fail-closed). Exit 0 = PASS / scored completely; 1 = FAIL or could not run. A check that cannot run is a FAIL. """ from __future__ import annotations import argparse import json import random import re import sys from dataclasses import dataclass, field from pathlib import Path from typing import Any from keyform import count_keys from multidoc import ( NUMBER_WORDS, RunError, SourceCache, as_list, count_files, inside_repo, key_holds, key_in, load_json, negated_in, normalise_quote, number_key, require_dir, sha256_bytes, tree_sha256, ) SET_SCHEMA = "okf-wiki/wholedoc-set/1" MANIFEST_SCHEMA = "okf-wiki/wholedoc-manifest/1" ANSWERS_SCHEMA_NOTE = '{"arm": "", "answers": [{"id", "answer", "count" (count questions), "citations": [{"file", "quote"}]}]}' TYPES = ("count", "whole") CUT_MAX_FILES = 20 SPAN_MIN, SPAN_MAX = 20, 400 KEY_MIN, KEY_MAX = 3, 60 WHOLE_MIN_LINES = 150 WHOLE_MIN_KEYS = 3 WHOLE_MIN_PARTS = 3 PART_GAP = 0.15 FIRST_THIRD, LAST_THIRD = 1 / 3, 2 / 3 COUNT_MIN = 5 def thresholds() -> dict[str, Any]: """The thresholds, as frozen in the manifest before any question is written.""" return { "span_chars": [SPAN_MIN, SPAN_MAX], "key_chars": [KEY_MIN, KEY_MAX], "span_occurrences": "exactly 1 in its file (so its position is defined)", "whole_min_file_lines": WHOLE_MIN_LINES, "whole_min_keys": WHOLE_MIN_KEYS, "whole_min_parts": WHOLE_MIN_PARTS, "whole_part_gap": PART_GAP, "whole_part_rule": "spans sorted by start offset; a new part begins where the gap between two starts is " ">= whole_part_gap x file length (characters)", "whole_coverage": "first span starts before 1/3 of the file, last span starts at or after 2/3", "whole_one_span": "no single span holds every answer key (case-folded)", "count_min": COUNT_MIN, "count_rule": "count == number of evidence spans == matches of item_pattern in the file; every match starts " "inside exactly one span", "count_not_verbatim": "the count stands nowhere in the file as a number (not part of a longer number, version " "or identifier) nor as an English number word", "cut_max_files": CUT_MAX_FILES, } def number_stands_in(n: int, text: str) -> bool: """True when n stands as its own number or as its English word, not inside a version, identifier or number.""" if re.search(rf"(? list[dict[str, Any]]: data = load_json(path, "set") if not isinstance(data, dict) or data.get("schema") != SET_SCHEMA or not isinstance(data.get("questions"), list): raise RunError(f"{path} is not a set with schema {SET_SCHEMA}") questions: list[dict[str, Any]] = data["questions"] return questions def dump_set(questions: list[dict[str, Any]]) -> bytes: return (json.dumps({"schema": SET_SCHEMA, "questions": questions}, ensure_ascii=False, indent=1) + "\n").encode("utf-8") def load_manifest(path: Path) -> dict[str, Any]: manifest = load_json(path, "manifest") if not isinstance(manifest, dict) or manifest.get("schema") != MANIFEST_SCHEMA: raise RunError(f"{path} is not a manifest with schema {MANIFEST_SCHEMA}") return manifest def write_json(path: Path, data: Any) -> None: path.parent.mkdir(parents=True, exist_ok=True) path.write_text(json.dumps(data, indent=1) + "\n", encoding="utf-8") # ---------------------------------------------------------------------------------------------- cut def compute_cut(questions: list[dict[str, Any]], max_files: int) -> tuple[list[str], list[str]]: """Repeatedly take the question needing the fewest new files (ties: lowest id) while the union fits.""" files = {q["id"]: {e["file"] for e in as_list(q.get("evidence"))} for q in questions} remaining = sorted(files) union: set[str] = set() chosen: list[str] = [] while remaining: best = min(remaining, key=lambda i: (len(files[i] - union), i)) if len(union | files[best]) > max_files: break union |= files[best] chosen.append(best) remaining.remove(best) return chosen, sorted(union) def cut_from(e2_path: Path, max_files: int) -> dict[str, Any]: data = load_json(e2_path, "E2 set") if not isinstance(data, dict) or not isinstance(data.get("questions"), list): raise RunError(f"{e2_path} is not an E2 set") chosen, files = compute_cut(data["questions"], max_files) return { "rule": "E2 dev questions sorted by id; repeatedly the question needing the fewest NEW source files " "(ties: lowest id), while the union stays <= max_files", "e2_set_sha256": sha256_bytes(e2_path.read_bytes()), "max_files": max_files, "questions": len(chosen), "files": len(files), "names_sha256": sha256_bytes("".join(f + "\n" for f in files).encode("utf-8")), "names": files, } def cmd_cut(args: argparse.Namespace) -> int: sources, manifest_path = Path(args.sources), Path(args.manifest) require_dir(sources, "sources") if manifest_path.exists(): raise RunError(f"manifest {manifest_path} already exists; the thresholds are frozen once written") cut = cut_from(Path(args.e2_set), args.max_files) missing = [f for f in cut["names"] if not (sources / f).is_file()] if missing: raise RunError(f"cut files missing from the sources: {', '.join(missing)}") write_json(manifest_path, { "schema": MANIFEST_SCHEMA, "thresholds": thresholds(), "cut": cut, "sources": {"files": count_files(sources), "tree_sha256": tree_sha256(sources)}, "seed": None, "dev": None, "held_out": None, }) print(f"cut: {cut['files']} files from {cut['questions']} E2 questions (max {args.max_files})") for name in cut["names"]: print(f" {name}") print(f"manifest: {manifest_path}") return 0 # ---------------------------------------------------------------------------------------------- check @dataclass class CheckReport: questions: int = 0 by_type: dict[str, int] = field(default_factory=lambda: {t: 0 for t in TYPES}) spans: int = 0 spans_ok: int = 0 in_cut: int = 0 whole_long: int = 0 whole_keyed: int = 0 whole_spread: int = 0 count_ok: int = 0 count_hidden: int = 0 not_one_span: int = 0 keys: int = 0 wording_keys: int = 0 negated_keys: int = 0 negated_ok: int = 0 errors: list[str] = field(default_factory=list) @property def passed(self) -> bool: n, w, c = self.questions, self.by_type["whole"], self.by_type["count"] return (n > 0 and not self.errors and self.spans_ok == self.spans and self.in_cut == n and self.whole_long == w and self.whole_keyed == w and self.whole_spread == w and self.count_ok == c and self.count_hidden == c and self.not_one_span == n and self.wording_keys == 0 and self.negated_ok == self.negated_keys) def lines(self) -> list[str]: n, w, c = self.questions, self.by_type["whole"], self.by_type["count"] return [f"error: {e}" for e in self.errors] + [ f"questions: {n} (whole {w}, count {c})", f"spans verbatim and unique: {self.spans_ok} of {self.spans}", f"questions whose file is in the cut: {self.in_cut} of {n}", f"whole: file >= {WHOLE_MIN_LINES} lines: {self.whole_long} of {w}", f"whole: every key in a span, every span keyed: {self.whole_keyed} of {w}", f"whole: spread over >= {WHOLE_MIN_PARTS} parts, first to last third: {self.whole_spread} of {w}", f"count: count = spans = pattern matches >= {COUNT_MIN}: {self.count_ok} of {c}", f"count: answer not verbatim in the file: {self.count_hidden} of {c}", f"not answerable from one span: {self.not_one_span} of {n}", f"answer keys that are wording: {self.wording_keys} of {self.keys}", f"negated keys negated in their span: {self.negated_ok} of {self.negated_keys}", "PASS" if self.passed else "FAIL", ] def parts(starts: list[int], length: int) -> int: ordered = sorted(starts) return 1 + sum(1 for a, b in zip(ordered, ordered[1:]) if b - a >= PART_GAP * length) if ordered else 0 def check_whole(q: dict[str, Any], text: str, spans: list[tuple[str, int]], rep: CheckReport) -> None: qid = q["id"] if text.count("\n") >= WHOLE_MIN_LINES: rep.whole_long += 1 keys = [k.get("text") if isinstance(k, dict) else None for k in as_list(q.get("answer_keys"))] if (not all(isinstance(k, str) and (number_key(k) is not None or KEY_MIN <= len(k.strip()) <= KEY_MAX) for k in keys) or len(keys) < WHOLE_MIN_KEYS): rep.errors.append(f"{qid}: needs >= {WHOLE_MIN_KEYS} answer keys of {KEY_MIN}-{KEY_MAX} characters") return texts = [str(k) for k in keys] span_texts = [s for s, _ in spans] if all(any(key_in(k, s) for s in span_texts) for k in texts) and all(any(key_in(k, s) for k in texts) for s in span_texts): rep.whole_keyed += 1 starts = [pos for _, pos in spans if pos >= 0] if (len(starts) == len(spans) and parts(starts, len(text)) >= WHOLE_MIN_PARTS and min(starts) < FIRST_THIRD * len(text) and max(starts) >= LAST_THIRD * len(text)): rep.whole_spread += 1 if not any(all(key_in(k, s) for k in texts) for s in span_texts): rep.not_one_span += 1 for k in as_list(q.get("answer_keys")): if "negated" not in k: continue if k["negated"] is not True: rep.errors.append(f"{qid}: negated must be true or absent") continue rep.negated_keys += 1 rep.negated_ok += any(negated_in(k["text"], s) for s in span_texts) def check_count(q: dict[str, Any], text: str, spans: list[tuple[str, int]], rep: CheckReport) -> None: qid, count, pattern = q["id"], q.get("count"), q.get("item_pattern") if type(count) is not int or not isinstance(pattern, str): rep.errors.append(f"{qid}: a count question needs an integer count and an item_pattern") return try: matches = [m.start() for m in re.finditer(pattern, text)] except re.error as exc: rep.errors.append(f"{qid}: item_pattern does not compile: {exc}") return ranges = [(pos, pos + len(s)) for s, pos in spans if pos >= 0] each_in_one = all(sum(1 for a, b in ranges if a <= m < b) == 1 for m in matches) if count >= COUNT_MIN and count == len(spans) == len(ranges) == len(matches) and each_in_one: rep.count_ok += 1 if not number_stands_in(count, text): rep.count_hidden += 1 if len(spans) >= 2 and not any(number_stands_in(count, s) for s, _ in spans): rep.not_one_span += 1 def check_questions(questions: list[dict[str, Any]], sources: Path, cut_names: list[str]) -> CheckReport: require_dir(sources, "sources") cache = SourceCache(sources) rep = CheckReport(questions=len(questions)) form = count_keys(questions) rep.keys, rep.wording_keys = form.keys, form.wording seen: set[str] = set() for q in questions: qid = q.get("id") if isinstance(q, dict) else None if not isinstance(qid, str) or not qid: rep.errors.append("question without an id") continue if qid in seen: rep.errors.append(f"duplicate id {qid}") seen.add(qid) qtype = q.get("type") if qtype not in TYPES: rep.errors.append(f"{qid}: type {qtype!r} is not one of {TYPES}") continue rep.by_type[qtype] += 1 if not isinstance(q.get("question"), str) or not q["question"].strip(): rep.errors.append(f"{qid}: empty question") name = q.get("file") if name in cut_names: rep.in_cut += 1 else: rep.errors.append(f"{qid}: file {name!r} is not in the cut") text = cache.text(name) if isinstance(name, str) else None if text is None: rep.errors.append(f"{qid}: file {name!r} is not a readable file in the sources") text = "" spans: list[tuple[str, int]] = [] for e in as_list(q.get("evidence")): span = e.get("span") if isinstance(e, dict) else None rep.spans += 1 if not isinstance(span, str): rep.errors.append(f"{qid}: evidence span is not text") continue if not SPAN_MIN <= len(span) <= SPAN_MAX: rep.errors.append(f"{qid}: span length {len(span)} outside {SPAN_MIN}-{SPAN_MAX}") unique = text.count(span) == 1 rep.spans_ok += unique spans.append((span, text.find(span) if unique else -1)) if qtype == "whole": check_whole(q, text, spans, rep) else: check_count(q, text, spans, rep) return rep def verified_cut(manifest: dict[str, Any], e2_path: Path) -> list[str]: cut = manifest.get("cut") if not isinstance(cut, dict): raise RunError("manifest has no cut") fresh = cut_from(e2_path, int(cut.get("max_files", CUT_MAX_FILES))) for key in ("e2_set_sha256", "names_sha256", "files"): if fresh[key] != cut.get(key): raise RunError(f"the cut from {e2_path} differs from the manifest ({key}); the group is frozen against it") names: list[str] = fresh["names"] return names def cmd_check(args: argparse.Namespace) -> int: manifest = load_manifest(Path(args.manifest)) cut_names = verified_cut(manifest, Path(args.e2_set)) rep = check_questions(load_set(Path(args.set)), Path(args.sources), cut_names) print("\n".join(rep.lines())) return 0 if rep.passed else 1 # ---------------------------------------------------------------------------------------------- split def cmd_split(args: argparse.Namespace) -> int: sources, held_path, manifest_path = Path(args.sources), Path(args.held_out), Path(args.manifest) dev_path = Path(args.dev_out) if inside_repo(held_path): raise RunError(f"held-out path {held_path} is inside the repository; it must live outside it") if held_path.exists(): raise RunError(f"held-out file {held_path} already exists; a frozen draw is never redrawn") manifest = load_manifest(manifest_path) if manifest.get("dev") is not None or manifest.get("held_out") is not None: raise RunError(f"manifest {manifest_path} already records a draw; a frozen draw is never redrawn") if manifest.get("thresholds") != thresholds(): raise RunError("the manifest's thresholds differ from the code; they were frozen before the questions") if tree_sha256(sources) != manifest.get("sources", {}).get("tree_sha256"): raise RunError(f"sources {sources} differ from the manifest's tree_sha256") questions = load_set(Path(args.set)) rep = check_questions(questions, sources, verified_cut(manifest, Path(args.e2_set))) if not rep.passed: print("\n".join(rep.lines())) raise RunError("the set fails the check; nothing was split") k = args.held_out_per_type rng = random.Random(args.seed) held_ids: set[str] = set() for t in TYPES: ids = sorted(q["id"] for q in questions if q["type"] == t) if not 0 < k < len(ids): raise RunError(f"cannot hold out {k} of {len(ids)} {t} questions") held_ids.update(rng.sample(ids, k)) by_id = {q["id"]: q for q in questions} ids = sorted(by_id) dev = [dict(by_id[i], id=f"d{n:02d}") for n, i in enumerate((i for i in ids if i not in held_ids), 1)] held = [dict(by_id[i], id=f"h{n:02d}") for n, i in enumerate(sorted(held_ids), 1)] dev_bytes, held_bytes = dump_set(dev), dump_set(held) held_path.parent.mkdir(parents=True, exist_ok=True) held_path.write_bytes(held_bytes) dev_path.parent.mkdir(parents=True, exist_ok=True) dev_path.write_bytes(dev_bytes) def per_type(part: list[dict[str, Any]]) -> dict[str, int]: return {t: sum(1 for q in part if q["type"] == t) for t in ("whole", "count")} manifest["seed"] = args.seed manifest["draw"] = ("rng = random.Random(seed); for type in ('count', 'whole'): rng.sample(sorted(ids of type), " "held_out_per_type); both parts renumbered d../h.. in sorted original-id order") manifest["dev"] = {"file": args.dev_label or dev_path.name, "sha256": sha256_bytes(dev_bytes), "questions": len(dev), **per_type(dev)} manifest["held_out"] = {"sha256": sha256_bytes(held_bytes), "questions": len(held), **per_type(held)} write_json(manifest_path, manifest) print(f"seed: {args.seed}") print(f"dev: {len(dev)} questions {per_type(dev)}, sha256 {manifest['dev']['sha256']}") print(f"held out: {len(held)} questions {per_type(held)}, sha256 {manifest['held_out']['sha256']}") print(f"manifest: {manifest_path}") return 0 # ---------------------------------------------------------------------------------------------- gate @dataclass class GateResult: qid: str qtype: str answered: bool answer_ok: bool source_ok: bool @property def correct(self) -> bool: return self.answer_ok and self.source_ok def score_one(q: dict[str, Any], a: dict[str, Any] | None, cache: SourceCache) -> GateResult: if a is None: return GateResult(q["id"], q["type"], False, False, False) if q["type"] == "count": given = a.get("count") answer_ok = type(given) is int and given == q["count"] else: answer = a["answer"] if isinstance(a.get("answer"), str) else "" answer_ok = all(key_holds(k, answer) for k in q["answer_keys"]) text = cache.text(q["file"]) source_ok = False for c in as_list(a.get("citations")): name, quote = (c.get("file"), c.get("quote")) if isinstance(c, dict) else (None, None) if name == q["file"] and text is not None and isinstance(quote, str) and quote.strip(): source_ok = source_ok or normalise_quote(quote) in normalise_quote(text) return GateResult(q["id"], q["type"], True, answer_ok, source_ok) def cmd_gate(args: argparse.Namespace) -> int: set_path, sources = Path(args.set), Path(args.sources) require_dir(sources, "sources") manifest = load_manifest(Path(args.manifest)) dev, held = manifest.get("dev"), manifest.get("held_out") if not isinstance(dev, dict) or not isinstance(held, dict): raise RunError("the manifest records no draw yet; run split first") set_hash = sha256_bytes(set_path.read_bytes()) if set_path.is_file() else "" part = {dev.get("sha256"): "dev", held.get("sha256"): "held_out"}.get(set_hash) if part is None: raise RunError(f"set {set_path} sha256 {set_hash or '(missing)'} matches neither part of the manifest") if tree_sha256(sources) != manifest.get("sources", {}).get("tree_sha256"): raise RunError(f"sources {sources} differ from the manifest's tree_sha256; the set is frozen against them") questions = load_set(set_path) cache = SourceCache(sources) answers: dict[str, dict[str, Any]] = {} fail_closed = "" answers_path = Path(args.answers) if not answers_path.is_file(): fail_closed = f"no answers for arm {args.arm} at {answers_path}" else: data = load_json(answers_path, "answers") if not isinstance(data, dict) or not isinstance(data.get("answers"), list): raise RunError(f"{answers_path} is not an answers file {ANSWERS_SCHEMA_NOTE}") if data.get("arm") != args.arm: raise RunError(f"answers file is for arm {data.get('arm')!r}, not {args.arm!r}") answers = {a["id"]: a for a in data["answers"] if isinstance(a, dict) and isinstance(a.get("id"), str)} results = [score_one(q, answers.get(q["id"]), cache) for q in questions] n = len(results) missing = sum(1 for r in results if not r.answered) print(f"set: {part}, {n} questions; arm: {args.arm}") if fail_closed: print(f"fail-closed: {fail_closed}") print(f"missing answers: {missing}") for t in ("whole", "count"): of_type = [r for r in results if r.qtype == t] print(f"{t}: correct {sum(r.correct for r in of_type)} of {len(of_type)}") print(f"correct answer: {sum(r.answer_ok for r in results)} of {n}") print(f"correct source: {sum(r.source_ok for r in results)} of {n}") correct = sum(r.correct for r in results) print(f"correct answer and source: {correct} of {n}") if args.out: write_json(Path(args.out), { "arm": args.arm, "set_sha256": set_hash, "results": [{"id": r.qid, "type": r.qtype, "answered": r.answered, "answer_ok": r.answer_ok, "source_ok": r.source_ok, "correct": r.correct} for r in results], }) scored = not fail_closed and missing == 0 print("GREEN" if scored and n > 0 and correct == n else "RED") print("SCORED" if scored else "UNSCORED (fail-closed)" if fail_closed else "INCOMPLETE") return 0 if scored else 1 # ---------------------------------------------------------------------------------------------- main def build_parser() -> argparse.ArgumentParser: p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) sub = p.add_subparsers(dest="command", required=True) u = sub.add_parser("cut", help="compute the source cut and freeze the thresholds in a new manifest") u.add_argument("--e2-set", required=True) u.add_argument("--sources", required=True) u.add_argument("--max-files", type=int, default=CUT_MAX_FILES) u.add_argument("--manifest", required=True) c = sub.add_parser("check", help="check every question's properties in the source files") c.add_argument("--set", required=True) c.add_argument("--sources", required=True) c.add_argument("--manifest", required=True) c.add_argument("--e2-set", required=True) s = sub.add_parser("split", help="draw the held-out questions per type with a fixed seed") s.add_argument("--set", required=True) s.add_argument("--sources", required=True) s.add_argument("--manifest", required=True) s.add_argument("--e2-set", required=True) s.add_argument("--seed", type=int, required=True) s.add_argument("--held-out-per-type", type=int, default=2) s.add_argument("--dev-out", required=True) s.add_argument("--dev-label", default="", help="path of the dev set as written in the manifest") s.add_argument("--held-out", required=True, help="file path OUTSIDE the repository") g = sub.add_parser("gate", help="score one arm's answers against the answer key") g.add_argument("--set", required=True) g.add_argument("--manifest", required=True) g.add_argument("--sources", required=True) g.add_argument("--arm", required=True) g.add_argument("--answers", required=True, help=ANSWERS_SCHEMA_NOTE) g.add_argument("--out", default="", help="write per-question results (for paired tests)") return p def main(argv: list[str] | None = None) -> int: args = build_parser().parse_args(argv) commands = {"cut": cmd_cut, "check": cmd_check, "split": cmd_split, "gate": cmd_gate} try: return commands[args.command](args) except RunError as exc: print(f"FAIL: {exc}", file=sys.stderr) return 1 if __name__ == "__main__": sys.exit(main())