okf-wiki/gates/wholedoc.py
Kjell Tore Guttormsen 0fd86a23a3
okf-wiki 1.1.1
First public release, published as one snapshot. What changed between
versions is in CHANGELOG.md.

Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
2026-11-14 21:03:20 +01:00

544 lines
25 KiB
Python

"""The whole-document and counting group (E2b): cut, checker, held-out split and answer gate. No model call.
The E2 set asks only multi-document questions with short spans. This group asks the other class: questions whose
answer needs one long file read as a whole, or a count over many places in one file. It is reported on its own.
cut computes the source cut from the E2 dev set (sorted by id, repeatedly the question needing the fewest
NEW files, while the union stays within --max-files) and writes the manifest with the thresholds,
BEFORE any question is written. The thresholds are fixed from that moment.
check the properties of every question, measured in the source files: spans verbatim and unique, the file
in the cut; whole: spans from >= 3 separate parts covering the first to the last third; count: the
count equals the spans and the item pattern's matches, is >= the minimum and does not stand in the file.
split draws the held-out questions per type with a fixed seed, writes them OUTSIDE the repository,
renumbers both parts and records sha256 values and counts in the manifest.
gate scores one arm. whole: the answer holds every answer key; count: the answer's `count` is the exact
integer. Both: a citation quotes the file verbatim. A missing arm scores 0 (fail-closed).
Exit 0 = PASS / scored completely; 1 = FAIL or could not run. A check that cannot run is a FAIL.
"""
from __future__ import annotations
import argparse
import json
import random
import re
import sys
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any
from keyform import count_keys
from multidoc import (
NUMBER_WORDS,
RunError,
SourceCache,
as_list,
count_files,
inside_repo,
key_holds,
key_in,
load_json,
negated_in,
normalise_quote,
number_key,
require_dir,
sha256_bytes,
tree_sha256,
)
SET_SCHEMA = "okf-wiki/wholedoc-set/1"
MANIFEST_SCHEMA = "okf-wiki/wholedoc-manifest/1"
ANSWERS_SCHEMA_NOTE = '{"arm": "<name>", "answers": [{"id", "answer", "count" (count questions), "citations": [{"file", "quote"}]}]}'
TYPES = ("count", "whole")
CUT_MAX_FILES = 20
SPAN_MIN, SPAN_MAX = 20, 400
KEY_MIN, KEY_MAX = 3, 60
WHOLE_MIN_LINES = 150
WHOLE_MIN_KEYS = 3
WHOLE_MIN_PARTS = 3
PART_GAP = 0.15
FIRST_THIRD, LAST_THIRD = 1 / 3, 2 / 3
COUNT_MIN = 5
def thresholds() -> dict[str, Any]:
"""The thresholds, as frozen in the manifest before any question is written."""
return {
"span_chars": [SPAN_MIN, SPAN_MAX],
"key_chars": [KEY_MIN, KEY_MAX],
"span_occurrences": "exactly 1 in its file (so its position is defined)",
"whole_min_file_lines": WHOLE_MIN_LINES,
"whole_min_keys": WHOLE_MIN_KEYS,
"whole_min_parts": WHOLE_MIN_PARTS,
"whole_part_gap": PART_GAP,
"whole_part_rule": "spans sorted by start offset; a new part begins where the gap between two starts is "
">= whole_part_gap x file length (characters)",
"whole_coverage": "first span starts before 1/3 of the file, last span starts at or after 2/3",
"whole_one_span": "no single span holds every answer key (case-folded)",
"count_min": COUNT_MIN,
"count_rule": "count == number of evidence spans == matches of item_pattern in the file; every match starts "
"inside exactly one span",
"count_not_verbatim": "the count stands nowhere in the file as a number (not part of a longer number, version "
"or identifier) nor as an English number word",
"cut_max_files": CUT_MAX_FILES,
}
def number_stands_in(n: int, text: str) -> bool:
"""True when n stands as its own number or as its English word, not inside a version, identifier or number."""
if re.search(rf"(?<![\w.\-]){n}(?!\w|\.\d)", text):
return True
return n < len(NUMBER_WORDS) and re.search(rf"\b{NUMBER_WORDS[n]}\b", text, re.IGNORECASE) is not None
def load_set(path: Path) -> list[dict[str, Any]]:
data = load_json(path, "set")
if not isinstance(data, dict) or data.get("schema") != SET_SCHEMA or not isinstance(data.get("questions"), list):
raise RunError(f"{path} is not a set with schema {SET_SCHEMA}")
questions: list[dict[str, Any]] = data["questions"]
return questions
def dump_set(questions: list[dict[str, Any]]) -> bytes:
return (json.dumps({"schema": SET_SCHEMA, "questions": questions}, ensure_ascii=False, indent=1) + "\n").encode("utf-8")
def load_manifest(path: Path) -> dict[str, Any]:
manifest = load_json(path, "manifest")
if not isinstance(manifest, dict) or manifest.get("schema") != MANIFEST_SCHEMA:
raise RunError(f"{path} is not a manifest with schema {MANIFEST_SCHEMA}")
return manifest
def write_json(path: Path, data: Any) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(data, indent=1) + "\n", encoding="utf-8")
# ---------------------------------------------------------------------------------------------- cut
def compute_cut(questions: list[dict[str, Any]], max_files: int) -> tuple[list[str], list[str]]:
"""Repeatedly take the question needing the fewest new files (ties: lowest id) while the union fits."""
files = {q["id"]: {e["file"] for e in as_list(q.get("evidence"))} for q in questions}
remaining = sorted(files)
union: set[str] = set()
chosen: list[str] = []
while remaining:
best = min(remaining, key=lambda i: (len(files[i] - union), i))
if len(union | files[best]) > max_files:
break
union |= files[best]
chosen.append(best)
remaining.remove(best)
return chosen, sorted(union)
def cut_from(e2_path: Path, max_files: int) -> dict[str, Any]:
data = load_json(e2_path, "E2 set")
if not isinstance(data, dict) or not isinstance(data.get("questions"), list):
raise RunError(f"{e2_path} is not an E2 set")
chosen, files = compute_cut(data["questions"], max_files)
return {
"rule": "E2 dev questions sorted by id; repeatedly the question needing the fewest NEW source files "
"(ties: lowest id), while the union stays <= max_files",
"e2_set_sha256": sha256_bytes(e2_path.read_bytes()),
"max_files": max_files,
"questions": len(chosen),
"files": len(files),
"names_sha256": sha256_bytes("".join(f + "\n" for f in files).encode("utf-8")),
"names": files,
}
def cmd_cut(args: argparse.Namespace) -> int:
sources, manifest_path = Path(args.sources), Path(args.manifest)
require_dir(sources, "sources")
if manifest_path.exists():
raise RunError(f"manifest {manifest_path} already exists; the thresholds are frozen once written")
cut = cut_from(Path(args.e2_set), args.max_files)
missing = [f for f in cut["names"] if not (sources / f).is_file()]
if missing:
raise RunError(f"cut files missing from the sources: {', '.join(missing)}")
write_json(manifest_path, {
"schema": MANIFEST_SCHEMA,
"thresholds": thresholds(),
"cut": cut,
"sources": {"files": count_files(sources), "tree_sha256": tree_sha256(sources)},
"seed": None,
"dev": None,
"held_out": None,
})
print(f"cut: {cut['files']} files from {cut['questions']} E2 questions (max {args.max_files})")
for name in cut["names"]:
print(f" {name}")
print(f"manifest: {manifest_path}")
return 0
# ---------------------------------------------------------------------------------------------- check
@dataclass
class CheckReport:
questions: int = 0
by_type: dict[str, int] = field(default_factory=lambda: {t: 0 for t in TYPES})
spans: int = 0
spans_ok: int = 0
in_cut: int = 0
whole_long: int = 0
whole_keyed: int = 0
whole_spread: int = 0
count_ok: int = 0
count_hidden: int = 0
not_one_span: int = 0
keys: int = 0
wording_keys: int = 0
negated_keys: int = 0
negated_ok: int = 0
errors: list[str] = field(default_factory=list)
@property
def passed(self) -> bool:
n, w, c = self.questions, self.by_type["whole"], self.by_type["count"]
return (n > 0 and not self.errors and self.spans_ok == self.spans and self.in_cut == n
and self.whole_long == w and self.whole_keyed == w and self.whole_spread == w
and self.count_ok == c and self.count_hidden == c and self.not_one_span == n
and self.wording_keys == 0 and self.negated_ok == self.negated_keys)
def lines(self) -> list[str]:
n, w, c = self.questions, self.by_type["whole"], self.by_type["count"]
return [f"error: {e}" for e in self.errors] + [
f"questions: {n} (whole {w}, count {c})",
f"spans verbatim and unique: {self.spans_ok} of {self.spans}",
f"questions whose file is in the cut: {self.in_cut} of {n}",
f"whole: file >= {WHOLE_MIN_LINES} lines: {self.whole_long} of {w}",
f"whole: every key in a span, every span keyed: {self.whole_keyed} of {w}",
f"whole: spread over >= {WHOLE_MIN_PARTS} parts, first to last third: {self.whole_spread} of {w}",
f"count: count = spans = pattern matches >= {COUNT_MIN}: {self.count_ok} of {c}",
f"count: answer not verbatim in the file: {self.count_hidden} of {c}",
f"not answerable from one span: {self.not_one_span} of {n}",
f"answer keys that are wording: {self.wording_keys} of {self.keys}",
f"negated keys negated in their span: {self.negated_ok} of {self.negated_keys}",
"PASS" if self.passed else "FAIL",
]
def parts(starts: list[int], length: int) -> int:
ordered = sorted(starts)
return 1 + sum(1 for a, b in zip(ordered, ordered[1:]) if b - a >= PART_GAP * length) if ordered else 0
def check_whole(q: dict[str, Any], text: str, spans: list[tuple[str, int]], rep: CheckReport) -> None:
qid = q["id"]
if text.count("\n") >= WHOLE_MIN_LINES:
rep.whole_long += 1
keys = [k.get("text") if isinstance(k, dict) else None for k in as_list(q.get("answer_keys"))]
if (not all(isinstance(k, str) and (number_key(k) is not None or KEY_MIN <= len(k.strip()) <= KEY_MAX) for k in keys)
or len(keys) < WHOLE_MIN_KEYS):
rep.errors.append(f"{qid}: needs >= {WHOLE_MIN_KEYS} answer keys of {KEY_MIN}-{KEY_MAX} characters")
return
texts = [str(k) for k in keys]
span_texts = [s for s, _ in spans]
if all(any(key_in(k, s) for s in span_texts) for k in texts) and all(any(key_in(k, s) for k in texts) for s in span_texts):
rep.whole_keyed += 1
starts = [pos for _, pos in spans if pos >= 0]
if (len(starts) == len(spans) and parts(starts, len(text)) >= WHOLE_MIN_PARTS
and min(starts) < FIRST_THIRD * len(text) and max(starts) >= LAST_THIRD * len(text)):
rep.whole_spread += 1
if not any(all(key_in(k, s) for k in texts) for s in span_texts):
rep.not_one_span += 1
for k in as_list(q.get("answer_keys")):
if "negated" not in k:
continue
if k["negated"] is not True:
rep.errors.append(f"{qid}: negated must be true or absent")
continue
rep.negated_keys += 1
rep.negated_ok += any(negated_in(k["text"], s) for s in span_texts)
def check_count(q: dict[str, Any], text: str, spans: list[tuple[str, int]], rep: CheckReport) -> None:
qid, count, pattern = q["id"], q.get("count"), q.get("item_pattern")
if type(count) is not int or not isinstance(pattern, str):
rep.errors.append(f"{qid}: a count question needs an integer count and an item_pattern")
return
try:
matches = [m.start() for m in re.finditer(pattern, text)]
except re.error as exc:
rep.errors.append(f"{qid}: item_pattern does not compile: {exc}")
return
ranges = [(pos, pos + len(s)) for s, pos in spans if pos >= 0]
each_in_one = all(sum(1 for a, b in ranges if a <= m < b) == 1 for m in matches)
if count >= COUNT_MIN and count == len(spans) == len(ranges) == len(matches) and each_in_one:
rep.count_ok += 1
if not number_stands_in(count, text):
rep.count_hidden += 1
if len(spans) >= 2 and not any(number_stands_in(count, s) for s, _ in spans):
rep.not_one_span += 1
def check_questions(questions: list[dict[str, Any]], sources: Path, cut_names: list[str]) -> CheckReport:
require_dir(sources, "sources")
cache = SourceCache(sources)
rep = CheckReport(questions=len(questions))
form = count_keys(questions)
rep.keys, rep.wording_keys = form.keys, form.wording
seen: set[str] = set()
for q in questions:
qid = q.get("id") if isinstance(q, dict) else None
if not isinstance(qid, str) or not qid:
rep.errors.append("question without an id")
continue
if qid in seen:
rep.errors.append(f"duplicate id {qid}")
seen.add(qid)
qtype = q.get("type")
if qtype not in TYPES:
rep.errors.append(f"{qid}: type {qtype!r} is not one of {TYPES}")
continue
rep.by_type[qtype] += 1
if not isinstance(q.get("question"), str) or not q["question"].strip():
rep.errors.append(f"{qid}: empty question")
name = q.get("file")
if name in cut_names:
rep.in_cut += 1
else:
rep.errors.append(f"{qid}: file {name!r} is not in the cut")
text = cache.text(name) if isinstance(name, str) else None
if text is None:
rep.errors.append(f"{qid}: file {name!r} is not a readable file in the sources")
text = ""
spans: list[tuple[str, int]] = []
for e in as_list(q.get("evidence")):
span = e.get("span") if isinstance(e, dict) else None
rep.spans += 1
if not isinstance(span, str):
rep.errors.append(f"{qid}: evidence span is not text")
continue
if not SPAN_MIN <= len(span) <= SPAN_MAX:
rep.errors.append(f"{qid}: span length {len(span)} outside {SPAN_MIN}-{SPAN_MAX}")
unique = text.count(span) == 1
rep.spans_ok += unique
spans.append((span, text.find(span) if unique else -1))
if qtype == "whole":
check_whole(q, text, spans, rep)
else:
check_count(q, text, spans, rep)
return rep
def verified_cut(manifest: dict[str, Any], e2_path: Path) -> list[str]:
cut = manifest.get("cut")
if not isinstance(cut, dict):
raise RunError("manifest has no cut")
fresh = cut_from(e2_path, int(cut.get("max_files", CUT_MAX_FILES)))
for key in ("e2_set_sha256", "names_sha256", "files"):
if fresh[key] != cut.get(key):
raise RunError(f"the cut from {e2_path} differs from the manifest ({key}); the group is frozen against it")
names: list[str] = fresh["names"]
return names
def cmd_check(args: argparse.Namespace) -> int:
manifest = load_manifest(Path(args.manifest))
cut_names = verified_cut(manifest, Path(args.e2_set))
rep = check_questions(load_set(Path(args.set)), Path(args.sources), cut_names)
print("\n".join(rep.lines()))
return 0 if rep.passed else 1
# ---------------------------------------------------------------------------------------------- split
def cmd_split(args: argparse.Namespace) -> int:
sources, held_path, manifest_path = Path(args.sources), Path(args.held_out), Path(args.manifest)
dev_path = Path(args.dev_out)
if inside_repo(held_path):
raise RunError(f"held-out path {held_path} is inside the repository; it must live outside it")
if held_path.exists():
raise RunError(f"held-out file {held_path} already exists; a frozen draw is never redrawn")
manifest = load_manifest(manifest_path)
if manifest.get("dev") is not None or manifest.get("held_out") is not None:
raise RunError(f"manifest {manifest_path} already records a draw; a frozen draw is never redrawn")
if manifest.get("thresholds") != thresholds():
raise RunError("the manifest's thresholds differ from the code; they were frozen before the questions")
if tree_sha256(sources) != manifest.get("sources", {}).get("tree_sha256"):
raise RunError(f"sources {sources} differ from the manifest's tree_sha256")
questions = load_set(Path(args.set))
rep = check_questions(questions, sources, verified_cut(manifest, Path(args.e2_set)))
if not rep.passed:
print("\n".join(rep.lines()))
raise RunError("the set fails the check; nothing was split")
k = args.held_out_per_type
rng = random.Random(args.seed)
held_ids: set[str] = set()
for t in TYPES:
ids = sorted(q["id"] for q in questions if q["type"] == t)
if not 0 < k < len(ids):
raise RunError(f"cannot hold out {k} of {len(ids)} {t} questions")
held_ids.update(rng.sample(ids, k))
by_id = {q["id"]: q for q in questions}
ids = sorted(by_id)
dev = [dict(by_id[i], id=f"d{n:02d}") for n, i in enumerate((i for i in ids if i not in held_ids), 1)]
held = [dict(by_id[i], id=f"h{n:02d}") for n, i in enumerate(sorted(held_ids), 1)]
dev_bytes, held_bytes = dump_set(dev), dump_set(held)
held_path.parent.mkdir(parents=True, exist_ok=True)
held_path.write_bytes(held_bytes)
dev_path.parent.mkdir(parents=True, exist_ok=True)
dev_path.write_bytes(dev_bytes)
def per_type(part: list[dict[str, Any]]) -> dict[str, int]:
return {t: sum(1 for q in part if q["type"] == t) for t in ("whole", "count")}
manifest["seed"] = args.seed
manifest["draw"] = ("rng = random.Random(seed); for type in ('count', 'whole'): rng.sample(sorted(ids of type), "
"held_out_per_type); both parts renumbered d../h.. in sorted original-id order")
manifest["dev"] = {"file": args.dev_label or dev_path.name, "sha256": sha256_bytes(dev_bytes),
"questions": len(dev), **per_type(dev)}
manifest["held_out"] = {"sha256": sha256_bytes(held_bytes), "questions": len(held), **per_type(held)}
write_json(manifest_path, manifest)
print(f"seed: {args.seed}")
print(f"dev: {len(dev)} questions {per_type(dev)}, sha256 {manifest['dev']['sha256']}")
print(f"held out: {len(held)} questions {per_type(held)}, sha256 {manifest['held_out']['sha256']}")
print(f"manifest: {manifest_path}")
return 0
# ---------------------------------------------------------------------------------------------- gate
@dataclass
class GateResult:
qid: str
qtype: str
answered: bool
answer_ok: bool
source_ok: bool
@property
def correct(self) -> bool:
return self.answer_ok and self.source_ok
def score_one(q: dict[str, Any], a: dict[str, Any] | None, cache: SourceCache) -> GateResult:
if a is None:
return GateResult(q["id"], q["type"], False, False, False)
if q["type"] == "count":
given = a.get("count")
answer_ok = type(given) is int and given == q["count"]
else:
answer = a["answer"] if isinstance(a.get("answer"), str) else ""
answer_ok = all(key_holds(k, answer) for k in q["answer_keys"])
text = cache.text(q["file"])
source_ok = False
for c in as_list(a.get("citations")):
name, quote = (c.get("file"), c.get("quote")) if isinstance(c, dict) else (None, None)
if name == q["file"] and text is not None and isinstance(quote, str) and quote.strip():
source_ok = source_ok or normalise_quote(quote) in normalise_quote(text)
return GateResult(q["id"], q["type"], True, answer_ok, source_ok)
def cmd_gate(args: argparse.Namespace) -> int:
set_path, sources = Path(args.set), Path(args.sources)
require_dir(sources, "sources")
manifest = load_manifest(Path(args.manifest))
dev, held = manifest.get("dev"), manifest.get("held_out")
if not isinstance(dev, dict) or not isinstance(held, dict):
raise RunError("the manifest records no draw yet; run split first")
set_hash = sha256_bytes(set_path.read_bytes()) if set_path.is_file() else ""
part = {dev.get("sha256"): "dev", held.get("sha256"): "held_out"}.get(set_hash)
if part is None:
raise RunError(f"set {set_path} sha256 {set_hash or '(missing)'} matches neither part of the manifest")
if tree_sha256(sources) != manifest.get("sources", {}).get("tree_sha256"):
raise RunError(f"sources {sources} differ from the manifest's tree_sha256; the set is frozen against them")
questions = load_set(set_path)
cache = SourceCache(sources)
answers: dict[str, dict[str, Any]] = {}
fail_closed = ""
answers_path = Path(args.answers)
if not answers_path.is_file():
fail_closed = f"no answers for arm {args.arm} at {answers_path}"
else:
data = load_json(answers_path, "answers")
if not isinstance(data, dict) or not isinstance(data.get("answers"), list):
raise RunError(f"{answers_path} is not an answers file {ANSWERS_SCHEMA_NOTE}")
if data.get("arm") != args.arm:
raise RunError(f"answers file is for arm {data.get('arm')!r}, not {args.arm!r}")
answers = {a["id"]: a for a in data["answers"] if isinstance(a, dict) and isinstance(a.get("id"), str)}
results = [score_one(q, answers.get(q["id"]), cache) for q in questions]
n = len(results)
missing = sum(1 for r in results if not r.answered)
print(f"set: {part}, {n} questions; arm: {args.arm}")
if fail_closed:
print(f"fail-closed: {fail_closed}")
print(f"missing answers: {missing}")
for t in ("whole", "count"):
of_type = [r for r in results if r.qtype == t]
print(f"{t}: correct {sum(r.correct for r in of_type)} of {len(of_type)}")
print(f"correct answer: {sum(r.answer_ok for r in results)} of {n}")
print(f"correct source: {sum(r.source_ok for r in results)} of {n}")
correct = sum(r.correct for r in results)
print(f"correct answer and source: {correct} of {n}")
if args.out:
write_json(Path(args.out), {
"arm": args.arm,
"set_sha256": set_hash,
"results": [{"id": r.qid, "type": r.qtype, "answered": r.answered, "answer_ok": r.answer_ok,
"source_ok": r.source_ok, "correct": r.correct} for r in results],
})
scored = not fail_closed and missing == 0
print("GREEN" if scored and n > 0 and correct == n else "RED")
print("SCORED" if scored else "UNSCORED (fail-closed)" if fail_closed else "INCOMPLETE")
return 0 if scored else 1
# ---------------------------------------------------------------------------------------------- main
def build_parser() -> argparse.ArgumentParser:
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
sub = p.add_subparsers(dest="command", required=True)
u = sub.add_parser("cut", help="compute the source cut and freeze the thresholds in a new manifest")
u.add_argument("--e2-set", required=True)
u.add_argument("--sources", required=True)
u.add_argument("--max-files", type=int, default=CUT_MAX_FILES)
u.add_argument("--manifest", required=True)
c = sub.add_parser("check", help="check every question's properties in the source files")
c.add_argument("--set", required=True)
c.add_argument("--sources", required=True)
c.add_argument("--manifest", required=True)
c.add_argument("--e2-set", required=True)
s = sub.add_parser("split", help="draw the held-out questions per type with a fixed seed")
s.add_argument("--set", required=True)
s.add_argument("--sources", required=True)
s.add_argument("--manifest", required=True)
s.add_argument("--e2-set", required=True)
s.add_argument("--seed", type=int, required=True)
s.add_argument("--held-out-per-type", type=int, default=2)
s.add_argument("--dev-out", required=True)
s.add_argument("--dev-label", default="", help="path of the dev set as written in the manifest")
s.add_argument("--held-out", required=True, help="file path OUTSIDE the repository")
g = sub.add_parser("gate", help="score one arm's answers against the answer key")
g.add_argument("--set", required=True)
g.add_argument("--manifest", required=True)
g.add_argument("--sources", required=True)
g.add_argument("--arm", required=True)
g.add_argument("--answers", required=True, help=ANSWERS_SCHEMA_NOTE)
g.add_argument("--out", default="", help="write per-question results (for paired tests)")
return p
def main(argv: list[str] | None = None) -> int:
args = build_parser().parse_args(argv)
commands = {"cut": cmd_cut, "check": cmd_check, "split": cmd_split, "gate": cmd_gate}
try:
return commands[args.command](args)
except RunError as exc:
print(f"FAIL: {exc}", file=sys.stderr)
return 1
if __name__ == "__main__":
sys.exit(main())