okf-wiki/gates/multidoc.py
Kjell Tore Guttormsen 0fd86a23a3
okf-wiki 1.1.1
First public release, published as one snapshot. What changed between
versions is in CHANGELOG.md.

Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
2026-11-14 21:03:20 +01:00

492 lines
21 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

"""The multi-document set for PLAN.md criterion 4: checker, held-out split and answer gate. No model call.
check every answer span stands verbatim in its source file, every question needs >= 4 distinct files,
and every file carries an answer key that stands inside that file's spans. The counts printed
here are the denominators; they come from the set, not from whoever wrote it.
split draws the held-out questions with a fixed seed, writes them OUTSIDE the repository, renumbers
both parts, and writes a manifest with sha256 only (nothing about the held-out questions).
gate scores one arm's answers: a question counts when the answer holds every answer key AND the
citations quote, verbatim, every file the answer key needs. A missing arm scores 0 (fail-closed).
Exit 0 = PASS / scored completely; 1 = FAIL or could not run. A check that cannot run is a FAIL.
"""
from __future__ import annotations
import argparse
import hashlib
import json
import random
import re
import sys
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any
from keyform import count_keys
SET_SCHEMA = "okf-wiki/multidoc-set/1"
MANIFEST_SCHEMA = "okf-wiki/multidoc-manifest/1"
ANSWERS_SCHEMA_NOTE = '{"arm": "<name>", "answers": [{"id", "answer", "citations": [{"file", "quote"}]}]}'
MIN_FILES = 4
SPAN_MIN, SPAN_MAX = 20, 400
KEY_MIN, KEY_MAX = 3, 60
NUMBER_WORDS = ("zero one two three four five six seven eight nine ten eleven twelve thirteen fourteen fifteen "
"sixteen seventeen eighteen nineteen twenty").split()
NEGATION_WORDS = frozenset("not no never none neither nor without cannot".split())
NEGATION_WINDOW = 6
REPO_ROOT = Path(__file__).resolve().parents[1]
class RunError(Exception):
"""The command cannot run; reported as FAIL with exit 1."""
# ---------------------------------------------------------------------------------------------- helpers
def sha256_bytes(data: bytes) -> str:
return hashlib.sha256(data).hexdigest()
def tree_sha256(sources: Path) -> str:
"""One hash over the flat sources directory: sorted "name NUL sha256 LF" lines."""
lines = []
for p in sorted(sources.iterdir(), key=lambda p: p.name):
if p.is_file():
lines.append(f"{p.name}\0{sha256_bytes(p.read_bytes())}\n")
return sha256_bytes("".join(lines).encode("utf-8"))
def count_files(sources: Path) -> int:
return sum(1 for p in sources.iterdir() if p.is_file())
def normalise_answer(text: str) -> str:
"""Case-folded, without backticks and asterisks, whitespace collapsed. Used for answer keys only."""
return re.sub(r"\s+", " ", text.replace("`", "").replace("*", "")).strip().casefold()
def number_key(key: str) -> tuple[int, bool] | None:
"""(n, is_percentage) when the key is a whole number, a percentage or an English number word; otherwise None."""
text = normalise_answer(key)
m = re.fullmatch(r"(\d+) ?(%?)", text)
if m:
return int(m.group(1)), m.group(2) == "%"
return (NUMBER_WORDS.index(text), False) if text in NUMBER_WORDS else None
def key_positions(key: str, norm: str) -> list[int]:
"""Start offsets of the answer key in normalised text. A number key matches as a number: its digits or its
English word, never inside a longer number, version or identifier. Any other key matches as normalised text."""
num = number_key(key)
if num is None:
needle = normalise_answer(key)
return [m.start() for m in re.finditer(re.escape(needle), norm)] if needle else []
n, percent = num
if percent:
return [m.start() for m in re.finditer(rf"(?<![\w.]){n} ?(?:%|percent\b)", norm)]
forms = [str(n)] + ([NUMBER_WORDS[n]] if n < len(NUMBER_WORDS) else [])
return [m.start() for m in re.finditer(rf"(?<![\w.])(?:{'|'.join(forms)})(?![\w%]|\.\d)", norm)]
def key_in(key: str, text: str) -> bool:
"""True when the answer key stands in the text (see key_positions)."""
return bool(key_positions(key, normalise_answer(text)))
def is_negation(word: str) -> bool:
"""True when the word is a negation: one of NEGATION_WORDS or a word ending in n't."""
w = word.strip(".,;:!?()[]{}\"'").replace("’", "'").casefold()
return w in NEGATION_WORDS or w.endswith("n't")
def sentences(text: str) -> list[str]:
"""The text split at line breaks and at a full stop, question mark, exclamation mark or semicolon ending a word."""
return [s for s in re.split(r"\n|[.!?;](?=\s|$)", text) if s.strip()]
def negated_at(norm: str, pos: int) -> bool:
"""True when one of the NEGATION_WINDOW words before offset pos in a normalised sentence is a negation."""
return any(is_negation(w) for w in norm[:pos].split()[-NEGATION_WINDOW:])
def negated_in(key: str, text: str) -> bool:
"""True when, within one sentence of the text, the key stands with a negation at most NEGATION_WINDOW words
before it."""
for s in sentences(text):
norm = normalise_answer(s)
if any(negated_at(norm, pos) for pos in key_positions(key, norm)):
return True
return False
def key_holds(key: dict[str, Any], text: str) -> bool:
"""The gates' one test of an answer key against an answer. A key marked "negated": true holds only where it
stands negated (negated_in); any other key holds where it stands (key_in), with or without a negation."""
if key.get("negated") is True:
return negated_in(key["text"], text)
return key_in(key["text"], text)
def normalise_quote(text: str) -> str:
"""Whitespace collapsed, otherwise exact. Used for an arm's citations."""
return re.sub(r"\s+", " ", text).strip()
def as_list(value: object) -> list[Any]:
return value if isinstance(value, list) else []
def is_plain_name(name: object) -> bool:
return isinstance(name, str) and name != "" and "/" not in name and "\\" not in name and name not in (".", "..")
def load_json(path: Path, what: str) -> Any:
try:
return json.loads(path.read_text(encoding="utf-8"))
except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc:
raise RunError(f"cannot read {what} {path}: {exc}") from exc
def load_questions(path: Path) -> list[dict[str, Any]]:
data = load_json(path, "set")
if not isinstance(data, dict) or data.get("schema") != SET_SCHEMA or not isinstance(data.get("questions"), list):
raise RunError(f"{path} is not a set with schema {SET_SCHEMA}")
questions: list[dict[str, Any]] = data["questions"]
return questions
def require_dir(path: Path, what: str) -> None:
if not path.is_dir():
raise RunError(f"{what} {path} is not a directory")
class SourceCache:
def __init__(self, root: Path) -> None:
self.root = root
self._text: dict[str, str | None] = {}
def text(self, name: str) -> str | None:
"""File text, or None when the name is not a plain file directly in the sources directory."""
if not is_plain_name(name):
return None
if name not in self._text:
p = self.root / name
try:
self._text[name] = p.read_text(encoding="utf-8") if p.is_file() else None
except (OSError, UnicodeDecodeError):
self._text[name] = None
return self._text[name]
# ---------------------------------------------------------------------------------------------- check
@dataclass
class CheckReport:
questions: int = 0
spans: int = 0
spans_verbatim: int = 0
all_spans_verbatim: int = 0
min_files: int = 0
key_per_file: int = 0
keys: int = 0
wording_keys: int = 0
negated_keys: int = 0
negated_ok: int = 0
errors: list[str] = field(default_factory=list)
files_per_question: dict[str, int] = field(default_factory=dict)
@property
def passed(self) -> bool:
n = self.questions
return (
n > 0
and not self.errors
and self.spans_verbatim == self.spans
and self.all_spans_verbatim == n
and self.min_files == n
and self.key_per_file == n
and self.wording_keys == 0
and self.negated_ok == self.negated_keys
)
def lines(self) -> list[str]:
n = self.questions
out = [f"error: {e}" for e in self.errors]
out += [
f"questions: {n}",
f"spans verbatim: {self.spans_verbatim} of {self.spans}",
f"questions with every span verbatim: {self.all_spans_verbatim} of {n}",
f"questions with >= {MIN_FILES} distinct files: {self.min_files} of {n}",
f"questions with a key per file: {self.key_per_file} of {n}",
f"answer keys that are wording: {self.wording_keys} of {self.keys}",
f"negated keys negated in their span: {self.negated_ok} of {self.negated_keys}",
"PASS" if self.passed else "FAIL",
]
return out
def check_questions(questions: list[dict[str, Any]], sources: Path) -> CheckReport:
require_dir(sources, "sources")
cache = SourceCache(sources)
rep = CheckReport(questions=len(questions))
form = count_keys(questions)
rep.keys, rep.wording_keys = form.keys, form.wording
seen: set[str] = set()
for q in questions:
qid = q.get("id") if isinstance(q, dict) else None
if not isinstance(qid, str) or not qid:
rep.errors.append("question without an id")
continue
if qid in seen:
rep.errors.append(f"duplicate id {qid}")
seen.add(qid)
if not isinstance(q.get("question"), str) or not q["question"].strip():
rep.errors.append(f"{qid}: empty question")
evidence = as_list(q.get("evidence"))
keys = as_list(q.get("answer_keys"))
spans_by_file: dict[str, list[str]] = {}
verbatim = 0
for e in evidence:
name, span = (e.get("file"), e.get("span")) if isinstance(e, dict) else (None, None)
rep.spans += 1
if not isinstance(name, str) or not is_plain_name(name):
rep.errors.append(f"{qid}: evidence file {name!r} is not a plain file name")
continue
if not isinstance(span, str):
rep.errors.append(f"{qid}: evidence span in {name} is not text")
continue
if not SPAN_MIN <= len(span) <= SPAN_MAX:
rep.errors.append(f"{qid}: span length {len(span)} in {name} outside {SPAN_MIN}-{SPAN_MAX}")
text = cache.text(name)
if text is not None and span in text:
verbatim += 1
spans_by_file.setdefault(name, []).append(span)
rep.spans_verbatim += verbatim
if evidence and verbatim == len(evidence):
rep.all_spans_verbatim += 1
files = set(spans_by_file)
rep.files_per_question[qid] = len(files)
if len(files) >= MIN_FILES:
rep.min_files += 1
keyed: set[str] = set()
keys_ok = bool(keys)
for k in keys:
name, key = (k.get("file"), k.get("text")) if isinstance(k, dict) else (None, None)
if not isinstance(key, str) or not isinstance(name, str):
keys_ok = False
rep.errors.append(f"{qid}: malformed answer key")
continue
if number_key(key) is None and not KEY_MIN <= len(key.strip()) <= KEY_MAX:
rep.errors.append(f"{qid}: key length {len(key.strip())} outside {KEY_MIN}-{KEY_MAX}")
if any(key_in(key, s) for s in spans_by_file.get(name, [])):
keyed.add(name)
else:
keys_ok = False
if "negated" in k:
if k["negated"] is not True:
rep.errors.append(f"{qid}: negated must be true or absent")
continue
rep.negated_keys += 1
rep.negated_ok += any(negated_in(key, s) for s in spans_by_file.get(name, []))
if keys_ok and files and keyed == files:
rep.key_per_file += 1
return rep
def cmd_check(args: argparse.Namespace) -> int:
rep = check_questions(load_questions(Path(args.set)), Path(args.sources))
if args.per_question:
for qid, n in rep.files_per_question.items():
print(f"{qid}\t{n} files")
print("\n".join(rep.lines()))
return 0 if rep.passed else 1
# ---------------------------------------------------------------------------------------------- split
def inside_repo(path: Path) -> bool:
try:
path.resolve().relative_to(REPO_ROOT)
return True
except ValueError:
return False
def dump_set(questions: list[dict[str, Any]]) -> bytes:
return (json.dumps({"schema": SET_SCHEMA, "questions": questions}, ensure_ascii=False, indent=1) + "\n").encode("utf-8")
def cmd_split(args: argparse.Namespace) -> int:
sources, held_path = Path(args.sources), Path(args.held_out)
dev_path, manifest_path = Path(args.dev_out), Path(args.manifest)
if inside_repo(held_path):
raise RunError(f"held-out path {held_path} is inside the repository; it must live outside it")
if held_path.exists():
raise RunError(f"held-out file {held_path} already exists; a frozen draw is never redrawn")
questions = load_questions(Path(args.set))
rep = check_questions(questions, sources)
if not rep.passed:
print("\n".join(rep.lines()))
raise RunError("the set fails the check; nothing was split")
n_held = args.held_out_count
if not 0 < n_held < len(questions):
raise RunError(f"cannot hold out {n_held} of {len(questions)}")
ids = sorted(q["id"] for q in questions)
held_ids = set(random.Random(args.seed).sample(ids, n_held))
by_id = {q["id"]: q for q in questions}
dev = [dict(by_id[i], id=f"d{n:02d}") for n, i in enumerate((i for i in ids if i not in held_ids), 1)]
held = [dict(by_id[i], id=f"h{n:02d}") for n, i in enumerate(sorted(held_ids), 1)]
dev_bytes, held_bytes = dump_set(dev), dump_set(held)
held_path.parent.mkdir(parents=True, exist_ok=True)
held_path.write_bytes(held_bytes)
dev_path.parent.mkdir(parents=True, exist_ok=True)
dev_path.write_bytes(dev_bytes)
manifest = {
"schema": MANIFEST_SCHEMA,
"seed": args.seed,
"draw": "random.Random(seed).sample(sorted(ids), held_out.questions); both parts renumbered d../h..",
"dev": {"file": args.dev_label or dev_path.name, "sha256": sha256_bytes(dev_bytes), "questions": len(dev)},
"held_out": {"sha256": sha256_bytes(held_bytes), "questions": len(held)},
"sources": {"files": count_files(sources), "tree_sha256": tree_sha256(sources)},
}
manifest_path.parent.mkdir(parents=True, exist_ok=True)
manifest_path.write_text(json.dumps(manifest, indent=1) + "\n", encoding="utf-8")
print(f"seed: {args.seed}")
print(f"dev: {len(dev)} questions, sha256 {manifest['dev']['sha256']}")
print(f"held out: {len(held)} questions, sha256 {manifest['held_out']['sha256']}")
print(f"manifest: {manifest_path}")
return 0
# ---------------------------------------------------------------------------------------------- gate
@dataclass
class GateResult:
qid: str
answered: bool
answer_ok: bool
source_ok: bool
@property
def correct(self) -> bool:
return self.answer_ok and self.source_ok
def score_one(q: dict[str, Any], a: dict[str, Any] | None, cache: SourceCache) -> GateResult:
if a is None:
return GateResult(q["id"], False, False, False)
answer = a.get("answer", "") if isinstance(a.get("answer"), str) else ""
answer_ok = all(key_holds(k, answer) for k in q["answer_keys"])
supported: set[str] = set()
citations = as_list(a.get("citations"))
for c in citations:
name, quote = (c.get("file"), c.get("quote")) if isinstance(c, dict) else (None, None)
if not isinstance(name, str) or not isinstance(quote, str) or not quote.strip():
continue
text = cache.text(name)
if text is not None and normalise_quote(quote) in normalise_quote(text):
supported.add(name)
needed = {e["file"] for e in q["evidence"]}
return GateResult(q["id"], True, answer_ok, needed <= supported)
def cmd_gate(args: argparse.Namespace) -> int:
set_path, sources, manifest_path = Path(args.set), Path(args.sources), Path(args.manifest)
require_dir(sources, "sources")
manifest = load_json(manifest_path, "manifest")
if not isinstance(manifest, dict) or manifest.get("schema") != MANIFEST_SCHEMA:
raise RunError(f"{manifest_path} is not a manifest with schema {MANIFEST_SCHEMA}")
set_hash = sha256_bytes(set_path.read_bytes()) if set_path.is_file() else ""
known = {manifest.get("dev", {}).get("sha256"): "dev", manifest.get("held_out", {}).get("sha256"): "held_out"}
part = known.get(set_hash)
if part is None:
raise RunError(f"set {set_path} sha256 {set_hash or '(missing)'} matches neither part of the manifest")
if tree_sha256(sources) != manifest.get("sources", {}).get("tree_sha256"):
raise RunError(f"sources {sources} differ from the manifest's tree_sha256; the set is frozen against them")
questions = load_questions(set_path)
n = len(questions)
cache = SourceCache(sources)
answers: dict[str, dict[str, Any]] = {}
fail_closed = ""
answers_path = Path(args.answers)
if not answers_path.is_file():
fail_closed = f"no answers for arm {args.arm} at {answers_path}"
else:
data = load_json(answers_path, "answers")
if not isinstance(data, dict) or not isinstance(data.get("answers"), list):
raise RunError(f"{answers_path} is not an answers file {ANSWERS_SCHEMA_NOTE}")
if data.get("arm") != args.arm:
raise RunError(f"answers file is for arm {data.get('arm')!r}, not {args.arm!r}")
for a in data["answers"]:
if isinstance(a, dict) and isinstance(a.get("id"), str):
answers[a["id"]] = a
results = [score_one(q, answers.get(q["id"]), cache) for q in questions]
missing = sum(1 for r in results if not r.answered)
print(f"set: {part}, {n} questions; arm: {args.arm}")
if fail_closed:
print(f"fail-closed: {fail_closed}")
print(f"missing answers: {missing}")
print(f"correct answer: {sum(r.answer_ok for r in results)} of {n}")
print(f"correct source: {sum(r.source_ok for r in results)} of {n}")
print(f"correct answer and source: {sum(r.correct for r in results)} of {n}")
if args.out:
out = {
"arm": args.arm,
"set_sha256": set_hash,
"results": [
{"id": r.qid, "answered": r.answered, "answer_ok": r.answer_ok, "source_ok": r.source_ok, "correct": r.correct}
for r in results
],
}
Path(args.out).write_text(json.dumps(out, indent=1) + "\n", encoding="utf-8")
scored = not fail_closed and missing == 0
print("SCORED" if scored else "UNSCORED (fail-closed)" if fail_closed else "INCOMPLETE")
return 0 if scored else 1
# ---------------------------------------------------------------------------------------------- main
def build_parser() -> argparse.ArgumentParser:
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
sub = p.add_subparsers(dest="command", required=True)
c = sub.add_parser("check", help="check spans, file counts and answer keys")
c.add_argument("--set", required=True)
c.add_argument("--sources", required=True)
c.add_argument("--per-question", action="store_true", help="also list distinct files per question")
s = sub.add_parser("split", help="draw the held-out questions with a fixed seed")
s.add_argument("--set", required=True)
s.add_argument("--sources", required=True)
s.add_argument("--seed", type=int, required=True)
s.add_argument("--held-out-count", type=int, default=20)
s.add_argument("--dev-out", required=True)
s.add_argument("--dev-label", default="", help="path of the dev set as written in the manifest")
s.add_argument("--held-out", required=True, help="file path OUTSIDE the repository")
s.add_argument("--manifest", required=True)
g = sub.add_parser("gate", help="score one arm's answers against the answer key")
g.add_argument("--set", required=True)
g.add_argument("--manifest", required=True)
g.add_argument("--sources", required=True)
g.add_argument("--arm", required=True)
g.add_argument("--answers", required=True, help=ANSWERS_SCHEMA_NOTE)
g.add_argument("--out", default="", help="write per-question results (for paired tests)")
return p
def main(argv: list[str] | None = None) -> int:
args = build_parser().parse_args(argv)
commands = {"check": cmd_check, "split": cmd_split, "gate": cmd_gate}
try:
return commands[args.command](args)
except RunError as exc:
print(f"FAIL: {exc}", file=sys.stderr)
return 1
if __name__ == "__main__":
sys.exit(main())