First public release, published as one snapshot. What changed between versions is in CHANGELOG.md. Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
544 lines
25 KiB
Python
544 lines
25 KiB
Python
"""The whole-document and counting group (E2b): cut, checker, held-out split and answer gate. No model call.
|
|
|
|
The E2 set asks only multi-document questions with short spans. This group asks the other class: questions whose
|
|
answer needs one long file read as a whole, or a count over many places in one file. It is reported on its own.
|
|
|
|
cut computes the source cut from the E2 dev set (sorted by id, repeatedly the question needing the fewest
|
|
NEW files, while the union stays within --max-files) and writes the manifest with the thresholds,
|
|
BEFORE any question is written. The thresholds are fixed from that moment.
|
|
check the properties of every question, measured in the source files: spans verbatim and unique, the file
|
|
in the cut; whole: spans from >= 3 separate parts covering the first to the last third; count: the
|
|
count equals the spans and the item pattern's matches, is >= the minimum and does not stand in the file.
|
|
split draws the held-out questions per type with a fixed seed, writes them OUTSIDE the repository,
|
|
renumbers both parts and records sha256 values and counts in the manifest.
|
|
gate scores one arm. whole: the answer holds every answer key; count: the answer's `count` is the exact
|
|
integer. Both: a citation quotes the file verbatim. A missing arm scores 0 (fail-closed).
|
|
|
|
Exit 0 = PASS / scored completely; 1 = FAIL or could not run. A check that cannot run is a FAIL.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import random
|
|
import re
|
|
import sys
|
|
from dataclasses import dataclass, field
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
from keyform import count_keys
|
|
from multidoc import (
|
|
NUMBER_WORDS,
|
|
RunError,
|
|
SourceCache,
|
|
as_list,
|
|
count_files,
|
|
inside_repo,
|
|
key_holds,
|
|
key_in,
|
|
load_json,
|
|
negated_in,
|
|
normalise_quote,
|
|
number_key,
|
|
require_dir,
|
|
sha256_bytes,
|
|
tree_sha256,
|
|
)
|
|
|
|
SET_SCHEMA = "okf-wiki/wholedoc-set/1"
|
|
MANIFEST_SCHEMA = "okf-wiki/wholedoc-manifest/1"
|
|
ANSWERS_SCHEMA_NOTE = '{"arm": "<name>", "answers": [{"id", "answer", "count" (count questions), "citations": [{"file", "quote"}]}]}'
|
|
TYPES = ("count", "whole")
|
|
CUT_MAX_FILES = 20
|
|
SPAN_MIN, SPAN_MAX = 20, 400
|
|
KEY_MIN, KEY_MAX = 3, 60
|
|
WHOLE_MIN_LINES = 150
|
|
WHOLE_MIN_KEYS = 3
|
|
WHOLE_MIN_PARTS = 3
|
|
PART_GAP = 0.15
|
|
FIRST_THIRD, LAST_THIRD = 1 / 3, 2 / 3
|
|
COUNT_MIN = 5
|
|
|
|
|
|
def thresholds() -> dict[str, Any]:
|
|
"""The thresholds, as frozen in the manifest before any question is written."""
|
|
return {
|
|
"span_chars": [SPAN_MIN, SPAN_MAX],
|
|
"key_chars": [KEY_MIN, KEY_MAX],
|
|
"span_occurrences": "exactly 1 in its file (so its position is defined)",
|
|
"whole_min_file_lines": WHOLE_MIN_LINES,
|
|
"whole_min_keys": WHOLE_MIN_KEYS,
|
|
"whole_min_parts": WHOLE_MIN_PARTS,
|
|
"whole_part_gap": PART_GAP,
|
|
"whole_part_rule": "spans sorted by start offset; a new part begins where the gap between two starts is "
|
|
">= whole_part_gap x file length (characters)",
|
|
"whole_coverage": "first span starts before 1/3 of the file, last span starts at or after 2/3",
|
|
"whole_one_span": "no single span holds every answer key (case-folded)",
|
|
"count_min": COUNT_MIN,
|
|
"count_rule": "count == number of evidence spans == matches of item_pattern in the file; every match starts "
|
|
"inside exactly one span",
|
|
"count_not_verbatim": "the count stands nowhere in the file as a number (not part of a longer number, version "
|
|
"or identifier) nor as an English number word",
|
|
"cut_max_files": CUT_MAX_FILES,
|
|
}
|
|
|
|
|
|
def number_stands_in(n: int, text: str) -> bool:
|
|
"""True when n stands as its own number or as its English word, not inside a version, identifier or number."""
|
|
if re.search(rf"(?<![\w.\-]){n}(?!\w|\.\d)", text):
|
|
return True
|
|
return n < len(NUMBER_WORDS) and re.search(rf"\b{NUMBER_WORDS[n]}\b", text, re.IGNORECASE) is not None
|
|
|
|
|
|
def load_set(path: Path) -> list[dict[str, Any]]:
|
|
data = load_json(path, "set")
|
|
if not isinstance(data, dict) or data.get("schema") != SET_SCHEMA or not isinstance(data.get("questions"), list):
|
|
raise RunError(f"{path} is not a set with schema {SET_SCHEMA}")
|
|
questions: list[dict[str, Any]] = data["questions"]
|
|
return questions
|
|
|
|
|
|
def dump_set(questions: list[dict[str, Any]]) -> bytes:
|
|
return (json.dumps({"schema": SET_SCHEMA, "questions": questions}, ensure_ascii=False, indent=1) + "\n").encode("utf-8")
|
|
|
|
|
|
def load_manifest(path: Path) -> dict[str, Any]:
|
|
manifest = load_json(path, "manifest")
|
|
if not isinstance(manifest, dict) or manifest.get("schema") != MANIFEST_SCHEMA:
|
|
raise RunError(f"{path} is not a manifest with schema {MANIFEST_SCHEMA}")
|
|
return manifest
|
|
|
|
|
|
def write_json(path: Path, data: Any) -> None:
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
path.write_text(json.dumps(data, indent=1) + "\n", encoding="utf-8")
|
|
|
|
|
|
# ---------------------------------------------------------------------------------------------- cut
|
|
|
|
|
|
def compute_cut(questions: list[dict[str, Any]], max_files: int) -> tuple[list[str], list[str]]:
|
|
"""Repeatedly take the question needing the fewest new files (ties: lowest id) while the union fits."""
|
|
files = {q["id"]: {e["file"] for e in as_list(q.get("evidence"))} for q in questions}
|
|
remaining = sorted(files)
|
|
union: set[str] = set()
|
|
chosen: list[str] = []
|
|
while remaining:
|
|
best = min(remaining, key=lambda i: (len(files[i] - union), i))
|
|
if len(union | files[best]) > max_files:
|
|
break
|
|
union |= files[best]
|
|
chosen.append(best)
|
|
remaining.remove(best)
|
|
return chosen, sorted(union)
|
|
|
|
|
|
def cut_from(e2_path: Path, max_files: int) -> dict[str, Any]:
|
|
data = load_json(e2_path, "E2 set")
|
|
if not isinstance(data, dict) or not isinstance(data.get("questions"), list):
|
|
raise RunError(f"{e2_path} is not an E2 set")
|
|
chosen, files = compute_cut(data["questions"], max_files)
|
|
return {
|
|
"rule": "E2 dev questions sorted by id; repeatedly the question needing the fewest NEW source files "
|
|
"(ties: lowest id), while the union stays <= max_files",
|
|
"e2_set_sha256": sha256_bytes(e2_path.read_bytes()),
|
|
"max_files": max_files,
|
|
"questions": len(chosen),
|
|
"files": len(files),
|
|
"names_sha256": sha256_bytes("".join(f + "\n" for f in files).encode("utf-8")),
|
|
"names": files,
|
|
}
|
|
|
|
|
|
def cmd_cut(args: argparse.Namespace) -> int:
|
|
sources, manifest_path = Path(args.sources), Path(args.manifest)
|
|
require_dir(sources, "sources")
|
|
if manifest_path.exists():
|
|
raise RunError(f"manifest {manifest_path} already exists; the thresholds are frozen once written")
|
|
cut = cut_from(Path(args.e2_set), args.max_files)
|
|
missing = [f for f in cut["names"] if not (sources / f).is_file()]
|
|
if missing:
|
|
raise RunError(f"cut files missing from the sources: {', '.join(missing)}")
|
|
write_json(manifest_path, {
|
|
"schema": MANIFEST_SCHEMA,
|
|
"thresholds": thresholds(),
|
|
"cut": cut,
|
|
"sources": {"files": count_files(sources), "tree_sha256": tree_sha256(sources)},
|
|
"seed": None,
|
|
"dev": None,
|
|
"held_out": None,
|
|
})
|
|
print(f"cut: {cut['files']} files from {cut['questions']} E2 questions (max {args.max_files})")
|
|
for name in cut["names"]:
|
|
print(f" {name}")
|
|
print(f"manifest: {manifest_path}")
|
|
return 0
|
|
|
|
|
|
# ---------------------------------------------------------------------------------------------- check
|
|
|
|
|
|
@dataclass
|
|
class CheckReport:
|
|
questions: int = 0
|
|
by_type: dict[str, int] = field(default_factory=lambda: {t: 0 for t in TYPES})
|
|
spans: int = 0
|
|
spans_ok: int = 0
|
|
in_cut: int = 0
|
|
whole_long: int = 0
|
|
whole_keyed: int = 0
|
|
whole_spread: int = 0
|
|
count_ok: int = 0
|
|
count_hidden: int = 0
|
|
not_one_span: int = 0
|
|
keys: int = 0
|
|
wording_keys: int = 0
|
|
negated_keys: int = 0
|
|
negated_ok: int = 0
|
|
errors: list[str] = field(default_factory=list)
|
|
|
|
@property
|
|
def passed(self) -> bool:
|
|
n, w, c = self.questions, self.by_type["whole"], self.by_type["count"]
|
|
return (n > 0 and not self.errors and self.spans_ok == self.spans and self.in_cut == n
|
|
and self.whole_long == w and self.whole_keyed == w and self.whole_spread == w
|
|
and self.count_ok == c and self.count_hidden == c and self.not_one_span == n
|
|
and self.wording_keys == 0 and self.negated_ok == self.negated_keys)
|
|
|
|
def lines(self) -> list[str]:
|
|
n, w, c = self.questions, self.by_type["whole"], self.by_type["count"]
|
|
return [f"error: {e}" for e in self.errors] + [
|
|
f"questions: {n} (whole {w}, count {c})",
|
|
f"spans verbatim and unique: {self.spans_ok} of {self.spans}",
|
|
f"questions whose file is in the cut: {self.in_cut} of {n}",
|
|
f"whole: file >= {WHOLE_MIN_LINES} lines: {self.whole_long} of {w}",
|
|
f"whole: every key in a span, every span keyed: {self.whole_keyed} of {w}",
|
|
f"whole: spread over >= {WHOLE_MIN_PARTS} parts, first to last third: {self.whole_spread} of {w}",
|
|
f"count: count = spans = pattern matches >= {COUNT_MIN}: {self.count_ok} of {c}",
|
|
f"count: answer not verbatim in the file: {self.count_hidden} of {c}",
|
|
f"not answerable from one span: {self.not_one_span} of {n}",
|
|
f"answer keys that are wording: {self.wording_keys} of {self.keys}",
|
|
f"negated keys negated in their span: {self.negated_ok} of {self.negated_keys}",
|
|
"PASS" if self.passed else "FAIL",
|
|
]
|
|
|
|
|
|
def parts(starts: list[int], length: int) -> int:
|
|
ordered = sorted(starts)
|
|
return 1 + sum(1 for a, b in zip(ordered, ordered[1:]) if b - a >= PART_GAP * length) if ordered else 0
|
|
|
|
|
|
def check_whole(q: dict[str, Any], text: str, spans: list[tuple[str, int]], rep: CheckReport) -> None:
|
|
qid = q["id"]
|
|
if text.count("\n") >= WHOLE_MIN_LINES:
|
|
rep.whole_long += 1
|
|
keys = [k.get("text") if isinstance(k, dict) else None for k in as_list(q.get("answer_keys"))]
|
|
if (not all(isinstance(k, str) and (number_key(k) is not None or KEY_MIN <= len(k.strip()) <= KEY_MAX) for k in keys)
|
|
or len(keys) < WHOLE_MIN_KEYS):
|
|
rep.errors.append(f"{qid}: needs >= {WHOLE_MIN_KEYS} answer keys of {KEY_MIN}-{KEY_MAX} characters")
|
|
return
|
|
texts = [str(k) for k in keys]
|
|
span_texts = [s for s, _ in spans]
|
|
if all(any(key_in(k, s) for s in span_texts) for k in texts) and all(any(key_in(k, s) for k in texts) for s in span_texts):
|
|
rep.whole_keyed += 1
|
|
starts = [pos for _, pos in spans if pos >= 0]
|
|
if (len(starts) == len(spans) and parts(starts, len(text)) >= WHOLE_MIN_PARTS
|
|
and min(starts) < FIRST_THIRD * len(text) and max(starts) >= LAST_THIRD * len(text)):
|
|
rep.whole_spread += 1
|
|
if not any(all(key_in(k, s) for k in texts) for s in span_texts):
|
|
rep.not_one_span += 1
|
|
for k in as_list(q.get("answer_keys")):
|
|
if "negated" not in k:
|
|
continue
|
|
if k["negated"] is not True:
|
|
rep.errors.append(f"{qid}: negated must be true or absent")
|
|
continue
|
|
rep.negated_keys += 1
|
|
rep.negated_ok += any(negated_in(k["text"], s) for s in span_texts)
|
|
|
|
|
|
def check_count(q: dict[str, Any], text: str, spans: list[tuple[str, int]], rep: CheckReport) -> None:
|
|
qid, count, pattern = q["id"], q.get("count"), q.get("item_pattern")
|
|
if type(count) is not int or not isinstance(pattern, str):
|
|
rep.errors.append(f"{qid}: a count question needs an integer count and an item_pattern")
|
|
return
|
|
try:
|
|
matches = [m.start() for m in re.finditer(pattern, text)]
|
|
except re.error as exc:
|
|
rep.errors.append(f"{qid}: item_pattern does not compile: {exc}")
|
|
return
|
|
ranges = [(pos, pos + len(s)) for s, pos in spans if pos >= 0]
|
|
each_in_one = all(sum(1 for a, b in ranges if a <= m < b) == 1 for m in matches)
|
|
if count >= COUNT_MIN and count == len(spans) == len(ranges) == len(matches) and each_in_one:
|
|
rep.count_ok += 1
|
|
if not number_stands_in(count, text):
|
|
rep.count_hidden += 1
|
|
if len(spans) >= 2 and not any(number_stands_in(count, s) for s, _ in spans):
|
|
rep.not_one_span += 1
|
|
|
|
|
|
def check_questions(questions: list[dict[str, Any]], sources: Path, cut_names: list[str]) -> CheckReport:
|
|
require_dir(sources, "sources")
|
|
cache = SourceCache(sources)
|
|
rep = CheckReport(questions=len(questions))
|
|
form = count_keys(questions)
|
|
rep.keys, rep.wording_keys = form.keys, form.wording
|
|
seen: set[str] = set()
|
|
for q in questions:
|
|
qid = q.get("id") if isinstance(q, dict) else None
|
|
if not isinstance(qid, str) or not qid:
|
|
rep.errors.append("question without an id")
|
|
continue
|
|
if qid in seen:
|
|
rep.errors.append(f"duplicate id {qid}")
|
|
seen.add(qid)
|
|
qtype = q.get("type")
|
|
if qtype not in TYPES:
|
|
rep.errors.append(f"{qid}: type {qtype!r} is not one of {TYPES}")
|
|
continue
|
|
rep.by_type[qtype] += 1
|
|
if not isinstance(q.get("question"), str) or not q["question"].strip():
|
|
rep.errors.append(f"{qid}: empty question")
|
|
name = q.get("file")
|
|
if name in cut_names:
|
|
rep.in_cut += 1
|
|
else:
|
|
rep.errors.append(f"{qid}: file {name!r} is not in the cut")
|
|
text = cache.text(name) if isinstance(name, str) else None
|
|
if text is None:
|
|
rep.errors.append(f"{qid}: file {name!r} is not a readable file in the sources")
|
|
text = ""
|
|
spans: list[tuple[str, int]] = []
|
|
for e in as_list(q.get("evidence")):
|
|
span = e.get("span") if isinstance(e, dict) else None
|
|
rep.spans += 1
|
|
if not isinstance(span, str):
|
|
rep.errors.append(f"{qid}: evidence span is not text")
|
|
continue
|
|
if not SPAN_MIN <= len(span) <= SPAN_MAX:
|
|
rep.errors.append(f"{qid}: span length {len(span)} outside {SPAN_MIN}-{SPAN_MAX}")
|
|
unique = text.count(span) == 1
|
|
rep.spans_ok += unique
|
|
spans.append((span, text.find(span) if unique else -1))
|
|
if qtype == "whole":
|
|
check_whole(q, text, spans, rep)
|
|
else:
|
|
check_count(q, text, spans, rep)
|
|
return rep
|
|
|
|
|
|
def verified_cut(manifest: dict[str, Any], e2_path: Path) -> list[str]:
|
|
cut = manifest.get("cut")
|
|
if not isinstance(cut, dict):
|
|
raise RunError("manifest has no cut")
|
|
fresh = cut_from(e2_path, int(cut.get("max_files", CUT_MAX_FILES)))
|
|
for key in ("e2_set_sha256", "names_sha256", "files"):
|
|
if fresh[key] != cut.get(key):
|
|
raise RunError(f"the cut from {e2_path} differs from the manifest ({key}); the group is frozen against it")
|
|
names: list[str] = fresh["names"]
|
|
return names
|
|
|
|
|
|
def cmd_check(args: argparse.Namespace) -> int:
|
|
manifest = load_manifest(Path(args.manifest))
|
|
cut_names = verified_cut(manifest, Path(args.e2_set))
|
|
rep = check_questions(load_set(Path(args.set)), Path(args.sources), cut_names)
|
|
print("\n".join(rep.lines()))
|
|
return 0 if rep.passed else 1
|
|
|
|
|
|
# ---------------------------------------------------------------------------------------------- split
|
|
|
|
|
|
def cmd_split(args: argparse.Namespace) -> int:
|
|
sources, held_path, manifest_path = Path(args.sources), Path(args.held_out), Path(args.manifest)
|
|
dev_path = Path(args.dev_out)
|
|
if inside_repo(held_path):
|
|
raise RunError(f"held-out path {held_path} is inside the repository; it must live outside it")
|
|
if held_path.exists():
|
|
raise RunError(f"held-out file {held_path} already exists; a frozen draw is never redrawn")
|
|
manifest = load_manifest(manifest_path)
|
|
if manifest.get("dev") is not None or manifest.get("held_out") is not None:
|
|
raise RunError(f"manifest {manifest_path} already records a draw; a frozen draw is never redrawn")
|
|
if manifest.get("thresholds") != thresholds():
|
|
raise RunError("the manifest's thresholds differ from the code; they were frozen before the questions")
|
|
if tree_sha256(sources) != manifest.get("sources", {}).get("tree_sha256"):
|
|
raise RunError(f"sources {sources} differ from the manifest's tree_sha256")
|
|
questions = load_set(Path(args.set))
|
|
rep = check_questions(questions, sources, verified_cut(manifest, Path(args.e2_set)))
|
|
if not rep.passed:
|
|
print("\n".join(rep.lines()))
|
|
raise RunError("the set fails the check; nothing was split")
|
|
k = args.held_out_per_type
|
|
rng = random.Random(args.seed)
|
|
held_ids: set[str] = set()
|
|
for t in TYPES:
|
|
ids = sorted(q["id"] for q in questions if q["type"] == t)
|
|
if not 0 < k < len(ids):
|
|
raise RunError(f"cannot hold out {k} of {len(ids)} {t} questions")
|
|
held_ids.update(rng.sample(ids, k))
|
|
by_id = {q["id"]: q for q in questions}
|
|
ids = sorted(by_id)
|
|
dev = [dict(by_id[i], id=f"d{n:02d}") for n, i in enumerate((i for i in ids if i not in held_ids), 1)]
|
|
held = [dict(by_id[i], id=f"h{n:02d}") for n, i in enumerate(sorted(held_ids), 1)]
|
|
dev_bytes, held_bytes = dump_set(dev), dump_set(held)
|
|
held_path.parent.mkdir(parents=True, exist_ok=True)
|
|
held_path.write_bytes(held_bytes)
|
|
dev_path.parent.mkdir(parents=True, exist_ok=True)
|
|
dev_path.write_bytes(dev_bytes)
|
|
|
|
def per_type(part: list[dict[str, Any]]) -> dict[str, int]:
|
|
return {t: sum(1 for q in part if q["type"] == t) for t in ("whole", "count")}
|
|
|
|
manifest["seed"] = args.seed
|
|
manifest["draw"] = ("rng = random.Random(seed); for type in ('count', 'whole'): rng.sample(sorted(ids of type), "
|
|
"held_out_per_type); both parts renumbered d../h.. in sorted original-id order")
|
|
manifest["dev"] = {"file": args.dev_label or dev_path.name, "sha256": sha256_bytes(dev_bytes),
|
|
"questions": len(dev), **per_type(dev)}
|
|
manifest["held_out"] = {"sha256": sha256_bytes(held_bytes), "questions": len(held), **per_type(held)}
|
|
write_json(manifest_path, manifest)
|
|
print(f"seed: {args.seed}")
|
|
print(f"dev: {len(dev)} questions {per_type(dev)}, sha256 {manifest['dev']['sha256']}")
|
|
print(f"held out: {len(held)} questions {per_type(held)}, sha256 {manifest['held_out']['sha256']}")
|
|
print(f"manifest: {manifest_path}")
|
|
return 0
|
|
|
|
|
|
# ---------------------------------------------------------------------------------------------- gate
|
|
|
|
|
|
@dataclass
|
|
class GateResult:
|
|
qid: str
|
|
qtype: str
|
|
answered: bool
|
|
answer_ok: bool
|
|
source_ok: bool
|
|
|
|
@property
|
|
def correct(self) -> bool:
|
|
return self.answer_ok and self.source_ok
|
|
|
|
|
|
def score_one(q: dict[str, Any], a: dict[str, Any] | None, cache: SourceCache) -> GateResult:
|
|
if a is None:
|
|
return GateResult(q["id"], q["type"], False, False, False)
|
|
if q["type"] == "count":
|
|
given = a.get("count")
|
|
answer_ok = type(given) is int and given == q["count"]
|
|
else:
|
|
answer = a["answer"] if isinstance(a.get("answer"), str) else ""
|
|
answer_ok = all(key_holds(k, answer) for k in q["answer_keys"])
|
|
text = cache.text(q["file"])
|
|
source_ok = False
|
|
for c in as_list(a.get("citations")):
|
|
name, quote = (c.get("file"), c.get("quote")) if isinstance(c, dict) else (None, None)
|
|
if name == q["file"] and text is not None and isinstance(quote, str) and quote.strip():
|
|
source_ok = source_ok or normalise_quote(quote) in normalise_quote(text)
|
|
return GateResult(q["id"], q["type"], True, answer_ok, source_ok)
|
|
|
|
|
|
def cmd_gate(args: argparse.Namespace) -> int:
|
|
set_path, sources = Path(args.set), Path(args.sources)
|
|
require_dir(sources, "sources")
|
|
manifest = load_manifest(Path(args.manifest))
|
|
dev, held = manifest.get("dev"), manifest.get("held_out")
|
|
if not isinstance(dev, dict) or not isinstance(held, dict):
|
|
raise RunError("the manifest records no draw yet; run split first")
|
|
set_hash = sha256_bytes(set_path.read_bytes()) if set_path.is_file() else ""
|
|
part = {dev.get("sha256"): "dev", held.get("sha256"): "held_out"}.get(set_hash)
|
|
if part is None:
|
|
raise RunError(f"set {set_path} sha256 {set_hash or '(missing)'} matches neither part of the manifest")
|
|
if tree_sha256(sources) != manifest.get("sources", {}).get("tree_sha256"):
|
|
raise RunError(f"sources {sources} differ from the manifest's tree_sha256; the set is frozen against them")
|
|
questions = load_set(set_path)
|
|
cache = SourceCache(sources)
|
|
answers: dict[str, dict[str, Any]] = {}
|
|
fail_closed = ""
|
|
answers_path = Path(args.answers)
|
|
if not answers_path.is_file():
|
|
fail_closed = f"no answers for arm {args.arm} at {answers_path}"
|
|
else:
|
|
data = load_json(answers_path, "answers")
|
|
if not isinstance(data, dict) or not isinstance(data.get("answers"), list):
|
|
raise RunError(f"{answers_path} is not an answers file {ANSWERS_SCHEMA_NOTE}")
|
|
if data.get("arm") != args.arm:
|
|
raise RunError(f"answers file is for arm {data.get('arm')!r}, not {args.arm!r}")
|
|
answers = {a["id"]: a for a in data["answers"] if isinstance(a, dict) and isinstance(a.get("id"), str)}
|
|
results = [score_one(q, answers.get(q["id"]), cache) for q in questions]
|
|
n = len(results)
|
|
missing = sum(1 for r in results if not r.answered)
|
|
print(f"set: {part}, {n} questions; arm: {args.arm}")
|
|
if fail_closed:
|
|
print(f"fail-closed: {fail_closed}")
|
|
print(f"missing answers: {missing}")
|
|
for t in ("whole", "count"):
|
|
of_type = [r for r in results if r.qtype == t]
|
|
print(f"{t}: correct {sum(r.correct for r in of_type)} of {len(of_type)}")
|
|
print(f"correct answer: {sum(r.answer_ok for r in results)} of {n}")
|
|
print(f"correct source: {sum(r.source_ok for r in results)} of {n}")
|
|
correct = sum(r.correct for r in results)
|
|
print(f"correct answer and source: {correct} of {n}")
|
|
if args.out:
|
|
write_json(Path(args.out), {
|
|
"arm": args.arm,
|
|
"set_sha256": set_hash,
|
|
"results": [{"id": r.qid, "type": r.qtype, "answered": r.answered, "answer_ok": r.answer_ok,
|
|
"source_ok": r.source_ok, "correct": r.correct} for r in results],
|
|
})
|
|
scored = not fail_closed and missing == 0
|
|
print("GREEN" if scored and n > 0 and correct == n else "RED")
|
|
print("SCORED" if scored else "UNSCORED (fail-closed)" if fail_closed else "INCOMPLETE")
|
|
return 0 if scored else 1
|
|
|
|
|
|
# ---------------------------------------------------------------------------------------------- main
|
|
|
|
|
|
def build_parser() -> argparse.ArgumentParser:
|
|
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
sub = p.add_subparsers(dest="command", required=True)
|
|
u = sub.add_parser("cut", help="compute the source cut and freeze the thresholds in a new manifest")
|
|
u.add_argument("--e2-set", required=True)
|
|
u.add_argument("--sources", required=True)
|
|
u.add_argument("--max-files", type=int, default=CUT_MAX_FILES)
|
|
u.add_argument("--manifest", required=True)
|
|
c = sub.add_parser("check", help="check every question's properties in the source files")
|
|
c.add_argument("--set", required=True)
|
|
c.add_argument("--sources", required=True)
|
|
c.add_argument("--manifest", required=True)
|
|
c.add_argument("--e2-set", required=True)
|
|
s = sub.add_parser("split", help="draw the held-out questions per type with a fixed seed")
|
|
s.add_argument("--set", required=True)
|
|
s.add_argument("--sources", required=True)
|
|
s.add_argument("--manifest", required=True)
|
|
s.add_argument("--e2-set", required=True)
|
|
s.add_argument("--seed", type=int, required=True)
|
|
s.add_argument("--held-out-per-type", type=int, default=2)
|
|
s.add_argument("--dev-out", required=True)
|
|
s.add_argument("--dev-label", default="", help="path of the dev set as written in the manifest")
|
|
s.add_argument("--held-out", required=True, help="file path OUTSIDE the repository")
|
|
g = sub.add_parser("gate", help="score one arm's answers against the answer key")
|
|
g.add_argument("--set", required=True)
|
|
g.add_argument("--manifest", required=True)
|
|
g.add_argument("--sources", required=True)
|
|
g.add_argument("--arm", required=True)
|
|
g.add_argument("--answers", required=True, help=ANSWERS_SCHEMA_NOTE)
|
|
g.add_argument("--out", default="", help="write per-question results (for paired tests)")
|
|
return p
|
|
|
|
|
|
def main(argv: list[str] | None = None) -> int:
|
|
args = build_parser().parse_args(argv)
|
|
commands = {"cut": cmd_cut, "check": cmd_check, "split": cmd_split, "gate": cmd_gate}
|
|
try:
|
|
return commands[args.command](args)
|
|
except RunError as exc:
|
|
print(f"FAIL: {exc}", file=sys.stderr)
|
|
return 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|