okf-wiki 1.1.1
First public release, published as one snapshot. What changed between versions is in CHANGELOG.md. Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
This commit is contained in:
commit
0fd86a23a3
227 changed files with 96077 additions and 0 deletions
544
gates/wholedoc.py
Normal file
544
gates/wholedoc.py
Normal file
|
|
@ -0,0 +1,544 @@
|
|||
"""The whole-document and counting group (E2b): cut, checker, held-out split and answer gate. No model call.
|
||||
|
||||
The E2 set asks only multi-document questions with short spans. This group asks the other class: questions whose
|
||||
answer needs one long file read as a whole, or a count over many places in one file. It is reported on its own.
|
||||
|
||||
cut computes the source cut from the E2 dev set (sorted by id, repeatedly the question needing the fewest
|
||||
NEW files, while the union stays within --max-files) and writes the manifest with the thresholds,
|
||||
BEFORE any question is written. The thresholds are fixed from that moment.
|
||||
check the properties of every question, measured in the source files: spans verbatim and unique, the file
|
||||
in the cut; whole: spans from >= 3 separate parts covering the first to the last third; count: the
|
||||
count equals the spans and the item pattern's matches, is >= the minimum and does not stand in the file.
|
||||
split draws the held-out questions per type with a fixed seed, writes them OUTSIDE the repository,
|
||||
renumbers both parts and records sha256 values and counts in the manifest.
|
||||
gate scores one arm. whole: the answer holds every answer key; count: the answer's `count` is the exact
|
||||
integer. Both: a citation quotes the file verbatim. A missing arm scores 0 (fail-closed).
|
||||
|
||||
Exit 0 = PASS / scored completely; 1 = FAIL or could not run. A check that cannot run is a FAIL.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import random
|
||||
import re
|
||||
import sys
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from keyform import count_keys
|
||||
from multidoc import (
|
||||
NUMBER_WORDS,
|
||||
RunError,
|
||||
SourceCache,
|
||||
as_list,
|
||||
count_files,
|
||||
inside_repo,
|
||||
key_holds,
|
||||
key_in,
|
||||
load_json,
|
||||
negated_in,
|
||||
normalise_quote,
|
||||
number_key,
|
||||
require_dir,
|
||||
sha256_bytes,
|
||||
tree_sha256,
|
||||
)
|
||||
|
||||
SET_SCHEMA = "okf-wiki/wholedoc-set/1"
|
||||
MANIFEST_SCHEMA = "okf-wiki/wholedoc-manifest/1"
|
||||
ANSWERS_SCHEMA_NOTE = '{"arm": "<name>", "answers": [{"id", "answer", "count" (count questions), "citations": [{"file", "quote"}]}]}'
|
||||
TYPES = ("count", "whole")
|
||||
CUT_MAX_FILES = 20
|
||||
SPAN_MIN, SPAN_MAX = 20, 400
|
||||
KEY_MIN, KEY_MAX = 3, 60
|
||||
WHOLE_MIN_LINES = 150
|
||||
WHOLE_MIN_KEYS = 3
|
||||
WHOLE_MIN_PARTS = 3
|
||||
PART_GAP = 0.15
|
||||
FIRST_THIRD, LAST_THIRD = 1 / 3, 2 / 3
|
||||
COUNT_MIN = 5
|
||||
|
||||
|
||||
def thresholds() -> dict[str, Any]:
|
||||
"""The thresholds, as frozen in the manifest before any question is written."""
|
||||
return {
|
||||
"span_chars": [SPAN_MIN, SPAN_MAX],
|
||||
"key_chars": [KEY_MIN, KEY_MAX],
|
||||
"span_occurrences": "exactly 1 in its file (so its position is defined)",
|
||||
"whole_min_file_lines": WHOLE_MIN_LINES,
|
||||
"whole_min_keys": WHOLE_MIN_KEYS,
|
||||
"whole_min_parts": WHOLE_MIN_PARTS,
|
||||
"whole_part_gap": PART_GAP,
|
||||
"whole_part_rule": "spans sorted by start offset; a new part begins where the gap between two starts is "
|
||||
">= whole_part_gap x file length (characters)",
|
||||
"whole_coverage": "first span starts before 1/3 of the file, last span starts at or after 2/3",
|
||||
"whole_one_span": "no single span holds every answer key (case-folded)",
|
||||
"count_min": COUNT_MIN,
|
||||
"count_rule": "count == number of evidence spans == matches of item_pattern in the file; every match starts "
|
||||
"inside exactly one span",
|
||||
"count_not_verbatim": "the count stands nowhere in the file as a number (not part of a longer number, version "
|
||||
"or identifier) nor as an English number word",
|
||||
"cut_max_files": CUT_MAX_FILES,
|
||||
}
|
||||
|
||||
|
||||
def number_stands_in(n: int, text: str) -> bool:
|
||||
"""True when n stands as its own number or as its English word, not inside a version, identifier or number."""
|
||||
if re.search(rf"(?<![\w.\-]){n}(?!\w|\.\d)", text):
|
||||
return True
|
||||
return n < len(NUMBER_WORDS) and re.search(rf"\b{NUMBER_WORDS[n]}\b", text, re.IGNORECASE) is not None
|
||||
|
||||
|
||||
def load_set(path: Path) -> list[dict[str, Any]]:
|
||||
data = load_json(path, "set")
|
||||
if not isinstance(data, dict) or data.get("schema") != SET_SCHEMA or not isinstance(data.get("questions"), list):
|
||||
raise RunError(f"{path} is not a set with schema {SET_SCHEMA}")
|
||||
questions: list[dict[str, Any]] = data["questions"]
|
||||
return questions
|
||||
|
||||
|
||||
def dump_set(questions: list[dict[str, Any]]) -> bytes:
|
||||
return (json.dumps({"schema": SET_SCHEMA, "questions": questions}, ensure_ascii=False, indent=1) + "\n").encode("utf-8")
|
||||
|
||||
|
||||
def load_manifest(path: Path) -> dict[str, Any]:
|
||||
manifest = load_json(path, "manifest")
|
||||
if not isinstance(manifest, dict) or manifest.get("schema") != MANIFEST_SCHEMA:
|
||||
raise RunError(f"{path} is not a manifest with schema {MANIFEST_SCHEMA}")
|
||||
return manifest
|
||||
|
||||
|
||||
def write_json(path: Path, data: Any) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(json.dumps(data, indent=1) + "\n", encoding="utf-8")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------------------------- cut
|
||||
|
||||
|
||||
def compute_cut(questions: list[dict[str, Any]], max_files: int) -> tuple[list[str], list[str]]:
|
||||
"""Repeatedly take the question needing the fewest new files (ties: lowest id) while the union fits."""
|
||||
files = {q["id"]: {e["file"] for e in as_list(q.get("evidence"))} for q in questions}
|
||||
remaining = sorted(files)
|
||||
union: set[str] = set()
|
||||
chosen: list[str] = []
|
||||
while remaining:
|
||||
best = min(remaining, key=lambda i: (len(files[i] - union), i))
|
||||
if len(union | files[best]) > max_files:
|
||||
break
|
||||
union |= files[best]
|
||||
chosen.append(best)
|
||||
remaining.remove(best)
|
||||
return chosen, sorted(union)
|
||||
|
||||
|
||||
def cut_from(e2_path: Path, max_files: int) -> dict[str, Any]:
|
||||
data = load_json(e2_path, "E2 set")
|
||||
if not isinstance(data, dict) or not isinstance(data.get("questions"), list):
|
||||
raise RunError(f"{e2_path} is not an E2 set")
|
||||
chosen, files = compute_cut(data["questions"], max_files)
|
||||
return {
|
||||
"rule": "E2 dev questions sorted by id; repeatedly the question needing the fewest NEW source files "
|
||||
"(ties: lowest id), while the union stays <= max_files",
|
||||
"e2_set_sha256": sha256_bytes(e2_path.read_bytes()),
|
||||
"max_files": max_files,
|
||||
"questions": len(chosen),
|
||||
"files": len(files),
|
||||
"names_sha256": sha256_bytes("".join(f + "\n" for f in files).encode("utf-8")),
|
||||
"names": files,
|
||||
}
|
||||
|
||||
|
||||
def cmd_cut(args: argparse.Namespace) -> int:
|
||||
sources, manifest_path = Path(args.sources), Path(args.manifest)
|
||||
require_dir(sources, "sources")
|
||||
if manifest_path.exists():
|
||||
raise RunError(f"manifest {manifest_path} already exists; the thresholds are frozen once written")
|
||||
cut = cut_from(Path(args.e2_set), args.max_files)
|
||||
missing = [f for f in cut["names"] if not (sources / f).is_file()]
|
||||
if missing:
|
||||
raise RunError(f"cut files missing from the sources: {', '.join(missing)}")
|
||||
write_json(manifest_path, {
|
||||
"schema": MANIFEST_SCHEMA,
|
||||
"thresholds": thresholds(),
|
||||
"cut": cut,
|
||||
"sources": {"files": count_files(sources), "tree_sha256": tree_sha256(sources)},
|
||||
"seed": None,
|
||||
"dev": None,
|
||||
"held_out": None,
|
||||
})
|
||||
print(f"cut: {cut['files']} files from {cut['questions']} E2 questions (max {args.max_files})")
|
||||
for name in cut["names"]:
|
||||
print(f" {name}")
|
||||
print(f"manifest: {manifest_path}")
|
||||
return 0
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------------------------- check
|
||||
|
||||
|
||||
@dataclass
|
||||
class CheckReport:
|
||||
questions: int = 0
|
||||
by_type: dict[str, int] = field(default_factory=lambda: {t: 0 for t in TYPES})
|
||||
spans: int = 0
|
||||
spans_ok: int = 0
|
||||
in_cut: int = 0
|
||||
whole_long: int = 0
|
||||
whole_keyed: int = 0
|
||||
whole_spread: int = 0
|
||||
count_ok: int = 0
|
||||
count_hidden: int = 0
|
||||
not_one_span: int = 0
|
||||
keys: int = 0
|
||||
wording_keys: int = 0
|
||||
negated_keys: int = 0
|
||||
negated_ok: int = 0
|
||||
errors: list[str] = field(default_factory=list)
|
||||
|
||||
@property
|
||||
def passed(self) -> bool:
|
||||
n, w, c = self.questions, self.by_type["whole"], self.by_type["count"]
|
||||
return (n > 0 and not self.errors and self.spans_ok == self.spans and self.in_cut == n
|
||||
and self.whole_long == w and self.whole_keyed == w and self.whole_spread == w
|
||||
and self.count_ok == c and self.count_hidden == c and self.not_one_span == n
|
||||
and self.wording_keys == 0 and self.negated_ok == self.negated_keys)
|
||||
|
||||
def lines(self) -> list[str]:
|
||||
n, w, c = self.questions, self.by_type["whole"], self.by_type["count"]
|
||||
return [f"error: {e}" for e in self.errors] + [
|
||||
f"questions: {n} (whole {w}, count {c})",
|
||||
f"spans verbatim and unique: {self.spans_ok} of {self.spans}",
|
||||
f"questions whose file is in the cut: {self.in_cut} of {n}",
|
||||
f"whole: file >= {WHOLE_MIN_LINES} lines: {self.whole_long} of {w}",
|
||||
f"whole: every key in a span, every span keyed: {self.whole_keyed} of {w}",
|
||||
f"whole: spread over >= {WHOLE_MIN_PARTS} parts, first to last third: {self.whole_spread} of {w}",
|
||||
f"count: count = spans = pattern matches >= {COUNT_MIN}: {self.count_ok} of {c}",
|
||||
f"count: answer not verbatim in the file: {self.count_hidden} of {c}",
|
||||
f"not answerable from one span: {self.not_one_span} of {n}",
|
||||
f"answer keys that are wording: {self.wording_keys} of {self.keys}",
|
||||
f"negated keys negated in their span: {self.negated_ok} of {self.negated_keys}",
|
||||
"PASS" if self.passed else "FAIL",
|
||||
]
|
||||
|
||||
|
||||
def parts(starts: list[int], length: int) -> int:
|
||||
ordered = sorted(starts)
|
||||
return 1 + sum(1 for a, b in zip(ordered, ordered[1:]) if b - a >= PART_GAP * length) if ordered else 0
|
||||
|
||||
|
||||
def check_whole(q: dict[str, Any], text: str, spans: list[tuple[str, int]], rep: CheckReport) -> None:
|
||||
qid = q["id"]
|
||||
if text.count("\n") >= WHOLE_MIN_LINES:
|
||||
rep.whole_long += 1
|
||||
keys = [k.get("text") if isinstance(k, dict) else None for k in as_list(q.get("answer_keys"))]
|
||||
if (not all(isinstance(k, str) and (number_key(k) is not None or KEY_MIN <= len(k.strip()) <= KEY_MAX) for k in keys)
|
||||
or len(keys) < WHOLE_MIN_KEYS):
|
||||
rep.errors.append(f"{qid}: needs >= {WHOLE_MIN_KEYS} answer keys of {KEY_MIN}-{KEY_MAX} characters")
|
||||
return
|
||||
texts = [str(k) for k in keys]
|
||||
span_texts = [s for s, _ in spans]
|
||||
if all(any(key_in(k, s) for s in span_texts) for k in texts) and all(any(key_in(k, s) for k in texts) for s in span_texts):
|
||||
rep.whole_keyed += 1
|
||||
starts = [pos for _, pos in spans if pos >= 0]
|
||||
if (len(starts) == len(spans) and parts(starts, len(text)) >= WHOLE_MIN_PARTS
|
||||
and min(starts) < FIRST_THIRD * len(text) and max(starts) >= LAST_THIRD * len(text)):
|
||||
rep.whole_spread += 1
|
||||
if not any(all(key_in(k, s) for k in texts) for s in span_texts):
|
||||
rep.not_one_span += 1
|
||||
for k in as_list(q.get("answer_keys")):
|
||||
if "negated" not in k:
|
||||
continue
|
||||
if k["negated"] is not True:
|
||||
rep.errors.append(f"{qid}: negated must be true or absent")
|
||||
continue
|
||||
rep.negated_keys += 1
|
||||
rep.negated_ok += any(negated_in(k["text"], s) for s in span_texts)
|
||||
|
||||
|
||||
def check_count(q: dict[str, Any], text: str, spans: list[tuple[str, int]], rep: CheckReport) -> None:
|
||||
qid, count, pattern = q["id"], q.get("count"), q.get("item_pattern")
|
||||
if type(count) is not int or not isinstance(pattern, str):
|
||||
rep.errors.append(f"{qid}: a count question needs an integer count and an item_pattern")
|
||||
return
|
||||
try:
|
||||
matches = [m.start() for m in re.finditer(pattern, text)]
|
||||
except re.error as exc:
|
||||
rep.errors.append(f"{qid}: item_pattern does not compile: {exc}")
|
||||
return
|
||||
ranges = [(pos, pos + len(s)) for s, pos in spans if pos >= 0]
|
||||
each_in_one = all(sum(1 for a, b in ranges if a <= m < b) == 1 for m in matches)
|
||||
if count >= COUNT_MIN and count == len(spans) == len(ranges) == len(matches) and each_in_one:
|
||||
rep.count_ok += 1
|
||||
if not number_stands_in(count, text):
|
||||
rep.count_hidden += 1
|
||||
if len(spans) >= 2 and not any(number_stands_in(count, s) for s, _ in spans):
|
||||
rep.not_one_span += 1
|
||||
|
||||
|
||||
def check_questions(questions: list[dict[str, Any]], sources: Path, cut_names: list[str]) -> CheckReport:
|
||||
require_dir(sources, "sources")
|
||||
cache = SourceCache(sources)
|
||||
rep = CheckReport(questions=len(questions))
|
||||
form = count_keys(questions)
|
||||
rep.keys, rep.wording_keys = form.keys, form.wording
|
||||
seen: set[str] = set()
|
||||
for q in questions:
|
||||
qid = q.get("id") if isinstance(q, dict) else None
|
||||
if not isinstance(qid, str) or not qid:
|
||||
rep.errors.append("question without an id")
|
||||
continue
|
||||
if qid in seen:
|
||||
rep.errors.append(f"duplicate id {qid}")
|
||||
seen.add(qid)
|
||||
qtype = q.get("type")
|
||||
if qtype not in TYPES:
|
||||
rep.errors.append(f"{qid}: type {qtype!r} is not one of {TYPES}")
|
||||
continue
|
||||
rep.by_type[qtype] += 1
|
||||
if not isinstance(q.get("question"), str) or not q["question"].strip():
|
||||
rep.errors.append(f"{qid}: empty question")
|
||||
name = q.get("file")
|
||||
if name in cut_names:
|
||||
rep.in_cut += 1
|
||||
else:
|
||||
rep.errors.append(f"{qid}: file {name!r} is not in the cut")
|
||||
text = cache.text(name) if isinstance(name, str) else None
|
||||
if text is None:
|
||||
rep.errors.append(f"{qid}: file {name!r} is not a readable file in the sources")
|
||||
text = ""
|
||||
spans: list[tuple[str, int]] = []
|
||||
for e in as_list(q.get("evidence")):
|
||||
span = e.get("span") if isinstance(e, dict) else None
|
||||
rep.spans += 1
|
||||
if not isinstance(span, str):
|
||||
rep.errors.append(f"{qid}: evidence span is not text")
|
||||
continue
|
||||
if not SPAN_MIN <= len(span) <= SPAN_MAX:
|
||||
rep.errors.append(f"{qid}: span length {len(span)} outside {SPAN_MIN}-{SPAN_MAX}")
|
||||
unique = text.count(span) == 1
|
||||
rep.spans_ok += unique
|
||||
spans.append((span, text.find(span) if unique else -1))
|
||||
if qtype == "whole":
|
||||
check_whole(q, text, spans, rep)
|
||||
else:
|
||||
check_count(q, text, spans, rep)
|
||||
return rep
|
||||
|
||||
|
||||
def verified_cut(manifest: dict[str, Any], e2_path: Path) -> list[str]:
|
||||
cut = manifest.get("cut")
|
||||
if not isinstance(cut, dict):
|
||||
raise RunError("manifest has no cut")
|
||||
fresh = cut_from(e2_path, int(cut.get("max_files", CUT_MAX_FILES)))
|
||||
for key in ("e2_set_sha256", "names_sha256", "files"):
|
||||
if fresh[key] != cut.get(key):
|
||||
raise RunError(f"the cut from {e2_path} differs from the manifest ({key}); the group is frozen against it")
|
||||
names: list[str] = fresh["names"]
|
||||
return names
|
||||
|
||||
|
||||
def cmd_check(args: argparse.Namespace) -> int:
|
||||
manifest = load_manifest(Path(args.manifest))
|
||||
cut_names = verified_cut(manifest, Path(args.e2_set))
|
||||
rep = check_questions(load_set(Path(args.set)), Path(args.sources), cut_names)
|
||||
print("\n".join(rep.lines()))
|
||||
return 0 if rep.passed else 1
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------------------------- split
|
||||
|
||||
|
||||
def cmd_split(args: argparse.Namespace) -> int:
|
||||
sources, held_path, manifest_path = Path(args.sources), Path(args.held_out), Path(args.manifest)
|
||||
dev_path = Path(args.dev_out)
|
||||
if inside_repo(held_path):
|
||||
raise RunError(f"held-out path {held_path} is inside the repository; it must live outside it")
|
||||
if held_path.exists():
|
||||
raise RunError(f"held-out file {held_path} already exists; a frozen draw is never redrawn")
|
||||
manifest = load_manifest(manifest_path)
|
||||
if manifest.get("dev") is not None or manifest.get("held_out") is not None:
|
||||
raise RunError(f"manifest {manifest_path} already records a draw; a frozen draw is never redrawn")
|
||||
if manifest.get("thresholds") != thresholds():
|
||||
raise RunError("the manifest's thresholds differ from the code; they were frozen before the questions")
|
||||
if tree_sha256(sources) != manifest.get("sources", {}).get("tree_sha256"):
|
||||
raise RunError(f"sources {sources} differ from the manifest's tree_sha256")
|
||||
questions = load_set(Path(args.set))
|
||||
rep = check_questions(questions, sources, verified_cut(manifest, Path(args.e2_set)))
|
||||
if not rep.passed:
|
||||
print("\n".join(rep.lines()))
|
||||
raise RunError("the set fails the check; nothing was split")
|
||||
k = args.held_out_per_type
|
||||
rng = random.Random(args.seed)
|
||||
held_ids: set[str] = set()
|
||||
for t in TYPES:
|
||||
ids = sorted(q["id"] for q in questions if q["type"] == t)
|
||||
if not 0 < k < len(ids):
|
||||
raise RunError(f"cannot hold out {k} of {len(ids)} {t} questions")
|
||||
held_ids.update(rng.sample(ids, k))
|
||||
by_id = {q["id"]: q for q in questions}
|
||||
ids = sorted(by_id)
|
||||
dev = [dict(by_id[i], id=f"d{n:02d}") for n, i in enumerate((i for i in ids if i not in held_ids), 1)]
|
||||
held = [dict(by_id[i], id=f"h{n:02d}") for n, i in enumerate(sorted(held_ids), 1)]
|
||||
dev_bytes, held_bytes = dump_set(dev), dump_set(held)
|
||||
held_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
held_path.write_bytes(held_bytes)
|
||||
dev_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
dev_path.write_bytes(dev_bytes)
|
||||
|
||||
def per_type(part: list[dict[str, Any]]) -> dict[str, int]:
|
||||
return {t: sum(1 for q in part if q["type"] == t) for t in ("whole", "count")}
|
||||
|
||||
manifest["seed"] = args.seed
|
||||
manifest["draw"] = ("rng = random.Random(seed); for type in ('count', 'whole'): rng.sample(sorted(ids of type), "
|
||||
"held_out_per_type); both parts renumbered d../h.. in sorted original-id order")
|
||||
manifest["dev"] = {"file": args.dev_label or dev_path.name, "sha256": sha256_bytes(dev_bytes),
|
||||
"questions": len(dev), **per_type(dev)}
|
||||
manifest["held_out"] = {"sha256": sha256_bytes(held_bytes), "questions": len(held), **per_type(held)}
|
||||
write_json(manifest_path, manifest)
|
||||
print(f"seed: {args.seed}")
|
||||
print(f"dev: {len(dev)} questions {per_type(dev)}, sha256 {manifest['dev']['sha256']}")
|
||||
print(f"held out: {len(held)} questions {per_type(held)}, sha256 {manifest['held_out']['sha256']}")
|
||||
print(f"manifest: {manifest_path}")
|
||||
return 0
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------------------------- gate
|
||||
|
||||
|
||||
@dataclass
|
||||
class GateResult:
|
||||
qid: str
|
||||
qtype: str
|
||||
answered: bool
|
||||
answer_ok: bool
|
||||
source_ok: bool
|
||||
|
||||
@property
|
||||
def correct(self) -> bool:
|
||||
return self.answer_ok and self.source_ok
|
||||
|
||||
|
||||
def score_one(q: dict[str, Any], a: dict[str, Any] | None, cache: SourceCache) -> GateResult:
|
||||
if a is None:
|
||||
return GateResult(q["id"], q["type"], False, False, False)
|
||||
if q["type"] == "count":
|
||||
given = a.get("count")
|
||||
answer_ok = type(given) is int and given == q["count"]
|
||||
else:
|
||||
answer = a["answer"] if isinstance(a.get("answer"), str) else ""
|
||||
answer_ok = all(key_holds(k, answer) for k in q["answer_keys"])
|
||||
text = cache.text(q["file"])
|
||||
source_ok = False
|
||||
for c in as_list(a.get("citations")):
|
||||
name, quote = (c.get("file"), c.get("quote")) if isinstance(c, dict) else (None, None)
|
||||
if name == q["file"] and text is not None and isinstance(quote, str) and quote.strip():
|
||||
source_ok = source_ok or normalise_quote(quote) in normalise_quote(text)
|
||||
return GateResult(q["id"], q["type"], True, answer_ok, source_ok)
|
||||
|
||||
|
||||
def cmd_gate(args: argparse.Namespace) -> int:
|
||||
set_path, sources = Path(args.set), Path(args.sources)
|
||||
require_dir(sources, "sources")
|
||||
manifest = load_manifest(Path(args.manifest))
|
||||
dev, held = manifest.get("dev"), manifest.get("held_out")
|
||||
if not isinstance(dev, dict) or not isinstance(held, dict):
|
||||
raise RunError("the manifest records no draw yet; run split first")
|
||||
set_hash = sha256_bytes(set_path.read_bytes()) if set_path.is_file() else ""
|
||||
part = {dev.get("sha256"): "dev", held.get("sha256"): "held_out"}.get(set_hash)
|
||||
if part is None:
|
||||
raise RunError(f"set {set_path} sha256 {set_hash or '(missing)'} matches neither part of the manifest")
|
||||
if tree_sha256(sources) != manifest.get("sources", {}).get("tree_sha256"):
|
||||
raise RunError(f"sources {sources} differ from the manifest's tree_sha256; the set is frozen against them")
|
||||
questions = load_set(set_path)
|
||||
cache = SourceCache(sources)
|
||||
answers: dict[str, dict[str, Any]] = {}
|
||||
fail_closed = ""
|
||||
answers_path = Path(args.answers)
|
||||
if not answers_path.is_file():
|
||||
fail_closed = f"no answers for arm {args.arm} at {answers_path}"
|
||||
else:
|
||||
data = load_json(answers_path, "answers")
|
||||
if not isinstance(data, dict) or not isinstance(data.get("answers"), list):
|
||||
raise RunError(f"{answers_path} is not an answers file {ANSWERS_SCHEMA_NOTE}")
|
||||
if data.get("arm") != args.arm:
|
||||
raise RunError(f"answers file is for arm {data.get('arm')!r}, not {args.arm!r}")
|
||||
answers = {a["id"]: a for a in data["answers"] if isinstance(a, dict) and isinstance(a.get("id"), str)}
|
||||
results = [score_one(q, answers.get(q["id"]), cache) for q in questions]
|
||||
n = len(results)
|
||||
missing = sum(1 for r in results if not r.answered)
|
||||
print(f"set: {part}, {n} questions; arm: {args.arm}")
|
||||
if fail_closed:
|
||||
print(f"fail-closed: {fail_closed}")
|
||||
print(f"missing answers: {missing}")
|
||||
for t in ("whole", "count"):
|
||||
of_type = [r for r in results if r.qtype == t]
|
||||
print(f"{t}: correct {sum(r.correct for r in of_type)} of {len(of_type)}")
|
||||
print(f"correct answer: {sum(r.answer_ok for r in results)} of {n}")
|
||||
print(f"correct source: {sum(r.source_ok for r in results)} of {n}")
|
||||
correct = sum(r.correct for r in results)
|
||||
print(f"correct answer and source: {correct} of {n}")
|
||||
if args.out:
|
||||
write_json(Path(args.out), {
|
||||
"arm": args.arm,
|
||||
"set_sha256": set_hash,
|
||||
"results": [{"id": r.qid, "type": r.qtype, "answered": r.answered, "answer_ok": r.answer_ok,
|
||||
"source_ok": r.source_ok, "correct": r.correct} for r in results],
|
||||
})
|
||||
scored = not fail_closed and missing == 0
|
||||
print("GREEN" if scored and n > 0 and correct == n else "RED")
|
||||
print("SCORED" if scored else "UNSCORED (fail-closed)" if fail_closed else "INCOMPLETE")
|
||||
return 0 if scored else 1
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------------------------- main
|
||||
|
||||
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
sub = p.add_subparsers(dest="command", required=True)
|
||||
u = sub.add_parser("cut", help="compute the source cut and freeze the thresholds in a new manifest")
|
||||
u.add_argument("--e2-set", required=True)
|
||||
u.add_argument("--sources", required=True)
|
||||
u.add_argument("--max-files", type=int, default=CUT_MAX_FILES)
|
||||
u.add_argument("--manifest", required=True)
|
||||
c = sub.add_parser("check", help="check every question's properties in the source files")
|
||||
c.add_argument("--set", required=True)
|
||||
c.add_argument("--sources", required=True)
|
||||
c.add_argument("--manifest", required=True)
|
||||
c.add_argument("--e2-set", required=True)
|
||||
s = sub.add_parser("split", help="draw the held-out questions per type with a fixed seed")
|
||||
s.add_argument("--set", required=True)
|
||||
s.add_argument("--sources", required=True)
|
||||
s.add_argument("--manifest", required=True)
|
||||
s.add_argument("--e2-set", required=True)
|
||||
s.add_argument("--seed", type=int, required=True)
|
||||
s.add_argument("--held-out-per-type", type=int, default=2)
|
||||
s.add_argument("--dev-out", required=True)
|
||||
s.add_argument("--dev-label", default="", help="path of the dev set as written in the manifest")
|
||||
s.add_argument("--held-out", required=True, help="file path OUTSIDE the repository")
|
||||
g = sub.add_parser("gate", help="score one arm's answers against the answer key")
|
||||
g.add_argument("--set", required=True)
|
||||
g.add_argument("--manifest", required=True)
|
||||
g.add_argument("--sources", required=True)
|
||||
g.add_argument("--arm", required=True)
|
||||
g.add_argument("--answers", required=True, help=ANSWERS_SCHEMA_NOTE)
|
||||
g.add_argument("--out", default="", help="write per-question results (for paired tests)")
|
||||
return p
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
args = build_parser().parse_args(argv)
|
||||
commands = {"cut": cmd_cut, "check": cmd_check, "split": cmd_split, "gate": cmd_gate}
|
||||
try:
|
||||
return commands[args.command](args)
|
||||
except RunError as exc:
|
||||
print(f"FAIL: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Loading…
Add table
Add a link
Reference in a new issue