First public release, published as one snapshot. What changed between versions is in CHANGELOG.md. Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
694 lines
33 KiB
Python
694 lines
33 KiB
Python
"""The question set for PLAN.md criterion 4 on collection 1 (contract K5 point 1): checker, part count, held-out
|
|
split, answer gate and the stub arms. No model call.
|
|
|
|
check every span stands byte for byte in its file in a K1 root, every question has at least two distinct
|
|
paths, its group follows K1 and the path count, every path carries an answer key that stands in that
|
|
path's spans, no key is wording, every negated key stands negated. PASS only N of N with N > 0.
|
|
For a harder set, --min-paths raises the paths every question needs and --min-many sets how many
|
|
questions need 4 or more.
|
|
inside the questions whose every span stands in a part root (a K1 subset root, here the first part).
|
|
split draws the held-out questions with a fixed seed, stratified by group, writes them OUTSIDE the
|
|
repository, renumbers both parts and writes the manifest (seed, counts, sha256 values only).
|
|
freeze freezes a set not made by split: it must pass check against its own root with the thresholds and
|
|
hold exactly --count questions. It is renumbered in a seeded order and gets a manifest of its own that
|
|
names it the part of that root (seed, rules, counts, sha256 values, the roots' K1 sha256; no text),
|
|
so gate judges it as it judges the part. A frozen set or manifest is never overwritten.
|
|
gate scores one arm: a question is correct when the answer holds every key (multidoc.key_holds) AND the
|
|
citations quote, verbatim, every path the question needs. A missing answers file scores 0
|
|
(fail-closed). The set and the root must match the manifest.
|
|
stub writes a known arm: `fasit` answers with the keys and quotes the spans; `no-citations` answers with
|
|
the keys and cites nothing.
|
|
|
|
The group is set by the sources, never by the writer: `history` when a path has `main_topic: false` in K1,
|
|
otherwise `docs-2-3` (2-3 distinct paths) or `docs-4` (4 or more).
|
|
|
|
Exit 0 = PASS / scored completely; 1 = FAIL or could not run. A check that cannot run is a FAIL.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
import math
|
|
import random
|
|
import re
|
|
import sys
|
|
from dataclasses import dataclass, field
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
from keyform import count_keys
|
|
from multidoc import KEY_MAX, KEY_MIN, SPAN_MAX, SPAN_MIN, key_holds, key_in, negated_in, normalise_quote, number_key
|
|
|
|
SET_SCHEMA = "okf-wiki/answer-set/1"
|
|
MANIFEST_SCHEMA = "okf-wiki/answer-set-manifest/1"
|
|
ANSWERS_SCHEMA = "okf-wiki/answers/1"
|
|
RESULTS_SCHEMA = "okf-wiki/answer-results/1"
|
|
K1_SCHEMA = "okf-wiki/source-manifest/1"
|
|
K1_FILE = "source-manifest.json"
|
|
GROUPS = ("docs-2-3", "docs-4", "history")
|
|
ARMS = ("filsok", "bundle")
|
|
MIN_PATHS = 2
|
|
MANY_PATHS = 4
|
|
PATH_RE = re.compile(r"^kilde/[a-z0-9._-]+$")
|
|
REPO_ROOT = Path(__file__).resolve().parents[1]
|
|
|
|
|
|
class RunError(Exception):
|
|
"""The command cannot run; reported as FAIL with exit 1."""
|
|
|
|
|
|
def dump(data: Any) -> bytes:
|
|
"""The K5 file form: two writings of the same content are byte-identical."""
|
|
return (json.dumps(data, indent=1, sort_keys=True, ensure_ascii=False) + "\n").encode("utf-8")
|
|
|
|
|
|
def sha256_bytes(data: bytes) -> str:
|
|
return hashlib.sha256(data).hexdigest()
|
|
|
|
|
|
def load_json(path: Path, what: str) -> Any:
|
|
try:
|
|
return json.loads(path.read_text(encoding="utf-8"))
|
|
except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc:
|
|
raise RunError(f"cannot read {what} {path}: {exc}") from exc
|
|
|
|
|
|
def as_list(value: object) -> list[Any]:
|
|
return value if isinstance(value, list) else []
|
|
|
|
|
|
def group_of(paths: list[str], main_topic: dict[str, bool]) -> str:
|
|
"""The group K1 gives a question with these distinct paths (see the module text)."""
|
|
distinct = set(paths)
|
|
if any(main_topic.get(p) is False for p in distinct):
|
|
return "history"
|
|
return "docs-4" if len(distinct) >= MANY_PATHS else "docs-2-3"
|
|
|
|
|
|
# ---------------------------------------------------------------------------------------------- K1 root
|
|
|
|
|
|
class Root:
|
|
"""A K1 root read with this module's own code (contract K1; okfwiki.sources is not imported)."""
|
|
|
|
def __init__(self, path: Path) -> None:
|
|
self.path = path
|
|
manifest = path / K1_FILE
|
|
if not path.is_dir() or not manifest.is_file():
|
|
raise RunError(f"{path} is not a K1 root: no {K1_FILE}")
|
|
self.manifest_bytes = manifest.read_bytes()
|
|
self.manifest_sha256 = sha256_bytes(self.manifest_bytes)
|
|
try:
|
|
data = json.loads(self.manifest_bytes.decode("utf-8"))
|
|
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
|
|
raise RunError(f"{manifest} is not JSON: {exc}") from exc
|
|
if not isinstance(data, dict) or data.get("schema") != K1_SCHEMA or not isinstance(data.get("sources"), list):
|
|
raise RunError(f"{manifest} is not a K1 manifest with schema {K1_SCHEMA}")
|
|
self.frozen_at = str(data.get("frozen_at", ""))
|
|
self.entries: dict[str, dict[str, Any]] = {}
|
|
for entry in data["sources"]:
|
|
if not (isinstance(entry, dict) and isinstance(entry.get("path"), str)
|
|
and isinstance(entry.get("main_topic"), bool) and isinstance(entry.get("sha256"), str)):
|
|
raise RunError(f"{manifest} has an entry without path, main_topic or sha256: {entry!r}")
|
|
self.entries[entry["path"]] = entry
|
|
self.main_topic = {p: bool(e["main_topic"]) for p, e in self.entries.items()}
|
|
self._text: dict[str, str | None] = {}
|
|
self._bad: dict[str, str] = {}
|
|
|
|
def ref(self) -> dict[str, str]:
|
|
return {"frozen_at": self.frozen_at, "manifest_sha256": self.manifest_sha256}
|
|
|
|
def text(self, path: str, verify: bool = True) -> str | None:
|
|
"""The text of a listed source, or None (with the reason in problem()) when it is not listed, missing, not
|
|
UTF-8, or (with verify) its sha256 differs from K1."""
|
|
key = f"{verify}:{path}"
|
|
if key not in self._text:
|
|
self._text[key] = self._read(path, verify, key)
|
|
return self._text[key]
|
|
|
|
def _read(self, path: str, verify: bool, key: str) -> str | None:
|
|
if not PATH_RE.match(path) or path not in self.entries:
|
|
self._bad[key] = f"{path} is not a path K1 lists in {self.path}"
|
|
return None
|
|
target = self.path / path
|
|
try:
|
|
data = target.read_bytes()
|
|
except OSError:
|
|
self._bad[key] = f"{path} is missing in {self.path}"
|
|
return None
|
|
if verify and sha256_bytes(data) != self.entries[path]["sha256"]:
|
|
self._bad[key] = f"{path} sha256 differs from K1 in {self.path}"
|
|
return None
|
|
try:
|
|
return data.decode("utf-8")
|
|
except UnicodeDecodeError:
|
|
self._bad[key] = f"{path} is not UTF-8"
|
|
return None
|
|
|
|
def problem(self, path: str, verify: bool = True) -> str:
|
|
return self._bad.get(f"{verify}:{path}", "")
|
|
|
|
|
|
def load_set(path: Path) -> dict[str, Any]:
|
|
data = load_json(path, "set")
|
|
if (not isinstance(data, dict) or data.get("schema") != SET_SCHEMA or not isinstance(data.get("questions"), list)
|
|
or not isinstance(data.get("sources"), dict)):
|
|
raise RunError(f"{path} is not a set with schema {SET_SCHEMA}")
|
|
return data
|
|
|
|
|
|
def evidence_of(q: dict[str, Any]) -> list[tuple[str, str]]:
|
|
"""(path, span) for every well-formed evidence entry."""
|
|
out = []
|
|
for e in as_list(q.get("evidence")):
|
|
if isinstance(e, dict) and isinstance(e.get("path"), str) and isinstance(e.get("span"), str):
|
|
out.append((e["path"], e["span"]))
|
|
return out
|
|
|
|
|
|
# ---------------------------------------------------------------------------------------------- check
|
|
|
|
|
|
@dataclass
|
|
class CheckReport:
|
|
questions: int = 0
|
|
spans: int = 0
|
|
spans_verbatim: int = 0
|
|
all_spans_verbatim: int = 0
|
|
min_paths: int = 0
|
|
group_ok: int = 0
|
|
key_per_path: int = 0
|
|
keys: int = 0
|
|
wording_keys: int = 0
|
|
negated_keys: int = 0
|
|
negated_ok: int = 0
|
|
groups: dict[str, int] = field(default_factory=lambda: {g: 0 for g in GROUPS})
|
|
errors: list[str] = field(default_factory=list)
|
|
paths_per_question: dict[str, int] = field(default_factory=dict)
|
|
need_paths: int = MIN_PATHS
|
|
need_many: int = 0
|
|
many: int = 0
|
|
|
|
@property
|
|
def passed(self) -> bool:
|
|
n = self.questions
|
|
return (n > 0 and not self.errors and self.spans_verbatim == self.spans and self.all_spans_verbatim == n
|
|
and self.min_paths == n and self.group_ok == n and self.key_per_path == n
|
|
and self.wording_keys == 0 and self.negated_ok == self.negated_keys and self.many >= self.need_many)
|
|
|
|
def lines(self) -> list[str]:
|
|
n = self.questions
|
|
out = [f"error: {e}" for e in self.errors]
|
|
out += [
|
|
f"questions: {n}",
|
|
"groups: " + ", ".join(f"{g} {self.groups[g]}" for g in GROUPS),
|
|
f"spans verbatim: {self.spans_verbatim} of {self.spans}",
|
|
f"questions with every span verbatim: {self.all_spans_verbatim} of {n}",
|
|
f"questions with >= {self.need_paths} distinct paths: {self.min_paths} of {n}",
|
|
]
|
|
if self.need_many:
|
|
out.append(f"questions with >= {MANY_PATHS} distinct paths: {self.many} of {n} (required >= {self.need_many})")
|
|
out += [
|
|
f"questions whose group matches K1 and the path count: {self.group_ok} of {n}",
|
|
f"questions with a key in every path's spans: {self.key_per_path} of {n}",
|
|
f"answer keys that are wording: {self.wording_keys} of {self.keys}",
|
|
f"negated keys negated in their span: {self.negated_ok} of {self.negated_keys}",
|
|
"PASS" if self.passed else "FAIL",
|
|
]
|
|
return out
|
|
|
|
|
|
def check_set(data: dict[str, Any], root: Root, need_paths: int = MIN_PATHS, need_many: int = 0) -> CheckReport:
|
|
questions = data["questions"]
|
|
rep = CheckReport(questions=len(questions), need_paths=need_paths, need_many=need_many)
|
|
ref = data["sources"]
|
|
if ref.get("manifest_sha256") != root.manifest_sha256 or ref.get("frozen_at") != root.frozen_at:
|
|
rep.errors.append(f"the set names sources {ref.get('frozen_at')!r} manifest_sha256 "
|
|
f"{ref.get('manifest_sha256')!r}, the root has {root.frozen_at!r} {root.manifest_sha256}")
|
|
form = count_keys(questions)
|
|
rep.keys, rep.wording_keys = form.keys, form.wording
|
|
seen: set[str] = set()
|
|
for q in questions:
|
|
qid = q.get("id") if isinstance(q, dict) else None
|
|
if not isinstance(qid, str) or not qid:
|
|
rep.errors.append("question without an id")
|
|
continue
|
|
if qid in seen:
|
|
rep.errors.append(f"duplicate id {qid}")
|
|
seen.add(qid)
|
|
if not isinstance(q.get("question"), str) or not q["question"].strip():
|
|
rep.errors.append(f"{qid}: empty question")
|
|
raw_evidence = as_list(q.get("evidence"))
|
|
evidence = evidence_of(q)
|
|
rep.spans += len(raw_evidence)
|
|
if len(evidence) != len(raw_evidence):
|
|
rep.errors.append(f"{qid}: an evidence entry without a path or a span")
|
|
spans_by_path: dict[str, list[str]] = {}
|
|
verbatim = 0
|
|
for path, span in evidence:
|
|
if not SPAN_MIN <= len(span) <= SPAN_MAX:
|
|
rep.errors.append(f"{qid}: span length {len(span)} in {path} outside {SPAN_MIN}-{SPAN_MAX}")
|
|
text = root.text(path)
|
|
if text is None:
|
|
rep.errors.append(f"{qid}: {root.problem(path)}")
|
|
elif span in text:
|
|
verbatim += 1
|
|
spans_by_path.setdefault(path, []).append(span)
|
|
rep.spans_verbatim += verbatim
|
|
if raw_evidence and verbatim == len(raw_evidence):
|
|
rep.all_spans_verbatim += 1
|
|
paths = sorted(spans_by_path)
|
|
rep.paths_per_question[qid] = len(paths)
|
|
if len(paths) >= need_paths:
|
|
rep.min_paths += 1
|
|
if len(paths) >= MANY_PATHS:
|
|
rep.many += 1
|
|
group = q.get("group")
|
|
if group in GROUPS:
|
|
rep.groups[group] += 1
|
|
else:
|
|
rep.errors.append(f"{qid}: group {group!r} is not one of {', '.join(GROUPS)}")
|
|
if paths and all(p in root.entries for p in paths) and group == group_of(paths, root.main_topic):
|
|
rep.group_ok += 1
|
|
keyed: set[str] = set()
|
|
keys_ok = bool(as_list(q.get("answer_keys")))
|
|
for k in as_list(q.get("answer_keys")):
|
|
kpath, key = (k.get("path"), k.get("text")) if isinstance(k, dict) else (None, None)
|
|
if not isinstance(key, str) or not isinstance(kpath, str):
|
|
keys_ok = False
|
|
rep.errors.append(f"{qid}: malformed answer key")
|
|
continue
|
|
if kpath not in spans_by_path:
|
|
rep.errors.append(f"{qid}: answer key path {kpath} is not an evidence path")
|
|
if number_key(key) is None and not KEY_MIN <= len(key.strip()) <= KEY_MAX:
|
|
rep.errors.append(f"{qid}: key length {len(key.strip())} outside {KEY_MIN}-{KEY_MAX}")
|
|
if any(key_in(key, s) for s in spans_by_path.get(kpath, [])):
|
|
keyed.add(kpath)
|
|
else:
|
|
keys_ok = False
|
|
if "negated" in k:
|
|
if k["negated"] is not True:
|
|
rep.errors.append(f"{qid}: negated must be true or absent")
|
|
continue
|
|
rep.negated_keys += 1
|
|
rep.negated_ok += any(negated_in(key, s) for s in spans_by_path.get(kpath, []))
|
|
if keys_ok and paths and keyed == set(paths):
|
|
rep.key_per_path += 1
|
|
return rep
|
|
|
|
|
|
def thresholds(args: argparse.Namespace) -> tuple[int, int]:
|
|
if args.min_paths < MIN_PATHS or args.min_many < 0:
|
|
raise RunError(f"--min-paths must be at least {MIN_PATHS} and --min-many at least 0")
|
|
return args.min_paths, args.min_many
|
|
|
|
|
|
def cmd_check(args: argparse.Namespace) -> int:
|
|
rep = check_set(load_set(Path(args.set)), Root(Path(args.sources)), *thresholds(args))
|
|
if args.per_question:
|
|
for qid, n in rep.paths_per_question.items():
|
|
print(f"{qid}\t{n} paths")
|
|
print("\n".join(rep.lines()))
|
|
return 0 if rep.passed else 1
|
|
|
|
|
|
# ---------------------------------------------------------------------------------------------- inside
|
|
|
|
|
|
def inside_ids(questions: list[dict[str, Any]], part: Root) -> list[str]:
|
|
"""The ids of the questions whose every span stands in its file in the part root (path listed there)."""
|
|
out = []
|
|
for q in questions:
|
|
evidence = evidence_of(q) if isinstance(q, dict) else []
|
|
if evidence and len(evidence) == len(as_list(q.get("evidence"))):
|
|
texts = [(part.text(p, verify=False), s) for p, s in evidence]
|
|
if all(t is not None and s in t for t, s in texts):
|
|
out.append(str(q.get("id")))
|
|
return out
|
|
|
|
|
|
def cmd_inside(args: argparse.Namespace) -> int:
|
|
questions = load_set(Path(args.set))["questions"]
|
|
part = Root(Path(args.part_root))
|
|
ids = set(inside_ids(questions, part))
|
|
n = len(questions)
|
|
print(f"questions with every span in the part root: {len(ids)} of {n}")
|
|
for g in GROUPS:
|
|
members = [q for q in questions if isinstance(q, dict) and q.get("group") == g]
|
|
print(f" {g}: {sum(1 for q in members if q.get('id') in ids)} of {len(members)}")
|
|
if not args.require_all:
|
|
return 0
|
|
passed = n > 0 and len(ids) == n
|
|
print("PASS" if passed else "FAIL")
|
|
return 0 if passed else 1
|
|
|
|
|
|
# ---------------------------------------------------------------------------------------------- split
|
|
|
|
|
|
def inside_repo(path: Path) -> bool:
|
|
try:
|
|
path.resolve().relative_to(REPO_ROOT)
|
|
return True
|
|
except ValueError:
|
|
return False
|
|
|
|
|
|
def quotas(counts: dict[str, int], n_held: int) -> dict[str, int]:
|
|
"""Held-out questions per group, proportional to the group's size (largest remainder, ties in GROUPS order)."""
|
|
total = sum(counts.values())
|
|
exact = {g: n_held * counts[g] / total for g in GROUPS}
|
|
out = {g: math.floor(exact[g]) for g in GROUPS}
|
|
for g in sorted(GROUPS, key=lambda g: (-(exact[g] - out[g]), GROUPS.index(g)))[: n_held - sum(out.values())]:
|
|
out[g] += 1
|
|
return out
|
|
|
|
|
|
def group_counts(questions: list[dict[str, Any]]) -> dict[str, int]:
|
|
return {g: sum(1 for q in questions if q.get("group") == g) for g in GROUPS}
|
|
|
|
|
|
def cmd_split(args: argparse.Namespace) -> int:
|
|
held_path = Path(args.held_out)
|
|
if inside_repo(held_path):
|
|
raise RunError(f"held-out path {held_path} is inside the repository; it must live outside it")
|
|
if held_path.exists():
|
|
raise RunError(f"held-out file {held_path} already exists; a frozen draw is never redrawn")
|
|
root, part_root = Root(Path(args.sources)), Root(Path(args.part_sources))
|
|
data, part_data = load_set(Path(args.set)), load_set(Path(args.part))
|
|
for label, d, r in (("set", data, root), ("part", part_data, part_root)):
|
|
rep = check_set(d, r)
|
|
if not rep.passed:
|
|
print("\n".join(rep.lines()))
|
|
raise RunError(f"the {label} fails the check; nothing was split")
|
|
part_q = part_data["questions"]
|
|
if len(inside_ids(part_q, part_root)) != len(part_q):
|
|
raise RunError("a question of the part has a span outside the part root; nothing was split")
|
|
questions = data["questions"]
|
|
n_held = args.held_out_count
|
|
if not 0 < n_held < len(questions):
|
|
raise RunError(f"cannot hold out {n_held} of {len(questions)}")
|
|
counts = group_counts(questions)
|
|
per_group = quotas(counts, n_held)
|
|
rng = random.Random(args.seed)
|
|
held_ids: set[str] = set()
|
|
for g in GROUPS:
|
|
ids = sorted(q["id"] for q in questions if q["group"] == g)
|
|
held_ids.update(rng.sample(ids, per_group[g]))
|
|
by_id = {q["id"]: q for q in questions}
|
|
ordered = sorted(by_id)
|
|
dev = [dict(by_id[i], id=f"d{n:02d}") for n, i in enumerate((i for i in ordered if i not in held_ids), 1)]
|
|
held = [dict(by_id[i], id=f"h{n:02d}") for n, i in enumerate((i for i in ordered if i in held_ids), 1)]
|
|
dev_bytes = dump({"schema": SET_SCHEMA, "sources": data["sources"], "questions": dev})
|
|
held_bytes = dump({"schema": SET_SCHEMA, "sources": data["sources"], "questions": held})
|
|
part_bytes = Path(args.part).read_bytes()
|
|
manifest = {
|
|
"schema": MANIFEST_SCHEMA,
|
|
"seed": args.seed,
|
|
"draw": "per group in the order " + ", ".join(GROUPS) + ": random.Random(seed).sample(sorted(ids of the "
|
|
"group), quota) with one generator; quotas proportional to the group sizes (largest remainder); "
|
|
"both parts renumbered d../h.. in the order of the original ids",
|
|
"groups": {"all": counts, "set": group_counts(dev), "held_out": group_counts(held),
|
|
"part": group_counts(part_q)},
|
|
"set": {"file": args.set_label or Path(args.set_out).name, "sha256": sha256_bytes(dev_bytes),
|
|
"questions": len(dev)},
|
|
"held_out": {"sha256": sha256_bytes(held_bytes), "questions": len(held)},
|
|
"part": {"file": args.part_label or Path(args.part).name, "sha256": sha256_bytes(part_bytes),
|
|
"questions": len(part_q)},
|
|
"sources": {"copy": root.ref(), "part": part_root.ref()},
|
|
}
|
|
held_path.parent.mkdir(parents=True, exist_ok=True)
|
|
held_path.write_bytes(held_bytes)
|
|
set_out = Path(args.set_out)
|
|
set_out.parent.mkdir(parents=True, exist_ok=True)
|
|
set_out.write_bytes(dev_bytes)
|
|
manifest_path = Path(args.manifest)
|
|
manifest_path.parent.mkdir(parents=True, exist_ok=True)
|
|
manifest_path.write_bytes(dump(manifest))
|
|
print(f"seed: {args.seed}")
|
|
print("held out per group: " + ", ".join(f"{g} {per_group[g]}" for g in GROUPS))
|
|
print(f"set: {len(dev)} questions, sha256 {manifest['set']['sha256']}")
|
|
print(f"held out: {len(held)} questions, sha256 {manifest['held_out']['sha256']}")
|
|
print(f"part: {len(part_q)} questions, sha256 {manifest['part']['sha256']}")
|
|
print(f"manifest: {manifest_path}")
|
|
return 0
|
|
|
|
|
|
# ---------------------------------------------------------------------------------------------- freeze
|
|
|
|
|
|
def cmd_freeze(args: argparse.Namespace) -> int:
|
|
set_out, manifest_path = Path(args.set_out), Path(args.manifest)
|
|
for target in (set_out, manifest_path):
|
|
if target.exists():
|
|
raise RunError(f"{target} already exists; a frozen set is never frozen again")
|
|
need_paths, need_many = thresholds(args)
|
|
root = Root(Path(args.sources))
|
|
data = load_set(Path(args.set))
|
|
rep = check_set(data, root, need_paths, need_many)
|
|
print("\n".join(rep.lines()))
|
|
if not rep.passed:
|
|
raise RunError("the set fails the check; nothing was frozen")
|
|
questions = data["questions"]
|
|
if len(questions) != args.count:
|
|
raise RunError(f"the set holds {len(questions)} questions, not {args.count}; nothing was frozen")
|
|
order = sorted(q["id"] for q in questions)
|
|
random.Random(args.seed).shuffle(order)
|
|
by_id = {q["id"]: q for q in questions}
|
|
frozen = [dict(by_id[i], id=f"{args.prefix}{n:02d}") for n, i in enumerate(order, 1)]
|
|
set_bytes = dump({"schema": SET_SCHEMA, "sources": data["sources"], "questions": frozen})
|
|
raw = json.loads(root.manifest_bytes.decode("utf-8")).get("subset_of")
|
|
sources = {"part": root.ref()}
|
|
if isinstance(raw, dict) and isinstance(raw.get("manifest_sha256"), str):
|
|
sources["copy"] = {"frozen_at": str(raw.get("frozen_at", "")), "manifest_sha256": raw["manifest_sha256"]}
|
|
manifest = {
|
|
"schema": MANIFEST_SCHEMA,
|
|
"seed": args.seed,
|
|
"draw": "not split: random.Random(seed).shuffle(sorted ids), then renumbered "
|
|
f"{args.prefix}01.. in that order; the set is the part of its root",
|
|
"rules": {"min_paths": need_paths, "many_paths": MANY_PATHS, "min_many": need_many},
|
|
"groups": {"part": group_counts(frozen)},
|
|
"part": {"file": args.set_label or set_out.name, "sha256": sha256_bytes(set_bytes), "questions": len(frozen)},
|
|
"sources": sources,
|
|
}
|
|
set_out.parent.mkdir(parents=True, exist_ok=True)
|
|
set_out.write_bytes(set_bytes)
|
|
manifest_path.parent.mkdir(parents=True, exist_ok=True)
|
|
manifest_path.write_bytes(dump(manifest))
|
|
print(f"seed: {args.seed}")
|
|
print(f"frozen: {len(frozen)} questions, sha256 {manifest['part']['sha256']}")
|
|
print(f"manifest: {manifest_path}")
|
|
return 0
|
|
|
|
|
|
# ---------------------------------------------------------------------------------------------- gate
|
|
|
|
|
|
@dataclass
|
|
class Scored:
|
|
qid: str
|
|
group: str
|
|
answered: bool
|
|
keys_held: int
|
|
keys: int
|
|
paths_cited: int
|
|
paths_needed: int
|
|
|
|
@property
|
|
def answer_ok(self) -> bool:
|
|
return self.answered and self.keys_held == self.keys
|
|
|
|
@property
|
|
def source_ok(self) -> bool:
|
|
return self.answered and self.paths_cited == self.paths_needed
|
|
|
|
@property
|
|
def correct(self) -> bool:
|
|
return self.answer_ok and self.source_ok
|
|
|
|
def row(self) -> dict[str, Any]:
|
|
return {"id": self.qid, "group": self.group, "correct": self.correct, "keys_held": self.keys_held,
|
|
"keys": self.keys, "paths_cited": self.paths_cited, "paths_needed": self.paths_needed}
|
|
|
|
|
|
def cited_path(value: object) -> str | None:
|
|
"""A citation's path as K1 writes it; a leading "/" (a page's resource form) is accepted."""
|
|
if not isinstance(value, str):
|
|
return None
|
|
return value[1:] if value.startswith("/") else value
|
|
|
|
|
|
def score_one(q: dict[str, Any], a: dict[str, Any] | None, root: Root) -> Scored:
|
|
keys = as_list(q.get("answer_keys"))
|
|
needed = sorted({p for p, _ in evidence_of(q)})
|
|
if a is None:
|
|
return Scored(q["id"], str(q.get("group")), False, 0, len(keys), 0, len(needed))
|
|
answer = a.get("answer") if isinstance(a.get("answer"), str) else ""
|
|
held = sum(1 for k in keys if isinstance(k, dict) and key_holds(k, str(answer)))
|
|
supported: set[str] = set()
|
|
for c in as_list(a.get("citations")):
|
|
path = cited_path(c.get("path")) if isinstance(c, dict) else None
|
|
quote = c.get("quote") if isinstance(c, dict) else None
|
|
if path not in needed or not isinstance(quote, str) or not quote.strip():
|
|
continue
|
|
text = root.text(path, verify=False)
|
|
if text is not None and normalise_quote(quote) in normalise_quote(text):
|
|
supported.add(path)
|
|
return Scored(q["id"], str(q.get("group")), True, held, len(keys), len(supported), len(needed))
|
|
|
|
|
|
def manifest_part(manifest: dict[str, Any], set_hash: str) -> tuple[str, str]:
|
|
"""(part name, the K1 manifest sha256 its root must have)."""
|
|
sources = manifest.get("sources", {})
|
|
for name, root_name in (("set", "copy"), ("held_out", "copy"), ("part", "part")):
|
|
if manifest.get(name, {}).get("sha256") == set_hash:
|
|
return name, str(sources.get(root_name, {}).get("manifest_sha256"))
|
|
raise RunError(f"set sha256 {set_hash or '(missing)'} matches no part of the manifest")
|
|
|
|
|
|
def cmd_gate(args: argparse.Namespace) -> int:
|
|
if args.arm not in ARMS:
|
|
raise RunError(f"unknown arm {args.arm!r}; the arms are {', '.join(ARMS)}")
|
|
set_path = Path(args.set)
|
|
manifest = load_json(Path(args.manifest), "manifest")
|
|
if not isinstance(manifest, dict) or manifest.get("schema") != MANIFEST_SCHEMA:
|
|
raise RunError(f"{args.manifest} is not a manifest with schema {MANIFEST_SCHEMA}")
|
|
set_hash = sha256_bytes(set_path.read_bytes()) if set_path.is_file() else ""
|
|
part, root_sha = manifest_part(manifest, set_hash)
|
|
root = Root(Path(args.sources))
|
|
if root.manifest_sha256 != root_sha:
|
|
raise RunError(f"the root's {K1_FILE} sha256 {root.manifest_sha256} is not the manifest's {root_sha} "
|
|
f"for the {part}")
|
|
questions = load_set(set_path)["questions"]
|
|
n = len(questions)
|
|
answers: dict[str, dict[str, Any]] = {}
|
|
fail_closed = ""
|
|
answers_path = Path(args.answers)
|
|
if not answers_path.is_file():
|
|
fail_closed = f"no answers for arm {args.arm} at {answers_path}"
|
|
else:
|
|
data = load_json(answers_path, "answers")
|
|
if not isinstance(data, dict) or data.get("schema") != ANSWERS_SCHEMA or not isinstance(data.get("answers"),
|
|
list):
|
|
raise RunError(f"{answers_path} is not an answers file with schema {ANSWERS_SCHEMA}")
|
|
if data.get("arm") != args.arm:
|
|
raise RunError(f"answers file is for arm {data.get('arm')!r}, not {args.arm!r}")
|
|
for a in data["answers"]:
|
|
if isinstance(a, dict) and isinstance(a.get("id"), str):
|
|
answers[a["id"]] = a
|
|
results = sorted((score_one(q, answers.get(q["id"]), root) for q in questions), key=lambda r: r.qid)
|
|
missing = sum(1 for r in results if not r.answered)
|
|
correct = sum(r.correct for r in results)
|
|
print(f"set: {part}, {n} questions; arm: {args.arm}")
|
|
if fail_closed:
|
|
print(f"fail-closed: {fail_closed}")
|
|
print(f"missing answers: {missing}")
|
|
print(f"correct answer: {sum(r.answer_ok for r in results)} of {n}")
|
|
print(f"correct source: {sum(r.source_ok for r in results)} of {n}")
|
|
print(f"correct answer and source: {correct} of {n}")
|
|
for g in GROUPS:
|
|
members = [r for r in results if r.group == g]
|
|
print(f" {g}: {sum(r.correct for r in members)} of {len(members)}")
|
|
if args.out:
|
|
out = {"schema": RESULTS_SCHEMA, "arm": args.arm, "set_sha256": set_hash, "results": [r.row() for r in results]}
|
|
Path(args.out).parent.mkdir(parents=True, exist_ok=True)
|
|
Path(args.out).write_bytes(dump(out))
|
|
scored = not fail_closed and missing == 0
|
|
print("GREEN" if scored and n > 0 and correct == n else "RED")
|
|
print("SCORED" if scored else "UNSCORED (fail-closed)" if fail_closed else "INCOMPLETE")
|
|
return 0 if scored else 1
|
|
|
|
|
|
# ---------------------------------------------------------------------------------------------- stub
|
|
|
|
|
|
def stub_answer(q: dict[str, Any], kind: str) -> dict[str, Any]:
|
|
"""fasit: every key as its own sentence ("not <key>" when negated) and every span quoted; no-citations: the
|
|
same answer with no citations."""
|
|
keys = [k for k in as_list(q.get("answer_keys")) if isinstance(k, dict) and isinstance(k.get("text"), str)]
|
|
text = " ".join(("not " if k.get("negated") is True else "") + k["text"] + "." for k in keys)
|
|
citations = [{"path": p, "quote": s} for p, s in evidence_of(q)] if kind == "fasit" else []
|
|
return {"id": q["id"], "answer": text, "citations": citations}
|
|
|
|
|
|
def cmd_stub(args: argparse.Namespace) -> int:
|
|
if args.arm not in ARMS:
|
|
raise RunError(f"unknown arm {args.arm!r}; the arms are {', '.join(ARMS)}")
|
|
questions = load_set(Path(args.set))["questions"]
|
|
doc = {"schema": ANSWERS_SCHEMA, "arm": args.arm, "answers": [stub_answer(q, args.kind) for q in questions]}
|
|
out = Path(args.out)
|
|
out.parent.mkdir(parents=True, exist_ok=True)
|
|
out.write_bytes(dump(doc))
|
|
print(f"{args.kind}: {len(questions)} answers for arm {args.arm} at {out}")
|
|
return 0
|
|
|
|
|
|
# ---------------------------------------------------------------------------------------------- main
|
|
|
|
|
|
def build_parser() -> argparse.ArgumentParser:
|
|
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
sub = p.add_subparsers(dest="command", required=True)
|
|
c = sub.add_parser("check", help="check spans, paths, groups and answer keys against a K1 root")
|
|
c.add_argument("--set", required=True)
|
|
c.add_argument("--sources", required=True, help="the K1 root the set was made from")
|
|
c.add_argument("--per-question", action="store_true", help="also list distinct paths per question")
|
|
c.add_argument("--min-paths", type=int, default=MIN_PATHS, help="distinct paths every question needs")
|
|
c.add_argument("--min-many", type=int, default=0, help=f"questions that need {MANY_PATHS} or more paths")
|
|
i = sub.add_parser("inside", help="count the questions whose every span stands in a part root")
|
|
i.add_argument("--set", required=True)
|
|
i.add_argument("--part-root", required=True)
|
|
i.add_argument("--require-all", action="store_true", help="FAIL unless every question is inside")
|
|
s = sub.add_parser("split", help="draw the held-out questions with a fixed seed, stratified by group")
|
|
s.add_argument("--set", required=True)
|
|
s.add_argument("--sources", required=True)
|
|
s.add_argument("--seed", type=int, required=True)
|
|
s.add_argument("--held-out-count", type=int, default=20)
|
|
s.add_argument("--set-out", required=True)
|
|
s.add_argument("--set-label", default="", help="path of the set as written in the manifest")
|
|
s.add_argument("--held-out", required=True, help="file path OUTSIDE the repository; never overwritten")
|
|
s.add_argument("--part", required=True, help="the separate set for the first part (not split)")
|
|
s.add_argument("--part-sources", required=True, help="the part's K1 root")
|
|
s.add_argument("--part-label", default="", help="path of the part as written in the manifest")
|
|
s.add_argument("--manifest", required=True)
|
|
f = sub.add_parser("freeze", help="freeze a set not made by split, with a manifest of its own")
|
|
f.add_argument("--set", required=True)
|
|
f.add_argument("--sources", required=True, help="the K1 root the set names (a subset root such as the part)")
|
|
f.add_argument("--seed", type=int, required=True)
|
|
f.add_argument("--count", type=int, required=True, help="the number of questions the set must hold")
|
|
f.add_argument("--min-paths", type=int, required=True)
|
|
f.add_argument("--min-many", type=int, required=True)
|
|
f.add_argument("--prefix", required=True, help="id prefix of the frozen questions")
|
|
f.add_argument("--set-out", required=True)
|
|
f.add_argument("--set-label", default="", help="path of the set as written in the manifest")
|
|
f.add_argument("--manifest", required=True)
|
|
g = sub.add_parser("gate", help="score one arm's answers")
|
|
g.add_argument("--set", required=True)
|
|
g.add_argument("--manifest", required=True)
|
|
g.add_argument("--sources", required=True)
|
|
g.add_argument("--arm", required=True)
|
|
g.add_argument("--answers", required=True)
|
|
g.add_argument("--out", default="", help="write the per-question results (K5 point 5)")
|
|
t = sub.add_parser("stub", help="write a known arm's answers (fasit or no-citations)")
|
|
t.add_argument("--set", required=True)
|
|
t.add_argument("--arm", required=True)
|
|
t.add_argument("--kind", required=True, choices=("fasit", "no-citations"))
|
|
t.add_argument("--out", required=True)
|
|
return p
|
|
|
|
|
|
def main(argv: list[str] | None = None) -> int:
|
|
args = build_parser().parse_args(argv)
|
|
commands = {"check": cmd_check, "inside": cmd_inside, "split": cmd_split, "freeze": cmd_freeze, "gate": cmd_gate,
|
|
"stub": cmd_stub}
|
|
try:
|
|
return commands[args.command](args)
|
|
except RunError as exc:
|
|
print(f"FAIL: {exc}", file=sys.stderr)
|
|
return 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|