"""The question set for PLAN.md criterion 4 on collection 1 (contract K5 point 1): checker, part count, held-out split, answer gate and the stub arms. No model call. check every span stands byte for byte in its file in a K1 root, every question has at least two distinct paths, its group follows K1 and the path count, every path carries an answer key that stands in that path's spans, no key is wording, every negated key stands negated. PASS only N of N with N > 0. For a harder set, --min-paths raises the paths every question needs and --min-many sets how many questions need 4 or more. inside the questions whose every span stands in a part root (a K1 subset root, here the first part). split draws the held-out questions with a fixed seed, stratified by group, writes them OUTSIDE the repository, renumbers both parts and writes the manifest (seed, counts, sha256 values only). freeze freezes a set not made by split: it must pass check against its own root with the thresholds and hold exactly --count questions. It is renumbered in a seeded order and gets a manifest of its own that names it the part of that root (seed, rules, counts, sha256 values, the roots' K1 sha256; no text), so gate judges it as it judges the part. A frozen set or manifest is never overwritten. gate scores one arm: a question is correct when the answer holds every key (multidoc.key_holds) AND the citations quote, verbatim, every path the question needs. A missing answers file scores 0 (fail-closed). The set and the root must match the manifest. stub writes a known arm: `fasit` answers with the keys and quotes the spans; `no-citations` answers with the keys and cites nothing. The group is set by the sources, never by the writer: `history` when a path has `main_topic: false` in K1, otherwise `docs-2-3` (2-3 distinct paths) or `docs-4` (4 or more). Exit 0 = PASS / scored completely; 1 = FAIL or could not run. A check that cannot run is a FAIL. """ from __future__ import annotations import argparse import hashlib import json import math import random import re import sys from dataclasses import dataclass, field from pathlib import Path from typing import Any from keyform import count_keys from multidoc import KEY_MAX, KEY_MIN, SPAN_MAX, SPAN_MIN, key_holds, key_in, negated_in, normalise_quote, number_key SET_SCHEMA = "okf-wiki/answer-set/1" MANIFEST_SCHEMA = "okf-wiki/answer-set-manifest/1" ANSWERS_SCHEMA = "okf-wiki/answers/1" RESULTS_SCHEMA = "okf-wiki/answer-results/1" K1_SCHEMA = "okf-wiki/source-manifest/1" K1_FILE = "source-manifest.json" GROUPS = ("docs-2-3", "docs-4", "history") ARMS = ("filsok", "bundle") MIN_PATHS = 2 MANY_PATHS = 4 PATH_RE = re.compile(r"^kilde/[a-z0-9._-]+$") REPO_ROOT = Path(__file__).resolve().parents[1] class RunError(Exception): """The command cannot run; reported as FAIL with exit 1.""" def dump(data: Any) -> bytes: """The K5 file form: two writings of the same content are byte-identical.""" return (json.dumps(data, indent=1, sort_keys=True, ensure_ascii=False) + "\n").encode("utf-8") def sha256_bytes(data: bytes) -> str: return hashlib.sha256(data).hexdigest() def load_json(path: Path, what: str) -> Any: try: return json.loads(path.read_text(encoding="utf-8")) except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc: raise RunError(f"cannot read {what} {path}: {exc}") from exc def as_list(value: object) -> list[Any]: return value if isinstance(value, list) else [] def group_of(paths: list[str], main_topic: dict[str, bool]) -> str: """The group K1 gives a question with these distinct paths (see the module text).""" distinct = set(paths) if any(main_topic.get(p) is False for p in distinct): return "history" return "docs-4" if len(distinct) >= MANY_PATHS else "docs-2-3" # ---------------------------------------------------------------------------------------------- K1 root class Root: """A K1 root read with this module's own code (contract K1; okfwiki.sources is not imported).""" def __init__(self, path: Path) -> None: self.path = path manifest = path / K1_FILE if not path.is_dir() or not manifest.is_file(): raise RunError(f"{path} is not a K1 root: no {K1_FILE}") self.manifest_bytes = manifest.read_bytes() self.manifest_sha256 = sha256_bytes(self.manifest_bytes) try: data = json.loads(self.manifest_bytes.decode("utf-8")) except (UnicodeDecodeError, json.JSONDecodeError) as exc: raise RunError(f"{manifest} is not JSON: {exc}") from exc if not isinstance(data, dict) or data.get("schema") != K1_SCHEMA or not isinstance(data.get("sources"), list): raise RunError(f"{manifest} is not a K1 manifest with schema {K1_SCHEMA}") self.frozen_at = str(data.get("frozen_at", "")) self.entries: dict[str, dict[str, Any]] = {} for entry in data["sources"]: if not (isinstance(entry, dict) and isinstance(entry.get("path"), str) and isinstance(entry.get("main_topic"), bool) and isinstance(entry.get("sha256"), str)): raise RunError(f"{manifest} has an entry without path, main_topic or sha256: {entry!r}") self.entries[entry["path"]] = entry self.main_topic = {p: bool(e["main_topic"]) for p, e in self.entries.items()} self._text: dict[str, str | None] = {} self._bad: dict[str, str] = {} def ref(self) -> dict[str, str]: return {"frozen_at": self.frozen_at, "manifest_sha256": self.manifest_sha256} def text(self, path: str, verify: bool = True) -> str | None: """The text of a listed source, or None (with the reason in problem()) when it is not listed, missing, not UTF-8, or (with verify) its sha256 differs from K1.""" key = f"{verify}:{path}" if key not in self._text: self._text[key] = self._read(path, verify, key) return self._text[key] def _read(self, path: str, verify: bool, key: str) -> str | None: if not PATH_RE.match(path) or path not in self.entries: self._bad[key] = f"{path} is not a path K1 lists in {self.path}" return None target = self.path / path try: data = target.read_bytes() except OSError: self._bad[key] = f"{path} is missing in {self.path}" return None if verify and sha256_bytes(data) != self.entries[path]["sha256"]: self._bad[key] = f"{path} sha256 differs from K1 in {self.path}" return None try: return data.decode("utf-8") except UnicodeDecodeError: self._bad[key] = f"{path} is not UTF-8" return None def problem(self, path: str, verify: bool = True) -> str: return self._bad.get(f"{verify}:{path}", "") def load_set(path: Path) -> dict[str, Any]: data = load_json(path, "set") if (not isinstance(data, dict) or data.get("schema") != SET_SCHEMA or not isinstance(data.get("questions"), list) or not isinstance(data.get("sources"), dict)): raise RunError(f"{path} is not a set with schema {SET_SCHEMA}") return data def evidence_of(q: dict[str, Any]) -> list[tuple[str, str]]: """(path, span) for every well-formed evidence entry.""" out = [] for e in as_list(q.get("evidence")): if isinstance(e, dict) and isinstance(e.get("path"), str) and isinstance(e.get("span"), str): out.append((e["path"], e["span"])) return out # ---------------------------------------------------------------------------------------------- check @dataclass class CheckReport: questions: int = 0 spans: int = 0 spans_verbatim: int = 0 all_spans_verbatim: int = 0 min_paths: int = 0 group_ok: int = 0 key_per_path: int = 0 keys: int = 0 wording_keys: int = 0 negated_keys: int = 0 negated_ok: int = 0 groups: dict[str, int] = field(default_factory=lambda: {g: 0 for g in GROUPS}) errors: list[str] = field(default_factory=list) paths_per_question: dict[str, int] = field(default_factory=dict) need_paths: int = MIN_PATHS need_many: int = 0 many: int = 0 @property def passed(self) -> bool: n = self.questions return (n > 0 and not self.errors and self.spans_verbatim == self.spans and self.all_spans_verbatim == n and self.min_paths == n and self.group_ok == n and self.key_per_path == n and self.wording_keys == 0 and self.negated_ok == self.negated_keys and self.many >= self.need_many) def lines(self) -> list[str]: n = self.questions out = [f"error: {e}" for e in self.errors] out += [ f"questions: {n}", "groups: " + ", ".join(f"{g} {self.groups[g]}" for g in GROUPS), f"spans verbatim: {self.spans_verbatim} of {self.spans}", f"questions with every span verbatim: {self.all_spans_verbatim} of {n}", f"questions with >= {self.need_paths} distinct paths: {self.min_paths} of {n}", ] if self.need_many: out.append(f"questions with >= {MANY_PATHS} distinct paths: {self.many} of {n} (required >= {self.need_many})") out += [ f"questions whose group matches K1 and the path count: {self.group_ok} of {n}", f"questions with a key in every path's spans: {self.key_per_path} of {n}", f"answer keys that are wording: {self.wording_keys} of {self.keys}", f"negated keys negated in their span: {self.negated_ok} of {self.negated_keys}", "PASS" if self.passed else "FAIL", ] return out def check_set(data: dict[str, Any], root: Root, need_paths: int = MIN_PATHS, need_many: int = 0) -> CheckReport: questions = data["questions"] rep = CheckReport(questions=len(questions), need_paths=need_paths, need_many=need_many) ref = data["sources"] if ref.get("manifest_sha256") != root.manifest_sha256 or ref.get("frozen_at") != root.frozen_at: rep.errors.append(f"the set names sources {ref.get('frozen_at')!r} manifest_sha256 " f"{ref.get('manifest_sha256')!r}, the root has {root.frozen_at!r} {root.manifest_sha256}") form = count_keys(questions) rep.keys, rep.wording_keys = form.keys, form.wording seen: set[str] = set() for q in questions: qid = q.get("id") if isinstance(q, dict) else None if not isinstance(qid, str) or not qid: rep.errors.append("question without an id") continue if qid in seen: rep.errors.append(f"duplicate id {qid}") seen.add(qid) if not isinstance(q.get("question"), str) or not q["question"].strip(): rep.errors.append(f"{qid}: empty question") raw_evidence = as_list(q.get("evidence")) evidence = evidence_of(q) rep.spans += len(raw_evidence) if len(evidence) != len(raw_evidence): rep.errors.append(f"{qid}: an evidence entry without a path or a span") spans_by_path: dict[str, list[str]] = {} verbatim = 0 for path, span in evidence: if not SPAN_MIN <= len(span) <= SPAN_MAX: rep.errors.append(f"{qid}: span length {len(span)} in {path} outside {SPAN_MIN}-{SPAN_MAX}") text = root.text(path) if text is None: rep.errors.append(f"{qid}: {root.problem(path)}") elif span in text: verbatim += 1 spans_by_path.setdefault(path, []).append(span) rep.spans_verbatim += verbatim if raw_evidence and verbatim == len(raw_evidence): rep.all_spans_verbatim += 1 paths = sorted(spans_by_path) rep.paths_per_question[qid] = len(paths) if len(paths) >= need_paths: rep.min_paths += 1 if len(paths) >= MANY_PATHS: rep.many += 1 group = q.get("group") if group in GROUPS: rep.groups[group] += 1 else: rep.errors.append(f"{qid}: group {group!r} is not one of {', '.join(GROUPS)}") if paths and all(p in root.entries for p in paths) and group == group_of(paths, root.main_topic): rep.group_ok += 1 keyed: set[str] = set() keys_ok = bool(as_list(q.get("answer_keys"))) for k in as_list(q.get("answer_keys")): kpath, key = (k.get("path"), k.get("text")) if isinstance(k, dict) else (None, None) if not isinstance(key, str) or not isinstance(kpath, str): keys_ok = False rep.errors.append(f"{qid}: malformed answer key") continue if kpath not in spans_by_path: rep.errors.append(f"{qid}: answer key path {kpath} is not an evidence path") if number_key(key) is None and not KEY_MIN <= len(key.strip()) <= KEY_MAX: rep.errors.append(f"{qid}: key length {len(key.strip())} outside {KEY_MIN}-{KEY_MAX}") if any(key_in(key, s) for s in spans_by_path.get(kpath, [])): keyed.add(kpath) else: keys_ok = False if "negated" in k: if k["negated"] is not True: rep.errors.append(f"{qid}: negated must be true or absent") continue rep.negated_keys += 1 rep.negated_ok += any(negated_in(key, s) for s in spans_by_path.get(kpath, [])) if keys_ok and paths and keyed == set(paths): rep.key_per_path += 1 return rep def thresholds(args: argparse.Namespace) -> tuple[int, int]: if args.min_paths < MIN_PATHS or args.min_many < 0: raise RunError(f"--min-paths must be at least {MIN_PATHS} and --min-many at least 0") return args.min_paths, args.min_many def cmd_check(args: argparse.Namespace) -> int: rep = check_set(load_set(Path(args.set)), Root(Path(args.sources)), *thresholds(args)) if args.per_question: for qid, n in rep.paths_per_question.items(): print(f"{qid}\t{n} paths") print("\n".join(rep.lines())) return 0 if rep.passed else 1 # ---------------------------------------------------------------------------------------------- inside def inside_ids(questions: list[dict[str, Any]], part: Root) -> list[str]: """The ids of the questions whose every span stands in its file in the part root (path listed there).""" out = [] for q in questions: evidence = evidence_of(q) if isinstance(q, dict) else [] if evidence and len(evidence) == len(as_list(q.get("evidence"))): texts = [(part.text(p, verify=False), s) for p, s in evidence] if all(t is not None and s in t for t, s in texts): out.append(str(q.get("id"))) return out def cmd_inside(args: argparse.Namespace) -> int: questions = load_set(Path(args.set))["questions"] part = Root(Path(args.part_root)) ids = set(inside_ids(questions, part)) n = len(questions) print(f"questions with every span in the part root: {len(ids)} of {n}") for g in GROUPS: members = [q for q in questions if isinstance(q, dict) and q.get("group") == g] print(f" {g}: {sum(1 for q in members if q.get('id') in ids)} of {len(members)}") if not args.require_all: return 0 passed = n > 0 and len(ids) == n print("PASS" if passed else "FAIL") return 0 if passed else 1 # ---------------------------------------------------------------------------------------------- split def inside_repo(path: Path) -> bool: try: path.resolve().relative_to(REPO_ROOT) return True except ValueError: return False def quotas(counts: dict[str, int], n_held: int) -> dict[str, int]: """Held-out questions per group, proportional to the group's size (largest remainder, ties in GROUPS order).""" total = sum(counts.values()) exact = {g: n_held * counts[g] / total for g in GROUPS} out = {g: math.floor(exact[g]) for g in GROUPS} for g in sorted(GROUPS, key=lambda g: (-(exact[g] - out[g]), GROUPS.index(g)))[: n_held - sum(out.values())]: out[g] += 1 return out def group_counts(questions: list[dict[str, Any]]) -> dict[str, int]: return {g: sum(1 for q in questions if q.get("group") == g) for g in GROUPS} def cmd_split(args: argparse.Namespace) -> int: held_path = Path(args.held_out) if inside_repo(held_path): raise RunError(f"held-out path {held_path} is inside the repository; it must live outside it") if held_path.exists(): raise RunError(f"held-out file {held_path} already exists; a frozen draw is never redrawn") root, part_root = Root(Path(args.sources)), Root(Path(args.part_sources)) data, part_data = load_set(Path(args.set)), load_set(Path(args.part)) for label, d, r in (("set", data, root), ("part", part_data, part_root)): rep = check_set(d, r) if not rep.passed: print("\n".join(rep.lines())) raise RunError(f"the {label} fails the check; nothing was split") part_q = part_data["questions"] if len(inside_ids(part_q, part_root)) != len(part_q): raise RunError("a question of the part has a span outside the part root; nothing was split") questions = data["questions"] n_held = args.held_out_count if not 0 < n_held < len(questions): raise RunError(f"cannot hold out {n_held} of {len(questions)}") counts = group_counts(questions) per_group = quotas(counts, n_held) rng = random.Random(args.seed) held_ids: set[str] = set() for g in GROUPS: ids = sorted(q["id"] for q in questions if q["group"] == g) held_ids.update(rng.sample(ids, per_group[g])) by_id = {q["id"]: q for q in questions} ordered = sorted(by_id) dev = [dict(by_id[i], id=f"d{n:02d}") for n, i in enumerate((i for i in ordered if i not in held_ids), 1)] held = [dict(by_id[i], id=f"h{n:02d}") for n, i in enumerate((i for i in ordered if i in held_ids), 1)] dev_bytes = dump({"schema": SET_SCHEMA, "sources": data["sources"], "questions": dev}) held_bytes = dump({"schema": SET_SCHEMA, "sources": data["sources"], "questions": held}) part_bytes = Path(args.part).read_bytes() manifest = { "schema": MANIFEST_SCHEMA, "seed": args.seed, "draw": "per group in the order " + ", ".join(GROUPS) + ": random.Random(seed).sample(sorted(ids of the " "group), quota) with one generator; quotas proportional to the group sizes (largest remainder); " "both parts renumbered d../h.. in the order of the original ids", "groups": {"all": counts, "set": group_counts(dev), "held_out": group_counts(held), "part": group_counts(part_q)}, "set": {"file": args.set_label or Path(args.set_out).name, "sha256": sha256_bytes(dev_bytes), "questions": len(dev)}, "held_out": {"sha256": sha256_bytes(held_bytes), "questions": len(held)}, "part": {"file": args.part_label or Path(args.part).name, "sha256": sha256_bytes(part_bytes), "questions": len(part_q)}, "sources": {"copy": root.ref(), "part": part_root.ref()}, } held_path.parent.mkdir(parents=True, exist_ok=True) held_path.write_bytes(held_bytes) set_out = Path(args.set_out) set_out.parent.mkdir(parents=True, exist_ok=True) set_out.write_bytes(dev_bytes) manifest_path = Path(args.manifest) manifest_path.parent.mkdir(parents=True, exist_ok=True) manifest_path.write_bytes(dump(manifest)) print(f"seed: {args.seed}") print("held out per group: " + ", ".join(f"{g} {per_group[g]}" for g in GROUPS)) print(f"set: {len(dev)} questions, sha256 {manifest['set']['sha256']}") print(f"held out: {len(held)} questions, sha256 {manifest['held_out']['sha256']}") print(f"part: {len(part_q)} questions, sha256 {manifest['part']['sha256']}") print(f"manifest: {manifest_path}") return 0 # ---------------------------------------------------------------------------------------------- freeze def cmd_freeze(args: argparse.Namespace) -> int: set_out, manifest_path = Path(args.set_out), Path(args.manifest) for target in (set_out, manifest_path): if target.exists(): raise RunError(f"{target} already exists; a frozen set is never frozen again") need_paths, need_many = thresholds(args) root = Root(Path(args.sources)) data = load_set(Path(args.set)) rep = check_set(data, root, need_paths, need_many) print("\n".join(rep.lines())) if not rep.passed: raise RunError("the set fails the check; nothing was frozen") questions = data["questions"] if len(questions) != args.count: raise RunError(f"the set holds {len(questions)} questions, not {args.count}; nothing was frozen") order = sorted(q["id"] for q in questions) random.Random(args.seed).shuffle(order) by_id = {q["id"]: q for q in questions} frozen = [dict(by_id[i], id=f"{args.prefix}{n:02d}") for n, i in enumerate(order, 1)] set_bytes = dump({"schema": SET_SCHEMA, "sources": data["sources"], "questions": frozen}) raw = json.loads(root.manifest_bytes.decode("utf-8")).get("subset_of") sources = {"part": root.ref()} if isinstance(raw, dict) and isinstance(raw.get("manifest_sha256"), str): sources["copy"] = {"frozen_at": str(raw.get("frozen_at", "")), "manifest_sha256": raw["manifest_sha256"]} manifest = { "schema": MANIFEST_SCHEMA, "seed": args.seed, "draw": "not split: random.Random(seed).shuffle(sorted ids), then renumbered " f"{args.prefix}01.. in that order; the set is the part of its root", "rules": {"min_paths": need_paths, "many_paths": MANY_PATHS, "min_many": need_many}, "groups": {"part": group_counts(frozen)}, "part": {"file": args.set_label or set_out.name, "sha256": sha256_bytes(set_bytes), "questions": len(frozen)}, "sources": sources, } set_out.parent.mkdir(parents=True, exist_ok=True) set_out.write_bytes(set_bytes) manifest_path.parent.mkdir(parents=True, exist_ok=True) manifest_path.write_bytes(dump(manifest)) print(f"seed: {args.seed}") print(f"frozen: {len(frozen)} questions, sha256 {manifest['part']['sha256']}") print(f"manifest: {manifest_path}") return 0 # ---------------------------------------------------------------------------------------------- gate @dataclass class Scored: qid: str group: str answered: bool keys_held: int keys: int paths_cited: int paths_needed: int @property def answer_ok(self) -> bool: return self.answered and self.keys_held == self.keys @property def source_ok(self) -> bool: return self.answered and self.paths_cited == self.paths_needed @property def correct(self) -> bool: return self.answer_ok and self.source_ok def row(self) -> dict[str, Any]: return {"id": self.qid, "group": self.group, "correct": self.correct, "keys_held": self.keys_held, "keys": self.keys, "paths_cited": self.paths_cited, "paths_needed": self.paths_needed} def cited_path(value: object) -> str | None: """A citation's path as K1 writes it; a leading "/" (a page's resource form) is accepted.""" if not isinstance(value, str): return None return value[1:] if value.startswith("/") else value def score_one(q: dict[str, Any], a: dict[str, Any] | None, root: Root) -> Scored: keys = as_list(q.get("answer_keys")) needed = sorted({p for p, _ in evidence_of(q)}) if a is None: return Scored(q["id"], str(q.get("group")), False, 0, len(keys), 0, len(needed)) answer = a.get("answer") if isinstance(a.get("answer"), str) else "" held = sum(1 for k in keys if isinstance(k, dict) and key_holds(k, str(answer))) supported: set[str] = set() for c in as_list(a.get("citations")): path = cited_path(c.get("path")) if isinstance(c, dict) else None quote = c.get("quote") if isinstance(c, dict) else None if path not in needed or not isinstance(quote, str) or not quote.strip(): continue text = root.text(path, verify=False) if text is not None and normalise_quote(quote) in normalise_quote(text): supported.add(path) return Scored(q["id"], str(q.get("group")), True, held, len(keys), len(supported), len(needed)) def manifest_part(manifest: dict[str, Any], set_hash: str) -> tuple[str, str]: """(part name, the K1 manifest sha256 its root must have).""" sources = manifest.get("sources", {}) for name, root_name in (("set", "copy"), ("held_out", "copy"), ("part", "part")): if manifest.get(name, {}).get("sha256") == set_hash: return name, str(sources.get(root_name, {}).get("manifest_sha256")) raise RunError(f"set sha256 {set_hash or '(missing)'} matches no part of the manifest") def cmd_gate(args: argparse.Namespace) -> int: if args.arm not in ARMS: raise RunError(f"unknown arm {args.arm!r}; the arms are {', '.join(ARMS)}") set_path = Path(args.set) manifest = load_json(Path(args.manifest), "manifest") if not isinstance(manifest, dict) or manifest.get("schema") != MANIFEST_SCHEMA: raise RunError(f"{args.manifest} is not a manifest with schema {MANIFEST_SCHEMA}") set_hash = sha256_bytes(set_path.read_bytes()) if set_path.is_file() else "" part, root_sha = manifest_part(manifest, set_hash) root = Root(Path(args.sources)) if root.manifest_sha256 != root_sha: raise RunError(f"the root's {K1_FILE} sha256 {root.manifest_sha256} is not the manifest's {root_sha} " f"for the {part}") questions = load_set(set_path)["questions"] n = len(questions) answers: dict[str, dict[str, Any]] = {} fail_closed = "" answers_path = Path(args.answers) if not answers_path.is_file(): fail_closed = f"no answers for arm {args.arm} at {answers_path}" else: data = load_json(answers_path, "answers") if not isinstance(data, dict) or data.get("schema") != ANSWERS_SCHEMA or not isinstance(data.get("answers"), list): raise RunError(f"{answers_path} is not an answers file with schema {ANSWERS_SCHEMA}") if data.get("arm") != args.arm: raise RunError(f"answers file is for arm {data.get('arm')!r}, not {args.arm!r}") for a in data["answers"]: if isinstance(a, dict) and isinstance(a.get("id"), str): answers[a["id"]] = a results = sorted((score_one(q, answers.get(q["id"]), root) for q in questions), key=lambda r: r.qid) missing = sum(1 for r in results if not r.answered) correct = sum(r.correct for r in results) print(f"set: {part}, {n} questions; arm: {args.arm}") if fail_closed: print(f"fail-closed: {fail_closed}") print(f"missing answers: {missing}") print(f"correct answer: {sum(r.answer_ok for r in results)} of {n}") print(f"correct source: {sum(r.source_ok for r in results)} of {n}") print(f"correct answer and source: {correct} of {n}") for g in GROUPS: members = [r for r in results if r.group == g] print(f" {g}: {sum(r.correct for r in members)} of {len(members)}") if args.out: out = {"schema": RESULTS_SCHEMA, "arm": args.arm, "set_sha256": set_hash, "results": [r.row() for r in results]} Path(args.out).parent.mkdir(parents=True, exist_ok=True) Path(args.out).write_bytes(dump(out)) scored = not fail_closed and missing == 0 print("GREEN" if scored and n > 0 and correct == n else "RED") print("SCORED" if scored else "UNSCORED (fail-closed)" if fail_closed else "INCOMPLETE") return 0 if scored else 1 # ---------------------------------------------------------------------------------------------- stub def stub_answer(q: dict[str, Any], kind: str) -> dict[str, Any]: """fasit: every key as its own sentence ("not " when negated) and every span quoted; no-citations: the same answer with no citations.""" keys = [k for k in as_list(q.get("answer_keys")) if isinstance(k, dict) and isinstance(k.get("text"), str)] text = " ".join(("not " if k.get("negated") is True else "") + k["text"] + "." for k in keys) citations = [{"path": p, "quote": s} for p, s in evidence_of(q)] if kind == "fasit" else [] return {"id": q["id"], "answer": text, "citations": citations} def cmd_stub(args: argparse.Namespace) -> int: if args.arm not in ARMS: raise RunError(f"unknown arm {args.arm!r}; the arms are {', '.join(ARMS)}") questions = load_set(Path(args.set))["questions"] doc = {"schema": ANSWERS_SCHEMA, "arm": args.arm, "answers": [stub_answer(q, args.kind) for q in questions]} out = Path(args.out) out.parent.mkdir(parents=True, exist_ok=True) out.write_bytes(dump(doc)) print(f"{args.kind}: {len(questions)} answers for arm {args.arm} at {out}") return 0 # ---------------------------------------------------------------------------------------------- main def build_parser() -> argparse.ArgumentParser: p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) sub = p.add_subparsers(dest="command", required=True) c = sub.add_parser("check", help="check spans, paths, groups and answer keys against a K1 root") c.add_argument("--set", required=True) c.add_argument("--sources", required=True, help="the K1 root the set was made from") c.add_argument("--per-question", action="store_true", help="also list distinct paths per question") c.add_argument("--min-paths", type=int, default=MIN_PATHS, help="distinct paths every question needs") c.add_argument("--min-many", type=int, default=0, help=f"questions that need {MANY_PATHS} or more paths") i = sub.add_parser("inside", help="count the questions whose every span stands in a part root") i.add_argument("--set", required=True) i.add_argument("--part-root", required=True) i.add_argument("--require-all", action="store_true", help="FAIL unless every question is inside") s = sub.add_parser("split", help="draw the held-out questions with a fixed seed, stratified by group") s.add_argument("--set", required=True) s.add_argument("--sources", required=True) s.add_argument("--seed", type=int, required=True) s.add_argument("--held-out-count", type=int, default=20) s.add_argument("--set-out", required=True) s.add_argument("--set-label", default="", help="path of the set as written in the manifest") s.add_argument("--held-out", required=True, help="file path OUTSIDE the repository; never overwritten") s.add_argument("--part", required=True, help="the separate set for the first part (not split)") s.add_argument("--part-sources", required=True, help="the part's K1 root") s.add_argument("--part-label", default="", help="path of the part as written in the manifest") s.add_argument("--manifest", required=True) f = sub.add_parser("freeze", help="freeze a set not made by split, with a manifest of its own") f.add_argument("--set", required=True) f.add_argument("--sources", required=True, help="the K1 root the set names (a subset root such as the part)") f.add_argument("--seed", type=int, required=True) f.add_argument("--count", type=int, required=True, help="the number of questions the set must hold") f.add_argument("--min-paths", type=int, required=True) f.add_argument("--min-many", type=int, required=True) f.add_argument("--prefix", required=True, help="id prefix of the frozen questions") f.add_argument("--set-out", required=True) f.add_argument("--set-label", default="", help="path of the set as written in the manifest") f.add_argument("--manifest", required=True) g = sub.add_parser("gate", help="score one arm's answers") g.add_argument("--set", required=True) g.add_argument("--manifest", required=True) g.add_argument("--sources", required=True) g.add_argument("--arm", required=True) g.add_argument("--answers", required=True) g.add_argument("--out", default="", help="write the per-question results (K5 point 5)") t = sub.add_parser("stub", help="write a known arm's answers (fasit or no-citations)") t.add_argument("--set", required=True) t.add_argument("--arm", required=True) t.add_argument("--kind", required=True, choices=("fasit", "no-citations")) t.add_argument("--out", required=True) return p def main(argv: list[str] | None = None) -> int: args = build_parser().parse_args(argv) commands = {"check": cmd_check, "inside": cmd_inside, "split": cmd_split, "freeze": cmd_freeze, "gate": cmd_gate, "stub": cmd_stub} try: return commands[args.command](args) except RunError as exc: print(f"FAIL: {exc}", file=sys.stderr) return 1 if __name__ == "__main__": sys.exit(main())