"""The reader calibration for PLAN.md criterion 2: set checker, seeded build with a call layout, stub reader and verdict gate. No model call. The reader judges up to ten claims per call and gives one verdict per claim (K3 in docs/bolge1/0-felles.local.md). The calibration therefore comes as three frozen files beside the manifest: the set (claims and excerpts, what the reader sees), the key (label and kind per claim) and the call layout (which claims stand together in a call, and where). Labels: "unsupported" (planted, must be caught), "supported" (counted controls, must be accepted) and "filler-supported" (supported claims that fill the calls so they resemble a real page; reported apart and never counted in the thresholds). check every excerpt stands verbatim in its source file (and, in a built set, at its stated line), every kind of the two counted labels is present and fits its label, a wrong-number claim carries a number the excerpts do not, a supported or filler claim is not a verbatim copy of its excerpt, and a twin_of names an unsupported pair. A built set is checked with its layout against the manifest: sha256 of set, key, layout and sources, the layout rules, and the counts per label, kind and position. build checks a labelled draft, lays it out in calls with a fixed seed, renumbers the claims (c001.., in call order) and the calls (k001..), and writes the set, the key, the layout and a manifest with sha256, counts and thresholds only. A built set is frozen: build never overwrites it. An earlier manifest at the same path moves into the new one's history. stub writes a verdict file without a reader: the same verdict for every claim, or the key's verdicts, and optionally "supported" for every claim at one position (the position-mild stub). Its model is "stub". gate scores one reader's verdicts against the key: totals, caught per kind and caught per position, and the fillers apart; never a result per call or per pair. A missing, invalid or duplicated verdict counts as wrong (fail-closed). GREEN only when every unsupported claim is caught at every position, the counted supported claims reach the manifest's threshold, no verdict is missing, and the verdicts name exactly one model (their "models", written by okfwiki.reader collect from the readers' transcripts; PLAN § Rammer 23). It prints the model it scored. With --passed FILE a GREEN run is added to FILE, the record of green runs (okf-wiki/calibration-passed/1: model, set_sha256, key_sha256, verdicts_sha256, reader; no time, and an entry already there is not added twice); a RED run never writes it. okfwiki.reader collect --gate-passed FILE turns a reading into verdicts only for a model in FILE with a run on the set and key of the tracked manifest, so a model change, or a new set, needs a new gate run. Both read FILE with okfwiki.reader.read_passed. Exit 0 = PASS / GREEN / written; 1 = FAIL / RED or could not run. A check that cannot run is a FAIL. """ from __future__ import annotations import argparse import hashlib import json import math import random import re import sys from dataclasses import dataclass, field from pathlib import Path from typing import Any sys.path.insert(0, str(Path(__file__).resolve().parents[1])) from okfwiki import reader # noqa: E402 DRAFT_SCHEMA = "okf-wiki/calibration-draft/1" SET_SCHEMA = "okf-wiki/calibration-set/1" KEY_SCHEMA = "okf-wiki/calibration-key/1" CALLS_SCHEMA = "okf-wiki/reader-calls/1" MANIFEST_SCHEMA = "okf-wiki/calibration-manifest/2" PASSED_SCHEMA = reader.PASSED_SCHEMA STUB_MODEL = "stub" VERDICTS_SCHEMA_NOTE = ( '{"reader": "", "models": [""], "verdicts": [{"id": "c001", "verdict": "supported" | "unsupported"}]}' ) UNSUPPORTED = "unsupported" SUPPORTED = "supported" FILLER = "filler-supported" LABELS = (UNSUPPORTED, SUPPORTED, FILLER) COUNTED = (UNSUPPORTED, SUPPORTED) VERDICTS = (SUPPORTED, UNSUPPORTED) UNSUPPORTED_KINDS = ( "overgeneralisation", "wrong-number", "swapped-cause", "not-mentioned", "reversed-polarity", "conflated-sources", ) SUPPORTED_KINDS = ("paraphrase", "inference", "synthesis") KINDS = {UNSUPPORTED: UNSUPPORTED_KINDS, SUPPORTED: SUPPORTED_KINDS, FILLER: SUPPORTED_KINDS} TWO_EXCERPT_KINDS = ("conflated-sources", "synthesis") MAX_EXCERPTS = 2 CLAIM_MIN, CLAIM_MAX = 10, 400 EXCERPT_MIN, EXCERPT_MAX = 20, 1500 CALL_SIZE_MAX = 10 # K3 point 1: up to ten claims per call, positions 1-10 ACCEPTED_SHARE = 0.9 # PM's choice (order E3): a reader that rejects more than 10 % of supported claims is unusable. NUMBER = re.compile(r"\d+(?:[.,]\d+)*") LAYOUT_STATS = ("calls", "call_size_max", "unsupported_alone", "unsupported_per_call") LAYOUT_RULE = ( "random.Random(seed): each label's pairs sorted by draft id and shuffled; the first `lone` unsupported pairs alone" " in a call at positions spread evenly over 1-10, the rest `dense` to a call; counted supported pairs one per call" " in call order; filler pairs fill every call to ten in a cycle, never twice in a call and never beside the" " unsupported pair they are the twin of; the other positions shuffled; ids renumbered c001.. in call order," " calls k001.." ) class RunError(Exception): """The command cannot run; reported as FAIL with exit 1.""" # ---------------------------------------------------------------------------------------------- helpers def sha256_bytes(data: bytes) -> str: return hashlib.sha256(data).hexdigest() def tree_sha256(sources: Path) -> str: """One hash over the flat sources directory: sorted "name NUL sha256 LF" lines (same as gates/multidoc.py).""" lines = [] for p in sorted(sources.iterdir(), key=lambda p: p.name): if p.is_file(): lines.append(f"{p.name}\0{sha256_bytes(p.read_bytes())}\n") return sha256_bytes("".join(lines).encode("utf-8")) def count_files(sources: Path) -> int: return sum(1 for p in sources.iterdir() if p.is_file()) def normalise(text: str) -> str: """Case-folded, whitespace collapsed, outer punctuation stripped. Used for the verbatim-copy test only.""" return re.sub(r"\s+", " ", text).strip().strip(".,;:!?").strip().casefold() def numbers(text: str) -> set[str]: return {n.replace(",", "") for n in NUMBER.findall(text)} def as_list(value: object) -> list[Any]: return value if isinstance(value, list) else [] def as_dict(value: object) -> dict[str, Any]: return value if isinstance(value, dict) else {} def is_plain_name(name: object) -> bool: return isinstance(name, str) and name != "" and "/" not in name and "\\" not in name and name not in (".", "..") def load_json(path: Path, what: str) -> Any: try: return json.loads(path.read_text(encoding="utf-8")) except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc: raise RunError(f"cannot read {what} {path}: {exc}") from exc def load_list(path: Path, schema: str, key: str, what: str) -> list[dict[str, Any]]: data = load_json(path, what) if not isinstance(data, dict) or data.get("schema") != schema or not isinstance(data.get(key), list): raise RunError(f"{path} is not a {what} with schema {schema}") items: list[dict[str, Any]] = data[key] return items def require_dir(path: Path, what: str) -> None: if not path.is_dir(): raise RunError(f"{what} {path} is not a directory") def file_sha256(path: Path) -> str: return sha256_bytes(path.read_bytes()) if path.is_file() else "(missing)" def line_of(text: str, excerpt: str, line: int) -> bool: """True when an occurrence of the excerpt starts on the given 1-based line.""" start = text.find(excerpt) while start != -1: if text.count("\n", 0, start) + 1 == line: return True start = text.find(excerpt, start + 1) return False def first_line(text: str, excerpt: str) -> int: return text.count("\n", 0, text.find(excerpt)) + 1 def verdict_for(label: object) -> str: """The right verdict for a label: only an unsupported claim should be rejected.""" return UNSUPPORTED if label == UNSUPPORTED else SUPPORTED class SourceCache: def __init__(self, root: Path) -> None: self.root = root self._text: dict[str, str | None] = {} def text(self, name: str) -> str | None: """File text, or None when the name is not a plain file directly in the sources directory.""" if not is_plain_name(name): return None if name not in self._text: p = self.root / name try: self._text[name] = p.read_text(encoding="utf-8") if p.is_file() else None except (OSError, UnicodeDecodeError): self._text[name] = None return self._text[name] # ---------------------------------------------------------------------------------------------- check @dataclass class CheckReport: pairs: int = 0 unsupported: int = 0 supported: int = 0 filler: int = 0 excerpts: int = 0 excerpts_verbatim: int = 0 lines_checked: int = 0 lines_correct: int = 0 by_kind: dict[str, int] = field(default_factory=lambda: {k: 0 for lab in COUNTED for k in KINDS[lab]}) filler_by_kind: dict[str, int] = field(default_factory=lambda: {k: 0 for k in KINDS[FILLER]}) errors: list[str] = field(default_factory=list) notes: list[str] = field(default_factory=list) @property def passed(self) -> bool: return ( self.pairs > 0 and not self.errors and self.excerpts_verbatim == self.excerpts and self.lines_correct == self.lines_checked ) def lines(self) -> list[str]: out = [f"error: {e}" for e in self.errors] out += [ f"pairs: {self.pairs} ({self.unsupported} {UNSUPPORTED}, {self.supported} {SUPPORTED}, {self.filler} {FILLER})", f"excerpts verbatim: {self.excerpts_verbatim} of {self.excerpts}", ] if self.lines_checked: out.append(f"excerpt lines correct: {self.lines_correct} of {self.lines_checked}") for label in COUNTED: out.append(f"{label} by kind: " + ", ".join(f"{k} {self.by_kind[k]}" for k in KINDS[label])) out.append(f"{FILLER} by kind: " + ", ".join(f"{k} {self.filler_by_kind[k]}" for k in KINDS[FILLER])) out += self.notes out.append("PASS" if self.passed else "FAIL") return out def check_pairs(pairs: list[dict[str, Any]], sources: Path, with_lines: bool = False) -> CheckReport: """Check labelled pairs ({id, label, kind, claim, excerpts: [{file, text[, line]}][, twin_of]}) against the sources.""" require_dir(sources, "sources") cache = SourceCache(sources) rep = CheckReport(pairs=len(pairs)) unsupported_ids = {p.get("id") for p in pairs if isinstance(p, dict) and p.get("label") == UNSUPPORTED} seen: set[str] = set() for p in pairs: pid = p.get("id") if isinstance(p, dict) else None if not isinstance(pid, str) or not pid: rep.errors.append("pair without an id") continue if pid in seen: rep.errors.append(f"duplicate id {pid}") seen.add(pid) label, kind, claim = p.get("label"), p.get("kind"), p.get("claim") if label not in KINDS: rep.errors.append(f"{pid}: label {label!r} is not one of {', '.join(LABELS)}") elif kind not in KINDS[label]: rep.errors.append(f"{pid}: kind {kind!r} does not belong to label {label}") elif label == UNSUPPORTED: rep.unsupported += 1 rep.by_kind[kind] += 1 elif label == SUPPORTED: rep.supported += 1 rep.by_kind[kind] += 1 else: rep.filler += 1 rep.filler_by_kind[kind] += 1 twin = p.get("twin_of") if twin is not None and (label != FILLER or twin not in unsupported_ids): rep.errors.append(f"{pid}: twin_of {twin!r} does not name an unsupported pair of a {FILLER} pair") if not isinstance(claim, str) or not CLAIM_MIN <= len(claim.strip()) <= CLAIM_MAX: rep.errors.append(f"{pid}: claim missing or length outside {CLAIM_MIN}-{CLAIM_MAX}") claim = "" excerpts = as_list(p.get("excerpts")) if not 1 <= len(excerpts) <= MAX_EXCERPTS: rep.errors.append(f"{pid}: {len(excerpts)} excerpts, expected 1-{MAX_EXCERPTS}") texts: list[str] = [] files: set[str] = set() for e in excerpts: rep.excerpts += 1 name, text = (e.get("file"), e.get("text")) if isinstance(e, dict) else (None, None) if not is_plain_name(name) or not isinstance(name, str): rep.errors.append(f"{pid}: excerpt file {name!r} is not a plain file name") continue if not isinstance(text, str): rep.errors.append(f"{pid}: excerpt in {name} is not text") continue if not EXCERPT_MIN <= len(text) <= EXCERPT_MAX: rep.errors.append(f"{pid}: excerpt length {len(text)} in {name} outside {EXCERPT_MIN}-{EXCERPT_MAX}") files.add(name) texts.append(text) source = cache.text(name) if source is not None and text in source: rep.excerpts_verbatim += 1 if with_lines: rep.lines_checked += 1 line = e.get("line") if source is not None and isinstance(line, int) and line_of(source, text, line): rep.lines_correct += 1 if kind in TWO_EXCERPT_KINDS and len(excerpts) != 2: rep.errors.append(f"{pid}: kind {kind} needs two excerpts") if kind == "conflated-sources" and len(files) != 2: rep.errors.append(f"{pid}: kind conflated-sources needs excerpts from two different files") if kind == "wrong-number" and not numbers(claim) - set().union(*(numbers(t) for t in texts)): rep.errors.append(f"{pid}: wrong-number claim has no number that is absent from the excerpts") if label in (SUPPORTED, FILLER) and claim and any(normalise(claim) in normalise(t) for t in texts): rep.errors.append(f"{pid}: {label} claim is a verbatim copy of its excerpt") for label in COUNTED: for k in KINDS[label]: if rep.by_kind[k] == 0: rep.errors.append(f"kind {k} has no pair") return rep def join_set_and_key(set_path: Path, key_path: Path) -> list[dict[str, Any]]: pairs = load_list(set_path, SET_SCHEMA, "pairs", "set") labels = load_list(key_path, KEY_SCHEMA, "labels", "key") by_id = {k.get("id"): k for k in labels if isinstance(k, dict)} ids = [p.get("id") for p in pairs if isinstance(p, dict)] if set(ids) != set(by_id) or len(ids) != len(labels) or len(ids) != len(pairs): raise RunError("the key does not label exactly the pairs of the set") return [ dict(p, label=by_id[p["id"]].get("label"), kind=by_id[p["id"]].get("kind"), draft_id=by_id[p["id"]].get("draft_id")) for p in pairs ] # ---------------------------------------------------------------------------------------------- layout @dataclass class Layout: """What a call layout says, measured against the labels: positions, counts, and every rule it breaks.""" position: dict[str, int] = field(default_factory=dict) stats: dict[str, Any] = field(default_factory=dict) by_position: dict[str, dict[str, int]] = field(default_factory=dict) errors: list[str] = field(default_factory=list) def shown(ids: list[str]) -> str: return ", ".join(ids[:5]) + (", ..." if len(ids) > 5 else "") def check_layout(layout: object, labels: dict[str, str], set_sha256: str) -> Layout: """K3 point 1: every claim of the set in exactly one call, at most call_size_max (<= 10) claims per call, the layout bound to the set by its sha256, and unsupported claims at every position 1-10.""" out = Layout() if not isinstance(layout, dict) or layout.get("schema") != CALLS_SCHEMA: out.errors.append(f"the layout is not a layout with schema {CALLS_SCHEMA}") return out size_max = layout.get("call_size_max") if not isinstance(size_max, int) or isinstance(size_max, bool) or not 1 <= size_max <= CALL_SIZE_MAX: out.errors.append(f"call_size_max {size_max!r} is not a whole number from 1 to {CALL_SIZE_MAX}") size_max = CALL_SIZE_MAX if layout.get("set_sha256") != set_sha256: out.errors.append(f"the layout's set_sha256 {layout.get('set_sha256')!r} is not the set's sha256 {set_sha256}") calls = layout.get("calls") if not isinstance(calls, list) or not calls: out.errors.append("the layout has no calls") return out seen: dict[str, int] = {} names: set[str] = set() per_call: dict[int, int] = {} for c in calls: name, ids = (c.get("call"), c.get("ids")) if isinstance(c, dict) else (None, None) if not isinstance(name, str) or not isinstance(ids, list) or not ids or not all(isinstance(i, str) for i in ids): out.errors.append(f"call {name!r} is not a named call with a list of claim ids") continue if name in names: out.errors.append(f"call name {name} is given twice") names.add(name) if len(ids) > size_max: out.errors.append(f"call {name} has {len(ids)} claims, more than {size_max}") for n, i in enumerate(ids, 1): seen[i] = seen.get(i, 0) + 1 out.position[i] = n u = sum(1 for i in ids if labels.get(i) == UNSUPPORTED) per_call[u] = per_call.get(u, 0) + 1 twice = sorted(i for i, n in seen.items() if n > 1) if twice: out.errors.append(f"{len(twice)} claim id(s) stand in more than one place: {shown(twice)}") unknown = sorted(i for i in seen if i not in labels) if unknown: out.errors.append(f"{len(unknown)} claim id(s) of the layout are not in the set: {shown(unknown)}") missing = sorted(i for i in labels if i not in seen) if missing: out.errors.append(f"{len(missing)} claim id(s) of the set stand in no call: {shown(missing)}") out.by_position = {str(n): {lab: 0 for lab in LABELS} for n in range(1, CALL_SIZE_MAX + 1)} for i, n in out.position.items(): if i in labels and str(n) in out.by_position and labels[i] in LABELS: out.by_position[str(n)][labels[i]] += 1 for n in range(1, CALL_SIZE_MAX + 1): if out.by_position[str(n)][UNSUPPORTED] == 0: out.errors.append(f"position {n} has no unsupported claim") out.stats = { "calls": len(calls), "call_size_max": size_max, "unsupported_alone": per_call.get(1, 0), "unsupported_per_call": {str(k): per_call[k] for k in sorted(per_call)}, } return out def layout_lines(lay: Layout) -> list[str]: per_call = lay.stats.get("unsupported_per_call", {}) return [ f"calls: {lay.stats.get('calls', 0)} (at most {lay.stats.get('call_size_max', CALL_SIZE_MAX)} claims each)", "unsupported per call: " + ", ".join(f"{k} in {v} calls" for k, v in per_call.items()), f"unsupported alone in their call: {lay.stats.get('unsupported_alone', 0)}", "unsupported by position: " + ", ".join(f"{n} {lay.by_position.get(str(n), {}).get(UNSUPPORTED, 0)}" for n in range(1, CALL_SIZE_MAX + 1)), ] def positions_of(layout: object) -> dict[str, int]: """Claim id -> position 1.. in its call, read leniently (the stub's use; the gate checks the layout itself).""" out: dict[str, int] = {} for c in as_list(as_dict(layout).get("calls")): for n, i in enumerate(as_list(as_dict(c).get("ids")), 1): if isinstance(i, str): out[i] = n return out def tally(pairs: list[dict[str, Any]]) -> dict[str, Any]: """Counts per label and per kind of the two counted labels, and the distinct filler pairs behind the slots.""" out: dict[str, Any] = {lab: sum(1 for p in pairs if p.get("label") == lab) for lab in LABELS} out["filler_distinct"] = len({str(p.get("draft_id", p.get("id"))) for p in pairs if p.get("label") == FILLER}) out["by_kind"] = { k: sum(1 for p in pairs if p.get("label") == lab and p.get("kind") == k) for lab in COUNTED for k in KINDS[lab] } return out @dataclass class Frozen: pairs: list[dict[str, Any]] layout: Layout manifest: dict[str, Any] errors: list[str] def load_frozen(set_path: Path, key_path: Path, calls_path: Path, manifest_path: Path, sources: Path) -> Frozen: """The set, key and layout with every disagreement with the manifest listed (sha256, sources, layout, counts).""" require_dir(sources, "sources") manifest = load_json(manifest_path, "manifest") if not isinstance(manifest, dict) or manifest.get("schema") != MANIFEST_SCHEMA: raise RunError(f"{manifest_path} is not a manifest with schema {MANIFEST_SCHEMA}") errors: list[str] = [] for path, part in ((set_path, "set"), (key_path, "key"), (calls_path, "calls")): digest = file_sha256(path) if digest != as_dict(manifest.get(part)).get("sha256"): errors.append(f"{part} {path} sha256 {digest} does not match the manifest") if tree_sha256(sources) != as_dict(manifest.get("sources")).get("tree_sha256"): errors.append(f"sources {sources} differ from the manifest's tree_sha256; the set is frozen against them") pairs = join_set_and_key(set_path, key_path) labels = {str(p["id"]): str(p.get("label")) for p in pairs} layout = check_layout(load_json(calls_path, "layout"), labels, file_sha256(set_path)) errors += layout.errors counts = as_dict(manifest.get("counts")) for k, v in {**tally(pairs), "by_position": layout.by_position}.items(): if counts.get(k) != v: errors.append(f"the manifest's counts.{k} differs from the set and layout" + ("" if isinstance(v, dict) else f" ({counts.get(k)!r}, measured {v!r})")) calls = as_dict(manifest.get("calls")) for k in LAYOUT_STATS: if k in layout.stats and calls.get(k) != layout.stats[k]: errors.append(f"the manifest's calls.{k} differs from the layout ({calls.get(k)!r}, measured {layout.stats[k]!r})") return Frozen(pairs, layout, manifest, errors) def cmd_check(args: argparse.Namespace) -> int: sources = Path(args.sources) if args.draft: rep = check_pairs(load_list(Path(args.draft), DRAFT_SCHEMA, "pairs", "draft"), sources) elif args.set and args.key and args.calls and args.manifest: frozen = load_frozen(Path(args.set), Path(args.key), Path(args.calls), Path(args.manifest), sources) rep = check_pairs(frozen.pairs, sources, with_lines=True) rep.errors = frozen.errors + rep.errors rep.notes = layout_lines(frozen.layout) else: raise RunError("check needs --draft, or --set, --key, --calls and --manifest") print("\n".join(rep.lines())) return 0 if rep.passed else 1 # ---------------------------------------------------------------------------------------------- build def dump(schema: str, key: str, items: list[dict[str, Any]]) -> bytes: return (json.dumps({"schema": schema, key: items}, ensure_ascii=False, indent=1) + "\n").encode("utf-8") def lay_out(draft: list[dict[str, Any]], seed: int, lone: int, dense: int) -> list[list[dict[str, Any]]]: """Calls of draft pairs, each in position order. With random.Random(seed): each label's pairs sorted by draft id and shuffled; the first `lone` unsupported pairs stand alone in a call, at positions spread evenly over 1-10; the rest stand `dense` to a call; counted supported pairs go one per call in call order; filler pairs fill every call to ten, taken in a cycle, never twice in one call and never in the call of the unsupported pair they are the twin of; the other positions are shuffled.""" rng = random.Random(seed) groups = {lab: sorted((p for p in draft if p.get("label") == lab), key=lambda p: str(p["id"])) for lab in LABELS} for lab in LABELS: rng.shuffle(groups[lab]) unsupported, supported, fillers = groups[UNSUPPORTED], groups[SUPPORTED], groups[FILLER] if not 0 <= lone <= len(unsupported): raise RunError(f"--lone {lone} is outside 0-{len(unsupported)} (the unsupported pairs in the draft)") if not 1 <= dense <= CALL_SIZE_MAX: raise RunError(f"--dense {dense} is outside 1-{CALL_SIZE_MAX}") rest = unsupported[lone:] calls = [[u] for u in unsupported[:lone]] + [rest[i : i + dense] for i in range(0, len(rest), dense)] pending = list(supported) while pending: free = [c for c in calls if len(c) < CALL_SIZE_MAX] if not free: raise RunError(f"{len(pending)} supported pairs find no free place in the calls") for c in free[: len(pending)]: c.append(pending.pop(0)) cursor = 0 for c in calls: need = CALL_SIZE_MAX - len(c) if need and not fillers: raise RunError(f"a call has {need} free places and the draft has no {FILLER} pairs to fill them") taken = {str(p["id"]) for p in c} twins_of = {str(p["id"]) for p in c if p.get("label") == UNSUPPORTED} misses = 0 while need > 0: f = fillers[cursor % len(fillers)] cursor += 1 if str(f["id"]) in taken or f.get("twin_of") in twins_of: misses += 1 if misses >= len(fillers): raise RunError(f"too few distinct {FILLER} pairs to fill a call without repeating one or meeting its twin") continue c.append(f) taken.add(str(f["id"])) need -= 1 misses = 0 at = [n % CALL_SIZE_MAX for n in range(lone)] rng.shuffle(at) ordered: list[list[dict[str, Any]]] = [] for n, c in enumerate(calls): if n < lone: head, others = c[0], c[1:] rng.shuffle(others) ordered.append(others[: at[n]] + [head] + others[at[n] :]) else: rng.shuffle(c) ordered.append(c) return ordered def history_of(manifest_path: Path) -> list[Any]: """The history for a new manifest: the old manifest's history, then the old manifest itself without it.""" if not manifest_path.exists(): return [] old = load_json(manifest_path, "manifest") if not isinstance(old, dict): raise RunError(f"{manifest_path} is not a manifest; it would be lost from the history") return as_list(old.get("history")) + [{k: v for k, v in old.items() if k != "history"}] def cmd_build(args: argparse.Namespace) -> int: sources = Path(args.sources) set_path, key_path, calls_path = Path(args.set_out), Path(args.key_out), Path(args.calls_out) manifest_path = Path(args.manifest) for out_path in (set_path, key_path, calls_path): if out_path.exists(): raise RunError(f"{out_path} already exists; a built set is frozen and never rebuilt") draft = load_list(Path(args.draft), DRAFT_SCHEMA, "pairs", "draft") rep = check_pairs(draft, sources) if not rep.passed: print("\n".join(rep.lines())) raise RunError("the draft fails the check; nothing was built") calls = lay_out(draft, args.seed, args.lone, args.dense) cache = SourceCache(sources) total = sum(len(c) for c in calls) width, call_width = max(3, len(str(total))), max(3, len(str(len(calls)))) pairs: list[dict[str, Any]] = [] labels: list[dict[str, Any]] = [] layout_calls: list[dict[str, Any]] = [] for k, members in enumerate(calls, 1): ids = [] for p in members: cid = f"c{len(pairs) + 1:0{width}d}" excerpts = [] for e in p["excerpts"]: source = cache.text(e["file"]) or "" excerpts.append({"file": e["file"], "line": first_line(source, e["text"]), "text": e["text"]}) pairs.append({"id": cid, "claim": p["claim"], "excerpts": excerpts}) labels.append({"id": cid, "label": p["label"], "kind": p["kind"], "draft_id": p["id"], "why": p.get("why", "")}) ids.append(cid) layout_calls.append({"call": f"k{k:0{call_width}d}", "ids": ids}) set_bytes, key_bytes = dump(SET_SCHEMA, "pairs", pairs), dump(KEY_SCHEMA, "labels", labels) layout = {"call_size_max": CALL_SIZE_MAX, "calls": layout_calls, "schema": CALLS_SCHEMA, "set_sha256": sha256_bytes(set_bytes)} calls_bytes = (json.dumps(layout, indent=1, sort_keys=True, ensure_ascii=False) + "\n").encode("utf-8") lay = check_layout(layout, {lab["id"]: lab["label"] for lab in labels}, sha256_bytes(set_bytes)) if lay.errors: raise RunError("the layout breaks K3 point 1, nothing was built: " + "; ".join(lay.errors)) joined = [dict(p, label=lab["label"], kind=lab["kind"], draft_id=lab["draft_id"]) for p, lab in zip(pairs, labels)] counts = tally(joined) manifest = { "schema": MANIFEST_SCHEMA, "seed": args.seed, "shuffle": LAYOUT_RULE, "set": {"file": args.set_label or set_path.name, "sha256": sha256_bytes(set_bytes), "pairs": len(pairs)}, "key": {"file": args.key_label or key_path.name, "sha256": sha256_bytes(key_bytes)}, "calls": { "file": args.calls_label or calls_path.name, "sha256": sha256_bytes(calls_bytes), **lay.stats, "built_with": {"lone": args.lone, "dense": args.dense}, }, "counts": {**counts, "by_position": lay.by_position}, "thresholds": { "caught_min": counts[UNSUPPORTED], "accepted_min": math.ceil(ACCEPTED_SHARE * counts[SUPPORTED]), }, "sources": {"files": count_files(sources), "tree_sha256": tree_sha256(sources)}, "history": history_of(manifest_path), } for path, data in ((set_path, set_bytes), (key_path, key_bytes), (calls_path, calls_bytes)): path.parent.mkdir(parents=True, exist_ok=True) path.write_bytes(data) manifest_path.parent.mkdir(parents=True, exist_ok=True) manifest_path.write_text(json.dumps(manifest, indent=1) + "\n", encoding="utf-8") rep.notes = layout_lines(lay) print("\n".join(rep.lines())) print(f"seed: {args.seed}") print(f"set: {len(pairs)} claims, sha256 {manifest['set']['sha256']}") print(f"key: sha256 {manifest['key']['sha256']}") print(f"layout: {len(layout_calls)} calls, sha256 {manifest['calls']['sha256']}") print(f"manifest: {manifest_path}") return 0 # ---------------------------------------------------------------------------------------------- stub def cmd_stub(args: argparse.Namespace) -> int: pairs = load_list(Path(args.set), SET_SCHEMA, "pairs", "set") if args.key: verdict = {str(p["id"]): verdict_for(p["label"]) for p in join_set_and_key(Path(args.set), Path(args.key))} reader = "stub-key" else: verdict = {str(p["id"]): args.verdict for p in pairs} reader = f"stub-all-{args.verdict}" if args.accept_at is not None: if not args.calls: raise RunError("--accept-at needs --calls, the layout that says where each claim stands") if not 1 <= args.accept_at <= CALL_SIZE_MAX: raise RunError(f"--accept-at {args.accept_at} is outside 1-{CALL_SIZE_MAX}") for i, n in positions_of(load_json(Path(args.calls), "layout")).items(): if n == args.accept_at and i in verdict: verdict[i] = SUPPORTED reader += f"-accept-at-{args.accept_at}" reader = args.reader or reader out = {"reader": reader, "models": [STUB_MODEL], "verdicts": [{"id": p["id"], "verdict": verdict[str(p["id"])]} for p in pairs]} Path(args.out).write_text(json.dumps(out, indent=1) + "\n", encoding="utf-8") print(f"stub {reader}: {len(pairs)} verdicts written to {args.out}") return 0 # ---------------------------------------------------------------------------------------------- gate def read_verdicts(path: Path) -> tuple[str, dict[str, str], str]: """(reader, verdict per id, fail-closed reason). An id given twice, or with an invalid verdict, is left out.""" if not path.is_file(): return "", {}, f"no verdicts at {path}" data = load_json(path, "verdicts") if not isinstance(data, dict) or not isinstance(data.get("verdicts"), list): raise RunError(f"{path} is not a verdicts file {VERDICTS_SCHEMA_NOTE}") counts: dict[str, int] = {} given: dict[str, str] = {} for v in data["verdicts"]: if isinstance(v, dict) and isinstance(v.get("id"), str): counts[v["id"]] = counts.get(v["id"], 0) + 1 if v.get("verdict") in VERDICTS: given[v["id"]] = v["verdict"] verdicts = {i: val for i, val in given.items() if counts[i] == 1} reader = data.get("reader") return reader if isinstance(reader, str) else "", verdicts, "" def read_models(path: Path) -> list[str]: """The sorted, distinct models a verdicts file names in its "models" list; none when the file or the list is missing or holds no model name.""" if not path.is_file(): return [] data = load_json(path, "verdicts") raw = data.get("models") if isinstance(data, dict) else None return sorted({m for m in as_list(raw) if isinstance(m, str) and m}) def record_passed(path: Path, entry: dict[str, str]) -> bool: """Add one green run to the record at path (created when missing); False when the same entry is there already. A record that cannot be read, or does not have the schema, is a RunError and is left as it was.""" entries: list[Any] = [] if path.exists(): try: entries = reader.read_passed(path) # the parser collect --gate-passed reads the record with except reader.RunError as exc: raise RunError(str(exc)) from exc if entry in entries: return False path.parent.mkdir(parents=True, exist_ok=True) path.write_text(json.dumps({"schema": PASSED_SCHEMA, "passed": [*entries, entry]}, indent=1) + "\n", encoding="utf-8") return True def cmd_gate(args: argparse.Namespace) -> int: frozen = load_frozen(Path(args.set), Path(args.key), Path(args.calls), Path(args.manifest), Path(args.sources)) if frozen.errors: raise RunError("the frozen set does not hold: " + "; ".join(frozen.errors)) thresholds = as_dict(frozen.manifest.get("thresholds")) caught_min, accepted_min = thresholds.get("caught_min"), thresholds.get("accepted_min") if not isinstance(caught_min, int) or not isinstance(accepted_min, int): raise RunError("the manifest has no integer thresholds caught_min and accepted_min") pairs, position = frozen.pairs, frozen.layout.position reader, verdicts, fail_closed = read_verdicts(Path(args.verdicts)) models = read_models(Path(args.verdicts)) model_reason = "" if len(models) == 1 else ( "the verdicts name no model" if not models else f"the verdicts name {len(models)} models") n = {lab: sum(1 for p in pairs if p["label"] == lab) for lab in LABELS} caught = sum(1 for p in pairs if p["label"] == UNSUPPORTED and verdicts.get(p["id"]) == UNSUPPORTED) accepted = sum(1 for p in pairs if p["label"] == SUPPORTED and verdicts.get(p["id"]) == SUPPORTED) filler_accepted = sum(1 for p in pairs if p["label"] == FILLER and verdicts.get(p["id"]) == SUPPORTED) missing = sum(1 for p in pairs if p["id"] not in verdicts) unknown = sum(1 for i in verdicts if i not in position) per_kind = [] for k in UNSUPPORTED_KINDS: of_kind = [p for p in pairs if p["label"] == UNSUPPORTED and p["kind"] == k] hit = sum(1 for p in of_kind if verdicts.get(p["id"]) == UNSUPPORTED) per_kind.append(f"{k} {hit} of {len(of_kind)}") per_position, short = [], [] for at in range(1, CALL_SIZE_MAX + 1): there = [p for p in pairs if p["label"] == UNSUPPORTED and position.get(p["id"]) == at] hit = sum(1 for p in there if verdicts.get(p["id"]) == UNSUPPORTED) per_position.append(f"{at} {hit} of {len(there)}") if hit < len(there) or not there: short.append(f"position {at}: caught {hit} of {len(there)}") green = ( not fail_closed and not model_reason and missing == 0 and caught >= caught_min and caught == n[UNSUPPORTED] and not short and accepted >= accepted_min ) print( f"set: {len(pairs)} claims ({n[UNSUPPORTED]} {UNSUPPORTED}, {n[SUPPORTED]} {SUPPORTED}, {n[FILLER]} {FILLER})" f" in {frozen.layout.stats['calls']} calls; reader: {reader or '(none)'}" ) print(f"model: {', '.join(models) if models else '(none)'}") if fail_closed: print(f"fail-closed: {fail_closed}") if model_reason: print(f"fail-closed: {model_reason}") print(f"missing verdicts: {missing}" + (f"; verdicts for unknown ids: {unknown}" if unknown else "")) print(f"unsupported caught {caught} of {n[UNSUPPORTED]} · supported accepted {accepted} of {n[SUPPORTED]}") print("caught by kind: " + ", ".join(per_kind)) print("caught by position: " + ", ".join(per_position)) print(f"filler accepted {filler_accepted} of {n[FILLER]} (reported apart, never counted in the thresholds)") print( f"required: caught >= {caught_min} of {n[UNSUPPORTED]} with every position 1-{CALL_SIZE_MAX} caught in full," f" accepted >= {accepted_min} of {n[SUPPORTED]}, no verdict missing" ) if not green: reasons = [] if fail_closed: reasons.append("fail-closed") if model_reason: reasons.append(model_reason) if missing: reasons.append(f"{missing} verdicts missing") if caught < caught_min or caught < n[UNSUPPORTED]: reasons.append(f"caught {caught} of {n[UNSUPPORTED]}") reasons += short if accepted < accepted_min: reasons.append(f"accepted {accepted} of {n[SUPPORTED]}, needs {accepted_min}") print("not met: " + "; ".join(reasons)) if green and args.passed: entry = {"model": models[0], "set_sha256": file_sha256(Path(args.set)), "key_sha256": file_sha256(Path(args.key)), "verdicts_sha256": file_sha256(Path(args.verdicts)), "reader": reader} added = record_passed(Path(args.passed), entry) print(f"passed: {models[0]} {'recorded in' if added else 'already recorded in'} {args.passed}") print("GREEN" if green else "RED") return 0 if green else 1 # ---------------------------------------------------------------------------------------------- main def build_parser() -> argparse.ArgumentParser: p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) sub = p.add_subparsers(dest="command", required=True) c = sub.add_parser("check", help="check a labelled draft, or a built set with its key, layout and manifest") c.add_argument("--draft", default="") c.add_argument("--set", default="") c.add_argument("--key", default="") c.add_argument("--calls", default="", help="the call layout (K3 point 1)") c.add_argument("--manifest", default="") c.add_argument("--sources", required=True) b = sub.add_parser("build", help="lay out a checked draft in calls with a fixed seed: set, key, layout and manifest") b.add_argument("--draft", required=True) b.add_argument("--sources", required=True) b.add_argument("--seed", type=int, required=True) b.add_argument("--lone", type=int, required=True, help="unsupported pairs that stand alone in their call") b.add_argument("--dense", type=int, required=True, help="unsupported pairs per call for the rest") b.add_argument("--set-out", required=True) b.add_argument("--key-out", required=True) b.add_argument("--calls-out", required=True) b.add_argument("--set-label", default="", help="path of the set as written in the manifest") b.add_argument("--key-label", default="", help="path of the key as written in the manifest") b.add_argument("--calls-label", default="", help="path of the layout as written in the manifest") b.add_argument("--manifest", required=True) s = sub.add_parser("stub", help="write verdicts without a reader") s.add_argument("--set", required=True) which = s.add_mutually_exclusive_group(required=True) which.add_argument("--verdict", choices=VERDICTS, help="the same verdict for every claim") which.add_argument("--key", help="the key's verdict for every claim (the known-positive)") s.add_argument("--calls", default="", help="the call layout, for --accept-at") s.add_argument("--accept-at", type=int, default=None, help="answer supported for every claim at this position") s.add_argument("--reader", default="") s.add_argument("--out", required=True) g = sub.add_parser("gate", help="score one reader's verdicts against the key") g.add_argument("--set", required=True) g.add_argument("--key", required=True) g.add_argument("--calls", required=True, help="the call layout (K3 point 1)") g.add_argument("--manifest", required=True) g.add_argument("--sources", required=True) g.add_argument("--verdicts", required=True, help=VERDICTS_SCHEMA_NOTE) g.add_argument("--passed", default="", metavar="FILE", help=f"on GREEN, add the run to this record of green runs ({PASSED_SCHEMA}); RED never writes it") return p def main(argv: list[str] | None = None) -> int: args = build_parser().parse_args(argv) commands = {"check": cmd_check, "build": cmd_build, "stub": cmd_stub, "gate": cmd_gate} try: return commands[args.command](args) except RunError as exc: print(f"FAIL: {exc}", file=sys.stderr) return 1 if __name__ == "__main__": sys.exit(main())