okf-wiki/gates/calibration.py
Kjell Tore Guttormsen 0fd86a23a3
okf-wiki 1.1.1
First public release, published as one snapshot. What changed between
versions is in CHANGELOG.md.

Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
2026-11-14 21:03:20 +01:00

860 lines
41 KiB
Python

"""The reader calibration for PLAN.md criterion 2: set checker, seeded build with a call layout, stub reader and verdict
gate. No model call.
The reader judges up to ten claims per call and gives one verdict per claim (K3 in docs/bolge1/0-felles.local.md). The
calibration therefore comes as three frozen files beside the manifest: the set (claims and excerpts, what the reader
sees), the key (label and kind per claim) and the call layout (which claims stand together in a call, and where).
Labels: "unsupported" (planted, must be caught), "supported" (counted controls, must be accepted) and
"filler-supported" (supported claims that fill the calls so they resemble a real page; reported apart and never
counted in the thresholds).
check every excerpt stands verbatim in its source file (and, in a built set, at its stated line), every kind of
the two counted labels is present and fits its label, a wrong-number claim carries a number the excerpts
do not, a supported or filler claim is not a verbatim copy of its excerpt, and a twin_of names an
unsupported pair. A built set is checked with its layout against the manifest: sha256 of set, key, layout
and sources, the layout rules, and the counts per label, kind and position.
build checks a labelled draft, lays it out in calls with a fixed seed, renumbers the claims (c001.., in call
order) and the calls (k001..), and writes the set, the key, the layout and a manifest with sha256, counts
and thresholds only. A built set is frozen: build never overwrites it. An earlier manifest at the same
path moves into the new one's history.
stub writes a verdict file without a reader: the same verdict for every claim, or the key's verdicts, and
optionally "supported" for every claim at one position (the position-mild stub). Its model is "stub".
gate scores one reader's verdicts against the key: totals, caught per kind and caught per position, and the
fillers apart; never a result per call or per pair. A missing, invalid or duplicated verdict counts as
wrong (fail-closed). GREEN only when every unsupported claim is caught at every position, the counted
supported claims reach the manifest's threshold, no verdict is missing, and the verdicts name exactly
one model (their "models", written by okfwiki.reader collect from the readers' transcripts; PLAN
§ Rammer 23). It prints the model it scored. With --passed FILE a GREEN run is added to FILE, the record
of green runs (okf-wiki/calibration-passed/1: model, set_sha256, key_sha256, verdicts_sha256, reader; no
time, and an entry already there is not added twice); a RED run never writes it. okfwiki.reader collect
--gate-passed FILE turns a reading into verdicts only for a model in FILE with a run on the set and key
of the tracked manifest, so a model change, or a new set, needs a new gate run. Both read FILE with
okfwiki.reader.read_passed.
Exit 0 = PASS / GREEN / written; 1 = FAIL / RED or could not run. A check that cannot run is a FAIL.
"""
from __future__ import annotations
import argparse
import hashlib
import json
import math
import random
import re
import sys
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from okfwiki import reader # noqa: E402
DRAFT_SCHEMA = "okf-wiki/calibration-draft/1"
SET_SCHEMA = "okf-wiki/calibration-set/1"
KEY_SCHEMA = "okf-wiki/calibration-key/1"
CALLS_SCHEMA = "okf-wiki/reader-calls/1"
MANIFEST_SCHEMA = "okf-wiki/calibration-manifest/2"
PASSED_SCHEMA = reader.PASSED_SCHEMA
STUB_MODEL = "stub"
VERDICTS_SCHEMA_NOTE = (
'{"reader": "<name>", "models": ["<model>"], "verdicts": [{"id": "c001", "verdict": "supported" | "unsupported"}]}'
)
UNSUPPORTED = "unsupported"
SUPPORTED = "supported"
FILLER = "filler-supported"
LABELS = (UNSUPPORTED, SUPPORTED, FILLER)
COUNTED = (UNSUPPORTED, SUPPORTED)
VERDICTS = (SUPPORTED, UNSUPPORTED)
UNSUPPORTED_KINDS = (
"overgeneralisation",
"wrong-number",
"swapped-cause",
"not-mentioned",
"reversed-polarity",
"conflated-sources",
)
SUPPORTED_KINDS = ("paraphrase", "inference", "synthesis")
KINDS = {UNSUPPORTED: UNSUPPORTED_KINDS, SUPPORTED: SUPPORTED_KINDS, FILLER: SUPPORTED_KINDS}
TWO_EXCERPT_KINDS = ("conflated-sources", "synthesis")
MAX_EXCERPTS = 2
CLAIM_MIN, CLAIM_MAX = 10, 400
EXCERPT_MIN, EXCERPT_MAX = 20, 1500
CALL_SIZE_MAX = 10 # K3 point 1: up to ten claims per call, positions 1-10
ACCEPTED_SHARE = 0.9 # PM's choice (order E3): a reader that rejects more than 10 % of supported claims is unusable.
NUMBER = re.compile(r"\d+(?:[.,]\d+)*")
LAYOUT_STATS = ("calls", "call_size_max", "unsupported_alone", "unsupported_per_call")
LAYOUT_RULE = (
"random.Random(seed): each label's pairs sorted by draft id and shuffled; the first `lone` unsupported pairs alone"
" in a call at positions spread evenly over 1-10, the rest `dense` to a call; counted supported pairs one per call"
" in call order; filler pairs fill every call to ten in a cycle, never twice in a call and never beside the"
" unsupported pair they are the twin of; the other positions shuffled; ids renumbered c001.. in call order,"
" calls k001.."
)
class RunError(Exception):
"""The command cannot run; reported as FAIL with exit 1."""
# ---------------------------------------------------------------------------------------------- helpers
def sha256_bytes(data: bytes) -> str:
return hashlib.sha256(data).hexdigest()
def tree_sha256(sources: Path) -> str:
"""One hash over the flat sources directory: sorted "name NUL sha256 LF" lines (same as gates/multidoc.py)."""
lines = []
for p in sorted(sources.iterdir(), key=lambda p: p.name):
if p.is_file():
lines.append(f"{p.name}\0{sha256_bytes(p.read_bytes())}\n")
return sha256_bytes("".join(lines).encode("utf-8"))
def count_files(sources: Path) -> int:
return sum(1 for p in sources.iterdir() if p.is_file())
def normalise(text: str) -> str:
"""Case-folded, whitespace collapsed, outer punctuation stripped. Used for the verbatim-copy test only."""
return re.sub(r"\s+", " ", text).strip().strip(".,;:!?").strip().casefold()
def numbers(text: str) -> set[str]:
return {n.replace(",", "") for n in NUMBER.findall(text)}
def as_list(value: object) -> list[Any]:
return value if isinstance(value, list) else []
def as_dict(value: object) -> dict[str, Any]:
return value if isinstance(value, dict) else {}
def is_plain_name(name: object) -> bool:
return isinstance(name, str) and name != "" and "/" not in name and "\\" not in name and name not in (".", "..")
def load_json(path: Path, what: str) -> Any:
try:
return json.loads(path.read_text(encoding="utf-8"))
except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc:
raise RunError(f"cannot read {what} {path}: {exc}") from exc
def load_list(path: Path, schema: str, key: str, what: str) -> list[dict[str, Any]]:
data = load_json(path, what)
if not isinstance(data, dict) or data.get("schema") != schema or not isinstance(data.get(key), list):
raise RunError(f"{path} is not a {what} with schema {schema}")
items: list[dict[str, Any]] = data[key]
return items
def require_dir(path: Path, what: str) -> None:
if not path.is_dir():
raise RunError(f"{what} {path} is not a directory")
def file_sha256(path: Path) -> str:
return sha256_bytes(path.read_bytes()) if path.is_file() else "(missing)"
def line_of(text: str, excerpt: str, line: int) -> bool:
"""True when an occurrence of the excerpt starts on the given 1-based line."""
start = text.find(excerpt)
while start != -1:
if text.count("\n", 0, start) + 1 == line:
return True
start = text.find(excerpt, start + 1)
return False
def first_line(text: str, excerpt: str) -> int:
return text.count("\n", 0, text.find(excerpt)) + 1
def verdict_for(label: object) -> str:
"""The right verdict for a label: only an unsupported claim should be rejected."""
return UNSUPPORTED if label == UNSUPPORTED else SUPPORTED
class SourceCache:
def __init__(self, root: Path) -> None:
self.root = root
self._text: dict[str, str | None] = {}
def text(self, name: str) -> str | None:
"""File text, or None when the name is not a plain file directly in the sources directory."""
if not is_plain_name(name):
return None
if name not in self._text:
p = self.root / name
try:
self._text[name] = p.read_text(encoding="utf-8") if p.is_file() else None
except (OSError, UnicodeDecodeError):
self._text[name] = None
return self._text[name]
# ---------------------------------------------------------------------------------------------- check
@dataclass
class CheckReport:
pairs: int = 0
unsupported: int = 0
supported: int = 0
filler: int = 0
excerpts: int = 0
excerpts_verbatim: int = 0
lines_checked: int = 0
lines_correct: int = 0
by_kind: dict[str, int] = field(default_factory=lambda: {k: 0 for lab in COUNTED for k in KINDS[lab]})
filler_by_kind: dict[str, int] = field(default_factory=lambda: {k: 0 for k in KINDS[FILLER]})
errors: list[str] = field(default_factory=list)
notes: list[str] = field(default_factory=list)
@property
def passed(self) -> bool:
return (
self.pairs > 0
and not self.errors
and self.excerpts_verbatim == self.excerpts
and self.lines_correct == self.lines_checked
)
def lines(self) -> list[str]:
out = [f"error: {e}" for e in self.errors]
out += [
f"pairs: {self.pairs} ({self.unsupported} {UNSUPPORTED}, {self.supported} {SUPPORTED}, {self.filler} {FILLER})",
f"excerpts verbatim: {self.excerpts_verbatim} of {self.excerpts}",
]
if self.lines_checked:
out.append(f"excerpt lines correct: {self.lines_correct} of {self.lines_checked}")
for label in COUNTED:
out.append(f"{label} by kind: " + ", ".join(f"{k} {self.by_kind[k]}" for k in KINDS[label]))
out.append(f"{FILLER} by kind: " + ", ".join(f"{k} {self.filler_by_kind[k]}" for k in KINDS[FILLER]))
out += self.notes
out.append("PASS" if self.passed else "FAIL")
return out
def check_pairs(pairs: list[dict[str, Any]], sources: Path, with_lines: bool = False) -> CheckReport:
"""Check labelled pairs ({id, label, kind, claim, excerpts: [{file, text[, line]}][, twin_of]}) against the sources."""
require_dir(sources, "sources")
cache = SourceCache(sources)
rep = CheckReport(pairs=len(pairs))
unsupported_ids = {p.get("id") for p in pairs if isinstance(p, dict) and p.get("label") == UNSUPPORTED}
seen: set[str] = set()
for p in pairs:
pid = p.get("id") if isinstance(p, dict) else None
if not isinstance(pid, str) or not pid:
rep.errors.append("pair without an id")
continue
if pid in seen:
rep.errors.append(f"duplicate id {pid}")
seen.add(pid)
label, kind, claim = p.get("label"), p.get("kind"), p.get("claim")
if label not in KINDS:
rep.errors.append(f"{pid}: label {label!r} is not one of {', '.join(LABELS)}")
elif kind not in KINDS[label]:
rep.errors.append(f"{pid}: kind {kind!r} does not belong to label {label}")
elif label == UNSUPPORTED:
rep.unsupported += 1
rep.by_kind[kind] += 1
elif label == SUPPORTED:
rep.supported += 1
rep.by_kind[kind] += 1
else:
rep.filler += 1
rep.filler_by_kind[kind] += 1
twin = p.get("twin_of")
if twin is not None and (label != FILLER or twin not in unsupported_ids):
rep.errors.append(f"{pid}: twin_of {twin!r} does not name an unsupported pair of a {FILLER} pair")
if not isinstance(claim, str) or not CLAIM_MIN <= len(claim.strip()) <= CLAIM_MAX:
rep.errors.append(f"{pid}: claim missing or length outside {CLAIM_MIN}-{CLAIM_MAX}")
claim = ""
excerpts = as_list(p.get("excerpts"))
if not 1 <= len(excerpts) <= MAX_EXCERPTS:
rep.errors.append(f"{pid}: {len(excerpts)} excerpts, expected 1-{MAX_EXCERPTS}")
texts: list[str] = []
files: set[str] = set()
for e in excerpts:
rep.excerpts += 1
name, text = (e.get("file"), e.get("text")) if isinstance(e, dict) else (None, None)
if not is_plain_name(name) or not isinstance(name, str):
rep.errors.append(f"{pid}: excerpt file {name!r} is not a plain file name")
continue
if not isinstance(text, str):
rep.errors.append(f"{pid}: excerpt in {name} is not text")
continue
if not EXCERPT_MIN <= len(text) <= EXCERPT_MAX:
rep.errors.append(f"{pid}: excerpt length {len(text)} in {name} outside {EXCERPT_MIN}-{EXCERPT_MAX}")
files.add(name)
texts.append(text)
source = cache.text(name)
if source is not None and text in source:
rep.excerpts_verbatim += 1
if with_lines:
rep.lines_checked += 1
line = e.get("line")
if source is not None and isinstance(line, int) and line_of(source, text, line):
rep.lines_correct += 1
if kind in TWO_EXCERPT_KINDS and len(excerpts) != 2:
rep.errors.append(f"{pid}: kind {kind} needs two excerpts")
if kind == "conflated-sources" and len(files) != 2:
rep.errors.append(f"{pid}: kind conflated-sources needs excerpts from two different files")
if kind == "wrong-number" and not numbers(claim) - set().union(*(numbers(t) for t in texts)):
rep.errors.append(f"{pid}: wrong-number claim has no number that is absent from the excerpts")
if label in (SUPPORTED, FILLER) and claim and any(normalise(claim) in normalise(t) for t in texts):
rep.errors.append(f"{pid}: {label} claim is a verbatim copy of its excerpt")
for label in COUNTED:
for k in KINDS[label]:
if rep.by_kind[k] == 0:
rep.errors.append(f"kind {k} has no pair")
return rep
def join_set_and_key(set_path: Path, key_path: Path) -> list[dict[str, Any]]:
pairs = load_list(set_path, SET_SCHEMA, "pairs", "set")
labels = load_list(key_path, KEY_SCHEMA, "labels", "key")
by_id = {k.get("id"): k for k in labels if isinstance(k, dict)}
ids = [p.get("id") for p in pairs if isinstance(p, dict)]
if set(ids) != set(by_id) or len(ids) != len(labels) or len(ids) != len(pairs):
raise RunError("the key does not label exactly the pairs of the set")
return [
dict(p, label=by_id[p["id"]].get("label"), kind=by_id[p["id"]].get("kind"), draft_id=by_id[p["id"]].get("draft_id"))
for p in pairs
]
# ---------------------------------------------------------------------------------------------- layout
@dataclass
class Layout:
"""What a call layout says, measured against the labels: positions, counts, and every rule it breaks."""
position: dict[str, int] = field(default_factory=dict)
stats: dict[str, Any] = field(default_factory=dict)
by_position: dict[str, dict[str, int]] = field(default_factory=dict)
errors: list[str] = field(default_factory=list)
def shown(ids: list[str]) -> str:
return ", ".join(ids[:5]) + (", ..." if len(ids) > 5 else "")
def check_layout(layout: object, labels: dict[str, str], set_sha256: str) -> Layout:
"""K3 point 1: every claim of the set in exactly one call, at most call_size_max (<= 10) claims per call, the
layout bound to the set by its sha256, and unsupported claims at every position 1-10."""
out = Layout()
if not isinstance(layout, dict) or layout.get("schema") != CALLS_SCHEMA:
out.errors.append(f"the layout is not a layout with schema {CALLS_SCHEMA}")
return out
size_max = layout.get("call_size_max")
if not isinstance(size_max, int) or isinstance(size_max, bool) or not 1 <= size_max <= CALL_SIZE_MAX:
out.errors.append(f"call_size_max {size_max!r} is not a whole number from 1 to {CALL_SIZE_MAX}")
size_max = CALL_SIZE_MAX
if layout.get("set_sha256") != set_sha256:
out.errors.append(f"the layout's set_sha256 {layout.get('set_sha256')!r} is not the set's sha256 {set_sha256}")
calls = layout.get("calls")
if not isinstance(calls, list) or not calls:
out.errors.append("the layout has no calls")
return out
seen: dict[str, int] = {}
names: set[str] = set()
per_call: dict[int, int] = {}
for c in calls:
name, ids = (c.get("call"), c.get("ids")) if isinstance(c, dict) else (None, None)
if not isinstance(name, str) or not isinstance(ids, list) or not ids or not all(isinstance(i, str) for i in ids):
out.errors.append(f"call {name!r} is not a named call with a list of claim ids")
continue
if name in names:
out.errors.append(f"call name {name} is given twice")
names.add(name)
if len(ids) > size_max:
out.errors.append(f"call {name} has {len(ids)} claims, more than {size_max}")
for n, i in enumerate(ids, 1):
seen[i] = seen.get(i, 0) + 1
out.position[i] = n
u = sum(1 for i in ids if labels.get(i) == UNSUPPORTED)
per_call[u] = per_call.get(u, 0) + 1
twice = sorted(i for i, n in seen.items() if n > 1)
if twice:
out.errors.append(f"{len(twice)} claim id(s) stand in more than one place: {shown(twice)}")
unknown = sorted(i for i in seen if i not in labels)
if unknown:
out.errors.append(f"{len(unknown)} claim id(s) of the layout are not in the set: {shown(unknown)}")
missing = sorted(i for i in labels if i not in seen)
if missing:
out.errors.append(f"{len(missing)} claim id(s) of the set stand in no call: {shown(missing)}")
out.by_position = {str(n): {lab: 0 for lab in LABELS} for n in range(1, CALL_SIZE_MAX + 1)}
for i, n in out.position.items():
if i in labels and str(n) in out.by_position and labels[i] in LABELS:
out.by_position[str(n)][labels[i]] += 1
for n in range(1, CALL_SIZE_MAX + 1):
if out.by_position[str(n)][UNSUPPORTED] == 0:
out.errors.append(f"position {n} has no unsupported claim")
out.stats = {
"calls": len(calls),
"call_size_max": size_max,
"unsupported_alone": per_call.get(1, 0),
"unsupported_per_call": {str(k): per_call[k] for k in sorted(per_call)},
}
return out
def layout_lines(lay: Layout) -> list[str]:
per_call = lay.stats.get("unsupported_per_call", {})
return [
f"calls: {lay.stats.get('calls', 0)} (at most {lay.stats.get('call_size_max', CALL_SIZE_MAX)} claims each)",
"unsupported per call: " + ", ".join(f"{k} in {v} calls" for k, v in per_call.items()),
f"unsupported alone in their call: {lay.stats.get('unsupported_alone', 0)}",
"unsupported by position: "
+ ", ".join(f"{n} {lay.by_position.get(str(n), {}).get(UNSUPPORTED, 0)}" for n in range(1, CALL_SIZE_MAX + 1)),
]
def positions_of(layout: object) -> dict[str, int]:
"""Claim id -> position 1.. in its call, read leniently (the stub's use; the gate checks the layout itself)."""
out: dict[str, int] = {}
for c in as_list(as_dict(layout).get("calls")):
for n, i in enumerate(as_list(as_dict(c).get("ids")), 1):
if isinstance(i, str):
out[i] = n
return out
def tally(pairs: list[dict[str, Any]]) -> dict[str, Any]:
"""Counts per label and per kind of the two counted labels, and the distinct filler pairs behind the slots."""
out: dict[str, Any] = {lab: sum(1 for p in pairs if p.get("label") == lab) for lab in LABELS}
out["filler_distinct"] = len({str(p.get("draft_id", p.get("id"))) for p in pairs if p.get("label") == FILLER})
out["by_kind"] = {
k: sum(1 for p in pairs if p.get("label") == lab and p.get("kind") == k) for lab in COUNTED for k in KINDS[lab]
}
return out
@dataclass
class Frozen:
pairs: list[dict[str, Any]]
layout: Layout
manifest: dict[str, Any]
errors: list[str]
def load_frozen(set_path: Path, key_path: Path, calls_path: Path, manifest_path: Path, sources: Path) -> Frozen:
"""The set, key and layout with every disagreement with the manifest listed (sha256, sources, layout, counts)."""
require_dir(sources, "sources")
manifest = load_json(manifest_path, "manifest")
if not isinstance(manifest, dict) or manifest.get("schema") != MANIFEST_SCHEMA:
raise RunError(f"{manifest_path} is not a manifest with schema {MANIFEST_SCHEMA}")
errors: list[str] = []
for path, part in ((set_path, "set"), (key_path, "key"), (calls_path, "calls")):
digest = file_sha256(path)
if digest != as_dict(manifest.get(part)).get("sha256"):
errors.append(f"{part} {path} sha256 {digest} does not match the manifest")
if tree_sha256(sources) != as_dict(manifest.get("sources")).get("tree_sha256"):
errors.append(f"sources {sources} differ from the manifest's tree_sha256; the set is frozen against them")
pairs = join_set_and_key(set_path, key_path)
labels = {str(p["id"]): str(p.get("label")) for p in pairs}
layout = check_layout(load_json(calls_path, "layout"), labels, file_sha256(set_path))
errors += layout.errors
counts = as_dict(manifest.get("counts"))
for k, v in {**tally(pairs), "by_position": layout.by_position}.items():
if counts.get(k) != v:
errors.append(f"the manifest's counts.{k} differs from the set and layout" + ("" if isinstance(v, dict) else f" ({counts.get(k)!r}, measured {v!r})"))
calls = as_dict(manifest.get("calls"))
for k in LAYOUT_STATS:
if k in layout.stats and calls.get(k) != layout.stats[k]:
errors.append(f"the manifest's calls.{k} differs from the layout ({calls.get(k)!r}, measured {layout.stats[k]!r})")
return Frozen(pairs, layout, manifest, errors)
def cmd_check(args: argparse.Namespace) -> int:
sources = Path(args.sources)
if args.draft:
rep = check_pairs(load_list(Path(args.draft), DRAFT_SCHEMA, "pairs", "draft"), sources)
elif args.set and args.key and args.calls and args.manifest:
frozen = load_frozen(Path(args.set), Path(args.key), Path(args.calls), Path(args.manifest), sources)
rep = check_pairs(frozen.pairs, sources, with_lines=True)
rep.errors = frozen.errors + rep.errors
rep.notes = layout_lines(frozen.layout)
else:
raise RunError("check needs --draft, or --set, --key, --calls and --manifest")
print("\n".join(rep.lines()))
return 0 if rep.passed else 1
# ---------------------------------------------------------------------------------------------- build
def dump(schema: str, key: str, items: list[dict[str, Any]]) -> bytes:
return (json.dumps({"schema": schema, key: items}, ensure_ascii=False, indent=1) + "\n").encode("utf-8")
def lay_out(draft: list[dict[str, Any]], seed: int, lone: int, dense: int) -> list[list[dict[str, Any]]]:
"""Calls of draft pairs, each in position order. With random.Random(seed): each label's pairs sorted by draft id and
shuffled; the first `lone` unsupported pairs stand alone in a call, at positions spread evenly over 1-10; the rest
stand `dense` to a call; counted supported pairs go one per call in call order; filler pairs fill every call to
ten, taken in a cycle, never twice in one call and never in the call of the unsupported pair they are the twin of;
the other positions are shuffled."""
rng = random.Random(seed)
groups = {lab: sorted((p for p in draft if p.get("label") == lab), key=lambda p: str(p["id"])) for lab in LABELS}
for lab in LABELS:
rng.shuffle(groups[lab])
unsupported, supported, fillers = groups[UNSUPPORTED], groups[SUPPORTED], groups[FILLER]
if not 0 <= lone <= len(unsupported):
raise RunError(f"--lone {lone} is outside 0-{len(unsupported)} (the unsupported pairs in the draft)")
if not 1 <= dense <= CALL_SIZE_MAX:
raise RunError(f"--dense {dense} is outside 1-{CALL_SIZE_MAX}")
rest = unsupported[lone:]
calls = [[u] for u in unsupported[:lone]] + [rest[i : i + dense] for i in range(0, len(rest), dense)]
pending = list(supported)
while pending:
free = [c for c in calls if len(c) < CALL_SIZE_MAX]
if not free:
raise RunError(f"{len(pending)} supported pairs find no free place in the calls")
for c in free[: len(pending)]:
c.append(pending.pop(0))
cursor = 0
for c in calls:
need = CALL_SIZE_MAX - len(c)
if need and not fillers:
raise RunError(f"a call has {need} free places and the draft has no {FILLER} pairs to fill them")
taken = {str(p["id"]) for p in c}
twins_of = {str(p["id"]) for p in c if p.get("label") == UNSUPPORTED}
misses = 0
while need > 0:
f = fillers[cursor % len(fillers)]
cursor += 1
if str(f["id"]) in taken or f.get("twin_of") in twins_of:
misses += 1
if misses >= len(fillers):
raise RunError(f"too few distinct {FILLER} pairs to fill a call without repeating one or meeting its twin")
continue
c.append(f)
taken.add(str(f["id"]))
need -= 1
misses = 0
at = [n % CALL_SIZE_MAX for n in range(lone)]
rng.shuffle(at)
ordered: list[list[dict[str, Any]]] = []
for n, c in enumerate(calls):
if n < lone:
head, others = c[0], c[1:]
rng.shuffle(others)
ordered.append(others[: at[n]] + [head] + others[at[n] :])
else:
rng.shuffle(c)
ordered.append(c)
return ordered
def history_of(manifest_path: Path) -> list[Any]:
"""The history for a new manifest: the old manifest's history, then the old manifest itself without it."""
if not manifest_path.exists():
return []
old = load_json(manifest_path, "manifest")
if not isinstance(old, dict):
raise RunError(f"{manifest_path} is not a manifest; it would be lost from the history")
return as_list(old.get("history")) + [{k: v for k, v in old.items() if k != "history"}]
def cmd_build(args: argparse.Namespace) -> int:
sources = Path(args.sources)
set_path, key_path, calls_path = Path(args.set_out), Path(args.key_out), Path(args.calls_out)
manifest_path = Path(args.manifest)
for out_path in (set_path, key_path, calls_path):
if out_path.exists():
raise RunError(f"{out_path} already exists; a built set is frozen and never rebuilt")
draft = load_list(Path(args.draft), DRAFT_SCHEMA, "pairs", "draft")
rep = check_pairs(draft, sources)
if not rep.passed:
print("\n".join(rep.lines()))
raise RunError("the draft fails the check; nothing was built")
calls = lay_out(draft, args.seed, args.lone, args.dense)
cache = SourceCache(sources)
total = sum(len(c) for c in calls)
width, call_width = max(3, len(str(total))), max(3, len(str(len(calls))))
pairs: list[dict[str, Any]] = []
labels: list[dict[str, Any]] = []
layout_calls: list[dict[str, Any]] = []
for k, members in enumerate(calls, 1):
ids = []
for p in members:
cid = f"c{len(pairs) + 1:0{width}d}"
excerpts = []
for e in p["excerpts"]:
source = cache.text(e["file"]) or ""
excerpts.append({"file": e["file"], "line": first_line(source, e["text"]), "text": e["text"]})
pairs.append({"id": cid, "claim": p["claim"], "excerpts": excerpts})
labels.append({"id": cid, "label": p["label"], "kind": p["kind"], "draft_id": p["id"], "why": p.get("why", "")})
ids.append(cid)
layout_calls.append({"call": f"k{k:0{call_width}d}", "ids": ids})
set_bytes, key_bytes = dump(SET_SCHEMA, "pairs", pairs), dump(KEY_SCHEMA, "labels", labels)
layout = {"call_size_max": CALL_SIZE_MAX, "calls": layout_calls, "schema": CALLS_SCHEMA, "set_sha256": sha256_bytes(set_bytes)}
calls_bytes = (json.dumps(layout, indent=1, sort_keys=True, ensure_ascii=False) + "\n").encode("utf-8")
lay = check_layout(layout, {lab["id"]: lab["label"] for lab in labels}, sha256_bytes(set_bytes))
if lay.errors:
raise RunError("the layout breaks K3 point 1, nothing was built: " + "; ".join(lay.errors))
joined = [dict(p, label=lab["label"], kind=lab["kind"], draft_id=lab["draft_id"]) for p, lab in zip(pairs, labels)]
counts = tally(joined)
manifest = {
"schema": MANIFEST_SCHEMA,
"seed": args.seed,
"shuffle": LAYOUT_RULE,
"set": {"file": args.set_label or set_path.name, "sha256": sha256_bytes(set_bytes), "pairs": len(pairs)},
"key": {"file": args.key_label or key_path.name, "sha256": sha256_bytes(key_bytes)},
"calls": {
"file": args.calls_label or calls_path.name,
"sha256": sha256_bytes(calls_bytes),
**lay.stats,
"built_with": {"lone": args.lone, "dense": args.dense},
},
"counts": {**counts, "by_position": lay.by_position},
"thresholds": {
"caught_min": counts[UNSUPPORTED],
"accepted_min": math.ceil(ACCEPTED_SHARE * counts[SUPPORTED]),
},
"sources": {"files": count_files(sources), "tree_sha256": tree_sha256(sources)},
"history": history_of(manifest_path),
}
for path, data in ((set_path, set_bytes), (key_path, key_bytes), (calls_path, calls_bytes)):
path.parent.mkdir(parents=True, exist_ok=True)
path.write_bytes(data)
manifest_path.parent.mkdir(parents=True, exist_ok=True)
manifest_path.write_text(json.dumps(manifest, indent=1) + "\n", encoding="utf-8")
rep.notes = layout_lines(lay)
print("\n".join(rep.lines()))
print(f"seed: {args.seed}")
print(f"set: {len(pairs)} claims, sha256 {manifest['set']['sha256']}")
print(f"key: sha256 {manifest['key']['sha256']}")
print(f"layout: {len(layout_calls)} calls, sha256 {manifest['calls']['sha256']}")
print(f"manifest: {manifest_path}")
return 0
# ---------------------------------------------------------------------------------------------- stub
def cmd_stub(args: argparse.Namespace) -> int:
pairs = load_list(Path(args.set), SET_SCHEMA, "pairs", "set")
if args.key:
verdict = {str(p["id"]): verdict_for(p["label"]) for p in join_set_and_key(Path(args.set), Path(args.key))}
reader = "stub-key"
else:
verdict = {str(p["id"]): args.verdict for p in pairs}
reader = f"stub-all-{args.verdict}"
if args.accept_at is not None:
if not args.calls:
raise RunError("--accept-at needs --calls, the layout that says where each claim stands")
if not 1 <= args.accept_at <= CALL_SIZE_MAX:
raise RunError(f"--accept-at {args.accept_at} is outside 1-{CALL_SIZE_MAX}")
for i, n in positions_of(load_json(Path(args.calls), "layout")).items():
if n == args.accept_at and i in verdict:
verdict[i] = SUPPORTED
reader += f"-accept-at-{args.accept_at}"
reader = args.reader or reader
out = {"reader": reader, "models": [STUB_MODEL],
"verdicts": [{"id": p["id"], "verdict": verdict[str(p["id"])]} for p in pairs]}
Path(args.out).write_text(json.dumps(out, indent=1) + "\n", encoding="utf-8")
print(f"stub {reader}: {len(pairs)} verdicts written to {args.out}")
return 0
# ---------------------------------------------------------------------------------------------- gate
def read_verdicts(path: Path) -> tuple[str, dict[str, str], str]:
"""(reader, verdict per id, fail-closed reason). An id given twice, or with an invalid verdict, is left out."""
if not path.is_file():
return "", {}, f"no verdicts at {path}"
data = load_json(path, "verdicts")
if not isinstance(data, dict) or not isinstance(data.get("verdicts"), list):
raise RunError(f"{path} is not a verdicts file {VERDICTS_SCHEMA_NOTE}")
counts: dict[str, int] = {}
given: dict[str, str] = {}
for v in data["verdicts"]:
if isinstance(v, dict) and isinstance(v.get("id"), str):
counts[v["id"]] = counts.get(v["id"], 0) + 1
if v.get("verdict") in VERDICTS:
given[v["id"]] = v["verdict"]
verdicts = {i: val for i, val in given.items() if counts[i] == 1}
reader = data.get("reader")
return reader if isinstance(reader, str) else "", verdicts, ""
def read_models(path: Path) -> list[str]:
"""The sorted, distinct models a verdicts file names in its "models" list; none when the file or the list is
missing or holds no model name."""
if not path.is_file():
return []
data = load_json(path, "verdicts")
raw = data.get("models") if isinstance(data, dict) else None
return sorted({m for m in as_list(raw) if isinstance(m, str) and m})
def record_passed(path: Path, entry: dict[str, str]) -> bool:
"""Add one green run to the record at path (created when missing); False when the same entry is there already.
A record that cannot be read, or does not have the schema, is a RunError and is left as it was."""
entries: list[Any] = []
if path.exists():
try:
entries = reader.read_passed(path) # the parser collect --gate-passed reads the record with
except reader.RunError as exc:
raise RunError(str(exc)) from exc
if entry in entries:
return False
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps({"schema": PASSED_SCHEMA, "passed": [*entries, entry]}, indent=1) + "\n", encoding="utf-8")
return True
def cmd_gate(args: argparse.Namespace) -> int:
frozen = load_frozen(Path(args.set), Path(args.key), Path(args.calls), Path(args.manifest), Path(args.sources))
if frozen.errors:
raise RunError("the frozen set does not hold: " + "; ".join(frozen.errors))
thresholds = as_dict(frozen.manifest.get("thresholds"))
caught_min, accepted_min = thresholds.get("caught_min"), thresholds.get("accepted_min")
if not isinstance(caught_min, int) or not isinstance(accepted_min, int):
raise RunError("the manifest has no integer thresholds caught_min and accepted_min")
pairs, position = frozen.pairs, frozen.layout.position
reader, verdicts, fail_closed = read_verdicts(Path(args.verdicts))
models = read_models(Path(args.verdicts))
model_reason = "" if len(models) == 1 else (
"the verdicts name no model" if not models else f"the verdicts name {len(models)} models")
n = {lab: sum(1 for p in pairs if p["label"] == lab) for lab in LABELS}
caught = sum(1 for p in pairs if p["label"] == UNSUPPORTED and verdicts.get(p["id"]) == UNSUPPORTED)
accepted = sum(1 for p in pairs if p["label"] == SUPPORTED and verdicts.get(p["id"]) == SUPPORTED)
filler_accepted = sum(1 for p in pairs if p["label"] == FILLER and verdicts.get(p["id"]) == SUPPORTED)
missing = sum(1 for p in pairs if p["id"] not in verdicts)
unknown = sum(1 for i in verdicts if i not in position)
per_kind = []
for k in UNSUPPORTED_KINDS:
of_kind = [p for p in pairs if p["label"] == UNSUPPORTED and p["kind"] == k]
hit = sum(1 for p in of_kind if verdicts.get(p["id"]) == UNSUPPORTED)
per_kind.append(f"{k} {hit} of {len(of_kind)}")
per_position, short = [], []
for at in range(1, CALL_SIZE_MAX + 1):
there = [p for p in pairs if p["label"] == UNSUPPORTED and position.get(p["id"]) == at]
hit = sum(1 for p in there if verdicts.get(p["id"]) == UNSUPPORTED)
per_position.append(f"{at} {hit} of {len(there)}")
if hit < len(there) or not there:
short.append(f"position {at}: caught {hit} of {len(there)}")
green = (
not fail_closed
and not model_reason
and missing == 0
and caught >= caught_min
and caught == n[UNSUPPORTED]
and not short
and accepted >= accepted_min
)
print(
f"set: {len(pairs)} claims ({n[UNSUPPORTED]} {UNSUPPORTED}, {n[SUPPORTED]} {SUPPORTED}, {n[FILLER]} {FILLER})"
f" in {frozen.layout.stats['calls']} calls; reader: {reader or '(none)'}"
)
print(f"model: {', '.join(models) if models else '(none)'}")
if fail_closed:
print(f"fail-closed: {fail_closed}")
if model_reason:
print(f"fail-closed: {model_reason}")
print(f"missing verdicts: {missing}" + (f"; verdicts for unknown ids: {unknown}" if unknown else ""))
print(f"unsupported caught {caught} of {n[UNSUPPORTED]} · supported accepted {accepted} of {n[SUPPORTED]}")
print("caught by kind: " + ", ".join(per_kind))
print("caught by position: " + ", ".join(per_position))
print(f"filler accepted {filler_accepted} of {n[FILLER]} (reported apart, never counted in the thresholds)")
print(
f"required: caught >= {caught_min} of {n[UNSUPPORTED]} with every position 1-{CALL_SIZE_MAX} caught in full,"
f" accepted >= {accepted_min} of {n[SUPPORTED]}, no verdict missing"
)
if not green:
reasons = []
if fail_closed:
reasons.append("fail-closed")
if model_reason:
reasons.append(model_reason)
if missing:
reasons.append(f"{missing} verdicts missing")
if caught < caught_min or caught < n[UNSUPPORTED]:
reasons.append(f"caught {caught} of {n[UNSUPPORTED]}")
reasons += short
if accepted < accepted_min:
reasons.append(f"accepted {accepted} of {n[SUPPORTED]}, needs {accepted_min}")
print("not met: " + "; ".join(reasons))
if green and args.passed:
entry = {"model": models[0], "set_sha256": file_sha256(Path(args.set)), "key_sha256": file_sha256(Path(args.key)),
"verdicts_sha256": file_sha256(Path(args.verdicts)), "reader": reader}
added = record_passed(Path(args.passed), entry)
print(f"passed: {models[0]} {'recorded in' if added else 'already recorded in'} {args.passed}")
print("GREEN" if green else "RED")
return 0 if green else 1
# ---------------------------------------------------------------------------------------------- main
def build_parser() -> argparse.ArgumentParser:
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
sub = p.add_subparsers(dest="command", required=True)
c = sub.add_parser("check", help="check a labelled draft, or a built set with its key, layout and manifest")
c.add_argument("--draft", default="")
c.add_argument("--set", default="")
c.add_argument("--key", default="")
c.add_argument("--calls", default="", help="the call layout (K3 point 1)")
c.add_argument("--manifest", default="")
c.add_argument("--sources", required=True)
b = sub.add_parser("build", help="lay out a checked draft in calls with a fixed seed: set, key, layout and manifest")
b.add_argument("--draft", required=True)
b.add_argument("--sources", required=True)
b.add_argument("--seed", type=int, required=True)
b.add_argument("--lone", type=int, required=True, help="unsupported pairs that stand alone in their call")
b.add_argument("--dense", type=int, required=True, help="unsupported pairs per call for the rest")
b.add_argument("--set-out", required=True)
b.add_argument("--key-out", required=True)
b.add_argument("--calls-out", required=True)
b.add_argument("--set-label", default="", help="path of the set as written in the manifest")
b.add_argument("--key-label", default="", help="path of the key as written in the manifest")
b.add_argument("--calls-label", default="", help="path of the layout as written in the manifest")
b.add_argument("--manifest", required=True)
s = sub.add_parser("stub", help="write verdicts without a reader")
s.add_argument("--set", required=True)
which = s.add_mutually_exclusive_group(required=True)
which.add_argument("--verdict", choices=VERDICTS, help="the same verdict for every claim")
which.add_argument("--key", help="the key's verdict for every claim (the known-positive)")
s.add_argument("--calls", default="", help="the call layout, for --accept-at")
s.add_argument("--accept-at", type=int, default=None, help="answer supported for every claim at this position")
s.add_argument("--reader", default="")
s.add_argument("--out", required=True)
g = sub.add_parser("gate", help="score one reader's verdicts against the key")
g.add_argument("--set", required=True)
g.add_argument("--key", required=True)
g.add_argument("--calls", required=True, help="the call layout (K3 point 1)")
g.add_argument("--manifest", required=True)
g.add_argument("--sources", required=True)
g.add_argument("--verdicts", required=True, help=VERDICTS_SCHEMA_NOTE)
g.add_argument("--passed", default="", metavar="FILE",
help=f"on GREEN, add the run to this record of green runs ({PASSED_SCHEMA}); RED never writes it")
return p
def main(argv: list[str] | None = None) -> int:
args = build_parser().parse_args(argv)
commands = {"check": cmd_check, "build": cmd_build, "stub": cmd_stub, "gate": cmd_gate}
try:
return commands[args.command](args)
except RunError as exc:
print(f"FAIL: {exc}", file=sys.stderr)
return 1
if __name__ == "__main__":
sys.exit(main())