First public release, published as one snapshot. What changed between versions is in CHANGELOG.md. Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
860 lines
41 KiB
Python
860 lines
41 KiB
Python
"""The reader calibration for PLAN.md criterion 2: set checker, seeded build with a call layout, stub reader and verdict
|
|
gate. No model call.
|
|
|
|
The reader judges up to ten claims per call and gives one verdict per claim (K3 in docs/bolge1/0-felles.local.md). The
|
|
calibration therefore comes as three frozen files beside the manifest: the set (claims and excerpts, what the reader
|
|
sees), the key (label and kind per claim) and the call layout (which claims stand together in a call, and where).
|
|
|
|
Labels: "unsupported" (planted, must be caught), "supported" (counted controls, must be accepted) and
|
|
"filler-supported" (supported claims that fill the calls so they resemble a real page; reported apart and never
|
|
counted in the thresholds).
|
|
|
|
check every excerpt stands verbatim in its source file (and, in a built set, at its stated line), every kind of
|
|
the two counted labels is present and fits its label, a wrong-number claim carries a number the excerpts
|
|
do not, a supported or filler claim is not a verbatim copy of its excerpt, and a twin_of names an
|
|
unsupported pair. A built set is checked with its layout against the manifest: sha256 of set, key, layout
|
|
and sources, the layout rules, and the counts per label, kind and position.
|
|
build checks a labelled draft, lays it out in calls with a fixed seed, renumbers the claims (c001.., in call
|
|
order) and the calls (k001..), and writes the set, the key, the layout and a manifest with sha256, counts
|
|
and thresholds only. A built set is frozen: build never overwrites it. An earlier manifest at the same
|
|
path moves into the new one's history.
|
|
stub writes a verdict file without a reader: the same verdict for every claim, or the key's verdicts, and
|
|
optionally "supported" for every claim at one position (the position-mild stub). Its model is "stub".
|
|
gate scores one reader's verdicts against the key: totals, caught per kind and caught per position, and the
|
|
fillers apart; never a result per call or per pair. A missing, invalid or duplicated verdict counts as
|
|
wrong (fail-closed). GREEN only when every unsupported claim is caught at every position, the counted
|
|
supported claims reach the manifest's threshold, no verdict is missing, and the verdicts name exactly
|
|
one model (their "models", written by okfwiki.reader collect from the readers' transcripts; PLAN
|
|
§ Rammer 23). It prints the model it scored. With --passed FILE a GREEN run is added to FILE, the record
|
|
of green runs (okf-wiki/calibration-passed/1: model, set_sha256, key_sha256, verdicts_sha256, reader; no
|
|
time, and an entry already there is not added twice); a RED run never writes it. okfwiki.reader collect
|
|
--gate-passed FILE turns a reading into verdicts only for a model in FILE with a run on the set and key
|
|
of the tracked manifest, so a model change, or a new set, needs a new gate run. Both read FILE with
|
|
okfwiki.reader.read_passed.
|
|
|
|
Exit 0 = PASS / GREEN / written; 1 = FAIL / RED or could not run. A check that cannot run is a FAIL.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
import math
|
|
import random
|
|
import re
|
|
import sys
|
|
from dataclasses import dataclass, field
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
|
|
|
from okfwiki import reader # noqa: E402
|
|
|
|
DRAFT_SCHEMA = "okf-wiki/calibration-draft/1"
|
|
SET_SCHEMA = "okf-wiki/calibration-set/1"
|
|
KEY_SCHEMA = "okf-wiki/calibration-key/1"
|
|
CALLS_SCHEMA = "okf-wiki/reader-calls/1"
|
|
MANIFEST_SCHEMA = "okf-wiki/calibration-manifest/2"
|
|
PASSED_SCHEMA = reader.PASSED_SCHEMA
|
|
STUB_MODEL = "stub"
|
|
VERDICTS_SCHEMA_NOTE = (
|
|
'{"reader": "<name>", "models": ["<model>"], "verdicts": [{"id": "c001", "verdict": "supported" | "unsupported"}]}'
|
|
)
|
|
|
|
UNSUPPORTED = "unsupported"
|
|
SUPPORTED = "supported"
|
|
FILLER = "filler-supported"
|
|
LABELS = (UNSUPPORTED, SUPPORTED, FILLER)
|
|
COUNTED = (UNSUPPORTED, SUPPORTED)
|
|
VERDICTS = (SUPPORTED, UNSUPPORTED)
|
|
UNSUPPORTED_KINDS = (
|
|
"overgeneralisation",
|
|
"wrong-number",
|
|
"swapped-cause",
|
|
"not-mentioned",
|
|
"reversed-polarity",
|
|
"conflated-sources",
|
|
)
|
|
SUPPORTED_KINDS = ("paraphrase", "inference", "synthesis")
|
|
KINDS = {UNSUPPORTED: UNSUPPORTED_KINDS, SUPPORTED: SUPPORTED_KINDS, FILLER: SUPPORTED_KINDS}
|
|
TWO_EXCERPT_KINDS = ("conflated-sources", "synthesis")
|
|
MAX_EXCERPTS = 2
|
|
CLAIM_MIN, CLAIM_MAX = 10, 400
|
|
EXCERPT_MIN, EXCERPT_MAX = 20, 1500
|
|
CALL_SIZE_MAX = 10 # K3 point 1: up to ten claims per call, positions 1-10
|
|
ACCEPTED_SHARE = 0.9 # PM's choice (order E3): a reader that rejects more than 10 % of supported claims is unusable.
|
|
NUMBER = re.compile(r"\d+(?:[.,]\d+)*")
|
|
LAYOUT_STATS = ("calls", "call_size_max", "unsupported_alone", "unsupported_per_call")
|
|
LAYOUT_RULE = (
|
|
"random.Random(seed): each label's pairs sorted by draft id and shuffled; the first `lone` unsupported pairs alone"
|
|
" in a call at positions spread evenly over 1-10, the rest `dense` to a call; counted supported pairs one per call"
|
|
" in call order; filler pairs fill every call to ten in a cycle, never twice in a call and never beside the"
|
|
" unsupported pair they are the twin of; the other positions shuffled; ids renumbered c001.. in call order,"
|
|
" calls k001.."
|
|
)
|
|
|
|
|
|
class RunError(Exception):
|
|
"""The command cannot run; reported as FAIL with exit 1."""
|
|
|
|
|
|
# ---------------------------------------------------------------------------------------------- helpers
|
|
|
|
|
|
def sha256_bytes(data: bytes) -> str:
|
|
return hashlib.sha256(data).hexdigest()
|
|
|
|
|
|
def tree_sha256(sources: Path) -> str:
|
|
"""One hash over the flat sources directory: sorted "name NUL sha256 LF" lines (same as gates/multidoc.py)."""
|
|
lines = []
|
|
for p in sorted(sources.iterdir(), key=lambda p: p.name):
|
|
if p.is_file():
|
|
lines.append(f"{p.name}\0{sha256_bytes(p.read_bytes())}\n")
|
|
return sha256_bytes("".join(lines).encode("utf-8"))
|
|
|
|
|
|
def count_files(sources: Path) -> int:
|
|
return sum(1 for p in sources.iterdir() if p.is_file())
|
|
|
|
|
|
def normalise(text: str) -> str:
|
|
"""Case-folded, whitespace collapsed, outer punctuation stripped. Used for the verbatim-copy test only."""
|
|
return re.sub(r"\s+", " ", text).strip().strip(".,;:!?").strip().casefold()
|
|
|
|
|
|
def numbers(text: str) -> set[str]:
|
|
return {n.replace(",", "") for n in NUMBER.findall(text)}
|
|
|
|
|
|
def as_list(value: object) -> list[Any]:
|
|
return value if isinstance(value, list) else []
|
|
|
|
|
|
def as_dict(value: object) -> dict[str, Any]:
|
|
return value if isinstance(value, dict) else {}
|
|
|
|
|
|
def is_plain_name(name: object) -> bool:
|
|
return isinstance(name, str) and name != "" and "/" not in name and "\\" not in name and name not in (".", "..")
|
|
|
|
|
|
def load_json(path: Path, what: str) -> Any:
|
|
try:
|
|
return json.loads(path.read_text(encoding="utf-8"))
|
|
except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc:
|
|
raise RunError(f"cannot read {what} {path}: {exc}") from exc
|
|
|
|
|
|
def load_list(path: Path, schema: str, key: str, what: str) -> list[dict[str, Any]]:
|
|
data = load_json(path, what)
|
|
if not isinstance(data, dict) or data.get("schema") != schema or not isinstance(data.get(key), list):
|
|
raise RunError(f"{path} is not a {what} with schema {schema}")
|
|
items: list[dict[str, Any]] = data[key]
|
|
return items
|
|
|
|
|
|
def require_dir(path: Path, what: str) -> None:
|
|
if not path.is_dir():
|
|
raise RunError(f"{what} {path} is not a directory")
|
|
|
|
|
|
def file_sha256(path: Path) -> str:
|
|
return sha256_bytes(path.read_bytes()) if path.is_file() else "(missing)"
|
|
|
|
|
|
def line_of(text: str, excerpt: str, line: int) -> bool:
|
|
"""True when an occurrence of the excerpt starts on the given 1-based line."""
|
|
start = text.find(excerpt)
|
|
while start != -1:
|
|
if text.count("\n", 0, start) + 1 == line:
|
|
return True
|
|
start = text.find(excerpt, start + 1)
|
|
return False
|
|
|
|
|
|
def first_line(text: str, excerpt: str) -> int:
|
|
return text.count("\n", 0, text.find(excerpt)) + 1
|
|
|
|
|
|
def verdict_for(label: object) -> str:
|
|
"""The right verdict for a label: only an unsupported claim should be rejected."""
|
|
return UNSUPPORTED if label == UNSUPPORTED else SUPPORTED
|
|
|
|
|
|
class SourceCache:
|
|
def __init__(self, root: Path) -> None:
|
|
self.root = root
|
|
self._text: dict[str, str | None] = {}
|
|
|
|
def text(self, name: str) -> str | None:
|
|
"""File text, or None when the name is not a plain file directly in the sources directory."""
|
|
if not is_plain_name(name):
|
|
return None
|
|
if name not in self._text:
|
|
p = self.root / name
|
|
try:
|
|
self._text[name] = p.read_text(encoding="utf-8") if p.is_file() else None
|
|
except (OSError, UnicodeDecodeError):
|
|
self._text[name] = None
|
|
return self._text[name]
|
|
|
|
|
|
# ---------------------------------------------------------------------------------------------- check
|
|
|
|
|
|
@dataclass
|
|
class CheckReport:
|
|
pairs: int = 0
|
|
unsupported: int = 0
|
|
supported: int = 0
|
|
filler: int = 0
|
|
excerpts: int = 0
|
|
excerpts_verbatim: int = 0
|
|
lines_checked: int = 0
|
|
lines_correct: int = 0
|
|
by_kind: dict[str, int] = field(default_factory=lambda: {k: 0 for lab in COUNTED for k in KINDS[lab]})
|
|
filler_by_kind: dict[str, int] = field(default_factory=lambda: {k: 0 for k in KINDS[FILLER]})
|
|
errors: list[str] = field(default_factory=list)
|
|
notes: list[str] = field(default_factory=list)
|
|
|
|
@property
|
|
def passed(self) -> bool:
|
|
return (
|
|
self.pairs > 0
|
|
and not self.errors
|
|
and self.excerpts_verbatim == self.excerpts
|
|
and self.lines_correct == self.lines_checked
|
|
)
|
|
|
|
def lines(self) -> list[str]:
|
|
out = [f"error: {e}" for e in self.errors]
|
|
out += [
|
|
f"pairs: {self.pairs} ({self.unsupported} {UNSUPPORTED}, {self.supported} {SUPPORTED}, {self.filler} {FILLER})",
|
|
f"excerpts verbatim: {self.excerpts_verbatim} of {self.excerpts}",
|
|
]
|
|
if self.lines_checked:
|
|
out.append(f"excerpt lines correct: {self.lines_correct} of {self.lines_checked}")
|
|
for label in COUNTED:
|
|
out.append(f"{label} by kind: " + ", ".join(f"{k} {self.by_kind[k]}" for k in KINDS[label]))
|
|
out.append(f"{FILLER} by kind: " + ", ".join(f"{k} {self.filler_by_kind[k]}" for k in KINDS[FILLER]))
|
|
out += self.notes
|
|
out.append("PASS" if self.passed else "FAIL")
|
|
return out
|
|
|
|
|
|
def check_pairs(pairs: list[dict[str, Any]], sources: Path, with_lines: bool = False) -> CheckReport:
|
|
"""Check labelled pairs ({id, label, kind, claim, excerpts: [{file, text[, line]}][, twin_of]}) against the sources."""
|
|
require_dir(sources, "sources")
|
|
cache = SourceCache(sources)
|
|
rep = CheckReport(pairs=len(pairs))
|
|
unsupported_ids = {p.get("id") for p in pairs if isinstance(p, dict) and p.get("label") == UNSUPPORTED}
|
|
seen: set[str] = set()
|
|
for p in pairs:
|
|
pid = p.get("id") if isinstance(p, dict) else None
|
|
if not isinstance(pid, str) or not pid:
|
|
rep.errors.append("pair without an id")
|
|
continue
|
|
if pid in seen:
|
|
rep.errors.append(f"duplicate id {pid}")
|
|
seen.add(pid)
|
|
label, kind, claim = p.get("label"), p.get("kind"), p.get("claim")
|
|
if label not in KINDS:
|
|
rep.errors.append(f"{pid}: label {label!r} is not one of {', '.join(LABELS)}")
|
|
elif kind not in KINDS[label]:
|
|
rep.errors.append(f"{pid}: kind {kind!r} does not belong to label {label}")
|
|
elif label == UNSUPPORTED:
|
|
rep.unsupported += 1
|
|
rep.by_kind[kind] += 1
|
|
elif label == SUPPORTED:
|
|
rep.supported += 1
|
|
rep.by_kind[kind] += 1
|
|
else:
|
|
rep.filler += 1
|
|
rep.filler_by_kind[kind] += 1
|
|
twin = p.get("twin_of")
|
|
if twin is not None and (label != FILLER or twin not in unsupported_ids):
|
|
rep.errors.append(f"{pid}: twin_of {twin!r} does not name an unsupported pair of a {FILLER} pair")
|
|
if not isinstance(claim, str) or not CLAIM_MIN <= len(claim.strip()) <= CLAIM_MAX:
|
|
rep.errors.append(f"{pid}: claim missing or length outside {CLAIM_MIN}-{CLAIM_MAX}")
|
|
claim = ""
|
|
excerpts = as_list(p.get("excerpts"))
|
|
if not 1 <= len(excerpts) <= MAX_EXCERPTS:
|
|
rep.errors.append(f"{pid}: {len(excerpts)} excerpts, expected 1-{MAX_EXCERPTS}")
|
|
texts: list[str] = []
|
|
files: set[str] = set()
|
|
for e in excerpts:
|
|
rep.excerpts += 1
|
|
name, text = (e.get("file"), e.get("text")) if isinstance(e, dict) else (None, None)
|
|
if not is_plain_name(name) or not isinstance(name, str):
|
|
rep.errors.append(f"{pid}: excerpt file {name!r} is not a plain file name")
|
|
continue
|
|
if not isinstance(text, str):
|
|
rep.errors.append(f"{pid}: excerpt in {name} is not text")
|
|
continue
|
|
if not EXCERPT_MIN <= len(text) <= EXCERPT_MAX:
|
|
rep.errors.append(f"{pid}: excerpt length {len(text)} in {name} outside {EXCERPT_MIN}-{EXCERPT_MAX}")
|
|
files.add(name)
|
|
texts.append(text)
|
|
source = cache.text(name)
|
|
if source is not None and text in source:
|
|
rep.excerpts_verbatim += 1
|
|
if with_lines:
|
|
rep.lines_checked += 1
|
|
line = e.get("line")
|
|
if source is not None and isinstance(line, int) and line_of(source, text, line):
|
|
rep.lines_correct += 1
|
|
if kind in TWO_EXCERPT_KINDS and len(excerpts) != 2:
|
|
rep.errors.append(f"{pid}: kind {kind} needs two excerpts")
|
|
if kind == "conflated-sources" and len(files) != 2:
|
|
rep.errors.append(f"{pid}: kind conflated-sources needs excerpts from two different files")
|
|
if kind == "wrong-number" and not numbers(claim) - set().union(*(numbers(t) for t in texts)):
|
|
rep.errors.append(f"{pid}: wrong-number claim has no number that is absent from the excerpts")
|
|
if label in (SUPPORTED, FILLER) and claim and any(normalise(claim) in normalise(t) for t in texts):
|
|
rep.errors.append(f"{pid}: {label} claim is a verbatim copy of its excerpt")
|
|
for label in COUNTED:
|
|
for k in KINDS[label]:
|
|
if rep.by_kind[k] == 0:
|
|
rep.errors.append(f"kind {k} has no pair")
|
|
return rep
|
|
|
|
|
|
def join_set_and_key(set_path: Path, key_path: Path) -> list[dict[str, Any]]:
|
|
pairs = load_list(set_path, SET_SCHEMA, "pairs", "set")
|
|
labels = load_list(key_path, KEY_SCHEMA, "labels", "key")
|
|
by_id = {k.get("id"): k for k in labels if isinstance(k, dict)}
|
|
ids = [p.get("id") for p in pairs if isinstance(p, dict)]
|
|
if set(ids) != set(by_id) or len(ids) != len(labels) or len(ids) != len(pairs):
|
|
raise RunError("the key does not label exactly the pairs of the set")
|
|
return [
|
|
dict(p, label=by_id[p["id"]].get("label"), kind=by_id[p["id"]].get("kind"), draft_id=by_id[p["id"]].get("draft_id"))
|
|
for p in pairs
|
|
]
|
|
|
|
|
|
# ---------------------------------------------------------------------------------------------- layout
|
|
|
|
|
|
@dataclass
|
|
class Layout:
|
|
"""What a call layout says, measured against the labels: positions, counts, and every rule it breaks."""
|
|
|
|
position: dict[str, int] = field(default_factory=dict)
|
|
stats: dict[str, Any] = field(default_factory=dict)
|
|
by_position: dict[str, dict[str, int]] = field(default_factory=dict)
|
|
errors: list[str] = field(default_factory=list)
|
|
|
|
|
|
def shown(ids: list[str]) -> str:
|
|
return ", ".join(ids[:5]) + (", ..." if len(ids) > 5 else "")
|
|
|
|
|
|
def check_layout(layout: object, labels: dict[str, str], set_sha256: str) -> Layout:
|
|
"""K3 point 1: every claim of the set in exactly one call, at most call_size_max (<= 10) claims per call, the
|
|
layout bound to the set by its sha256, and unsupported claims at every position 1-10."""
|
|
out = Layout()
|
|
if not isinstance(layout, dict) or layout.get("schema") != CALLS_SCHEMA:
|
|
out.errors.append(f"the layout is not a layout with schema {CALLS_SCHEMA}")
|
|
return out
|
|
size_max = layout.get("call_size_max")
|
|
if not isinstance(size_max, int) or isinstance(size_max, bool) or not 1 <= size_max <= CALL_SIZE_MAX:
|
|
out.errors.append(f"call_size_max {size_max!r} is not a whole number from 1 to {CALL_SIZE_MAX}")
|
|
size_max = CALL_SIZE_MAX
|
|
if layout.get("set_sha256") != set_sha256:
|
|
out.errors.append(f"the layout's set_sha256 {layout.get('set_sha256')!r} is not the set's sha256 {set_sha256}")
|
|
calls = layout.get("calls")
|
|
if not isinstance(calls, list) or not calls:
|
|
out.errors.append("the layout has no calls")
|
|
return out
|
|
seen: dict[str, int] = {}
|
|
names: set[str] = set()
|
|
per_call: dict[int, int] = {}
|
|
for c in calls:
|
|
name, ids = (c.get("call"), c.get("ids")) if isinstance(c, dict) else (None, None)
|
|
if not isinstance(name, str) or not isinstance(ids, list) or not ids or not all(isinstance(i, str) for i in ids):
|
|
out.errors.append(f"call {name!r} is not a named call with a list of claim ids")
|
|
continue
|
|
if name in names:
|
|
out.errors.append(f"call name {name} is given twice")
|
|
names.add(name)
|
|
if len(ids) > size_max:
|
|
out.errors.append(f"call {name} has {len(ids)} claims, more than {size_max}")
|
|
for n, i in enumerate(ids, 1):
|
|
seen[i] = seen.get(i, 0) + 1
|
|
out.position[i] = n
|
|
u = sum(1 for i in ids if labels.get(i) == UNSUPPORTED)
|
|
per_call[u] = per_call.get(u, 0) + 1
|
|
twice = sorted(i for i, n in seen.items() if n > 1)
|
|
if twice:
|
|
out.errors.append(f"{len(twice)} claim id(s) stand in more than one place: {shown(twice)}")
|
|
unknown = sorted(i for i in seen if i not in labels)
|
|
if unknown:
|
|
out.errors.append(f"{len(unknown)} claim id(s) of the layout are not in the set: {shown(unknown)}")
|
|
missing = sorted(i for i in labels if i not in seen)
|
|
if missing:
|
|
out.errors.append(f"{len(missing)} claim id(s) of the set stand in no call: {shown(missing)}")
|
|
out.by_position = {str(n): {lab: 0 for lab in LABELS} for n in range(1, CALL_SIZE_MAX + 1)}
|
|
for i, n in out.position.items():
|
|
if i in labels and str(n) in out.by_position and labels[i] in LABELS:
|
|
out.by_position[str(n)][labels[i]] += 1
|
|
for n in range(1, CALL_SIZE_MAX + 1):
|
|
if out.by_position[str(n)][UNSUPPORTED] == 0:
|
|
out.errors.append(f"position {n} has no unsupported claim")
|
|
out.stats = {
|
|
"calls": len(calls),
|
|
"call_size_max": size_max,
|
|
"unsupported_alone": per_call.get(1, 0),
|
|
"unsupported_per_call": {str(k): per_call[k] for k in sorted(per_call)},
|
|
}
|
|
return out
|
|
|
|
|
|
def layout_lines(lay: Layout) -> list[str]:
|
|
per_call = lay.stats.get("unsupported_per_call", {})
|
|
return [
|
|
f"calls: {lay.stats.get('calls', 0)} (at most {lay.stats.get('call_size_max', CALL_SIZE_MAX)} claims each)",
|
|
"unsupported per call: " + ", ".join(f"{k} in {v} calls" for k, v in per_call.items()),
|
|
f"unsupported alone in their call: {lay.stats.get('unsupported_alone', 0)}",
|
|
"unsupported by position: "
|
|
+ ", ".join(f"{n} {lay.by_position.get(str(n), {}).get(UNSUPPORTED, 0)}" for n in range(1, CALL_SIZE_MAX + 1)),
|
|
]
|
|
|
|
|
|
def positions_of(layout: object) -> dict[str, int]:
|
|
"""Claim id -> position 1.. in its call, read leniently (the stub's use; the gate checks the layout itself)."""
|
|
out: dict[str, int] = {}
|
|
for c in as_list(as_dict(layout).get("calls")):
|
|
for n, i in enumerate(as_list(as_dict(c).get("ids")), 1):
|
|
if isinstance(i, str):
|
|
out[i] = n
|
|
return out
|
|
|
|
|
|
def tally(pairs: list[dict[str, Any]]) -> dict[str, Any]:
|
|
"""Counts per label and per kind of the two counted labels, and the distinct filler pairs behind the slots."""
|
|
out: dict[str, Any] = {lab: sum(1 for p in pairs if p.get("label") == lab) for lab in LABELS}
|
|
out["filler_distinct"] = len({str(p.get("draft_id", p.get("id"))) for p in pairs if p.get("label") == FILLER})
|
|
out["by_kind"] = {
|
|
k: sum(1 for p in pairs if p.get("label") == lab and p.get("kind") == k) for lab in COUNTED for k in KINDS[lab]
|
|
}
|
|
return out
|
|
|
|
|
|
@dataclass
|
|
class Frozen:
|
|
pairs: list[dict[str, Any]]
|
|
layout: Layout
|
|
manifest: dict[str, Any]
|
|
errors: list[str]
|
|
|
|
|
|
def load_frozen(set_path: Path, key_path: Path, calls_path: Path, manifest_path: Path, sources: Path) -> Frozen:
|
|
"""The set, key and layout with every disagreement with the manifest listed (sha256, sources, layout, counts)."""
|
|
require_dir(sources, "sources")
|
|
manifest = load_json(manifest_path, "manifest")
|
|
if not isinstance(manifest, dict) or manifest.get("schema") != MANIFEST_SCHEMA:
|
|
raise RunError(f"{manifest_path} is not a manifest with schema {MANIFEST_SCHEMA}")
|
|
errors: list[str] = []
|
|
for path, part in ((set_path, "set"), (key_path, "key"), (calls_path, "calls")):
|
|
digest = file_sha256(path)
|
|
if digest != as_dict(manifest.get(part)).get("sha256"):
|
|
errors.append(f"{part} {path} sha256 {digest} does not match the manifest")
|
|
if tree_sha256(sources) != as_dict(manifest.get("sources")).get("tree_sha256"):
|
|
errors.append(f"sources {sources} differ from the manifest's tree_sha256; the set is frozen against them")
|
|
pairs = join_set_and_key(set_path, key_path)
|
|
labels = {str(p["id"]): str(p.get("label")) for p in pairs}
|
|
layout = check_layout(load_json(calls_path, "layout"), labels, file_sha256(set_path))
|
|
errors += layout.errors
|
|
counts = as_dict(manifest.get("counts"))
|
|
for k, v in {**tally(pairs), "by_position": layout.by_position}.items():
|
|
if counts.get(k) != v:
|
|
errors.append(f"the manifest's counts.{k} differs from the set and layout" + ("" if isinstance(v, dict) else f" ({counts.get(k)!r}, measured {v!r})"))
|
|
calls = as_dict(manifest.get("calls"))
|
|
for k in LAYOUT_STATS:
|
|
if k in layout.stats and calls.get(k) != layout.stats[k]:
|
|
errors.append(f"the manifest's calls.{k} differs from the layout ({calls.get(k)!r}, measured {layout.stats[k]!r})")
|
|
return Frozen(pairs, layout, manifest, errors)
|
|
|
|
|
|
def cmd_check(args: argparse.Namespace) -> int:
|
|
sources = Path(args.sources)
|
|
if args.draft:
|
|
rep = check_pairs(load_list(Path(args.draft), DRAFT_SCHEMA, "pairs", "draft"), sources)
|
|
elif args.set and args.key and args.calls and args.manifest:
|
|
frozen = load_frozen(Path(args.set), Path(args.key), Path(args.calls), Path(args.manifest), sources)
|
|
rep = check_pairs(frozen.pairs, sources, with_lines=True)
|
|
rep.errors = frozen.errors + rep.errors
|
|
rep.notes = layout_lines(frozen.layout)
|
|
else:
|
|
raise RunError("check needs --draft, or --set, --key, --calls and --manifest")
|
|
print("\n".join(rep.lines()))
|
|
return 0 if rep.passed else 1
|
|
|
|
|
|
# ---------------------------------------------------------------------------------------------- build
|
|
|
|
|
|
def dump(schema: str, key: str, items: list[dict[str, Any]]) -> bytes:
|
|
return (json.dumps({"schema": schema, key: items}, ensure_ascii=False, indent=1) + "\n").encode("utf-8")
|
|
|
|
|
|
def lay_out(draft: list[dict[str, Any]], seed: int, lone: int, dense: int) -> list[list[dict[str, Any]]]:
|
|
"""Calls of draft pairs, each in position order. With random.Random(seed): each label's pairs sorted by draft id and
|
|
shuffled; the first `lone` unsupported pairs stand alone in a call, at positions spread evenly over 1-10; the rest
|
|
stand `dense` to a call; counted supported pairs go one per call in call order; filler pairs fill every call to
|
|
ten, taken in a cycle, never twice in one call and never in the call of the unsupported pair they are the twin of;
|
|
the other positions are shuffled."""
|
|
rng = random.Random(seed)
|
|
groups = {lab: sorted((p for p in draft if p.get("label") == lab), key=lambda p: str(p["id"])) for lab in LABELS}
|
|
for lab in LABELS:
|
|
rng.shuffle(groups[lab])
|
|
unsupported, supported, fillers = groups[UNSUPPORTED], groups[SUPPORTED], groups[FILLER]
|
|
if not 0 <= lone <= len(unsupported):
|
|
raise RunError(f"--lone {lone} is outside 0-{len(unsupported)} (the unsupported pairs in the draft)")
|
|
if not 1 <= dense <= CALL_SIZE_MAX:
|
|
raise RunError(f"--dense {dense} is outside 1-{CALL_SIZE_MAX}")
|
|
rest = unsupported[lone:]
|
|
calls = [[u] for u in unsupported[:lone]] + [rest[i : i + dense] for i in range(0, len(rest), dense)]
|
|
pending = list(supported)
|
|
while pending:
|
|
free = [c for c in calls if len(c) < CALL_SIZE_MAX]
|
|
if not free:
|
|
raise RunError(f"{len(pending)} supported pairs find no free place in the calls")
|
|
for c in free[: len(pending)]:
|
|
c.append(pending.pop(0))
|
|
cursor = 0
|
|
for c in calls:
|
|
need = CALL_SIZE_MAX - len(c)
|
|
if need and not fillers:
|
|
raise RunError(f"a call has {need} free places and the draft has no {FILLER} pairs to fill them")
|
|
taken = {str(p["id"]) for p in c}
|
|
twins_of = {str(p["id"]) for p in c if p.get("label") == UNSUPPORTED}
|
|
misses = 0
|
|
while need > 0:
|
|
f = fillers[cursor % len(fillers)]
|
|
cursor += 1
|
|
if str(f["id"]) in taken or f.get("twin_of") in twins_of:
|
|
misses += 1
|
|
if misses >= len(fillers):
|
|
raise RunError(f"too few distinct {FILLER} pairs to fill a call without repeating one or meeting its twin")
|
|
continue
|
|
c.append(f)
|
|
taken.add(str(f["id"]))
|
|
need -= 1
|
|
misses = 0
|
|
at = [n % CALL_SIZE_MAX for n in range(lone)]
|
|
rng.shuffle(at)
|
|
ordered: list[list[dict[str, Any]]] = []
|
|
for n, c in enumerate(calls):
|
|
if n < lone:
|
|
head, others = c[0], c[1:]
|
|
rng.shuffle(others)
|
|
ordered.append(others[: at[n]] + [head] + others[at[n] :])
|
|
else:
|
|
rng.shuffle(c)
|
|
ordered.append(c)
|
|
return ordered
|
|
|
|
|
|
def history_of(manifest_path: Path) -> list[Any]:
|
|
"""The history for a new manifest: the old manifest's history, then the old manifest itself without it."""
|
|
if not manifest_path.exists():
|
|
return []
|
|
old = load_json(manifest_path, "manifest")
|
|
if not isinstance(old, dict):
|
|
raise RunError(f"{manifest_path} is not a manifest; it would be lost from the history")
|
|
return as_list(old.get("history")) + [{k: v for k, v in old.items() if k != "history"}]
|
|
|
|
|
|
def cmd_build(args: argparse.Namespace) -> int:
|
|
sources = Path(args.sources)
|
|
set_path, key_path, calls_path = Path(args.set_out), Path(args.key_out), Path(args.calls_out)
|
|
manifest_path = Path(args.manifest)
|
|
for out_path in (set_path, key_path, calls_path):
|
|
if out_path.exists():
|
|
raise RunError(f"{out_path} already exists; a built set is frozen and never rebuilt")
|
|
draft = load_list(Path(args.draft), DRAFT_SCHEMA, "pairs", "draft")
|
|
rep = check_pairs(draft, sources)
|
|
if not rep.passed:
|
|
print("\n".join(rep.lines()))
|
|
raise RunError("the draft fails the check; nothing was built")
|
|
calls = lay_out(draft, args.seed, args.lone, args.dense)
|
|
cache = SourceCache(sources)
|
|
total = sum(len(c) for c in calls)
|
|
width, call_width = max(3, len(str(total))), max(3, len(str(len(calls))))
|
|
pairs: list[dict[str, Any]] = []
|
|
labels: list[dict[str, Any]] = []
|
|
layout_calls: list[dict[str, Any]] = []
|
|
for k, members in enumerate(calls, 1):
|
|
ids = []
|
|
for p in members:
|
|
cid = f"c{len(pairs) + 1:0{width}d}"
|
|
excerpts = []
|
|
for e in p["excerpts"]:
|
|
source = cache.text(e["file"]) or ""
|
|
excerpts.append({"file": e["file"], "line": first_line(source, e["text"]), "text": e["text"]})
|
|
pairs.append({"id": cid, "claim": p["claim"], "excerpts": excerpts})
|
|
labels.append({"id": cid, "label": p["label"], "kind": p["kind"], "draft_id": p["id"], "why": p.get("why", "")})
|
|
ids.append(cid)
|
|
layout_calls.append({"call": f"k{k:0{call_width}d}", "ids": ids})
|
|
set_bytes, key_bytes = dump(SET_SCHEMA, "pairs", pairs), dump(KEY_SCHEMA, "labels", labels)
|
|
layout = {"call_size_max": CALL_SIZE_MAX, "calls": layout_calls, "schema": CALLS_SCHEMA, "set_sha256": sha256_bytes(set_bytes)}
|
|
calls_bytes = (json.dumps(layout, indent=1, sort_keys=True, ensure_ascii=False) + "\n").encode("utf-8")
|
|
lay = check_layout(layout, {lab["id"]: lab["label"] for lab in labels}, sha256_bytes(set_bytes))
|
|
if lay.errors:
|
|
raise RunError("the layout breaks K3 point 1, nothing was built: " + "; ".join(lay.errors))
|
|
joined = [dict(p, label=lab["label"], kind=lab["kind"], draft_id=lab["draft_id"]) for p, lab in zip(pairs, labels)]
|
|
counts = tally(joined)
|
|
manifest = {
|
|
"schema": MANIFEST_SCHEMA,
|
|
"seed": args.seed,
|
|
"shuffle": LAYOUT_RULE,
|
|
"set": {"file": args.set_label or set_path.name, "sha256": sha256_bytes(set_bytes), "pairs": len(pairs)},
|
|
"key": {"file": args.key_label or key_path.name, "sha256": sha256_bytes(key_bytes)},
|
|
"calls": {
|
|
"file": args.calls_label or calls_path.name,
|
|
"sha256": sha256_bytes(calls_bytes),
|
|
**lay.stats,
|
|
"built_with": {"lone": args.lone, "dense": args.dense},
|
|
},
|
|
"counts": {**counts, "by_position": lay.by_position},
|
|
"thresholds": {
|
|
"caught_min": counts[UNSUPPORTED],
|
|
"accepted_min": math.ceil(ACCEPTED_SHARE * counts[SUPPORTED]),
|
|
},
|
|
"sources": {"files": count_files(sources), "tree_sha256": tree_sha256(sources)},
|
|
"history": history_of(manifest_path),
|
|
}
|
|
for path, data in ((set_path, set_bytes), (key_path, key_bytes), (calls_path, calls_bytes)):
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
path.write_bytes(data)
|
|
manifest_path.parent.mkdir(parents=True, exist_ok=True)
|
|
manifest_path.write_text(json.dumps(manifest, indent=1) + "\n", encoding="utf-8")
|
|
rep.notes = layout_lines(lay)
|
|
print("\n".join(rep.lines()))
|
|
print(f"seed: {args.seed}")
|
|
print(f"set: {len(pairs)} claims, sha256 {manifest['set']['sha256']}")
|
|
print(f"key: sha256 {manifest['key']['sha256']}")
|
|
print(f"layout: {len(layout_calls)} calls, sha256 {manifest['calls']['sha256']}")
|
|
print(f"manifest: {manifest_path}")
|
|
return 0
|
|
|
|
|
|
# ---------------------------------------------------------------------------------------------- stub
|
|
|
|
|
|
def cmd_stub(args: argparse.Namespace) -> int:
|
|
pairs = load_list(Path(args.set), SET_SCHEMA, "pairs", "set")
|
|
if args.key:
|
|
verdict = {str(p["id"]): verdict_for(p["label"]) for p in join_set_and_key(Path(args.set), Path(args.key))}
|
|
reader = "stub-key"
|
|
else:
|
|
verdict = {str(p["id"]): args.verdict for p in pairs}
|
|
reader = f"stub-all-{args.verdict}"
|
|
if args.accept_at is not None:
|
|
if not args.calls:
|
|
raise RunError("--accept-at needs --calls, the layout that says where each claim stands")
|
|
if not 1 <= args.accept_at <= CALL_SIZE_MAX:
|
|
raise RunError(f"--accept-at {args.accept_at} is outside 1-{CALL_SIZE_MAX}")
|
|
for i, n in positions_of(load_json(Path(args.calls), "layout")).items():
|
|
if n == args.accept_at and i in verdict:
|
|
verdict[i] = SUPPORTED
|
|
reader += f"-accept-at-{args.accept_at}"
|
|
reader = args.reader or reader
|
|
out = {"reader": reader, "models": [STUB_MODEL],
|
|
"verdicts": [{"id": p["id"], "verdict": verdict[str(p["id"])]} for p in pairs]}
|
|
Path(args.out).write_text(json.dumps(out, indent=1) + "\n", encoding="utf-8")
|
|
print(f"stub {reader}: {len(pairs)} verdicts written to {args.out}")
|
|
return 0
|
|
|
|
|
|
# ---------------------------------------------------------------------------------------------- gate
|
|
|
|
|
|
def read_verdicts(path: Path) -> tuple[str, dict[str, str], str]:
|
|
"""(reader, verdict per id, fail-closed reason). An id given twice, or with an invalid verdict, is left out."""
|
|
if not path.is_file():
|
|
return "", {}, f"no verdicts at {path}"
|
|
data = load_json(path, "verdicts")
|
|
if not isinstance(data, dict) or not isinstance(data.get("verdicts"), list):
|
|
raise RunError(f"{path} is not a verdicts file {VERDICTS_SCHEMA_NOTE}")
|
|
counts: dict[str, int] = {}
|
|
given: dict[str, str] = {}
|
|
for v in data["verdicts"]:
|
|
if isinstance(v, dict) and isinstance(v.get("id"), str):
|
|
counts[v["id"]] = counts.get(v["id"], 0) + 1
|
|
if v.get("verdict") in VERDICTS:
|
|
given[v["id"]] = v["verdict"]
|
|
verdicts = {i: val for i, val in given.items() if counts[i] == 1}
|
|
reader = data.get("reader")
|
|
return reader if isinstance(reader, str) else "", verdicts, ""
|
|
|
|
|
|
def read_models(path: Path) -> list[str]:
|
|
"""The sorted, distinct models a verdicts file names in its "models" list; none when the file or the list is
|
|
missing or holds no model name."""
|
|
if not path.is_file():
|
|
return []
|
|
data = load_json(path, "verdicts")
|
|
raw = data.get("models") if isinstance(data, dict) else None
|
|
return sorted({m for m in as_list(raw) if isinstance(m, str) and m})
|
|
|
|
|
|
def record_passed(path: Path, entry: dict[str, str]) -> bool:
|
|
"""Add one green run to the record at path (created when missing); False when the same entry is there already.
|
|
A record that cannot be read, or does not have the schema, is a RunError and is left as it was."""
|
|
entries: list[Any] = []
|
|
if path.exists():
|
|
try:
|
|
entries = reader.read_passed(path) # the parser collect --gate-passed reads the record with
|
|
except reader.RunError as exc:
|
|
raise RunError(str(exc)) from exc
|
|
if entry in entries:
|
|
return False
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
path.write_text(json.dumps({"schema": PASSED_SCHEMA, "passed": [*entries, entry]}, indent=1) + "\n", encoding="utf-8")
|
|
return True
|
|
|
|
|
|
def cmd_gate(args: argparse.Namespace) -> int:
|
|
frozen = load_frozen(Path(args.set), Path(args.key), Path(args.calls), Path(args.manifest), Path(args.sources))
|
|
if frozen.errors:
|
|
raise RunError("the frozen set does not hold: " + "; ".join(frozen.errors))
|
|
thresholds = as_dict(frozen.manifest.get("thresholds"))
|
|
caught_min, accepted_min = thresholds.get("caught_min"), thresholds.get("accepted_min")
|
|
if not isinstance(caught_min, int) or not isinstance(accepted_min, int):
|
|
raise RunError("the manifest has no integer thresholds caught_min and accepted_min")
|
|
pairs, position = frozen.pairs, frozen.layout.position
|
|
reader, verdicts, fail_closed = read_verdicts(Path(args.verdicts))
|
|
models = read_models(Path(args.verdicts))
|
|
model_reason = "" if len(models) == 1 else (
|
|
"the verdicts name no model" if not models else f"the verdicts name {len(models)} models")
|
|
n = {lab: sum(1 for p in pairs if p["label"] == lab) for lab in LABELS}
|
|
caught = sum(1 for p in pairs if p["label"] == UNSUPPORTED and verdicts.get(p["id"]) == UNSUPPORTED)
|
|
accepted = sum(1 for p in pairs if p["label"] == SUPPORTED and verdicts.get(p["id"]) == SUPPORTED)
|
|
filler_accepted = sum(1 for p in pairs if p["label"] == FILLER and verdicts.get(p["id"]) == SUPPORTED)
|
|
missing = sum(1 for p in pairs if p["id"] not in verdicts)
|
|
unknown = sum(1 for i in verdicts if i not in position)
|
|
per_kind = []
|
|
for k in UNSUPPORTED_KINDS:
|
|
of_kind = [p for p in pairs if p["label"] == UNSUPPORTED and p["kind"] == k]
|
|
hit = sum(1 for p in of_kind if verdicts.get(p["id"]) == UNSUPPORTED)
|
|
per_kind.append(f"{k} {hit} of {len(of_kind)}")
|
|
per_position, short = [], []
|
|
for at in range(1, CALL_SIZE_MAX + 1):
|
|
there = [p for p in pairs if p["label"] == UNSUPPORTED and position.get(p["id"]) == at]
|
|
hit = sum(1 for p in there if verdicts.get(p["id"]) == UNSUPPORTED)
|
|
per_position.append(f"{at} {hit} of {len(there)}")
|
|
if hit < len(there) or not there:
|
|
short.append(f"position {at}: caught {hit} of {len(there)}")
|
|
green = (
|
|
not fail_closed
|
|
and not model_reason
|
|
and missing == 0
|
|
and caught >= caught_min
|
|
and caught == n[UNSUPPORTED]
|
|
and not short
|
|
and accepted >= accepted_min
|
|
)
|
|
print(
|
|
f"set: {len(pairs)} claims ({n[UNSUPPORTED]} {UNSUPPORTED}, {n[SUPPORTED]} {SUPPORTED}, {n[FILLER]} {FILLER})"
|
|
f" in {frozen.layout.stats['calls']} calls; reader: {reader or '(none)'}"
|
|
)
|
|
print(f"model: {', '.join(models) if models else '(none)'}")
|
|
if fail_closed:
|
|
print(f"fail-closed: {fail_closed}")
|
|
if model_reason:
|
|
print(f"fail-closed: {model_reason}")
|
|
print(f"missing verdicts: {missing}" + (f"; verdicts for unknown ids: {unknown}" if unknown else ""))
|
|
print(f"unsupported caught {caught} of {n[UNSUPPORTED]} · supported accepted {accepted} of {n[SUPPORTED]}")
|
|
print("caught by kind: " + ", ".join(per_kind))
|
|
print("caught by position: " + ", ".join(per_position))
|
|
print(f"filler accepted {filler_accepted} of {n[FILLER]} (reported apart, never counted in the thresholds)")
|
|
print(
|
|
f"required: caught >= {caught_min} of {n[UNSUPPORTED]} with every position 1-{CALL_SIZE_MAX} caught in full,"
|
|
f" accepted >= {accepted_min} of {n[SUPPORTED]}, no verdict missing"
|
|
)
|
|
if not green:
|
|
reasons = []
|
|
if fail_closed:
|
|
reasons.append("fail-closed")
|
|
if model_reason:
|
|
reasons.append(model_reason)
|
|
if missing:
|
|
reasons.append(f"{missing} verdicts missing")
|
|
if caught < caught_min or caught < n[UNSUPPORTED]:
|
|
reasons.append(f"caught {caught} of {n[UNSUPPORTED]}")
|
|
reasons += short
|
|
if accepted < accepted_min:
|
|
reasons.append(f"accepted {accepted} of {n[SUPPORTED]}, needs {accepted_min}")
|
|
print("not met: " + "; ".join(reasons))
|
|
if green and args.passed:
|
|
entry = {"model": models[0], "set_sha256": file_sha256(Path(args.set)), "key_sha256": file_sha256(Path(args.key)),
|
|
"verdicts_sha256": file_sha256(Path(args.verdicts)), "reader": reader}
|
|
added = record_passed(Path(args.passed), entry)
|
|
print(f"passed: {models[0]} {'recorded in' if added else 'already recorded in'} {args.passed}")
|
|
print("GREEN" if green else "RED")
|
|
return 0 if green else 1
|
|
|
|
|
|
# ---------------------------------------------------------------------------------------------- main
|
|
|
|
|
|
def build_parser() -> argparse.ArgumentParser:
|
|
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
sub = p.add_subparsers(dest="command", required=True)
|
|
c = sub.add_parser("check", help="check a labelled draft, or a built set with its key, layout and manifest")
|
|
c.add_argument("--draft", default="")
|
|
c.add_argument("--set", default="")
|
|
c.add_argument("--key", default="")
|
|
c.add_argument("--calls", default="", help="the call layout (K3 point 1)")
|
|
c.add_argument("--manifest", default="")
|
|
c.add_argument("--sources", required=True)
|
|
b = sub.add_parser("build", help="lay out a checked draft in calls with a fixed seed: set, key, layout and manifest")
|
|
b.add_argument("--draft", required=True)
|
|
b.add_argument("--sources", required=True)
|
|
b.add_argument("--seed", type=int, required=True)
|
|
b.add_argument("--lone", type=int, required=True, help="unsupported pairs that stand alone in their call")
|
|
b.add_argument("--dense", type=int, required=True, help="unsupported pairs per call for the rest")
|
|
b.add_argument("--set-out", required=True)
|
|
b.add_argument("--key-out", required=True)
|
|
b.add_argument("--calls-out", required=True)
|
|
b.add_argument("--set-label", default="", help="path of the set as written in the manifest")
|
|
b.add_argument("--key-label", default="", help="path of the key as written in the manifest")
|
|
b.add_argument("--calls-label", default="", help="path of the layout as written in the manifest")
|
|
b.add_argument("--manifest", required=True)
|
|
s = sub.add_parser("stub", help="write verdicts without a reader")
|
|
s.add_argument("--set", required=True)
|
|
which = s.add_mutually_exclusive_group(required=True)
|
|
which.add_argument("--verdict", choices=VERDICTS, help="the same verdict for every claim")
|
|
which.add_argument("--key", help="the key's verdict for every claim (the known-positive)")
|
|
s.add_argument("--calls", default="", help="the call layout, for --accept-at")
|
|
s.add_argument("--accept-at", type=int, default=None, help="answer supported for every claim at this position")
|
|
s.add_argument("--reader", default="")
|
|
s.add_argument("--out", required=True)
|
|
g = sub.add_parser("gate", help="score one reader's verdicts against the key")
|
|
g.add_argument("--set", required=True)
|
|
g.add_argument("--key", required=True)
|
|
g.add_argument("--calls", required=True, help="the call layout (K3 point 1)")
|
|
g.add_argument("--manifest", required=True)
|
|
g.add_argument("--sources", required=True)
|
|
g.add_argument("--verdicts", required=True, help=VERDICTS_SCHEMA_NOTE)
|
|
g.add_argument("--passed", default="", metavar="FILE",
|
|
help=f"on GREEN, add the run to this record of green runs ({PASSED_SCHEMA}); RED never writes it")
|
|
return p
|
|
|
|
|
|
def main(argv: list[str] | None = None) -> int:
|
|
args = build_parser().parse_args(argv)
|
|
commands = {"check": cmd_check, "build": cmd_build, "stub": cmd_stub, "gate": cmd_gate}
|
|
try:
|
|
return commands[args.command](args)
|
|
except RunError as exc:
|
|
print(f"FAIL: {exc}", file=sys.stderr)
|
|
return 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|