First public release, published as one snapshot. What changed between versions is in CHANGELOG.md. Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
546 lines
29 KiB
Python
546 lines
29 KiB
Python
"""The tool around the reader pass for PLAN.md criterion 2 ("slutning støttet"). It calls no model.
|
||
|
||
The reader judges up to ten claims per call and gives one verdict per claim (K3 in the wave-1 contracts). The model
|
||
call happens in the user's own Claude Code session: one isolated subagent per call, described "V1-<run>-read-<call>",
|
||
which reads one rendered task file and replies with one line per claim. The agent definition
|
||
(agents/okf-wiki-reader.md) is thin: it only tells the subagent to read its task file and follow it. The instructions
|
||
travel in the task (INSTRUCTIONS below), so a change between runs reaches the reader whatever the session loaded, and
|
||
the transcript shows word for word what the reader was told.
|
||
|
||
render writes one task per call of a call layout (okf-wiki/reader-calls/1, bound to the set by its sha256):
|
||
the instructions, then the claims numbered 1. to n. in layout order, each with its excerpts (file, line,
|
||
text). Nothing else reaches the reader: no id, no call name, no label, no other call. It refuses to write
|
||
when the agent's system text or the instructions share a run of eight words with any claim or excerpt of
|
||
the set, so the reader cannot carry text from the pairs it is judged on, and it never changes a task
|
||
already written.
|
||
collect reads every reader's reply from the subagent transcripts (<transcripts>/<session>/subagents/, the
|
||
or <session>/subagents/workflows/<id>/ for a workflow; the agent type okf-wiki-reader, bare or
|
||
plugin-namespaced as <plugin>:okf-wiki-reader, and the
|
||
description "V1-<run>-read-<call>"), the message it handed back or else its last text, and turns it
|
||
into the verdicts file that gates/calibration.py scores ({"reader": "<name>", "verdicts": [{"id",
|
||
"verdict"}]}). One rule, fail-closed (K3 point 3): every stripped, non-empty line must be
|
||
"<1-10>: supported" or "<1-10>: unsupported"; one line that does not fit, or a number outside 1..n,
|
||
gives no verdict for any claim of the call; a number given twice or not at all gives none for that
|
||
claim. A call answered by two transcripts gets no verdict.
|
||
It also names the models the readers ran on (PLAN § Rammer 23): the sorted, distinct `message.model`
|
||
of the assistant records in the transcripts it turns into verdicts (records without one, and
|
||
"<synthetic>", are left out), written as "models" beside "reader" and "verdicts" and printed as
|
||
"models: ...". One of two choices is required: --calibration says this run is a calibration gate run
|
||
and is not checked; --gate-passed FILE names the record of green gate runs (gates/calibration.py gate
|
||
--passed; --gate-passed alone reads the tracked gates/calibration.passed.json, which holds only models
|
||
and sha256, so a fresh clone without the set can still read), and when the transcripts name no model, or a model the record does not hold with a run on the
|
||
set and key sha256 of every calibration manifest (--manifest, once per manifest; default the tracked
|
||
gates/calibration.manifest.json, the Claude Code docs set, and gates/calibration-folder.manifest.json,
|
||
the folder set of Norwegian and English PDF/Office documents), collect writes nothing and exits 1, so a
|
||
reading on a model that never passed both gates gives no verdicts.
|
||
|
||
Exit 0 = written with every verdict; 1 = written with verdicts missing, refused, or could not run.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import hashlib
|
||
import json
|
||
import re
|
||
import sys
|
||
from dataclasses import dataclass
|
||
from pathlib import Path
|
||
from typing import Any
|
||
|
||
from okfwiki.transcript import is_agent_type, read_transcript
|
||
|
||
SUPPORTED = "supported"
|
||
UNSUPPORTED = "unsupported"
|
||
CALLS_SCHEMA = "okf-wiki/reader-calls/1"
|
||
CALL_SIZE_MAX = 10
|
||
RUN_WORDS = 8
|
||
WORD = re.compile(r"[a-z0-9]+(?:['’.-][a-z0-9]+)*")
|
||
LINE = re.compile(r"([1-9]|10)\s*:\s*(supported|unsupported)")
|
||
CALL_NAME = re.compile(r"[A-Za-z0-9][A-Za-z0-9_-]*")
|
||
HANDBACK = "SubagentHandback"
|
||
AGENT_TYPE = "okf-wiki-reader"
|
||
SYNTHETIC_MODEL = "<synthetic>"
|
||
PASSED_SCHEMA = "okf-wiki/calibration-passed/1"
|
||
CALIBRATION_MANIFEST = Path(__file__).resolve().parents[1] / "gates" / "calibration.manifest.json"
|
||
CALIBRATION_FOLDER_MANIFEST = Path(__file__).resolve().parents[1] / "gates" / "calibration-folder.manifest.json"
|
||
CALIBRATION_PASSED = Path(__file__).resolve().parents[1] / "gates" / "calibration.passed.json" # models and sha256 only
|
||
|
||
INSTRUCTIONS = """\
|
||
# How to judge the claims
|
||
|
||
You are an independent reader. This task gives you a numbered list of claims from a knowledge collection. Under each
|
||
claim stand the source excerpts it cites. For every claim, decide whether its own excerpts support it, and give one
|
||
verdict per claim.
|
||
|
||
Judge each claim on its own. Only the excerpts printed under a claim count as evidence for that claim. The other claims
|
||
in this task and their excerpts are not evidence for it, and the verdict on one claim must never lean on the verdict on
|
||
another. Any number of the claims may be unsupported: none, one, several or all of them. Do not expect a pattern, and
|
||
do not let a run of supported claims lower your guard on the next one.
|
||
|
||
You have not seen the pages the claims come from, and you must not look for them. What you know about the subject from
|
||
anywhere else does not count, even when you are sure it is true: a claim that is true in the world but not stated by
|
||
its excerpts is unsupported.
|
||
|
||
## What "supported" means
|
||
|
||
A claim is **supported** when everything it says is stated by its excerpts, or follows from them in one step that no
|
||
careful reader of the excerpts could deny. Otherwise it is **unsupported**. There is no middle verdict: a claim that is
|
||
mostly right is unsupported.
|
||
|
||
Allowed, and still supported:
|
||
- Different words for the same thing: synonyms, reordering, active for passive, a plain-language summary.
|
||
- Saying less than the excerpt: leaving out details, examples, conditions that only narrow what the excerpt says
|
||
further, or items from a list, as long as what remains is still exactly true. A claim that is less specific than its
|
||
source is fine; a claim that is more general is not.
|
||
- One necessary step: a conclusion that the excerpt forces (if it says every X has Y and that Z is an X, then Z has Y;
|
||
if it gives two numbers, their plain sum or difference). If the step needs any assumption of your own, it is not
|
||
necessary.
|
||
- Combining excerpts: when the claim joins facts from two excerpts, each part is backed by the excerpt that states it,
|
||
and the join itself adds nothing new.
|
||
|
||
## How to read one claim
|
||
|
||
1. Break the claim into its parts: every subject, every thing said about it, every number and unit, every qualifier
|
||
(all, always, only, never, some, may, by default, at most, at least, when, unless), every stated reason or order of
|
||
events, and every negation.
|
||
2. For each part, find the words in this claim's own excerpts that state it. If you cannot point to them, that part is
|
||
not supported.
|
||
3. Then run the six checks below on the claim as a whole. Each one describes a way a claim fails even though it sounds
|
||
close to its source. One failed check makes the claim unsupported.
|
||
|
||
## The six checks
|
||
|
||
1. **Broader than the source.** The claim widens what the excerpt says: "some", "can", "may", "often", "by default" or
|
||
"in this case" becomes "all", "always", "must", "every" or "in general"; a condition or exception in the excerpt is
|
||
dropped so the claim covers cases the excerpt does not; one example is turned into a rule; one product, platform,
|
||
version or setting is turned into all of them.
|
||
2. **A number that differs.** Every number, count, limit, size, duration, date, version, percentage and unit in the
|
||
claim must be the one in the excerpts, or the result of one plain calculation on them. A different figure, a rounded
|
||
figure presented as exact, a swapped unit, or "at least" where the excerpt says "at most" fails. When the claim
|
||
gives a figure the excerpts do not print (a difference, sum, total, share, ratio, count of listed items, or a span
|
||
between two dates), work it out yourself from the excerpts' own figures before you judge; if your result is not
|
||
the claim's figure, the claim fails, however close it is and however natural the sentence reads.
|
||
3. **Cause, purpose or order that the source does not give.** The claim says A causes, enables, requires, prevents or
|
||
happens before B, or that A is done in order to get B. The excerpts must state that same link in the same direction.
|
||
Two facts that the excerpt only lists side by side are not a cause. A reversed direction (B causes A, B is needed
|
||
for A) fails.
|
||
4. **Something not in the excerpts.** Any subject, feature, option, value, reason, consequence, comparison or
|
||
recommendation in the claim that the excerpts do not mention fails, however plausible it is. Look hardest at the
|
||
last clause of the claim and at small added words ("also", "instead", "automatically", "only").
|
||
5. **Negation or direction reversed.** The claim says the opposite of the excerpt on any point: can versus cannot, is
|
||
versus is not, enabled versus disabled, allowed versus blocked, recommended versus discouraged, required versus
|
||
optional, increases versus decreases, included versus excluded.
|
||
6. **A property moved between different files.** When a claim's excerpts come from different files, they may be about
|
||
different subjects. Each property in the claim must belong to the subject that its own excerpt attaches it to. A
|
||
claim that gives one file's subject a property that only the other file states for its own subject fails, even
|
||
when every word appears somewhere in the excerpts.
|
||
|
||
## Deciding
|
||
|
||
- If every part is backed and all six checks pass, the verdict is supported. Do not reject a claim for being short,
|
||
for leaving things out, for style, or because you would have written it differently.
|
||
- If any part is not backed, or any check fails, the verdict is unsupported.
|
||
- When you are unsure whether a step is necessary or only likely, it is not necessary: unsupported.
|
||
- Give every claim the same full reading, the last one as much as the first.
|
||
|
||
## Your reply
|
||
|
||
Work through every claim before you reply. Your reply is the list of verdicts and nothing else: one line per claim,
|
||
`<number>: supported` or `<number>: unsupported`, where `<number>` is the claim's number in this task (1, 2, 3 and so
|
||
on, without a leading zero) and the verdict is written in lower case. Give every number exactly once. Write nothing
|
||
else in the reply: no heading, no explanation, no reasoning, no code block, no markup, and no text before or after the
|
||
lines. Any other text in the reply makes the whole reply count as unanswered.
|
||
|
||
For a task with three claims, the reply has this form:
|
||
|
||
1: supported
|
||
2: unsupported
|
||
3: supported
|
||
"""
|
||
|
||
|
||
class RunError(Exception):
|
||
"""The command cannot run; reported as FAIL with exit 1."""
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class Excerpt:
|
||
file: str
|
||
line: int
|
||
text: str
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class Pair:
|
||
id: str
|
||
claim: str
|
||
excerpts: tuple[Excerpt, ...]
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class Call:
|
||
name: str
|
||
ids: tuple[str, ...]
|
||
|
||
|
||
def instructions(agent_text: str) -> str:
|
||
"""The body of an agent definition, without its YAML frontmatter."""
|
||
if agent_text.startswith("---\n"):
|
||
end = agent_text.find("\n---\n", 4)
|
||
if end < 0:
|
||
raise RunError("the agent definition opens a frontmatter block that never closes")
|
||
agent_text = agent_text[end + len("\n---\n") :]
|
||
return agent_text.strip()
|
||
|
||
|
||
def _read_json(path: Path, what: str) -> Any:
|
||
try:
|
||
return json.loads(path.read_text(encoding="utf-8"))
|
||
except (OSError, ValueError) as exc:
|
||
raise RunError(f"cannot read the {what} {path}: {exc}") from exc
|
||
|
||
|
||
def load_pairs(path: Path) -> list[Pair]:
|
||
"""Claims with their excerpts from a set file ({"pairs": [{id, claim, excerpts: [{file, line, text}]}]})."""
|
||
data = _read_json(path, "set")
|
||
raw = data.get("pairs") if isinstance(data, dict) else None
|
||
if not isinstance(raw, list):
|
||
raise RunError(f"{path} has no list of pairs")
|
||
pairs: list[Pair] = []
|
||
seen: set[str] = set()
|
||
for p in raw:
|
||
pid, claim, excerpts = (p.get(k) if isinstance(p, dict) else None for k in ("id", "claim", "excerpts"))
|
||
if not isinstance(pid, str) or not isinstance(claim, str) or not isinstance(excerpts, list) or not excerpts:
|
||
raise RunError(f"pair {pid!r} needs a string id, a string claim and at least one excerpt")
|
||
if pid in seen:
|
||
raise RunError(f"pair id {pid} is given twice")
|
||
seen.add(pid)
|
||
parsed = []
|
||
for e in excerpts:
|
||
if not (isinstance(e, dict) and isinstance(e.get("file"), str) and isinstance(e.get("text"), str)
|
||
and isinstance(e.get("line"), int)):
|
||
raise RunError(f"pair {pid}: every excerpt needs a file, a line and a text")
|
||
parsed.append(Excerpt(e["file"], e["line"], e["text"]))
|
||
pairs.append(Pair(pid, claim, tuple(parsed)))
|
||
return pairs
|
||
|
||
|
||
def load_calls(path: Path, set_path: Path) -> list[Call]:
|
||
"""The calls of a layout (K3 point 1), bound to the set file by its sha256. A layout may hold part of the set
|
||
(a sample); every id it names must be in the set and stand in exactly one call of at most call_size_max claims."""
|
||
data = _read_json(path, "call layout")
|
||
if not isinstance(data, dict) or data.get("schema") != CALLS_SCHEMA:
|
||
raise RunError(f"{path} is not a call layout with schema {CALLS_SCHEMA}")
|
||
size_max = data.get("call_size_max")
|
||
if not isinstance(size_max, int) or isinstance(size_max, bool) or not 1 <= size_max <= CALL_SIZE_MAX:
|
||
raise RunError(f"call_size_max {size_max!r} is not a whole number from 1 to {CALL_SIZE_MAX}")
|
||
digest = hashlib.sha256(set_path.read_bytes()).hexdigest()
|
||
if data.get("set_sha256") != digest:
|
||
raise RunError(f"the layout is bound to set {data.get('set_sha256')!r}, not to {set_path} ({digest})")
|
||
raw = data.get("calls")
|
||
if not isinstance(raw, list) or not raw:
|
||
raise RunError(f"{path} has no calls")
|
||
known = {p.id for p in load_pairs(set_path)}
|
||
calls: list[Call] = []
|
||
names: set[str] = set()
|
||
placed: set[str] = set()
|
||
for c in raw:
|
||
name, ids = (c.get("call"), c.get("ids")) if isinstance(c, dict) else (None, None)
|
||
if not isinstance(name, str) or not CALL_NAME.fullmatch(name):
|
||
raise RunError(f"call name {name!r} is not letters, digits, '-' and '_'")
|
||
if name in names:
|
||
raise RunError(f"call name {name} is given twice")
|
||
names.add(name)
|
||
if not isinstance(ids, list) or not ids or not all(isinstance(i, str) for i in ids):
|
||
raise RunError(f"call {name} has no list of claim ids")
|
||
if len(ids) > size_max:
|
||
raise RunError(f"call {name} has {len(ids)} claims, more than {size_max}")
|
||
for i in ids:
|
||
if i not in known:
|
||
raise RunError(f"call {name} names claim {i}, which is not in the set")
|
||
if i in placed:
|
||
raise RunError(f"claim {i} stands in more than one place")
|
||
placed.add(i)
|
||
calls.append(Call(name, tuple(ids)))
|
||
return calls
|
||
|
||
|
||
def render_call(call: Call, pairs: dict[str, Pair]) -> str:
|
||
"""One task: the instructions, then the claims numbered 1. to n., each with its excerpts."""
|
||
n = len(call.ids)
|
||
parts = [INSTRUCTIONS.strip(), "", "---", "", "## The claims", "",
|
||
f"This task has {n} claim{'s' if n != 1 else ''}, numbered 1 to {n}."]
|
||
for number, cid in enumerate(call.ids, 1):
|
||
p = pairs[cid]
|
||
parts += ["", f"{number}. <claim>{p.claim}</claim>"]
|
||
for e in p.excerpts:
|
||
parts += [f'<excerpt file="{e.file}" line="{e.line}">', e.text, "</excerpt>"]
|
||
return "\n".join(parts) + "\n"
|
||
|
||
|
||
def _runs(text: str) -> set[tuple[str, ...]]:
|
||
words = WORD.findall(text.lower())
|
||
return {tuple(words[i : i + RUN_WORDS]) for i in range(len(words) - RUN_WORDS + 1)}
|
||
|
||
|
||
def shared_runs(text: str, pairs: list[Pair]) -> list[str]:
|
||
"""Pair ids whose claim or excerpts share a run of RUN_WORDS words with an instruction text."""
|
||
mine = _runs(text)
|
||
return [p.id for p in pairs if mine & _runs(" ".join([p.claim, *(e.text for e in p.excerpts)]))]
|
||
|
||
|
||
def parse_reply(reply: str, n: int) -> dict[int, str]:
|
||
"""K3 point 3: the verdict per claim number of one call of n claims; a claim with no verdict is left out."""
|
||
lines = [line.strip() for line in reply.splitlines() if line.strip()]
|
||
found: dict[int, list[str]] = {}
|
||
for line in lines:
|
||
m = LINE.fullmatch(line)
|
||
if m is None:
|
||
return {}
|
||
number = int(m.group(1))
|
||
if number > n:
|
||
return {}
|
||
found.setdefault(number, []).append(m.group(2))
|
||
return {k: v[0] for k, v in sorted(found.items()) if len(v) == 1}
|
||
|
||
|
||
def final_reply(path: Path) -> str:
|
||
"""The reader's reply in its transcript: the message of its last SubagentHandback call (how Claude Code 2.1.295
|
||
delivers a subagent's final report), or else the text of its last assistant message (its text blocks joined in
|
||
order). Thinking is never part of the reply."""
|
||
records = [r for r in read_transcript(path) if r.get("type") == "assistant" and isinstance(r.get("message"), dict)]
|
||
handed: list[Any] = []
|
||
for r in records:
|
||
content = r["message"].get("content")
|
||
for block in content if isinstance(content, list) else []:
|
||
if isinstance(block, dict) and block.get("type") == "tool_use" and block.get("name") == HANDBACK:
|
||
given = block.get("input")
|
||
handed.append(given.get("message") if isinstance(given, dict) else None)
|
||
if handed:
|
||
return handed[-1] if isinstance(handed[-1], str) else ""
|
||
if not records:
|
||
return ""
|
||
last = records[-1]["message"].get("id")
|
||
texts: list[str] = []
|
||
for r in records:
|
||
if r["message"].get("id") != last:
|
||
continue
|
||
content = r["message"].get("content")
|
||
for block in content if isinstance(content, list) else []:
|
||
if isinstance(block, dict) and block.get("type") == "text" and isinstance(block.get("text"), str):
|
||
texts.append(block["text"])
|
||
return "\n".join(texts)
|
||
|
||
|
||
def transcript_models(path: Path) -> set[str]:
|
||
"""The models named by the assistant records of one transcript (message.model, as Claude Code writes it); records
|
||
without one, and the "<synthetic>" records Claude Code writes itself, name none."""
|
||
found: set[str] = set()
|
||
for r in read_transcript(path):
|
||
message = r.get("message") if r.get("type") == "assistant" else None
|
||
model = message.get("model") if isinstance(message, dict) else None
|
||
if isinstance(model, str) and model and model != SYNTHETIC_MODEL:
|
||
found.add(model)
|
||
return found
|
||
|
||
|
||
PASSED_FIELDS = ("model", "set_sha256", "key_sha256")
|
||
|
||
|
||
def read_passed(path: Path) -> list[dict[str, Any]]:
|
||
"""The entries of a record of green calibration runs (okf-wiki/calibration-passed/1), the one parser both
|
||
gates/calibration.py gate --passed (which adds to it) and collect --gate-passed use. A record that cannot be read,
|
||
does not have that form, or has an entry without a model, set_sha256 or key_sha256 string, is a RunError."""
|
||
data = _read_json(path, "record of green calibration runs")
|
||
entries = data.get("passed") if isinstance(data, dict) and data.get("schema") == PASSED_SCHEMA else None
|
||
if not isinstance(entries, list):
|
||
raise RunError(f"{path} is not a record of green calibration runs with schema {PASSED_SCHEMA}")
|
||
for e in entries:
|
||
for name in PASSED_FIELDS:
|
||
value = e.get(name) if isinstance(e, dict) else None
|
||
if not isinstance(value, str) or not value:
|
||
raise RunError(f"{path}: an entry of the record of green calibration runs names no {name}")
|
||
return entries
|
||
|
||
|
||
def manifest_shas(path: Path) -> tuple[str, str]:
|
||
"""(set sha256, key sha256) of a calibration manifest (gates/calibration.manifest.json)."""
|
||
data = _read_json(path, "calibration manifest")
|
||
shas = [data.get(part, {}).get("sha256") if isinstance(data, dict) and isinstance(data.get(part), dict) else None
|
||
for part in ("set", "key")]
|
||
if not all(isinstance(s, str) and s for s in shas):
|
||
raise RunError(f"{path} is not a calibration manifest with a set and a key sha256")
|
||
return str(shas[0]), str(shas[1])
|
||
|
||
|
||
def passed_models(path: Path, manifest: Path = CALIBRATION_MANIFEST) -> set[str]:
|
||
"""The models with a green run on the set and key the calibration manifest names: an entry whose set_sha256 or
|
||
key_sha256 differs (a toy set, an older set, a set rebuilt later) clears no model."""
|
||
set_sha, key_sha = manifest_shas(manifest)
|
||
return {e["model"] for e in read_passed(path) if e["set_sha256"] == set_sha and e["key_sha256"] == key_sha}
|
||
|
||
|
||
def manifests_of(args: argparse.Namespace) -> list[Path]:
|
||
"""The calibration manifests a green run must be on: those given with --manifest, else both tracked sets (the
|
||
Claude Code docs set and the folder set of Norwegian and English PDF/Office documents, motor B16)."""
|
||
given: list[str] | None = args.manifest
|
||
return [Path(m) for m in given] if given else [CALIBRATION_MANIFEST, CALIBRATION_FOLDER_MANIFEST]
|
||
|
||
|
||
def reader_transcripts(transcripts: Path, run: str) -> dict[str, list[Path]]:
|
||
"""Transcripts per call name of the reader agents of one run, found by their description and agent type, whether
|
||
started from the session (<session>/subagents/) or from a workflow (<session>/subagents/workflows/<id>/)."""
|
||
prefix = re.compile(rf"^V1-{re.escape(run)}-read-(\S+)$")
|
||
found: dict[str, list[Path]] = {}
|
||
metas = [*transcripts.glob("*/subagents/agent-*.meta.json"),
|
||
*transcripts.glob("*/subagents/workflows/*/agent-*.meta.json")]
|
||
for meta_path in sorted(metas):
|
||
try:
|
||
meta = json.loads(meta_path.read_text(encoding="utf-8"))
|
||
except (OSError, ValueError):
|
||
continue
|
||
if not isinstance(meta, dict) or not is_agent_type(meta.get("agentType"), AGENT_TYPE):
|
||
continue
|
||
m = prefix.match(str(meta.get("description", "")))
|
||
if m is None:
|
||
continue
|
||
path = meta_path.with_name(meta_path.name.removesuffix(".meta.json") + ".jsonl")
|
||
found.setdefault(m.group(1), []).append(path)
|
||
return found
|
||
|
||
|
||
@dataclass
|
||
class Collected:
|
||
doc: dict[str, Any]
|
||
claims: int
|
||
answered: int = 0
|
||
broke_rule: int = 0
|
||
twice: int = 0
|
||
unanswered: int = 0
|
||
stray: int = 0
|
||
|
||
|
||
def collect(name: str, calls: list[Call], found: dict[str, list[Path]]) -> Collected:
|
||
verdicts: list[dict[str, str]] = []
|
||
models: set[str] = set()
|
||
out = Collected({"reader": name, "models": [], "verdicts": verdicts}, sum(len(c.ids) for c in calls))
|
||
names = {c.name for c in calls}
|
||
out.stray = sum(len(paths) for call, paths in found.items() if call not in names)
|
||
for call in calls:
|
||
paths = found.get(call.name, [])
|
||
if not paths:
|
||
out.unanswered += 1
|
||
continue
|
||
if len(paths) > 1:
|
||
out.twice += 1
|
||
continue
|
||
out.answered += 1
|
||
if paths[0].is_file():
|
||
models |= transcript_models(paths[0])
|
||
given = parse_reply(final_reply(paths[0]) if paths[0].is_file() else "", len(call.ids))
|
||
if not given:
|
||
out.broke_rule += 1
|
||
for number, cid in enumerate(call.ids, 1):
|
||
if number in given:
|
||
verdicts.append({"id": cid, "verdict": given[number]})
|
||
out.doc["models"] = sorted(models)
|
||
return out
|
||
|
||
|
||
def cmd_render(args: argparse.Namespace) -> int:
|
||
set_path = Path(args.set)
|
||
pairs = load_pairs(set_path)
|
||
calls = load_calls(Path(args.calls), set_path)
|
||
try:
|
||
system = instructions(Path(args.agent).read_text(encoding="utf-8"))
|
||
except OSError as exc:
|
||
raise RunError(f"cannot read the agent definition {args.agent}: {exc}") from exc
|
||
for what, text in (("the agent's system text", system), ("the instructions in the task", INSTRUCTIONS)):
|
||
leaks = shared_runs(text, pairs)
|
||
if leaks:
|
||
raise RunError(f"{what} and the set share a run of {RUN_WORDS} words in {len(leaks)} pair(s); "
|
||
"nothing written")
|
||
by_id = {p.id: p for p in pairs}
|
||
out = Path(args.out_dir)
|
||
tasks = {out / f"{c.name}.md": render_call(c, by_id) for c in calls}
|
||
changed = sorted(p.name for p, t in tasks.items() if p.exists() and p.read_text(encoding="utf-8") != t)
|
||
if changed:
|
||
raise RunError(f"{len(changed)} task(s) already written with other text, first {changed[0]}; nothing written")
|
||
out.mkdir(parents=True, exist_ok=True)
|
||
for path, task in tasks.items():
|
||
path.write_text(task, encoding="utf-8")
|
||
print(f"render: {len(tasks)} tasks for {sum(len(c.ids) for c in calls)} claims written to {out}")
|
||
return 0
|
||
|
||
|
||
def cmd_collect(args: argparse.Namespace) -> int:
|
||
set_path = Path(args.set)
|
||
calls = load_calls(Path(args.calls), set_path)
|
||
transcripts = Path(args.transcripts)
|
||
if not transcripts.is_dir():
|
||
raise RunError(f"the transcripts directory {transcripts} does not exist")
|
||
got = collect(args.reader, calls, reader_transcripts(transcripts, args.run))
|
||
models: list[str] = got.doc["models"]
|
||
if args.gate_passed is not None:
|
||
for manifest in manifests_of(args):
|
||
passed = passed_models(Path(args.gate_passed), manifest)
|
||
lacking = [m for m in models if m not in passed] if models else ["(none)"]
|
||
if lacking:
|
||
print(f"collect cannot run: model(s) {', '.join(lacking)} have no green calibration run in "
|
||
f"{args.gate_passed} on the set and key of {manifest}; run the calibration gate on them first",
|
||
file=sys.stderr)
|
||
return 1
|
||
Path(args.out).write_text(json.dumps(got.doc, indent=1) + "\n", encoding="utf-8")
|
||
written = len(got.doc["verdicts"])
|
||
stray = f"; {got.stray} transcript{'s' if got.stray != 1 else ''} for a call not in the layout" if got.stray else ""
|
||
print(f"collect {args.reader}: {written} of {got.claims} verdicts written to {args.out}")
|
||
print(f"calls: {got.answered} of {len(calls)} answered; {got.broke_rule} broke the reply rule; "
|
||
f"{got.twice} answered twice; {got.unanswered} not answered{stray}")
|
||
print(f"missing: {got.claims - written}")
|
||
print(f"models: {', '.join(models) if models else '(none)'}")
|
||
return 0 if written == got.claims else 1
|
||
|
||
|
||
def build_parser() -> argparse.ArgumentParser:
|
||
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||
sub = p.add_subparsers(dest="command", required=True)
|
||
r = sub.add_parser("render", help="write one reader task per call")
|
||
r.add_argument("--set", required=True)
|
||
r.add_argument("--calls", required=True, help="the call layout (okf-wiki/reader-calls/1)")
|
||
r.add_argument("--agent", required=True, help="the reader's agent definition (agents/okf-wiki-reader.md)")
|
||
r.add_argument("--out-dir", required=True)
|
||
c = sub.add_parser("collect", help="turn the readers' replies in the transcripts into a verdicts file")
|
||
c.add_argument("--set", required=True)
|
||
c.add_argument("--calls", required=True)
|
||
c.add_argument("--transcripts", required=True,
|
||
help="~/.claude/projects/<slug>/ of the session that ran the readers")
|
||
c.add_argument("--run", required=True, help="the run name in the descriptions V1-<run>-read-<call>")
|
||
c.add_argument("--reader", required=True)
|
||
c.add_argument("--out", required=True)
|
||
check = c.add_mutually_exclusive_group(required=True)
|
||
check.add_argument("--calibration", action="store_true",
|
||
help="this run is a calibration gate run; the readers' model is not checked")
|
||
check.add_argument("--gate-passed", metavar="FILE", default=None, nargs="?", const=str(CALIBRATION_PASSED),
|
||
help="the record of green calibration runs (gates/calibration.py gate --passed; without FILE "
|
||
"the tracked gates/calibration.passed.json); collect writes nothing unless every model "
|
||
"the readers ran on is in it")
|
||
c.add_argument("--manifest", action="append", default=None,
|
||
help="a calibration manifest whose set and key a green run must be on, once per manifest; every "
|
||
"one must hold (default: the tracked gates/calibration.manifest.json and "
|
||
"gates/calibration-folder.manifest.json)")
|
||
return p
|
||
|
||
|
||
def main(argv: list[str] | None = None) -> int:
|
||
args = build_parser().parse_args(argv)
|
||
commands = {"render": cmd_render, "collect": cmd_collect}
|
||
try:
|
||
return commands[args.command](args)
|
||
except (RunError, OSError) as exc:
|
||
print(f"FAIL: {exc}", file=sys.stderr)
|
||
return 1
|
||
|
||
|
||
if __name__ == "__main__":
|
||
sys.exit(main())
|