okf-wiki 1.1.1
First public release, published as one snapshot. What changed between versions is in CHANGELOG.md. Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
This commit is contained in:
commit
0fd86a23a3
227 changed files with 96077 additions and 0 deletions
546
okfwiki/reader.py
Normal file
546
okfwiki/reader.py
Normal file
|
|
@ -0,0 +1,546 @@
|
|||
"""The tool around the reader pass for PLAN.md criterion 2 ("slutning støttet"). It calls no model.
|
||||
|
||||
The reader judges up to ten claims per call and gives one verdict per claim (K3 in the wave-1 contracts). The model
|
||||
call happens in the user's own Claude Code session: one isolated subagent per call, described "V1-<run>-read-<call>",
|
||||
which reads one rendered task file and replies with one line per claim. The agent definition
|
||||
(agents/okf-wiki-reader.md) is thin: it only tells the subagent to read its task file and follow it. The instructions
|
||||
travel in the task (INSTRUCTIONS below), so a change between runs reaches the reader whatever the session loaded, and
|
||||
the transcript shows word for word what the reader was told.
|
||||
|
||||
render writes one task per call of a call layout (okf-wiki/reader-calls/1, bound to the set by its sha256):
|
||||
the instructions, then the claims numbered 1. to n. in layout order, each with its excerpts (file, line,
|
||||
text). Nothing else reaches the reader: no id, no call name, no label, no other call. It refuses to write
|
||||
when the agent's system text or the instructions share a run of eight words with any claim or excerpt of
|
||||
the set, so the reader cannot carry text from the pairs it is judged on, and it never changes a task
|
||||
already written.
|
||||
collect reads every reader's reply from the subagent transcripts (<transcripts>/<session>/subagents/, the
|
||||
or <session>/subagents/workflows/<id>/ for a workflow; the agent type okf-wiki-reader, bare or
|
||||
plugin-namespaced as <plugin>:okf-wiki-reader, and the
|
||||
description "V1-<run>-read-<call>"), the message it handed back or else its last text, and turns it
|
||||
into the verdicts file that gates/calibration.py scores ({"reader": "<name>", "verdicts": [{"id",
|
||||
"verdict"}]}). One rule, fail-closed (K3 point 3): every stripped, non-empty line must be
|
||||
"<1-10>: supported" or "<1-10>: unsupported"; one line that does not fit, or a number outside 1..n,
|
||||
gives no verdict for any claim of the call; a number given twice or not at all gives none for that
|
||||
claim. A call answered by two transcripts gets no verdict.
|
||||
It also names the models the readers ran on (PLAN § Rammer 23): the sorted, distinct `message.model`
|
||||
of the assistant records in the transcripts it turns into verdicts (records without one, and
|
||||
"<synthetic>", are left out), written as "models" beside "reader" and "verdicts" and printed as
|
||||
"models: ...". One of two choices is required: --calibration says this run is a calibration gate run
|
||||
and is not checked; --gate-passed FILE names the record of green gate runs (gates/calibration.py gate
|
||||
--passed; --gate-passed alone reads the tracked gates/calibration.passed.json, which holds only models
|
||||
and sha256, so a fresh clone without the set can still read), and when the transcripts name no model, or a model the record does not hold with a run on the
|
||||
set and key sha256 of every calibration manifest (--manifest, once per manifest; default the tracked
|
||||
gates/calibration.manifest.json, the Claude Code docs set, and gates/calibration-folder.manifest.json,
|
||||
the folder set of Norwegian and English PDF/Office documents), collect writes nothing and exits 1, so a
|
||||
reading on a model that never passed both gates gives no verdicts.
|
||||
|
||||
Exit 0 = written with every verdict; 1 = written with verdicts missing, refused, or could not run.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from okfwiki.transcript import is_agent_type, read_transcript
|
||||
|
||||
SUPPORTED = "supported"
|
||||
UNSUPPORTED = "unsupported"
|
||||
CALLS_SCHEMA = "okf-wiki/reader-calls/1"
|
||||
CALL_SIZE_MAX = 10
|
||||
RUN_WORDS = 8
|
||||
WORD = re.compile(r"[a-z0-9]+(?:['’.-][a-z0-9]+)*")
|
||||
LINE = re.compile(r"([1-9]|10)\s*:\s*(supported|unsupported)")
|
||||
CALL_NAME = re.compile(r"[A-Za-z0-9][A-Za-z0-9_-]*")
|
||||
HANDBACK = "SubagentHandback"
|
||||
AGENT_TYPE = "okf-wiki-reader"
|
||||
SYNTHETIC_MODEL = "<synthetic>"
|
||||
PASSED_SCHEMA = "okf-wiki/calibration-passed/1"
|
||||
CALIBRATION_MANIFEST = Path(__file__).resolve().parents[1] / "gates" / "calibration.manifest.json"
|
||||
CALIBRATION_FOLDER_MANIFEST = Path(__file__).resolve().parents[1] / "gates" / "calibration-folder.manifest.json"
|
||||
CALIBRATION_PASSED = Path(__file__).resolve().parents[1] / "gates" / "calibration.passed.json" # models and sha256 only
|
||||
|
||||
INSTRUCTIONS = """\
|
||||
# How to judge the claims
|
||||
|
||||
You are an independent reader. This task gives you a numbered list of claims from a knowledge collection. Under each
|
||||
claim stand the source excerpts it cites. For every claim, decide whether its own excerpts support it, and give one
|
||||
verdict per claim.
|
||||
|
||||
Judge each claim on its own. Only the excerpts printed under a claim count as evidence for that claim. The other claims
|
||||
in this task and their excerpts are not evidence for it, and the verdict on one claim must never lean on the verdict on
|
||||
another. Any number of the claims may be unsupported: none, one, several or all of them. Do not expect a pattern, and
|
||||
do not let a run of supported claims lower your guard on the next one.
|
||||
|
||||
You have not seen the pages the claims come from, and you must not look for them. What you know about the subject from
|
||||
anywhere else does not count, even when you are sure it is true: a claim that is true in the world but not stated by
|
||||
its excerpts is unsupported.
|
||||
|
||||
## What "supported" means
|
||||
|
||||
A claim is **supported** when everything it says is stated by its excerpts, or follows from them in one step that no
|
||||
careful reader of the excerpts could deny. Otherwise it is **unsupported**. There is no middle verdict: a claim that is
|
||||
mostly right is unsupported.
|
||||
|
||||
Allowed, and still supported:
|
||||
- Different words for the same thing: synonyms, reordering, active for passive, a plain-language summary.
|
||||
- Saying less than the excerpt: leaving out details, examples, conditions that only narrow what the excerpt says
|
||||
further, or items from a list, as long as what remains is still exactly true. A claim that is less specific than its
|
||||
source is fine; a claim that is more general is not.
|
||||
- One necessary step: a conclusion that the excerpt forces (if it says every X has Y and that Z is an X, then Z has Y;
|
||||
if it gives two numbers, their plain sum or difference). If the step needs any assumption of your own, it is not
|
||||
necessary.
|
||||
- Combining excerpts: when the claim joins facts from two excerpts, each part is backed by the excerpt that states it,
|
||||
and the join itself adds nothing new.
|
||||
|
||||
## How to read one claim
|
||||
|
||||
1. Break the claim into its parts: every subject, every thing said about it, every number and unit, every qualifier
|
||||
(all, always, only, never, some, may, by default, at most, at least, when, unless), every stated reason or order of
|
||||
events, and every negation.
|
||||
2. For each part, find the words in this claim's own excerpts that state it. If you cannot point to them, that part is
|
||||
not supported.
|
||||
3. Then run the six checks below on the claim as a whole. Each one describes a way a claim fails even though it sounds
|
||||
close to its source. One failed check makes the claim unsupported.
|
||||
|
||||
## The six checks
|
||||
|
||||
1. **Broader than the source.** The claim widens what the excerpt says: "some", "can", "may", "often", "by default" or
|
||||
"in this case" becomes "all", "always", "must", "every" or "in general"; a condition or exception in the excerpt is
|
||||
dropped so the claim covers cases the excerpt does not; one example is turned into a rule; one product, platform,
|
||||
version or setting is turned into all of them.
|
||||
2. **A number that differs.** Every number, count, limit, size, duration, date, version, percentage and unit in the
|
||||
claim must be the one in the excerpts, or the result of one plain calculation on them. A different figure, a rounded
|
||||
figure presented as exact, a swapped unit, or "at least" where the excerpt says "at most" fails. When the claim
|
||||
gives a figure the excerpts do not print (a difference, sum, total, share, ratio, count of listed items, or a span
|
||||
between two dates), work it out yourself from the excerpts' own figures before you judge; if your result is not
|
||||
the claim's figure, the claim fails, however close it is and however natural the sentence reads.
|
||||
3. **Cause, purpose or order that the source does not give.** The claim says A causes, enables, requires, prevents or
|
||||
happens before B, or that A is done in order to get B. The excerpts must state that same link in the same direction.
|
||||
Two facts that the excerpt only lists side by side are not a cause. A reversed direction (B causes A, B is needed
|
||||
for A) fails.
|
||||
4. **Something not in the excerpts.** Any subject, feature, option, value, reason, consequence, comparison or
|
||||
recommendation in the claim that the excerpts do not mention fails, however plausible it is. Look hardest at the
|
||||
last clause of the claim and at small added words ("also", "instead", "automatically", "only").
|
||||
5. **Negation or direction reversed.** The claim says the opposite of the excerpt on any point: can versus cannot, is
|
||||
versus is not, enabled versus disabled, allowed versus blocked, recommended versus discouraged, required versus
|
||||
optional, increases versus decreases, included versus excluded.
|
||||
6. **A property moved between different files.** When a claim's excerpts come from different files, they may be about
|
||||
different subjects. Each property in the claim must belong to the subject that its own excerpt attaches it to. A
|
||||
claim that gives one file's subject a property that only the other file states for its own subject fails, even
|
||||
when every word appears somewhere in the excerpts.
|
||||
|
||||
## Deciding
|
||||
|
||||
- If every part is backed and all six checks pass, the verdict is supported. Do not reject a claim for being short,
|
||||
for leaving things out, for style, or because you would have written it differently.
|
||||
- If any part is not backed, or any check fails, the verdict is unsupported.
|
||||
- When you are unsure whether a step is necessary or only likely, it is not necessary: unsupported.
|
||||
- Give every claim the same full reading, the last one as much as the first.
|
||||
|
||||
## Your reply
|
||||
|
||||
Work through every claim before you reply. Your reply is the list of verdicts and nothing else: one line per claim,
|
||||
`<number>: supported` or `<number>: unsupported`, where `<number>` is the claim's number in this task (1, 2, 3 and so
|
||||
on, without a leading zero) and the verdict is written in lower case. Give every number exactly once. Write nothing
|
||||
else in the reply: no heading, no explanation, no reasoning, no code block, no markup, and no text before or after the
|
||||
lines. Any other text in the reply makes the whole reply count as unanswered.
|
||||
|
||||
For a task with three claims, the reply has this form:
|
||||
|
||||
1: supported
|
||||
2: unsupported
|
||||
3: supported
|
||||
"""
|
||||
|
||||
|
||||
class RunError(Exception):
|
||||
"""The command cannot run; reported as FAIL with exit 1."""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Excerpt:
|
||||
file: str
|
||||
line: int
|
||||
text: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Pair:
|
||||
id: str
|
||||
claim: str
|
||||
excerpts: tuple[Excerpt, ...]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Call:
|
||||
name: str
|
||||
ids: tuple[str, ...]
|
||||
|
||||
|
||||
def instructions(agent_text: str) -> str:
|
||||
"""The body of an agent definition, without its YAML frontmatter."""
|
||||
if agent_text.startswith("---\n"):
|
||||
end = agent_text.find("\n---\n", 4)
|
||||
if end < 0:
|
||||
raise RunError("the agent definition opens a frontmatter block that never closes")
|
||||
agent_text = agent_text[end + len("\n---\n") :]
|
||||
return agent_text.strip()
|
||||
|
||||
|
||||
def _read_json(path: Path, what: str) -> Any:
|
||||
try:
|
||||
return json.loads(path.read_text(encoding="utf-8"))
|
||||
except (OSError, ValueError) as exc:
|
||||
raise RunError(f"cannot read the {what} {path}: {exc}") from exc
|
||||
|
||||
|
||||
def load_pairs(path: Path) -> list[Pair]:
|
||||
"""Claims with their excerpts from a set file ({"pairs": [{id, claim, excerpts: [{file, line, text}]}]})."""
|
||||
data = _read_json(path, "set")
|
||||
raw = data.get("pairs") if isinstance(data, dict) else None
|
||||
if not isinstance(raw, list):
|
||||
raise RunError(f"{path} has no list of pairs")
|
||||
pairs: list[Pair] = []
|
||||
seen: set[str] = set()
|
||||
for p in raw:
|
||||
pid, claim, excerpts = (p.get(k) if isinstance(p, dict) else None for k in ("id", "claim", "excerpts"))
|
||||
if not isinstance(pid, str) or not isinstance(claim, str) or not isinstance(excerpts, list) or not excerpts:
|
||||
raise RunError(f"pair {pid!r} needs a string id, a string claim and at least one excerpt")
|
||||
if pid in seen:
|
||||
raise RunError(f"pair id {pid} is given twice")
|
||||
seen.add(pid)
|
||||
parsed = []
|
||||
for e in excerpts:
|
||||
if not (isinstance(e, dict) and isinstance(e.get("file"), str) and isinstance(e.get("text"), str)
|
||||
and isinstance(e.get("line"), int)):
|
||||
raise RunError(f"pair {pid}: every excerpt needs a file, a line and a text")
|
||||
parsed.append(Excerpt(e["file"], e["line"], e["text"]))
|
||||
pairs.append(Pair(pid, claim, tuple(parsed)))
|
||||
return pairs
|
||||
|
||||
|
||||
def load_calls(path: Path, set_path: Path) -> list[Call]:
|
||||
"""The calls of a layout (K3 point 1), bound to the set file by its sha256. A layout may hold part of the set
|
||||
(a sample); every id it names must be in the set and stand in exactly one call of at most call_size_max claims."""
|
||||
data = _read_json(path, "call layout")
|
||||
if not isinstance(data, dict) or data.get("schema") != CALLS_SCHEMA:
|
||||
raise RunError(f"{path} is not a call layout with schema {CALLS_SCHEMA}")
|
||||
size_max = data.get("call_size_max")
|
||||
if not isinstance(size_max, int) or isinstance(size_max, bool) or not 1 <= size_max <= CALL_SIZE_MAX:
|
||||
raise RunError(f"call_size_max {size_max!r} is not a whole number from 1 to {CALL_SIZE_MAX}")
|
||||
digest = hashlib.sha256(set_path.read_bytes()).hexdigest()
|
||||
if data.get("set_sha256") != digest:
|
||||
raise RunError(f"the layout is bound to set {data.get('set_sha256')!r}, not to {set_path} ({digest})")
|
||||
raw = data.get("calls")
|
||||
if not isinstance(raw, list) or not raw:
|
||||
raise RunError(f"{path} has no calls")
|
||||
known = {p.id for p in load_pairs(set_path)}
|
||||
calls: list[Call] = []
|
||||
names: set[str] = set()
|
||||
placed: set[str] = set()
|
||||
for c in raw:
|
||||
name, ids = (c.get("call"), c.get("ids")) if isinstance(c, dict) else (None, None)
|
||||
if not isinstance(name, str) or not CALL_NAME.fullmatch(name):
|
||||
raise RunError(f"call name {name!r} is not letters, digits, '-' and '_'")
|
||||
if name in names:
|
||||
raise RunError(f"call name {name} is given twice")
|
||||
names.add(name)
|
||||
if not isinstance(ids, list) or not ids or not all(isinstance(i, str) for i in ids):
|
||||
raise RunError(f"call {name} has no list of claim ids")
|
||||
if len(ids) > size_max:
|
||||
raise RunError(f"call {name} has {len(ids)} claims, more than {size_max}")
|
||||
for i in ids:
|
||||
if i not in known:
|
||||
raise RunError(f"call {name} names claim {i}, which is not in the set")
|
||||
if i in placed:
|
||||
raise RunError(f"claim {i} stands in more than one place")
|
||||
placed.add(i)
|
||||
calls.append(Call(name, tuple(ids)))
|
||||
return calls
|
||||
|
||||
|
||||
def render_call(call: Call, pairs: dict[str, Pair]) -> str:
|
||||
"""One task: the instructions, then the claims numbered 1. to n., each with its excerpts."""
|
||||
n = len(call.ids)
|
||||
parts = [INSTRUCTIONS.strip(), "", "---", "", "## The claims", "",
|
||||
f"This task has {n} claim{'s' if n != 1 else ''}, numbered 1 to {n}."]
|
||||
for number, cid in enumerate(call.ids, 1):
|
||||
p = pairs[cid]
|
||||
parts += ["", f"{number}. <claim>{p.claim}</claim>"]
|
||||
for e in p.excerpts:
|
||||
parts += [f'<excerpt file="{e.file}" line="{e.line}">', e.text, "</excerpt>"]
|
||||
return "\n".join(parts) + "\n"
|
||||
|
||||
|
||||
def _runs(text: str) -> set[tuple[str, ...]]:
|
||||
words = WORD.findall(text.lower())
|
||||
return {tuple(words[i : i + RUN_WORDS]) for i in range(len(words) - RUN_WORDS + 1)}
|
||||
|
||||
|
||||
def shared_runs(text: str, pairs: list[Pair]) -> list[str]:
|
||||
"""Pair ids whose claim or excerpts share a run of RUN_WORDS words with an instruction text."""
|
||||
mine = _runs(text)
|
||||
return [p.id for p in pairs if mine & _runs(" ".join([p.claim, *(e.text for e in p.excerpts)]))]
|
||||
|
||||
|
||||
def parse_reply(reply: str, n: int) -> dict[int, str]:
|
||||
"""K3 point 3: the verdict per claim number of one call of n claims; a claim with no verdict is left out."""
|
||||
lines = [line.strip() for line in reply.splitlines() if line.strip()]
|
||||
found: dict[int, list[str]] = {}
|
||||
for line in lines:
|
||||
m = LINE.fullmatch(line)
|
||||
if m is None:
|
||||
return {}
|
||||
number = int(m.group(1))
|
||||
if number > n:
|
||||
return {}
|
||||
found.setdefault(number, []).append(m.group(2))
|
||||
return {k: v[0] for k, v in sorted(found.items()) if len(v) == 1}
|
||||
|
||||
|
||||
def final_reply(path: Path) -> str:
|
||||
"""The reader's reply in its transcript: the message of its last SubagentHandback call (how Claude Code 2.1.295
|
||||
delivers a subagent's final report), or else the text of its last assistant message (its text blocks joined in
|
||||
order). Thinking is never part of the reply."""
|
||||
records = [r for r in read_transcript(path) if r.get("type") == "assistant" and isinstance(r.get("message"), dict)]
|
||||
handed: list[Any] = []
|
||||
for r in records:
|
||||
content = r["message"].get("content")
|
||||
for block in content if isinstance(content, list) else []:
|
||||
if isinstance(block, dict) and block.get("type") == "tool_use" and block.get("name") == HANDBACK:
|
||||
given = block.get("input")
|
||||
handed.append(given.get("message") if isinstance(given, dict) else None)
|
||||
if handed:
|
||||
return handed[-1] if isinstance(handed[-1], str) else ""
|
||||
if not records:
|
||||
return ""
|
||||
last = records[-1]["message"].get("id")
|
||||
texts: list[str] = []
|
||||
for r in records:
|
||||
if r["message"].get("id") != last:
|
||||
continue
|
||||
content = r["message"].get("content")
|
||||
for block in content if isinstance(content, list) else []:
|
||||
if isinstance(block, dict) and block.get("type") == "text" and isinstance(block.get("text"), str):
|
||||
texts.append(block["text"])
|
||||
return "\n".join(texts)
|
||||
|
||||
|
||||
def transcript_models(path: Path) -> set[str]:
|
||||
"""The models named by the assistant records of one transcript (message.model, as Claude Code writes it); records
|
||||
without one, and the "<synthetic>" records Claude Code writes itself, name none."""
|
||||
found: set[str] = set()
|
||||
for r in read_transcript(path):
|
||||
message = r.get("message") if r.get("type") == "assistant" else None
|
||||
model = message.get("model") if isinstance(message, dict) else None
|
||||
if isinstance(model, str) and model and model != SYNTHETIC_MODEL:
|
||||
found.add(model)
|
||||
return found
|
||||
|
||||
|
||||
PASSED_FIELDS = ("model", "set_sha256", "key_sha256")
|
||||
|
||||
|
||||
def read_passed(path: Path) -> list[dict[str, Any]]:
|
||||
"""The entries of a record of green calibration runs (okf-wiki/calibration-passed/1), the one parser both
|
||||
gates/calibration.py gate --passed (which adds to it) and collect --gate-passed use. A record that cannot be read,
|
||||
does not have that form, or has an entry without a model, set_sha256 or key_sha256 string, is a RunError."""
|
||||
data = _read_json(path, "record of green calibration runs")
|
||||
entries = data.get("passed") if isinstance(data, dict) and data.get("schema") == PASSED_SCHEMA else None
|
||||
if not isinstance(entries, list):
|
||||
raise RunError(f"{path} is not a record of green calibration runs with schema {PASSED_SCHEMA}")
|
||||
for e in entries:
|
||||
for name in PASSED_FIELDS:
|
||||
value = e.get(name) if isinstance(e, dict) else None
|
||||
if not isinstance(value, str) or not value:
|
||||
raise RunError(f"{path}: an entry of the record of green calibration runs names no {name}")
|
||||
return entries
|
||||
|
||||
|
||||
def manifest_shas(path: Path) -> tuple[str, str]:
|
||||
"""(set sha256, key sha256) of a calibration manifest (gates/calibration.manifest.json)."""
|
||||
data = _read_json(path, "calibration manifest")
|
||||
shas = [data.get(part, {}).get("sha256") if isinstance(data, dict) and isinstance(data.get(part), dict) else None
|
||||
for part in ("set", "key")]
|
||||
if not all(isinstance(s, str) and s for s in shas):
|
||||
raise RunError(f"{path} is not a calibration manifest with a set and a key sha256")
|
||||
return str(shas[0]), str(shas[1])
|
||||
|
||||
|
||||
def passed_models(path: Path, manifest: Path = CALIBRATION_MANIFEST) -> set[str]:
|
||||
"""The models with a green run on the set and key the calibration manifest names: an entry whose set_sha256 or
|
||||
key_sha256 differs (a toy set, an older set, a set rebuilt later) clears no model."""
|
||||
set_sha, key_sha = manifest_shas(manifest)
|
||||
return {e["model"] for e in read_passed(path) if e["set_sha256"] == set_sha and e["key_sha256"] == key_sha}
|
||||
|
||||
|
||||
def manifests_of(args: argparse.Namespace) -> list[Path]:
|
||||
"""The calibration manifests a green run must be on: those given with --manifest, else both tracked sets (the
|
||||
Claude Code docs set and the folder set of Norwegian and English PDF/Office documents, motor B16)."""
|
||||
given: list[str] | None = args.manifest
|
||||
return [Path(m) for m in given] if given else [CALIBRATION_MANIFEST, CALIBRATION_FOLDER_MANIFEST]
|
||||
|
||||
|
||||
def reader_transcripts(transcripts: Path, run: str) -> dict[str, list[Path]]:
|
||||
"""Transcripts per call name of the reader agents of one run, found by their description and agent type, whether
|
||||
started from the session (<session>/subagents/) or from a workflow (<session>/subagents/workflows/<id>/)."""
|
||||
prefix = re.compile(rf"^V1-{re.escape(run)}-read-(\S+)$")
|
||||
found: dict[str, list[Path]] = {}
|
||||
metas = [*transcripts.glob("*/subagents/agent-*.meta.json"),
|
||||
*transcripts.glob("*/subagents/workflows/*/agent-*.meta.json")]
|
||||
for meta_path in sorted(metas):
|
||||
try:
|
||||
meta = json.loads(meta_path.read_text(encoding="utf-8"))
|
||||
except (OSError, ValueError):
|
||||
continue
|
||||
if not isinstance(meta, dict) or not is_agent_type(meta.get("agentType"), AGENT_TYPE):
|
||||
continue
|
||||
m = prefix.match(str(meta.get("description", "")))
|
||||
if m is None:
|
||||
continue
|
||||
path = meta_path.with_name(meta_path.name.removesuffix(".meta.json") + ".jsonl")
|
||||
found.setdefault(m.group(1), []).append(path)
|
||||
return found
|
||||
|
||||
|
||||
@dataclass
|
||||
class Collected:
|
||||
doc: dict[str, Any]
|
||||
claims: int
|
||||
answered: int = 0
|
||||
broke_rule: int = 0
|
||||
twice: int = 0
|
||||
unanswered: int = 0
|
||||
stray: int = 0
|
||||
|
||||
|
||||
def collect(name: str, calls: list[Call], found: dict[str, list[Path]]) -> Collected:
|
||||
verdicts: list[dict[str, str]] = []
|
||||
models: set[str] = set()
|
||||
out = Collected({"reader": name, "models": [], "verdicts": verdicts}, sum(len(c.ids) for c in calls))
|
||||
names = {c.name for c in calls}
|
||||
out.stray = sum(len(paths) for call, paths in found.items() if call not in names)
|
||||
for call in calls:
|
||||
paths = found.get(call.name, [])
|
||||
if not paths:
|
||||
out.unanswered += 1
|
||||
continue
|
||||
if len(paths) > 1:
|
||||
out.twice += 1
|
||||
continue
|
||||
out.answered += 1
|
||||
if paths[0].is_file():
|
||||
models |= transcript_models(paths[0])
|
||||
given = parse_reply(final_reply(paths[0]) if paths[0].is_file() else "", len(call.ids))
|
||||
if not given:
|
||||
out.broke_rule += 1
|
||||
for number, cid in enumerate(call.ids, 1):
|
||||
if number in given:
|
||||
verdicts.append({"id": cid, "verdict": given[number]})
|
||||
out.doc["models"] = sorted(models)
|
||||
return out
|
||||
|
||||
|
||||
def cmd_render(args: argparse.Namespace) -> int:
|
||||
set_path = Path(args.set)
|
||||
pairs = load_pairs(set_path)
|
||||
calls = load_calls(Path(args.calls), set_path)
|
||||
try:
|
||||
system = instructions(Path(args.agent).read_text(encoding="utf-8"))
|
||||
except OSError as exc:
|
||||
raise RunError(f"cannot read the agent definition {args.agent}: {exc}") from exc
|
||||
for what, text in (("the agent's system text", system), ("the instructions in the task", INSTRUCTIONS)):
|
||||
leaks = shared_runs(text, pairs)
|
||||
if leaks:
|
||||
raise RunError(f"{what} and the set share a run of {RUN_WORDS} words in {len(leaks)} pair(s); "
|
||||
"nothing written")
|
||||
by_id = {p.id: p for p in pairs}
|
||||
out = Path(args.out_dir)
|
||||
tasks = {out / f"{c.name}.md": render_call(c, by_id) for c in calls}
|
||||
changed = sorted(p.name for p, t in tasks.items() if p.exists() and p.read_text(encoding="utf-8") != t)
|
||||
if changed:
|
||||
raise RunError(f"{len(changed)} task(s) already written with other text, first {changed[0]}; nothing written")
|
||||
out.mkdir(parents=True, exist_ok=True)
|
||||
for path, task in tasks.items():
|
||||
path.write_text(task, encoding="utf-8")
|
||||
print(f"render: {len(tasks)} tasks for {sum(len(c.ids) for c in calls)} claims written to {out}")
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_collect(args: argparse.Namespace) -> int:
|
||||
set_path = Path(args.set)
|
||||
calls = load_calls(Path(args.calls), set_path)
|
||||
transcripts = Path(args.transcripts)
|
||||
if not transcripts.is_dir():
|
||||
raise RunError(f"the transcripts directory {transcripts} does not exist")
|
||||
got = collect(args.reader, calls, reader_transcripts(transcripts, args.run))
|
||||
models: list[str] = got.doc["models"]
|
||||
if args.gate_passed is not None:
|
||||
for manifest in manifests_of(args):
|
||||
passed = passed_models(Path(args.gate_passed), manifest)
|
||||
lacking = [m for m in models if m not in passed] if models else ["(none)"]
|
||||
if lacking:
|
||||
print(f"collect cannot run: model(s) {', '.join(lacking)} have no green calibration run in "
|
||||
f"{args.gate_passed} on the set and key of {manifest}; run the calibration gate on them first",
|
||||
file=sys.stderr)
|
||||
return 1
|
||||
Path(args.out).write_text(json.dumps(got.doc, indent=1) + "\n", encoding="utf-8")
|
||||
written = len(got.doc["verdicts"])
|
||||
stray = f"; {got.stray} transcript{'s' if got.stray != 1 else ''} for a call not in the layout" if got.stray else ""
|
||||
print(f"collect {args.reader}: {written} of {got.claims} verdicts written to {args.out}")
|
||||
print(f"calls: {got.answered} of {len(calls)} answered; {got.broke_rule} broke the reply rule; "
|
||||
f"{got.twice} answered twice; {got.unanswered} not answered{stray}")
|
||||
print(f"missing: {got.claims - written}")
|
||||
print(f"models: {', '.join(models) if models else '(none)'}")
|
||||
return 0 if written == got.claims else 1
|
||||
|
||||
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
sub = p.add_subparsers(dest="command", required=True)
|
||||
r = sub.add_parser("render", help="write one reader task per call")
|
||||
r.add_argument("--set", required=True)
|
||||
r.add_argument("--calls", required=True, help="the call layout (okf-wiki/reader-calls/1)")
|
||||
r.add_argument("--agent", required=True, help="the reader's agent definition (agents/okf-wiki-reader.md)")
|
||||
r.add_argument("--out-dir", required=True)
|
||||
c = sub.add_parser("collect", help="turn the readers' replies in the transcripts into a verdicts file")
|
||||
c.add_argument("--set", required=True)
|
||||
c.add_argument("--calls", required=True)
|
||||
c.add_argument("--transcripts", required=True,
|
||||
help="~/.claude/projects/<slug>/ of the session that ran the readers")
|
||||
c.add_argument("--run", required=True, help="the run name in the descriptions V1-<run>-read-<call>")
|
||||
c.add_argument("--reader", required=True)
|
||||
c.add_argument("--out", required=True)
|
||||
check = c.add_mutually_exclusive_group(required=True)
|
||||
check.add_argument("--calibration", action="store_true",
|
||||
help="this run is a calibration gate run; the readers' model is not checked")
|
||||
check.add_argument("--gate-passed", metavar="FILE", default=None, nargs="?", const=str(CALIBRATION_PASSED),
|
||||
help="the record of green calibration runs (gates/calibration.py gate --passed; without FILE "
|
||||
"the tracked gates/calibration.passed.json); collect writes nothing unless every model "
|
||||
"the readers ran on is in it")
|
||||
c.add_argument("--manifest", action="append", default=None,
|
||||
help="a calibration manifest whose set and key a green run must be on, once per manifest; every "
|
||||
"one must hold (default: the tracked gates/calibration.manifest.json and "
|
||||
"gates/calibration-folder.manifest.json)")
|
||||
return p
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
args = build_parser().parse_args(argv)
|
||||
commands = {"render": cmd_render, "collect": cmd_collect}
|
||||
try:
|
||||
return commands[args.command](args)
|
||||
except (RunError, OSError) as exc:
|
||||
print(f"FAIL: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Loading…
Add table
Add a link
Reference in a new issue