okf-wiki/gates/answers.py
Kjell Tore Guttormsen 0fd86a23a3
okf-wiki 1.1.1
First public release, published as one snapshot. What changed between
versions is in CHANGELOG.md.

Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
2026-11-14 21:03:20 +01:00

911 lines
43 KiB
Python

"""The answer harness for PLAN.md criterion 4 (contract K5 points 2-7), adapted from evals/direction_probe.py.
Each question is answered by one isolated subagent per arm in the user's session, described
`S1-<run>-<arm>-<id>`. This module never calls a model and starts nothing:
setup one arm's basis (its own copy of the root's source files, and for `bundle` the collection beside
them) and one task file per question, `<run>/tasks/<arm>/<id>.md`, which carries the question and
the basis and never the answer keys, the spans or the number of documents
collect the per-question answer files into the arm's answers file (K5 point 4)
harvest tokens per question (each message id once, all four usage fields, plus a lower bound for an output
the transcript never recorded) and isolation breaches from the subagent transcripts (K5 point 6)
score two arms' results from `answerset.py gate --out`: per arm and group, by path count, the pairs with
the exact McNemar p, the difference with its paired 95 % interval (Newcombe 1998, method 10), and the
smallest difference the set can show at 80 % power and alpha 0.05 (K5 point 7)
The arms are `filsok` (file search alone over the root's source files) and `bundle` (the collection plus file
search over the same files). Exit 0 = done; 1 = could not run, or (collect, harvest) something is missing.
"""
from __future__ import annotations
import argparse
import json
import os
import re
import shlex
import shutil
import sys
from math import comb, sqrt
from pathlib import Path
from statistics import NormalDist
from typing import Any
from answerset import (
ANSWERS_SCHEMA,
ARMS,
GROUPS,
K1_FILE,
MANY_PATHS,
RESULTS_SCHEMA,
Root,
RunError,
as_list,
dump,
load_json,
load_set,
)
TOKENS_SCHEMA = "okf-wiki/answer-tokens/1"
SCORE_SCHEMA = "okf-wiki/answer-score/1"
NAME_RE = re.compile(r"^[A-Za-z0-9]+$")
ALLOWED_TOOLS = ("Read", "Grep", "Glob", "Bash", "Write", "SubagentHandback", "StructuredOutput", "ToolSearch")
READ_ONLY = ("grep", "egrep", "fgrep", "rg", "find", "ls", "wc", "head", "tail", "sed", "cat", "sort", "uniq", "cut",
"tr", "nl")
LITERAL = ("echo",) # accepted in a chain (order 31 A2: order 13's separators), never offered in the task
OWN_ANSWER = ("jq",) # accepted on the helper's own answer file only (order 31 A2b), never offered in the task
SEPARATORS = ("|", "&&", ";", "||")
FIND_ACTIONS = ("-exec", "-execdir", "-ok", "-okdir", "-delete", "-fprint", "-fprint0", "-fprintf", "-fls")
# Order 31 A2b: the search word of grep and rg, the script of sed and the filter of jq are not paths. Per command: the
# short options that take a value, those of them whose value is the search word, those whose value is a file of
# search words, the long ones of each, and (order 39) every other long option that takes a value, GNU and BSD. Such a
# value is consumed and checked as a path, so a file option's value outside the basis (--exclude-from, --ignore-file)
# is a breach. An option missing here makes its value the search word, and when that value is a path the path goes
# unchecked: the lists must be whole. grep's --context is left out because BSD grep takes its value only after `=`
# (`--context 1 /x f` searches for 1 in /x), so a separate value stays the search word and the check stays stricter.
GREP_LONG = ("after-context", "before-context", "binary-files", "devices", "directories", "exclude", "exclude-dir",
"exclude-from", "group-separator", "include", "include-dir", "label", "max-count")
RG_LONG = ("after-context", "before-context", "color", "colors", "context", "context-separator", "dfa-size-limit",
"encoding", "engine", "field-context-separator", "field-match-separator", "generate", "glob", "iglob",
"ignore-file", "hostname-bin", "hyperlink-format", "max-columns", "max-count", "max-depth", "max-filesize",
"path-separator", "pre", "pre-glob", "regex-size-limit", "replace", "sort", "sortr", "threads", "type",
"type-add", "type-clear", "type-not")
SEARCH_WORDS: dict[str, tuple[str, str, str, tuple[str, ...], tuple[str, ...], tuple[str, ...]]] = {
**{grep: ("ABCDdefm", "e", "f", ("regexp",), ("file",), GREP_LONG) for grep in ("grep", "egrep", "fgrep")},
"rg": ("ABCEMTdefgjmrt", "e", "f", ("regexp",), ("file",), RG_LONG),
"sed": ("efl", "e", "f", ("expression",), ("file",), ("line-length",)),
"jq": ("", "", "f", (), ("from-file",), ()), # the filter; -f reads it from the first file instead
}
RUNS_A_PROGRAM = {"rg": ("pre", "hostname-bin")} # long options whose value is a program the command runs
# Order 39: a sed script is exempt from the path check, so its commands are read instead. w, W and e (also as flags
# of s) write a file or run a program: a breach. r and R read the file named after them: checked as a path.
SED_IN_PLACE = "iI" # GNU -i[SUFFIX], BSD -i SUFFIX and -I SUFFIX: either one in a flag cluster edits in place
SED_ADDRESS = re.compile(r"\d+(~\d+)?|\$")
SED_PLAIN = set("=dDgGhHlnNpPxzFqQ") # commands without an argument (l, q and Q may take a number)
SED_LABEL = set("btT:") # commands whose label runs to a semicolon or a newline
SED_TEXT = set("aic") # commands whose text runs to the end of the line
SED_S_FLAGS = set("gpiImM0123456789 \t")
USAGE_FIELDS = (
"input_tokens", # uncached input
"cache_creation_input_tokens", # written to the prompt cache
"cache_read_input_tokens", # read from it
"output_tokens",
)
INPUT_SIDE = USAGE_FIELDS[:3]
CHARS_PER_TOKEN_LOW = 2.5 # injected text counted high, so a derived output is a lower bound (okfwiki/cost.py)
ALPHA, POWER = 0.05, 0.80
def write(path: Path, data: Any) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_bytes(dump(data))
# ---------------------------------------------------------------------------------------------- setup
SEARCH = {
"filsok": """BASIS holds the source files under `kilde/` and nothing else. Search them with grep and find (Bash,
as above), and read what you find with Read.""",
"bundle": """BASIS holds `wiki/`, a knowledge collection compiled from the sources, and `kilde/`, the source
files. A page names its source as `/kilde/<name>`, which is the file `BASIS/kilde/<name>`. Follow this order:
1. Search `BASIS/wiki/index.md` with grep for the question's words: it is never read whole, because it can be over
the size one Read allows. Each line is one page: its title and its description.
2. Open the main page of the area the question is about: a main page carries `x-main-for` with the source it covers.
3. Read it, follow its links to the pages the question needs, and go on from those pages where the question needs it.
4. Open the source behind a footnote when the quote has to be checked or when the page does not say enough.
You may also search `wiki/` and `kilde/` with grep and find (Bash, as above) at any point.""",
}
def render_task(arm: str, basis: Path, qid: str, question: str, answer_path: Path) -> str:
return f"""# Answer one question
BASIS: `{basis}`
You answer ONE question from the files in BASIS. Read nothing outside BASIS apart from this task file: no other
folder, no web, no earlier work. Answer only from what the files say, and say plainly when they do not say something.
Tools: Read (a file inside BASIS), Bash for read-only search, and Write once, for the answer file. No other tool.
Every Bash command starts with `cd {basis} && ` and then uses only {', '.join(READ_ONLY)}, joined by `|` or `&&`,
with paths relative to BASIS: never `..`, `~` or a path outside BASIS, no `$` expansion, no output redirection except
`2>/dev/null`, no `find -exec` or `-delete`, no `sed -i`. For example: `cd {basis} && grep -rn "term" kilde`
## How to search
{SEARCH[arm]}
## The question
{question}
## The answer
Write one JSON file to `{answer_path}`:
```json
{{"id": "{qid}", "answer": "<your answer in prose>", "citations": [{{"path": "kilde/<name>", "quote": "<text>"}}]}}
```
- `answer`: answer every part of the question.
- `citations`: one or more per source file your answer relies on, and cite every such file.
- `path`: `kilde/` and the file name, as the file lies in BASIS, for example `kilde/notes.md`: never a page in
`wiki/`.
- `quote`: text copied character for character from that file (only whitespace may differ), long enough to carry
the fact, for example the whole line or sentence.
When the file is written, reply with the one word `done`.
"""
def copy_sources(root: Root, target: Path) -> int:
"""Every source K1 lists, copied byte-exact under target/kilde/, each checked against its K1 sha256."""
for path in sorted(root.entries):
text = root.text(path)
if text is None:
raise RunError(root.problem(path))
dest = target / path
dest.parent.mkdir(parents=True, exist_ok=True)
shutil.copyfile(root.path / path, dest)
return len(root.entries)
def check_part(ref: dict[str, Any], root: Root, part_path: Path) -> None:
"""Order 31 A3: the set names the part's K1, the part is a subset_of the root, and every source of the part
stands in the root with the same path and sha256."""
part = Root(part_path)
if ref.get("manifest_sha256") != part.manifest_sha256:
raise RunError(f"the set names manifest_sha256 {ref.get('manifest_sha256')!r}, --part-root {part_path} has "
f"{part.manifest_sha256}; --part-root is the root the set was made from")
raw = json.loads(part.manifest_bytes.decode("utf-8")).get("subset_of")
named = raw.get("manifest_sha256") if isinstance(raw, dict) else None
if named != root.manifest_sha256:
raise RunError(f"--part-root {part_path} names subset_of manifest_sha256 {named!r}, the root has "
f"{root.manifest_sha256}")
for path, entry in sorted(part.entries.items()):
if path not in root.entries:
raise RunError(f"{path} of --part-root is not in the root {root.path}")
if root.entries[path]["sha256"] != entry["sha256"]:
raise RunError(f"{path} has sha256 {entry['sha256']} in --part-root and {root.entries[path]['sha256']} "
f"in the root")
def setup_arm(run: Path, name: str, arm: str, set_path: Path, sources: Path,
bundle: Path | None, part_root: Path | None = None) -> list[tuple[str, Path]]:
if arm not in ARMS:
raise RunError(f"unknown arm {arm!r}; the arms are {', '.join(ARMS)}")
if not NAME_RE.match(name):
raise RunError(f"run name {name!r} must be letters and digits only (it is part of S1-<run>-<arm>-<id>)")
if arm == "bundle" and bundle is None:
raise RunError("the bundle arm needs --bundle")
if arm == "bundle" and bundle is not None and not (bundle / "index.md").is_file():
raise RunError(f"{bundle} has no index.md")
data = load_set(set_path)
root = Root(sources)
ref = data["sources"]
if part_root is not None:
check_part(ref, root, part_root)
elif ref.get("manifest_sha256") != root.manifest_sha256:
raise RunError(f"the set names manifest_sha256 {ref.get('manifest_sha256')!r}, the root has "
f"{root.manifest_sha256}; an arm is set up on the root the set was made from")
basis = run / "arms" / arm
if basis.exists():
raise RunError(f"{basis} exists already; an arm is set up once, from fresh")
if arm == "bundle" and bundle is not None:
shutil.copytree(bundle, basis / "wiki")
copy_sources(root, basis)
tasks = []
for q in data["questions"]:
qid = str(q["id"])
task = run / "tasks" / arm / f"{qid}.md"
answer = run / "answers" / arm / f"{qid}.json"
task.parent.mkdir(parents=True, exist_ok=True)
task.write_text(render_task(arm, basis, qid, str(q["question"]), answer), encoding="utf-8")
tasks.append((f"S1-{name}-{arm}-{qid}", task))
(run / "answers" / arm).mkdir(parents=True, exist_ok=True)
return tasks
# ---------------------------------------------------------------------------------------------- collect
def collect(run: Path, arm: str) -> tuple[Path, list[str]]:
"""The per-question answers into one answers file; the id comes from the file name."""
if arm not in ARMS:
raise RunError(f"unknown arm {arm!r}; the arms are {', '.join(ARMS)}")
folder = run / "answers" / arm
if not folder.is_dir():
raise RunError(f"{folder} does not exist; run setup first")
gathered: list[dict[str, Any]] = []
problems = []
for path in sorted(folder.glob("*.json")):
try:
data = json.loads(path.read_text(encoding="utf-8"))
except (OSError, UnicodeDecodeError, json.JSONDecodeError):
data = None
if not isinstance(data, dict):
problems.append(f"{path.name}: not a JSON object")
continue
gathered.append({**data, "id": path.stem})
out = run / "answers" / f"{arm}.json"
write(out, {"schema": ANSWERS_SCHEMA, "arm": arm, "answers": gathered})
return out, problems
# ---------------------------------------------------------------------------------------------- harvest
def read_transcript(path: Path) -> list[dict[str, Any]]:
lines = []
for raw in path.read_text(encoding="utf-8").splitlines():
try:
line = json.loads(raw)
except json.JSONDecodeError:
continue
if isinstance(line, dict):
lines.append(line)
return lines
def first_prompt(lines: list[dict[str, Any]]) -> str:
for line in lines:
if line.get("type") == "user":
content = line.get("message", {}).get("content")
if isinstance(content, list):
content = " ".join(str(b.get("text", "")) for b in content if isinstance(b, dict))
return content if isinstance(content, str) else ""
return ""
def inside(path: object, root: Path) -> bool:
if not isinstance(path, str) or not os.path.isabs(path):
return False
norm = Path(os.path.normpath(path))
return norm == root or root in norm.parents
def outside_single_quotes(command: str) -> str:
"""The command with the text inside single quotes blanked out, where the shell takes every character literally
(order 31 A2b). A backslash outside them keeps the next character, so an escaped backtick still shows."""
out: list[str] = []
quote = ""
i = 0
while i < len(command):
ch = command[i]
if quote == "'":
quote = "" if ch == "'" else quote
out.append(" ")
elif ch == "\\":
out.append(command[i:i + 2])
i += 1
elif ch == "'" and not quote:
quote = "'"
out.append(" ")
else:
if ch == '"':
quote = "" if quote else '"'
out.append(ch)
i += 1
return "".join(out)
def search_words(command: str, args: list[str]) -> set[int]:
"""The indexes in args of the search words: the values of -e and --regexp (sed: --expression), or else the first
argument that is not an option. With -f or --file the search words come from a file and none is positional,
wherever the option stands (GNU getopt reorders the arguments). The value of any other option that takes one is
skipped here, so it is checked as a path."""
if command not in SEARCH_WORDS:
return set()
valued, word_opts, file_opts, word_long, file_long, other_long = SEARCH_WORDS[command]
words: set[int] = set()
first: int | None = None
positional, options = True, True
i = 0
while i < len(args):
arg = args[i]
if options and arg == "--":
options = False
elif options and arg.startswith("--"):
name, eq, _ = arg[2:].partition("=")
if name in word_long + file_long:
positional = False
if name in word_long:
words.add(i if eq else i + 1)
if name in word_long + file_long + other_long:
i += 0 if eq else 1
elif options and arg.startswith("-") and arg != "-":
for j, ch in enumerate(arg[1:]):
if ch in word_opts + file_opts:
positional = False
if ch in valued:
attached = j + 2 < len(arg)
if ch in word_opts:
words.add(i if attached else i + 1)
i += 0 if attached else 1
break
elif first is None:
first = i
i += 1
if positional and first is not None:
words.add(first)
return words
def word_text(command: str, token: str) -> str:
"""The search word or script a search-word token carries: the token, or the value attached to -e or --expression."""
if token.startswith("--"):
return token.partition("=")[2]
if token.startswith("-") and token != "-":
valued = SEARCH_WORDS[command][0]
for j, ch in enumerate(token[1:]):
if ch in valued:
return token[j + 2:]
return token
def in_place(token: str) -> bool:
"""A sed flag cluster with -i or -I before the first option that takes a value (`-ni`, `-Ei`, `-I`)."""
if not token.startswith("-") or token.startswith("--"):
return False
for ch in token[1:]:
if ch in SED_IN_PLACE:
return True
if ch in SEARCH_WORDS["sed"][0]:
return False
return False
class SedParseError(ValueError):
pass
def sed_breach(script: str) -> tuple[str | None, list[str]]:
"""The breach of one sed script, if any (w, W or e as a command or as a flag of s, or a script that cannot be
parsed), and the files it reads with r or R, which the caller checks as paths. POSIX, GNU and BSD commands."""
n = len(script)
reads: list[str] = []
def regex_end(k: int, delim: str) -> int:
while k < n and script[k] != delim:
k += 2 if script[k] == "\\" else 1
if k >= n:
raise SedParseError(script)
return k + 1
def address(k: int) -> int:
m = SED_ADDRESS.match(script, k)
if m:
return m.end()
if k < n and script[k] in "/\\":
delim = "/"
if script[k] == "\\":
if k + 1 >= n:
raise SedParseError(script)
k += 1
delim = script[k]
k = regex_end(k + 1, delim)
while k < n and script[k] in "IM":
k += 1
return k
def line_end(k: int) -> int:
end = script.find("\n", k)
return n if end < 0 else end
depth, i = 0, 0
try:
while i < n:
if script[i] in " \t\n;":
i += 1
continue
if script[i] == "}":
depth -= 1
if depth < 0:
raise SedParseError(script)
i += 1
continue
start = i
i = address(i)
if start < i < n and script[i] == ",":
i += 1
if i < n and script[i] in "+~":
i += 1
while i < n and script[i].isdigit():
i += 1
else:
i = address(i)
while i < n and script[i] in " \t!":
i += 1
if i >= n:
raise SedParseError(script)
cmd = script[i]
i += 1
if cmd == "{":
depth += 1
elif cmd in "wWe":
return f"Bash sed command {cmd}", reads
elif cmd in "rR":
end = line_end(i)
reads.append(script[i:end].strip())
i = end
elif cmd in SED_TEXT:
i = line_end(i)
elif cmd in SED_LABEL:
while i < n and script[i] not in ";\n":
i += 1
elif cmd in SED_PLAIN:
while i < n and script[i] in " \t0123456789":
i += 1
elif cmd in "sy":
if i >= n or script[i] in "\\\n":
raise SedParseError(script)
delim = script[i]
i = regex_end(regex_end(i + 1, delim), delim)
while cmd == "s" and i < n and script[i] not in ";\n}":
if script[i] in "wWe":
return f"Bash sed s flag {script[i]}", reads
if script[i] not in SED_S_FLAGS:
raise SedParseError(script)
i += 1
else:
raise SedParseError(script)
except SedParseError:
return "Bash sed script that cannot be parsed", reads
if depth:
return "Bash sed script that cannot be parsed", reads
return None, reads
def bash_breach(command: str, basis: Path, answer: Path | None = None) -> str | None:
"""None when the command is read-only search that starts with `cd <basis> &&` and stays inside the basis, where
the only path outside it may be the answer file; or when it reads the answer file alone, by its absolute path or
after `cd` into its folder (order 31 A2b)."""
unquoted = outside_single_quotes(command)
if "`" in unquoted or re.search(r"\$[A-Za-z_{(]", unquoted):
return "Bash with an expansion"
try:
# Order 39: bash takes '#' as a comment only at the start of a word, so shlex's comment rule is off (a comment
# is then checked as arguments, which is stricter); an unquoted newline separates commands as ';' does.
lexer = shlex.shlex(re.sub(r"\s2>/dev/null(?=\s|$|[;|&])", "", command), posix=True,
punctuation_chars="|&;<>()\n")
lexer.whitespace_split = True
lexer.commenters = ""
lexer.whitespace = " \t\r"
tokens = [piece for token in lexer for piece in
(re.split(r"(\n)", token) if set(token) <= set("|&;<>()\n") else [token]) if piece]
except ValueError:
return "Bash that cannot be parsed"
segments: list[list[str]] = [[]]
newline = False # a newline ends a command; after a separator, on a blank line or at the end it changes nothing
for token in tokens:
if token == "\n":
newline = bool(segments[-1])
continue
if newline and token not in SEPARATORS:
segments.append([])
newline = False
if token in SEPARATORS:
segments.append([])
elif token and set(token) <= set("&<>()"):
return f"Bash with {token!r}"
else:
segments[-1].append(token)
first = segments[0]
own = os.path.normpath(answer) if answer is not None else ""
cd = first[1] if len(first) == 2 and first[0] == "cd" and os.path.isabs(first[1]) else ""
here: str | None = None # where a relative path is, when it can name the answer file
if cd and inside(cd, basis):
own_only, chain = False, segments[1:]
elif own and cd and os.path.normpath(cd) == os.path.dirname(own):
own_only, chain, here = True, segments[1:], os.path.dirname(own)
elif own and first and first[0] in READ_ONLY + OWN_ANSWER:
own_only, chain = True, segments
else:
return "Bash that does not start with cd into the basis"
for segment in chain:
if not segment or segment[0] not in READ_ONLY + LITERAL + OWN_ANSWER:
return f"Bash command not allowed: {segment[0] if segment else '(empty)'}"
words = search_words(segment[0], segment[1:])
names_own = False
checked: list[tuple[str, str]] = [] # (token, the value in it that is checked as a path)
for index, token in enumerate(segment[1:]):
if segment[0] == "sed" and (in_place(token) or token.startswith("--in-place")):
return "Bash sed -i"
if index in words:
if segment[0] == "sed":
breach, reads = sed_breach(word_text("sed", token))
if breach:
return breach
checked.extend((name, name) for name in reads)
continue
if segment[0] == "find" and token in FIND_ACTIONS:
return f"Bash find action {token}"
if token.startswith("--") and token[2:].partition("=")[0] in RUNS_A_PROGRAM.get(segment[0], ()):
return f"Bash {segment[0]} {token.partition('=')[0]} runs a program"
checked.append((token, token.split("=", 1)[1] if token.startswith("-") and "=" in token else token))
for token, value in checked:
if value.startswith("~") or ".." in value.split("/"):
return f"Bash path outside the basis: {token}"
path = value if os.path.isabs(value) else os.path.join(here, value) if here else ""
if own and path and os.path.normpath(path) == own:
names_own = True
elif own_only and (not token.startswith("-") or "=" in token):
return f"Bash path other than the own answer: {token}"
elif value.startswith("/") and value != "/dev/null" and not inside(value, basis):
return f"Bash path outside the basis: {token}"
if own_only and not names_own:
return "Bash outside the basis that does not name the own answer"
if segment[0] in OWN_ANSWER and not names_own:
return f"Bash command not allowed: {segment[0]}"
return None
def breaches_of(tools: list[tuple[str, dict[str, Any]]], basis: Path, task: Path, answer: Path) -> list[str]:
"""K5 point 3: every tool call that reads or lists a path outside the basis and the task file, every tool that
is not allowed, and every write to another file than the answer."""
found = []
for name, inp in tools:
if name not in ALLOWED_TOOLS:
found.append(f"{name}: tool not allowed")
elif name == "Read":
path = inp.get("file_path")
if not (inside(path, basis)
or (isinstance(path, str) and os.path.normpath(path) in (str(task), str(answer)))):
found.append(f"Read outside the basis: {path}")
elif name in ("Grep", "Glob"):
path = inp.get("path")
pattern = inp.get("pattern", "")
if path is None:
found.append(f"{name} without a path (searches the working directory)")
elif not inside(path, basis):
found.append(f"{name} outside the basis: {path}")
elif name == "Glob" and isinstance(pattern, str) and os.path.isabs(pattern) and not inside(pattern, basis):
found.append(f"Glob pattern outside the basis: {pattern}")
elif name == "Bash":
breach = bash_breach(str(inp.get("command", "")), basis, answer)
if breach:
found.append(breach)
elif name == "Write" and inp.get("file_path") != str(answer):
found.append(f"Write to another file: {inp.get('file_path')}")
return found
def _injected_chars(line: dict[str, Any]) -> int:
"""Text put into the context between two assistant messages: tool results and rendered attachments."""
if line.get("type") == "attachment":
return len(str(line.get("rendered") or ""))
content = line.get("message", {}).get("content") if isinstance(line.get("message"), dict) else None
total = 0
for item in content if isinstance(content, list) else []:
if isinstance(item, dict) and item.get("type") == "tool_result":
inner = item.get("content")
if isinstance(inner, str):
total += len(inner)
elif isinstance(inner, list):
total += sum(len(str(x.get("text", ""))) for x in inner if isinstance(x, dict))
return total
def usage_and_tools(lines: list[dict[str, Any]]) -> tuple[dict[str, int], list[tuple[str, dict[str, Any]]]]:
"""The four usage fields summed with each message id once at its largest value, the derived output (a message
whose records all lack a stop reason never got its final usage written: its output is the growth of the next
message's input side less the text injected between them, counted high), and every tool call."""
per: dict[str, dict[str, int]] = {}
order: list[str] = []
complete: set[str] = set()
between: dict[str, int] = {}
tools: list[tuple[str, dict[str, Any]]] = []
for line in lines:
message = line.get("message")
if line.get("type") != "assistant" or not isinstance(message, dict):
if order:
between[order[-1]] = between.get(order[-1], 0) + _injected_chars(line)
continue
mid = str(message.get("id", ""))
if mid not in per:
order.append(mid)
seen = per.setdefault(mid, {})
fields = message.get("usage") or {}
for f in USAGE_FIELDS:
value = fields.get(f)
if isinstance(value, int):
seen[f] = max(seen.get(f, 0), value)
if isinstance(message.get("stop_reason"), str):
complete.add(mid)
for block in message.get("content") or []:
if isinstance(block, dict) and block.get("type") == "tool_use":
inp = block.get("input")
tools.append((str(block.get("name")), inp if isinstance(inp, dict) else {}))
out = {f: sum(v.get(f, 0) for v in per.values()) for f in USAGE_FIELDS}
derived = 0
for i, mid in enumerate(order[:-1]):
if mid in complete:
continue
growth = (sum(per[order[i + 1]].get(f, 0) for f in INPUT_SIDE) - sum(per[mid].get(f, 0) for f in INPUT_SIDE)
- int(between.get(mid, 0) / CHARS_PER_TOKEN_LOW))
derived += max(0, growth - per[mid].get("output_tokens", 0))
out["output_tokens_derived"] = derived
return out, tools
def harvest(run: Path, name: str, arm: str, transcripts: Path) -> tuple[dict[str, Any], list[str]]:
"""Per task of the arm: tokens over all its runs and every isolation breach. Matched by the subagent's
description S1-<run>-<arm>-<id> (the .meta.json beside the transcript); a transcript with no description is
matched by the task path in its first prompt, one with another description is not this arm's. Returns the K5 tokens document and the tasks no transcript was found for."""
if arm not in ARMS:
raise RunError(f"unknown arm {arm!r}; the arms are {', '.join(ARMS)}")
if not transcripts.is_dir():
raise RunError(f"{transcripts} is not a directory")
tasks_dir = run / "tasks" / arm
ids = sorted(p.stem for p in tasks_dir.glob("*.md"))
if not ids:
raise RunError(f"no tasks in {tasks_dir}; run setup first")
described = re.compile(rf"^S1-{re.escape(name)}-{re.escape(arm)}-(\S+)$")
prompted = re.compile(re.escape(str(tasks_dir)) + r"/([A-Za-z0-9]+)\.md")
basis = run / "arms" / arm
rows: dict[str, dict[str, Any]] = {}
for path in sorted(transcripts.rglob("*.jsonl")):
meta_path = path.with_name(path.stem + ".meta.json")
description = ""
if meta_path.is_file():
meta = load_json(meta_path, "agent meta")
description = str(meta.get("description", "")) if isinstance(meta, dict) else ""
lines = read_transcript(path)
m = described.match(description) if description else prompted.search(first_prompt(lines))
if m is None or m.group(1) not in ids:
continue
qid = m.group(1)
counts, tools = usage_and_tools(lines)
breaches = breaches_of(tools, basis, tasks_dir / f"{qid}.md", run / "answers" / arm / f"{qid}.json")
row = rows.setdefault(qid, {"id": qid, **{f: 0 for f in counts}, "isolation_breaches": 0,
"breach_detail": [], "runs": 0})
for f, value in counts.items():
row[f] += value
row["breach_detail"] += breaches
row["isolation_breaches"] = len(row["breach_detail"])
row["runs"] += 1
doc = {"schema": TOKENS_SCHEMA, "arm": arm, "questions": [rows[i] for i in sorted(rows)]}
return doc, [i for i in ids if i not in rows]
# ---------------------------------------------------------------------------------------------- score
def mcnemar_exact(only_a: int, only_b: int) -> float:
"""Two-sided exact McNemar: the binomial test over the discordant pairs, p = 0.5."""
n, k = only_a + only_b, min(only_a, only_b)
if n == 0:
return 1.0
return min(1.0, 2 * sum(comb(n, i) for i in range(k + 1)) / float(2 ** n))
def wilson(x: int, n: int, z: float) -> tuple[float, float]:
centre = (x + z * z / 2) / (n + z * z)
half = z * sqrt(x * (n - x) / n + z * z / 4) / (n + z * z)
return centre - half, centre + half
def paired_difference_ci(e: int, f: int, g: int, h: int, level: float = 0.95) -> tuple[float, float]:
"""The interval for (e + f)/n - (e + g)/n from paired data: e both, f first only, g second only, h neither.
Newcombe (1998), Statist. Med. 17:2635-2650, method 10: Wilson score limits for each margin combined with the
phi coefficient, its numerator max(eh - fg - n/2, 0) when eh > fg, and phi 0 when a margin is 0."""
n = e + f + g + h
if n == 0:
raise RunError("no pairs")
z = NormalDist().inv_cdf(1 - (1 - level) / 2)
p1, p2 = (e + f) / n, (e + g) / n
l1, u1 = wilson(e + f, n, z)
l2, u2 = wilson(e + g, n, z)
denominator = (e + f) * (g + h) * (e + g) * (f + h)
numerator = float(e * h - f * g)
if numerator > 0:
numerator = max(numerator - n / 2, 0.0)
phi = numerator / sqrt(denominator) if denominator else 0.0
dl1, du1, dl2, du2 = p1 - l1, u1 - p1, p2 - l2, u2 - p2
low = sqrt(max(0.0, dl1 * dl1 - 2 * phi * dl1 * du2 + du2 * du2))
high = sqrt(max(0.0, du1 * du1 - 2 * phi * du1 * dl2 + dl2 * dl2))
return p1 - p2 - low, p1 - p2 + high
def min_detectable(n: int, discordant_share: float, alpha: float = ALPHA, power: float = POWER) -> float | None:
"""The smallest difference between paired proportions that n pairs show at the given power with a two-sided
McNemar test, for a share of discordant pairs psi (Connor 1987: n = (z_a sqrt(psi) + z_b sqrt(psi - d^2))^2 /
d^2, solved for d by bisection). None when psi is 0 or no d up to psi is visible."""
if n <= 0 or discordant_share <= 0:
return None
za, zb = NormalDist().inv_cdf(1 - alpha / 2), NormalDist().inv_cdf(power)
psi = min(discordant_share, 1.0)
def needed(d: float) -> float:
return (za * sqrt(psi) + zb * sqrt(max(0.0, psi - d * d))) ** 2 / (d * d)
if needed(psi) > n:
return None
low, high = 1e-9, psi
for _ in range(200):
mid = (low + high) / 2
if needed(mid) > n:
low = mid
else:
high = mid
return high
def load_results(path: Path) -> dict[str, Any]:
data = load_json(path, "results")
if not isinstance(data, dict) or data.get("schema") != RESULTS_SCHEMA or not isinstance(data.get("results"), list):
raise RunError(f"{path} is not a results file with schema {RESULTS_SCHEMA}")
if data.get("arm") not in ARMS:
raise RunError(f"{path} is for arm {data.get('arm')!r}; the arms are {', '.join(ARMS)}")
return data
def path_band(paths: int) -> str:
return "4+" if paths >= MANY_PATHS else "2-3"
def score(results: list[dict[str, Any]], tokens: list[dict[str, Any]]) -> dict[str, Any]:
by_arm = {r["arm"]: r for r in results}
if sorted(by_arm) != sorted(ARMS) or len(results) != len(ARMS):
raise RunError(f"score needs one results file for each arm: {', '.join(ARMS)}")
a, b = by_arm["filsok"], by_arm["bundle"]
if a.get("set_sha256") != b.get("set_sha256"):
raise RunError("the two results files are for different sets")
rows = {arm: {str(r["id"]): r for r in by_arm[arm]["results"]} for arm in ARMS}
if sorted(rows["filsok"]) != sorted(rows["bundle"]):
raise RunError("the two results files hold different questions")
ids = sorted(rows["filsok"])
doc: dict[str, Any] = {"schema": SCORE_SCHEMA, "set_sha256": a.get("set_sha256"), "questions": len(ids),
"arms": {}, "pairs": {}}
for arm in ARMS:
groups = {g: [sum(1 for i in ids if rows[arm][i]["group"] == g and rows[arm][i]["correct"] is True),
sum(1 for i in ids if rows[arm][i]["group"] == g)] for g in GROUPS}
bands = {band: [sum(1 for i in ids if path_band(int(rows[arm][i]["paths_needed"])) == band
and rows[arm][i]["correct"] is True),
sum(1 for i in ids if path_band(int(rows[arm][i]["paths_needed"])) == band)]
for band in ("2-3", "4+")}
doc["arms"][arm] = {"groups": groups, "paths": bands,
"all": [sum(1 for i in ids if rows[arm][i]["correct"] is True), len(ids)]}
for label, members in [("all", ids)] + [(g, [i for i in ids if rows["filsok"][i]["group"] == g]) for g in GROUPS]:
counts = {"both": 0, "only filsok": 0, "only bundle": 0, "neither": 0}
for i in members:
fa, bu = rows["filsok"][i]["correct"] is True, rows["bundle"][i]["correct"] is True
counts["both" if fa and bu else "only filsok" if fa else "only bundle" if bu else "neither"] += 1
doc["pairs"][label] = counts
pairs = doc["pairs"]["all"]
doc["mcnemar_p"] = mcnemar_exact(pairs["only filsok"], pairs["only bundle"])
n = len(ids)
if n:
low, high = paired_difference_ci(pairs["both"], pairs["only bundle"], pairs["only filsok"], pairs["neither"])
doc["difference"] = {"of": "bundle - filsok", "point": (pairs["only bundle"] - pairs["only filsok"]) / n,
"low": low, "high": high, "level": 0.95, "method": "Newcombe 1998 method 10"}
share = (pairs["only filsok"] + pairs["only bundle"]) / n
doc["smallest_visible"] = {"discordant_share": share, "difference": min_detectable(n, share),
"power": POWER, "alpha": ALPHA}
doc["tokens"] = {}
for t in tokens:
if not isinstance(t, dict) or t.get("schema") != TOKENS_SCHEMA or t.get("arm") not in ARMS:
raise RunError(f"a tokens file is not {TOKENS_SCHEMA} for one of the arms")
qs = as_list(t.get("questions"))
total = sum(int(q.get(f, 0)) for q in qs for f in (*USAGE_FIELDS, "output_tokens_derived"))
doc["tokens"][t["arm"]] = {"questions": len(qs), "per_question": total / len(qs) if qs else None,
"isolation_breaches": sum(int(q.get("isolation_breaches", 0)) for q in qs)}
return doc
def render(doc: dict[str, Any]) -> list[str]:
lines = []
for arm in ARMS:
row = doc["arms"][arm]
lines.append(f"{arm}: " + ", ".join(f"{g} {c} of {n}" for g, (c, n) in row["groups"].items())
+ f", all {row['all'][0]} of {row['all'][1]}")
for arm in ARMS:
lines.append(f"{arm} by paths: " + ", ".join(f"{b} {c} of {n}" for b, (c, n) in doc["arms"][arm]["paths"].items()))
for label, counts in doc["pairs"].items():
lines.append(("pairs" if label == "all" else f"pairs {label}") + ": "
+ ", ".join(f"{k} {v}" for k, v in counts.items()))
lines.append(f"exact McNemar p = {doc['mcnemar_p']:.3g}")
if "difference" in doc:
d = doc["difference"]
lines.append(f"bundle - filsok = {d['point']:.3f}, 95 % CI {d['low']:.3f} to {d['high']:.3f} "
f"(paired, Newcombe 1998 method 10)")
s = doc["smallest_visible"]
visible = "none" if s["difference"] is None else f"{s['difference']:.3f}"
lines.append(f"smallest difference visible at 80 % power, alpha 0.05: {visible} "
f"(discordant share measured {s['discordant_share']:.3f}, N = {doc['questions']})")
for arm, t in doc["tokens"].items():
per = "none" if t["per_question"] is None else f"{t['per_question']:.0f}"
lines.append(f"{arm} tokens: {t['questions']} questions harvested, {per} per question, "
f"isolation breaches {t['isolation_breaches']}")
return lines
# ---------------------------------------------------------------------------------------------- main
def build_parser() -> argparse.ArgumentParser:
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
sub = p.add_subparsers(dest="command", required=True)
s = sub.add_parser("setup", help="one arm's basis and task files")
s.add_argument("--run", required=True)
s.add_argument("--name", required=True, help="the run's name, letters and digits (S1-<name>-<arm>-<id>)")
s.add_argument("--arm", required=True)
s.add_argument("--set", required=True)
s.add_argument("--sources", required=True, help=f"the K1 root measured (its kilde/ files and {K1_FILE})")
s.add_argument("--bundle", help="the collection, for the bundle arm")
s.add_argument("--part-root", help="the K1 root the set was made from, when it is a subset_of --sources: the arm "
"searches the whole of --sources")
c = sub.add_parser("collect", help="the answer files into the arm's answers file")
c.add_argument("--run", required=True)
c.add_argument("--arm", required=True)
h = sub.add_parser("harvest", help="tokens and isolation breaches per question from the transcripts")
h.add_argument("--run", required=True)
h.add_argument("--name", required=True)
h.add_argument("--arm", required=True)
h.add_argument("--transcripts", required=True, help="a directory searched for *.jsonl at any depth")
r = sub.add_parser("score", help="two arms' results: groups, paths, pairs, McNemar, interval, power")
r.add_argument("--results", action="append", required=True, help="answerset.py gate --out, once per arm")
r.add_argument("--tokens", action="append", default=[], help="harvest output, once per arm (optional)")
r.add_argument("--out", default="")
return p
def main(argv: list[str] | None = None) -> int:
args = build_parser().parse_args(argv)
try:
if args.command == "setup":
run = Path(args.run).resolve()
tasks = setup_arm(run, args.name, args.arm, Path(args.set), Path(args.sources),
Path(args.bundle).resolve() if args.bundle else None,
Path(args.part_root) if args.part_root else None)
for description, task in tasks:
print(f"{description}\t{task}")
return 0
if args.command == "collect":
out, problems = collect(Path(args.run).resolve(), args.arm)
print(out)
for problem in problems:
print(f"problem: {problem}")
return 1 if problems else 0
if args.command == "harvest":
run = Path(args.run).resolve()
doc, missing = harvest(run, args.name, args.arm, Path(args.transcripts))
write(run / "tokens" / f"{args.arm}.json", doc)
breaches = sum(q["isolation_breaches"] for q in doc["questions"])
print(f"tasks harvested: {len(doc['questions'])}; breaches: {breaches}")
if missing:
print(f"tasks without a transcript: {len(missing)}: {', '.join(missing)}")
return 1 if missing else 0
doc = score([load_results(Path(p)) for p in args.results],
[load_json(Path(p), "tokens") for p in args.tokens])
print("\n".join(render(doc)))
if args.out:
write(Path(args.out), doc)
return 0
except RunError as exc:
print(f"FAIL: {exc}", file=sys.stderr)
return 1
if __name__ == "__main__":
sys.exit(main())