First public release, published as one snapshot. What changed between versions is in CHANGELOG.md. Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
911 lines
43 KiB
Python
911 lines
43 KiB
Python
"""The answer harness for PLAN.md criterion 4 (contract K5 points 2-7), adapted from evals/direction_probe.py.
|
|
|
|
Each question is answered by one isolated subagent per arm in the user's session, described
|
|
`S1-<run>-<arm>-<id>`. This module never calls a model and starts nothing:
|
|
|
|
setup one arm's basis (its own copy of the root's source files, and for `bundle` the collection beside
|
|
them) and one task file per question, `<run>/tasks/<arm>/<id>.md`, which carries the question and
|
|
the basis and never the answer keys, the spans or the number of documents
|
|
collect the per-question answer files into the arm's answers file (K5 point 4)
|
|
harvest tokens per question (each message id once, all four usage fields, plus a lower bound for an output
|
|
the transcript never recorded) and isolation breaches from the subagent transcripts (K5 point 6)
|
|
score two arms' results from `answerset.py gate --out`: per arm and group, by path count, the pairs with
|
|
the exact McNemar p, the difference with its paired 95 % interval (Newcombe 1998, method 10), and the
|
|
smallest difference the set can show at 80 % power and alpha 0.05 (K5 point 7)
|
|
|
|
The arms are `filsok` (file search alone over the root's source files) and `bundle` (the collection plus file
|
|
search over the same files). Exit 0 = done; 1 = could not run, or (collect, harvest) something is missing.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import os
|
|
import re
|
|
import shlex
|
|
import shutil
|
|
import sys
|
|
from math import comb, sqrt
|
|
from pathlib import Path
|
|
from statistics import NormalDist
|
|
from typing import Any
|
|
|
|
from answerset import (
|
|
ANSWERS_SCHEMA,
|
|
ARMS,
|
|
GROUPS,
|
|
K1_FILE,
|
|
MANY_PATHS,
|
|
RESULTS_SCHEMA,
|
|
Root,
|
|
RunError,
|
|
as_list,
|
|
dump,
|
|
load_json,
|
|
load_set,
|
|
)
|
|
|
|
TOKENS_SCHEMA = "okf-wiki/answer-tokens/1"
|
|
SCORE_SCHEMA = "okf-wiki/answer-score/1"
|
|
NAME_RE = re.compile(r"^[A-Za-z0-9]+$")
|
|
ALLOWED_TOOLS = ("Read", "Grep", "Glob", "Bash", "Write", "SubagentHandback", "StructuredOutput", "ToolSearch")
|
|
READ_ONLY = ("grep", "egrep", "fgrep", "rg", "find", "ls", "wc", "head", "tail", "sed", "cat", "sort", "uniq", "cut",
|
|
"tr", "nl")
|
|
LITERAL = ("echo",) # accepted in a chain (order 31 A2: order 13's separators), never offered in the task
|
|
OWN_ANSWER = ("jq",) # accepted on the helper's own answer file only (order 31 A2b), never offered in the task
|
|
SEPARATORS = ("|", "&&", ";", "||")
|
|
FIND_ACTIONS = ("-exec", "-execdir", "-ok", "-okdir", "-delete", "-fprint", "-fprint0", "-fprintf", "-fls")
|
|
# Order 31 A2b: the search word of grep and rg, the script of sed and the filter of jq are not paths. Per command: the
|
|
# short options that take a value, those of them whose value is the search word, those whose value is a file of
|
|
# search words, the long ones of each, and (order 39) every other long option that takes a value, GNU and BSD. Such a
|
|
# value is consumed and checked as a path, so a file option's value outside the basis (--exclude-from, --ignore-file)
|
|
# is a breach. An option missing here makes its value the search word, and when that value is a path the path goes
|
|
# unchecked: the lists must be whole. grep's --context is left out because BSD grep takes its value only after `=`
|
|
# (`--context 1 /x f` searches for 1 in /x), so a separate value stays the search word and the check stays stricter.
|
|
GREP_LONG = ("after-context", "before-context", "binary-files", "devices", "directories", "exclude", "exclude-dir",
|
|
"exclude-from", "group-separator", "include", "include-dir", "label", "max-count")
|
|
RG_LONG = ("after-context", "before-context", "color", "colors", "context", "context-separator", "dfa-size-limit",
|
|
"encoding", "engine", "field-context-separator", "field-match-separator", "generate", "glob", "iglob",
|
|
"ignore-file", "hostname-bin", "hyperlink-format", "max-columns", "max-count", "max-depth", "max-filesize",
|
|
"path-separator", "pre", "pre-glob", "regex-size-limit", "replace", "sort", "sortr", "threads", "type",
|
|
"type-add", "type-clear", "type-not")
|
|
SEARCH_WORDS: dict[str, tuple[str, str, str, tuple[str, ...], tuple[str, ...], tuple[str, ...]]] = {
|
|
**{grep: ("ABCDdefm", "e", "f", ("regexp",), ("file",), GREP_LONG) for grep in ("grep", "egrep", "fgrep")},
|
|
"rg": ("ABCEMTdefgjmrt", "e", "f", ("regexp",), ("file",), RG_LONG),
|
|
"sed": ("efl", "e", "f", ("expression",), ("file",), ("line-length",)),
|
|
"jq": ("", "", "f", (), ("from-file",), ()), # the filter; -f reads it from the first file instead
|
|
}
|
|
RUNS_A_PROGRAM = {"rg": ("pre", "hostname-bin")} # long options whose value is a program the command runs
|
|
# Order 39: a sed script is exempt from the path check, so its commands are read instead. w, W and e (also as flags
|
|
# of s) write a file or run a program: a breach. r and R read the file named after them: checked as a path.
|
|
SED_IN_PLACE = "iI" # GNU -i[SUFFIX], BSD -i SUFFIX and -I SUFFIX: either one in a flag cluster edits in place
|
|
SED_ADDRESS = re.compile(r"\d+(~\d+)?|\$")
|
|
SED_PLAIN = set("=dDgGhHlnNpPxzFqQ") # commands without an argument (l, q and Q may take a number)
|
|
SED_LABEL = set("btT:") # commands whose label runs to a semicolon or a newline
|
|
SED_TEXT = set("aic") # commands whose text runs to the end of the line
|
|
SED_S_FLAGS = set("gpiImM0123456789 \t")
|
|
USAGE_FIELDS = (
|
|
"input_tokens", # uncached input
|
|
"cache_creation_input_tokens", # written to the prompt cache
|
|
"cache_read_input_tokens", # read from it
|
|
"output_tokens",
|
|
)
|
|
INPUT_SIDE = USAGE_FIELDS[:3]
|
|
CHARS_PER_TOKEN_LOW = 2.5 # injected text counted high, so a derived output is a lower bound (okfwiki/cost.py)
|
|
ALPHA, POWER = 0.05, 0.80
|
|
|
|
|
|
def write(path: Path, data: Any) -> None:
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
path.write_bytes(dump(data))
|
|
|
|
|
|
# ---------------------------------------------------------------------------------------------- setup
|
|
|
|
|
|
SEARCH = {
|
|
"filsok": """BASIS holds the source files under `kilde/` and nothing else. Search them with grep and find (Bash,
|
|
as above), and read what you find with Read.""",
|
|
"bundle": """BASIS holds `wiki/`, a knowledge collection compiled from the sources, and `kilde/`, the source
|
|
files. A page names its source as `/kilde/<name>`, which is the file `BASIS/kilde/<name>`. Follow this order:
|
|
1. Search `BASIS/wiki/index.md` with grep for the question's words: it is never read whole, because it can be over
|
|
the size one Read allows. Each line is one page: its title and its description.
|
|
2. Open the main page of the area the question is about: a main page carries `x-main-for` with the source it covers.
|
|
3. Read it, follow its links to the pages the question needs, and go on from those pages where the question needs it.
|
|
4. Open the source behind a footnote when the quote has to be checked or when the page does not say enough.
|
|
You may also search `wiki/` and `kilde/` with grep and find (Bash, as above) at any point.""",
|
|
}
|
|
|
|
|
|
def render_task(arm: str, basis: Path, qid: str, question: str, answer_path: Path) -> str:
|
|
return f"""# Answer one question
|
|
|
|
BASIS: `{basis}`
|
|
|
|
You answer ONE question from the files in BASIS. Read nothing outside BASIS apart from this task file: no other
|
|
folder, no web, no earlier work. Answer only from what the files say, and say plainly when they do not say something.
|
|
|
|
Tools: Read (a file inside BASIS), Bash for read-only search, and Write once, for the answer file. No other tool.
|
|
Every Bash command starts with `cd {basis} && ` and then uses only {', '.join(READ_ONLY)}, joined by `|` or `&&`,
|
|
with paths relative to BASIS: never `..`, `~` or a path outside BASIS, no `$` expansion, no output redirection except
|
|
`2>/dev/null`, no `find -exec` or `-delete`, no `sed -i`. For example: `cd {basis} && grep -rn "term" kilde`
|
|
|
|
## How to search
|
|
|
|
{SEARCH[arm]}
|
|
|
|
## The question
|
|
|
|
{question}
|
|
|
|
## The answer
|
|
|
|
Write one JSON file to `{answer_path}`:
|
|
|
|
```json
|
|
{{"id": "{qid}", "answer": "<your answer in prose>", "citations": [{{"path": "kilde/<name>", "quote": "<text>"}}]}}
|
|
```
|
|
|
|
- `answer`: answer every part of the question.
|
|
- `citations`: one or more per source file your answer relies on, and cite every such file.
|
|
- `path`: `kilde/` and the file name, as the file lies in BASIS, for example `kilde/notes.md`: never a page in
|
|
`wiki/`.
|
|
- `quote`: text copied character for character from that file (only whitespace may differ), long enough to carry
|
|
the fact, for example the whole line or sentence.
|
|
|
|
When the file is written, reply with the one word `done`.
|
|
"""
|
|
|
|
|
|
def copy_sources(root: Root, target: Path) -> int:
|
|
"""Every source K1 lists, copied byte-exact under target/kilde/, each checked against its K1 sha256."""
|
|
for path in sorted(root.entries):
|
|
text = root.text(path)
|
|
if text is None:
|
|
raise RunError(root.problem(path))
|
|
dest = target / path
|
|
dest.parent.mkdir(parents=True, exist_ok=True)
|
|
shutil.copyfile(root.path / path, dest)
|
|
return len(root.entries)
|
|
|
|
|
|
def check_part(ref: dict[str, Any], root: Root, part_path: Path) -> None:
|
|
"""Order 31 A3: the set names the part's K1, the part is a subset_of the root, and every source of the part
|
|
stands in the root with the same path and sha256."""
|
|
part = Root(part_path)
|
|
if ref.get("manifest_sha256") != part.manifest_sha256:
|
|
raise RunError(f"the set names manifest_sha256 {ref.get('manifest_sha256')!r}, --part-root {part_path} has "
|
|
f"{part.manifest_sha256}; --part-root is the root the set was made from")
|
|
raw = json.loads(part.manifest_bytes.decode("utf-8")).get("subset_of")
|
|
named = raw.get("manifest_sha256") if isinstance(raw, dict) else None
|
|
if named != root.manifest_sha256:
|
|
raise RunError(f"--part-root {part_path} names subset_of manifest_sha256 {named!r}, the root has "
|
|
f"{root.manifest_sha256}")
|
|
for path, entry in sorted(part.entries.items()):
|
|
if path not in root.entries:
|
|
raise RunError(f"{path} of --part-root is not in the root {root.path}")
|
|
if root.entries[path]["sha256"] != entry["sha256"]:
|
|
raise RunError(f"{path} has sha256 {entry['sha256']} in --part-root and {root.entries[path]['sha256']} "
|
|
f"in the root")
|
|
|
|
|
|
def setup_arm(run: Path, name: str, arm: str, set_path: Path, sources: Path,
|
|
bundle: Path | None, part_root: Path | None = None) -> list[tuple[str, Path]]:
|
|
if arm not in ARMS:
|
|
raise RunError(f"unknown arm {arm!r}; the arms are {', '.join(ARMS)}")
|
|
if not NAME_RE.match(name):
|
|
raise RunError(f"run name {name!r} must be letters and digits only (it is part of S1-<run>-<arm>-<id>)")
|
|
if arm == "bundle" and bundle is None:
|
|
raise RunError("the bundle arm needs --bundle")
|
|
if arm == "bundle" and bundle is not None and not (bundle / "index.md").is_file():
|
|
raise RunError(f"{bundle} has no index.md")
|
|
data = load_set(set_path)
|
|
root = Root(sources)
|
|
ref = data["sources"]
|
|
if part_root is not None:
|
|
check_part(ref, root, part_root)
|
|
elif ref.get("manifest_sha256") != root.manifest_sha256:
|
|
raise RunError(f"the set names manifest_sha256 {ref.get('manifest_sha256')!r}, the root has "
|
|
f"{root.manifest_sha256}; an arm is set up on the root the set was made from")
|
|
basis = run / "arms" / arm
|
|
if basis.exists():
|
|
raise RunError(f"{basis} exists already; an arm is set up once, from fresh")
|
|
if arm == "bundle" and bundle is not None:
|
|
shutil.copytree(bundle, basis / "wiki")
|
|
copy_sources(root, basis)
|
|
tasks = []
|
|
for q in data["questions"]:
|
|
qid = str(q["id"])
|
|
task = run / "tasks" / arm / f"{qid}.md"
|
|
answer = run / "answers" / arm / f"{qid}.json"
|
|
task.parent.mkdir(parents=True, exist_ok=True)
|
|
task.write_text(render_task(arm, basis, qid, str(q["question"]), answer), encoding="utf-8")
|
|
tasks.append((f"S1-{name}-{arm}-{qid}", task))
|
|
(run / "answers" / arm).mkdir(parents=True, exist_ok=True)
|
|
return tasks
|
|
|
|
|
|
# ---------------------------------------------------------------------------------------------- collect
|
|
|
|
|
|
def collect(run: Path, arm: str) -> tuple[Path, list[str]]:
|
|
"""The per-question answers into one answers file; the id comes from the file name."""
|
|
if arm not in ARMS:
|
|
raise RunError(f"unknown arm {arm!r}; the arms are {', '.join(ARMS)}")
|
|
folder = run / "answers" / arm
|
|
if not folder.is_dir():
|
|
raise RunError(f"{folder} does not exist; run setup first")
|
|
gathered: list[dict[str, Any]] = []
|
|
problems = []
|
|
for path in sorted(folder.glob("*.json")):
|
|
try:
|
|
data = json.loads(path.read_text(encoding="utf-8"))
|
|
except (OSError, UnicodeDecodeError, json.JSONDecodeError):
|
|
data = None
|
|
if not isinstance(data, dict):
|
|
problems.append(f"{path.name}: not a JSON object")
|
|
continue
|
|
gathered.append({**data, "id": path.stem})
|
|
out = run / "answers" / f"{arm}.json"
|
|
write(out, {"schema": ANSWERS_SCHEMA, "arm": arm, "answers": gathered})
|
|
return out, problems
|
|
|
|
|
|
# ---------------------------------------------------------------------------------------------- harvest
|
|
|
|
|
|
def read_transcript(path: Path) -> list[dict[str, Any]]:
|
|
lines = []
|
|
for raw in path.read_text(encoding="utf-8").splitlines():
|
|
try:
|
|
line = json.loads(raw)
|
|
except json.JSONDecodeError:
|
|
continue
|
|
if isinstance(line, dict):
|
|
lines.append(line)
|
|
return lines
|
|
|
|
|
|
def first_prompt(lines: list[dict[str, Any]]) -> str:
|
|
for line in lines:
|
|
if line.get("type") == "user":
|
|
content = line.get("message", {}).get("content")
|
|
if isinstance(content, list):
|
|
content = " ".join(str(b.get("text", "")) for b in content if isinstance(b, dict))
|
|
return content if isinstance(content, str) else ""
|
|
return ""
|
|
|
|
|
|
def inside(path: object, root: Path) -> bool:
|
|
if not isinstance(path, str) or not os.path.isabs(path):
|
|
return False
|
|
norm = Path(os.path.normpath(path))
|
|
return norm == root or root in norm.parents
|
|
|
|
|
|
def outside_single_quotes(command: str) -> str:
|
|
"""The command with the text inside single quotes blanked out, where the shell takes every character literally
|
|
(order 31 A2b). A backslash outside them keeps the next character, so an escaped backtick still shows."""
|
|
out: list[str] = []
|
|
quote = ""
|
|
i = 0
|
|
while i < len(command):
|
|
ch = command[i]
|
|
if quote == "'":
|
|
quote = "" if ch == "'" else quote
|
|
out.append(" ")
|
|
elif ch == "\\":
|
|
out.append(command[i:i + 2])
|
|
i += 1
|
|
elif ch == "'" and not quote:
|
|
quote = "'"
|
|
out.append(" ")
|
|
else:
|
|
if ch == '"':
|
|
quote = "" if quote else '"'
|
|
out.append(ch)
|
|
i += 1
|
|
return "".join(out)
|
|
|
|
|
|
def search_words(command: str, args: list[str]) -> set[int]:
|
|
"""The indexes in args of the search words: the values of -e and --regexp (sed: --expression), or else the first
|
|
argument that is not an option. With -f or --file the search words come from a file and none is positional,
|
|
wherever the option stands (GNU getopt reorders the arguments). The value of any other option that takes one is
|
|
skipped here, so it is checked as a path."""
|
|
if command not in SEARCH_WORDS:
|
|
return set()
|
|
valued, word_opts, file_opts, word_long, file_long, other_long = SEARCH_WORDS[command]
|
|
words: set[int] = set()
|
|
first: int | None = None
|
|
positional, options = True, True
|
|
i = 0
|
|
while i < len(args):
|
|
arg = args[i]
|
|
if options and arg == "--":
|
|
options = False
|
|
elif options and arg.startswith("--"):
|
|
name, eq, _ = arg[2:].partition("=")
|
|
if name in word_long + file_long:
|
|
positional = False
|
|
if name in word_long:
|
|
words.add(i if eq else i + 1)
|
|
if name in word_long + file_long + other_long:
|
|
i += 0 if eq else 1
|
|
elif options and arg.startswith("-") and arg != "-":
|
|
for j, ch in enumerate(arg[1:]):
|
|
if ch in word_opts + file_opts:
|
|
positional = False
|
|
if ch in valued:
|
|
attached = j + 2 < len(arg)
|
|
if ch in word_opts:
|
|
words.add(i if attached else i + 1)
|
|
i += 0 if attached else 1
|
|
break
|
|
elif first is None:
|
|
first = i
|
|
i += 1
|
|
if positional and first is not None:
|
|
words.add(first)
|
|
return words
|
|
|
|
|
|
def word_text(command: str, token: str) -> str:
|
|
"""The search word or script a search-word token carries: the token, or the value attached to -e or --expression."""
|
|
if token.startswith("--"):
|
|
return token.partition("=")[2]
|
|
if token.startswith("-") and token != "-":
|
|
valued = SEARCH_WORDS[command][0]
|
|
for j, ch in enumerate(token[1:]):
|
|
if ch in valued:
|
|
return token[j + 2:]
|
|
return token
|
|
|
|
|
|
def in_place(token: str) -> bool:
|
|
"""A sed flag cluster with -i or -I before the first option that takes a value (`-ni`, `-Ei`, `-I`)."""
|
|
if not token.startswith("-") or token.startswith("--"):
|
|
return False
|
|
for ch in token[1:]:
|
|
if ch in SED_IN_PLACE:
|
|
return True
|
|
if ch in SEARCH_WORDS["sed"][0]:
|
|
return False
|
|
return False
|
|
|
|
|
|
class SedParseError(ValueError):
|
|
pass
|
|
|
|
|
|
def sed_breach(script: str) -> tuple[str | None, list[str]]:
|
|
"""The breach of one sed script, if any (w, W or e as a command or as a flag of s, or a script that cannot be
|
|
parsed), and the files it reads with r or R, which the caller checks as paths. POSIX, GNU and BSD commands."""
|
|
n = len(script)
|
|
reads: list[str] = []
|
|
|
|
def regex_end(k: int, delim: str) -> int:
|
|
while k < n and script[k] != delim:
|
|
k += 2 if script[k] == "\\" else 1
|
|
if k >= n:
|
|
raise SedParseError(script)
|
|
return k + 1
|
|
|
|
def address(k: int) -> int:
|
|
m = SED_ADDRESS.match(script, k)
|
|
if m:
|
|
return m.end()
|
|
if k < n and script[k] in "/\\":
|
|
delim = "/"
|
|
if script[k] == "\\":
|
|
if k + 1 >= n:
|
|
raise SedParseError(script)
|
|
k += 1
|
|
delim = script[k]
|
|
k = regex_end(k + 1, delim)
|
|
while k < n and script[k] in "IM":
|
|
k += 1
|
|
return k
|
|
|
|
def line_end(k: int) -> int:
|
|
end = script.find("\n", k)
|
|
return n if end < 0 else end
|
|
|
|
depth, i = 0, 0
|
|
try:
|
|
while i < n:
|
|
if script[i] in " \t\n;":
|
|
i += 1
|
|
continue
|
|
if script[i] == "}":
|
|
depth -= 1
|
|
if depth < 0:
|
|
raise SedParseError(script)
|
|
i += 1
|
|
continue
|
|
start = i
|
|
i = address(i)
|
|
if start < i < n and script[i] == ",":
|
|
i += 1
|
|
if i < n and script[i] in "+~":
|
|
i += 1
|
|
while i < n and script[i].isdigit():
|
|
i += 1
|
|
else:
|
|
i = address(i)
|
|
while i < n and script[i] in " \t!":
|
|
i += 1
|
|
if i >= n:
|
|
raise SedParseError(script)
|
|
cmd = script[i]
|
|
i += 1
|
|
if cmd == "{":
|
|
depth += 1
|
|
elif cmd in "wWe":
|
|
return f"Bash sed command {cmd}", reads
|
|
elif cmd in "rR":
|
|
end = line_end(i)
|
|
reads.append(script[i:end].strip())
|
|
i = end
|
|
elif cmd in SED_TEXT:
|
|
i = line_end(i)
|
|
elif cmd in SED_LABEL:
|
|
while i < n and script[i] not in ";\n":
|
|
i += 1
|
|
elif cmd in SED_PLAIN:
|
|
while i < n and script[i] in " \t0123456789":
|
|
i += 1
|
|
elif cmd in "sy":
|
|
if i >= n or script[i] in "\\\n":
|
|
raise SedParseError(script)
|
|
delim = script[i]
|
|
i = regex_end(regex_end(i + 1, delim), delim)
|
|
while cmd == "s" and i < n and script[i] not in ";\n}":
|
|
if script[i] in "wWe":
|
|
return f"Bash sed s flag {script[i]}", reads
|
|
if script[i] not in SED_S_FLAGS:
|
|
raise SedParseError(script)
|
|
i += 1
|
|
else:
|
|
raise SedParseError(script)
|
|
except SedParseError:
|
|
return "Bash sed script that cannot be parsed", reads
|
|
if depth:
|
|
return "Bash sed script that cannot be parsed", reads
|
|
return None, reads
|
|
|
|
|
|
def bash_breach(command: str, basis: Path, answer: Path | None = None) -> str | None:
|
|
"""None when the command is read-only search that starts with `cd <basis> &&` and stays inside the basis, where
|
|
the only path outside it may be the answer file; or when it reads the answer file alone, by its absolute path or
|
|
after `cd` into its folder (order 31 A2b)."""
|
|
unquoted = outside_single_quotes(command)
|
|
if "`" in unquoted or re.search(r"\$[A-Za-z_{(]", unquoted):
|
|
return "Bash with an expansion"
|
|
try:
|
|
# Order 39: bash takes '#' as a comment only at the start of a word, so shlex's comment rule is off (a comment
|
|
# is then checked as arguments, which is stricter); an unquoted newline separates commands as ';' does.
|
|
lexer = shlex.shlex(re.sub(r"\s2>/dev/null(?=\s|$|[;|&])", "", command), posix=True,
|
|
punctuation_chars="|&;<>()\n")
|
|
lexer.whitespace_split = True
|
|
lexer.commenters = ""
|
|
lexer.whitespace = " \t\r"
|
|
tokens = [piece for token in lexer for piece in
|
|
(re.split(r"(\n)", token) if set(token) <= set("|&;<>()\n") else [token]) if piece]
|
|
except ValueError:
|
|
return "Bash that cannot be parsed"
|
|
segments: list[list[str]] = [[]]
|
|
newline = False # a newline ends a command; after a separator, on a blank line or at the end it changes nothing
|
|
for token in tokens:
|
|
if token == "\n":
|
|
newline = bool(segments[-1])
|
|
continue
|
|
if newline and token not in SEPARATORS:
|
|
segments.append([])
|
|
newline = False
|
|
if token in SEPARATORS:
|
|
segments.append([])
|
|
elif token and set(token) <= set("&<>()"):
|
|
return f"Bash with {token!r}"
|
|
else:
|
|
segments[-1].append(token)
|
|
first = segments[0]
|
|
own = os.path.normpath(answer) if answer is not None else ""
|
|
cd = first[1] if len(first) == 2 and first[0] == "cd" and os.path.isabs(first[1]) else ""
|
|
here: str | None = None # where a relative path is, when it can name the answer file
|
|
if cd and inside(cd, basis):
|
|
own_only, chain = False, segments[1:]
|
|
elif own and cd and os.path.normpath(cd) == os.path.dirname(own):
|
|
own_only, chain, here = True, segments[1:], os.path.dirname(own)
|
|
elif own and first and first[0] in READ_ONLY + OWN_ANSWER:
|
|
own_only, chain = True, segments
|
|
else:
|
|
return "Bash that does not start with cd into the basis"
|
|
for segment in chain:
|
|
if not segment or segment[0] not in READ_ONLY + LITERAL + OWN_ANSWER:
|
|
return f"Bash command not allowed: {segment[0] if segment else '(empty)'}"
|
|
words = search_words(segment[0], segment[1:])
|
|
names_own = False
|
|
checked: list[tuple[str, str]] = [] # (token, the value in it that is checked as a path)
|
|
for index, token in enumerate(segment[1:]):
|
|
if segment[0] == "sed" and (in_place(token) or token.startswith("--in-place")):
|
|
return "Bash sed -i"
|
|
if index in words:
|
|
if segment[0] == "sed":
|
|
breach, reads = sed_breach(word_text("sed", token))
|
|
if breach:
|
|
return breach
|
|
checked.extend((name, name) for name in reads)
|
|
continue
|
|
if segment[0] == "find" and token in FIND_ACTIONS:
|
|
return f"Bash find action {token}"
|
|
if token.startswith("--") and token[2:].partition("=")[0] in RUNS_A_PROGRAM.get(segment[0], ()):
|
|
return f"Bash {segment[0]} {token.partition('=')[0]} runs a program"
|
|
checked.append((token, token.split("=", 1)[1] if token.startswith("-") and "=" in token else token))
|
|
for token, value in checked:
|
|
if value.startswith("~") or ".." in value.split("/"):
|
|
return f"Bash path outside the basis: {token}"
|
|
path = value if os.path.isabs(value) else os.path.join(here, value) if here else ""
|
|
if own and path and os.path.normpath(path) == own:
|
|
names_own = True
|
|
elif own_only and (not token.startswith("-") or "=" in token):
|
|
return f"Bash path other than the own answer: {token}"
|
|
elif value.startswith("/") and value != "/dev/null" and not inside(value, basis):
|
|
return f"Bash path outside the basis: {token}"
|
|
if own_only and not names_own:
|
|
return "Bash outside the basis that does not name the own answer"
|
|
if segment[0] in OWN_ANSWER and not names_own:
|
|
return f"Bash command not allowed: {segment[0]}"
|
|
return None
|
|
|
|
|
|
def breaches_of(tools: list[tuple[str, dict[str, Any]]], basis: Path, task: Path, answer: Path) -> list[str]:
|
|
"""K5 point 3: every tool call that reads or lists a path outside the basis and the task file, every tool that
|
|
is not allowed, and every write to another file than the answer."""
|
|
found = []
|
|
for name, inp in tools:
|
|
if name not in ALLOWED_TOOLS:
|
|
found.append(f"{name}: tool not allowed")
|
|
elif name == "Read":
|
|
path = inp.get("file_path")
|
|
if not (inside(path, basis)
|
|
or (isinstance(path, str) and os.path.normpath(path) in (str(task), str(answer)))):
|
|
found.append(f"Read outside the basis: {path}")
|
|
elif name in ("Grep", "Glob"):
|
|
path = inp.get("path")
|
|
pattern = inp.get("pattern", "")
|
|
if path is None:
|
|
found.append(f"{name} without a path (searches the working directory)")
|
|
elif not inside(path, basis):
|
|
found.append(f"{name} outside the basis: {path}")
|
|
elif name == "Glob" and isinstance(pattern, str) and os.path.isabs(pattern) and not inside(pattern, basis):
|
|
found.append(f"Glob pattern outside the basis: {pattern}")
|
|
elif name == "Bash":
|
|
breach = bash_breach(str(inp.get("command", "")), basis, answer)
|
|
if breach:
|
|
found.append(breach)
|
|
elif name == "Write" and inp.get("file_path") != str(answer):
|
|
found.append(f"Write to another file: {inp.get('file_path')}")
|
|
return found
|
|
|
|
|
|
def _injected_chars(line: dict[str, Any]) -> int:
|
|
"""Text put into the context between two assistant messages: tool results and rendered attachments."""
|
|
if line.get("type") == "attachment":
|
|
return len(str(line.get("rendered") or ""))
|
|
content = line.get("message", {}).get("content") if isinstance(line.get("message"), dict) else None
|
|
total = 0
|
|
for item in content if isinstance(content, list) else []:
|
|
if isinstance(item, dict) and item.get("type") == "tool_result":
|
|
inner = item.get("content")
|
|
if isinstance(inner, str):
|
|
total += len(inner)
|
|
elif isinstance(inner, list):
|
|
total += sum(len(str(x.get("text", ""))) for x in inner if isinstance(x, dict))
|
|
return total
|
|
|
|
|
|
def usage_and_tools(lines: list[dict[str, Any]]) -> tuple[dict[str, int], list[tuple[str, dict[str, Any]]]]:
|
|
"""The four usage fields summed with each message id once at its largest value, the derived output (a message
|
|
whose records all lack a stop reason never got its final usage written: its output is the growth of the next
|
|
message's input side less the text injected between them, counted high), and every tool call."""
|
|
per: dict[str, dict[str, int]] = {}
|
|
order: list[str] = []
|
|
complete: set[str] = set()
|
|
between: dict[str, int] = {}
|
|
tools: list[tuple[str, dict[str, Any]]] = []
|
|
for line in lines:
|
|
message = line.get("message")
|
|
if line.get("type") != "assistant" or not isinstance(message, dict):
|
|
if order:
|
|
between[order[-1]] = between.get(order[-1], 0) + _injected_chars(line)
|
|
continue
|
|
mid = str(message.get("id", ""))
|
|
if mid not in per:
|
|
order.append(mid)
|
|
seen = per.setdefault(mid, {})
|
|
fields = message.get("usage") or {}
|
|
for f in USAGE_FIELDS:
|
|
value = fields.get(f)
|
|
if isinstance(value, int):
|
|
seen[f] = max(seen.get(f, 0), value)
|
|
if isinstance(message.get("stop_reason"), str):
|
|
complete.add(mid)
|
|
for block in message.get("content") or []:
|
|
if isinstance(block, dict) and block.get("type") == "tool_use":
|
|
inp = block.get("input")
|
|
tools.append((str(block.get("name")), inp if isinstance(inp, dict) else {}))
|
|
out = {f: sum(v.get(f, 0) for v in per.values()) for f in USAGE_FIELDS}
|
|
derived = 0
|
|
for i, mid in enumerate(order[:-1]):
|
|
if mid in complete:
|
|
continue
|
|
growth = (sum(per[order[i + 1]].get(f, 0) for f in INPUT_SIDE) - sum(per[mid].get(f, 0) for f in INPUT_SIDE)
|
|
- int(between.get(mid, 0) / CHARS_PER_TOKEN_LOW))
|
|
derived += max(0, growth - per[mid].get("output_tokens", 0))
|
|
out["output_tokens_derived"] = derived
|
|
return out, tools
|
|
|
|
|
|
def harvest(run: Path, name: str, arm: str, transcripts: Path) -> tuple[dict[str, Any], list[str]]:
|
|
"""Per task of the arm: tokens over all its runs and every isolation breach. Matched by the subagent's
|
|
description S1-<run>-<arm>-<id> (the .meta.json beside the transcript); a transcript with no description is
|
|
matched by the task path in its first prompt, one with another description is not this arm's. Returns the K5 tokens document and the tasks no transcript was found for."""
|
|
if arm not in ARMS:
|
|
raise RunError(f"unknown arm {arm!r}; the arms are {', '.join(ARMS)}")
|
|
if not transcripts.is_dir():
|
|
raise RunError(f"{transcripts} is not a directory")
|
|
tasks_dir = run / "tasks" / arm
|
|
ids = sorted(p.stem for p in tasks_dir.glob("*.md"))
|
|
if not ids:
|
|
raise RunError(f"no tasks in {tasks_dir}; run setup first")
|
|
described = re.compile(rf"^S1-{re.escape(name)}-{re.escape(arm)}-(\S+)$")
|
|
prompted = re.compile(re.escape(str(tasks_dir)) + r"/([A-Za-z0-9]+)\.md")
|
|
basis = run / "arms" / arm
|
|
rows: dict[str, dict[str, Any]] = {}
|
|
for path in sorted(transcripts.rglob("*.jsonl")):
|
|
meta_path = path.with_name(path.stem + ".meta.json")
|
|
description = ""
|
|
if meta_path.is_file():
|
|
meta = load_json(meta_path, "agent meta")
|
|
description = str(meta.get("description", "")) if isinstance(meta, dict) else ""
|
|
lines = read_transcript(path)
|
|
m = described.match(description) if description else prompted.search(first_prompt(lines))
|
|
if m is None or m.group(1) not in ids:
|
|
continue
|
|
qid = m.group(1)
|
|
counts, tools = usage_and_tools(lines)
|
|
breaches = breaches_of(tools, basis, tasks_dir / f"{qid}.md", run / "answers" / arm / f"{qid}.json")
|
|
row = rows.setdefault(qid, {"id": qid, **{f: 0 for f in counts}, "isolation_breaches": 0,
|
|
"breach_detail": [], "runs": 0})
|
|
for f, value in counts.items():
|
|
row[f] += value
|
|
row["breach_detail"] += breaches
|
|
row["isolation_breaches"] = len(row["breach_detail"])
|
|
row["runs"] += 1
|
|
doc = {"schema": TOKENS_SCHEMA, "arm": arm, "questions": [rows[i] for i in sorted(rows)]}
|
|
return doc, [i for i in ids if i not in rows]
|
|
|
|
|
|
# ---------------------------------------------------------------------------------------------- score
|
|
|
|
|
|
def mcnemar_exact(only_a: int, only_b: int) -> float:
|
|
"""Two-sided exact McNemar: the binomial test over the discordant pairs, p = 0.5."""
|
|
n, k = only_a + only_b, min(only_a, only_b)
|
|
if n == 0:
|
|
return 1.0
|
|
return min(1.0, 2 * sum(comb(n, i) for i in range(k + 1)) / float(2 ** n))
|
|
|
|
|
|
def wilson(x: int, n: int, z: float) -> tuple[float, float]:
|
|
centre = (x + z * z / 2) / (n + z * z)
|
|
half = z * sqrt(x * (n - x) / n + z * z / 4) / (n + z * z)
|
|
return centre - half, centre + half
|
|
|
|
|
|
def paired_difference_ci(e: int, f: int, g: int, h: int, level: float = 0.95) -> tuple[float, float]:
|
|
"""The interval for (e + f)/n - (e + g)/n from paired data: e both, f first only, g second only, h neither.
|
|
Newcombe (1998), Statist. Med. 17:2635-2650, method 10: Wilson score limits for each margin combined with the
|
|
phi coefficient, its numerator max(eh - fg - n/2, 0) when eh > fg, and phi 0 when a margin is 0."""
|
|
n = e + f + g + h
|
|
if n == 0:
|
|
raise RunError("no pairs")
|
|
z = NormalDist().inv_cdf(1 - (1 - level) / 2)
|
|
p1, p2 = (e + f) / n, (e + g) / n
|
|
l1, u1 = wilson(e + f, n, z)
|
|
l2, u2 = wilson(e + g, n, z)
|
|
denominator = (e + f) * (g + h) * (e + g) * (f + h)
|
|
numerator = float(e * h - f * g)
|
|
if numerator > 0:
|
|
numerator = max(numerator - n / 2, 0.0)
|
|
phi = numerator / sqrt(denominator) if denominator else 0.0
|
|
dl1, du1, dl2, du2 = p1 - l1, u1 - p1, p2 - l2, u2 - p2
|
|
low = sqrt(max(0.0, dl1 * dl1 - 2 * phi * dl1 * du2 + du2 * du2))
|
|
high = sqrt(max(0.0, du1 * du1 - 2 * phi * du1 * dl2 + dl2 * dl2))
|
|
return p1 - p2 - low, p1 - p2 + high
|
|
|
|
|
|
def min_detectable(n: int, discordant_share: float, alpha: float = ALPHA, power: float = POWER) -> float | None:
|
|
"""The smallest difference between paired proportions that n pairs show at the given power with a two-sided
|
|
McNemar test, for a share of discordant pairs psi (Connor 1987: n = (z_a sqrt(psi) + z_b sqrt(psi - d^2))^2 /
|
|
d^2, solved for d by bisection). None when psi is 0 or no d up to psi is visible."""
|
|
if n <= 0 or discordant_share <= 0:
|
|
return None
|
|
za, zb = NormalDist().inv_cdf(1 - alpha / 2), NormalDist().inv_cdf(power)
|
|
psi = min(discordant_share, 1.0)
|
|
|
|
def needed(d: float) -> float:
|
|
return (za * sqrt(psi) + zb * sqrt(max(0.0, psi - d * d))) ** 2 / (d * d)
|
|
|
|
if needed(psi) > n:
|
|
return None
|
|
low, high = 1e-9, psi
|
|
for _ in range(200):
|
|
mid = (low + high) / 2
|
|
if needed(mid) > n:
|
|
low = mid
|
|
else:
|
|
high = mid
|
|
return high
|
|
|
|
|
|
def load_results(path: Path) -> dict[str, Any]:
|
|
data = load_json(path, "results")
|
|
if not isinstance(data, dict) or data.get("schema") != RESULTS_SCHEMA or not isinstance(data.get("results"), list):
|
|
raise RunError(f"{path} is not a results file with schema {RESULTS_SCHEMA}")
|
|
if data.get("arm") not in ARMS:
|
|
raise RunError(f"{path} is for arm {data.get('arm')!r}; the arms are {', '.join(ARMS)}")
|
|
return data
|
|
|
|
|
|
def path_band(paths: int) -> str:
|
|
return "4+" if paths >= MANY_PATHS else "2-3"
|
|
|
|
|
|
def score(results: list[dict[str, Any]], tokens: list[dict[str, Any]]) -> dict[str, Any]:
|
|
by_arm = {r["arm"]: r for r in results}
|
|
if sorted(by_arm) != sorted(ARMS) or len(results) != len(ARMS):
|
|
raise RunError(f"score needs one results file for each arm: {', '.join(ARMS)}")
|
|
a, b = by_arm["filsok"], by_arm["bundle"]
|
|
if a.get("set_sha256") != b.get("set_sha256"):
|
|
raise RunError("the two results files are for different sets")
|
|
rows = {arm: {str(r["id"]): r for r in by_arm[arm]["results"]} for arm in ARMS}
|
|
if sorted(rows["filsok"]) != sorted(rows["bundle"]):
|
|
raise RunError("the two results files hold different questions")
|
|
ids = sorted(rows["filsok"])
|
|
doc: dict[str, Any] = {"schema": SCORE_SCHEMA, "set_sha256": a.get("set_sha256"), "questions": len(ids),
|
|
"arms": {}, "pairs": {}}
|
|
for arm in ARMS:
|
|
groups = {g: [sum(1 for i in ids if rows[arm][i]["group"] == g and rows[arm][i]["correct"] is True),
|
|
sum(1 for i in ids if rows[arm][i]["group"] == g)] for g in GROUPS}
|
|
bands = {band: [sum(1 for i in ids if path_band(int(rows[arm][i]["paths_needed"])) == band
|
|
and rows[arm][i]["correct"] is True),
|
|
sum(1 for i in ids if path_band(int(rows[arm][i]["paths_needed"])) == band)]
|
|
for band in ("2-3", "4+")}
|
|
doc["arms"][arm] = {"groups": groups, "paths": bands,
|
|
"all": [sum(1 for i in ids if rows[arm][i]["correct"] is True), len(ids)]}
|
|
for label, members in [("all", ids)] + [(g, [i for i in ids if rows["filsok"][i]["group"] == g]) for g in GROUPS]:
|
|
counts = {"both": 0, "only filsok": 0, "only bundle": 0, "neither": 0}
|
|
for i in members:
|
|
fa, bu = rows["filsok"][i]["correct"] is True, rows["bundle"][i]["correct"] is True
|
|
counts["both" if fa and bu else "only filsok" if fa else "only bundle" if bu else "neither"] += 1
|
|
doc["pairs"][label] = counts
|
|
pairs = doc["pairs"]["all"]
|
|
doc["mcnemar_p"] = mcnemar_exact(pairs["only filsok"], pairs["only bundle"])
|
|
n = len(ids)
|
|
if n:
|
|
low, high = paired_difference_ci(pairs["both"], pairs["only bundle"], pairs["only filsok"], pairs["neither"])
|
|
doc["difference"] = {"of": "bundle - filsok", "point": (pairs["only bundle"] - pairs["only filsok"]) / n,
|
|
"low": low, "high": high, "level": 0.95, "method": "Newcombe 1998 method 10"}
|
|
share = (pairs["only filsok"] + pairs["only bundle"]) / n
|
|
doc["smallest_visible"] = {"discordant_share": share, "difference": min_detectable(n, share),
|
|
"power": POWER, "alpha": ALPHA}
|
|
doc["tokens"] = {}
|
|
for t in tokens:
|
|
if not isinstance(t, dict) or t.get("schema") != TOKENS_SCHEMA or t.get("arm") not in ARMS:
|
|
raise RunError(f"a tokens file is not {TOKENS_SCHEMA} for one of the arms")
|
|
qs = as_list(t.get("questions"))
|
|
total = sum(int(q.get(f, 0)) for q in qs for f in (*USAGE_FIELDS, "output_tokens_derived"))
|
|
doc["tokens"][t["arm"]] = {"questions": len(qs), "per_question": total / len(qs) if qs else None,
|
|
"isolation_breaches": sum(int(q.get("isolation_breaches", 0)) for q in qs)}
|
|
return doc
|
|
|
|
|
|
def render(doc: dict[str, Any]) -> list[str]:
|
|
lines = []
|
|
for arm in ARMS:
|
|
row = doc["arms"][arm]
|
|
lines.append(f"{arm}: " + ", ".join(f"{g} {c} of {n}" for g, (c, n) in row["groups"].items())
|
|
+ f", all {row['all'][0]} of {row['all'][1]}")
|
|
for arm in ARMS:
|
|
lines.append(f"{arm} by paths: " + ", ".join(f"{b} {c} of {n}" for b, (c, n) in doc["arms"][arm]["paths"].items()))
|
|
for label, counts in doc["pairs"].items():
|
|
lines.append(("pairs" if label == "all" else f"pairs {label}") + ": "
|
|
+ ", ".join(f"{k} {v}" for k, v in counts.items()))
|
|
lines.append(f"exact McNemar p = {doc['mcnemar_p']:.3g}")
|
|
if "difference" in doc:
|
|
d = doc["difference"]
|
|
lines.append(f"bundle - filsok = {d['point']:.3f}, 95 % CI {d['low']:.3f} to {d['high']:.3f} "
|
|
f"(paired, Newcombe 1998 method 10)")
|
|
s = doc["smallest_visible"]
|
|
visible = "none" if s["difference"] is None else f"{s['difference']:.3f}"
|
|
lines.append(f"smallest difference visible at 80 % power, alpha 0.05: {visible} "
|
|
f"(discordant share measured {s['discordant_share']:.3f}, N = {doc['questions']})")
|
|
for arm, t in doc["tokens"].items():
|
|
per = "none" if t["per_question"] is None else f"{t['per_question']:.0f}"
|
|
lines.append(f"{arm} tokens: {t['questions']} questions harvested, {per} per question, "
|
|
f"isolation breaches {t['isolation_breaches']}")
|
|
return lines
|
|
|
|
|
|
# ---------------------------------------------------------------------------------------------- main
|
|
|
|
|
|
def build_parser() -> argparse.ArgumentParser:
|
|
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
sub = p.add_subparsers(dest="command", required=True)
|
|
s = sub.add_parser("setup", help="one arm's basis and task files")
|
|
s.add_argument("--run", required=True)
|
|
s.add_argument("--name", required=True, help="the run's name, letters and digits (S1-<name>-<arm>-<id>)")
|
|
s.add_argument("--arm", required=True)
|
|
s.add_argument("--set", required=True)
|
|
s.add_argument("--sources", required=True, help=f"the K1 root measured (its kilde/ files and {K1_FILE})")
|
|
s.add_argument("--bundle", help="the collection, for the bundle arm")
|
|
s.add_argument("--part-root", help="the K1 root the set was made from, when it is a subset_of --sources: the arm "
|
|
"searches the whole of --sources")
|
|
c = sub.add_parser("collect", help="the answer files into the arm's answers file")
|
|
c.add_argument("--run", required=True)
|
|
c.add_argument("--arm", required=True)
|
|
h = sub.add_parser("harvest", help="tokens and isolation breaches per question from the transcripts")
|
|
h.add_argument("--run", required=True)
|
|
h.add_argument("--name", required=True)
|
|
h.add_argument("--arm", required=True)
|
|
h.add_argument("--transcripts", required=True, help="a directory searched for *.jsonl at any depth")
|
|
r = sub.add_parser("score", help="two arms' results: groups, paths, pairs, McNemar, interval, power")
|
|
r.add_argument("--results", action="append", required=True, help="answerset.py gate --out, once per arm")
|
|
r.add_argument("--tokens", action="append", default=[], help="harvest output, once per arm (optional)")
|
|
r.add_argument("--out", default="")
|
|
return p
|
|
|
|
|
|
def main(argv: list[str] | None = None) -> int:
|
|
args = build_parser().parse_args(argv)
|
|
try:
|
|
if args.command == "setup":
|
|
run = Path(args.run).resolve()
|
|
tasks = setup_arm(run, args.name, args.arm, Path(args.set), Path(args.sources),
|
|
Path(args.bundle).resolve() if args.bundle else None,
|
|
Path(args.part_root) if args.part_root else None)
|
|
for description, task in tasks:
|
|
print(f"{description}\t{task}")
|
|
return 0
|
|
if args.command == "collect":
|
|
out, problems = collect(Path(args.run).resolve(), args.arm)
|
|
print(out)
|
|
for problem in problems:
|
|
print(f"problem: {problem}")
|
|
return 1 if problems else 0
|
|
if args.command == "harvest":
|
|
run = Path(args.run).resolve()
|
|
doc, missing = harvest(run, args.name, args.arm, Path(args.transcripts))
|
|
write(run / "tokens" / f"{args.arm}.json", doc)
|
|
breaches = sum(q["isolation_breaches"] for q in doc["questions"])
|
|
print(f"tasks harvested: {len(doc['questions'])}; breaches: {breaches}")
|
|
if missing:
|
|
print(f"tasks without a transcript: {len(missing)}: {', '.join(missing)}")
|
|
return 1 if missing else 0
|
|
doc = score([load_results(Path(p)) for p in args.results],
|
|
[load_json(Path(p), "tokens") for p in args.tokens])
|
|
print("\n".join(render(doc)))
|
|
if args.out:
|
|
write(Path(args.out), doc)
|
|
return 0
|
|
except RunError as exc:
|
|
print(f"FAIL: {exc}", file=sys.stderr)
|
|
return 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|