"""The answer harness for PLAN.md criterion 4 (contract K5 points 2-7), adapted from evals/direction_probe.py. Each question is answered by one isolated subagent per arm in the user's session, described `S1---`. This module never calls a model and starts nothing: setup one arm's basis (its own copy of the root's source files, and for `bundle` the collection beside them) and one task file per question, `/tasks//.md`, which carries the question and the basis and never the answer keys, the spans or the number of documents collect the per-question answer files into the arm's answers file (K5 point 4) harvest tokens per question (each message id once, all four usage fields, plus a lower bound for an output the transcript never recorded) and isolation breaches from the subagent transcripts (K5 point 6) score two arms' results from `answerset.py gate --out`: per arm and group, by path count, the pairs with the exact McNemar p, the difference with its paired 95 % interval (Newcombe 1998, method 10), and the smallest difference the set can show at 80 % power and alpha 0.05 (K5 point 7) The arms are `filsok` (file search alone over the root's source files) and `bundle` (the collection plus file search over the same files). Exit 0 = done; 1 = could not run, or (collect, harvest) something is missing. """ from __future__ import annotations import argparse import json import os import re import shlex import shutil import sys from math import comb, sqrt from pathlib import Path from statistics import NormalDist from typing import Any from answerset import ( ANSWERS_SCHEMA, ARMS, GROUPS, K1_FILE, MANY_PATHS, RESULTS_SCHEMA, Root, RunError, as_list, dump, load_json, load_set, ) TOKENS_SCHEMA = "okf-wiki/answer-tokens/1" SCORE_SCHEMA = "okf-wiki/answer-score/1" NAME_RE = re.compile(r"^[A-Za-z0-9]+$") ALLOWED_TOOLS = ("Read", "Grep", "Glob", "Bash", "Write", "SubagentHandback", "StructuredOutput", "ToolSearch") READ_ONLY = ("grep", "egrep", "fgrep", "rg", "find", "ls", "wc", "head", "tail", "sed", "cat", "sort", "uniq", "cut", "tr", "nl") LITERAL = ("echo",) # accepted in a chain (order 31 A2: order 13's separators), never offered in the task OWN_ANSWER = ("jq",) # accepted on the helper's own answer file only (order 31 A2b), never offered in the task SEPARATORS = ("|", "&&", ";", "||") FIND_ACTIONS = ("-exec", "-execdir", "-ok", "-okdir", "-delete", "-fprint", "-fprint0", "-fprintf", "-fls") # Order 31 A2b: the search word of grep and rg, the script of sed and the filter of jq are not paths. Per command: the # short options that take a value, those of them whose value is the search word, those whose value is a file of # search words, the long ones of each, and (order 39) every other long option that takes a value, GNU and BSD. Such a # value is consumed and checked as a path, so a file option's value outside the basis (--exclude-from, --ignore-file) # is a breach. An option missing here makes its value the search word, and when that value is a path the path goes # unchecked: the lists must be whole. grep's --context is left out because BSD grep takes its value only after `=` # (`--context 1 /x f` searches for 1 in /x), so a separate value stays the search word and the check stays stricter. GREP_LONG = ("after-context", "before-context", "binary-files", "devices", "directories", "exclude", "exclude-dir", "exclude-from", "group-separator", "include", "include-dir", "label", "max-count") RG_LONG = ("after-context", "before-context", "color", "colors", "context", "context-separator", "dfa-size-limit", "encoding", "engine", "field-context-separator", "field-match-separator", "generate", "glob", "iglob", "ignore-file", "hostname-bin", "hyperlink-format", "max-columns", "max-count", "max-depth", "max-filesize", "path-separator", "pre", "pre-glob", "regex-size-limit", "replace", "sort", "sortr", "threads", "type", "type-add", "type-clear", "type-not") SEARCH_WORDS: dict[str, tuple[str, str, str, tuple[str, ...], tuple[str, ...], tuple[str, ...]]] = { **{grep: ("ABCDdefm", "e", "f", ("regexp",), ("file",), GREP_LONG) for grep in ("grep", "egrep", "fgrep")}, "rg": ("ABCEMTdefgjmrt", "e", "f", ("regexp",), ("file",), RG_LONG), "sed": ("efl", "e", "f", ("expression",), ("file",), ("line-length",)), "jq": ("", "", "f", (), ("from-file",), ()), # the filter; -f reads it from the first file instead } RUNS_A_PROGRAM = {"rg": ("pre", "hostname-bin")} # long options whose value is a program the command runs # Order 39: a sed script is exempt from the path check, so its commands are read instead. w, W and e (also as flags # of s) write a file or run a program: a breach. r and R read the file named after them: checked as a path. SED_IN_PLACE = "iI" # GNU -i[SUFFIX], BSD -i SUFFIX and -I SUFFIX: either one in a flag cluster edits in place SED_ADDRESS = re.compile(r"\d+(~\d+)?|\$") SED_PLAIN = set("=dDgGhHlnNpPxzFqQ") # commands without an argument (l, q and Q may take a number) SED_LABEL = set("btT:") # commands whose label runs to a semicolon or a newline SED_TEXT = set("aic") # commands whose text runs to the end of the line SED_S_FLAGS = set("gpiImM0123456789 \t") USAGE_FIELDS = ( "input_tokens", # uncached input "cache_creation_input_tokens", # written to the prompt cache "cache_read_input_tokens", # read from it "output_tokens", ) INPUT_SIDE = USAGE_FIELDS[:3] CHARS_PER_TOKEN_LOW = 2.5 # injected text counted high, so a derived output is a lower bound (okfwiki/cost.py) ALPHA, POWER = 0.05, 0.80 def write(path: Path, data: Any) -> None: path.parent.mkdir(parents=True, exist_ok=True) path.write_bytes(dump(data)) # ---------------------------------------------------------------------------------------------- setup SEARCH = { "filsok": """BASIS holds the source files under `kilde/` and nothing else. Search them with grep and find (Bash, as above), and read what you find with Read.""", "bundle": """BASIS holds `wiki/`, a knowledge collection compiled from the sources, and `kilde/`, the source files. A page names its source as `/kilde/`, which is the file `BASIS/kilde/`. Follow this order: 1. Search `BASIS/wiki/index.md` with grep for the question's words: it is never read whole, because it can be over the size one Read allows. Each line is one page: its title and its description. 2. Open the main page of the area the question is about: a main page carries `x-main-for` with the source it covers. 3. Read it, follow its links to the pages the question needs, and go on from those pages where the question needs it. 4. Open the source behind a footnote when the quote has to be checked or when the page does not say enough. You may also search `wiki/` and `kilde/` with grep and find (Bash, as above) at any point.""", } def render_task(arm: str, basis: Path, qid: str, question: str, answer_path: Path) -> str: return f"""# Answer one question BASIS: `{basis}` You answer ONE question from the files in BASIS. Read nothing outside BASIS apart from this task file: no other folder, no web, no earlier work. Answer only from what the files say, and say plainly when they do not say something. Tools: Read (a file inside BASIS), Bash for read-only search, and Write once, for the answer file. No other tool. Every Bash command starts with `cd {basis} && ` and then uses only {', '.join(READ_ONLY)}, joined by `|` or `&&`, with paths relative to BASIS: never `..`, `~` or a path outside BASIS, no `$` expansion, no output redirection except `2>/dev/null`, no `find -exec` or `-delete`, no `sed -i`. For example: `cd {basis} && grep -rn "term" kilde` ## How to search {SEARCH[arm]} ## The question {question} ## The answer Write one JSON file to `{answer_path}`: ```json {{"id": "{qid}", "answer": "", "citations": [{{"path": "kilde/", "quote": ""}}]}} ``` - `answer`: answer every part of the question. - `citations`: one or more per source file your answer relies on, and cite every such file. - `path`: `kilde/` and the file name, as the file lies in BASIS, for example `kilde/notes.md`: never a page in `wiki/`. - `quote`: text copied character for character from that file (only whitespace may differ), long enough to carry the fact, for example the whole line or sentence. When the file is written, reply with the one word `done`. """ def copy_sources(root: Root, target: Path) -> int: """Every source K1 lists, copied byte-exact under target/kilde/, each checked against its K1 sha256.""" for path in sorted(root.entries): text = root.text(path) if text is None: raise RunError(root.problem(path)) dest = target / path dest.parent.mkdir(parents=True, exist_ok=True) shutil.copyfile(root.path / path, dest) return len(root.entries) def check_part(ref: dict[str, Any], root: Root, part_path: Path) -> None: """Order 31 A3: the set names the part's K1, the part is a subset_of the root, and every source of the part stands in the root with the same path and sha256.""" part = Root(part_path) if ref.get("manifest_sha256") != part.manifest_sha256: raise RunError(f"the set names manifest_sha256 {ref.get('manifest_sha256')!r}, --part-root {part_path} has " f"{part.manifest_sha256}; --part-root is the root the set was made from") raw = json.loads(part.manifest_bytes.decode("utf-8")).get("subset_of") named = raw.get("manifest_sha256") if isinstance(raw, dict) else None if named != root.manifest_sha256: raise RunError(f"--part-root {part_path} names subset_of manifest_sha256 {named!r}, the root has " f"{root.manifest_sha256}") for path, entry in sorted(part.entries.items()): if path not in root.entries: raise RunError(f"{path} of --part-root is not in the root {root.path}") if root.entries[path]["sha256"] != entry["sha256"]: raise RunError(f"{path} has sha256 {entry['sha256']} in --part-root and {root.entries[path]['sha256']} " f"in the root") def setup_arm(run: Path, name: str, arm: str, set_path: Path, sources: Path, bundle: Path | None, part_root: Path | None = None) -> list[tuple[str, Path]]: if arm not in ARMS: raise RunError(f"unknown arm {arm!r}; the arms are {', '.join(ARMS)}") if not NAME_RE.match(name): raise RunError(f"run name {name!r} must be letters and digits only (it is part of S1---)") if arm == "bundle" and bundle is None: raise RunError("the bundle arm needs --bundle") if arm == "bundle" and bundle is not None and not (bundle / "index.md").is_file(): raise RunError(f"{bundle} has no index.md") data = load_set(set_path) root = Root(sources) ref = data["sources"] if part_root is not None: check_part(ref, root, part_root) elif ref.get("manifest_sha256") != root.manifest_sha256: raise RunError(f"the set names manifest_sha256 {ref.get('manifest_sha256')!r}, the root has " f"{root.manifest_sha256}; an arm is set up on the root the set was made from") basis = run / "arms" / arm if basis.exists(): raise RunError(f"{basis} exists already; an arm is set up once, from fresh") if arm == "bundle" and bundle is not None: shutil.copytree(bundle, basis / "wiki") copy_sources(root, basis) tasks = [] for q in data["questions"]: qid = str(q["id"]) task = run / "tasks" / arm / f"{qid}.md" answer = run / "answers" / arm / f"{qid}.json" task.parent.mkdir(parents=True, exist_ok=True) task.write_text(render_task(arm, basis, qid, str(q["question"]), answer), encoding="utf-8") tasks.append((f"S1-{name}-{arm}-{qid}", task)) (run / "answers" / arm).mkdir(parents=True, exist_ok=True) return tasks # ---------------------------------------------------------------------------------------------- collect def collect(run: Path, arm: str) -> tuple[Path, list[str]]: """The per-question answers into one answers file; the id comes from the file name.""" if arm not in ARMS: raise RunError(f"unknown arm {arm!r}; the arms are {', '.join(ARMS)}") folder = run / "answers" / arm if not folder.is_dir(): raise RunError(f"{folder} does not exist; run setup first") gathered: list[dict[str, Any]] = [] problems = [] for path in sorted(folder.glob("*.json")): try: data = json.loads(path.read_text(encoding="utf-8")) except (OSError, UnicodeDecodeError, json.JSONDecodeError): data = None if not isinstance(data, dict): problems.append(f"{path.name}: not a JSON object") continue gathered.append({**data, "id": path.stem}) out = run / "answers" / f"{arm}.json" write(out, {"schema": ANSWERS_SCHEMA, "arm": arm, "answers": gathered}) return out, problems # ---------------------------------------------------------------------------------------------- harvest def read_transcript(path: Path) -> list[dict[str, Any]]: lines = [] for raw in path.read_text(encoding="utf-8").splitlines(): try: line = json.loads(raw) except json.JSONDecodeError: continue if isinstance(line, dict): lines.append(line) return lines def first_prompt(lines: list[dict[str, Any]]) -> str: for line in lines: if line.get("type") == "user": content = line.get("message", {}).get("content") if isinstance(content, list): content = " ".join(str(b.get("text", "")) for b in content if isinstance(b, dict)) return content if isinstance(content, str) else "" return "" def inside(path: object, root: Path) -> bool: if not isinstance(path, str) or not os.path.isabs(path): return False norm = Path(os.path.normpath(path)) return norm == root or root in norm.parents def outside_single_quotes(command: str) -> str: """The command with the text inside single quotes blanked out, where the shell takes every character literally (order 31 A2b). A backslash outside them keeps the next character, so an escaped backtick still shows.""" out: list[str] = [] quote = "" i = 0 while i < len(command): ch = command[i] if quote == "'": quote = "" if ch == "'" else quote out.append(" ") elif ch == "\\": out.append(command[i:i + 2]) i += 1 elif ch == "'" and not quote: quote = "'" out.append(" ") else: if ch == '"': quote = "" if quote else '"' out.append(ch) i += 1 return "".join(out) def search_words(command: str, args: list[str]) -> set[int]: """The indexes in args of the search words: the values of -e and --regexp (sed: --expression), or else the first argument that is not an option. With -f or --file the search words come from a file and none is positional, wherever the option stands (GNU getopt reorders the arguments). The value of any other option that takes one is skipped here, so it is checked as a path.""" if command not in SEARCH_WORDS: return set() valued, word_opts, file_opts, word_long, file_long, other_long = SEARCH_WORDS[command] words: set[int] = set() first: int | None = None positional, options = True, True i = 0 while i < len(args): arg = args[i] if options and arg == "--": options = False elif options and arg.startswith("--"): name, eq, _ = arg[2:].partition("=") if name in word_long + file_long: positional = False if name in word_long: words.add(i if eq else i + 1) if name in word_long + file_long + other_long: i += 0 if eq else 1 elif options and arg.startswith("-") and arg != "-": for j, ch in enumerate(arg[1:]): if ch in word_opts + file_opts: positional = False if ch in valued: attached = j + 2 < len(arg) if ch in word_opts: words.add(i if attached else i + 1) i += 0 if attached else 1 break elif first is None: first = i i += 1 if positional and first is not None: words.add(first) return words def word_text(command: str, token: str) -> str: """The search word or script a search-word token carries: the token, or the value attached to -e or --expression.""" if token.startswith("--"): return token.partition("=")[2] if token.startswith("-") and token != "-": valued = SEARCH_WORDS[command][0] for j, ch in enumerate(token[1:]): if ch in valued: return token[j + 2:] return token def in_place(token: str) -> bool: """A sed flag cluster with -i or -I before the first option that takes a value (`-ni`, `-Ei`, `-I`).""" if not token.startswith("-") or token.startswith("--"): return False for ch in token[1:]: if ch in SED_IN_PLACE: return True if ch in SEARCH_WORDS["sed"][0]: return False return False class SedParseError(ValueError): pass def sed_breach(script: str) -> tuple[str | None, list[str]]: """The breach of one sed script, if any (w, W or e as a command or as a flag of s, or a script that cannot be parsed), and the files it reads with r or R, which the caller checks as paths. POSIX, GNU and BSD commands.""" n = len(script) reads: list[str] = [] def regex_end(k: int, delim: str) -> int: while k < n and script[k] != delim: k += 2 if script[k] == "\\" else 1 if k >= n: raise SedParseError(script) return k + 1 def address(k: int) -> int: m = SED_ADDRESS.match(script, k) if m: return m.end() if k < n and script[k] in "/\\": delim = "/" if script[k] == "\\": if k + 1 >= n: raise SedParseError(script) k += 1 delim = script[k] k = regex_end(k + 1, delim) while k < n and script[k] in "IM": k += 1 return k def line_end(k: int) -> int: end = script.find("\n", k) return n if end < 0 else end depth, i = 0, 0 try: while i < n: if script[i] in " \t\n;": i += 1 continue if script[i] == "}": depth -= 1 if depth < 0: raise SedParseError(script) i += 1 continue start = i i = address(i) if start < i < n and script[i] == ",": i += 1 if i < n and script[i] in "+~": i += 1 while i < n and script[i].isdigit(): i += 1 else: i = address(i) while i < n and script[i] in " \t!": i += 1 if i >= n: raise SedParseError(script) cmd = script[i] i += 1 if cmd == "{": depth += 1 elif cmd in "wWe": return f"Bash sed command {cmd}", reads elif cmd in "rR": end = line_end(i) reads.append(script[i:end].strip()) i = end elif cmd in SED_TEXT: i = line_end(i) elif cmd in SED_LABEL: while i < n and script[i] not in ";\n": i += 1 elif cmd in SED_PLAIN: while i < n and script[i] in " \t0123456789": i += 1 elif cmd in "sy": if i >= n or script[i] in "\\\n": raise SedParseError(script) delim = script[i] i = regex_end(regex_end(i + 1, delim), delim) while cmd == "s" and i < n and script[i] not in ";\n}": if script[i] in "wWe": return f"Bash sed s flag {script[i]}", reads if script[i] not in SED_S_FLAGS: raise SedParseError(script) i += 1 else: raise SedParseError(script) except SedParseError: return "Bash sed script that cannot be parsed", reads if depth: return "Bash sed script that cannot be parsed", reads return None, reads def bash_breach(command: str, basis: Path, answer: Path | None = None) -> str | None: """None when the command is read-only search that starts with `cd &&` and stays inside the basis, where the only path outside it may be the answer file; or when it reads the answer file alone, by its absolute path or after `cd` into its folder (order 31 A2b).""" unquoted = outside_single_quotes(command) if "`" in unquoted or re.search(r"\$[A-Za-z_{(]", unquoted): return "Bash with an expansion" try: # Order 39: bash takes '#' as a comment only at the start of a word, so shlex's comment rule is off (a comment # is then checked as arguments, which is stricter); an unquoted newline separates commands as ';' does. lexer = shlex.shlex(re.sub(r"\s2>/dev/null(?=\s|$|[;|&])", "", command), posix=True, punctuation_chars="|&;<>()\n") lexer.whitespace_split = True lexer.commenters = "" lexer.whitespace = " \t\r" tokens = [piece for token in lexer for piece in (re.split(r"(\n)", token) if set(token) <= set("|&;<>()\n") else [token]) if piece] except ValueError: return "Bash that cannot be parsed" segments: list[list[str]] = [[]] newline = False # a newline ends a command; after a separator, on a blank line or at the end it changes nothing for token in tokens: if token == "\n": newline = bool(segments[-1]) continue if newline and token not in SEPARATORS: segments.append([]) newline = False if token in SEPARATORS: segments.append([]) elif token and set(token) <= set("&<>()"): return f"Bash with {token!r}" else: segments[-1].append(token) first = segments[0] own = os.path.normpath(answer) if answer is not None else "" cd = first[1] if len(first) == 2 and first[0] == "cd" and os.path.isabs(first[1]) else "" here: str | None = None # where a relative path is, when it can name the answer file if cd and inside(cd, basis): own_only, chain = False, segments[1:] elif own and cd and os.path.normpath(cd) == os.path.dirname(own): own_only, chain, here = True, segments[1:], os.path.dirname(own) elif own and first and first[0] in READ_ONLY + OWN_ANSWER: own_only, chain = True, segments else: return "Bash that does not start with cd into the basis" for segment in chain: if not segment or segment[0] not in READ_ONLY + LITERAL + OWN_ANSWER: return f"Bash command not allowed: {segment[0] if segment else '(empty)'}" words = search_words(segment[0], segment[1:]) names_own = False checked: list[tuple[str, str]] = [] # (token, the value in it that is checked as a path) for index, token in enumerate(segment[1:]): if segment[0] == "sed" and (in_place(token) or token.startswith("--in-place")): return "Bash sed -i" if index in words: if segment[0] == "sed": breach, reads = sed_breach(word_text("sed", token)) if breach: return breach checked.extend((name, name) for name in reads) continue if segment[0] == "find" and token in FIND_ACTIONS: return f"Bash find action {token}" if token.startswith("--") and token[2:].partition("=")[0] in RUNS_A_PROGRAM.get(segment[0], ()): return f"Bash {segment[0]} {token.partition('=')[0]} runs a program" checked.append((token, token.split("=", 1)[1] if token.startswith("-") and "=" in token else token)) for token, value in checked: if value.startswith("~") or ".." in value.split("/"): return f"Bash path outside the basis: {token}" path = value if os.path.isabs(value) else os.path.join(here, value) if here else "" if own and path and os.path.normpath(path) == own: names_own = True elif own_only and (not token.startswith("-") or "=" in token): return f"Bash path other than the own answer: {token}" elif value.startswith("/") and value != "/dev/null" and not inside(value, basis): return f"Bash path outside the basis: {token}" if own_only and not names_own: return "Bash outside the basis that does not name the own answer" if segment[0] in OWN_ANSWER and not names_own: return f"Bash command not allowed: {segment[0]}" return None def breaches_of(tools: list[tuple[str, dict[str, Any]]], basis: Path, task: Path, answer: Path) -> list[str]: """K5 point 3: every tool call that reads or lists a path outside the basis and the task file, every tool that is not allowed, and every write to another file than the answer.""" found = [] for name, inp in tools: if name not in ALLOWED_TOOLS: found.append(f"{name}: tool not allowed") elif name == "Read": path = inp.get("file_path") if not (inside(path, basis) or (isinstance(path, str) and os.path.normpath(path) in (str(task), str(answer)))): found.append(f"Read outside the basis: {path}") elif name in ("Grep", "Glob"): path = inp.get("path") pattern = inp.get("pattern", "") if path is None: found.append(f"{name} without a path (searches the working directory)") elif not inside(path, basis): found.append(f"{name} outside the basis: {path}") elif name == "Glob" and isinstance(pattern, str) and os.path.isabs(pattern) and not inside(pattern, basis): found.append(f"Glob pattern outside the basis: {pattern}") elif name == "Bash": breach = bash_breach(str(inp.get("command", "")), basis, answer) if breach: found.append(breach) elif name == "Write" and inp.get("file_path") != str(answer): found.append(f"Write to another file: {inp.get('file_path')}") return found def _injected_chars(line: dict[str, Any]) -> int: """Text put into the context between two assistant messages: tool results and rendered attachments.""" if line.get("type") == "attachment": return len(str(line.get("rendered") or "")) content = line.get("message", {}).get("content") if isinstance(line.get("message"), dict) else None total = 0 for item in content if isinstance(content, list) else []: if isinstance(item, dict) and item.get("type") == "tool_result": inner = item.get("content") if isinstance(inner, str): total += len(inner) elif isinstance(inner, list): total += sum(len(str(x.get("text", ""))) for x in inner if isinstance(x, dict)) return total def usage_and_tools(lines: list[dict[str, Any]]) -> tuple[dict[str, int], list[tuple[str, dict[str, Any]]]]: """The four usage fields summed with each message id once at its largest value, the derived output (a message whose records all lack a stop reason never got its final usage written: its output is the growth of the next message's input side less the text injected between them, counted high), and every tool call.""" per: dict[str, dict[str, int]] = {} order: list[str] = [] complete: set[str] = set() between: dict[str, int] = {} tools: list[tuple[str, dict[str, Any]]] = [] for line in lines: message = line.get("message") if line.get("type") != "assistant" or not isinstance(message, dict): if order: between[order[-1]] = between.get(order[-1], 0) + _injected_chars(line) continue mid = str(message.get("id", "")) if mid not in per: order.append(mid) seen = per.setdefault(mid, {}) fields = message.get("usage") or {} for f in USAGE_FIELDS: value = fields.get(f) if isinstance(value, int): seen[f] = max(seen.get(f, 0), value) if isinstance(message.get("stop_reason"), str): complete.add(mid) for block in message.get("content") or []: if isinstance(block, dict) and block.get("type") == "tool_use": inp = block.get("input") tools.append((str(block.get("name")), inp if isinstance(inp, dict) else {})) out = {f: sum(v.get(f, 0) for v in per.values()) for f in USAGE_FIELDS} derived = 0 for i, mid in enumerate(order[:-1]): if mid in complete: continue growth = (sum(per[order[i + 1]].get(f, 0) for f in INPUT_SIDE) - sum(per[mid].get(f, 0) for f in INPUT_SIDE) - int(between.get(mid, 0) / CHARS_PER_TOKEN_LOW)) derived += max(0, growth - per[mid].get("output_tokens", 0)) out["output_tokens_derived"] = derived return out, tools def harvest(run: Path, name: str, arm: str, transcripts: Path) -> tuple[dict[str, Any], list[str]]: """Per task of the arm: tokens over all its runs and every isolation breach. Matched by the subagent's description S1--- (the .meta.json beside the transcript); a transcript with no description is matched by the task path in its first prompt, one with another description is not this arm's. Returns the K5 tokens document and the tasks no transcript was found for.""" if arm not in ARMS: raise RunError(f"unknown arm {arm!r}; the arms are {', '.join(ARMS)}") if not transcripts.is_dir(): raise RunError(f"{transcripts} is not a directory") tasks_dir = run / "tasks" / arm ids = sorted(p.stem for p in tasks_dir.glob("*.md")) if not ids: raise RunError(f"no tasks in {tasks_dir}; run setup first") described = re.compile(rf"^S1-{re.escape(name)}-{re.escape(arm)}-(\S+)$") prompted = re.compile(re.escape(str(tasks_dir)) + r"/([A-Za-z0-9]+)\.md") basis = run / "arms" / arm rows: dict[str, dict[str, Any]] = {} for path in sorted(transcripts.rglob("*.jsonl")): meta_path = path.with_name(path.stem + ".meta.json") description = "" if meta_path.is_file(): meta = load_json(meta_path, "agent meta") description = str(meta.get("description", "")) if isinstance(meta, dict) else "" lines = read_transcript(path) m = described.match(description) if description else prompted.search(first_prompt(lines)) if m is None or m.group(1) not in ids: continue qid = m.group(1) counts, tools = usage_and_tools(lines) breaches = breaches_of(tools, basis, tasks_dir / f"{qid}.md", run / "answers" / arm / f"{qid}.json") row = rows.setdefault(qid, {"id": qid, **{f: 0 for f in counts}, "isolation_breaches": 0, "breach_detail": [], "runs": 0}) for f, value in counts.items(): row[f] += value row["breach_detail"] += breaches row["isolation_breaches"] = len(row["breach_detail"]) row["runs"] += 1 doc = {"schema": TOKENS_SCHEMA, "arm": arm, "questions": [rows[i] for i in sorted(rows)]} return doc, [i for i in ids if i not in rows] # ---------------------------------------------------------------------------------------------- score def mcnemar_exact(only_a: int, only_b: int) -> float: """Two-sided exact McNemar: the binomial test over the discordant pairs, p = 0.5.""" n, k = only_a + only_b, min(only_a, only_b) if n == 0: return 1.0 return min(1.0, 2 * sum(comb(n, i) for i in range(k + 1)) / float(2 ** n)) def wilson(x: int, n: int, z: float) -> tuple[float, float]: centre = (x + z * z / 2) / (n + z * z) half = z * sqrt(x * (n - x) / n + z * z / 4) / (n + z * z) return centre - half, centre + half def paired_difference_ci(e: int, f: int, g: int, h: int, level: float = 0.95) -> tuple[float, float]: """The interval for (e + f)/n - (e + g)/n from paired data: e both, f first only, g second only, h neither. Newcombe (1998), Statist. Med. 17:2635-2650, method 10: Wilson score limits for each margin combined with the phi coefficient, its numerator max(eh - fg - n/2, 0) when eh > fg, and phi 0 when a margin is 0.""" n = e + f + g + h if n == 0: raise RunError("no pairs") z = NormalDist().inv_cdf(1 - (1 - level) / 2) p1, p2 = (e + f) / n, (e + g) / n l1, u1 = wilson(e + f, n, z) l2, u2 = wilson(e + g, n, z) denominator = (e + f) * (g + h) * (e + g) * (f + h) numerator = float(e * h - f * g) if numerator > 0: numerator = max(numerator - n / 2, 0.0) phi = numerator / sqrt(denominator) if denominator else 0.0 dl1, du1, dl2, du2 = p1 - l1, u1 - p1, p2 - l2, u2 - p2 low = sqrt(max(0.0, dl1 * dl1 - 2 * phi * dl1 * du2 + du2 * du2)) high = sqrt(max(0.0, du1 * du1 - 2 * phi * du1 * dl2 + dl2 * dl2)) return p1 - p2 - low, p1 - p2 + high def min_detectable(n: int, discordant_share: float, alpha: float = ALPHA, power: float = POWER) -> float | None: """The smallest difference between paired proportions that n pairs show at the given power with a two-sided McNemar test, for a share of discordant pairs psi (Connor 1987: n = (z_a sqrt(psi) + z_b sqrt(psi - d^2))^2 / d^2, solved for d by bisection). None when psi is 0 or no d up to psi is visible.""" if n <= 0 or discordant_share <= 0: return None za, zb = NormalDist().inv_cdf(1 - alpha / 2), NormalDist().inv_cdf(power) psi = min(discordant_share, 1.0) def needed(d: float) -> float: return (za * sqrt(psi) + zb * sqrt(max(0.0, psi - d * d))) ** 2 / (d * d) if needed(psi) > n: return None low, high = 1e-9, psi for _ in range(200): mid = (low + high) / 2 if needed(mid) > n: low = mid else: high = mid return high def load_results(path: Path) -> dict[str, Any]: data = load_json(path, "results") if not isinstance(data, dict) or data.get("schema") != RESULTS_SCHEMA or not isinstance(data.get("results"), list): raise RunError(f"{path} is not a results file with schema {RESULTS_SCHEMA}") if data.get("arm") not in ARMS: raise RunError(f"{path} is for arm {data.get('arm')!r}; the arms are {', '.join(ARMS)}") return data def path_band(paths: int) -> str: return "4+" if paths >= MANY_PATHS else "2-3" def score(results: list[dict[str, Any]], tokens: list[dict[str, Any]]) -> dict[str, Any]: by_arm = {r["arm"]: r for r in results} if sorted(by_arm) != sorted(ARMS) or len(results) != len(ARMS): raise RunError(f"score needs one results file for each arm: {', '.join(ARMS)}") a, b = by_arm["filsok"], by_arm["bundle"] if a.get("set_sha256") != b.get("set_sha256"): raise RunError("the two results files are for different sets") rows = {arm: {str(r["id"]): r for r in by_arm[arm]["results"]} for arm in ARMS} if sorted(rows["filsok"]) != sorted(rows["bundle"]): raise RunError("the two results files hold different questions") ids = sorted(rows["filsok"]) doc: dict[str, Any] = {"schema": SCORE_SCHEMA, "set_sha256": a.get("set_sha256"), "questions": len(ids), "arms": {}, "pairs": {}} for arm in ARMS: groups = {g: [sum(1 for i in ids if rows[arm][i]["group"] == g and rows[arm][i]["correct"] is True), sum(1 for i in ids if rows[arm][i]["group"] == g)] for g in GROUPS} bands = {band: [sum(1 for i in ids if path_band(int(rows[arm][i]["paths_needed"])) == band and rows[arm][i]["correct"] is True), sum(1 for i in ids if path_band(int(rows[arm][i]["paths_needed"])) == band)] for band in ("2-3", "4+")} doc["arms"][arm] = {"groups": groups, "paths": bands, "all": [sum(1 for i in ids if rows[arm][i]["correct"] is True), len(ids)]} for label, members in [("all", ids)] + [(g, [i for i in ids if rows["filsok"][i]["group"] == g]) for g in GROUPS]: counts = {"both": 0, "only filsok": 0, "only bundle": 0, "neither": 0} for i in members: fa, bu = rows["filsok"][i]["correct"] is True, rows["bundle"][i]["correct"] is True counts["both" if fa and bu else "only filsok" if fa else "only bundle" if bu else "neither"] += 1 doc["pairs"][label] = counts pairs = doc["pairs"]["all"] doc["mcnemar_p"] = mcnemar_exact(pairs["only filsok"], pairs["only bundle"]) n = len(ids) if n: low, high = paired_difference_ci(pairs["both"], pairs["only bundle"], pairs["only filsok"], pairs["neither"]) doc["difference"] = {"of": "bundle - filsok", "point": (pairs["only bundle"] - pairs["only filsok"]) / n, "low": low, "high": high, "level": 0.95, "method": "Newcombe 1998 method 10"} share = (pairs["only filsok"] + pairs["only bundle"]) / n doc["smallest_visible"] = {"discordant_share": share, "difference": min_detectable(n, share), "power": POWER, "alpha": ALPHA} doc["tokens"] = {} for t in tokens: if not isinstance(t, dict) or t.get("schema") != TOKENS_SCHEMA or t.get("arm") not in ARMS: raise RunError(f"a tokens file is not {TOKENS_SCHEMA} for one of the arms") qs = as_list(t.get("questions")) total = sum(int(q.get(f, 0)) for q in qs for f in (*USAGE_FIELDS, "output_tokens_derived")) doc["tokens"][t["arm"]] = {"questions": len(qs), "per_question": total / len(qs) if qs else None, "isolation_breaches": sum(int(q.get("isolation_breaches", 0)) for q in qs)} return doc def render(doc: dict[str, Any]) -> list[str]: lines = [] for arm in ARMS: row = doc["arms"][arm] lines.append(f"{arm}: " + ", ".join(f"{g} {c} of {n}" for g, (c, n) in row["groups"].items()) + f", all {row['all'][0]} of {row['all'][1]}") for arm in ARMS: lines.append(f"{arm} by paths: " + ", ".join(f"{b} {c} of {n}" for b, (c, n) in doc["arms"][arm]["paths"].items())) for label, counts in doc["pairs"].items(): lines.append(("pairs" if label == "all" else f"pairs {label}") + ": " + ", ".join(f"{k} {v}" for k, v in counts.items())) lines.append(f"exact McNemar p = {doc['mcnemar_p']:.3g}") if "difference" in doc: d = doc["difference"] lines.append(f"bundle - filsok = {d['point']:.3f}, 95 % CI {d['low']:.3f} to {d['high']:.3f} " f"(paired, Newcombe 1998 method 10)") s = doc["smallest_visible"] visible = "none" if s["difference"] is None else f"{s['difference']:.3f}" lines.append(f"smallest difference visible at 80 % power, alpha 0.05: {visible} " f"(discordant share measured {s['discordant_share']:.3f}, N = {doc['questions']})") for arm, t in doc["tokens"].items(): per = "none" if t["per_question"] is None else f"{t['per_question']:.0f}" lines.append(f"{arm} tokens: {t['questions']} questions harvested, {per} per question, " f"isolation breaches {t['isolation_breaches']}") return lines # ---------------------------------------------------------------------------------------------- main def build_parser() -> argparse.ArgumentParser: p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) sub = p.add_subparsers(dest="command", required=True) s = sub.add_parser("setup", help="one arm's basis and task files") s.add_argument("--run", required=True) s.add_argument("--name", required=True, help="the run's name, letters and digits (S1---)") s.add_argument("--arm", required=True) s.add_argument("--set", required=True) s.add_argument("--sources", required=True, help=f"the K1 root measured (its kilde/ files and {K1_FILE})") s.add_argument("--bundle", help="the collection, for the bundle arm") s.add_argument("--part-root", help="the K1 root the set was made from, when it is a subset_of --sources: the arm " "searches the whole of --sources") c = sub.add_parser("collect", help="the answer files into the arm's answers file") c.add_argument("--run", required=True) c.add_argument("--arm", required=True) h = sub.add_parser("harvest", help="tokens and isolation breaches per question from the transcripts") h.add_argument("--run", required=True) h.add_argument("--name", required=True) h.add_argument("--arm", required=True) h.add_argument("--transcripts", required=True, help="a directory searched for *.jsonl at any depth") r = sub.add_parser("score", help="two arms' results: groups, paths, pairs, McNemar, interval, power") r.add_argument("--results", action="append", required=True, help="answerset.py gate --out, once per arm") r.add_argument("--tokens", action="append", default=[], help="harvest output, once per arm (optional)") r.add_argument("--out", default="") return p def main(argv: list[str] | None = None) -> int: args = build_parser().parse_args(argv) try: if args.command == "setup": run = Path(args.run).resolve() tasks = setup_arm(run, args.name, args.arm, Path(args.set), Path(args.sources), Path(args.bundle).resolve() if args.bundle else None, Path(args.part_root) if args.part_root else None) for description, task in tasks: print(f"{description}\t{task}") return 0 if args.command == "collect": out, problems = collect(Path(args.run).resolve(), args.arm) print(out) for problem in problems: print(f"problem: {problem}") return 1 if problems else 0 if args.command == "harvest": run = Path(args.run).resolve() doc, missing = harvest(run, args.name, args.arm, Path(args.transcripts)) write(run / "tokens" / f"{args.arm}.json", doc) breaches = sum(q["isolation_breaches"] for q in doc["questions"]) print(f"tasks harvested: {len(doc['questions'])}; breaches: {breaches}") if missing: print(f"tasks without a transcript: {len(missing)}: {', '.join(missing)}") return 1 if missing else 0 doc = score([load_results(Path(p)) for p in args.results], [load_json(Path(p), "tokens") for p in args.tokens]) print("\n".join(render(doc))) if args.out: write(Path(args.out), doc) return 0 except RunError as exc: print(f"FAIL: {exc}", file=sys.stderr) return 1 if __name__ == "__main__": sys.exit(main())