"""The direction probe R (PLAN.md ยง Leveranser, row R): three arms answer the same questions from the same sources. filsok file search alone over the sources in the cut wiki the okf-wiki plus file search, following the skill's router (index.md, the pages, the raw source) astro a wiki built with Astro-Han's karpathy-llm-wiki skill on the same sources, following its Query steps Each question is answered by one isolated subagent per arm in the user's session. This module never calls a model: fetch-astro the skill's Fetch step done mechanically: each source copied byte-exact under the raw template's header into /raw// (the sources are local, clean Markdown already) setup one arm's basis (a copy, so no arm reads another's folder) and one task file per question, which carries the question and never the answer key, the evidence or the count collect the per-question answer files into the answers files the gates read harvest tokens (every usage field, each message id once) and isolation breaches from the transcripts score the gates' per-question results for the probe questions only: per arm and group, and in pairs The gates are run unchanged (`gates/multidoc.py gate`, `gates/wholedoc.py gate`, each with --out). """ from __future__ import annotations import argparse import json import os import re import shlex import shutil import sys from itertools import combinations from math import comb from pathlib import Path from typing import Any ARMS = ("filsok", "wiki", "astro") GROUPS = ("flerdokument", "helt dokument", "telling") MD_SCHEMA = "okf-wiki/multidoc-set/1" WD_SCHEMA = "okf-wiki/wholedoc-set/1" ALLOWED_TOOLS = ("Read", "Grep", "Glob", "Bash", "Write", "SubagentHandback") READ_ONLY = ("grep", "egrep", "fgrep", "rg", "find", "ls", "wc", "head", "tail", "sed", "cat", "sort", "uniq", "cut", "tr", "nl") SEPARATORS = ("|", "&&", ";", "||") FIND_ACTIONS = ("-exec", "-execdir", "-ok", "-okdir", "-delete", "-fprint", "-fprint0", "-fprintf", "-fls") USAGE_FIELDS = ("input_tokens", "cache_creation_input_tokens", "cache_read_input_tokens", "output_tokens") class ProbeError(Exception): """The command cannot run; reported as FAIL with exit 1.""" def load_json(path: Path) -> Any: try: return json.loads(path.read_text(encoding="utf-8")) except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc: raise ProbeError(f"cannot read {path}: {exc}") from exc def write_json(path: Path, data: Any) -> None: path.parent.mkdir(parents=True, exist_ok=True) path.write_text(json.dumps(data, ensure_ascii=False, indent=1) + "\n", encoding="utf-8") def copy_names(sources: Path, names: list[str], target: Path) -> None: target.mkdir(parents=True, exist_ok=True) for name in names: src = sources / name if not src.is_file(): raise ProbeError(f"source {src} is missing") shutil.copyfile(src, target / name) # ---------------------------------------------------------------------------------------------- fetch-astro def fetch_astro(sources: Path, names: list[str], project: Path, topic: str, collected: str) -> list[Path]: """The Fetch step of karpathy-llm-wiki, per its raw template, with the source text preserved byte-exact.""" folder = project / "raw" / topic targets = [folder / name for name in names] existing = [str(t) for t in targets if t.exists()] if existing: raise ProbeError(f"raw files exist already and are never overwritten: {', '.join(existing)}") folder.mkdir(parents=True, exist_ok=True) for name, target in zip(names, targets): src = sources / name if not src.is_file(): raise ProbeError(f"source {src} is missing") header = (f"# {Path(name).stem}\n\n> Source: kilde/{name}\n> Collected: {collected}\n" "> Published: Unknown\n\n") target.write_bytes(header.encode("utf-8") + src.read_bytes()) return targets # ---------------------------------------------------------------------------------------------- setup SEARCH = { "filsok": """BASIS holds the source files and nothing else. Search them with grep and find (Bash, as above), and read what you find with Read.""", "wiki": """BASIS holds `wiki/`, a knowledge wiki compiled from the sources, and `kilde/`, the source files. A page names its source as `/kilde/`, which is the file `BASIS/kilde/`. Follow this order: 1. Read `BASIS/wiki/index.md` first. 2. Read the pages the index points to. 3. Open the raw source behind a footnote (`resource` at `x-locator`) when the quote has to be checked or when the page does not say enough. You may also search `wiki/` and `kilde/` with grep and find (Bash, as above) at any point.""", "astro": """BASIS holds `wiki/`, a knowledge wiki compiled from the sources with the karpathy-llm-wiki skill, and `raw/`, the source files it was compiled from (each raw file starts with a short metadata header; the source text follows it unchanged). Follow the skill's Query steps: 1. Read `BASIS/wiki/index.md` to locate candidate articles, then full-text search `wiki/` with the topic's key terms and their synonyms (grep through Bash, as above). 2. Read the articles you found and synthesize an answer. 3. Prefer wiki content over your own training knowledge. Each article links the raw files it was compiled from in its `Raw` field. Open those raw files when you need an exact value or a verbatim quote.""", } CITE_NAME = { "filsok": "the bare name of a source file in BASIS", "wiki": "the bare name of a source file in `BASIS/kilde`", "astro": "the bare name of a raw file in `BASIS/raw//`", } def render_task(arm: str, basis: Path, qid: str, question: str, is_count: bool, answer_path: Path) -> str: count_line = '"count": , ' if is_count else "" count_rule = ("\n- `count`: the number the question asks for, as a JSON integer. Say in `answer` how you counted." if is_count else "") return f"""# Answer one question BASIS: `{basis}` You answer ONE question from the files in BASIS. Read nothing outside BASIS apart from this task file: no other folder, no web, no earlier work. Answer only from what the files say, and say plainly when they do not say something. Tools: Read (a file inside BASIS), Bash for read-only search, and Write once, for the answer file. No other tool. Every Bash command starts with `cd {basis} && ` and then uses only {', '.join(READ_ONLY)}, joined by `|` or `&&`, with paths relative to BASIS: never `..`, `~` or a path outside BASIS, no `$` expansion, no output redirection except `2>/dev/null`, no `find -exec` or `-delete`, no `sed -i`. For example: `cd {basis} && grep -rn "term" .` ## How to search {SEARCH[arm]} ## The question {question} ## The answer Write one JSON file to `{answer_path}`: ```json {{"id": "{qid}", "answer": "", {count_line}"citations": [{{"file": "", "quote": ""}}]}} ``` - `answer`: answer every part of the question.{count_rule} - `citations`: one or more per source file your answer relies on, and cite every such file. - `file`: {CITE_NAME[arm]}, for example `notes.md`: never a path and never a wiki page. - `quote`: text copied character for character from that file (only whitespace may differ), long enough to carry the fact, for example the whole line or sentence. When the file is written, reply with the one word `done`. """ def probe_questions(md_set: Path, md_ids: list[str], wd_set: Path) -> list[tuple[str, dict[str, Any]]]: md, wd = load_json(md_set), load_json(wd_set) if not isinstance(md, dict) or md.get("schema") != MD_SCHEMA: raise ProbeError(f"{md_set} is not a set with schema {MD_SCHEMA}") if not isinstance(wd, dict) or wd.get("schema") != WD_SCHEMA: raise ProbeError(f"{wd_set} is not a set with schema {WD_SCHEMA}") by_id = {q["id"]: q for q in md["questions"]} missing = [i for i in md_ids if i not in by_id] if missing: raise ProbeError(f"probe ids missing from {md_set}: {', '.join(missing)}") return [("md", by_id[i]) for i in md_ids] + [("wd", q) for q in wd["questions"]] def setup_arm(run: Path, arm: str, sources: Path, names: list[str], md_set: Path, md_ids: list[str], wd_set: Path, wiki: Path | None = None, astro: Path | None = None) -> list[Path]: if arm not in ARMS: raise ProbeError(f"unknown arm {arm!r}; the arms are {', '.join(ARMS)}") if arm == "wiki" and wiki is None: raise ProbeError("the wiki arm needs --wiki") if arm == "astro" and astro is None: raise ProbeError("the astro arm needs --astro") questions = probe_questions(md_set, md_ids, wd_set) basis = run / "arms" / arm if basis.exists(): raise ProbeError(f"{basis} exists already; an arm is set up once, from fresh") if arm == "filsok": copy_names(sources, names, basis) elif arm == "wiki" and wiki is not None: shutil.copytree(wiki, basis / "wiki") copy_names(sources, names, basis / "kilde") elif astro is not None: shutil.copytree(astro / "raw", basis / "raw") shutil.copytree(astro / "wiki", basis / "wiki") tasks = [] for group, q in questions: key = f"{group}-{q['id']}" task = run / "tasks" / arm / f"{key}.md" answer = run / "answers" / arm / f"{key}.json" task.parent.mkdir(parents=True, exist_ok=True) task.write_text(render_task(arm, basis, q["id"], q["question"], q.get("type") == "count", answer), encoding="utf-8") tasks.append(task) (run / "answers" / arm).mkdir(parents=True, exist_ok=True) return tasks # ---------------------------------------------------------------------------------------------- collect def collect(run: Path, arm: str) -> tuple[Path, Path, list[str]]: """The per-question answers into one answers file per gate. The id comes from the file name.""" folder = run / "answers" / arm gathered: dict[str, list[dict[str, Any]]] = {"md": [], "wd": []} problems = [] for path in sorted(folder.glob("*.json")): group, _, qid = path.stem.partition("-") if group not in gathered or not qid: problems.append(f"{path.name}: not a probe answer name") continue try: data = json.loads(path.read_text(encoding="utf-8")) except (OSError, UnicodeDecodeError, json.JSONDecodeError): data = None if not isinstance(data, dict): problems.append(f"{path.name}: not a JSON object") continue gathered[group].append({**data, "id": qid}) md_path, wd_path = run / "answers" / f"{arm}.md.json", run / "answers" / f"{arm}.wd.json" write_json(md_path, {"arm": arm, "answers": gathered["md"]}) write_json(wd_path, {"arm": arm, "answers": gathered["wd"]}) return md_path, wd_path, problems # ---------------------------------------------------------------------------------------------- harvest def first_prompt(lines: list[dict[str, Any]]) -> tuple[str, str]: for line in lines: if line.get("type") == "user": content = line.get("message", {}).get("content") if isinstance(content, list): content = " ".join(b.get("text", "") for b in content if isinstance(b, dict)) return (content if isinstance(content, str) else ""), str(line.get("timestamp", "")) return "", "" def inside(path: object, root: Path) -> bool: if not isinstance(path, str) or not os.path.isabs(path): return False norm = Path(os.path.normpath(path)) return norm == root or root in norm.parents def bash_breach(command: str, basis: Path) -> str | None: """None when the command is read-only search that starts with `cd &&` and stays inside the basis.""" if "`" in command or re.search(r"\$[A-Za-z_{(]", command): return "Bash with an expansion" try: lexer = shlex.shlex(re.sub(r"\s2>/dev/null(?=\s|$)", "", command), posix=True, punctuation_chars="|&;<>()") lexer.whitespace_split = True tokens = list(lexer) except ValueError: return "Bash that cannot be parsed" segments: list[list[str]] = [[]] for token in tokens: if token in SEPARATORS: segments.append([]) elif token and set(token) <= set("&<>()"): return f"Bash with {token!r}" else: segments[-1].append(token) first = segments[0] if len(first) != 2 or first[0] != "cd" or not inside(first[1], basis): return "Bash that does not start with cd into the basis" for segment in segments[1:]: if not segment or segment[0] not in READ_ONLY: return f"Bash command not allowed: {segment[0] if segment else '(empty)'}" for token in segment[1:]: value = token.split("=", 1)[1] if token.startswith("-") and "=" in token else token if segment[0] == "find" and token in FIND_ACTIONS: return f"Bash find action {token}" if segment[0] == "sed" and (token.startswith("-i") or token.startswith("--in-place")): return "Bash sed -i" if value.startswith("~") or ".." in value.split("/"): return f"Bash path outside the basis: {token}" if value.startswith("/") and value != "/dev/null" and not inside(value, basis): return f"Bash path outside the basis: {token}" return None def breaches_of(tools: list[tuple[str, dict[str, Any]]], basis: Path, task: Path, answer: Path) -> list[str]: found = [] for name, inp in tools: if name not in ALLOWED_TOOLS: found.append(f"{name}: tool not allowed") elif name == "Read": path = inp.get("file_path") if not (inside(path, basis) or (isinstance(path, str) and os.path.normpath(path) == str(task))): found.append(f"Read outside the basis: {path}") elif name in ("Grep", "Glob"): path = inp.get("path") pattern = inp.get("pattern", "") if path is None: found.append(f"{name} without a path (searches the working directory)") elif not inside(path, basis): found.append(f"{name} outside the basis: {path}") elif name == "Glob" and isinstance(pattern, str) and os.path.isabs(pattern) and not inside(pattern, basis): found.append(f"Glob pattern outside the basis: {pattern}") elif name == "Bash": breach = bash_breach(str(inp.get("command", "")), basis) if breach: found.append(breach) elif name == "Write" and inp.get("file_path") != str(answer): found.append(f"Write to another file: {inp.get('file_path')}") return found def read_transcript(path: Path) -> list[dict[str, Any]]: lines = [] for raw in path.read_text(encoding="utf-8").splitlines(): try: line = json.loads(raw) except json.JSONDecodeError: continue if isinstance(line, dict): lines.append(line) return lines def harvest(transcripts: Path, run: Path) -> dict[str, dict[str, Any]]: """Per task: all tokens its transcript records (each message id once), and every isolation breach.""" pattern = re.compile(re.escape(str(run / "tasks")) + r"/(" + "|".join(ARMS) + r")/((?:md|wd)-[A-Za-z0-9]+)\.md") rows: dict[str, dict[str, Any]] = {} for path in sorted(transcripts.glob("*.jsonl")): lines = read_transcript(path) prompt, started = first_prompt(lines) match = pattern.search(prompt) if match is None: continue arm, key = match.group(1), match.group(2) usage: dict[str, dict[str, int]] = {} tools: list[tuple[str, dict[str, Any]]] = [] for line in lines: message = line.get("message", {}) if line.get("type") != "assistant" or not isinstance(message, dict): continue mid = str(message.get("id", "")) fields = message.get("usage") or {} seen = usage.setdefault(mid, {}) for f in USAGE_FIELDS: value = fields.get(f) if isinstance(value, int): seen[f] = max(seen.get(f, 0), value) for block in message.get("content") or []: if isinstance(block, dict) and block.get("type") == "tool_use": tools.append((str(block.get("name")), block.get("input") or {})) row = { "tokens": sum(sum(v.values()) for v in usage.values()), "breaches": breaches_of(tools, run / "arms" / arm, run / "tasks" / arm / f"{key}.md", run / "answers" / arm / f"{key}.json"), "transcript": path.name, "started": started, } name = f"{arm}/{key}" previous = rows.get(name) runs = 1 + (previous["runs"] if previous else 0) if previous is None or started >= previous["started"]: rows[name] = {**row, "runs": runs} else: previous["runs"] = runs for row in rows.values(): row.pop("started") return rows # ---------------------------------------------------------------------------------------------- score def gate_results(path: Path) -> dict[str, dict[str, Any]]: if not path.is_file(): raise ProbeError(f"no gate results at {path}; an arm that was not scored is a failure, not a zero") data = load_json(path) return {r["id"]: r for r in data.get("results", [])} def mcnemar_exact(only_a: int, only_b: int) -> float: """Two-sided exact McNemar: the binomial test over the discordant pairs, p = 0.5.""" n, k = only_a + only_b, min(only_a, only_b) if n == 0: return 1.0 return min(1.0, 2 * sum(comb(n, i) for i in range(k + 1)) / float(2 ** n)) def score(run: Path, arms: list[str], md_ids: list[str]) -> dict[str, Any]: tokens_path = run / "tokens.json" tokens = load_json(tokens_path) if tokens_path.is_file() else {} correct: dict[str, dict[str, dict[str, bool]]] = {} table: dict[str, Any] = {"arms": {}, "pairs": {}, "mcnemar": {}} for arm in arms: md = gate_results(run / "results" / f"{arm}.md.json") wd = gate_results(run / "results" / f"{arm}.wd.json") by_group: dict[str, dict[str, bool]] = {g: {} for g in GROUPS} answer_ok = source_ok = 0 for qid in md_ids: r = md.get(qid, {}) by_group["flerdokument"][f"md-{qid}"] = bool(r.get("correct")) answer_ok += bool(r.get("answer_ok")) source_ok += bool(r.get("source_ok")) for qid, r in wd.items(): group = "telling" if r.get("type") == "count" else "helt dokument" by_group[group][f"wd-{qid}"] = bool(r.get("correct")) answer_ok += bool(r.get("answer_ok")) source_ok += bool(r.get("source_ok")) correct[arm] = by_group n = sum(len(v) for v in by_group.values()) own = [v for k, v in tokens.items() if k.startswith(f"{arm}/")] table["arms"][arm] = { "groups": {g: [sum(v.values()), len(v)] for g, v in by_group.items()}, "total": [sum(sum(v.values()) for v in by_group.values()), n], "answer_ok": answer_ok, "source_ok": source_ok, "tokens": {"sum": sum(int(v["tokens"]) for v in own), "questions": len(own)}, "breaches": sum(len(v["breaches"]) for v in own), } for a, b in combinations(arms, 2): a, b = (b, a) if b == "wiki" else (a, b) pair: dict[str, dict[str, int]] = {} for g in GROUPS: counts = {"both": 0, f"only {a}": 0, f"only {b}": 0, "neither": 0} for key, ok_a in correct[a][g].items(): ok_b = correct[b][g].get(key, False) label = "both" if ok_a and ok_b else f"only {a}" if ok_a else f"only {b}" if ok_b else "neither" counts[label] += 1 pair[g] = counts table["pairs"][f"{a} vs {b}"] = pair only_a, only_b = sum(c[f"only {a}"] for c in pair.values()), sum(c[f"only {b}"] for c in pair.values()) table["mcnemar"][f"{a} vs {b}"] = {f"only {a}": only_a, f"only {b}": only_b, "p": mcnemar_exact(only_a, only_b)} table["per_question"] = {arm: {k: v for g in GROUPS for k, v in correct[arm][g].items()} for arm in arms} return table def render_table(table: dict[str, Any]) -> str: head = "| arm | " + " | ".join(f"{g} (N)" for g in GROUPS) + " | total | answer ok | source ok | tokens/question |" lines = [head, "|" + "---|" * (len(GROUPS) + 5)] for arm, row in table["arms"].items(): t = row["tokens"] per_q = f"{t['sum'] / t['questions'] / 1e6:.2f} M" if t["questions"] else "not harvested" cells = [f"{c} of {n}" for c, n in (row["groups"][g] for g in GROUPS)] total = row["total"] lines.append(f"| {arm} | " + " | ".join(cells) + f" | {total[0]} of {total[1]} | {row['answer_ok']} | " f"{row['source_ok']} | {per_q} |") lines.append("") for name, pair in table["pairs"].items(): lines.append(f"{name}: " + "; ".join(f"{g}: " + ", ".join(f"{k} {v}" for k, v in c.items()) for g, c in pair.items())) m = table["mcnemar"][name] lines.append(f"{name}, all groups: " + ", ".join(f"{k} {v}" for k, v in m.items() if k != "p") + f"; exact McNemar p = {m['p']:.3g}") return "\n".join(lines) # ---------------------------------------------------------------------------------------------- main def cut_names(manifest: Path) -> list[str]: names = load_json(manifest).get("cut", {}).get("names") if not isinstance(names, list) or not names: raise ProbeError(f"{manifest} records no cut names") return [str(n) for n in names] def build_parser() -> argparse.ArgumentParser: p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) sub = p.add_subparsers(dest="command", required=True) f = sub.add_parser("fetch-astro") f.add_argument("--sources", required=True) f.add_argument("--cut-manifest", required=True) f.add_argument("--project", required=True) f.add_argument("--topic", required=True) f.add_argument("--collected", required=True) s = sub.add_parser("setup") s.add_argument("--run", required=True) s.add_argument("--arm", required=True) s.add_argument("--sources", required=True) s.add_argument("--cut-manifest", required=True) s.add_argument("--md-set", required=True) s.add_argument("--md-ids", required=True, help="comma-separated") s.add_argument("--wd-set", required=True) s.add_argument("--wiki") s.add_argument("--astro") c = sub.add_parser("collect") c.add_argument("--run", required=True) c.add_argument("--arm", required=True) h = sub.add_parser("harvest") h.add_argument("--run", required=True) h.add_argument("--transcripts", required=True) r = sub.add_parser("score") r.add_argument("--run", required=True) r.add_argument("--arms", required=True, help="comma-separated") r.add_argument("--md-ids", required=True, help="comma-separated") return p def main(argv: list[str] | None = None) -> int: args = build_parser().parse_args(argv) try: if args.command == "fetch-astro": written = fetch_astro(Path(args.sources), cut_names(Path(args.cut_manifest)), Path(args.project), args.topic, args.collected) print(f"raw files: {len(written)}") elif args.command == "setup": tasks = setup_arm(Path(args.run), args.arm, Path(args.sources), cut_names(Path(args.cut_manifest)), Path(args.md_set), args.md_ids.split(","), Path(args.wd_set), Path(args.wiki) if args.wiki else None, Path(args.astro) if args.astro else None) print("\n".join(str(t) for t in tasks)) elif args.command == "collect": md_path, wd_path, problems = collect(Path(args.run), args.arm) print(f"{md_path}\n{wd_path}") for problem in problems: print(f"problem: {problem}") elif args.command == "harvest": rows = harvest(Path(args.transcripts), Path(args.run)) write_json(Path(args.run) / "tokens.json", rows) print(f"tasks harvested: {len(rows)}; breaches: {sum(len(r['breaches']) for r in rows.values())}") else: table = score(Path(args.run), args.arms.split(","), args.md_ids.split(",")) write_json(Path(args.run) / "table.json", table) print(render_table(table)) except ProbeError as exc: print(f"FAIL: {exc}", file=sys.stderr) return 1 return 0 if __name__ == "__main__": sys.exit(main())