"""Token cost of a v1 run from its sessions' transcripts: cost lines, stop checks and the isolation count. No model. Usage: python3 -m okfwiki.cost harvest --transcripts DIR --okter FILE --work DIR --out kost.json [--packs FILE] [--margin M] python3 -m okfwiki.cost stop --lines L[,L] --cap N --unit blocks|passes|claims --done D --total T --kost FILE [--blocks FILE] [--batches-done B --batches-total N] [--next B [--next-chars C]] [--margin M] [--name NAME] python3 -m okfwiki.cost summarize --kost FILE [--cap-group NAME=L[,L]:CAP]... python3 -m okfwiki.cost isolation --transcripts DIR --run R --tasks DIR --out isolation.json [--line read|diagnose|repair] harvest Reads the sessions listed in okter.txt (" " per line, kind build, read, diagnose, merge or repair) under the transcripts directory (~/.claude/projects//): each session's main transcript is orchestration, on the line orchestration-, with the input side of every turn kept by turn number, the first turn that starts helpers (an Agent, Task or Workflow call) and the tokens of the turns before it; every subagent transcript under /subagents/ or, for a workflow's agents, /subagents/workflows// (the agent() label is the description) is attributed by its description "V1---", line pack (kept as build), link, read, diagnose, merge (step 18a's pair helper, order 29), repair (the repair helper, order 34) or calib. Any other description is an unattributed run: it is listed and the exit is 1, because a model-run session spawns no helper agents. Tokens are counted as the trial did: per message id once, at its largest value; an output never recorded is derived from the next message's input growth (a lower bound). With --packs, a pack log with a gap (a logged pack with no build run, or a build run for a pack the log lacks) cannot run. The margin is max(0.10, measured), 0.10 when nothing is measured, because the measurement is itself a lower bound. Recorded cap groups and stop rows survive a new harvest. stop The stop check after a pack or a pass. Measured = the sum of the given lines (calib is reported, never capped). Flat projection = measured / done x total. --unit blocks also fits cost per block against blocks read before (a + b*R) over the packs in --blocks (a pack log with "pack" and "blocks"), orchestration spread over the packs in proportion to their attributed cost, and projects a*T + b*T*T/2, once at least four packs are measured (the operator 11.10.2026); before that the flat projection alone. --unit claims scales only the helper lines per claim: an orchestration line counts the session's part before its first helper once and the rest per batch (a message that starts helpers, or a phase of a workflow), measured / --batches-done x --batches-total, both required with an orchestration line. STOP when the projection x (1 + margin) exceeds the cap. With --next B (--unit blocks only) the projection is recorded and printed but does not stop: STOP when (measured + measured / done x B) x (1 + margin) exceeds the cap, B being the blocks of the pack the model works on before the next check (the operator 12.10.2026: a cost line stops on measured cost plus a margin, the projection is reported only; order 22 session A3: plus the next pack, so the cap is not passed). With --next-chars C as well, the next pack's share is by characters, not blocks (motor B17: a block of a folder's PDF or spreadsheet holds about ten times the characters of a docs block, and the cost follows the characters): measured / the `block_chars` of the packs with a compiler run here x C, ROUND2_PER_CHAR before the first; a log whose built packs have no `block_chars` keeps the rule by blocks. The cap group and the stop row are recorded in the kost file. Exit 0 continue, 1 STOP, 2 cannot run; a caller treats every non-zero exit as STOP. summarize Per line ": + margin = "; per cap group (recorded by stop, or given with --cap-group) ": + margin = of "; orchestration tokens per turn, per pack and the input per turn fitted against the turn number. Exit 1 when a group is over its cap. isolation Reader agents (or, with --line diagnose, diagnosis agents) are one per task, described "V1---", each with the task file /.md, started from a session or from a workflow (the same two places as harvest). Every Read call of another path is a break. 0 breaks over at least one agent = exit 0. With --line repair (order 34), each repair helper "V1--repair-" may read exactly the parts and write exactly the one answer path of /.parts.json (krepair task's part list); every other tool call, a tool other than Read and Write among them, is a break (CLAUDE.md: the breach count reads every tool call), and so is a part never read and an answer written other than once (order 39). A helper whose task has no part list cannot run (exit 2). """ from __future__ import annotations import argparse import json import math import re import sys from collections.abc import Mapping from fractions import Fraction from pathlib import Path from typing import Any from okfwiki.common import RunError from okfwiki.transcript import USAGE_FIELDS, read_transcript SCHEMA = "okf-wiki/cost/1" ISOLATION_SCHEMA = "okf-wiki/isolation/1" LINES = ("build", "link", "orchestration-build", "read", "orchestration-read", "diagnose", "orchestration-diagnose", "merge", "orchestration-merge", "repair", "orchestration-repair", "calib") UNCAPPED = ("calib",) LINE_OF = {"pack": "build", "link": "link", "read": "read", "diagnose": "diagnose", "merge": "merge", "repair": "repair", "calib": "calib"} KINDS = ("build", "read", "diagnose", "merge", "repair") UNITS = ("blocks", "passes", "claims") ISOLATED = ("read", "diagnose", "repair") PREFIX = re.compile(r"^V1-(.+?)-(pack|link|read|diagnose|merge|repair|calib)-(\S+)$") PACK_UNIT = re.compile(r"^p(\d+)(?:-|$)") MARGIN_FLOOR = 0.10 CATEGORIES = {"input_tokens": "input", "cache_read_input_tokens": "cache_read", "cache_creation_input_tokens": "cache_write", "output_tokens": "output"} INPUT_SIDE = ("input_tokens", "cache_read_input_tokens", "cache_creation_input_tokens") CHARS_PER_TOKEN_LOW = 2.5 # injected text counted high, so a derived output is a lower bound SPAWNS = ("Agent", "Task", "Workflow") # the tool calls that start helpers or a workflow's agents FIT_FROM_PACKS = 4 # the operator 11.10.2026: the fitted curve counts from pack 4 ROUND2_PER_BLOCK = 3_259 # round 2 of the first part, 14 741 606 over 4 523 blocks (order 25 session C): the next pack's # share before a ledger's first compiler run (order 27 A3) ROUND2_PER_CHAR = Fraction(14_741_606, 1_111_796) # the same cost over the 1 111 796 characters of round 2's blocks # (about 13.3 per character): the share by characters before the first compiler run (B17) def margin(measured: float | None) -> float: """The margin on a lower-bound measurement: the measured one, never below the 10 % floor.""" return MARGIN_FLOOR if measured is None else max(MARGIN_FLOOR, measured) def sessions(okter: Path) -> list[tuple[str, str]]: """(session id, kind) per line of okter.txt, in order, each id once.""" out: list[tuple[str, str]] = [] for line in okter.read_text(encoding="utf-8").splitlines(): parts = line.split() if not parts: continue if len(parts) != 2 or parts[1] not in KINDS: raise RunError(f"okter line {line!r} is not ' ' with kind in {', '.join(KINDS)}") known = dict(out) if parts[0] in known and known[parts[0]] != parts[1]: raise RunError(f"session {parts[0]} is listed as both {known[parts[0]]} and {parts[1]}") if parts[0] not in known: out.append((parts[0], parts[1])) return out def _result_chars(message: dict[str, Any]) -> int: content = message.get("content") if not isinstance(content, list): return 0 total = 0 for item in content: if not (isinstance(item, dict) and item.get("type") == "tool_result"): continue inner = item.get("content") if isinstance(inner, str): total += len(inner) elif isinstance(inner, list): total += sum(len(str(x.get("text", ""))) for x in inner if isinstance(x, dict)) return total def usage(path: Path, per_turn: bool = False) -> dict[str, Any]: """Tokens of one transcript, ported from the trial. A message whose records all lack a stop reason never got its final usage written; when another message follows, its output is the growth of the next message's input side less the text injected between them (counted high at CHARS_PER_TOKEN_LOW), a lower bound. With per_turn, the input side of every turn is kept with its turn number.""" per: dict[str, dict[str, int]] = {} order: list[str] = [] complete: set[str] = set() spawning: set[str] = set() between: dict[str, int] = {} last_stop: str | None = None for line in read_transcript(path): message = line.get("message") if line.get("type") == "attachment" and order: between[order[-1]] = between.get(order[-1], 0) + len(str(line.get("rendered") or "")) continue if not isinstance(message, dict): continue if line.get("type") == "user" and order: between[order[-1]] = between.get(order[-1], 0) + _result_chars(message) continue if line.get("type") != "assistant": continue mid = str(message.get("id", "")) if mid not in per: order.append(mid) seen = per.setdefault(mid, {}) fields = message.get("usage") or {} for f in USAGE_FIELDS: value = fields.get(f) if isinstance(value, int): seen[f] = max(seen.get(f, 0), value) content = message.get("content") if isinstance(content, list) and any(isinstance(b, dict) and b.get("type") == "tool_use" and b.get("name") in SPAWNS for b in content): spawning.add(mid) stop = message.get("stop_reason") last_stop = stop if isinstance(stop, str) else None if isinstance(stop, str): complete.add(mid) out: dict[str, Any] = {CATEGORIES[f]: sum(v.get(f, 0) for v in per.values()) for f in USAGE_FIELDS} out["output_recorded"] = out["output"] derived = unrecorded = 0 for i, mid in enumerate(order): if mid in complete: continue if i + 1 == len(order): unrecorded += 1 continue growth = (sum(per[order[i + 1]].get(f, 0) for f in INPUT_SIDE) - sum(per[mid].get(f, 0) for f in INPUT_SIDE) - int(between.get(mid, 0) / CHARS_PER_TOKEN_LOW)) recorded = per[mid].get("output_tokens", 0) if growth > recorded: out["output"] += growth - recorded derived += 1 out["output_derived_messages"] = derived out["output_unrecorded"] = unrecorded out["total"] = sum(out[c] for c in CATEGORIES.values()) out["turns"] = len(order) first = per[order[0]] if order else {} out["startup"] = sum(first.get(f, 0) for f in INPUT_SIDE) out["stop_reason"] = last_stop if per_turn: out["per_turn"] = [{"turn": i, "input": sum(per[mid].get(f, 0) for f in INPUT_SIDE)} for i, mid in enumerate(order, 1)] spawn_turn = next((i for i, mid in enumerate(order, 1) if mid in spawning), None) before = order if spawn_turn is None else order[:spawn_turn - 1] out["first_helper_turn"] = spawn_turn out["before_first_helper"] = sum(per[mid].get(f, 0) for mid in before for f in USAGE_FIELDS) return out def pack_of(unit: str) -> int | None: m = PACK_UNIT.match(unit) return int(m.group(1)) if m else None def line_totals(runs: list[dict[str, Any]], orchestration: Mapping[str, dict[str, Any]]) -> dict[str, dict[str, int]]: out: dict[str, dict[str, int]] = {} for name in LINES: out[name] = {c: 0 for c in (*CATEGORIES.values(), "total", "turns", "agents")} rows = [(r["line"], r) for r in runs] + [(f"orchestration-{o['kind']}", o) for o in orchestration.values()] for name, row in rows: for c in (*CATEGORIES.values(), "total", "turns"): out[name][c] += int(row[c]) out[name]["agents"] += 1 return out def agent_transcripts(subagents: Path, suffix: str) -> list[Path]: """The agent files of one session: its own helpers and every workflow's (measured 09.10, Claude Code 2.1.295).""" return sorted([*subagents.glob(f"agent-*.{suffix}"), *subagents.glob(f"workflows/*/agent-*.{suffix}")]) def harvest(transcripts: Path, okter: list[tuple[str, str]]) -> dict[str, Any]: runs: list[dict[str, Any]] = [] unattributed: list[dict[str, Any]] = [] orchestration: dict[str, dict[str, Any]] = {} for sid, kind in okter: top = transcripts / f"{sid}.jsonl" if not top.is_file(): # order 27 A3: a wrong transcripts folder would give 0 for a whole ledger without a word raise RunError(f"session {sid} in okter has no transcript {top}") orchestration[sid] = {"kind": kind, **usage(top, per_turn=True)} for path in agent_transcripts(transcripts / sid / "subagents", "jsonl"): meta_path = path.with_name(path.stem + ".meta.json") meta = json.loads(meta_path.read_text(encoding="utf-8")) if meta_path.is_file() else {} row = {"session": sid, "agent": path.stem, "agentType": meta.get("agentType"), "description": str(meta.get("description", "")), **usage(path)} m = PREFIX.match(row["description"]) if m is None: unattributed.append(row) continue row["run"], row["line"], row["unit"] = m.group(1), LINE_OF[m.group(2)], m.group(3) row["pack"] = pack_of(m.group(3)) runs.append(row) return {"sessions": [{"id": sid, "kind": kind} for sid, kind in okter], "runs": runs, "unattributed": unattributed, "orchestration": orchestration, "lines": line_totals(runs, orchestration)} def check_packs(runs: list[dict[str, Any]], packs_log: dict[str, Any]) -> None: logged = sorted(int(p["pack"]) for p in packs_log["packs"]) if logged != list(range(1, len(logged) + 1)): raise RunError(f"the pack log is not packs 1..{len(logged)}: {logged}") built = {int(r["pack"]) for r in runs if r["line"] == "build" and r["pack"] is not None} missing, extra = sorted(set(logged) - built), sorted(built - set(logged)) if missing or extra: raise RunError(f"pack log gap: logged without a build run {missing}, built but not logged {extra}") def allocate(pack_costs: Mapping[int, float], packs: list[list[str]], blocks_doc: dict[str, Any]) -> dict[str, float]: """Tokens per source: each pack's cost split over its sources by their share of the pack's blocks.""" count: dict[str, int] = {} for b in blocks_doc["blocks"]: count[b["file"]] = count.get(b["file"], 0) + 1 per: dict[str, float] = {} for n, names in enumerate(packs, 1): if n not in pack_costs: continue total = sum(count.get(x, 0) for x in names) or 1 for x in names: per[x] = pack_costs[n] * count.get(x, 0) / total return per def fit_line(points: list[tuple[float, float]]) -> tuple[float, float]: """Least squares y = a + b*x; one distinct x gives b = 0.""" xs, ys = [x for x, _ in points], [y for _, y in points] mx, my = sum(xs) / len(xs), sum(ys) / len(ys) if len(set(xs)) == 1: return my, 0.0 b = sum((x - mx) * (y - my) for x, y in points) / sum((x - mx) ** 2 for x in xs) return my - b * mx, b def project(points: list[tuple[float, int, int]], total_blocks: int) -> dict[str, Any]: """points: (cost, blocks in the unit, blocks read before it). Cost per block a + b*R, summed to a*T + b*T*T/2.""" if not points: return {"fit": None, "flat": None, "T": total_blocks} a, b = fit_line([(float(r), c / bl) for c, bl, r in points]) fit: dict[str, Any] = {"a": a, "b": b, "note": "one point, no growth term" if b == 0 and len(points) == 1 else f"{len(points)} points, not validated"} fit["value"] = a * total_blocks + b * total_blocks * total_blocks / 2 flat = sum(c for c, _, _ in points) / sum(bl for _, bl, _ in points) * total_blocks return {"fit": fit, "flat": flat, "T": total_blocks} def capped_lines(spec: str) -> list[str]: names = [x.strip() for x in spec.split(",") if x.strip()] for name in names: if name not in LINES: raise RunError(f"unknown cost line {name!r}; lines are {', '.join(LINES)}") if name in UNCAPPED: raise RunError(f"{name} is reported, never capped") if not names: raise RunError("no cost line given") return names def claims_projection(kost: dict[str, Any], lines: list[str], done: int, total: int, batches: tuple[int, int] | None) -> tuple[Fraction, dict[str, Any]]: """Helper lines per claim; each orchestration line's part before its session's first helper once, the rest per batch. Measured 11.10 in order 12: scaling the whole sum per claim projected 162.8 M where the reading cost 10.5 M, because the session's fixed start was scaled with the claims.""" helpers = sum(int(kost["lines"][name]["total"]) for name in lines if not name.startswith("orchestration-")) projected = Fraction(helpers) * total / done before = after = 0 for name in (x for x in lines if x.startswith("orchestration-")): kind = name.removeprefix("orchestration-") line_before = sum(int(o["before_first_helper"]) for o in kost.get("orchestration", {}).values() if o.get("kind") == kind) before += line_before after += int(kost["lines"][name]["total"]) - line_before parts: dict[str, Any] = {"helpers": helpers} if any(x.startswith("orchestration-") for x in lines): if batches is None: raise RunError("--unit claims with an orchestration line needs --batches-done and --batches-total") b_done, b_total = batches if b_done <= 0 or b_done > b_total: raise ValueError(f"--batches-done {b_done} --batches-total {b_total}: need 0 < done <= total") projected += before + Fraction(after) * b_total / b_done parts.update({"before_first_helper": before, "after_first_helper": after, "batches_done": b_done, "batches_total": b_total}) return projected, parts def ledger_rate(kost: dict[str, Any], packs_log: dict[str, Any], measured: int) -> tuple[Fraction, int]: """(cost per block, the blocks it is over): the measured cost over the blocks of the packs with a compiler run in this ledger, so packs built in another ledger (a continued collection) never thin it; ROUND2_PER_BLOCK before the ledger's first compiler run (order 27 A3). A compiler run for a pack the log lacks cannot run.""" blocks = {int(p["pack"]): int(p["blocks"]) for p in packs_log["packs"]} over = sum(blocks[n] for n in built_packs(kost, packs_log)) return (Fraction(measured, over) if over else Fraction(ROUND2_PER_BLOCK)), over def built_packs(kost: dict[str, Any], packs_log: dict[str, Any]) -> set[int]: """The packs with a compiler run in this ledger. A compiler run for a pack the log lacks cannot run.""" logged = {int(p["pack"]) for p in packs_log["packs"]} built: set[int] = set() for r in kost.get("runs", []): if r.get("line") == "build" and r.get("pack") is not None: if int(r["pack"]) not in logged: raise RunError(f"a compiler run is for pack {r['pack']}, which the pack log lacks") built.add(int(r["pack"])) return built def ledger_chars_rate(kost: dict[str, Any], packs_log: dict[str, Any], measured: int) -> tuple[Fraction, int] | None: """(cost per character, the characters it is over), as ledger_rate by the characters of the packs' blocks (`block_chars`), ROUND2_PER_CHAR before the ledger's first compiler run; None when a built pack has no `block_chars` (a log written before blocks had characters), which keeps the rule by blocks (B17).""" chars = {int(p["pack"]): p.get("block_chars") for p in packs_log["packs"]} built = built_packs(kost, packs_log) if any(not isinstance(chars[n], int) for n in built): return None over = sum(int(chars[n] or 0) for n in built) return (Fraction(measured, over) if over else ROUND2_PER_CHAR), over def stop_row(kost: dict[str, Any], lines: list[str], cap: int, unit: str, done: int, total: int, packs_log: dict[str, Any] | None, used_margin: float, batches: tuple[int, int] | None = None, following: int | None = None, next_chars: int | None = None) -> dict[str, Any]: if next_chars is not None and (following is None or next_chars < 0): raise ValueError(f"--next-chars {next_chars}: the characters of the next pack, with --next only") if done <= 0 or total <= 0 or done > total: raise ValueError(f"--done {done} --total {total}: need 0 < done <= total") measured = sum(int(kost["lines"][name]["total"]) for name in lines) flat = Fraction(measured) * total / done rate = Fraction(measured, done) parts: dict[str, Any] = {} if unit == "claims": flat, parts = claims_projection(kost, lines, done, total, batches) fit: float | None = None if unit == "blocks": if packs_log is None: raise RunError("--unit blocks needs --blocks (a pack log with pack and blocks)") rate, over = ledger_rate(kost, packs_log, measured) flat = measured + rate * (total - done) # the blocks not yet done at this ledger's rate parts.update(rate=float(rate), rate_blocks=over) attributed: dict[int, int] = {} for r in kost.get("runs", []): if r.get("line") in lines and r.get("pack") is not None: attributed[int(r["pack"])] = attributed.get(int(r["pack"]), 0) + int(r["total"]) if attributed: scale = measured / sum(attributed.values()) points: list[tuple[float, int, int]] = [] read = 0 for p in sorted(packs_log["packs"], key=lambda p: int(p["pack"])): n, blocks = int(p["pack"]), int(p["blocks"]) if n in attributed: points.append((attributed[n] * scale, blocks, read)) read += blocks fitted = project(points, total)["fit"] if len(points) >= FIT_FROM_PACKS else None fit = float(fitted["value"]) if fitted else None worst = max(flat, Fraction(fit)) if fit is not None else flat with_margin = worst * (1 + Fraction(str(used_margin))) row = {"lines": lines, "cap": cap, "unit": unit, "done": done, "total": total, "measured": measured, "flat": float(flat), "fit": fit, "margin": used_margin, "projected_with_margin": math.ceil(with_margin), "stop": with_margin > cap, **parts} if following is not None: if unit != "blocks" or following < 0: raise ValueError(f"--next {following}: the blocks of the next pack, with --unit blocks only") measured_next = (measured + rate * following) * (1 + Fraction(str(used_margin))) row.update(rule="measured", next=following, measured_next_with_margin=math.ceil(measured_next), stop=measured_next > cap) by_chars = ledger_chars_rate(kost, packs_log, measured) if next_chars is not None and packs_log else None if by_chars is not None: rate_c, over_c = by_chars measured_next = (measured + rate_c * int(next_chars or 0)) * (1 + Fraction(str(used_margin))) row.update(rule="measured-chars", next_chars=next_chars, rate_chars=float(rate_c), rate_chars_over=over_c, measured_next_with_margin=math.ceil(measured_next), stop=measured_next > cap) return row def with_margin(lower: int, used: float) -> int: return math.ceil(Fraction(lower) * (1 + Fraction(str(used)))) def summarize(kost: dict[str, Any], extra_groups: Mapping[str, dict[str, Any]]) -> tuple[list[str], bool]: used = float(kost["margin"]["used"]) out: list[str] = [] for name in LINES: lower = int(kost["lines"].get(name, {}).get("total", 0)) # a kost file from before a line existed note = " (reported, never capped)" if name in UNCAPPED else "" out.append(f"{name}: {lower} + margin {used:.2f} = {with_margin(lower, used)}{note}") within = True for name, group in {**kost.get("caps", {}), **extra_groups}.items(): lower = sum(int(kost["lines"][x]["total"]) for x in group["lines"]) total = with_margin(lower, used) within = within and total <= int(group["cap"]) out.append(f"{name}: {lower} + margin {used:.2f} = {total} of {group['cap']}") packs = {int(r["pack"]) for r in kost.get("runs", []) if r.get("pack") is not None} orchestration = kost.get("orchestration", {}) for kind in KINDS: mine = [o for o in orchestration.values() if o.get("kind") == kind] turns = sum(int(o["turns"]) for o in mine) if not turns: continue tokens = int(kost["lines"].get(f"orchestration-{kind}", {}).get("total", 0)) per_pack = f"; per pack {tokens // len(packs)} over {len(packs)} packs" if packs else "" out.append(f"orchestration-{kind} per turn: {tokens // turns} over {turns} turns{per_pack}") points = [(float(t["turn"]), float(t["input"])) for o in mine for t in o.get("per_turn", [])] if len({x for x, _ in points}) > 1: a, b = fit_line(points) out.append(f"orchestration-{kind} input per turn: {round(a)} + {round(b)} x turn ({len(points)} turns)") return out, within def parse_cap_group(text: str) -> tuple[str, dict[str, Any]]: name, sep, rest = text.partition("=") lines, sep2, cap = rest.rpartition(":") if not (sep and sep2 and name.strip()): raise RunError(f"--cap-group {text!r} is not NAME=LINES:CAP") return name.strip(), {"lines": capped_lines(lines), "cap": int(cap)} def part_list(tasks: Path, task: str) -> tuple[set[Path], Path]: """(the parts a repair helper may read, the one answer path it writes) from krepair task's part list.""" path = tasks / f"{task}.parts.json" if not path.is_file(): raise RunError(f"repair task {task} has no part list {path}") doc = load_json(path) parts, answer = doc.get("parts"), doc.get("answer") if not (isinstance(parts, list) and parts and all(isinstance(p, str) for p in parts) and isinstance(answer, str)): raise RunError(f"{path} is not a part list with parts and one answer path") return {(tasks / p).resolve() for p in parts}, (tasks / answer).resolve() def repair_breaks(path: Path, reads: set[Path], answer: Path) -> tuple[int, int, list[str]]: """(reads, writes, breaks) of one repair helper: every tool call is checked, not only Read, and a part it never read or an answer it wrote other than once is a break of its own (order 39).""" n_reads = n_writes = 0 seen: set[Path] = set() breaks: list[str] = [] for record in read_transcript(path) if path.is_file() else []: message = record.get("message") content = message.get("content") if isinstance(message, dict) else None for block in content if isinstance(content, list) else []: if not (isinstance(block, dict) and block.get("type") == "tool_use"): continue name = str(block.get("name")) target = (block.get("input") or {}).get("file_path") where = Path(str(target)).resolve() if isinstance(target, str) else None if name == "Read" and where in reads: n_reads += 1 seen.add(where) elif name == "Write" and where == answer: n_writes += 1 else: breaks.append(f"{name} {target}" if isinstance(target, str) else name) breaks.extend(f"part not read: {p}" for p in sorted(reads - seen)) if n_writes != 1: breaks.append("answer not written" if n_writes == 0 else f"answer written {n_writes} times") return n_reads, n_writes, breaks def isolation(transcripts: Path, run: str, tasks: Path, line: str = "read") -> dict[str, Any]: prefix = re.compile(rf"^V1-{re.escape(run)}-{re.escape(line)}-(\S+)$") per_agent: list[dict[str, Any]] = [] for meta_path in sorted(p for d in transcripts.glob("*/subagents") for p in agent_transcripts(d, "meta.json")): meta = json.loads(meta_path.read_text(encoding="utf-8")) m = prefix.match(str(meta.get("description", ""))) if m is None: continue path = meta_path.with_name(meta_path.name.removesuffix(".meta.json") + ".jsonl") if line == "repair": n_reads, n_writes, found = repair_breaks(path, *part_list(tasks, m.group(1))) per_agent.append({"agent": meta_path.name.removesuffix(".meta.json"), "claim": m.group(1), "reads": n_reads, "writes": n_writes, "breaks": found}) continue task = (tasks / f"{m.group(1)}.md").resolve() reads: dict[str, str] = {} for record in read_transcript(path) if path.is_file() else []: message = record.get("message") content = message.get("content") if isinstance(message, dict) else None for block in content if isinstance(content, list) else []: if isinstance(block, dict) and block.get("type") == "tool_use" and block.get("name") == "Read": target = str((block.get("input") or {}).get("file_path", "")) reads[str(block.get("id") or len(reads))] = target breaks = [t for t in reads.values() if Path(t).resolve() != task] per_agent.append({"agent": meta_path.name.removesuffix(".meta.json"), "claim": m.group(1), "reads": len(reads), "breaks": breaks}) return {"schema": ISOLATION_SCHEMA, "run": run, "line": line, "tasks": str(tasks), "agents": len(per_agent), "breaks": sum(len(a["breaks"]) for a in per_agent), "per_agent": per_agent} def load_json(path: Path) -> Any: return json.loads(path.read_text(encoding="utf-8")) def write_json(path: Path, doc: Any) -> None: path.parent.mkdir(parents=True, exist_ok=True) path.write_text(json.dumps(doc, ensure_ascii=False, indent=1) + "\n", encoding="utf-8") def build_parser() -> argparse.ArgumentParser: p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) sub = p.add_subparsers(dest="command", required=True) h = sub.add_parser("harvest") h.add_argument("--transcripts", required=True) h.add_argument("--okter", required=True) h.add_argument("--work", required=True) h.add_argument("--out", required=True) h.add_argument("--packs") h.add_argument("--margin", type=float) s = sub.add_parser("stop") s.add_argument("--lines", required=True) s.add_argument("--cap", type=int, required=True) s.add_argument("--unit", choices=UNITS, required=True) s.add_argument("--done", type=int, required=True) s.add_argument("--total", type=int, required=True) s.add_argument("--kost", required=True) s.add_argument("--blocks") s.add_argument("--batches-done", type=int) s.add_argument("--batches-total", type=int) s.add_argument("--next", type=int) s.add_argument("--next-chars", type=int) s.add_argument("--margin", type=float) s.add_argument("--name") m = sub.add_parser("summarize") m.add_argument("--kost", required=True) m.add_argument("--cap-group", action="append", default=[]) i = sub.add_parser("isolation") i.add_argument("--transcripts", required=True) i.add_argument("--run", required=True) i.add_argument("--tasks", required=True) i.add_argument("--out", required=True) i.add_argument("--line", choices=ISOLATED, default="read") return p def main(argv: list[str] | None = None) -> int: args = build_parser().parse_args(argv) try: if args.command == "harvest": out = Path(args.out) h = harvest(Path(args.transcripts), sessions(Path(args.okter))) if args.packs: check_packs(h["runs"], load_json(Path(args.packs))) old = load_json(out) if out.is_file() else {} doc = {"schema": SCHEMA, "work": str(args.work), **h, "margin": {"measured": args.margin, "used": margin(args.margin)}, "caps": old.get("caps", {}), "stops": old.get("stops", {})} write_json(out, doc) print(f"harvest: {len(h['runs'])} attributed runs, {len(h['unattributed'])} unattributed, " f"{len(h['orchestration'])} orchestrating sessions -> {out}") for u in h["unattributed"]: print(f" unattributed: {u['agent']} {u['description']!r}") return 1 if h["unattributed"] else 0 if args.command == "stop": kost_path = Path(args.kost) kost = load_json(kost_path) lines = capped_lines(args.lines) used = margin(args.margin) if args.margin is not None else float(kost["margin"]["used"]) packs_log = load_json(Path(args.blocks)) if args.blocks else None given = (args.batches_done, args.batches_total) if (given[0] is None) != (given[1] is None): raise RunError("--batches-done and --batches-total go together") batches = None if given[0] is None else (int(given[0]), int(given[1])) row = stop_row(kost, lines, args.cap, args.unit, args.done, args.total, packs_log, used, batches, args.next, args.next_chars) name = args.name or ",".join(lines) kost.setdefault("caps", {})[name] = {"lines": lines, "cap": args.cap} kost.setdefault("stops", {})[name] = row write_json(kost_path, kost) fit = f", fit {row['fit']:.0f}" if row["fit"] is not None else "" if "batches_total" in row: fit += (f" (helpers {row['helpers']} x {args.total}/{args.done}, before the first helper " f"{row['before_first_helper']} once, after it {row['after_first_helper']} x " f"{row['batches_total']}/{row['batches_done']} batches)") if row.get("rule") == "measured-chars": rate = (f"{row['measured']}/{row['rate_chars_over']} (the characters with a compiler run here)" if row["rate_chars_over"] else f"{float(ROUND2_PER_CHAR):.2f} (round 2's price per character; no compiler run here)") print(f"{name}: measured {row['measured']} + next {row['next_chars']} characters x {rate}, " f"x (1 + {used:.2f}) = {row['measured_next_with_margin']} of {args.cap}: " f"{'STOP' if row['stop'] else 'continue'}; projection (reported only): flat {row['flat']:.0f}{fit}, " f"x (1 + {used:.2f}) = {row['projected_with_margin']}") elif "rule" in row: rate = (f"{row['measured']}/{row['rate_blocks']} (the blocks with a compiler run here)" if row["rate_blocks"] else f"{ROUND2_PER_BLOCK} (round 2's price per block; no compiler run here)") print(f"{name}: measured {row['measured']} + next {row['next']} blocks x {rate}, " f"x (1 + {used:.2f}) = {row['measured_next_with_margin']} of {args.cap}: " f"{'STOP' if row['stop'] else 'continue'}; projection (reported only): flat {row['flat']:.0f}{fit}, " f"x (1 + {used:.2f}) = {row['projected_with_margin']}") else: print(f"{name}: measured {row['measured']}, flat {row['flat']:.0f}{fit}, x (1 + {used:.2f}) = " f"{row['projected_with_margin']} of {args.cap}: {'STOP' if row['stop'] else 'continue'}") return 1 if row["stop"] else 0 if args.command == "summarize": extra = dict(parse_cap_group(g) for g in args.cap_group) text, within = summarize(load_json(Path(args.kost)), extra) print("\n".join(text)) return 0 if within else 1 doc = isolation(Path(args.transcripts), args.run, Path(args.tasks), args.line) write_json(Path(args.out), doc) kind = {"read": "reader", "diagnose": "diagnosis", "repair": "repair"}[args.line] print(f"isolation: {doc['breaks']} breaks over {doc['agents']} {kind} agents") return 0 if doc["agents"] and doc["breaks"] == 0 else 1 except (RunError, OSError, json.JSONDecodeError, KeyError, ValueError, ZeroDivisionError, TypeError) as exc: print(f"cost cannot run: {exc}", file=sys.stderr) return 2 if __name__ == "__main__": sys.exit(main())