"""The trial's cost from the run sessions' transcripts (TRIAL, P1): harvest, allocation, projection, stop. No model. harvest --project-dir D --okter FILE --blocks F --total N [--packs FILE] --out FILE Reads only the sessions listed in okter.txt (" " per line): each session's main transcript (orchestration, reported apart) and every subagent transcript under /subagents/. Per subagent: input, cache read, cache write and output tokens kept apart (each message id counted once, at its largest value, because a streamed message repeats its id with growing output), turns (distinct message ids), startup (the first message's input side), agentType and description from its .meta.json. A message whose final usage was never written (no stop reason on any record) gets its output from the growth of the next message's input side less the injected text counted high (a lower bound); the recorded output stays beside it (output_recorded). A description that starts "P1--" attributes the run; any other is listed as unattributed, never dropped. Per variant: totals, tokens per source (a pack's cost split by block share), the means of sources 1-5 and 15-19, and the projection to all sources (N blocks). The file is rewritten from the transcripts on every harvest; stop rows already in it are kept. stop VARIANT --kost FILE --blocks F [--pages-total N --pages-first M] [--record] The stop rule (cap 1 M tokens, all four categories): K after pack 1, projected per block processed and, beside it, naively per source; K+ after run 1, projected over the pages to consolidate. Exit 1 = stop. With --record the stop row is written into the kost file. Projection: cost per block c = a + b*R by least squares over the measured units, R = blocks read before the unit; the projected total for T blocks is a*T + b*T*T/2 (the sum as R grows from 0 to T). One unit: b = 0. The flat projection (mean cost per block times T) stands beside it. Exit 0 = done, 1 = stop (stop only), 2 = could not run. """ from __future__ import annotations import argparse import json import re import sys from collections.abc import Mapping from pathlib import Path from typing import Any from direction_probe import USAGE_FIELDS, read_transcript SCHEMA = "okf-wiki/p1-cost/1" CAP = 1_000_000 CATEGORIES = {"input_tokens": "input", "cache_read_input_tokens": "cache_read", "cache_creation_input_tokens": "cache_write", "output_tokens": "output"} PREFIX = re.compile(r"^P1-([A-Za-z0-9+]+)-(\S+)") PACK_UNIT = re.compile(r"^pack(\d+)") CLASS = {"LOAD": "load", "K": "write", "K+": "write"} class RunError(Exception): """The command cannot run as asked; nothing was written.""" def sessions(okter: Path) -> list[str]: ids: list[str] = [] for line in okter.read_text(encoding="utf-8").splitlines(): parts = line.split() if parts and parts[0] not in ids: ids.append(parts[0]) return ids INPUT_SIDE = ("input_tokens", "cache_read_input_tokens", "cache_creation_input_tokens") CHARS_PER_TOKEN_LOW = 2.5 # injected text counted high, so a derived output is a lower bound def _result_chars(message: dict[str, Any]) -> int: content = message.get("content") if not isinstance(content, list): return 0 total = 0 for item in content: if not (isinstance(item, dict) and item.get("type") == "tool_result"): continue inner = item.get("content") if isinstance(inner, str): total += len(inner) elif isinstance(inner, list): total += sum(len(str(x.get("text", ""))) for x in inner if isinstance(x, dict)) return total def usage(path: Path) -> dict[str, Any]: """Tokens of one transcript. A message whose records all lack a stop reason never got its final usage written (seen for a long streamed message: output 2); when another message follows, its output is the growth of the next message's input side less the text injected between them (tool results and rendered attachments, counted high at CHARS_PER_TOKEN_LOW), a lower bound, and never below what was recorded.""" per: dict[str, dict[str, int]] = {} order: list[str] = [] complete: set[str] = set() between: dict[str, int] = {} last_stop: str | None = None for line in read_transcript(path): message = line.get("message") if line.get("type") == "attachment" and order: between[order[-1]] = between.get(order[-1], 0) + len(str(line.get("rendered") or "")) continue if not isinstance(message, dict): continue if line.get("type") == "user" and order: between[order[-1]] = between.get(order[-1], 0) + _result_chars(message) continue if line.get("type") != "assistant": continue mid = str(message.get("id", "")) if mid not in per: order.append(mid) seen = per.setdefault(mid, {}) fields = message.get("usage") or {} for f in USAGE_FIELDS: value = fields.get(f) if isinstance(value, int): seen[f] = max(seen.get(f, 0), value) stop = message.get("stop_reason") last_stop = stop if isinstance(stop, str) else None if isinstance(stop, str): complete.add(mid) out: dict[str, Any] = {CATEGORIES[f]: sum(v.get(f, 0) for v in per.values()) for f in USAGE_FIELDS} out["output_recorded"] = out["output"] derived = unrecorded = 0 for i, mid in enumerate(order): if mid in complete: continue if i + 1 == len(order): unrecorded += 1 continue growth = (sum(per[order[i + 1]].get(f, 0) for f in INPUT_SIDE) - sum(per[mid].get(f, 0) for f in INPUT_SIDE) - int(between.get(mid, 0) / CHARS_PER_TOKEN_LOW)) recorded = per[mid].get("output_tokens", 0) if growth > recorded: out["output"] += growth - recorded derived += 1 out["output_derived_messages"] = derived out["output_unrecorded"] = unrecorded out["total"] = sum(out[c] for c in CATEGORIES.values()) out["turns"] = len(order) first = per[order[0]] if order else {} out["startup"] = sum(first.get(f, 0) for f in INPUT_SIDE) out["stop_reason"] = last_stop return out def harvest(project_dir: Path, ids: list[str]) -> dict[str, Any]: runs: list[dict[str, Any]] = [] unattributed: list[dict[str, Any]] = [] main: dict[str, Any] = {} for sid in ids: top = project_dir / f"{sid}.jsonl" if top.is_file(): main[sid] = usage(top) for path in sorted((project_dir / sid / "subagents").glob("agent-*.jsonl")): meta_path = path.with_name(path.stem + ".meta.json") meta = json.loads(meta_path.read_text(encoding="utf-8")) if meta_path.is_file() else {} row = {"session": sid, "agent": path.stem, "agentType": meta.get("agentType"), "description": str(meta.get("description", "")), **usage(path)} m = PREFIX.match(row["description"]) if m is None: unattributed.append(row) continue row["variant"], row["unit"] = m.group(1), m.group(2) row["class"] = CLASS.get(m.group(1), "other") runs.append(row) return {"sessions_read": ids, "runs": runs, "unattributed": unattributed, "main": main} def allocate(pack_costs: Mapping[int, float], packs: list[list[str]], blocks_doc: dict[str, Any]) -> dict[str, float]: """Tokens per source: each pack's cost split over its sources by their share of the pack's blocks.""" count: dict[str, int] = {} for b in blocks_doc["blocks"]: count[b["file"]] = count.get(b["file"], 0) + 1 per: dict[str, float] = {} for n, names in enumerate(packs, 1): if n not in pack_costs: continue total = sum(count.get(x, 0) for x in names) or 1 for x in names: per[x] = pack_costs[n] * count.get(x, 0) / total return per def means(per: dict[str, float], order: list[str]) -> dict[str, float | None]: def mean(names: list[str]) -> float | None: vals = [per[x] for x in names if x in per] return sum(vals) / len(vals) if vals else None return {"1-5": mean(order[0:5]), "15-19": mean(order[14:19])} def project(points: list[tuple[float, int, int]], total_blocks: int) -> dict[str, Any]: """points: (cost, blocks in the unit, blocks read before it).""" if not points: return {"fit": None, "flat": None, "T": total_blocks} xs = [float(r) for _, _, r in points] ys = [c / b for c, b, _ in points] fit: dict[str, Any] if len(points) == 1 or len(set(xs)) == 1: a, b = sum(ys) / len(ys), 0.0 fit = {"a": a, "b": 0, "note": "one point, no growth term"} else: mx, my = sum(xs) / len(xs), sum(ys) / len(ys) b = sum((x - mx) * (y - my) for x, y in zip(xs, ys)) / sum((x - mx) ** 2 for x in xs) a = my - b * mx fit = {"a": a, "b": b, "note": f"{len(points)} points, not validated"} fit["value"] = a * total_blocks + b * total_blocks * total_blocks / 2 flat = sum(c for c, _, _ in points) / sum(bl for _, bl, _ in points) * total_blocks return {"fit": fit, "flat": flat, "T": total_blocks} def stop_k(pack1_cost: int, pack1_blocks: int, cut_blocks: int, pack1_sources: int, cut_sources: int ) -> dict[str, Any]: per_block = pack1_cost * cut_blocks / pack1_blocks per_source = pack1_cost * cut_sources / pack1_sources return {"per_block": per_block, "per_source": per_source, "cap": CAP, "stop": per_block > CAP} def stop_kplus(run1_cost: int, run1_pages: int, pages_total: int) -> dict[str, Any]: projected = run1_cost * pages_total / run1_pages return {"projected": projected, "cap": CAP, "stop": projected > CAP} def _pack_number(unit: str) -> int | None: m = PACK_UNIT.match(unit) return int(m.group(1)) if m else None def summarize(h: dict[str, Any], blocks_doc: dict[str, Any], packs_log: dict[str, Any] | None, total_blocks: int) -> dict[str, Any]: order = [str(x) for x in blocks_doc["order"]] count: dict[str, int] = {} for b in blocks_doc["blocks"]: count[b["file"]] = count.get(b["file"], 0) + 1 variants: dict[str, Any] = {} for v in sorted({r["variant"] for r in h["runs"]}): mine = [r for r in h["runs"] if r["variant"] == v] row: dict[str, Any] = {c: sum(r[c] for r in mine) for c in (*CATEGORIES.values(), "total", "turns", "startup")} row["subagents"] = len(mine) row["class"] = mine[0]["class"] row["counts_toward_cap"] = v in ("K", "K+") if v == "K" and packs_log: packs = [list(p["sources"]) for p in sorted(packs_log["packs"], key=lambda p: int(p["pack"]))] pack_costs: dict[int, float] = {} per_cat: dict[str, dict[int, float]] = {c: {} for c in CATEGORIES.values()} for r in mine: n = _pack_number(r["unit"]) if n is not None: pack_costs[n] = pack_costs.get(n, 0) + r["total"] for c in CATEGORIES.values(): per_cat[c][n] = per_cat[c].get(n, 0) + r[c] row["per_pack"] = {str(n): c for n, c in sorted(pack_costs.items())} row["per_source"] = allocate(pack_costs, packs, blocks_doc) row["per_source_by_category"] = {c: allocate(per_cat[c], packs, blocks_doc) for c in CATEGORIES.values()} row["means"] = means(row["per_source"], order) points, read = [], 0 for n, names in enumerate(packs, 1): blocks_n = sum(count.get(x, 0) for x in names) if n in pack_costs: points.append((pack_costs[n], blocks_n, read)) read += blocks_n row["projection"] = project(points, total_blocks) variants[v] = row return variants def build_parser() -> argparse.ArgumentParser: p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) sub = p.add_subparsers(dest="command", required=True) h = sub.add_parser("harvest") h.add_argument("--project-dir", required=True) h.add_argument("--okter", required=True) h.add_argument("--blocks", required=True) h.add_argument("--total", type=int, required=True) h.add_argument("--packs") h.add_argument("--out", required=True) s = sub.add_parser("stop") s.add_argument("variant", choices=("K", "K+")) s.add_argument("--kost", required=True) s.add_argument("--blocks", required=True) s.add_argument("--packs") s.add_argument("--pages-total", type=int) s.add_argument("--pages-first", type=int) s.add_argument("--record", action="store_true") return p def main(argv: list[str] | None = None) -> int: args = build_parser().parse_args(argv) try: blocks_doc = json.loads(Path(args.blocks).read_text(encoding="utf-8")) if args.command == "harvest": out = Path(args.out) old = json.loads(out.read_text(encoding="utf-8")) if out.is_file() else {} h = harvest(Path(args.project_dir), sessions(Path(args.okter))) packs_log = json.loads(Path(args.packs).read_text(encoding="utf-8")) if args.packs else None doc = {"schema": SCHEMA, **h, "variants": summarize(h, blocks_doc, packs_log, args.total), "stops": old.get("stops", {})} out.parent.mkdir(parents=True, exist_ok=True) out.write_text(json.dumps(doc, ensure_ascii=False, indent=1) + "\n", encoding="utf-8") print(f"harvest: {len(h['runs'])} attributed runs, {len(h['unattributed'])} unattributed, sessions read: " f"{', '.join(h['sessions_read'])} -> {out}") return 0 kost_path = Path(args.kost) kost = json.loads(kost_path.read_text(encoding="utf-8")) runs = [r for r in kost["runs"] if r["variant"] == args.variant] if args.variant == "K": first = [r for r in runs if _pack_number(r["unit"]) == 1] if not first: raise RunError("no pack 1 run in the kost file; harvest after pack 1") if not args.packs: raise RunError("--packs is needed for K") log = json.loads(Path(args.packs).read_text(encoding="utf-8")) p1 = next(p for p in log["packs"] if int(p["pack"]) == 1) row = stop_k(sum(r["total"] for r in first), int(p1["blocks"]), len(blocks_doc["blocks"]), len(p1["sources"]), len(blocks_doc["order"])) else: first = [r for r in runs if r["unit"].startswith("run1")] if not first or not args.pages_total or not args.pages_first: raise RunError("K+ needs a run 1 in the kost file, --pages-total and --pages-first") row = stop_kplus(sum(r["total"] for r in first), args.pages_first, args.pages_total) print(f"{args.variant}: projected {row.get('per_block', row.get('projected'))} tokens" + (f" (naive per source {row['per_source']})" if "per_source" in row else "") + f", cap {CAP}: {'STOP' if row['stop'] else 'continue'}") if args.record: kost.setdefault("stops", {})[args.variant] = row kost_path.write_text(json.dumps(kost, ensure_ascii=False, indent=1) + "\n", encoding="utf-8") return 1 if row["stop"] else 0 except (RunError, OSError, json.JSONDecodeError, KeyError, StopIteration) as exc: print(f"cost cannot run: {exc}", file=sys.stderr) return 2 if __name__ == "__main__": sys.exit(main())