First public release, published as one snapshot. What changed between versions is in CHANGELOG.md. Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
329 lines
16 KiB
Python
329 lines
16 KiB
Python
"""The trial's cost from the run sessions' transcripts (TRIAL, P1): harvest, allocation, projection, stop. No model.
|
|
|
|
harvest --project-dir D --okter FILE --blocks F --total N [--packs FILE] --out FILE
|
|
Reads only the sessions listed in okter.txt ("<session-id> <step>" per line): each session's main
|
|
transcript (orchestration, reported apart) and every subagent transcript under <session>/subagents/. Per
|
|
subagent: input, cache read, cache write and output tokens kept apart (each message id counted once, at
|
|
its largest value, because a streamed message repeats its id with growing output), turns (distinct
|
|
message ids), startup (the first message's input side), agentType and description from its .meta.json.
|
|
A message whose final usage was never written (no stop reason on any record) gets its output from the
|
|
growth of the next message's input side less the injected text counted high (a lower bound); the
|
|
recorded output stays beside it (output_recorded).
|
|
A description that starts "P1-<variant>-<unit>" attributes the run; any other is listed as unattributed,
|
|
never dropped. Per variant: totals, tokens per source (a pack's cost split by block share), the means of
|
|
sources 1-5 and 15-19, and the projection to all sources (N blocks). The file is rewritten from the
|
|
transcripts on every harvest; stop rows already in it are kept.
|
|
stop VARIANT --kost FILE --blocks F [--pages-total N --pages-first M] [--record]
|
|
The stop rule (cap 1 M tokens, all four categories): K after pack 1, projected per block processed and,
|
|
beside it, naively per source; K+ after run 1, projected over the pages to consolidate. Exit 1 = stop.
|
|
With --record the stop row is written into the kost file.
|
|
|
|
Projection: cost per block c = a + b*R by least squares over the measured units, R = blocks read before the unit;
|
|
the projected total for T blocks is a*T + b*T*T/2 (the sum as R grows from 0 to T). One unit: b = 0. The flat
|
|
projection (mean cost per block times T) stands beside it. Exit 0 = done, 1 = stop (stop only), 2 = could not run.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import re
|
|
import sys
|
|
from collections.abc import Mapping
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
from direction_probe import USAGE_FIELDS, read_transcript
|
|
|
|
SCHEMA = "okf-wiki/p1-cost/1"
|
|
CAP = 1_000_000
|
|
CATEGORIES = {"input_tokens": "input", "cache_read_input_tokens": "cache_read",
|
|
"cache_creation_input_tokens": "cache_write", "output_tokens": "output"}
|
|
PREFIX = re.compile(r"^P1-([A-Za-z0-9+]+)-(\S+)")
|
|
PACK_UNIT = re.compile(r"^pack(\d+)")
|
|
CLASS = {"LOAD": "load", "K": "write", "K+": "write"}
|
|
|
|
|
|
class RunError(Exception):
|
|
"""The command cannot run as asked; nothing was written."""
|
|
|
|
|
|
def sessions(okter: Path) -> list[str]:
|
|
ids: list[str] = []
|
|
for line in okter.read_text(encoding="utf-8").splitlines():
|
|
parts = line.split()
|
|
if parts and parts[0] not in ids:
|
|
ids.append(parts[0])
|
|
return ids
|
|
|
|
|
|
INPUT_SIDE = ("input_tokens", "cache_read_input_tokens", "cache_creation_input_tokens")
|
|
CHARS_PER_TOKEN_LOW = 2.5 # injected text counted high, so a derived output is a lower bound
|
|
|
|
|
|
def _result_chars(message: dict[str, Any]) -> int:
|
|
content = message.get("content")
|
|
if not isinstance(content, list):
|
|
return 0
|
|
total = 0
|
|
for item in content:
|
|
if not (isinstance(item, dict) and item.get("type") == "tool_result"):
|
|
continue
|
|
inner = item.get("content")
|
|
if isinstance(inner, str):
|
|
total += len(inner)
|
|
elif isinstance(inner, list):
|
|
total += sum(len(str(x.get("text", ""))) for x in inner if isinstance(x, dict))
|
|
return total
|
|
|
|
|
|
def usage(path: Path) -> dict[str, Any]:
|
|
"""Tokens of one transcript. A message whose records all lack a stop reason never got its final usage written
|
|
(seen for a long streamed message: output 2); when another message follows, its output is the growth of the
|
|
next message's input side less the text injected between them (tool results and rendered attachments, counted
|
|
high at CHARS_PER_TOKEN_LOW), a lower bound, and never below what was recorded."""
|
|
per: dict[str, dict[str, int]] = {}
|
|
order: list[str] = []
|
|
complete: set[str] = set()
|
|
between: dict[str, int] = {}
|
|
last_stop: str | None = None
|
|
for line in read_transcript(path):
|
|
message = line.get("message")
|
|
if line.get("type") == "attachment" and order:
|
|
between[order[-1]] = between.get(order[-1], 0) + len(str(line.get("rendered") or ""))
|
|
continue
|
|
if not isinstance(message, dict):
|
|
continue
|
|
if line.get("type") == "user" and order:
|
|
between[order[-1]] = between.get(order[-1], 0) + _result_chars(message)
|
|
continue
|
|
if line.get("type") != "assistant":
|
|
continue
|
|
mid = str(message.get("id", ""))
|
|
if mid not in per:
|
|
order.append(mid)
|
|
seen = per.setdefault(mid, {})
|
|
fields = message.get("usage") or {}
|
|
for f in USAGE_FIELDS:
|
|
value = fields.get(f)
|
|
if isinstance(value, int):
|
|
seen[f] = max(seen.get(f, 0), value)
|
|
stop = message.get("stop_reason")
|
|
last_stop = stop if isinstance(stop, str) else None
|
|
if isinstance(stop, str):
|
|
complete.add(mid)
|
|
out: dict[str, Any] = {CATEGORIES[f]: sum(v.get(f, 0) for v in per.values()) for f in USAGE_FIELDS}
|
|
out["output_recorded"] = out["output"]
|
|
derived = unrecorded = 0
|
|
for i, mid in enumerate(order):
|
|
if mid in complete:
|
|
continue
|
|
if i + 1 == len(order):
|
|
unrecorded += 1
|
|
continue
|
|
growth = (sum(per[order[i + 1]].get(f, 0) for f in INPUT_SIDE) - sum(per[mid].get(f, 0) for f in INPUT_SIDE)
|
|
- int(between.get(mid, 0) / CHARS_PER_TOKEN_LOW))
|
|
recorded = per[mid].get("output_tokens", 0)
|
|
if growth > recorded:
|
|
out["output"] += growth - recorded
|
|
derived += 1
|
|
out["output_derived_messages"] = derived
|
|
out["output_unrecorded"] = unrecorded
|
|
out["total"] = sum(out[c] for c in CATEGORIES.values())
|
|
out["turns"] = len(order)
|
|
first = per[order[0]] if order else {}
|
|
out["startup"] = sum(first.get(f, 0) for f in INPUT_SIDE)
|
|
out["stop_reason"] = last_stop
|
|
return out
|
|
|
|
|
|
def harvest(project_dir: Path, ids: list[str]) -> dict[str, Any]:
|
|
runs: list[dict[str, Any]] = []
|
|
unattributed: list[dict[str, Any]] = []
|
|
main: dict[str, Any] = {}
|
|
for sid in ids:
|
|
top = project_dir / f"{sid}.jsonl"
|
|
if top.is_file():
|
|
main[sid] = usage(top)
|
|
for path in sorted((project_dir / sid / "subagents").glob("agent-*.jsonl")):
|
|
meta_path = path.with_name(path.stem + ".meta.json")
|
|
meta = json.loads(meta_path.read_text(encoding="utf-8")) if meta_path.is_file() else {}
|
|
row = {"session": sid, "agent": path.stem, "agentType": meta.get("agentType"),
|
|
"description": str(meta.get("description", "")), **usage(path)}
|
|
m = PREFIX.match(row["description"])
|
|
if m is None:
|
|
unattributed.append(row)
|
|
continue
|
|
row["variant"], row["unit"] = m.group(1), m.group(2)
|
|
row["class"] = CLASS.get(m.group(1), "other")
|
|
runs.append(row)
|
|
return {"sessions_read": ids, "runs": runs, "unattributed": unattributed, "main": main}
|
|
|
|
|
|
def allocate(pack_costs: Mapping[int, float], packs: list[list[str]], blocks_doc: dict[str, Any]) -> dict[str, float]:
|
|
"""Tokens per source: each pack's cost split over its sources by their share of the pack's blocks."""
|
|
count: dict[str, int] = {}
|
|
for b in blocks_doc["blocks"]:
|
|
count[b["file"]] = count.get(b["file"], 0) + 1
|
|
per: dict[str, float] = {}
|
|
for n, names in enumerate(packs, 1):
|
|
if n not in pack_costs:
|
|
continue
|
|
total = sum(count.get(x, 0) for x in names) or 1
|
|
for x in names:
|
|
per[x] = pack_costs[n] * count.get(x, 0) / total
|
|
return per
|
|
|
|
|
|
def means(per: dict[str, float], order: list[str]) -> dict[str, float | None]:
|
|
def mean(names: list[str]) -> float | None:
|
|
vals = [per[x] for x in names if x in per]
|
|
return sum(vals) / len(vals) if vals else None
|
|
|
|
return {"1-5": mean(order[0:5]), "15-19": mean(order[14:19])}
|
|
|
|
|
|
def project(points: list[tuple[float, int, int]], total_blocks: int) -> dict[str, Any]:
|
|
"""points: (cost, blocks in the unit, blocks read before it)."""
|
|
if not points:
|
|
return {"fit": None, "flat": None, "T": total_blocks}
|
|
xs = [float(r) for _, _, r in points]
|
|
ys = [c / b for c, b, _ in points]
|
|
fit: dict[str, Any]
|
|
if len(points) == 1 or len(set(xs)) == 1:
|
|
a, b = sum(ys) / len(ys), 0.0
|
|
fit = {"a": a, "b": 0, "note": "one point, no growth term"}
|
|
else:
|
|
mx, my = sum(xs) / len(xs), sum(ys) / len(ys)
|
|
b = sum((x - mx) * (y - my) for x, y in zip(xs, ys)) / sum((x - mx) ** 2 for x in xs)
|
|
a = my - b * mx
|
|
fit = {"a": a, "b": b, "note": f"{len(points)} points, not validated"}
|
|
fit["value"] = a * total_blocks + b * total_blocks * total_blocks / 2
|
|
flat = sum(c for c, _, _ in points) / sum(bl for _, bl, _ in points) * total_blocks
|
|
return {"fit": fit, "flat": flat, "T": total_blocks}
|
|
|
|
|
|
def stop_k(pack1_cost: int, pack1_blocks: int, cut_blocks: int, pack1_sources: int, cut_sources: int
|
|
) -> dict[str, Any]:
|
|
per_block = pack1_cost * cut_blocks / pack1_blocks
|
|
per_source = pack1_cost * cut_sources / pack1_sources
|
|
return {"per_block": per_block, "per_source": per_source, "cap": CAP, "stop": per_block > CAP}
|
|
|
|
|
|
def stop_kplus(run1_cost: int, run1_pages: int, pages_total: int) -> dict[str, Any]:
|
|
projected = run1_cost * pages_total / run1_pages
|
|
return {"projected": projected, "cap": CAP, "stop": projected > CAP}
|
|
|
|
|
|
def _pack_number(unit: str) -> int | None:
|
|
m = PACK_UNIT.match(unit)
|
|
return int(m.group(1)) if m else None
|
|
|
|
|
|
def summarize(h: dict[str, Any], blocks_doc: dict[str, Any], packs_log: dict[str, Any] | None,
|
|
total_blocks: int) -> dict[str, Any]:
|
|
order = [str(x) for x in blocks_doc["order"]]
|
|
count: dict[str, int] = {}
|
|
for b in blocks_doc["blocks"]:
|
|
count[b["file"]] = count.get(b["file"], 0) + 1
|
|
variants: dict[str, Any] = {}
|
|
for v in sorted({r["variant"] for r in h["runs"]}):
|
|
mine = [r for r in h["runs"] if r["variant"] == v]
|
|
row: dict[str, Any] = {c: sum(r[c] for r in mine) for c in (*CATEGORIES.values(), "total", "turns", "startup")}
|
|
row["subagents"] = len(mine)
|
|
row["class"] = mine[0]["class"]
|
|
row["counts_toward_cap"] = v in ("K", "K+")
|
|
if v == "K" and packs_log:
|
|
packs = [list(p["sources"]) for p in sorted(packs_log["packs"], key=lambda p: int(p["pack"]))]
|
|
pack_costs: dict[int, float] = {}
|
|
per_cat: dict[str, dict[int, float]] = {c: {} for c in CATEGORIES.values()}
|
|
for r in mine:
|
|
n = _pack_number(r["unit"])
|
|
if n is not None:
|
|
pack_costs[n] = pack_costs.get(n, 0) + r["total"]
|
|
for c in CATEGORIES.values():
|
|
per_cat[c][n] = per_cat[c].get(n, 0) + r[c]
|
|
row["per_pack"] = {str(n): c for n, c in sorted(pack_costs.items())}
|
|
row["per_source"] = allocate(pack_costs, packs, blocks_doc)
|
|
row["per_source_by_category"] = {c: allocate(per_cat[c], packs, blocks_doc) for c in CATEGORIES.values()}
|
|
row["means"] = means(row["per_source"], order)
|
|
points, read = [], 0
|
|
for n, names in enumerate(packs, 1):
|
|
blocks_n = sum(count.get(x, 0) for x in names)
|
|
if n in pack_costs:
|
|
points.append((pack_costs[n], blocks_n, read))
|
|
read += blocks_n
|
|
row["projection"] = project(points, total_blocks)
|
|
variants[v] = row
|
|
return variants
|
|
|
|
|
|
def build_parser() -> argparse.ArgumentParser:
|
|
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
sub = p.add_subparsers(dest="command", required=True)
|
|
h = sub.add_parser("harvest")
|
|
h.add_argument("--project-dir", required=True)
|
|
h.add_argument("--okter", required=True)
|
|
h.add_argument("--blocks", required=True)
|
|
h.add_argument("--total", type=int, required=True)
|
|
h.add_argument("--packs")
|
|
h.add_argument("--out", required=True)
|
|
s = sub.add_parser("stop")
|
|
s.add_argument("variant", choices=("K", "K+"))
|
|
s.add_argument("--kost", required=True)
|
|
s.add_argument("--blocks", required=True)
|
|
s.add_argument("--packs")
|
|
s.add_argument("--pages-total", type=int)
|
|
s.add_argument("--pages-first", type=int)
|
|
s.add_argument("--record", action="store_true")
|
|
return p
|
|
|
|
|
|
def main(argv: list[str] | None = None) -> int:
|
|
args = build_parser().parse_args(argv)
|
|
try:
|
|
blocks_doc = json.loads(Path(args.blocks).read_text(encoding="utf-8"))
|
|
if args.command == "harvest":
|
|
out = Path(args.out)
|
|
old = json.loads(out.read_text(encoding="utf-8")) if out.is_file() else {}
|
|
h = harvest(Path(args.project_dir), sessions(Path(args.okter)))
|
|
packs_log = json.loads(Path(args.packs).read_text(encoding="utf-8")) if args.packs else None
|
|
doc = {"schema": SCHEMA, **h, "variants": summarize(h, blocks_doc, packs_log, args.total),
|
|
"stops": old.get("stops", {})}
|
|
out.parent.mkdir(parents=True, exist_ok=True)
|
|
out.write_text(json.dumps(doc, ensure_ascii=False, indent=1) + "\n", encoding="utf-8")
|
|
print(f"harvest: {len(h['runs'])} attributed runs, {len(h['unattributed'])} unattributed, sessions read: "
|
|
f"{', '.join(h['sessions_read'])} -> {out}")
|
|
return 0
|
|
kost_path = Path(args.kost)
|
|
kost = json.loads(kost_path.read_text(encoding="utf-8"))
|
|
runs = [r for r in kost["runs"] if r["variant"] == args.variant]
|
|
if args.variant == "K":
|
|
first = [r for r in runs if _pack_number(r["unit"]) == 1]
|
|
if not first:
|
|
raise RunError("no pack 1 run in the kost file; harvest after pack 1")
|
|
if not args.packs:
|
|
raise RunError("--packs is needed for K")
|
|
log = json.loads(Path(args.packs).read_text(encoding="utf-8"))
|
|
p1 = next(p for p in log["packs"] if int(p["pack"]) == 1)
|
|
row = stop_k(sum(r["total"] for r in first), int(p1["blocks"]), len(blocks_doc["blocks"]),
|
|
len(p1["sources"]), len(blocks_doc["order"]))
|
|
else:
|
|
first = [r for r in runs if r["unit"].startswith("run1")]
|
|
if not first or not args.pages_total or not args.pages_first:
|
|
raise RunError("K+ needs a run 1 in the kost file, --pages-total and --pages-first")
|
|
row = stop_kplus(sum(r["total"] for r in first), args.pages_first, args.pages_total)
|
|
print(f"{args.variant}: projected {row.get('per_block', row.get('projected'))} tokens"
|
|
+ (f" (naive per source {row['per_source']})" if "per_source" in row else "")
|
|
+ f", cap {CAP}: {'STOP' if row['stop'] else 'continue'}")
|
|
if args.record:
|
|
kost.setdefault("stops", {})[args.variant] = row
|
|
kost_path.write_text(json.dumps(kost, ensure_ascii=False, indent=1) + "\n", encoding="utf-8")
|
|
return 1 if row["stop"] else 0
|
|
except (RunError, OSError, json.JSONDecodeError, KeyError, StopIteration) as exc:
|
|
print(f"cost cannot run: {exc}", file=sys.stderr)
|
|
return 2
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|