okf-wiki/evals/p1/cost.py
Kjell Tore Guttormsen 0fd86a23a3
okf-wiki 1.1.1
First public release, published as one snapshot. What changed between
versions is in CHANGELOG.md.

Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
2026-11-14 21:03:20 +01:00

329 lines
16 KiB
Python

"""The trial's cost from the run sessions' transcripts (TRIAL, P1): harvest, allocation, projection, stop. No model.
harvest --project-dir D --okter FILE --blocks F --total N [--packs FILE] --out FILE
Reads only the sessions listed in okter.txt ("<session-id> <step>" per line): each session's main
transcript (orchestration, reported apart) and every subagent transcript under <session>/subagents/. Per
subagent: input, cache read, cache write and output tokens kept apart (each message id counted once, at
its largest value, because a streamed message repeats its id with growing output), turns (distinct
message ids), startup (the first message's input side), agentType and description from its .meta.json.
A message whose final usage was never written (no stop reason on any record) gets its output from the
growth of the next message's input side less the injected text counted high (a lower bound); the
recorded output stays beside it (output_recorded).
A description that starts "P1-<variant>-<unit>" attributes the run; any other is listed as unattributed,
never dropped. Per variant: totals, tokens per source (a pack's cost split by block share), the means of
sources 1-5 and 15-19, and the projection to all sources (N blocks). The file is rewritten from the
transcripts on every harvest; stop rows already in it are kept.
stop VARIANT --kost FILE --blocks F [--pages-total N --pages-first M] [--record]
The stop rule (cap 1 M tokens, all four categories): K after pack 1, projected per block processed and,
beside it, naively per source; K+ after run 1, projected over the pages to consolidate. Exit 1 = stop.
With --record the stop row is written into the kost file.
Projection: cost per block c = a + b*R by least squares over the measured units, R = blocks read before the unit;
the projected total for T blocks is a*T + b*T*T/2 (the sum as R grows from 0 to T). One unit: b = 0. The flat
projection (mean cost per block times T) stands beside it. Exit 0 = done, 1 = stop (stop only), 2 = could not run.
"""
from __future__ import annotations
import argparse
import json
import re
import sys
from collections.abc import Mapping
from pathlib import Path
from typing import Any
from direction_probe import USAGE_FIELDS, read_transcript
SCHEMA = "okf-wiki/p1-cost/1"
CAP = 1_000_000
CATEGORIES = {"input_tokens": "input", "cache_read_input_tokens": "cache_read",
"cache_creation_input_tokens": "cache_write", "output_tokens": "output"}
PREFIX = re.compile(r"^P1-([A-Za-z0-9+]+)-(\S+)")
PACK_UNIT = re.compile(r"^pack(\d+)")
CLASS = {"LOAD": "load", "K": "write", "K+": "write"}
class RunError(Exception):
"""The command cannot run as asked; nothing was written."""
def sessions(okter: Path) -> list[str]:
ids: list[str] = []
for line in okter.read_text(encoding="utf-8").splitlines():
parts = line.split()
if parts and parts[0] not in ids:
ids.append(parts[0])
return ids
INPUT_SIDE = ("input_tokens", "cache_read_input_tokens", "cache_creation_input_tokens")
CHARS_PER_TOKEN_LOW = 2.5 # injected text counted high, so a derived output is a lower bound
def _result_chars(message: dict[str, Any]) -> int:
content = message.get("content")
if not isinstance(content, list):
return 0
total = 0
for item in content:
if not (isinstance(item, dict) and item.get("type") == "tool_result"):
continue
inner = item.get("content")
if isinstance(inner, str):
total += len(inner)
elif isinstance(inner, list):
total += sum(len(str(x.get("text", ""))) for x in inner if isinstance(x, dict))
return total
def usage(path: Path) -> dict[str, Any]:
"""Tokens of one transcript. A message whose records all lack a stop reason never got its final usage written
(seen for a long streamed message: output 2); when another message follows, its output is the growth of the
next message's input side less the text injected between them (tool results and rendered attachments, counted
high at CHARS_PER_TOKEN_LOW), a lower bound, and never below what was recorded."""
per: dict[str, dict[str, int]] = {}
order: list[str] = []
complete: set[str] = set()
between: dict[str, int] = {}
last_stop: str | None = None
for line in read_transcript(path):
message = line.get("message")
if line.get("type") == "attachment" and order:
between[order[-1]] = between.get(order[-1], 0) + len(str(line.get("rendered") or ""))
continue
if not isinstance(message, dict):
continue
if line.get("type") == "user" and order:
between[order[-1]] = between.get(order[-1], 0) + _result_chars(message)
continue
if line.get("type") != "assistant":
continue
mid = str(message.get("id", ""))
if mid not in per:
order.append(mid)
seen = per.setdefault(mid, {})
fields = message.get("usage") or {}
for f in USAGE_FIELDS:
value = fields.get(f)
if isinstance(value, int):
seen[f] = max(seen.get(f, 0), value)
stop = message.get("stop_reason")
last_stop = stop if isinstance(stop, str) else None
if isinstance(stop, str):
complete.add(mid)
out: dict[str, Any] = {CATEGORIES[f]: sum(v.get(f, 0) for v in per.values()) for f in USAGE_FIELDS}
out["output_recorded"] = out["output"]
derived = unrecorded = 0
for i, mid in enumerate(order):
if mid in complete:
continue
if i + 1 == len(order):
unrecorded += 1
continue
growth = (sum(per[order[i + 1]].get(f, 0) for f in INPUT_SIDE) - sum(per[mid].get(f, 0) for f in INPUT_SIDE)
- int(between.get(mid, 0) / CHARS_PER_TOKEN_LOW))
recorded = per[mid].get("output_tokens", 0)
if growth > recorded:
out["output"] += growth - recorded
derived += 1
out["output_derived_messages"] = derived
out["output_unrecorded"] = unrecorded
out["total"] = sum(out[c] for c in CATEGORIES.values())
out["turns"] = len(order)
first = per[order[0]] if order else {}
out["startup"] = sum(first.get(f, 0) for f in INPUT_SIDE)
out["stop_reason"] = last_stop
return out
def harvest(project_dir: Path, ids: list[str]) -> dict[str, Any]:
runs: list[dict[str, Any]] = []
unattributed: list[dict[str, Any]] = []
main: dict[str, Any] = {}
for sid in ids:
top = project_dir / f"{sid}.jsonl"
if top.is_file():
main[sid] = usage(top)
for path in sorted((project_dir / sid / "subagents").glob("agent-*.jsonl")):
meta_path = path.with_name(path.stem + ".meta.json")
meta = json.loads(meta_path.read_text(encoding="utf-8")) if meta_path.is_file() else {}
row = {"session": sid, "agent": path.stem, "agentType": meta.get("agentType"),
"description": str(meta.get("description", "")), **usage(path)}
m = PREFIX.match(row["description"])
if m is None:
unattributed.append(row)
continue
row["variant"], row["unit"] = m.group(1), m.group(2)
row["class"] = CLASS.get(m.group(1), "other")
runs.append(row)
return {"sessions_read": ids, "runs": runs, "unattributed": unattributed, "main": main}
def allocate(pack_costs: Mapping[int, float], packs: list[list[str]], blocks_doc: dict[str, Any]) -> dict[str, float]:
"""Tokens per source: each pack's cost split over its sources by their share of the pack's blocks."""
count: dict[str, int] = {}
for b in blocks_doc["blocks"]:
count[b["file"]] = count.get(b["file"], 0) + 1
per: dict[str, float] = {}
for n, names in enumerate(packs, 1):
if n not in pack_costs:
continue
total = sum(count.get(x, 0) for x in names) or 1
for x in names:
per[x] = pack_costs[n] * count.get(x, 0) / total
return per
def means(per: dict[str, float], order: list[str]) -> dict[str, float | None]:
def mean(names: list[str]) -> float | None:
vals = [per[x] for x in names if x in per]
return sum(vals) / len(vals) if vals else None
return {"1-5": mean(order[0:5]), "15-19": mean(order[14:19])}
def project(points: list[tuple[float, int, int]], total_blocks: int) -> dict[str, Any]:
"""points: (cost, blocks in the unit, blocks read before it)."""
if not points:
return {"fit": None, "flat": None, "T": total_blocks}
xs = [float(r) for _, _, r in points]
ys = [c / b for c, b, _ in points]
fit: dict[str, Any]
if len(points) == 1 or len(set(xs)) == 1:
a, b = sum(ys) / len(ys), 0.0
fit = {"a": a, "b": 0, "note": "one point, no growth term"}
else:
mx, my = sum(xs) / len(xs), sum(ys) / len(ys)
b = sum((x - mx) * (y - my) for x, y in zip(xs, ys)) / sum((x - mx) ** 2 for x in xs)
a = my - b * mx
fit = {"a": a, "b": b, "note": f"{len(points)} points, not validated"}
fit["value"] = a * total_blocks + b * total_blocks * total_blocks / 2
flat = sum(c for c, _, _ in points) / sum(bl for _, bl, _ in points) * total_blocks
return {"fit": fit, "flat": flat, "T": total_blocks}
def stop_k(pack1_cost: int, pack1_blocks: int, cut_blocks: int, pack1_sources: int, cut_sources: int
) -> dict[str, Any]:
per_block = pack1_cost * cut_blocks / pack1_blocks
per_source = pack1_cost * cut_sources / pack1_sources
return {"per_block": per_block, "per_source": per_source, "cap": CAP, "stop": per_block > CAP}
def stop_kplus(run1_cost: int, run1_pages: int, pages_total: int) -> dict[str, Any]:
projected = run1_cost * pages_total / run1_pages
return {"projected": projected, "cap": CAP, "stop": projected > CAP}
def _pack_number(unit: str) -> int | None:
m = PACK_UNIT.match(unit)
return int(m.group(1)) if m else None
def summarize(h: dict[str, Any], blocks_doc: dict[str, Any], packs_log: dict[str, Any] | None,
total_blocks: int) -> dict[str, Any]:
order = [str(x) for x in blocks_doc["order"]]
count: dict[str, int] = {}
for b in blocks_doc["blocks"]:
count[b["file"]] = count.get(b["file"], 0) + 1
variants: dict[str, Any] = {}
for v in sorted({r["variant"] for r in h["runs"]}):
mine = [r for r in h["runs"] if r["variant"] == v]
row: dict[str, Any] = {c: sum(r[c] for r in mine) for c in (*CATEGORIES.values(), "total", "turns", "startup")}
row["subagents"] = len(mine)
row["class"] = mine[0]["class"]
row["counts_toward_cap"] = v in ("K", "K+")
if v == "K" and packs_log:
packs = [list(p["sources"]) for p in sorted(packs_log["packs"], key=lambda p: int(p["pack"]))]
pack_costs: dict[int, float] = {}
per_cat: dict[str, dict[int, float]] = {c: {} for c in CATEGORIES.values()}
for r in mine:
n = _pack_number(r["unit"])
if n is not None:
pack_costs[n] = pack_costs.get(n, 0) + r["total"]
for c in CATEGORIES.values():
per_cat[c][n] = per_cat[c].get(n, 0) + r[c]
row["per_pack"] = {str(n): c for n, c in sorted(pack_costs.items())}
row["per_source"] = allocate(pack_costs, packs, blocks_doc)
row["per_source_by_category"] = {c: allocate(per_cat[c], packs, blocks_doc) for c in CATEGORIES.values()}
row["means"] = means(row["per_source"], order)
points, read = [], 0
for n, names in enumerate(packs, 1):
blocks_n = sum(count.get(x, 0) for x in names)
if n in pack_costs:
points.append((pack_costs[n], blocks_n, read))
read += blocks_n
row["projection"] = project(points, total_blocks)
variants[v] = row
return variants
def build_parser() -> argparse.ArgumentParser:
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
sub = p.add_subparsers(dest="command", required=True)
h = sub.add_parser("harvest")
h.add_argument("--project-dir", required=True)
h.add_argument("--okter", required=True)
h.add_argument("--blocks", required=True)
h.add_argument("--total", type=int, required=True)
h.add_argument("--packs")
h.add_argument("--out", required=True)
s = sub.add_parser("stop")
s.add_argument("variant", choices=("K", "K+"))
s.add_argument("--kost", required=True)
s.add_argument("--blocks", required=True)
s.add_argument("--packs")
s.add_argument("--pages-total", type=int)
s.add_argument("--pages-first", type=int)
s.add_argument("--record", action="store_true")
return p
def main(argv: list[str] | None = None) -> int:
args = build_parser().parse_args(argv)
try:
blocks_doc = json.loads(Path(args.blocks).read_text(encoding="utf-8"))
if args.command == "harvest":
out = Path(args.out)
old = json.loads(out.read_text(encoding="utf-8")) if out.is_file() else {}
h = harvest(Path(args.project_dir), sessions(Path(args.okter)))
packs_log = json.loads(Path(args.packs).read_text(encoding="utf-8")) if args.packs else None
doc = {"schema": SCHEMA, **h, "variants": summarize(h, blocks_doc, packs_log, args.total),
"stops": old.get("stops", {})}
out.parent.mkdir(parents=True, exist_ok=True)
out.write_text(json.dumps(doc, ensure_ascii=False, indent=1) + "\n", encoding="utf-8")
print(f"harvest: {len(h['runs'])} attributed runs, {len(h['unattributed'])} unattributed, sessions read: "
f"{', '.join(h['sessions_read'])} -> {out}")
return 0
kost_path = Path(args.kost)
kost = json.loads(kost_path.read_text(encoding="utf-8"))
runs = [r for r in kost["runs"] if r["variant"] == args.variant]
if args.variant == "K":
first = [r for r in runs if _pack_number(r["unit"]) == 1]
if not first:
raise RunError("no pack 1 run in the kost file; harvest after pack 1")
if not args.packs:
raise RunError("--packs is needed for K")
log = json.loads(Path(args.packs).read_text(encoding="utf-8"))
p1 = next(p for p in log["packs"] if int(p["pack"]) == 1)
row = stop_k(sum(r["total"] for r in first), int(p1["blocks"]), len(blocks_doc["blocks"]),
len(p1["sources"]), len(blocks_doc["order"]))
else:
first = [r for r in runs if r["unit"].startswith("run1")]
if not first or not args.pages_total or not args.pages_first:
raise RunError("K+ needs a run 1 in the kost file, --pages-total and --pages-first")
row = stop_kplus(sum(r["total"] for r in first), args.pages_first, args.pages_total)
print(f"{args.variant}: projected {row.get('per_block', row.get('projected'))} tokens"
+ (f" (naive per source {row['per_source']})" if "per_source" in row else "")
+ f", cap {CAP}: {'STOP' if row['stop'] else 'continue'}")
if args.record:
kost.setdefault("stops", {})[args.variant] = row
kost_path.write_text(json.dumps(kost, ensure_ascii=False, indent=1) + "\n", encoding="utf-8")
return 1 if row["stop"] else 0
except (RunError, OSError, json.JSONDecodeError, KeyError, StopIteration) as exc:
print(f"cost cannot run: {exc}", file=sys.stderr)
return 2
if __name__ == "__main__":
sys.exit(main())