First public release, published as one snapshot. What changed between versions is in CHANGELOG.md. Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
661 lines
37 KiB
Python
661 lines
37 KiB
Python
"""Token cost of a v1 run from its sessions' transcripts: cost lines, stop checks and the isolation count. No model.
|
|
|
|
Usage:
|
|
python3 -m okfwiki.cost harvest --transcripts DIR --okter FILE --work DIR --out kost.json [--packs FILE] [--margin M]
|
|
python3 -m okfwiki.cost stop --lines L[,L] --cap N --unit blocks|passes|claims --done D --total T --kost FILE
|
|
[--blocks FILE] [--batches-done B --batches-total N] [--next B [--next-chars C]]
|
|
[--margin M]
|
|
[--name NAME]
|
|
python3 -m okfwiki.cost summarize --kost FILE [--cap-group NAME=L[,L]:CAP]...
|
|
python3 -m okfwiki.cost isolation --transcripts DIR --run R --tasks DIR --out isolation.json
|
|
[--line read|diagnose|repair]
|
|
|
|
harvest Reads the sessions listed in okter.txt ("<session-id> <kind>" per line, kind build, read, diagnose, merge or
|
|
repair) under
|
|
the transcripts directory (~/.claude/projects/<slug>/): each session's main transcript is orchestration, on
|
|
the line orchestration-<kind>, with the input side of every turn kept by turn number, the first turn that
|
|
starts helpers (an Agent, Task or Workflow call) and the tokens of the turns before it; every subagent
|
|
transcript under <session>/subagents/ or, for a workflow's agents, <session>/subagents/workflows/<id>/ (the
|
|
agent() label is the description) is attributed by its description "V1-<run>-<line>-<unit>", line pack (kept
|
|
as build), link, read, diagnose, merge (step 18a's pair helper, order 29), repair (the repair helper, order 34)
|
|
or calib. Any other description is an unattributed run: it is listed and the exit is 1,
|
|
because a model-run session spawns no helper agents. Tokens are counted as the trial did: per message id
|
|
once, at its largest value; an output never recorded is derived from the next message's input growth (a
|
|
lower bound). With --packs, a pack log with a gap (a logged pack with no build run, or a build run for a pack
|
|
the log lacks) cannot run. The margin is max(0.10, measured), 0.10 when nothing is measured, because the
|
|
measurement is itself a lower bound. Recorded cap groups and stop rows survive a new harvest.
|
|
stop The stop check after a pack or a pass. Measured = the sum of the given lines (calib is reported, never
|
|
capped). Flat projection = measured / done x total. --unit blocks also fits cost per block against blocks
|
|
read before (a + b*R) over the packs in --blocks (a pack log with "pack" and "blocks"), orchestration spread
|
|
over the packs in proportion to their attributed cost, and projects a*T + b*T*T/2, once at least four packs
|
|
are measured (the operator 11.10.2026); before that the flat projection alone. --unit claims scales only the
|
|
helper lines per claim: an orchestration line counts the session's part before its first helper once and the
|
|
rest per batch (a message that starts helpers, or a phase of a workflow), measured / --batches-done x
|
|
--batches-total, both required with an orchestration line. STOP when the projection x (1 + margin) exceeds
|
|
the cap. With --next B (--unit blocks only) the projection is recorded and printed but does not stop:
|
|
STOP when (measured + measured / done x B) x (1 + margin) exceeds the cap, B being the blocks of the pack
|
|
the model works on before the next check (the operator 12.10.2026: a cost line stops on measured cost plus a
|
|
margin, the projection is reported only; order 22 session A3: plus the next pack, so the cap is not passed).
|
|
With --next-chars C as well, the next pack's share is by characters, not blocks (motor B17: a block of a
|
|
folder's PDF or spreadsheet holds about ten times the characters of a docs block, and the cost follows the
|
|
characters): measured / the `block_chars` of the packs with a compiler run here x C, ROUND2_PER_CHAR before
|
|
the first; a log whose built packs have no `block_chars` keeps the rule by blocks.
|
|
The cap group and the stop row are recorded in the kost file.
|
|
Exit 0 continue, 1 STOP, 2 cannot run; a caller treats every non-zero exit as STOP.
|
|
summarize Per line "<line>: <lower bound> + margin <m> = <sum>"; per cap group (recorded by stop, or given with
|
|
--cap-group) "<name>: <lower bound> + margin <m> = <sum> of <cap>"; orchestration tokens per turn, per pack
|
|
and the input per turn fitted against the turn number. Exit 1 when a group is over its cap.
|
|
isolation Reader agents (or, with --line diagnose, diagnosis agents) are one per task, described
|
|
"V1-<run>-<line>-<task-id>", each with the task file <tasks>/<task-id>.md, started from a session or from a workflow (the same two places as harvest). Every
|
|
Read call of another path is a break. 0 breaks over at least one agent = exit 0. With --line repair (order
|
|
34), each repair helper "V1-<run>-repair-<task-id>" may read exactly the parts and write exactly the one
|
|
answer path of <tasks>/<task-id>.parts.json (krepair task's part list); every other tool call, a tool other
|
|
than Read and Write among them, is a break (CLAUDE.md: the breach count reads every tool call), and so is a
|
|
part never read and an answer written other than once (order 39). A helper
|
|
whose task has no part list cannot run (exit 2).
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import math
|
|
import re
|
|
import sys
|
|
from collections.abc import Mapping
|
|
from fractions import Fraction
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
from okfwiki.common import RunError
|
|
from okfwiki.transcript import USAGE_FIELDS, read_transcript
|
|
|
|
SCHEMA = "okf-wiki/cost/1"
|
|
ISOLATION_SCHEMA = "okf-wiki/isolation/1"
|
|
LINES = ("build", "link", "orchestration-build", "read", "orchestration-read", "diagnose", "orchestration-diagnose",
|
|
"merge", "orchestration-merge", "repair", "orchestration-repair", "calib")
|
|
UNCAPPED = ("calib",)
|
|
LINE_OF = {"pack": "build", "link": "link", "read": "read", "diagnose": "diagnose", "merge": "merge",
|
|
"repair": "repair", "calib": "calib"}
|
|
KINDS = ("build", "read", "diagnose", "merge", "repair")
|
|
UNITS = ("blocks", "passes", "claims")
|
|
ISOLATED = ("read", "diagnose", "repair")
|
|
PREFIX = re.compile(r"^V1-(.+?)-(pack|link|read|diagnose|merge|repair|calib)-(\S+)$")
|
|
PACK_UNIT = re.compile(r"^p(\d+)(?:-|$)")
|
|
MARGIN_FLOOR = 0.10
|
|
CATEGORIES = {"input_tokens": "input", "cache_read_input_tokens": "cache_read",
|
|
"cache_creation_input_tokens": "cache_write", "output_tokens": "output"}
|
|
INPUT_SIDE = ("input_tokens", "cache_read_input_tokens", "cache_creation_input_tokens")
|
|
CHARS_PER_TOKEN_LOW = 2.5 # injected text counted high, so a derived output is a lower bound
|
|
SPAWNS = ("Agent", "Task", "Workflow") # the tool calls that start helpers or a workflow's agents
|
|
FIT_FROM_PACKS = 4 # the operator 11.10.2026: the fitted curve counts from pack 4
|
|
ROUND2_PER_BLOCK = 3_259 # round 2 of the first part, 14 741 606 over 4 523 blocks (order 25 session C): the next pack's
|
|
# share before a ledger's first compiler run (order 27 A3)
|
|
ROUND2_PER_CHAR = Fraction(14_741_606, 1_111_796) # the same cost over the 1 111 796 characters of round 2's blocks
|
|
# (about 13.3 per character): the share by characters before the first compiler run (B17)
|
|
|
|
|
|
def margin(measured: float | None) -> float:
|
|
"""The margin on a lower-bound measurement: the measured one, never below the 10 % floor."""
|
|
return MARGIN_FLOOR if measured is None else max(MARGIN_FLOOR, measured)
|
|
|
|
|
|
def sessions(okter: Path) -> list[tuple[str, str]]:
|
|
"""(session id, kind) per line of okter.txt, in order, each id once."""
|
|
out: list[tuple[str, str]] = []
|
|
for line in okter.read_text(encoding="utf-8").splitlines():
|
|
parts = line.split()
|
|
if not parts:
|
|
continue
|
|
if len(parts) != 2 or parts[1] not in KINDS:
|
|
raise RunError(f"okter line {line!r} is not '<session-id> <kind>' with kind in {', '.join(KINDS)}")
|
|
known = dict(out)
|
|
if parts[0] in known and known[parts[0]] != parts[1]:
|
|
raise RunError(f"session {parts[0]} is listed as both {known[parts[0]]} and {parts[1]}")
|
|
if parts[0] not in known:
|
|
out.append((parts[0], parts[1]))
|
|
return out
|
|
|
|
|
|
def _result_chars(message: dict[str, Any]) -> int:
|
|
content = message.get("content")
|
|
if not isinstance(content, list):
|
|
return 0
|
|
total = 0
|
|
for item in content:
|
|
if not (isinstance(item, dict) and item.get("type") == "tool_result"):
|
|
continue
|
|
inner = item.get("content")
|
|
if isinstance(inner, str):
|
|
total += len(inner)
|
|
elif isinstance(inner, list):
|
|
total += sum(len(str(x.get("text", ""))) for x in inner if isinstance(x, dict))
|
|
return total
|
|
|
|
|
|
def usage(path: Path, per_turn: bool = False) -> dict[str, Any]:
|
|
"""Tokens of one transcript, ported from the trial. A message whose records all lack a stop reason never got its
|
|
final usage written; when another message follows, its output is the growth of the next message's input side
|
|
less the text injected between them (counted high at CHARS_PER_TOKEN_LOW), a lower bound. With per_turn, the
|
|
input side of every turn is kept with its turn number."""
|
|
per: dict[str, dict[str, int]] = {}
|
|
order: list[str] = []
|
|
complete: set[str] = set()
|
|
spawning: set[str] = set()
|
|
between: dict[str, int] = {}
|
|
last_stop: str | None = None
|
|
for line in read_transcript(path):
|
|
message = line.get("message")
|
|
if line.get("type") == "attachment" and order:
|
|
between[order[-1]] = between.get(order[-1], 0) + len(str(line.get("rendered") or ""))
|
|
continue
|
|
if not isinstance(message, dict):
|
|
continue
|
|
if line.get("type") == "user" and order:
|
|
between[order[-1]] = between.get(order[-1], 0) + _result_chars(message)
|
|
continue
|
|
if line.get("type") != "assistant":
|
|
continue
|
|
mid = str(message.get("id", ""))
|
|
if mid not in per:
|
|
order.append(mid)
|
|
seen = per.setdefault(mid, {})
|
|
fields = message.get("usage") or {}
|
|
for f in USAGE_FIELDS:
|
|
value = fields.get(f)
|
|
if isinstance(value, int):
|
|
seen[f] = max(seen.get(f, 0), value)
|
|
content = message.get("content")
|
|
if isinstance(content, list) and any(isinstance(b, dict) and b.get("type") == "tool_use"
|
|
and b.get("name") in SPAWNS for b in content):
|
|
spawning.add(mid)
|
|
stop = message.get("stop_reason")
|
|
last_stop = stop if isinstance(stop, str) else None
|
|
if isinstance(stop, str):
|
|
complete.add(mid)
|
|
out: dict[str, Any] = {CATEGORIES[f]: sum(v.get(f, 0) for v in per.values()) for f in USAGE_FIELDS}
|
|
out["output_recorded"] = out["output"]
|
|
derived = unrecorded = 0
|
|
for i, mid in enumerate(order):
|
|
if mid in complete:
|
|
continue
|
|
if i + 1 == len(order):
|
|
unrecorded += 1
|
|
continue
|
|
growth = (sum(per[order[i + 1]].get(f, 0) for f in INPUT_SIDE) - sum(per[mid].get(f, 0) for f in INPUT_SIDE)
|
|
- int(between.get(mid, 0) / CHARS_PER_TOKEN_LOW))
|
|
recorded = per[mid].get("output_tokens", 0)
|
|
if growth > recorded:
|
|
out["output"] += growth - recorded
|
|
derived += 1
|
|
out["output_derived_messages"] = derived
|
|
out["output_unrecorded"] = unrecorded
|
|
out["total"] = sum(out[c] for c in CATEGORIES.values())
|
|
out["turns"] = len(order)
|
|
first = per[order[0]] if order else {}
|
|
out["startup"] = sum(first.get(f, 0) for f in INPUT_SIDE)
|
|
out["stop_reason"] = last_stop
|
|
if per_turn:
|
|
out["per_turn"] = [{"turn": i, "input": sum(per[mid].get(f, 0) for f in INPUT_SIDE)}
|
|
for i, mid in enumerate(order, 1)]
|
|
spawn_turn = next((i for i, mid in enumerate(order, 1) if mid in spawning), None)
|
|
before = order if spawn_turn is None else order[:spawn_turn - 1]
|
|
out["first_helper_turn"] = spawn_turn
|
|
out["before_first_helper"] = sum(per[mid].get(f, 0) for mid in before for f in USAGE_FIELDS)
|
|
return out
|
|
|
|
|
|
def pack_of(unit: str) -> int | None:
|
|
m = PACK_UNIT.match(unit)
|
|
return int(m.group(1)) if m else None
|
|
|
|
|
|
def line_totals(runs: list[dict[str, Any]], orchestration: Mapping[str, dict[str, Any]]) -> dict[str, dict[str, int]]:
|
|
out: dict[str, dict[str, int]] = {}
|
|
for name in LINES:
|
|
out[name] = {c: 0 for c in (*CATEGORIES.values(), "total", "turns", "agents")}
|
|
rows = [(r["line"], r) for r in runs] + [(f"orchestration-{o['kind']}", o) for o in orchestration.values()]
|
|
for name, row in rows:
|
|
for c in (*CATEGORIES.values(), "total", "turns"):
|
|
out[name][c] += int(row[c])
|
|
out[name]["agents"] += 1
|
|
return out
|
|
|
|
|
|
def agent_transcripts(subagents: Path, suffix: str) -> list[Path]:
|
|
"""The agent files of one session: its own helpers and every workflow's (measured 09.10, Claude Code 2.1.295)."""
|
|
return sorted([*subagents.glob(f"agent-*.{suffix}"), *subagents.glob(f"workflows/*/agent-*.{suffix}")])
|
|
|
|
|
|
def harvest(transcripts: Path, okter: list[tuple[str, str]]) -> dict[str, Any]:
|
|
runs: list[dict[str, Any]] = []
|
|
unattributed: list[dict[str, Any]] = []
|
|
orchestration: dict[str, dict[str, Any]] = {}
|
|
for sid, kind in okter:
|
|
top = transcripts / f"{sid}.jsonl"
|
|
if not top.is_file(): # order 27 A3: a wrong transcripts folder would give 0 for a whole ledger without a word
|
|
raise RunError(f"session {sid} in okter has no transcript {top}")
|
|
orchestration[sid] = {"kind": kind, **usage(top, per_turn=True)}
|
|
for path in agent_transcripts(transcripts / sid / "subagents", "jsonl"):
|
|
meta_path = path.with_name(path.stem + ".meta.json")
|
|
meta = json.loads(meta_path.read_text(encoding="utf-8")) if meta_path.is_file() else {}
|
|
row = {"session": sid, "agent": path.stem, "agentType": meta.get("agentType"),
|
|
"description": str(meta.get("description", "")), **usage(path)}
|
|
m = PREFIX.match(row["description"])
|
|
if m is None:
|
|
unattributed.append(row)
|
|
continue
|
|
row["run"], row["line"], row["unit"] = m.group(1), LINE_OF[m.group(2)], m.group(3)
|
|
row["pack"] = pack_of(m.group(3))
|
|
runs.append(row)
|
|
return {"sessions": [{"id": sid, "kind": kind} for sid, kind in okter], "runs": runs,
|
|
"unattributed": unattributed, "orchestration": orchestration, "lines": line_totals(runs, orchestration)}
|
|
|
|
|
|
def check_packs(runs: list[dict[str, Any]], packs_log: dict[str, Any]) -> None:
|
|
logged = sorted(int(p["pack"]) for p in packs_log["packs"])
|
|
if logged != list(range(1, len(logged) + 1)):
|
|
raise RunError(f"the pack log is not packs 1..{len(logged)}: {logged}")
|
|
built = {int(r["pack"]) for r in runs if r["line"] == "build" and r["pack"] is not None}
|
|
missing, extra = sorted(set(logged) - built), sorted(built - set(logged))
|
|
if missing or extra:
|
|
raise RunError(f"pack log gap: logged without a build run {missing}, built but not logged {extra}")
|
|
|
|
|
|
def allocate(pack_costs: Mapping[int, float], packs: list[list[str]], blocks_doc: dict[str, Any]) -> dict[str, float]:
|
|
"""Tokens per source: each pack's cost split over its sources by their share of the pack's blocks."""
|
|
count: dict[str, int] = {}
|
|
for b in blocks_doc["blocks"]:
|
|
count[b["file"]] = count.get(b["file"], 0) + 1
|
|
per: dict[str, float] = {}
|
|
for n, names in enumerate(packs, 1):
|
|
if n not in pack_costs:
|
|
continue
|
|
total = sum(count.get(x, 0) for x in names) or 1
|
|
for x in names:
|
|
per[x] = pack_costs[n] * count.get(x, 0) / total
|
|
return per
|
|
|
|
|
|
def fit_line(points: list[tuple[float, float]]) -> tuple[float, float]:
|
|
"""Least squares y = a + b*x; one distinct x gives b = 0."""
|
|
xs, ys = [x for x, _ in points], [y for _, y in points]
|
|
mx, my = sum(xs) / len(xs), sum(ys) / len(ys)
|
|
if len(set(xs)) == 1:
|
|
return my, 0.0
|
|
b = sum((x - mx) * (y - my) for x, y in points) / sum((x - mx) ** 2 for x in xs)
|
|
return my - b * mx, b
|
|
|
|
|
|
def project(points: list[tuple[float, int, int]], total_blocks: int) -> dict[str, Any]:
|
|
"""points: (cost, blocks in the unit, blocks read before it). Cost per block a + b*R, summed to a*T + b*T*T/2."""
|
|
if not points:
|
|
return {"fit": None, "flat": None, "T": total_blocks}
|
|
a, b = fit_line([(float(r), c / bl) for c, bl, r in points])
|
|
fit: dict[str, Any] = {"a": a, "b": b, "note": "one point, no growth term" if b == 0 and len(points) == 1
|
|
else f"{len(points)} points, not validated"}
|
|
fit["value"] = a * total_blocks + b * total_blocks * total_blocks / 2
|
|
flat = sum(c for c, _, _ in points) / sum(bl for _, bl, _ in points) * total_blocks
|
|
return {"fit": fit, "flat": flat, "T": total_blocks}
|
|
|
|
|
|
def capped_lines(spec: str) -> list[str]:
|
|
names = [x.strip() for x in spec.split(",") if x.strip()]
|
|
for name in names:
|
|
if name not in LINES:
|
|
raise RunError(f"unknown cost line {name!r}; lines are {', '.join(LINES)}")
|
|
if name in UNCAPPED:
|
|
raise RunError(f"{name} is reported, never capped")
|
|
if not names:
|
|
raise RunError("no cost line given")
|
|
return names
|
|
|
|
|
|
def claims_projection(kost: dict[str, Any], lines: list[str], done: int, total: int,
|
|
batches: tuple[int, int] | None) -> tuple[Fraction, dict[str, Any]]:
|
|
"""Helper lines per claim; each orchestration line's part before its session's first helper once, the rest per
|
|
batch. Measured 11.10 in order 12: scaling the whole sum per claim projected 162.8 M where the reading cost 10.5 M,
|
|
because the session's fixed start was scaled with the claims."""
|
|
helpers = sum(int(kost["lines"][name]["total"]) for name in lines if not name.startswith("orchestration-"))
|
|
projected = Fraction(helpers) * total / done
|
|
before = after = 0
|
|
for name in (x for x in lines if x.startswith("orchestration-")):
|
|
kind = name.removeprefix("orchestration-")
|
|
line_before = sum(int(o["before_first_helper"]) for o in kost.get("orchestration", {}).values()
|
|
if o.get("kind") == kind)
|
|
before += line_before
|
|
after += int(kost["lines"][name]["total"]) - line_before
|
|
parts: dict[str, Any] = {"helpers": helpers}
|
|
if any(x.startswith("orchestration-") for x in lines):
|
|
if batches is None:
|
|
raise RunError("--unit claims with an orchestration line needs --batches-done and --batches-total")
|
|
b_done, b_total = batches
|
|
if b_done <= 0 or b_done > b_total:
|
|
raise ValueError(f"--batches-done {b_done} --batches-total {b_total}: need 0 < done <= total")
|
|
projected += before + Fraction(after) * b_total / b_done
|
|
parts.update({"before_first_helper": before, "after_first_helper": after, "batches_done": b_done,
|
|
"batches_total": b_total})
|
|
return projected, parts
|
|
|
|
|
|
def ledger_rate(kost: dict[str, Any], packs_log: dict[str, Any], measured: int) -> tuple[Fraction, int]:
|
|
"""(cost per block, the blocks it is over): the measured cost over the blocks of the packs with a compiler run in
|
|
this ledger, so packs built in another ledger (a continued collection) never thin it; ROUND2_PER_BLOCK before the
|
|
ledger's first compiler run (order 27 A3). A compiler run for a pack the log lacks cannot run."""
|
|
blocks = {int(p["pack"]): int(p["blocks"]) for p in packs_log["packs"]}
|
|
over = sum(blocks[n] for n in built_packs(kost, packs_log))
|
|
return (Fraction(measured, over) if over else Fraction(ROUND2_PER_BLOCK)), over
|
|
|
|
|
|
def built_packs(kost: dict[str, Any], packs_log: dict[str, Any]) -> set[int]:
|
|
"""The packs with a compiler run in this ledger. A compiler run for a pack the log lacks cannot run."""
|
|
logged = {int(p["pack"]) for p in packs_log["packs"]}
|
|
built: set[int] = set()
|
|
for r in kost.get("runs", []):
|
|
if r.get("line") == "build" and r.get("pack") is not None:
|
|
if int(r["pack"]) not in logged:
|
|
raise RunError(f"a compiler run is for pack {r['pack']}, which the pack log lacks")
|
|
built.add(int(r["pack"]))
|
|
return built
|
|
|
|
|
|
def ledger_chars_rate(kost: dict[str, Any], packs_log: dict[str, Any], measured: int) -> tuple[Fraction, int] | None:
|
|
"""(cost per character, the characters it is over), as ledger_rate by the characters of the packs' blocks
|
|
(`block_chars`), ROUND2_PER_CHAR before the ledger's first compiler run; None when a built pack has no
|
|
`block_chars` (a log written before blocks had characters), which keeps the rule by blocks (B17)."""
|
|
chars = {int(p["pack"]): p.get("block_chars") for p in packs_log["packs"]}
|
|
built = built_packs(kost, packs_log)
|
|
if any(not isinstance(chars[n], int) for n in built):
|
|
return None
|
|
over = sum(int(chars[n] or 0) for n in built)
|
|
return (Fraction(measured, over) if over else ROUND2_PER_CHAR), over
|
|
|
|
|
|
def stop_row(kost: dict[str, Any], lines: list[str], cap: int, unit: str, done: int, total: int,
|
|
packs_log: dict[str, Any] | None, used_margin: float,
|
|
batches: tuple[int, int] | None = None, following: int | None = None,
|
|
next_chars: int | None = None) -> dict[str, Any]:
|
|
if next_chars is not None and (following is None or next_chars < 0):
|
|
raise ValueError(f"--next-chars {next_chars}: the characters of the next pack, with --next only")
|
|
if done <= 0 or total <= 0 or done > total:
|
|
raise ValueError(f"--done {done} --total {total}: need 0 < done <= total")
|
|
measured = sum(int(kost["lines"][name]["total"]) for name in lines)
|
|
flat = Fraction(measured) * total / done
|
|
rate = Fraction(measured, done)
|
|
parts: dict[str, Any] = {}
|
|
if unit == "claims":
|
|
flat, parts = claims_projection(kost, lines, done, total, batches)
|
|
fit: float | None = None
|
|
if unit == "blocks":
|
|
if packs_log is None:
|
|
raise RunError("--unit blocks needs --blocks (a pack log with pack and blocks)")
|
|
rate, over = ledger_rate(kost, packs_log, measured)
|
|
flat = measured + rate * (total - done) # the blocks not yet done at this ledger's rate
|
|
parts.update(rate=float(rate), rate_blocks=over)
|
|
attributed: dict[int, int] = {}
|
|
for r in kost.get("runs", []):
|
|
if r.get("line") in lines and r.get("pack") is not None:
|
|
attributed[int(r["pack"])] = attributed.get(int(r["pack"]), 0) + int(r["total"])
|
|
if attributed:
|
|
scale = measured / sum(attributed.values())
|
|
points: list[tuple[float, int, int]] = []
|
|
read = 0
|
|
for p in sorted(packs_log["packs"], key=lambda p: int(p["pack"])):
|
|
n, blocks = int(p["pack"]), int(p["blocks"])
|
|
if n in attributed:
|
|
points.append((attributed[n] * scale, blocks, read))
|
|
read += blocks
|
|
fitted = project(points, total)["fit"] if len(points) >= FIT_FROM_PACKS else None
|
|
fit = float(fitted["value"]) if fitted else None
|
|
worst = max(flat, Fraction(fit)) if fit is not None else flat
|
|
with_margin = worst * (1 + Fraction(str(used_margin)))
|
|
row = {"lines": lines, "cap": cap, "unit": unit, "done": done, "total": total, "measured": measured,
|
|
"flat": float(flat), "fit": fit, "margin": used_margin, "projected_with_margin": math.ceil(with_margin),
|
|
"stop": with_margin > cap, **parts}
|
|
if following is not None:
|
|
if unit != "blocks" or following < 0:
|
|
raise ValueError(f"--next {following}: the blocks of the next pack, with --unit blocks only")
|
|
measured_next = (measured + rate * following) * (1 + Fraction(str(used_margin)))
|
|
row.update(rule="measured", next=following, measured_next_with_margin=math.ceil(measured_next),
|
|
stop=measured_next > cap)
|
|
by_chars = ledger_chars_rate(kost, packs_log, measured) if next_chars is not None and packs_log else None
|
|
if by_chars is not None:
|
|
rate_c, over_c = by_chars
|
|
measured_next = (measured + rate_c * int(next_chars or 0)) * (1 + Fraction(str(used_margin)))
|
|
row.update(rule="measured-chars", next_chars=next_chars, rate_chars=float(rate_c), rate_chars_over=over_c,
|
|
measured_next_with_margin=math.ceil(measured_next), stop=measured_next > cap)
|
|
return row
|
|
|
|
|
|
def with_margin(lower: int, used: float) -> int:
|
|
return math.ceil(Fraction(lower) * (1 + Fraction(str(used))))
|
|
|
|
|
|
def summarize(kost: dict[str, Any], extra_groups: Mapping[str, dict[str, Any]]) -> tuple[list[str], bool]:
|
|
used = float(kost["margin"]["used"])
|
|
out: list[str] = []
|
|
for name in LINES:
|
|
lower = int(kost["lines"].get(name, {}).get("total", 0)) # a kost file from before a line existed
|
|
note = " (reported, never capped)" if name in UNCAPPED else ""
|
|
out.append(f"{name}: {lower} + margin {used:.2f} = {with_margin(lower, used)}{note}")
|
|
within = True
|
|
for name, group in {**kost.get("caps", {}), **extra_groups}.items():
|
|
lower = sum(int(kost["lines"][x]["total"]) for x in group["lines"])
|
|
total = with_margin(lower, used)
|
|
within = within and total <= int(group["cap"])
|
|
out.append(f"{name}: {lower} + margin {used:.2f} = {total} of {group['cap']}")
|
|
packs = {int(r["pack"]) for r in kost.get("runs", []) if r.get("pack") is not None}
|
|
orchestration = kost.get("orchestration", {})
|
|
for kind in KINDS:
|
|
mine = [o for o in orchestration.values() if o.get("kind") == kind]
|
|
turns = sum(int(o["turns"]) for o in mine)
|
|
if not turns:
|
|
continue
|
|
tokens = int(kost["lines"].get(f"orchestration-{kind}", {}).get("total", 0))
|
|
per_pack = f"; per pack {tokens // len(packs)} over {len(packs)} packs" if packs else ""
|
|
out.append(f"orchestration-{kind} per turn: {tokens // turns} over {turns} turns{per_pack}")
|
|
points = [(float(t["turn"]), float(t["input"])) for o in mine for t in o.get("per_turn", [])]
|
|
if len({x for x, _ in points}) > 1:
|
|
a, b = fit_line(points)
|
|
out.append(f"orchestration-{kind} input per turn: {round(a)} + {round(b)} x turn ({len(points)} turns)")
|
|
return out, within
|
|
|
|
|
|
def parse_cap_group(text: str) -> tuple[str, dict[str, Any]]:
|
|
name, sep, rest = text.partition("=")
|
|
lines, sep2, cap = rest.rpartition(":")
|
|
if not (sep and sep2 and name.strip()):
|
|
raise RunError(f"--cap-group {text!r} is not NAME=LINES:CAP")
|
|
return name.strip(), {"lines": capped_lines(lines), "cap": int(cap)}
|
|
|
|
|
|
def part_list(tasks: Path, task: str) -> tuple[set[Path], Path]:
|
|
"""(the parts a repair helper may read, the one answer path it writes) from krepair task's part list."""
|
|
path = tasks / f"{task}.parts.json"
|
|
if not path.is_file():
|
|
raise RunError(f"repair task {task} has no part list {path}")
|
|
doc = load_json(path)
|
|
parts, answer = doc.get("parts"), doc.get("answer")
|
|
if not (isinstance(parts, list) and parts and all(isinstance(p, str) for p in parts) and isinstance(answer, str)):
|
|
raise RunError(f"{path} is not a part list with parts and one answer path")
|
|
return {(tasks / p).resolve() for p in parts}, (tasks / answer).resolve()
|
|
|
|
|
|
def repair_breaks(path: Path, reads: set[Path], answer: Path) -> tuple[int, int, list[str]]:
|
|
"""(reads, writes, breaks) of one repair helper: every tool call is checked, not only Read, and a part it never
|
|
read or an answer it wrote other than once is a break of its own (order 39)."""
|
|
n_reads = n_writes = 0
|
|
seen: set[Path] = set()
|
|
breaks: list[str] = []
|
|
for record in read_transcript(path) if path.is_file() else []:
|
|
message = record.get("message")
|
|
content = message.get("content") if isinstance(message, dict) else None
|
|
for block in content if isinstance(content, list) else []:
|
|
if not (isinstance(block, dict) and block.get("type") == "tool_use"):
|
|
continue
|
|
name = str(block.get("name"))
|
|
target = (block.get("input") or {}).get("file_path")
|
|
where = Path(str(target)).resolve() if isinstance(target, str) else None
|
|
if name == "Read" and where in reads:
|
|
n_reads += 1
|
|
seen.add(where)
|
|
elif name == "Write" and where == answer:
|
|
n_writes += 1
|
|
else:
|
|
breaks.append(f"{name} {target}" if isinstance(target, str) else name)
|
|
breaks.extend(f"part not read: {p}" for p in sorted(reads - seen))
|
|
if n_writes != 1:
|
|
breaks.append("answer not written" if n_writes == 0 else f"answer written {n_writes} times")
|
|
return n_reads, n_writes, breaks
|
|
|
|
|
|
def isolation(transcripts: Path, run: str, tasks: Path, line: str = "read") -> dict[str, Any]:
|
|
prefix = re.compile(rf"^V1-{re.escape(run)}-{re.escape(line)}-(\S+)$")
|
|
per_agent: list[dict[str, Any]] = []
|
|
for meta_path in sorted(p for d in transcripts.glob("*/subagents") for p in agent_transcripts(d, "meta.json")):
|
|
meta = json.loads(meta_path.read_text(encoding="utf-8"))
|
|
m = prefix.match(str(meta.get("description", "")))
|
|
if m is None:
|
|
continue
|
|
path = meta_path.with_name(meta_path.name.removesuffix(".meta.json") + ".jsonl")
|
|
if line == "repair":
|
|
n_reads, n_writes, found = repair_breaks(path, *part_list(tasks, m.group(1)))
|
|
per_agent.append({"agent": meta_path.name.removesuffix(".meta.json"), "claim": m.group(1),
|
|
"reads": n_reads, "writes": n_writes, "breaks": found})
|
|
continue
|
|
task = (tasks / f"{m.group(1)}.md").resolve()
|
|
reads: dict[str, str] = {}
|
|
for record in read_transcript(path) if path.is_file() else []:
|
|
message = record.get("message")
|
|
content = message.get("content") if isinstance(message, dict) else None
|
|
for block in content if isinstance(content, list) else []:
|
|
if isinstance(block, dict) and block.get("type") == "tool_use" and block.get("name") == "Read":
|
|
target = str((block.get("input") or {}).get("file_path", ""))
|
|
reads[str(block.get("id") or len(reads))] = target
|
|
breaks = [t for t in reads.values() if Path(t).resolve() != task]
|
|
per_agent.append({"agent": meta_path.name.removesuffix(".meta.json"), "claim": m.group(1),
|
|
"reads": len(reads), "breaks": breaks})
|
|
return {"schema": ISOLATION_SCHEMA, "run": run, "line": line, "tasks": str(tasks), "agents": len(per_agent),
|
|
"breaks": sum(len(a["breaks"]) for a in per_agent), "per_agent": per_agent}
|
|
|
|
|
|
def load_json(path: Path) -> Any:
|
|
return json.loads(path.read_text(encoding="utf-8"))
|
|
|
|
|
|
def write_json(path: Path, doc: Any) -> None:
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
path.write_text(json.dumps(doc, ensure_ascii=False, indent=1) + "\n", encoding="utf-8")
|
|
|
|
|
|
def build_parser() -> argparse.ArgumentParser:
|
|
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
sub = p.add_subparsers(dest="command", required=True)
|
|
h = sub.add_parser("harvest")
|
|
h.add_argument("--transcripts", required=True)
|
|
h.add_argument("--okter", required=True)
|
|
h.add_argument("--work", required=True)
|
|
h.add_argument("--out", required=True)
|
|
h.add_argument("--packs")
|
|
h.add_argument("--margin", type=float)
|
|
s = sub.add_parser("stop")
|
|
s.add_argument("--lines", required=True)
|
|
s.add_argument("--cap", type=int, required=True)
|
|
s.add_argument("--unit", choices=UNITS, required=True)
|
|
s.add_argument("--done", type=int, required=True)
|
|
s.add_argument("--total", type=int, required=True)
|
|
s.add_argument("--kost", required=True)
|
|
s.add_argument("--blocks")
|
|
s.add_argument("--batches-done", type=int)
|
|
s.add_argument("--batches-total", type=int)
|
|
s.add_argument("--next", type=int)
|
|
s.add_argument("--next-chars", type=int)
|
|
s.add_argument("--margin", type=float)
|
|
s.add_argument("--name")
|
|
m = sub.add_parser("summarize")
|
|
m.add_argument("--kost", required=True)
|
|
m.add_argument("--cap-group", action="append", default=[])
|
|
i = sub.add_parser("isolation")
|
|
i.add_argument("--transcripts", required=True)
|
|
i.add_argument("--run", required=True)
|
|
i.add_argument("--tasks", required=True)
|
|
i.add_argument("--out", required=True)
|
|
i.add_argument("--line", choices=ISOLATED, default="read")
|
|
return p
|
|
|
|
|
|
def main(argv: list[str] | None = None) -> int:
|
|
args = build_parser().parse_args(argv)
|
|
try:
|
|
if args.command == "harvest":
|
|
out = Path(args.out)
|
|
h = harvest(Path(args.transcripts), sessions(Path(args.okter)))
|
|
if args.packs:
|
|
check_packs(h["runs"], load_json(Path(args.packs)))
|
|
old = load_json(out) if out.is_file() else {}
|
|
doc = {"schema": SCHEMA, "work": str(args.work), **h,
|
|
"margin": {"measured": args.margin, "used": margin(args.margin)},
|
|
"caps": old.get("caps", {}), "stops": old.get("stops", {})}
|
|
write_json(out, doc)
|
|
print(f"harvest: {len(h['runs'])} attributed runs, {len(h['unattributed'])} unattributed, "
|
|
f"{len(h['orchestration'])} orchestrating sessions -> {out}")
|
|
for u in h["unattributed"]:
|
|
print(f" unattributed: {u['agent']} {u['description']!r}")
|
|
return 1 if h["unattributed"] else 0
|
|
if args.command == "stop":
|
|
kost_path = Path(args.kost)
|
|
kost = load_json(kost_path)
|
|
lines = capped_lines(args.lines)
|
|
used = margin(args.margin) if args.margin is not None else float(kost["margin"]["used"])
|
|
packs_log = load_json(Path(args.blocks)) if args.blocks else None
|
|
given = (args.batches_done, args.batches_total)
|
|
if (given[0] is None) != (given[1] is None):
|
|
raise RunError("--batches-done and --batches-total go together")
|
|
batches = None if given[0] is None else (int(given[0]), int(given[1]))
|
|
row = stop_row(kost, lines, args.cap, args.unit, args.done, args.total, packs_log, used, batches, args.next,
|
|
args.next_chars)
|
|
name = args.name or ",".join(lines)
|
|
kost.setdefault("caps", {})[name] = {"lines": lines, "cap": args.cap}
|
|
kost.setdefault("stops", {})[name] = row
|
|
write_json(kost_path, kost)
|
|
fit = f", fit {row['fit']:.0f}" if row["fit"] is not None else ""
|
|
if "batches_total" in row:
|
|
fit += (f" (helpers {row['helpers']} x {args.total}/{args.done}, before the first helper "
|
|
f"{row['before_first_helper']} once, after it {row['after_first_helper']} x "
|
|
f"{row['batches_total']}/{row['batches_done']} batches)")
|
|
if row.get("rule") == "measured-chars":
|
|
rate = (f"{row['measured']}/{row['rate_chars_over']} (the characters with a compiler run here)"
|
|
if row["rate_chars_over"] else
|
|
f"{float(ROUND2_PER_CHAR):.2f} (round 2's price per character; no compiler run here)")
|
|
print(f"{name}: measured {row['measured']} + next {row['next_chars']} characters x {rate}, "
|
|
f"x (1 + {used:.2f}) = {row['measured_next_with_margin']} of {args.cap}: "
|
|
f"{'STOP' if row['stop'] else 'continue'}; projection (reported only): flat {row['flat']:.0f}{fit}, "
|
|
f"x (1 + {used:.2f}) = {row['projected_with_margin']}")
|
|
elif "rule" in row:
|
|
rate = (f"{row['measured']}/{row['rate_blocks']} (the blocks with a compiler run here)"
|
|
if row["rate_blocks"] else f"{ROUND2_PER_BLOCK} (round 2's price per block; no compiler run here)")
|
|
print(f"{name}: measured {row['measured']} + next {row['next']} blocks x {rate}, "
|
|
f"x (1 + {used:.2f}) = {row['measured_next_with_margin']} of {args.cap}: "
|
|
f"{'STOP' if row['stop'] else 'continue'}; projection (reported only): flat {row['flat']:.0f}{fit}, "
|
|
f"x (1 + {used:.2f}) = {row['projected_with_margin']}")
|
|
else:
|
|
print(f"{name}: measured {row['measured']}, flat {row['flat']:.0f}{fit}, x (1 + {used:.2f}) = "
|
|
f"{row['projected_with_margin']} of {args.cap}: {'STOP' if row['stop'] else 'continue'}")
|
|
return 1 if row["stop"] else 0
|
|
if args.command == "summarize":
|
|
extra = dict(parse_cap_group(g) for g in args.cap_group)
|
|
text, within = summarize(load_json(Path(args.kost)), extra)
|
|
print("\n".join(text))
|
|
return 0 if within else 1
|
|
doc = isolation(Path(args.transcripts), args.run, Path(args.tasks), args.line)
|
|
write_json(Path(args.out), doc)
|
|
kind = {"read": "reader", "diagnose": "diagnosis", "repair": "repair"}[args.line]
|
|
print(f"isolation: {doc['breaks']} breaks over {doc['agents']} {kind} agents")
|
|
return 0 if doc["agents"] and doc["breaks"] == 0 else 1
|
|
except (RunError, OSError, json.JSONDecodeError, KeyError, ValueError, ZeroDivisionError, TypeError) as exc:
|
|
print(f"cost cannot run: {exc}", file=sys.stderr)
|
|
return 2
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|