okf-wiki/tests/test_cost.py
Kjell Tore Guttormsen 0fd86a23a3
okf-wiki 1.1.1
First public release, published as one snapshot. What changed between
versions is in CHANGELOG.md.

Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
2026-11-14 21:03:20 +01:00

840 lines
50 KiB
Python

"""Tests for okfwiki/cost.py, okfwiki/transcript.py and okfwiki/common.py: cost lines, stop checks and isolation.
Every transcript here is synthetic. The usage reading is ported from the trial (evals/p1/cost.py) with its tests.
Run: python3 -m unittest discover -s tests -p 'test_cost.py'
"""
from __future__ import annotations
import io
import json
import math
import shutil
import sys
import tempfile
import unittest
from contextlib import redirect_stderr, redirect_stdout
from fractions import Fraction
from pathlib import Path
from typing import Any
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT))
from okfwiki import common, cost # noqa: E402
def msg(mid: str, inp: int, cr: int, cw: int, out: int, stop: str | None = "end_turn",
tools: list[dict[str, Any]] | None = None) -> dict[str, Any]:
return {"type": "assistant", "message": {"id": mid, "stop_reason": stop, "content": tools or [], "usage": {
"input_tokens": inp, "cache_read_input_tokens": cr, "cache_creation_input_tokens": cw, "output_tokens": out}}}
def jsonl(path: Path, lines: list[dict[str, Any]]) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text("\n".join(json.dumps(x) for x in lines) + "\n", encoding="utf-8")
USER = {"type": "user", "message": {"content": "task"}}
class Transcripts(unittest.TestCase):
"""A transcripts directory laid out as Claude Code writes it: <id>.jsonl and <id>/subagents/agent-*.jsonl."""
def setUp(self) -> None:
self._tmp = tempfile.TemporaryDirectory()
self.root = Path(self._tmp.name)
self.proj = self.root / "transcripts"
self.work = self.root / "run"
self.work.mkdir()
self.okter = self.work / "okter.txt"
def tearDown(self) -> None:
self._tmp.cleanup()
def agent(self, sid: str, name: str, description: str, lines: list[dict[str, Any]],
agent_type: str = "okf-wiki-linker", workflow: str | None = None) -> None:
sub = self.proj / sid / "subagents"
if workflow is not None: # a Workflow's agents lie in <id>/subagents/workflows/<wf>/ (measured 09.10, 2.1.295)
sub = sub / "workflows" / workflow
jsonl(sub / f"agent-{name}.jsonl", [USER, *lines])
(sub / f"agent-{name}.meta.json").write_text(json.dumps({"agentType": agent_type, "description": description}))
def harvest(self, *extra: str) -> tuple[int, dict[str, Any], str]:
out = self.work / "kost.json"
code, text = run_main(["harvest", "--transcripts", str(self.proj), "--okter", str(self.okter),
"--work", str(self.work), "--out", str(out), *extra])
return code, json.loads(out.read_text(encoding="utf-8")) if out.is_file() else {}, text
def run_main(argv: list[str]) -> tuple[int, str]:
buf = io.StringIO()
with redirect_stdout(buf), redirect_stderr(io.StringIO()):
code = cost.main(argv)
return code, buf.getvalue()
class HarvestTest(Transcripts):
def setUp(self) -> None:
super().setUp()
jsonl(self.proj / "s1.jsonl", [USER, msg("m0", 5, 100, 10, 7)])
# a streamed message id appears twice with growing output: counted once, at its max
self.agent("s1", "a1", "V1-utsnitt-pack-p1-1", [msg("x1", 3, 20_000, 2_000, 10, None),
msg("x1", 3, 20_000, 2_000, 50), msg("x2", 1, 22_000, 500, 900)],
agent_type="okf-wiki-compiler")
self.agent("s1", "a2", "V1-utsnitt-link-p1-1", [msg("y1", 2, 19_000, 1_000, 5)])
self.agent("s2", "a3", "V1-utsnitt-read-c007", [msg("z1", 1, 1_000, 100, 3)], agent_type="okf-wiki-reader")
self.agent("s2", "a4", "V1-utsnitt-calib-pair12", [msg("q1", 1, 500, 50, 2)], agent_type="okf-wiki-reader")
jsonl(self.proj / "s2.jsonl", [USER, msg("r0", 4, 300, 30, 9)])
# a session not listed in okter.txt: its transcript is never opened
self.agent("s9", "b1", "V1-utsnitt-pack-p2-1", [msg("q", 1, 1, 1, 1)])
self.okter.write_text("s1 build\ns2 read\n", encoding="utf-8")
def test_four_categories_turns_startup_and_attribution(self) -> None:
code, kost, _ = self.harvest()
self.assertEqual(0, code)
self.assertEqual([{"id": "s1", "kind": "build"}, {"id": "s2", "kind": "read"}], kost["sessions"])
runs = {r["agent"]: r for r in kost["runs"]}
k = runs["agent-a1"]
self.assertEqual(("utsnitt", "build", "p1-1", 1), (k["run"], k["line"], k["unit"], k["pack"]))
self.assertEqual((4, 42_000, 2_500, 950), (k["input"], k["cache_read"], k["cache_write"], k["output"]))
self.assertEqual((2, 3 + 20_000 + 2_000), (k["turns"], k["startup"]))
self.assertEqual(("link", 1), (runs["agent-a2"]["line"], runs["agent-a2"]["pack"]))
self.assertEqual(("read", None), (runs["agent-a3"]["line"], runs["agent-a3"]["pack"]))
self.assertNotIn("agent-b1", runs)
self.assertEqual([], kost["unattributed"])
def test_per_line_totals_by_hand(self) -> None:
_, kost, _ = self.harvest()
lines = kost["lines"]
self.assertEqual(4 + 42_000 + 2_500 + 950, lines["build"]["total"])
self.assertEqual(2 + 19_000 + 1_000 + 5, lines["link"]["total"])
self.assertEqual(1 + 1_000 + 100 + 3, lines["read"]["total"])
self.assertEqual(1 + 500 + 50 + 2, lines["calib"]["total"])
# the orchestrating sessions land on their own lines by the kind okter.txt gives them
self.assertEqual(5 + 100 + 10 + 7, lines["orchestration-build"]["total"])
self.assertEqual(4 + 300 + 30 + 9, lines["orchestration-read"]["total"])
self.assertEqual(list(cost.LINES), list(lines))
def test_input_per_orchestration_turn_is_kept_with_its_turn_number(self) -> None:
# Plankontroll 05.10, tillegg 3: does the cost per orchestrating turn grow with the session's length?
jsonl(self.proj / "s1.jsonl", [USER, msg("t1", 2, 70_000, 6_000, 40), {"type": "user", "message": {"content": "go"}},
msg("t2", 2, 76_000, 3_000, 60), msg("t3", 2, 79_000, 9_000, 80)])
_, kost, _ = self.harvest()
turns = kost["orchestration"]["s1"]["per_turn"]
self.assertEqual([{"turn": 1, "input": 76_002}, {"turn": 2, "input": 79_002}, {"turn": 3, "input": 88_002}], turns)
code, text = run_main(["summarize", "--kost", str(self.work / "kost.json")])
self.assertEqual(0, code)
self.assertIn("orchestration-build input per turn: 69002 + 6000 x turn", text)
def test_a_workflows_agents_are_harvested_like_the_sessions_own(self) -> None:
self.agent("s1", "w1", "V1-utsnitt-pack-p3-1", [msg("w1", 2, 10_000, 1_000, 30)],
agent_type="okf-wiki-compiler", workflow="wf_abc123")
self.agent("s2", "w2", "V1-utsnitt-read-c010", [msg("w2", 1, 2_000, 200, 4)],
agent_type="okf-wiki-reader", workflow="wf_def456")
self.agent("s9", "w9", "V1-utsnitt-pack-p4-1", [msg("w9", 1, 1, 1, 1)], workflow="wf_zzz")
code, kost, _ = self.harvest()
self.assertEqual(0, code)
runs = {r["agent"]: r for r in kost["runs"]}
self.assertEqual(("build", 3, "s1"), (runs["agent-w1"]["line"], runs["agent-w1"]["pack"], runs["agent-w1"]["session"]))
self.assertEqual(("read", "s2"), (runs["agent-w2"]["line"], runs["agent-w2"]["session"]))
self.assertNotIn("agent-w9", runs) # its session is not in okter.txt
self.assertEqual(4 + 42_000 + 2_500 + 950 + 2 + 10_000 + 1_000 + 30, kost["lines"]["build"]["total"])
def test_an_unattributed_workflow_agent_exits_1(self) -> None:
self.agent("s1", "w8", "explore the sources", [msg("h8", 1, 1, 1, 1)], agent_type="Explore", workflow="wf_abc123")
code, kost, _ = self.harvest()
self.assertEqual(1, code)
self.assertEqual(["explore the sources"], [u["description"] for u in kost["unattributed"]])
def test_an_unattributed_run_exits_1(self) -> None:
self.agent("s1", "a9", "look around the repository", [msg("h1", 1, 1, 1, 1)], agent_type="Explore")
code, kost, _ = self.harvest()
self.assertEqual(1, code)
self.assertEqual(["look around the repository"], [u["description"] for u in kost["unattributed"]])
def test_a_session_okter_lists_that_the_transcripts_lack_cannot_run(self) -> None:
# Order 27 A3: a wrong transcripts folder gave 0 for a whole variant without a word
self.okter.write_text("s1 build\ns3 build\n", encoding="utf-8")
code, _, _ = self.harvest()
self.assertEqual(2, code)
self.assertFalse((self.work / "kost.json").exists())
def test_an_unknown_session_kind_cannot_run(self) -> None:
self.okter.write_text("s1 build\ns2 other\n", encoding="utf-8")
self.assertEqual(2, self.harvest()[0])
def test_a_pack_log_gap_cannot_run(self) -> None:
packs = self.work / "packs.json"
packs.write_text(json.dumps({"packs": [{"pack": 1, "blocks": 10}, {"pack": 2, "blocks": 10}]}))
self.assertEqual(2, self.harvest("--packs", str(packs))[0]) # pack 2 is logged but has no build run
packs.write_text(json.dumps({"packs": [{"pack": 1, "blocks": 10}]}))
self.assertEqual(0, self.harvest("--packs", str(packs))[0])
def test_a_link_run_does_not_fill_a_build_gap(self) -> None:
self.agent("s1", "a7", "V1-utsnitt-link-p2-1", [msg("l2", 1, 1, 1, 1)])
packs = self.work / "packs.json"
packs.write_text(json.dumps({"packs": [{"pack": 1, "blocks": 10}, {"pack": 2, "blocks": 10}]}))
self.assertEqual(2, self.harvest("--packs", str(packs))[0])
def test_output_never_recorded_is_derived_from_the_next_messages_input_growth(self) -> None:
def result(text: str) -> dict[str, Any]:
return {"type": "user", "message": {"content": [{"type": "tool_result", "content": text}]}}
self.agent("s1", "a5", "V1-utsnitt-pack-p3-1", [msg("w1", 2, 0, 10_000, 300, "tool_use"), result("x" * 400),
msg("w2", 2, 10_000, 50_000, 2, None),
msg("w2", 2, 10_000, 50_000, 2, None),
result("y" * 40), msg("w3", 2, 60_000, 5_010, 20)])
_, kost, _ = self.harvest()
w = {r["agent"]: r for r in kost["runs"]}["agent-a5"]
self.assertEqual(300 + 2 + 20, w["output_recorded"])
self.assertEqual(300 + (5_010 - 40 * 2 // 5) + 20, w["output"])
self.assertEqual((1, 0), (w["output_derived_messages"], w["output_unrecorded"]))
def test_growth_from_injected_attachments_is_not_counted_as_output(self) -> None:
self.agent("s1", "a6", "V1-utsnitt-pack-p4-1", [msg("v1", 2, 0, 18_602, 16, None),
{"type": "attachment", "rendered": "z" * 5_133},
{"type": "user", "message": {"content": [
{"type": "tool_result", "content": "w" * 151}]}},
msg("v2", 2, 18_602, 1_894, 24, None)])
_, kost, _ = self.harvest()
v = {r["agent"]: r for r in kost["runs"]}["agent-a6"]
self.assertEqual((16 + 24, 0), (v["output"], v["output_derived_messages"]))
def test_harvest_writes_a_margin_and_keeps_recorded_caps(self) -> None:
_, kost, _ = self.harvest()
self.assertEqual({"measured": None, "used": 0.10}, kost["margin"])
kost["caps"] = {"build": {"lines": ["build"], "cap": 9}}
(self.work / "kost.json").write_text(json.dumps(kost))
_, again, _ = self.harvest("--margin", "0.12")
self.assertEqual({"build": {"lines": ["build"], "cap": 9}}, again["caps"])
self.assertEqual({"measured": 0.12, "used": 0.12}, again["margin"])
def kost_file(path: Path, lines: dict[str, int], runs: list[dict[str, Any]] | None = None,
margin: float = 0.10) -> Path:
doc = {"schema": cost.SCHEMA, "lines": {name: {"total": lines.get(name, 0)} for name in cost.LINES},
"runs": runs or [], "margin": {"measured": None, "used": margin}, "caps": {}, "stops": {}}
path.write_text(json.dumps(doc), encoding="utf-8")
return path
class StopTest(unittest.TestCase):
def setUp(self) -> None:
self._tmp = tempfile.TemporaryDirectory()
self.kost = Path(self._tmp.name) / "kost.json"
def tearDown(self) -> None:
self._tmp.cleanup()
def stop(self, lines: str, cap: int, unit: str, done: int, total: int, *extra: str) -> int:
return run_main(["stop", "--lines", lines, "--cap", str(cap), "--unit", unit, "--done", str(done),
"--total", str(total), "--kost", str(self.kost), *extra])[0]
def test_at_the_cap_continues_one_over_stops(self) -> None:
kost_file(self.kost, {"link": 1_000})
self.assertEqual(0, self.stop("link", 1_100, "passes", 1, 1)) # 1000 x 1.10 = 1100
self.assertEqual(1, self.stop("link", 1_099, "passes", 1, 1))
def test_the_build_cap_sums_build_link_and_build_orchestration(self) -> None:
kost_file(self.kost, {"build": 400, "link": 300, "orchestration-build": 300, "orchestration-read": 10_000,
"read": 10_000})
self.assertEqual(0, self.stop("build,link,orchestration-build", 2_200, "passes", 1, 2))
self.assertEqual(1, self.stop("build,link,orchestration-build", 2_199, "passes", 1, 2))
def test_blocks_stop_on_the_fit_when_the_flat_projection_is_under_the_cap(self) -> None:
runs = [{"line": "build", "pack": n, "total": t}
for n, t in ((1, 100_000), (2, 300_000), (3, 500_000), (4, 700_000))]
kost_file(self.kost, {"build": 1_600_000}, runs)
packs = Path(self._tmp.name) / "packs.json"
packs.write_text(json.dumps({"packs": [{"pack": n, "blocks": 100} for n in (1, 2, 3, 4)]}))
# flat: 1600000 / 400 x 1000 = 4.0 M; fit: a = 1000, b = 20, 1000 x 1000 + 20 x 1000^2 / 2 = 11 M
self.assertEqual(1, self.stop("build", 5_000_000, "blocks", 400, 1_000, "--blocks", str(packs)))
self.assertEqual(0, self.stop("build", 12_100_000, "blocks", 400, 1_000, "--blocks", str(packs)))
row = json.loads(self.kost.read_text(encoding="utf-8"))["stops"]["build"]
self.assertAlmostEqual(4_000_000, row["flat"])
self.assertAlmostEqual(11_000_000, row["fit"])
def test_the_growth_curve_counts_only_from_the_fourth_pack(self) -> None:
# Wave 3, order 11, after the link pass of pack 2 (logg/p2.txt): pack 1 cost 756203 over 871 blocks, pack 2
# 534095 over 466, measured 3397056 after 1337 of 4509 blocks: "flat 11456489, fit 18846464 ... = 20731111 of
# 16000000: STOP". The operator 11.10.2026: the fitted curve counts from pack 4; before that the flat one alone.
runs = [{"line": "build", "pack": 1, "total": 756_203}, {"line": "build", "pack": 2, "total": 534_095}]
kost_file(self.kost, {"build": 1_290_298, "orchestration-build": 2_106_758}, runs)
packs = Path(self._tmp.name) / "packs.json"
packs.write_text(json.dumps({"packs": [{"pack": 1, "blocks": 871}, {"pack": 2, "blocks": 466}]}))
scale = 3_397_056 / 1_290_298
old = cost.project([(756_203 * scale, 871, 0), (534_095 * scale, 466, 871)], 4_509)["fit"]["value"]
self.assertGreater(old * 1.10, 16_000_000) # the old rule: the fit said STOP
self.assertEqual(0, self.stop("build,link,orchestration-build", 16_000_000, "blocks", 1_337, 4_509,
"--blocks", str(packs)))
row = json.loads(self.kost.read_text(encoding="utf-8"))["stops"]["build,link,orchestration-build"]
self.assertIsNone(row["fit"])
self.assertEqual(math.ceil(Fraction(3_397_056) * 4_509 / 1_337 * Fraction(11, 10)), row["projected_with_margin"])
self.assertEqual(11_456_489, round(row["flat"]))
# a third pack still leaves the curve out; a fourth brings it in as before
runs.append({"line": "build", "pack": 3, "total": 600_000})
kost_file(self.kost, {"build": 1_890_298, "orchestration-build": 2_106_758}, runs)
packs.write_text(json.dumps({"packs": [{"pack": 1, "blocks": 871}, {"pack": 2, "blocks": 466},
{"pack": 3, "blocks": 400}]}))
self.stop("build,link,orchestration-build", 16_000_000, "blocks", 1_737, 4_509, "--blocks", str(packs))
self.assertIsNone(json.loads(self.kost.read_text(encoding="utf-8"))["stops"]["build,link,orchestration-build"]["fit"])
runs.append({"line": "build", "pack": 4, "total": 900_000})
kost_file(self.kost, {"build": 2_790_298, "orchestration-build": 2_106_758}, runs)
packs.write_text(json.dumps({"packs": [{"pack": 1, "blocks": 871}, {"pack": 2, "blocks": 466},
{"pack": 3, "blocks": 400}, {"pack": 4, "blocks": 500}]}))
self.stop("build,link,orchestration-build", 16_000_000, "blocks", 2_237, 4_509, "--blocks", str(packs))
self.assertIsNotNone(json.loads(self.kost.read_text(encoding="utf-8"))["stops"]["build,link,orchestration-build"]["fit"])
def test_with_next_the_stop_is_the_measured_cost_and_the_next_pack(self) -> None:
# The operator 12.10.2026: a cost line stops on measured cost plus a margin; the projection is reported only
# (order 22 session A3: plus the next pack's expected cost, so the cap is never passed). Measured 1000 over 10
# blocks, next pack 5 blocks: (1000 + 1000 x 5 / 10) x 1.1 = 1650; the flat projection 1000 x 100 / 10 x 1.1
# = 11000 would have stopped. (Order 27 A3: the 10 blocks are pack 1's, whose compiler run is in the ledger.)
kost_file(self.kost, {"build": 1_000}, [{"line": "build", "pack": 1, "total": 1_000}])
packs = Path(self._tmp.name) / "packs.json"
packs.write_text(json.dumps({"packs": [{"pack": 1, "blocks": 10}]}))
argv = ("build", 1_650, "blocks", 10, 100, "--blocks", str(packs), "--next", "5")
self.assertEqual(0, self.stop_or_exit(*argv))
row = json.loads(self.kost.read_text(encoding="utf-8"))["stops"]["build"]
self.assertEqual(("measured", 5, 1_650, 11_000, False), (row.get("rule"), row.get("next"),
row.get("measured_next_with_margin"), row["projected_with_margin"], row["stop"]))
self.assertEqual(1, self.stop_or_exit("build", 1_649, *argv[2:]))
self.assertEqual(2, self.stop_or_exit("build", 10**9, "passes", 10, 100, "--next", "5")) # blocks only
self.assertEqual(2, self.stop_or_exit("build", 10**9, *argv[2:-1], "-1"))
def stop_or_exit(self, lines: str, cap: int, unit: str, done: int, total: int, *extra: str) -> int:
try:
return self.stop(lines, cap, unit, done, total, *extra)
except SystemExit as exc: # an option the parser does not know
return int(exc.code or 0)
def test_calib_is_reported_never_capped(self) -> None:
kost_file(self.kost, {"calib": 5})
self.assertEqual(2, self.stop("calib", 100, "claims", 1, 1))
def test_zero_done_and_bad_numbers_cannot_run(self) -> None:
kost_file(self.kost, {"link": 1})
self.assertEqual(2, self.stop("link", 100, "passes", 0, 5))
self.assertEqual(2, self.stop("link", 100, "passes", 3, 2))
self.assertEqual(2, self.stop("link", 100, "blocks", 1, 2)) # blocks needs --blocks
def test_the_stop_records_its_cap_group_for_summarize(self) -> None:
kost_file(self.kost, {"build": 10, "link": 10, "orchestration-build": 10})
self.assertEqual(0, self.stop("build,link,orchestration-build", 1_000, "passes", 1, 1, "--name", "bygging"))
code, text = run_main(["summarize", "--kost", str(self.kost)])
self.assertEqual(0, code)
self.assertIn("bygging: 30 + margin 0.10 = 33 of 1000", text)
class ContinuedBuildStopTest(unittest.TestCase):
"""Order 27 A3: a build that continues a collection built in another ledger (packs 1-8 of round 2 cost nothing in
the variant's ledger). The next pack's share is the measured cost over the blocks of the packs with a compiler run
in this ledger, times the next pack's blocks; before the ledger's first, round 2's price per block. Wave 4's use,
every pack in one ledger, gives the numbers it gave."""
def setUp(self) -> None:
self._tmp = tempfile.TemporaryDirectory()
self.kost = Path(self._tmp.name) / "kost.json"
self.packs = Path(self._tmp.name) / "packs.json"
def tearDown(self) -> None:
self._tmp.cleanup()
def row(self, cap: int, done: int, total: int, following: int) -> tuple[int, dict[str, Any]]:
code, _ = run_main(["stop", "--lines", "build,link,orchestration-build", "--cap", str(cap), "--unit", "blocks",
"--done", str(done), "--total", str(total), "--kost", str(self.kost), "--blocks",
str(self.packs), "--next", str(following)])
doc = json.loads(self.kost.read_text(encoding="utf-8"))
return code, dict(doc["stops"].get("build,link,orchestration-build", {})) # none when it could not run
def test_the_share_comes_from_this_ledgers_packs_not_from_the_packs_built_before(self) -> None:
# packs 1 and 2 built before (4000 blocks, no run here), pack 3 here: 1000 by its compiler, 500 orchestration.
# Rate 1500 / 500 = 3 per block; next pack 100 blocks: (1500 + 300) x 1.1 = 1980 (over the done blocks it
# would be 1500 / 4500 x 100 = 33, and 1687).
self.packs.write_text(json.dumps({"packs": [{"pack": 1, "blocks": 3000, "accepted": 1},
{"pack": 2, "blocks": 1000, "accepted": 1},
{"pack": 3, "blocks": 500, "accepted": 1}]}))
kost_file(self.kost, {"build": 1_000, "orchestration-build": 500},
[{"line": "build", "pack": 3, "total": 1_000}])
code, row = self.row(1_980, 4_500, 4_600, 100)
self.assertEqual((0, 1_980), (code, row["measured_next_with_margin"]))
self.assertEqual((500, 3), (row["rate_blocks"], row["rate"]))
self.assertAlmostEqual(1_800, row["flat"]) # 1500 + 3 x (4600 - 4500), reported only
_, text = run_main(["stop", "--lines", "build,link,orchestration-build", "--cap", "1980", "--unit", "blocks",
"--done", "4500", "--total", "4600", "--kost", str(self.kost), "--blocks", str(self.packs),
"--next", "100"])
self.assertIn("measured 1500 + next 100 blocks x 1500/500 (the blocks with a compiler run here)", text)
self.assertEqual(1, self.row(1_979, 4_500, 4_600, 100)[0])
def test_before_the_ledger_s_first_compiler_run_the_share_is_round_2_s_price_per_block(self) -> None:
self.assertEqual(cost.ROUND2_PER_BLOCK, 3_259) # round 2: 14 741 606 over 4 523 blocks (order 25 session C)
self.packs.write_text(json.dumps({"packs": [{"pack": 1, "blocks": 4_000, "accepted": 1}]}))
kost_file(self.kost, {"orchestration-build": 500})
code, row = self.row(10**9, 4_000, 4_100, 100)
self.assertEqual(0, code)
self.assertEqual((0, 3_259), (row["rate_blocks"], row["rate"]))
self.assertEqual(math.ceil((500 + 3_259 * 100) * Fraction(11, 10)), row["measured_next_with_margin"])
def test_wave_4_s_use_gives_the_numbers_it_gave(self) -> None:
# every pack in one ledger, as round 2 ran: the rate is the measured cost over the done blocks, as before
self.packs.write_text(json.dumps({"packs": [{"pack": 1, "blocks": 885, "accepted": 1},
{"pack": 2, "blocks": 466, "accepted": 1}]}))
kost_file(self.kost, {"build": 2_580_808, "link": 249_822, "orchestration-build": 1_538_241},
[{"line": "build", "pack": 1, "total": 1_682_681}, {"line": "build", "pack": 2, "total": 898_127},
{"line": "link", "pack": 1, "total": 249_822}])
measured = 2_580_808 + 249_822 + 1_538_241
code, row = self.row(22_000_000, 1_351, 4_523, 235)
self.assertEqual(0, code)
self.assertEqual(math.ceil((measured + Fraction(measured) * 235 / 1_351) * Fraction(11, 10)),
row["measured_next_with_margin"])
self.assertAlmostEqual(float(Fraction(measured) * 4_523 / 1_351), row["flat"])
def test_a_compiler_run_for_a_pack_the_log_lacks_cannot_run(self) -> None:
self.packs.write_text(json.dumps({"packs": [{"pack": 1, "blocks": 10, "accepted": 1}]}))
kost_file(self.kost, {"build": 100}, [{"line": "build", "pack": 2, "total": 100}])
self.assertEqual(2, self.row(10**9, 10, 20, 5)[0])
class CharsStopTest(unittest.TestCase):
"""Motor B17: the next pack's share by the characters of its blocks, not their number. A block of collection 1
holds 229 characters on average, a PDF block of a real mixed folder 2 310 and a spreadsheet block 2 150, while the
compiler's cost per character stays near (6.9 on collection 1's round 2, 5.8 and 8.5 on two folder trials)."""
def setUp(self) -> None:
self._tmp = tempfile.TemporaryDirectory()
self.kost = Path(self._tmp.name) / "kost.json"
self.packs = Path(self._tmp.name) / "packs.json"
def tearDown(self) -> None:
self._tmp.cleanup()
def chars_row(self, cap: int, following: int, chars: int, *extra: str) -> tuple[int, dict[str, Any], str]:
code, text = run_main(["stop", "--lines", "build,link,orchestration-build", "--cap", str(cap), "--unit",
"blocks", "--done", "10", "--total", "100", "--kost", str(self.kost), "--blocks",
str(self.packs), "--next", str(following), "--next-chars", str(chars), *extra])
doc = json.loads(self.kost.read_text(encoding="utf-8"))
return code, dict(doc["stops"].get("build,link,orchestration-build", {})), text
def test_the_share_is_the_measured_cost_per_character_times_the_next_pack_s_characters(self) -> None:
# 1000 over pack 1's 2000 characters: 0.5 per character; the next pack 5 blocks of 4000 characters:
# (1000 + 2000) x 1.1 = 3300, where by blocks it would be (1000 + 500) x 1.1 = 1650.
self.packs.write_text(json.dumps({"packs": [{"pack": 1, "blocks": 10, "block_chars": 2_000, "accepted": 1}]}))
kost_file(self.kost, {"build": 1_000}, [{"line": "build", "pack": 1, "total": 1_000}])
code, row, text = self.chars_row(3_300, 5, 4_000)
self.assertEqual((0, "measured-chars", 4_000, 3_300, 2_000), (code, row["rule"], row["next_chars"],
row["measured_next_with_margin"], row["rate_chars_over"]))
self.assertIn("measured 1000 + next 4000 characters x 1000/2000 (the characters with a compiler run here)", text)
self.assertEqual(1, self.chars_row(3_299, 5, 4_000)[0])
def test_before_the_ledger_s_first_compiler_run_the_share_is_round_2_s_price_per_character(self) -> None:
self.assertEqual(cost.ROUND2_PER_CHAR, Fraction(14_741_606, 1_111_796)) # round 2's blocks, 1 111 796 chars
self.packs.write_text(json.dumps({"packs": [{"pack": 1, "blocks": 10, "block_chars": 2_000, "accepted": 1}]}))
kost_file(self.kost, {"orchestration-build": 500})
code, row, _ = self.chars_row(10**9, 5, 10_000)
self.assertEqual((0, 0), (code, row["rate_chars_over"]))
self.assertEqual(math.ceil((500 + cost.ROUND2_PER_CHAR * 10_000) * Fraction(11, 10)),
row["measured_next_with_margin"])
def test_a_built_pack_without_characters_keeps_the_rule_by_blocks(self) -> None:
# a pack log written before blocks had characters (collection 1's builds) is priced as before
self.packs.write_text(json.dumps({"packs": [{"pack": 1, "blocks": 10, "accepted": 1}]}))
kost_file(self.kost, {"build": 1_000}, [{"line": "build", "pack": 1, "total": 1_000}])
code, row, _ = self.chars_row(1_650, 5, 4_000)
self.assertEqual((0, "measured", 1_650), (code, row["rule"], row["measured_next_with_margin"]))
def test_next_chars_needs_next(self) -> None:
self.packs.write_text(json.dumps({"packs": [{"pack": 1, "blocks": 10, "block_chars": 2_000}]}))
kost_file(self.kost, {"build": 1_000}, [{"line": "build", "pack": 1, "total": 1_000}])
code, _ = run_main(["stop", "--lines", "build", "--cap", "10", "--unit", "blocks", "--done", "10", "--total",
"100", "--kost", str(self.kost), "--blocks", str(self.packs), "--next-chars", "5"])
self.assertEqual(2, code)
def spawn(mid: str, inp: int, cr: int, cw: int, out: int, tool: str = "Agent") -> dict[str, Any]:
"""An orchestrating turn that starts helpers (Agent) or a workflow (Workflow)."""
return msg(mid, inp, cr, cw, out, "tool_use", [{"type": "tool_use", "name": tool, "input": {}}])
# Wave 3, order 12, the price measurement (pris/kost-k010.json and the session's turns): ten readers, 34 of 2132 claims,
# 115687 tokens; the main session 2480218 over 24 turns, of which turns 1-21 (2074324) came before the turn that
# started the ten readers, and turns 22-24 (405894) were that batch. The rest ran as one workflow with one phase, so
# the reading had two batches; it ended at 10492384 measured.
READERS_K001_K010 = (11_183, 11_208, 11_307, 11_436, 11_215, 11_510, 12_028, 12_164, 11_335, 12_301)
class ClaimsProjectionTest(Transcripts):
def setUp(self) -> None:
super().setUp()
jsonl(self.proj / "s1.jsonl", [USER, msg("o1", 0, 2_000_000, 74_324, 0, "tool_use"),
spawn("o2", 0, 120_000, 9_000, 694),
msg("o3", 0, 130_000, 4_000, 565, "tool_use"),
msg("o4", 0, 140_000, 1_000, 635, "tool_use")])
for i, total in enumerate(READERS_K001_K010, 1):
self.agent("s1", f"r{i}", f"V1-del1l-read-k{i:03d}", [msg(f"k{i}", 0, total - 100, 0, 100)],
agent_type="okf-wiki-reader")
self.okter.write_text("s1 read\n", encoding="utf-8")
self.kost = self.work / "kost.json"
def stop(self, *extra: str) -> tuple[int, str]:
return run_main(["stop", "--lines", "read,orchestration-read", "--cap", "13000000", "--unit", "claims",
"--done", "34", "--total", "2132", "--kost", str(self.kost), *extra])
def test_the_harvest_keeps_the_sessions_part_before_its_first_helper(self) -> None:
_, kost, _ = self.harvest()
o = kost["orchestration"]["s1"]
self.assertEqual((2, 2_074_324), (o["first_helper_turn"], o["before_first_helper"]))
self.assertEqual((115_687, 2_480_218), (kost["lines"]["read"]["total"], kost["lines"]["orchestration-read"]["total"]))
def test_order_12s_measurement_projects_between_10_and_13_million(self) -> None:
self.harvest()
self.assertGreater(Fraction(2_595_905) * 2_132 / 34 * Fraction(11, 10), 13_000_000) # the old rule said STOP
code, text = self.stop("--batches-done", "1", "--batches-total", "2")
self.assertEqual(0, code, text)
row = json.loads(self.kost.read_text(encoding="utf-8"))["stops"]["read,orchestration-read"]
# readers 115687 x 2132 / 34, the session before its first helper once, the batch 405894 x 2 / 1
expected = Fraction(115_687) * 2_132 / 34 + 2_074_324 + Fraction(405_894) * 2
self.assertAlmostEqual(float(expected), row["flat"], places=3)
self.assertTrue(10_000_000 <= row["flat"] <= 13_000_000)
self.assertEqual((2_074_324, 405_894, 1, 2), (row["before_first_helper"], row["after_first_helper"],
row["batches_done"], row["batches_total"]))
def test_batches_of_ten_from_the_main_session_would_have_stopped(self) -> None:
self.harvest()
self.assertEqual(1, self.stop("--batches-done", "1", "--batches-total", "53")[0])
def test_claims_with_an_orchestration_line_need_the_batches(self) -> None:
self.harvest()
self.assertEqual(2, self.stop()[0])
self.assertEqual(2, self.stop("--batches-done", "0", "--batches-total", "2")[0])
self.assertEqual(2, self.stop("--batches-done", "3", "--batches-total", "2")[0])
def test_helper_lines_alone_scale_per_claim(self) -> None:
self.harvest()
code, _ = run_main(["stop", "--lines", "read", "--cap", "8000000", "--unit", "claims", "--done", "34",
"--total", "2132", "--kost", str(self.kost)])
self.assertEqual(0, code) # 115687 x 2132 / 34 x 1.10 = 7979681
row = json.loads(self.kost.read_text(encoding="utf-8"))["stops"]["read"]
self.assertAlmostEqual(115_687 * 2_132 / 34, row["flat"], places=3)
def test_a_workflow_turn_starts_a_batch_too(self) -> None:
jsonl(self.proj / "s1.jsonl", [USER, msg("o1", 0, 500, 0, 0, "tool_use"), spawn("o2", 0, 700, 0, 0, "Workflow")])
_, kost, _ = self.harvest()
self.assertEqual((2, 500), (kost["orchestration"]["s1"]["first_helper_turn"],
kost["orchestration"]["s1"]["before_first_helper"]))
def test_a_session_with_no_helper_yet_is_all_before(self) -> None:
jsonl(self.proj / "s1.jsonl", [USER, msg("o1", 0, 500, 0, 0, "tool_use"), msg("o2", 0, 700, 0, 0)])
_, kost, _ = self.harvest()
self.assertEqual((None, 1_200), (kost["orchestration"]["s1"]["first_helper_turn"],
kost["orchestration"]["s1"]["before_first_helper"]))
class DiagnoseKindTest(Transcripts):
def setUp(self) -> None:
super().setUp()
jsonl(self.proj / "s3.jsonl", [USER, msg("d0", 1, 900, 90, 9)])
self.tasks = self.work / "tasks"
read = [msg("r", 1, 1, 1, 1, "tool_use", [{"type": "tool_use", "name": "Read",
"input": {"file_path": str(self.tasks / "d001.md")}}]),
msg("d1", 1, 2_000, 200, 20)]
self.agent("s3", "g1", "V1-del1d-diagnose-d001", read, agent_type="okf-wiki-diagnoser")
self.okter.write_text("s3 diagnose\n", encoding="utf-8")
def test_a_diagnosis_helper_lands_on_its_own_line(self) -> None:
code, kost, _ = self.harvest()
self.assertEqual(0, code)
self.assertEqual([], kost["unattributed"])
(run,) = kost["runs"]
self.assertEqual(("del1d", "diagnose", "d001"), (run["run"], run["line"], run["unit"]))
self.assertEqual(1 + 1 + 1 + 1 + 1 + 2_000 + 200 + 20, kost["lines"]["diagnose"]["total"])
self.assertEqual(1 + 900 + 90 + 9, kost["lines"]["orchestration-diagnose"]["total"])
code, text = run_main(["summarize", "--kost", str(self.work / "kost.json"), "--cap-group",
"diagnose=diagnose,orchestration-diagnose:5000000"])
self.assertEqual(0, code)
self.assertIn("diagnose: 3225 + margin 0.10 = 3548 of 5000000", text)
def test_isolation_with_line_diagnose(self) -> None:
out = self.work / "isolation.json"
args = ["isolation", "--transcripts", str(self.proj), "--run", "del1d", "--tasks", str(self.tasks), "--out", str(out)]
self.assertEqual(1, run_main(args)[0]) # the default line is read: no reader agents
self.assertEqual(0, run_main([*args, "--line", "diagnose"])[0])
self.assertEqual((1, 0), (json.loads(out.read_text(encoding="utf-8"))["agents"],
json.loads(out.read_text(encoding="utf-8"))["breaks"]))
self.agent("s3", "g2", "V1-del1d-diagnose-d002", [msg("x", 1, 1, 1, 1, "tool_use", [
{"type": "tool_use", "name": "Read", "input": {"file_path": str(self.tasks / "d001.md")}}])],
agent_type="okf-wiki-diagnoser")
self.assertEqual(1, run_main([*args, "--line", "diagnose"])[0]) # d002 read d001's task: a break
def test_an_older_kost_file_without_the_diagnose_lines_still_summarizes(self) -> None:
kost = self.work / "old.json"
doc = {"schema": cost.SCHEMA, "lines": {n: {"total": 10} for n in cost.LINES if "diagnose" not in n},
"runs": [], "margin": {"measured": None, "used": 0.10}, "caps": {}, "stops": {}}
kost.write_text(json.dumps(doc), encoding="utf-8")
code, text = run_main(["summarize", "--kost", str(kost)])
self.assertEqual(0, code)
self.assertIn("diagnose: 0 + margin 0.10 = 0", text)
class MergeKindTest(Transcripts):
"""Order 29 A5: the merge helper of step 18a on its own line, its main session on orchestration-merge."""
def setUp(self) -> None:
super().setUp()
jsonl(self.proj / "s4.jsonl", [USER, spawn("m0", 1, 900, 90, 9, "Agent")])
self.agent("s4", "h1", "V1-b5m-merge-k001", [msg("h1", 1, 2_000, 200, 20)], agent_type="okf-wiki-merger")
self.okter.write_text("s4 merge\n", encoding="utf-8")
def test_a_merge_helper_lands_on_the_merge_line(self) -> None:
code, kost, _ = self.harvest()
self.assertEqual(0, code)
self.assertEqual([], kost["unattributed"])
(run,) = kost["runs"]
self.assertEqual(("b5m", "merge", "k001"), (run["run"], run["line"], run["unit"]))
self.assertEqual(1 + 2_000 + 200 + 20, kost["lines"]["merge"]["total"])
self.assertEqual(1 + 900 + 90 + 9, kost["lines"]["orchestration-merge"]["total"])
def test_the_stop_line_of_session_b_runs_on_the_merge_lines(self) -> None:
self.harvest()
argv = ["stop", "--lines", "merge,orchestration-merge", "--unit", "passes", "--done", "4", "--total", "4",
"--kost", str(self.work / "kost.json")]
self.assertEqual(0, run_main([*argv, "--cap", "3544"])[0]) # (2221 + 1000) x 1.10 = 3543.1
self.assertEqual(1, run_main([*argv, "--cap", "3543"])[0])
def test_an_okter_line_of_kind_merge_is_read(self) -> None:
self.assertEqual([("s4", "merge")], cost.sessions(self.okter))
class RepairKindTest(Transcripts):
"""Order 34 B3: the repair helper (okf-wiki-repairer, described V1-<run>-repair-<task>) on its own line `repair`,
its main session on orchestration-repair, and the breach count against each task's part list."""
def setUp(self) -> None:
super().setUp()
self.tasks = self.work / "tasks"
self.tasks.mkdir()
self.parts = [self.tasks / "r001-1.md", self.tasks / "r001-2.md"]
self.answer = self.tasks / "r001.answers.json"
(self.tasks / "r001.parts.json").write_text(json.dumps(
{"schema": "okf-wiki/repair-parts/1", "task": "r001", "parts": [str(p) for p in self.parts],
"answer": str(self.answer)}), encoding="utf-8")
jsonl(self.proj / "s5.jsonl", [USER, spawn("o0", 1, 900, 90, 9, "Agent")])
self.okter.write_text("s5 repair\n", encoding="utf-8")
def call(self, mid: str, tool: str, path: Path | None = None) -> dict[str, Any]:
args = {"file_path": str(path)} if path is not None else {"command": "ls"}
return msg(mid, 1, 1, 1, 1, "tool_use", [{"type": "tool_use", "id": f"t-{mid}", "name": tool, "input": args}])
def helper(self, name: str, task: str, lines: list[dict[str, Any]]) -> None:
self.agent("s5", name, f"V1-b5r-repair-{task}", lines, agent_type="okf-wiki-repairer")
def isolation(self) -> tuple[int, dict[str, Any]]:
out = self.work / "isolation.json"
code, _ = run_main(["isolation", "--transcripts", str(self.proj), "--run", "b5r", "--tasks", str(self.tasks),
"--out", str(out), "--line", "repair"])
return code, json.loads(out.read_text(encoding="utf-8")) if out.is_file() else {}
def faithful(self) -> list[dict[str, Any]]:
return [self.call("a", "Read", self.parts[0]), self.call("b", "Read", self.parts[1]),
self.call("c", "Write", self.answer), msg("d", 1, 2_000, 200, 20)]
def test_a_repair_helper_lands_on_the_repair_line(self) -> None:
self.helper("h1", "r001", [msg("h1", 1, 2_000, 200, 20)])
code, kost, _ = self.harvest()
self.assertEqual(0, code)
self.assertEqual([], kost["unattributed"])
(run,) = kost["runs"]
self.assertEqual(("b5r", "repair", "r001"), (run["run"], run["line"], run["unit"]))
self.assertEqual(1 + 2_000 + 200 + 20, kost["lines"]["repair"]["total"])
self.assertEqual(1 + 900 + 90 + 9, kost["lines"]["orchestration-repair"]["total"])
code, text = run_main(["summarize", "--kost", str(self.work / "kost.json"), "--cap-group",
"repair=repair,orchestration-repair:5000000"])
self.assertEqual(0, code)
self.assertIn("repair: 3221 + margin 0.10 = 3544 of 5000000", text)
self.assertIn("orchestration-repair per turn:", text)
def test_an_okter_line_of_kind_repair_is_read(self) -> None:
self.assertEqual([("s5", "repair")], cost.sessions(self.okter))
def test_reading_the_parts_and_writing_the_answer_is_no_breach(self) -> None:
self.helper("h1", "r001", self.faithful())
code, doc = self.isolation()
self.assertEqual(0, code)
self.assertEqual(("repair", 1, 0), (doc["line"], doc["agents"], doc["breaks"]))
(agent,) = doc["per_agent"]
self.assertEqual(("r001", 2, 1, []), (agent["claim"], agent["reads"], agent["writes"], agent["breaks"]))
def test_a_read_or_a_write_outside_the_part_list_is_a_breach(self) -> None:
other = self.tasks / "r002-1.md"
self.helper("h1", "r001", [*self.faithful(), self.call("e", "Read", other),
self.call("f", "Write", self.tasks / "r002.answers.json")])
code, doc = self.isolation()
self.assertEqual(1, code)
self.assertEqual([f"Read {other}", f"Write {self.tasks / 'r002.answers.json'}"], doc["per_agent"][0]["breaks"])
def test_a_tool_outside_read_and_write_is_a_breach_of_its_own(self) -> None:
self.helper("h1", "r001", [*self.faithful(), self.call("e", "Bash"), self.call("f", "Edit", self.answer)])
code, doc = self.isolation()
self.assertEqual(1, code)
self.assertEqual(["Bash", f"Edit {self.answer}"], doc["per_agent"][0]["breaks"])
def test_an_answer_not_written_or_written_twice_is_a_breach(self) -> None:
"""Review finding 24 (order 39): the helper must write exactly the one answer path."""
reads = [self.call("a", "Read", self.parts[0]), self.call("b", "Read", self.parts[1])]
for name, lines, expected in (
("none", [*reads, msg("d", 1, 2_000, 200, 20)], ["answer not written"]),
("twice", [*reads, self.call("c", "Write", self.answer), self.call("e", "Write", self.answer)],
["answer written 2 times"])):
with self.subTest(name=name):
shutil.rmtree(self.proj / "s5", ignore_errors=True)
self.helper("h1", "r001", lines)
code, doc = self.isolation()
self.assertEqual(1, code)
self.assertEqual(expected, doc["per_agent"][0]["breaks"])
def test_a_part_not_read_is_a_breach(self) -> None:
"""Review finding 24 (order 39): the helper must read exactly the parts of its task."""
write = [self.call("c", "Write", self.answer)]
first, second = (p.resolve() for p in self.parts)
for name, lines, expected in (
("none", write, [f"part not read: {first}", f"part not read: {second}"]),
("one", [self.call("a", "Read", self.parts[0]), *write], [f"part not read: {second}"])):
with self.subTest(name=name):
shutil.rmtree(self.proj / "s5", ignore_errors=True)
self.helper("h1", "r001", lines)
code, doc = self.isolation()
self.assertEqual(1, code)
self.assertEqual(expected, doc["per_agent"][0]["breaks"])
def test_a_workflow_s_repair_helpers_are_counted(self) -> None:
self.agent("s5", "w1", "V1-b5r-repair-r001", self.faithful(), agent_type="okf-wiki-repairer",
workflow="wf_abc123")
code, doc = self.isolation()
self.assertEqual((0, 1), (code, doc["agents"]))
def test_a_helper_whose_task_has_no_part_list_cannot_run(self) -> None:
self.helper("h1", "r009", self.faithful())
code, _ = self.isolation()
self.assertEqual(2, code)
def test_the_read_line_still_counts_only_reads(self) -> None:
tasks = self.work / "tasks"
self.agent("s5", "r1", "V1-b5r-read-c001", [self.call("a", "Read", tasks / "c001.md"), self.call("b", "Bash")],
agent_type="okf-wiki-reader")
out = self.work / "isolation.json"
code, _ = run_main(["isolation", "--transcripts", str(self.proj), "--run", "b5r", "--tasks", str(tasks),
"--out", str(out)])
self.assertEqual(0, code)
class MarginTest(unittest.TestCase):
def test_the_floor_is_ten_percent(self) -> None:
self.assertEqual(0.10, cost.margin(None))
self.assertEqual(0.10, cost.margin(0.04))
self.assertEqual(0.12, cost.margin(0.12))
class SummarizeTest(unittest.TestCase):
def test_lines_and_cap_groups(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
kost = kost_file(Path(tmp) / "kost.json", {"build": 1_000, "link": 500, "calib": 70},
[{"line": "build", "pack": 1, "total": 1_000}])
code, text = run_main(["summarize", "--kost", str(kost), "--cap-group", "build=build,link:2000"])
self.assertEqual(0, code)
self.assertIn("build: 1000 + margin 0.10 = 1100", text)
self.assertIn("calib: 70 + margin 0.10 = 77 (reported, never capped)", text)
self.assertIn("build: 1500 + margin 0.10 = 1650 of 2000", text)
def test_a_group_over_its_cap_exits_1(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
kost = kost_file(Path(tmp) / "kost.json", {"build": 1_000})
self.assertEqual(1, run_main(["summarize", "--kost", str(kost), "--cap-group", "b=build:1099"])[0])
class IsolationTest(Transcripts):
def read(self, mid: str, path: Path) -> dict[str, Any]:
return msg(mid, 1, 1, 1, 1, "tool_use", [{"type": "tool_use", "name": "Read", "input": {"file_path": str(path)}}])
def test_a_read_outside_the_claims_task_file_is_a_break(self) -> None:
tasks = self.work / "tasks"
self.agent("s2", "r1", "V1-utsnitt-read-c001", [self.read("a", tasks / "c001.md")], agent_type="okf-wiki-reader")
self.agent("s2", "r2", "V1-utsnitt-read-c002", [self.read("b", tasks / "c002.md"), self.read("c", tasks / "c001.md")],
agent_type="okf-wiki-reader")
self.agent("s2", "r3", "V1-other-read-c003", [self.read("d", Path("/etc/hosts"))], agent_type="okf-wiki-reader")
out = self.work / "isolation.json"
code, _ = run_main(["isolation", "--transcripts", str(self.proj), "--run", "utsnitt", "--tasks", str(tasks),
"--out", str(out)])
self.assertEqual(1, code)
doc = json.loads(out.read_text(encoding="utf-8"))
self.assertEqual((2, 1), (doc["agents"], doc["breaks"]))
self.assertEqual([str(tasks / "c001.md")], [b for a in doc["per_agent"] for b in a["breaks"]])
def test_a_workflows_reader_agents_are_counted(self) -> None:
tasks = self.work / "tasks"
self.agent("s2", "r1", "V1-utsnitt-read-c001", [self.read("a", tasks / "c001.md")], agent_type="okf-wiki-reader")
self.agent("s2", "r4", "V1-utsnitt-read-c004", [self.read("e", tasks / "c004.md"), self.read("f", Path("/etc/hosts"))],
agent_type="okf-wiki-reader", workflow="wf_abc123")
out = self.work / "isolation.json"
code, _ = run_main(["isolation", "--transcripts", str(self.proj), "--run", "utsnitt", "--tasks", str(tasks),
"--out", str(out)])
self.assertEqual(1, code)
doc = json.loads(out.read_text(encoding="utf-8"))
self.assertEqual((2, 1), (doc["agents"], doc["breaks"]))
self.assertEqual(["/etc/hosts"], [b for a in doc["per_agent"] for b in a["breaks"]])
def test_no_reader_agents_is_a_fail(self) -> None:
out = self.work / "isolation.json"
code, _ = run_main(["isolation", "--transcripts", str(self.proj), "--run", "utsnitt", "--tasks",
str(self.work / "tasks"), "--out", str(out)])
self.assertEqual(1, code)
class ProjectionTest(unittest.TestCase):
def test_least_squares_on_cost_per_block_against_blocks_read(self) -> None:
p = cost.project([(600_000, 600, 0), (220_000, 200, 600)], total_blocks=3000)
self.assertAlmostEqual(1000, p["fit"]["a"])
self.assertAlmostEqual(1 / 6, p["fit"]["b"])
self.assertAlmostEqual(3_750_000, p["fit"]["value"])
self.assertAlmostEqual(3_075_000, p["flat"])
def test_one_point_has_no_growth_term(self) -> None:
p = cost.project([(600_000, 600, 0)], total_blocks=3000)
self.assertEqual(0, p["fit"]["b"])
self.assertAlmostEqual(3_000_000, p["fit"]["value"])
class AllocationTest(unittest.TestCase):
def test_tokens_per_source_by_block_share(self) -> None:
names = [f"s{i:02d}.md" for i in range(1, 20)]
blocks_doc = {"order": names, "blocks": [{"file": n, "start": k, "end": k} for n in names for k in (1, 2)]}
packs = [names[i:i + 5] for i in range(0, 19, 5)]
per = cost.allocate({1: 1000, 2: 500, 3: 500, 4: 800}, packs, blocks_doc)
self.assertEqual(200, per["s01.md"])
self.assertEqual(200, per["s16.md"])
class CommonTest(unittest.TestCase):
def test_write_new_refuses_to_overwrite(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
p = Path(tmp) / "sub" / "a.json"
common.write_new(p, "one")
self.assertEqual("one", p.read_text(encoding="utf-8"))
with self.assertRaises(common.RunError):
common.write_new(p, "two")
self.assertEqual("one", p.read_text(encoding="utf-8"))
def test_jaccard(self) -> None:
self.assertEqual(0.5, common.jaccard({"a", "b"}, {"b", "c", "a", "d"}))
self.assertEqual(0.0, common.jaccard(set(), set()))
class NoModelTest(unittest.TestCase):
def test_tools_import_no_network_or_model_client(self) -> None:
for name in ("cost.py", "transcript.py", "common.py"):
source = (ROOT / "okfwiki" / name).read_text(encoding="utf-8")
for banned in ("anthropic", "openai", "urllib", "http.client", "requests", "subprocess", "socket"):
self.assertNotIn(f"import {banned}", source, name)
if __name__ == "__main__":
unittest.main()