"""Tests for okfwiki/cost.py, okfwiki/transcript.py and okfwiki/common.py: cost lines, stop checks and isolation. Every transcript here is synthetic. The usage reading is ported from the trial (evals/p1/cost.py) with its tests. Run: python3 -m unittest discover -s tests -p 'test_cost.py' """ from __future__ import annotations import io import json import math import shutil import sys import tempfile import unittest from contextlib import redirect_stderr, redirect_stdout from fractions import Fraction from pathlib import Path from typing import Any ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT)) from okfwiki import common, cost # noqa: E402 def msg(mid: str, inp: int, cr: int, cw: int, out: int, stop: str | None = "end_turn", tools: list[dict[str, Any]] | None = None) -> dict[str, Any]: return {"type": "assistant", "message": {"id": mid, "stop_reason": stop, "content": tools or [], "usage": { "input_tokens": inp, "cache_read_input_tokens": cr, "cache_creation_input_tokens": cw, "output_tokens": out}}} def jsonl(path: Path, lines: list[dict[str, Any]]) -> None: path.parent.mkdir(parents=True, exist_ok=True) path.write_text("\n".join(json.dumps(x) for x in lines) + "\n", encoding="utf-8") USER = {"type": "user", "message": {"content": "task"}} class Transcripts(unittest.TestCase): """A transcripts directory laid out as Claude Code writes it: .jsonl and /subagents/agent-*.jsonl.""" def setUp(self) -> None: self._tmp = tempfile.TemporaryDirectory() self.root = Path(self._tmp.name) self.proj = self.root / "transcripts" self.work = self.root / "run" self.work.mkdir() self.okter = self.work / "okter.txt" def tearDown(self) -> None: self._tmp.cleanup() def agent(self, sid: str, name: str, description: str, lines: list[dict[str, Any]], agent_type: str = "okf-wiki-linker", workflow: str | None = None) -> None: sub = self.proj / sid / "subagents" if workflow is not None: # a Workflow's agents lie in /subagents/workflows// (measured 09.10, 2.1.295) sub = sub / "workflows" / workflow jsonl(sub / f"agent-{name}.jsonl", [USER, *lines]) (sub / f"agent-{name}.meta.json").write_text(json.dumps({"agentType": agent_type, "description": description})) def harvest(self, *extra: str) -> tuple[int, dict[str, Any], str]: out = self.work / "kost.json" code, text = run_main(["harvest", "--transcripts", str(self.proj), "--okter", str(self.okter), "--work", str(self.work), "--out", str(out), *extra]) return code, json.loads(out.read_text(encoding="utf-8")) if out.is_file() else {}, text def run_main(argv: list[str]) -> tuple[int, str]: buf = io.StringIO() with redirect_stdout(buf), redirect_stderr(io.StringIO()): code = cost.main(argv) return code, buf.getvalue() class HarvestTest(Transcripts): def setUp(self) -> None: super().setUp() jsonl(self.proj / "s1.jsonl", [USER, msg("m0", 5, 100, 10, 7)]) # a streamed message id appears twice with growing output: counted once, at its max self.agent("s1", "a1", "V1-utsnitt-pack-p1-1", [msg("x1", 3, 20_000, 2_000, 10, None), msg("x1", 3, 20_000, 2_000, 50), msg("x2", 1, 22_000, 500, 900)], agent_type="okf-wiki-compiler") self.agent("s1", "a2", "V1-utsnitt-link-p1-1", [msg("y1", 2, 19_000, 1_000, 5)]) self.agent("s2", "a3", "V1-utsnitt-read-c007", [msg("z1", 1, 1_000, 100, 3)], agent_type="okf-wiki-reader") self.agent("s2", "a4", "V1-utsnitt-calib-pair12", [msg("q1", 1, 500, 50, 2)], agent_type="okf-wiki-reader") jsonl(self.proj / "s2.jsonl", [USER, msg("r0", 4, 300, 30, 9)]) # a session not listed in okter.txt: its transcript is never opened self.agent("s9", "b1", "V1-utsnitt-pack-p2-1", [msg("q", 1, 1, 1, 1)]) self.okter.write_text("s1 build\ns2 read\n", encoding="utf-8") def test_four_categories_turns_startup_and_attribution(self) -> None: code, kost, _ = self.harvest() self.assertEqual(0, code) self.assertEqual([{"id": "s1", "kind": "build"}, {"id": "s2", "kind": "read"}], kost["sessions"]) runs = {r["agent"]: r for r in kost["runs"]} k = runs["agent-a1"] self.assertEqual(("utsnitt", "build", "p1-1", 1), (k["run"], k["line"], k["unit"], k["pack"])) self.assertEqual((4, 42_000, 2_500, 950), (k["input"], k["cache_read"], k["cache_write"], k["output"])) self.assertEqual((2, 3 + 20_000 + 2_000), (k["turns"], k["startup"])) self.assertEqual(("link", 1), (runs["agent-a2"]["line"], runs["agent-a2"]["pack"])) self.assertEqual(("read", None), (runs["agent-a3"]["line"], runs["agent-a3"]["pack"])) self.assertNotIn("agent-b1", runs) self.assertEqual([], kost["unattributed"]) def test_per_line_totals_by_hand(self) -> None: _, kost, _ = self.harvest() lines = kost["lines"] self.assertEqual(4 + 42_000 + 2_500 + 950, lines["build"]["total"]) self.assertEqual(2 + 19_000 + 1_000 + 5, lines["link"]["total"]) self.assertEqual(1 + 1_000 + 100 + 3, lines["read"]["total"]) self.assertEqual(1 + 500 + 50 + 2, lines["calib"]["total"]) # the orchestrating sessions land on their own lines by the kind okter.txt gives them self.assertEqual(5 + 100 + 10 + 7, lines["orchestration-build"]["total"]) self.assertEqual(4 + 300 + 30 + 9, lines["orchestration-read"]["total"]) self.assertEqual(list(cost.LINES), list(lines)) def test_input_per_orchestration_turn_is_kept_with_its_turn_number(self) -> None: # Plankontroll 05.10, tillegg 3: does the cost per orchestrating turn grow with the session's length? jsonl(self.proj / "s1.jsonl", [USER, msg("t1", 2, 70_000, 6_000, 40), {"type": "user", "message": {"content": "go"}}, msg("t2", 2, 76_000, 3_000, 60), msg("t3", 2, 79_000, 9_000, 80)]) _, kost, _ = self.harvest() turns = kost["orchestration"]["s1"]["per_turn"] self.assertEqual([{"turn": 1, "input": 76_002}, {"turn": 2, "input": 79_002}, {"turn": 3, "input": 88_002}], turns) code, text = run_main(["summarize", "--kost", str(self.work / "kost.json")]) self.assertEqual(0, code) self.assertIn("orchestration-build input per turn: 69002 + 6000 x turn", text) def test_a_workflows_agents_are_harvested_like_the_sessions_own(self) -> None: self.agent("s1", "w1", "V1-utsnitt-pack-p3-1", [msg("w1", 2, 10_000, 1_000, 30)], agent_type="okf-wiki-compiler", workflow="wf_abc123") self.agent("s2", "w2", "V1-utsnitt-read-c010", [msg("w2", 1, 2_000, 200, 4)], agent_type="okf-wiki-reader", workflow="wf_def456") self.agent("s9", "w9", "V1-utsnitt-pack-p4-1", [msg("w9", 1, 1, 1, 1)], workflow="wf_zzz") code, kost, _ = self.harvest() self.assertEqual(0, code) runs = {r["agent"]: r for r in kost["runs"]} self.assertEqual(("build", 3, "s1"), (runs["agent-w1"]["line"], runs["agent-w1"]["pack"], runs["agent-w1"]["session"])) self.assertEqual(("read", "s2"), (runs["agent-w2"]["line"], runs["agent-w2"]["session"])) self.assertNotIn("agent-w9", runs) # its session is not in okter.txt self.assertEqual(4 + 42_000 + 2_500 + 950 + 2 + 10_000 + 1_000 + 30, kost["lines"]["build"]["total"]) def test_an_unattributed_workflow_agent_exits_1(self) -> None: self.agent("s1", "w8", "explore the sources", [msg("h8", 1, 1, 1, 1)], agent_type="Explore", workflow="wf_abc123") code, kost, _ = self.harvest() self.assertEqual(1, code) self.assertEqual(["explore the sources"], [u["description"] for u in kost["unattributed"]]) def test_an_unattributed_run_exits_1(self) -> None: self.agent("s1", "a9", "look around the repository", [msg("h1", 1, 1, 1, 1)], agent_type="Explore") code, kost, _ = self.harvest() self.assertEqual(1, code) self.assertEqual(["look around the repository"], [u["description"] for u in kost["unattributed"]]) def test_a_session_okter_lists_that_the_transcripts_lack_cannot_run(self) -> None: # Order 27 A3: a wrong transcripts folder gave 0 for a whole variant without a word self.okter.write_text("s1 build\ns3 build\n", encoding="utf-8") code, _, _ = self.harvest() self.assertEqual(2, code) self.assertFalse((self.work / "kost.json").exists()) def test_an_unknown_session_kind_cannot_run(self) -> None: self.okter.write_text("s1 build\ns2 other\n", encoding="utf-8") self.assertEqual(2, self.harvest()[0]) def test_a_pack_log_gap_cannot_run(self) -> None: packs = self.work / "packs.json" packs.write_text(json.dumps({"packs": [{"pack": 1, "blocks": 10}, {"pack": 2, "blocks": 10}]})) self.assertEqual(2, self.harvest("--packs", str(packs))[0]) # pack 2 is logged but has no build run packs.write_text(json.dumps({"packs": [{"pack": 1, "blocks": 10}]})) self.assertEqual(0, self.harvest("--packs", str(packs))[0]) def test_a_link_run_does_not_fill_a_build_gap(self) -> None: self.agent("s1", "a7", "V1-utsnitt-link-p2-1", [msg("l2", 1, 1, 1, 1)]) packs = self.work / "packs.json" packs.write_text(json.dumps({"packs": [{"pack": 1, "blocks": 10}, {"pack": 2, "blocks": 10}]})) self.assertEqual(2, self.harvest("--packs", str(packs))[0]) def test_output_never_recorded_is_derived_from_the_next_messages_input_growth(self) -> None: def result(text: str) -> dict[str, Any]: return {"type": "user", "message": {"content": [{"type": "tool_result", "content": text}]}} self.agent("s1", "a5", "V1-utsnitt-pack-p3-1", [msg("w1", 2, 0, 10_000, 300, "tool_use"), result("x" * 400), msg("w2", 2, 10_000, 50_000, 2, None), msg("w2", 2, 10_000, 50_000, 2, None), result("y" * 40), msg("w3", 2, 60_000, 5_010, 20)]) _, kost, _ = self.harvest() w = {r["agent"]: r for r in kost["runs"]}["agent-a5"] self.assertEqual(300 + 2 + 20, w["output_recorded"]) self.assertEqual(300 + (5_010 - 40 * 2 // 5) + 20, w["output"]) self.assertEqual((1, 0), (w["output_derived_messages"], w["output_unrecorded"])) def test_growth_from_injected_attachments_is_not_counted_as_output(self) -> None: self.agent("s1", "a6", "V1-utsnitt-pack-p4-1", [msg("v1", 2, 0, 18_602, 16, None), {"type": "attachment", "rendered": "z" * 5_133}, {"type": "user", "message": {"content": [ {"type": "tool_result", "content": "w" * 151}]}}, msg("v2", 2, 18_602, 1_894, 24, None)]) _, kost, _ = self.harvest() v = {r["agent"]: r for r in kost["runs"]}["agent-a6"] self.assertEqual((16 + 24, 0), (v["output"], v["output_derived_messages"])) def test_harvest_writes_a_margin_and_keeps_recorded_caps(self) -> None: _, kost, _ = self.harvest() self.assertEqual({"measured": None, "used": 0.10}, kost["margin"]) kost["caps"] = {"build": {"lines": ["build"], "cap": 9}} (self.work / "kost.json").write_text(json.dumps(kost)) _, again, _ = self.harvest("--margin", "0.12") self.assertEqual({"build": {"lines": ["build"], "cap": 9}}, again["caps"]) self.assertEqual({"measured": 0.12, "used": 0.12}, again["margin"]) def kost_file(path: Path, lines: dict[str, int], runs: list[dict[str, Any]] | None = None, margin: float = 0.10) -> Path: doc = {"schema": cost.SCHEMA, "lines": {name: {"total": lines.get(name, 0)} for name in cost.LINES}, "runs": runs or [], "margin": {"measured": None, "used": margin}, "caps": {}, "stops": {}} path.write_text(json.dumps(doc), encoding="utf-8") return path class StopTest(unittest.TestCase): def setUp(self) -> None: self._tmp = tempfile.TemporaryDirectory() self.kost = Path(self._tmp.name) / "kost.json" def tearDown(self) -> None: self._tmp.cleanup() def stop(self, lines: str, cap: int, unit: str, done: int, total: int, *extra: str) -> int: return run_main(["stop", "--lines", lines, "--cap", str(cap), "--unit", unit, "--done", str(done), "--total", str(total), "--kost", str(self.kost), *extra])[0] def test_at_the_cap_continues_one_over_stops(self) -> None: kost_file(self.kost, {"link": 1_000}) self.assertEqual(0, self.stop("link", 1_100, "passes", 1, 1)) # 1000 x 1.10 = 1100 self.assertEqual(1, self.stop("link", 1_099, "passes", 1, 1)) def test_the_build_cap_sums_build_link_and_build_orchestration(self) -> None: kost_file(self.kost, {"build": 400, "link": 300, "orchestration-build": 300, "orchestration-read": 10_000, "read": 10_000}) self.assertEqual(0, self.stop("build,link,orchestration-build", 2_200, "passes", 1, 2)) self.assertEqual(1, self.stop("build,link,orchestration-build", 2_199, "passes", 1, 2)) def test_blocks_stop_on_the_fit_when_the_flat_projection_is_under_the_cap(self) -> None: runs = [{"line": "build", "pack": n, "total": t} for n, t in ((1, 100_000), (2, 300_000), (3, 500_000), (4, 700_000))] kost_file(self.kost, {"build": 1_600_000}, runs) packs = Path(self._tmp.name) / "packs.json" packs.write_text(json.dumps({"packs": [{"pack": n, "blocks": 100} for n in (1, 2, 3, 4)]})) # flat: 1600000 / 400 x 1000 = 4.0 M; fit: a = 1000, b = 20, 1000 x 1000 + 20 x 1000^2 / 2 = 11 M self.assertEqual(1, self.stop("build", 5_000_000, "blocks", 400, 1_000, "--blocks", str(packs))) self.assertEqual(0, self.stop("build", 12_100_000, "blocks", 400, 1_000, "--blocks", str(packs))) row = json.loads(self.kost.read_text(encoding="utf-8"))["stops"]["build"] self.assertAlmostEqual(4_000_000, row["flat"]) self.assertAlmostEqual(11_000_000, row["fit"]) def test_the_growth_curve_counts_only_from_the_fourth_pack(self) -> None: # Wave 3, order 11, after the link pass of pack 2 (logg/p2.txt): pack 1 cost 756203 over 871 blocks, pack 2 # 534095 over 466, measured 3397056 after 1337 of 4509 blocks: "flat 11456489, fit 18846464 ... = 20731111 of # 16000000: STOP". The operator 11.10.2026: the fitted curve counts from pack 4; before that the flat one alone. runs = [{"line": "build", "pack": 1, "total": 756_203}, {"line": "build", "pack": 2, "total": 534_095}] kost_file(self.kost, {"build": 1_290_298, "orchestration-build": 2_106_758}, runs) packs = Path(self._tmp.name) / "packs.json" packs.write_text(json.dumps({"packs": [{"pack": 1, "blocks": 871}, {"pack": 2, "blocks": 466}]})) scale = 3_397_056 / 1_290_298 old = cost.project([(756_203 * scale, 871, 0), (534_095 * scale, 466, 871)], 4_509)["fit"]["value"] self.assertGreater(old * 1.10, 16_000_000) # the old rule: the fit said STOP self.assertEqual(0, self.stop("build,link,orchestration-build", 16_000_000, "blocks", 1_337, 4_509, "--blocks", str(packs))) row = json.loads(self.kost.read_text(encoding="utf-8"))["stops"]["build,link,orchestration-build"] self.assertIsNone(row["fit"]) self.assertEqual(math.ceil(Fraction(3_397_056) * 4_509 / 1_337 * Fraction(11, 10)), row["projected_with_margin"]) self.assertEqual(11_456_489, round(row["flat"])) # a third pack still leaves the curve out; a fourth brings it in as before runs.append({"line": "build", "pack": 3, "total": 600_000}) kost_file(self.kost, {"build": 1_890_298, "orchestration-build": 2_106_758}, runs) packs.write_text(json.dumps({"packs": [{"pack": 1, "blocks": 871}, {"pack": 2, "blocks": 466}, {"pack": 3, "blocks": 400}]})) self.stop("build,link,orchestration-build", 16_000_000, "blocks", 1_737, 4_509, "--blocks", str(packs)) self.assertIsNone(json.loads(self.kost.read_text(encoding="utf-8"))["stops"]["build,link,orchestration-build"]["fit"]) runs.append({"line": "build", "pack": 4, "total": 900_000}) kost_file(self.kost, {"build": 2_790_298, "orchestration-build": 2_106_758}, runs) packs.write_text(json.dumps({"packs": [{"pack": 1, "blocks": 871}, {"pack": 2, "blocks": 466}, {"pack": 3, "blocks": 400}, {"pack": 4, "blocks": 500}]})) self.stop("build,link,orchestration-build", 16_000_000, "blocks", 2_237, 4_509, "--blocks", str(packs)) self.assertIsNotNone(json.loads(self.kost.read_text(encoding="utf-8"))["stops"]["build,link,orchestration-build"]["fit"]) def test_with_next_the_stop_is_the_measured_cost_and_the_next_pack(self) -> None: # The operator 12.10.2026: a cost line stops on measured cost plus a margin; the projection is reported only # (order 22 session A3: plus the next pack's expected cost, so the cap is never passed). Measured 1000 over 10 # blocks, next pack 5 blocks: (1000 + 1000 x 5 / 10) x 1.1 = 1650; the flat projection 1000 x 100 / 10 x 1.1 # = 11000 would have stopped. (Order 27 A3: the 10 blocks are pack 1's, whose compiler run is in the ledger.) kost_file(self.kost, {"build": 1_000}, [{"line": "build", "pack": 1, "total": 1_000}]) packs = Path(self._tmp.name) / "packs.json" packs.write_text(json.dumps({"packs": [{"pack": 1, "blocks": 10}]})) argv = ("build", 1_650, "blocks", 10, 100, "--blocks", str(packs), "--next", "5") self.assertEqual(0, self.stop_or_exit(*argv)) row = json.loads(self.kost.read_text(encoding="utf-8"))["stops"]["build"] self.assertEqual(("measured", 5, 1_650, 11_000, False), (row.get("rule"), row.get("next"), row.get("measured_next_with_margin"), row["projected_with_margin"], row["stop"])) self.assertEqual(1, self.stop_or_exit("build", 1_649, *argv[2:])) self.assertEqual(2, self.stop_or_exit("build", 10**9, "passes", 10, 100, "--next", "5")) # blocks only self.assertEqual(2, self.stop_or_exit("build", 10**9, *argv[2:-1], "-1")) def stop_or_exit(self, lines: str, cap: int, unit: str, done: int, total: int, *extra: str) -> int: try: return self.stop(lines, cap, unit, done, total, *extra) except SystemExit as exc: # an option the parser does not know return int(exc.code or 0) def test_calib_is_reported_never_capped(self) -> None: kost_file(self.kost, {"calib": 5}) self.assertEqual(2, self.stop("calib", 100, "claims", 1, 1)) def test_zero_done_and_bad_numbers_cannot_run(self) -> None: kost_file(self.kost, {"link": 1}) self.assertEqual(2, self.stop("link", 100, "passes", 0, 5)) self.assertEqual(2, self.stop("link", 100, "passes", 3, 2)) self.assertEqual(2, self.stop("link", 100, "blocks", 1, 2)) # blocks needs --blocks def test_the_stop_records_its_cap_group_for_summarize(self) -> None: kost_file(self.kost, {"build": 10, "link": 10, "orchestration-build": 10}) self.assertEqual(0, self.stop("build,link,orchestration-build", 1_000, "passes", 1, 1, "--name", "bygging")) code, text = run_main(["summarize", "--kost", str(self.kost)]) self.assertEqual(0, code) self.assertIn("bygging: 30 + margin 0.10 = 33 of 1000", text) class ContinuedBuildStopTest(unittest.TestCase): """Order 27 A3: a build that continues a collection built in another ledger (packs 1-8 of round 2 cost nothing in the variant's ledger). The next pack's share is the measured cost over the blocks of the packs with a compiler run in this ledger, times the next pack's blocks; before the ledger's first, round 2's price per block. Wave 4's use, every pack in one ledger, gives the numbers it gave.""" def setUp(self) -> None: self._tmp = tempfile.TemporaryDirectory() self.kost = Path(self._tmp.name) / "kost.json" self.packs = Path(self._tmp.name) / "packs.json" def tearDown(self) -> None: self._tmp.cleanup() def row(self, cap: int, done: int, total: int, following: int) -> tuple[int, dict[str, Any]]: code, _ = run_main(["stop", "--lines", "build,link,orchestration-build", "--cap", str(cap), "--unit", "blocks", "--done", str(done), "--total", str(total), "--kost", str(self.kost), "--blocks", str(self.packs), "--next", str(following)]) doc = json.loads(self.kost.read_text(encoding="utf-8")) return code, dict(doc["stops"].get("build,link,orchestration-build", {})) # none when it could not run def test_the_share_comes_from_this_ledgers_packs_not_from_the_packs_built_before(self) -> None: # packs 1 and 2 built before (4000 blocks, no run here), pack 3 here: 1000 by its compiler, 500 orchestration. # Rate 1500 / 500 = 3 per block; next pack 100 blocks: (1500 + 300) x 1.1 = 1980 (over the done blocks it # would be 1500 / 4500 x 100 = 33, and 1687). self.packs.write_text(json.dumps({"packs": [{"pack": 1, "blocks": 3000, "accepted": 1}, {"pack": 2, "blocks": 1000, "accepted": 1}, {"pack": 3, "blocks": 500, "accepted": 1}]})) kost_file(self.kost, {"build": 1_000, "orchestration-build": 500}, [{"line": "build", "pack": 3, "total": 1_000}]) code, row = self.row(1_980, 4_500, 4_600, 100) self.assertEqual((0, 1_980), (code, row["measured_next_with_margin"])) self.assertEqual((500, 3), (row["rate_blocks"], row["rate"])) self.assertAlmostEqual(1_800, row["flat"]) # 1500 + 3 x (4600 - 4500), reported only _, text = run_main(["stop", "--lines", "build,link,orchestration-build", "--cap", "1980", "--unit", "blocks", "--done", "4500", "--total", "4600", "--kost", str(self.kost), "--blocks", str(self.packs), "--next", "100"]) self.assertIn("measured 1500 + next 100 blocks x 1500/500 (the blocks with a compiler run here)", text) self.assertEqual(1, self.row(1_979, 4_500, 4_600, 100)[0]) def test_before_the_ledger_s_first_compiler_run_the_share_is_round_2_s_price_per_block(self) -> None: self.assertEqual(cost.ROUND2_PER_BLOCK, 3_259) # round 2: 14 741 606 over 4 523 blocks (order 25 session C) self.packs.write_text(json.dumps({"packs": [{"pack": 1, "blocks": 4_000, "accepted": 1}]})) kost_file(self.kost, {"orchestration-build": 500}) code, row = self.row(10**9, 4_000, 4_100, 100) self.assertEqual(0, code) self.assertEqual((0, 3_259), (row["rate_blocks"], row["rate"])) self.assertEqual(math.ceil((500 + 3_259 * 100) * Fraction(11, 10)), row["measured_next_with_margin"]) def test_wave_4_s_use_gives_the_numbers_it_gave(self) -> None: # every pack in one ledger, as round 2 ran: the rate is the measured cost over the done blocks, as before self.packs.write_text(json.dumps({"packs": [{"pack": 1, "blocks": 885, "accepted": 1}, {"pack": 2, "blocks": 466, "accepted": 1}]})) kost_file(self.kost, {"build": 2_580_808, "link": 249_822, "orchestration-build": 1_538_241}, [{"line": "build", "pack": 1, "total": 1_682_681}, {"line": "build", "pack": 2, "total": 898_127}, {"line": "link", "pack": 1, "total": 249_822}]) measured = 2_580_808 + 249_822 + 1_538_241 code, row = self.row(22_000_000, 1_351, 4_523, 235) self.assertEqual(0, code) self.assertEqual(math.ceil((measured + Fraction(measured) * 235 / 1_351) * Fraction(11, 10)), row["measured_next_with_margin"]) self.assertAlmostEqual(float(Fraction(measured) * 4_523 / 1_351), row["flat"]) def test_a_compiler_run_for_a_pack_the_log_lacks_cannot_run(self) -> None: self.packs.write_text(json.dumps({"packs": [{"pack": 1, "blocks": 10, "accepted": 1}]})) kost_file(self.kost, {"build": 100}, [{"line": "build", "pack": 2, "total": 100}]) self.assertEqual(2, self.row(10**9, 10, 20, 5)[0]) class CharsStopTest(unittest.TestCase): """Motor B17: the next pack's share by the characters of its blocks, not their number. A block of collection 1 holds 229 characters on average, a PDF block of a real mixed folder 2 310 and a spreadsheet block 2 150, while the compiler's cost per character stays near (6.9 on collection 1's round 2, 5.8 and 8.5 on two folder trials).""" def setUp(self) -> None: self._tmp = tempfile.TemporaryDirectory() self.kost = Path(self._tmp.name) / "kost.json" self.packs = Path(self._tmp.name) / "packs.json" def tearDown(self) -> None: self._tmp.cleanup() def chars_row(self, cap: int, following: int, chars: int, *extra: str) -> tuple[int, dict[str, Any], str]: code, text = run_main(["stop", "--lines", "build,link,orchestration-build", "--cap", str(cap), "--unit", "blocks", "--done", "10", "--total", "100", "--kost", str(self.kost), "--blocks", str(self.packs), "--next", str(following), "--next-chars", str(chars), *extra]) doc = json.loads(self.kost.read_text(encoding="utf-8")) return code, dict(doc["stops"].get("build,link,orchestration-build", {})), text def test_the_share_is_the_measured_cost_per_character_times_the_next_pack_s_characters(self) -> None: # 1000 over pack 1's 2000 characters: 0.5 per character; the next pack 5 blocks of 4000 characters: # (1000 + 2000) x 1.1 = 3300, where by blocks it would be (1000 + 500) x 1.1 = 1650. self.packs.write_text(json.dumps({"packs": [{"pack": 1, "blocks": 10, "block_chars": 2_000, "accepted": 1}]})) kost_file(self.kost, {"build": 1_000}, [{"line": "build", "pack": 1, "total": 1_000}]) code, row, text = self.chars_row(3_300, 5, 4_000) self.assertEqual((0, "measured-chars", 4_000, 3_300, 2_000), (code, row["rule"], row["next_chars"], row["measured_next_with_margin"], row["rate_chars_over"])) self.assertIn("measured 1000 + next 4000 characters x 1000/2000 (the characters with a compiler run here)", text) self.assertEqual(1, self.chars_row(3_299, 5, 4_000)[0]) def test_before_the_ledger_s_first_compiler_run_the_share_is_round_2_s_price_per_character(self) -> None: self.assertEqual(cost.ROUND2_PER_CHAR, Fraction(14_741_606, 1_111_796)) # round 2's blocks, 1 111 796 chars self.packs.write_text(json.dumps({"packs": [{"pack": 1, "blocks": 10, "block_chars": 2_000, "accepted": 1}]})) kost_file(self.kost, {"orchestration-build": 500}) code, row, _ = self.chars_row(10**9, 5, 10_000) self.assertEqual((0, 0), (code, row["rate_chars_over"])) self.assertEqual(math.ceil((500 + cost.ROUND2_PER_CHAR * 10_000) * Fraction(11, 10)), row["measured_next_with_margin"]) def test_a_built_pack_without_characters_keeps_the_rule_by_blocks(self) -> None: # a pack log written before blocks had characters (collection 1's builds) is priced as before self.packs.write_text(json.dumps({"packs": [{"pack": 1, "blocks": 10, "accepted": 1}]})) kost_file(self.kost, {"build": 1_000}, [{"line": "build", "pack": 1, "total": 1_000}]) code, row, _ = self.chars_row(1_650, 5, 4_000) self.assertEqual((0, "measured", 1_650), (code, row["rule"], row["measured_next_with_margin"])) def test_next_chars_needs_next(self) -> None: self.packs.write_text(json.dumps({"packs": [{"pack": 1, "blocks": 10, "block_chars": 2_000}]})) kost_file(self.kost, {"build": 1_000}, [{"line": "build", "pack": 1, "total": 1_000}]) code, _ = run_main(["stop", "--lines", "build", "--cap", "10", "--unit", "blocks", "--done", "10", "--total", "100", "--kost", str(self.kost), "--blocks", str(self.packs), "--next-chars", "5"]) self.assertEqual(2, code) def spawn(mid: str, inp: int, cr: int, cw: int, out: int, tool: str = "Agent") -> dict[str, Any]: """An orchestrating turn that starts helpers (Agent) or a workflow (Workflow).""" return msg(mid, inp, cr, cw, out, "tool_use", [{"type": "tool_use", "name": tool, "input": {}}]) # Wave 3, order 12, the price measurement (pris/kost-k010.json and the session's turns): ten readers, 34 of 2132 claims, # 115687 tokens; the main session 2480218 over 24 turns, of which turns 1-21 (2074324) came before the turn that # started the ten readers, and turns 22-24 (405894) were that batch. The rest ran as one workflow with one phase, so # the reading had two batches; it ended at 10492384 measured. READERS_K001_K010 = (11_183, 11_208, 11_307, 11_436, 11_215, 11_510, 12_028, 12_164, 11_335, 12_301) class ClaimsProjectionTest(Transcripts): def setUp(self) -> None: super().setUp() jsonl(self.proj / "s1.jsonl", [USER, msg("o1", 0, 2_000_000, 74_324, 0, "tool_use"), spawn("o2", 0, 120_000, 9_000, 694), msg("o3", 0, 130_000, 4_000, 565, "tool_use"), msg("o4", 0, 140_000, 1_000, 635, "tool_use")]) for i, total in enumerate(READERS_K001_K010, 1): self.agent("s1", f"r{i}", f"V1-del1l-read-k{i:03d}", [msg(f"k{i}", 0, total - 100, 0, 100)], agent_type="okf-wiki-reader") self.okter.write_text("s1 read\n", encoding="utf-8") self.kost = self.work / "kost.json" def stop(self, *extra: str) -> tuple[int, str]: return run_main(["stop", "--lines", "read,orchestration-read", "--cap", "13000000", "--unit", "claims", "--done", "34", "--total", "2132", "--kost", str(self.kost), *extra]) def test_the_harvest_keeps_the_sessions_part_before_its_first_helper(self) -> None: _, kost, _ = self.harvest() o = kost["orchestration"]["s1"] self.assertEqual((2, 2_074_324), (o["first_helper_turn"], o["before_first_helper"])) self.assertEqual((115_687, 2_480_218), (kost["lines"]["read"]["total"], kost["lines"]["orchestration-read"]["total"])) def test_order_12s_measurement_projects_between_10_and_13_million(self) -> None: self.harvest() self.assertGreater(Fraction(2_595_905) * 2_132 / 34 * Fraction(11, 10), 13_000_000) # the old rule said STOP code, text = self.stop("--batches-done", "1", "--batches-total", "2") self.assertEqual(0, code, text) row = json.loads(self.kost.read_text(encoding="utf-8"))["stops"]["read,orchestration-read"] # readers 115687 x 2132 / 34, the session before its first helper once, the batch 405894 x 2 / 1 expected = Fraction(115_687) * 2_132 / 34 + 2_074_324 + Fraction(405_894) * 2 self.assertAlmostEqual(float(expected), row["flat"], places=3) self.assertTrue(10_000_000 <= row["flat"] <= 13_000_000) self.assertEqual((2_074_324, 405_894, 1, 2), (row["before_first_helper"], row["after_first_helper"], row["batches_done"], row["batches_total"])) def test_batches_of_ten_from_the_main_session_would_have_stopped(self) -> None: self.harvest() self.assertEqual(1, self.stop("--batches-done", "1", "--batches-total", "53")[0]) def test_claims_with_an_orchestration_line_need_the_batches(self) -> None: self.harvest() self.assertEqual(2, self.stop()[0]) self.assertEqual(2, self.stop("--batches-done", "0", "--batches-total", "2")[0]) self.assertEqual(2, self.stop("--batches-done", "3", "--batches-total", "2")[0]) def test_helper_lines_alone_scale_per_claim(self) -> None: self.harvest() code, _ = run_main(["stop", "--lines", "read", "--cap", "8000000", "--unit", "claims", "--done", "34", "--total", "2132", "--kost", str(self.kost)]) self.assertEqual(0, code) # 115687 x 2132 / 34 x 1.10 = 7979681 row = json.loads(self.kost.read_text(encoding="utf-8"))["stops"]["read"] self.assertAlmostEqual(115_687 * 2_132 / 34, row["flat"], places=3) def test_a_workflow_turn_starts_a_batch_too(self) -> None: jsonl(self.proj / "s1.jsonl", [USER, msg("o1", 0, 500, 0, 0, "tool_use"), spawn("o2", 0, 700, 0, 0, "Workflow")]) _, kost, _ = self.harvest() self.assertEqual((2, 500), (kost["orchestration"]["s1"]["first_helper_turn"], kost["orchestration"]["s1"]["before_first_helper"])) def test_a_session_with_no_helper_yet_is_all_before(self) -> None: jsonl(self.proj / "s1.jsonl", [USER, msg("o1", 0, 500, 0, 0, "tool_use"), msg("o2", 0, 700, 0, 0)]) _, kost, _ = self.harvest() self.assertEqual((None, 1_200), (kost["orchestration"]["s1"]["first_helper_turn"], kost["orchestration"]["s1"]["before_first_helper"])) class DiagnoseKindTest(Transcripts): def setUp(self) -> None: super().setUp() jsonl(self.proj / "s3.jsonl", [USER, msg("d0", 1, 900, 90, 9)]) self.tasks = self.work / "tasks" read = [msg("r", 1, 1, 1, 1, "tool_use", [{"type": "tool_use", "name": "Read", "input": {"file_path": str(self.tasks / "d001.md")}}]), msg("d1", 1, 2_000, 200, 20)] self.agent("s3", "g1", "V1-del1d-diagnose-d001", read, agent_type="okf-wiki-diagnoser") self.okter.write_text("s3 diagnose\n", encoding="utf-8") def test_a_diagnosis_helper_lands_on_its_own_line(self) -> None: code, kost, _ = self.harvest() self.assertEqual(0, code) self.assertEqual([], kost["unattributed"]) (run,) = kost["runs"] self.assertEqual(("del1d", "diagnose", "d001"), (run["run"], run["line"], run["unit"])) self.assertEqual(1 + 1 + 1 + 1 + 1 + 2_000 + 200 + 20, kost["lines"]["diagnose"]["total"]) self.assertEqual(1 + 900 + 90 + 9, kost["lines"]["orchestration-diagnose"]["total"]) code, text = run_main(["summarize", "--kost", str(self.work / "kost.json"), "--cap-group", "diagnose=diagnose,orchestration-diagnose:5000000"]) self.assertEqual(0, code) self.assertIn("diagnose: 3225 + margin 0.10 = 3548 of 5000000", text) def test_isolation_with_line_diagnose(self) -> None: out = self.work / "isolation.json" args = ["isolation", "--transcripts", str(self.proj), "--run", "del1d", "--tasks", str(self.tasks), "--out", str(out)] self.assertEqual(1, run_main(args)[0]) # the default line is read: no reader agents self.assertEqual(0, run_main([*args, "--line", "diagnose"])[0]) self.assertEqual((1, 0), (json.loads(out.read_text(encoding="utf-8"))["agents"], json.loads(out.read_text(encoding="utf-8"))["breaks"])) self.agent("s3", "g2", "V1-del1d-diagnose-d002", [msg("x", 1, 1, 1, 1, "tool_use", [ {"type": "tool_use", "name": "Read", "input": {"file_path": str(self.tasks / "d001.md")}}])], agent_type="okf-wiki-diagnoser") self.assertEqual(1, run_main([*args, "--line", "diagnose"])[0]) # d002 read d001's task: a break def test_an_older_kost_file_without_the_diagnose_lines_still_summarizes(self) -> None: kost = self.work / "old.json" doc = {"schema": cost.SCHEMA, "lines": {n: {"total": 10} for n in cost.LINES if "diagnose" not in n}, "runs": [], "margin": {"measured": None, "used": 0.10}, "caps": {}, "stops": {}} kost.write_text(json.dumps(doc), encoding="utf-8") code, text = run_main(["summarize", "--kost", str(kost)]) self.assertEqual(0, code) self.assertIn("diagnose: 0 + margin 0.10 = 0", text) class MergeKindTest(Transcripts): """Order 29 A5: the merge helper of step 18a on its own line, its main session on orchestration-merge.""" def setUp(self) -> None: super().setUp() jsonl(self.proj / "s4.jsonl", [USER, spawn("m0", 1, 900, 90, 9, "Agent")]) self.agent("s4", "h1", "V1-b5m-merge-k001", [msg("h1", 1, 2_000, 200, 20)], agent_type="okf-wiki-merger") self.okter.write_text("s4 merge\n", encoding="utf-8") def test_a_merge_helper_lands_on_the_merge_line(self) -> None: code, kost, _ = self.harvest() self.assertEqual(0, code) self.assertEqual([], kost["unattributed"]) (run,) = kost["runs"] self.assertEqual(("b5m", "merge", "k001"), (run["run"], run["line"], run["unit"])) self.assertEqual(1 + 2_000 + 200 + 20, kost["lines"]["merge"]["total"]) self.assertEqual(1 + 900 + 90 + 9, kost["lines"]["orchestration-merge"]["total"]) def test_the_stop_line_of_session_b_runs_on_the_merge_lines(self) -> None: self.harvest() argv = ["stop", "--lines", "merge,orchestration-merge", "--unit", "passes", "--done", "4", "--total", "4", "--kost", str(self.work / "kost.json")] self.assertEqual(0, run_main([*argv, "--cap", "3544"])[0]) # (2221 + 1000) x 1.10 = 3543.1 self.assertEqual(1, run_main([*argv, "--cap", "3543"])[0]) def test_an_okter_line_of_kind_merge_is_read(self) -> None: self.assertEqual([("s4", "merge")], cost.sessions(self.okter)) class RepairKindTest(Transcripts): """Order 34 B3: the repair helper (okf-wiki-repairer, described V1--repair-) on its own line `repair`, its main session on orchestration-repair, and the breach count against each task's part list.""" def setUp(self) -> None: super().setUp() self.tasks = self.work / "tasks" self.tasks.mkdir() self.parts = [self.tasks / "r001-1.md", self.tasks / "r001-2.md"] self.answer = self.tasks / "r001.answers.json" (self.tasks / "r001.parts.json").write_text(json.dumps( {"schema": "okf-wiki/repair-parts/1", "task": "r001", "parts": [str(p) for p in self.parts], "answer": str(self.answer)}), encoding="utf-8") jsonl(self.proj / "s5.jsonl", [USER, spawn("o0", 1, 900, 90, 9, "Agent")]) self.okter.write_text("s5 repair\n", encoding="utf-8") def call(self, mid: str, tool: str, path: Path | None = None) -> dict[str, Any]: args = {"file_path": str(path)} if path is not None else {"command": "ls"} return msg(mid, 1, 1, 1, 1, "tool_use", [{"type": "tool_use", "id": f"t-{mid}", "name": tool, "input": args}]) def helper(self, name: str, task: str, lines: list[dict[str, Any]]) -> None: self.agent("s5", name, f"V1-b5r-repair-{task}", lines, agent_type="okf-wiki-repairer") def isolation(self) -> tuple[int, dict[str, Any]]: out = self.work / "isolation.json" code, _ = run_main(["isolation", "--transcripts", str(self.proj), "--run", "b5r", "--tasks", str(self.tasks), "--out", str(out), "--line", "repair"]) return code, json.loads(out.read_text(encoding="utf-8")) if out.is_file() else {} def faithful(self) -> list[dict[str, Any]]: return [self.call("a", "Read", self.parts[0]), self.call("b", "Read", self.parts[1]), self.call("c", "Write", self.answer), msg("d", 1, 2_000, 200, 20)] def test_a_repair_helper_lands_on_the_repair_line(self) -> None: self.helper("h1", "r001", [msg("h1", 1, 2_000, 200, 20)]) code, kost, _ = self.harvest() self.assertEqual(0, code) self.assertEqual([], kost["unattributed"]) (run,) = kost["runs"] self.assertEqual(("b5r", "repair", "r001"), (run["run"], run["line"], run["unit"])) self.assertEqual(1 + 2_000 + 200 + 20, kost["lines"]["repair"]["total"]) self.assertEqual(1 + 900 + 90 + 9, kost["lines"]["orchestration-repair"]["total"]) code, text = run_main(["summarize", "--kost", str(self.work / "kost.json"), "--cap-group", "repair=repair,orchestration-repair:5000000"]) self.assertEqual(0, code) self.assertIn("repair: 3221 + margin 0.10 = 3544 of 5000000", text) self.assertIn("orchestration-repair per turn:", text) def test_an_okter_line_of_kind_repair_is_read(self) -> None: self.assertEqual([("s5", "repair")], cost.sessions(self.okter)) def test_reading_the_parts_and_writing_the_answer_is_no_breach(self) -> None: self.helper("h1", "r001", self.faithful()) code, doc = self.isolation() self.assertEqual(0, code) self.assertEqual(("repair", 1, 0), (doc["line"], doc["agents"], doc["breaks"])) (agent,) = doc["per_agent"] self.assertEqual(("r001", 2, 1, []), (agent["claim"], agent["reads"], agent["writes"], agent["breaks"])) def test_a_read_or_a_write_outside_the_part_list_is_a_breach(self) -> None: other = self.tasks / "r002-1.md" self.helper("h1", "r001", [*self.faithful(), self.call("e", "Read", other), self.call("f", "Write", self.tasks / "r002.answers.json")]) code, doc = self.isolation() self.assertEqual(1, code) self.assertEqual([f"Read {other}", f"Write {self.tasks / 'r002.answers.json'}"], doc["per_agent"][0]["breaks"]) def test_a_tool_outside_read_and_write_is_a_breach_of_its_own(self) -> None: self.helper("h1", "r001", [*self.faithful(), self.call("e", "Bash"), self.call("f", "Edit", self.answer)]) code, doc = self.isolation() self.assertEqual(1, code) self.assertEqual(["Bash", f"Edit {self.answer}"], doc["per_agent"][0]["breaks"]) def test_an_answer_not_written_or_written_twice_is_a_breach(self) -> None: """Review finding 24 (order 39): the helper must write exactly the one answer path.""" reads = [self.call("a", "Read", self.parts[0]), self.call("b", "Read", self.parts[1])] for name, lines, expected in ( ("none", [*reads, msg("d", 1, 2_000, 200, 20)], ["answer not written"]), ("twice", [*reads, self.call("c", "Write", self.answer), self.call("e", "Write", self.answer)], ["answer written 2 times"])): with self.subTest(name=name): shutil.rmtree(self.proj / "s5", ignore_errors=True) self.helper("h1", "r001", lines) code, doc = self.isolation() self.assertEqual(1, code) self.assertEqual(expected, doc["per_agent"][0]["breaks"]) def test_a_part_not_read_is_a_breach(self) -> None: """Review finding 24 (order 39): the helper must read exactly the parts of its task.""" write = [self.call("c", "Write", self.answer)] first, second = (p.resolve() for p in self.parts) for name, lines, expected in ( ("none", write, [f"part not read: {first}", f"part not read: {second}"]), ("one", [self.call("a", "Read", self.parts[0]), *write], [f"part not read: {second}"])): with self.subTest(name=name): shutil.rmtree(self.proj / "s5", ignore_errors=True) self.helper("h1", "r001", lines) code, doc = self.isolation() self.assertEqual(1, code) self.assertEqual(expected, doc["per_agent"][0]["breaks"]) def test_a_workflow_s_repair_helpers_are_counted(self) -> None: self.agent("s5", "w1", "V1-b5r-repair-r001", self.faithful(), agent_type="okf-wiki-repairer", workflow="wf_abc123") code, doc = self.isolation() self.assertEqual((0, 1), (code, doc["agents"])) def test_a_helper_whose_task_has_no_part_list_cannot_run(self) -> None: self.helper("h1", "r009", self.faithful()) code, _ = self.isolation() self.assertEqual(2, code) def test_the_read_line_still_counts_only_reads(self) -> None: tasks = self.work / "tasks" self.agent("s5", "r1", "V1-b5r-read-c001", [self.call("a", "Read", tasks / "c001.md"), self.call("b", "Bash")], agent_type="okf-wiki-reader") out = self.work / "isolation.json" code, _ = run_main(["isolation", "--transcripts", str(self.proj), "--run", "b5r", "--tasks", str(tasks), "--out", str(out)]) self.assertEqual(0, code) class MarginTest(unittest.TestCase): def test_the_floor_is_ten_percent(self) -> None: self.assertEqual(0.10, cost.margin(None)) self.assertEqual(0.10, cost.margin(0.04)) self.assertEqual(0.12, cost.margin(0.12)) class SummarizeTest(unittest.TestCase): def test_lines_and_cap_groups(self) -> None: with tempfile.TemporaryDirectory() as tmp: kost = kost_file(Path(tmp) / "kost.json", {"build": 1_000, "link": 500, "calib": 70}, [{"line": "build", "pack": 1, "total": 1_000}]) code, text = run_main(["summarize", "--kost", str(kost), "--cap-group", "build=build,link:2000"]) self.assertEqual(0, code) self.assertIn("build: 1000 + margin 0.10 = 1100", text) self.assertIn("calib: 70 + margin 0.10 = 77 (reported, never capped)", text) self.assertIn("build: 1500 + margin 0.10 = 1650 of 2000", text) def test_a_group_over_its_cap_exits_1(self) -> None: with tempfile.TemporaryDirectory() as tmp: kost = kost_file(Path(tmp) / "kost.json", {"build": 1_000}) self.assertEqual(1, run_main(["summarize", "--kost", str(kost), "--cap-group", "b=build:1099"])[0]) class IsolationTest(Transcripts): def read(self, mid: str, path: Path) -> dict[str, Any]: return msg(mid, 1, 1, 1, 1, "tool_use", [{"type": "tool_use", "name": "Read", "input": {"file_path": str(path)}}]) def test_a_read_outside_the_claims_task_file_is_a_break(self) -> None: tasks = self.work / "tasks" self.agent("s2", "r1", "V1-utsnitt-read-c001", [self.read("a", tasks / "c001.md")], agent_type="okf-wiki-reader") self.agent("s2", "r2", "V1-utsnitt-read-c002", [self.read("b", tasks / "c002.md"), self.read("c", tasks / "c001.md")], agent_type="okf-wiki-reader") self.agent("s2", "r3", "V1-other-read-c003", [self.read("d", Path("/etc/hosts"))], agent_type="okf-wiki-reader") out = self.work / "isolation.json" code, _ = run_main(["isolation", "--transcripts", str(self.proj), "--run", "utsnitt", "--tasks", str(tasks), "--out", str(out)]) self.assertEqual(1, code) doc = json.loads(out.read_text(encoding="utf-8")) self.assertEqual((2, 1), (doc["agents"], doc["breaks"])) self.assertEqual([str(tasks / "c001.md")], [b for a in doc["per_agent"] for b in a["breaks"]]) def test_a_workflows_reader_agents_are_counted(self) -> None: tasks = self.work / "tasks" self.agent("s2", "r1", "V1-utsnitt-read-c001", [self.read("a", tasks / "c001.md")], agent_type="okf-wiki-reader") self.agent("s2", "r4", "V1-utsnitt-read-c004", [self.read("e", tasks / "c004.md"), self.read("f", Path("/etc/hosts"))], agent_type="okf-wiki-reader", workflow="wf_abc123") out = self.work / "isolation.json" code, _ = run_main(["isolation", "--transcripts", str(self.proj), "--run", "utsnitt", "--tasks", str(tasks), "--out", str(out)]) self.assertEqual(1, code) doc = json.loads(out.read_text(encoding="utf-8")) self.assertEqual((2, 1), (doc["agents"], doc["breaks"])) self.assertEqual(["/etc/hosts"], [b for a in doc["per_agent"] for b in a["breaks"]]) def test_no_reader_agents_is_a_fail(self) -> None: out = self.work / "isolation.json" code, _ = run_main(["isolation", "--transcripts", str(self.proj), "--run", "utsnitt", "--tasks", str(self.work / "tasks"), "--out", str(out)]) self.assertEqual(1, code) class ProjectionTest(unittest.TestCase): def test_least_squares_on_cost_per_block_against_blocks_read(self) -> None: p = cost.project([(600_000, 600, 0), (220_000, 200, 600)], total_blocks=3000) self.assertAlmostEqual(1000, p["fit"]["a"]) self.assertAlmostEqual(1 / 6, p["fit"]["b"]) self.assertAlmostEqual(3_750_000, p["fit"]["value"]) self.assertAlmostEqual(3_075_000, p["flat"]) def test_one_point_has_no_growth_term(self) -> None: p = cost.project([(600_000, 600, 0)], total_blocks=3000) self.assertEqual(0, p["fit"]["b"]) self.assertAlmostEqual(3_000_000, p["fit"]["value"]) class AllocationTest(unittest.TestCase): def test_tokens_per_source_by_block_share(self) -> None: names = [f"s{i:02d}.md" for i in range(1, 20)] blocks_doc = {"order": names, "blocks": [{"file": n, "start": k, "end": k} for n in names for k in (1, 2)]} packs = [names[i:i + 5] for i in range(0, 19, 5)] per = cost.allocate({1: 1000, 2: 500, 3: 500, 4: 800}, packs, blocks_doc) self.assertEqual(200, per["s01.md"]) self.assertEqual(200, per["s16.md"]) class CommonTest(unittest.TestCase): def test_write_new_refuses_to_overwrite(self) -> None: with tempfile.TemporaryDirectory() as tmp: p = Path(tmp) / "sub" / "a.json" common.write_new(p, "one") self.assertEqual("one", p.read_text(encoding="utf-8")) with self.assertRaises(common.RunError): common.write_new(p, "two") self.assertEqual("one", p.read_text(encoding="utf-8")) def test_jaccard(self) -> None: self.assertEqual(0.5, common.jaccard({"a", "b"}, {"b", "c", "a", "d"})) self.assertEqual(0.0, common.jaccard(set(), set())) class NoModelTest(unittest.TestCase): def test_tools_import_no_network_or_model_client(self) -> None: for name in ("cost.py", "transcript.py", "common.py"): source = (ROOT / "okfwiki" / name).read_text(encoding="utf-8") for banned in ("anthropic", "openai", "urllib", "http.client", "requests", "subprocess", "socket"): self.assertNotIn(f"import {banned}", source, name) if __name__ == "__main__": unittest.main()