"""Tests for the P1 method trial's report side (TRIAL, branch p1-metodeprove, not v1 code). Covers the measures (evals/p1/measure.py), the cost harvester (cost.py), the side-by-side page (sidebyside.py) and the two verification scripts. None of these call a model. The fixtures are synthetic text about gardening (CLAUDE.md rule 3: no text from the Claude Code documentation is committed). Run: python3 -m unittest discover -s evals/tests -p 'test_p1_report.py' """ from __future__ import annotations import json import os import subprocess import sys import tempfile import unittest from pathlib import Path from typing import Any import yaml ROOT = Path(__file__).resolve().parents[2] sys.path.insert(0, str(ROOT)) sys.path.insert(0, str(ROOT / "evals")) from okfwiki import audit, view # noqa: E402 from p1 import cost, measure, sidebyside, split # noqa: E402 BANNED = ("anthropic", "openai", "urllib", "http.client", "requests", "subprocess", "socket") DOC_A = """# Alpha ## Beds Raised beds drain well. They warm early in spring. - Line one item. - Line two item. ## Paths Gravel paths stay dry. """ REL = """## 2.0.0 - Beds got labels. """ def src(sid: str, file: str, line: int, quote: str, span: tuple[int, int] | None = None) -> dict[str, Any]: s: dict[str, Any] = {"id": sid, "resource": f"/kilde/{file}", "title": file, "x-locator": {"line": line}, "x-quote": quote} if span: s["x-span"] = list(span) return s def write_page(folder: Path, name: str, title: str, body: str, sources: list[dict[str, Any]], **extra: Any) -> None: fm = {"type": "Concept", "title": title, "description": title, "sources": sources, **extra, "generated": {"by": "test", "at": "2026-10-02T12:00:00Z"}} text = "---\n" + yaml.safe_dump(fm, sort_keys=False) + "---\n\n# " + title + "\n\n" + body + "\n" folder.mkdir(parents=True, exist_ok=True) (folder / name).write_text(text, encoding="utf-8") class Fixture(unittest.TestCase): def setUp(self) -> None: self._tmp = tempfile.TemporaryDirectory() self.tmp = Path(self._tmp.name) self.root = self.tmp / "wiki" self.kilde = self.root / "kilde" self.kilde.mkdir(parents=True) (self.kilde / "docs-en-a.md").write_text(DOC_A, encoding="utf-8") (self.kilde / "release-v2.0.0.md").write_text(REL, encoding="utf-8") self.order = ["docs-en-a.md", "release-v2.0.0.md"] self.blocks = split.blocks_document(self.kilde, self.order) self.bundle = self.tmp / "B" write_page(self.bundle, "beds.md", "Beds", "Raised beds drain well.[^b1] The beds have a heading.[^b2] Items begin.[^b3] Two items.[^b4]\n\n" "## History\n\n- Beds got labels.[^r1][^v1]\n\n| a | b |", [src("b1", "docs-en-a.md", 5, "Raised beds drain well.", (5, 6)), src("b2", "docs-en-a.md", 3, "Beds", (3, 3)), src("b3", "docs-en-a.md", 8, "Line one item.", (8, 13)), src("b4", "docs-en-a.md", 9, "Line two item.", (8, 9)), src("v1", "release-v2.0.0.md", 1, "2.0.0"), src("r1", "release-v2.0.0.md", 3, "Beds got labels.", (3, 3)), src("u1", "docs-en-a.md", 13, "Gravel paths stay dry.", (13, 13))], **{"x-aliases": ["Beds", "Raised beds"]}) write_page(self.bundle, "labels.md", "Labels", "Beds got labels.[^r1][^v1] See also: [Beds](beds.md)", [src("v1", "release-v2.0.0.md", 1, "2.0.0"), src("r1", "release-v2.0.0.md", 3, "Beds got labels.")]) (self.bundle / "index.md").write_text('---\nokf_version: "0.2"\n---\n\n# Concepts\n\n- [Beds](beds.md)\n' '- [Labels](labels.md)\n', encoding="utf-8") (self.bundle / "log.md").write_text("# Log\n\n## 2026-10-02\n* built\n", encoding="utf-8") def tearDown(self) -> None: self._tmp.cleanup() # ---------------------------------------------------------------------------------------------- step 5: measures class CoverageTest(Fixture): def test_cited_entries_by_locator_line_only(self) -> None: got = measure.coverage(self.bundle, self.root, self.blocks, self.order) # covered: 5-6 (b1), 8 (b3's locator; its wide span is ignored), 9 (b4), release 3 (r1); not 13 (u1 uncited) self.assertEqual((got.covered, got.total), (4, 5)) def test_a_stopped_variant_counts_only_the_sources_it_built(self) -> None: got = measure.coverage(self.bundle, self.root, self.blocks, self.order[:1]) self.assertEqual((got.covered, got.total), (3, 4)) def test_per_pack_position(self) -> None: got = measure.coverage(self.bundle, self.root, self.blocks, self.order, pack_size=5) self.assertEqual(got.per_position, {1: (3, 4, 1), 2: (1, 1, 1)}) class StructureTest(Fixture): def test_shares_precision_and_concepts_per_source(self) -> None: s = measure.structure(self.bundle, self.root, self.blocks, self.order) self.assertEqual(s["pages"], 2) self.assertEqual(s["concepts_per_source"], {"docs-en-a.md": 1, "release-v2.0.0.md": 1}) # version sources are left out of the quote and precision counts self.assertEqual(s["quotes"]["entries"], 6) self.assertEqual(s["quotes"]["structural"], 1) # b2 quotes a heading line self.assertEqual(s["precision"], {"line": 3, "block": 1, "section": 1, "document": 1}) self.assertEqual(s["islands"]["total"], sum(not v for v in view.related( audit.load_bundle(self.bundle, self.root).concepts).values())) def test_duplicates_by_norm_alias_and_jaccard_in_any_bundle(self) -> None: b = self.tmp / "dups" for name, title in (("a.md", "Beds"), ("b.md", "Bed"), ("c.md", "Raised beds"), ("d.md", "Gravel path"), ("e.md", "Gravel path rules"), ("f.md", "Tools")): write_page(b, name, title, "x[^s].", [src("s", "docs-en-a.md", 5, "Raised")], **({"x-aliases": ["Beds", "Raised beds"]} if name == "a.md" else {})) pairs = measure.duplicates(audit.load_bundle(b, self.root).concepts) self.assertIn(("a.md", "b.md", "norm"), pairs) self.assertIn(("a.md", "c.md", "alias"), pairs) self.assertIn(("d.md", "e.md", "jaccard"), pairs) self.assertFalse(any("f.md" in p for p in pairs)) def test_a_wrong_root_raises(self) -> None: with self.assertRaises(measure.RunError): measure.structure(self.bundle, self.tmp / "nowhere", self.blocks, self.order) class AuditRecordTest(Fixture): def test_uncited_sentences_are_classed_by_cause_and_yaml_form_apart(self) -> None: write_page(self.bundle, "odd.md", "Odd", "```bash\n\nPlain words here.\n\nSee also: [Beds](beds.md)", [{"id": "y", "resource": "/kilde/docs-en-a.md", "x-locator": {"line": 5}, "x-quote": False}]) out = self.tmp / "resultat.json" rec = measure.audit_record(self.bundle, self.root, "K", out) self.assertEqual(rec["exit"], 1) # labels.md in the shared fixture holds one more link lead-in ("See also: [Beds](beds.md)") self.assertEqual(rec["uncited"], {"fence line": 1, "link lead-in": 2, "prose": 1, "table row": 1}) self.assertEqual(rec["yaml_form"], 1) self.assertEqual(json.loads(out.read_text(encoding="utf-8"))["variants"]["K"]["exit"], 1) with self.assertRaises(measure.RunError): measure.audit_record(self.bundle, self.root, "K", out) # one record per variant # ---------------------------------------------------------------------------------------------- step 6: cost def msg(mid: str, inp: int, cr: int, cw: int, out: int, stop: str | None = "end_turn") -> dict[str, Any]: return {"type": "assistant", "message": {"id": mid, "stop_reason": stop, "usage": { "input_tokens": inp, "cache_read_input_tokens": cr, "cache_creation_input_tokens": cw, "output_tokens": out}}} def jsonl(path: Path, lines: list[dict[str, Any]]) -> None: path.parent.mkdir(parents=True, exist_ok=True) path.write_text("\n".join(json.dumps(x) for x in lines) + "\n", encoding="utf-8") class HarvestTest(unittest.TestCase): def setUp(self) -> None: self._tmp = tempfile.TemporaryDirectory() self.proj = Path(self._tmp.name) user = {"type": "user", "message": {"content": "task"}} jsonl(self.proj / "s1.jsonl", [user, msg("m0", 5, 100, 10, 7)]) sub = self.proj / "s1" / "subagents" # a streamed message id appears twice with growing output: counted once, at its max jsonl(sub / "agent-a1.jsonl", [user, msg("x1", 3, 20_000, 2_000, 10, None), msg("x1", 3, 20_000, 2_000, 50), msg("x2", 1, 22_000, 500, 900)]) (sub / "agent-a1.meta.json").write_text(json.dumps({"agentType": "p1-compiler", "description": "P1-K-pack1"})) jsonl(sub / "agent-a2.jsonl", [user, msg("y1", 2, 19_000, 1_000, 5)]) (sub / "agent-a2.meta.json").write_text(json.dumps({"agentType": "p1-compiler", "description": "load test"})) jsonl(sub / "agent-a3.jsonl", [user, msg("z1", 1, 1, 1, 1, None)]) (sub / "agent-a3.meta.json").write_text(json.dumps({"agentType": "p1-consolidator", "description": "P1-K+-run1"})) # a session not listed in okter.txt: its transcript is never opened jsonl(self.proj / "s9" / "subagents" / "agent-b1.jsonl", [user, msg("q", 1, 1, 1, 1)]) (self.proj / "s9" / "subagents" / "agent-b1.meta.json").write_text( json.dumps({"agentType": "p1-compiler", "description": "P1-K-pack2"})) self.okter = self.proj / "okter.txt" self.okter.write_text("s1 step8\ns1 step9\n", encoding="utf-8") def tearDown(self) -> None: self._tmp.cleanup() def test_four_categories_turns_startup_and_attribution(self) -> None: got = cost.harvest(self.proj, cost.sessions(self.okter)) self.assertEqual(got["sessions_read"], ["s1"]) runs = {r["unit"]: r for r in got["runs"]} k = runs["pack1"] self.assertEqual((k["variant"], k["input"], k["cache_read"], k["cache_write"], k["output"]), ("K", 4, 42_000, 2_500, 950)) self.assertEqual(k["turns"], 2) self.assertEqual(k["startup"], 3 + 20_000 + 2_000) self.assertEqual(runs["run1"]["variant"], "K+") self.assertEqual(runs["run1"]["stop_reason"], None) # a stopped agent is listed, not dropped self.assertEqual([u["description"] for u in got["unattributed"]], ["load test"]) self.assertEqual(got["main"]["s1"]["cache_read"], 100) self.assertNotIn("pack2", runs) def test_output_never_recorded_is_derived_from_the_next_messages_input_growth(self) -> None: # Found in session 2: a long streamed message keeps only its first usage record (output 2, no stop # reason). The next message's input side grew by that output plus the text injected between them (tool # results and rendered attachments), counted high at 2.5 characters per token so the output is a lower bound. def result(text: str) -> dict[str, Any]: return {"type": "user", "message": {"content": [{"type": "tool_result", "content": text}]}} sub = self.proj / "s1" / "subagents" jsonl(sub / "agent-a4.jsonl", [{"type": "user", "message": {"content": "task"}}, msg("w1", 2, 0, 10_000, 300, "tool_use"), result("x" * 400), msg("w2", 2, 10_000, 50_000, 2, None), msg("w2", 2, 10_000, 50_000, 2, None), result("y" * 40), msg("w3", 2, 60_000, 5_010, 20)]) (sub / "agent-a4.meta.json").write_text(json.dumps({"agentType": "p1-compiler", "description": "P1-K-pack3"})) runs = {r["unit"]: r for r in cost.harvest(self.proj, cost.sessions(self.okter))["runs"]} w = runs["pack3"] self.assertEqual(w["output_recorded"], 300 + 2 + 20) self.assertEqual(w["output"], 300 + (5_010 - 40 * 2 // 5) + 20) self.assertEqual((w["output_derived_messages"], w["output_unrecorded"]), (1, 0)) self.assertEqual(w["total"], 2 + 2 + 2 + 70_000 + 65_010 + w["output"]) # the last message of a run has no next message: its output stays as recorded and is flagged self.assertEqual((runs["run1"]["output"], runs["run1"]["output_unrecorded"]), (1, 1)) self.assertEqual(runs["pack1"]["output_derived_messages"], 0) def test_growth_from_injected_attachments_is_not_counted_as_output(self) -> None: # The load test's pattern: a short call, then 5133 characters of rendered attachments before the next turn. sub = self.proj / "s1" / "subagents" jsonl(sub / "agent-a5.jsonl", [{"type": "user", "message": {"content": "task"}}, msg("v1", 2, 0, 18_602, 16, None), {"type": "attachment", "rendered": "z" * 5_133}, {"type": "user", "message": {"content": [{"type": "tool_result", "content": "w" * 151}]}}, msg("v2", 2, 18_602, 1_894, 24, None)]) (sub / "agent-a5.meta.json").write_text(json.dumps({"agentType": "p1-compiler", "description": "P1-LOAD-x"})) runs = {r["unit"]: r for r in cost.harvest(self.proj, cost.sessions(self.okter))["runs"]} self.assertEqual((runs["x"]["output"], runs["x"]["output_derived_messages"]), (16 + 24, 0)) class ProjectionTest(unittest.TestCase): def test_least_squares_on_cost_per_block_against_blocks_read(self) -> None: p = cost.project([(600_000, 600, 0), (220_000, 200, 600)], total_blocks=3000) self.assertAlmostEqual(p["fit"]["a"], 1000) self.assertAlmostEqual(p["fit"]["b"], 1 / 6) self.assertAlmostEqual(p["fit"]["value"], 3_750_000) self.assertAlmostEqual(p["flat"], 3_075_000) def test_one_point_has_no_growth_term(self) -> None: p = cost.project([(600_000, 600, 0)], total_blocks=3000) self.assertEqual(p["fit"]["b"], 0) self.assertEqual(p["fit"]["note"], "one point, no growth term") self.assertAlmostEqual(p["fit"]["value"], 3_000_000) def test_stop_per_block_against_naive_per_source(self) -> None: s = cost.stop_k(pack1_cost=600_000, pack1_blocks=600, cut_blocks=1200, pack1_sources=5, cut_sources=19) self.assertEqual(s["per_block"], 1_200_000) self.assertEqual(s["per_source"], 2_280_000) self.assertTrue(s["stop"]) self.assertFalse(cost.stop_k(500_000, 600, 1200, 5, 19)["stop"]) # exactly 1 M does not stop self.assertTrue(cost.stop_k(500_001, 600, 1200, 5, 19)["stop"]) def test_kplus_stop(self) -> None: self.assertFalse(cost.stop_kplus(run1_cost=200_000, run1_pages=8, pages_total=40)["stop"]) self.assertTrue(cost.stop_kplus(run1_cost=200_001, run1_pages=8, pages_total=40)["stop"]) class AllocationTest(unittest.TestCase): def test_tokens_per_source_by_block_share_and_the_two_means(self) -> None: names = [f"s{i:02d}.md" for i in range(1, 20)] blocks_doc = {"order": names, "blocks": [{"file": n, "start": k, "end": k} for n in names for k in (1, 2)]} packs = [names[i:i + 5] for i in range(0, 19, 5)] costs = {1: 1000, 2: 500, 3: 500, 4: 800} per = cost.allocate(costs, packs, blocks_doc) self.assertEqual(per["s01.md"], 200) self.assertEqual(per["s16.md"], 200) means = cost.means(per, names) self.assertEqual(means["1-5"], 200) self.assertEqual(means["15-19"], 180) # source 15 in pack 3 (100), 16-19 in pack 4 (200 each) # ---------------------------------------------------------------------------------------------- step 7: side by side class SideBySideTest(Fixture): def setUp(self) -> None: super().setUp() self.begreper = self.tmp / "begreper.txt" self.begreper.write_text("Beds\tBeds;Raised beds\nLabels\tLabels\nCompost\tCompost\n", encoding="utf-8") self.d = self.tmp / "D" write_page(self.d, "r-beds.md", "Raised beds", "- [Alpha](k.md)", [src("o1", "docs-en-a.md", 5, "Raised beds")]) self.l4 = self.tmp / "L4" write_page(self.l4, "x.md", "THE raised-beds, explained!", "Beds drain.[^b1]", [src("b1", "docs-en-a.md", 5, "Raised beds drain well.")]) write_page(self.l4, "y.md", "Compost", "Compost helps.[^b1]", [src("b1", "docs-en-a.md", 99, "Compost")]) # a link past the end of the file def columns(self) -> list[tuple[str, Path | None]]: return [("K", self.bundle), ("K+", None), ("D", self.d), ("L4", self.l4)] def test_three_concepts_in_four_columns_with_the_empty_states(self) -> None: out = self.tmp / "side.html" sidebyside.render(sidebyside.read_begreper(self.begreper), self.columns(), self.root, out) html = out.read_text(encoding="utf-8") self.assertEqual(html.count('