okf-wiki/evals/tests/test_p1_report.py
Kjell Tore Guttormsen 0fd86a23a3
okf-wiki 1.1.1
First public release, published as one snapshot. What changed between
versions is in CHANGELOG.md.

Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
2026-11-14 21:03:20 +01:00

366 lines
19 KiB
Python

"""Tests for the P1 method trial's report side (TRIAL, branch p1-metodeprove, not v1 code).
Covers the measures (evals/p1/measure.py), the cost harvester (cost.py), the side-by-side page (sidebyside.py) and the
two verification scripts. None of these call a model. The fixtures are synthetic text about gardening (CLAUDE.md
rule 3: no text from the Claude Code documentation is committed).
Run: python3 -m unittest discover -s evals/tests -p 'test_p1_report.py'
"""
from __future__ import annotations
import json
import os
import subprocess
import sys
import tempfile
import unittest
from pathlib import Path
from typing import Any
import yaml
ROOT = Path(__file__).resolve().parents[2]
sys.path.insert(0, str(ROOT))
sys.path.insert(0, str(ROOT / "evals"))
from okfwiki import audit, view # noqa: E402
from p1 import cost, measure, sidebyside, split # noqa: E402
BANNED = ("anthropic", "openai", "urllib", "http.client", "requests", "subprocess", "socket")
DOC_A = """# Alpha
## Beds
Raised beds drain well.
They warm early in spring.
- Line one item.
- Line two item.
## Paths
Gravel paths stay dry.
"""
REL = """## 2.0.0
- Beds got labels.
"""
def src(sid: str, file: str, line: int, quote: str, span: tuple[int, int] | None = None) -> dict[str, Any]:
s: dict[str, Any] = {"id": sid, "resource": f"/kilde/{file}", "title": file, "x-locator": {"line": line},
"x-quote": quote}
if span:
s["x-span"] = list(span)
return s
def write_page(folder: Path, name: str, title: str, body: str, sources: list[dict[str, Any]],
**extra: Any) -> None:
fm = {"type": "Concept", "title": title, "description": title, "sources": sources, **extra,
"generated": {"by": "test", "at": "2026-10-02T12:00:00Z"}}
text = "---\n" + yaml.safe_dump(fm, sort_keys=False) + "---\n\n# " + title + "\n\n" + body + "\n"
folder.mkdir(parents=True, exist_ok=True)
(folder / name).write_text(text, encoding="utf-8")
class Fixture(unittest.TestCase):
def setUp(self) -> None:
self._tmp = tempfile.TemporaryDirectory()
self.tmp = Path(self._tmp.name)
self.root = self.tmp / "wiki"
self.kilde = self.root / "kilde"
self.kilde.mkdir(parents=True)
(self.kilde / "docs-en-a.md").write_text(DOC_A, encoding="utf-8")
(self.kilde / "release-v2.0.0.md").write_text(REL, encoding="utf-8")
self.order = ["docs-en-a.md", "release-v2.0.0.md"]
self.blocks = split.blocks_document(self.kilde, self.order)
self.bundle = self.tmp / "B"
write_page(self.bundle, "beds.md", "Beds",
"Raised beds drain well.[^b1] The beds have a heading.[^b2] Items begin.[^b3] Two items.[^b4]\n\n"
"## History\n\n- Beds got labels.[^r1][^v1]\n\n| a | b |",
[src("b1", "docs-en-a.md", 5, "Raised beds drain well.", (5, 6)),
src("b2", "docs-en-a.md", 3, "Beds", (3, 3)),
src("b3", "docs-en-a.md", 8, "Line one item.", (8, 13)),
src("b4", "docs-en-a.md", 9, "Line two item.", (8, 9)),
src("v1", "release-v2.0.0.md", 1, "2.0.0"),
src("r1", "release-v2.0.0.md", 3, "Beds got labels.", (3, 3)),
src("u1", "docs-en-a.md", 13, "Gravel paths stay dry.", (13, 13))],
**{"x-aliases": ["Beds", "Raised beds"]})
write_page(self.bundle, "labels.md", "Labels", "Beds got labels.[^r1][^v1] See also: [Beds](beds.md)",
[src("v1", "release-v2.0.0.md", 1, "2.0.0"), src("r1", "release-v2.0.0.md", 3, "Beds got labels.")])
(self.bundle / "index.md").write_text('---\nokf_version: "0.2"\n---\n\n# Concepts\n\n- [Beds](beds.md)\n'
'- [Labels](labels.md)\n', encoding="utf-8")
(self.bundle / "log.md").write_text("# Log\n\n## 2026-10-02\n* built\n", encoding="utf-8")
def tearDown(self) -> None:
self._tmp.cleanup()
# ---------------------------------------------------------------------------------------------- step 5: measures
class CoverageTest(Fixture):
def test_cited_entries_by_locator_line_only(self) -> None:
got = measure.coverage(self.bundle, self.root, self.blocks, self.order)
# covered: 5-6 (b1), 8 (b3's locator; its wide span is ignored), 9 (b4), release 3 (r1); not 13 (u1 uncited)
self.assertEqual((got.covered, got.total), (4, 5))
def test_a_stopped_variant_counts_only_the_sources_it_built(self) -> None:
got = measure.coverage(self.bundle, self.root, self.blocks, self.order[:1])
self.assertEqual((got.covered, got.total), (3, 4))
def test_per_pack_position(self) -> None:
got = measure.coverage(self.bundle, self.root, self.blocks, self.order, pack_size=5)
self.assertEqual(got.per_position, {1: (3, 4, 1), 2: (1, 1, 1)})
class StructureTest(Fixture):
def test_shares_precision_and_concepts_per_source(self) -> None:
s = measure.structure(self.bundle, self.root, self.blocks, self.order)
self.assertEqual(s["pages"], 2)
self.assertEqual(s["concepts_per_source"], {"docs-en-a.md": 1, "release-v2.0.0.md": 1})
# version sources are left out of the quote and precision counts
self.assertEqual(s["quotes"]["entries"], 6)
self.assertEqual(s["quotes"]["structural"], 1) # b2 quotes a heading line
self.assertEqual(s["precision"], {"line": 3, "block": 1, "section": 1, "document": 1})
self.assertEqual(s["islands"]["total"], sum(not v for v in view.related(
audit.load_bundle(self.bundle, self.root).concepts).values()))
def test_duplicates_by_norm_alias_and_jaccard_in_any_bundle(self) -> None:
b = self.tmp / "dups"
for name, title in (("a.md", "Beds"), ("b.md", "Bed"), ("c.md", "Raised beds"), ("d.md", "Gravel path"),
("e.md", "Gravel path rules"), ("f.md", "Tools")):
write_page(b, name, title, "x[^s].", [src("s", "docs-en-a.md", 5, "Raised")],
**({"x-aliases": ["Beds", "Raised beds"]} if name == "a.md" else {}))
pairs = measure.duplicates(audit.load_bundle(b, self.root).concepts)
self.assertIn(("a.md", "b.md", "norm"), pairs)
self.assertIn(("a.md", "c.md", "alias"), pairs)
self.assertIn(("d.md", "e.md", "jaccard"), pairs)
self.assertFalse(any("f.md" in p for p in pairs))
def test_a_wrong_root_raises(self) -> None:
with self.assertRaises(measure.RunError):
measure.structure(self.bundle, self.tmp / "nowhere", self.blocks, self.order)
class AuditRecordTest(Fixture):
def test_uncited_sentences_are_classed_by_cause_and_yaml_form_apart(self) -> None:
write_page(self.bundle, "odd.md", "Odd", "```bash\n\nPlain words here.\n\nSee also: [Beds](beds.md)",
[{"id": "y", "resource": "/kilde/docs-en-a.md", "x-locator": {"line": 5}, "x-quote": False}])
out = self.tmp / "resultat.json"
rec = measure.audit_record(self.bundle, self.root, "K", out)
self.assertEqual(rec["exit"], 1)
# labels.md in the shared fixture holds one more link lead-in ("See also: [Beds](beds.md)")
self.assertEqual(rec["uncited"], {"fence line": 1, "link lead-in": 2, "prose": 1, "table row": 1})
self.assertEqual(rec["yaml_form"], 1)
self.assertEqual(json.loads(out.read_text(encoding="utf-8"))["variants"]["K"]["exit"], 1)
with self.assertRaises(measure.RunError):
measure.audit_record(self.bundle, self.root, "K", out) # one record per variant
# ---------------------------------------------------------------------------------------------- step 6: cost
def msg(mid: str, inp: int, cr: int, cw: int, out: int, stop: str | None = "end_turn") -> dict[str, Any]:
return {"type": "assistant", "message": {"id": mid, "stop_reason": stop, "usage": {
"input_tokens": inp, "cache_read_input_tokens": cr, "cache_creation_input_tokens": cw, "output_tokens": out}}}
def jsonl(path: Path, lines: list[dict[str, Any]]) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text("\n".join(json.dumps(x) for x in lines) + "\n", encoding="utf-8")
class HarvestTest(unittest.TestCase):
def setUp(self) -> None:
self._tmp = tempfile.TemporaryDirectory()
self.proj = Path(self._tmp.name)
user = {"type": "user", "message": {"content": "task"}}
jsonl(self.proj / "s1.jsonl", [user, msg("m0", 5, 100, 10, 7)])
sub = self.proj / "s1" / "subagents"
# a streamed message id appears twice with growing output: counted once, at its max
jsonl(sub / "agent-a1.jsonl", [user, msg("x1", 3, 20_000, 2_000, 10, None), msg("x1", 3, 20_000, 2_000, 50),
msg("x2", 1, 22_000, 500, 900)])
(sub / "agent-a1.meta.json").write_text(json.dumps({"agentType": "p1-compiler", "description": "P1-K-pack1"}))
jsonl(sub / "agent-a2.jsonl", [user, msg("y1", 2, 19_000, 1_000, 5)])
(sub / "agent-a2.meta.json").write_text(json.dumps({"agentType": "p1-compiler", "description": "load test"}))
jsonl(sub / "agent-a3.jsonl", [user, msg("z1", 1, 1, 1, 1, None)])
(sub / "agent-a3.meta.json").write_text(json.dumps({"agentType": "p1-consolidator",
"description": "P1-K+-run1"}))
# a session not listed in okter.txt: its transcript is never opened
jsonl(self.proj / "s9" / "subagents" / "agent-b1.jsonl", [user, msg("q", 1, 1, 1, 1)])
(self.proj / "s9" / "subagents" / "agent-b1.meta.json").write_text(
json.dumps({"agentType": "p1-compiler", "description": "P1-K-pack2"}))
self.okter = self.proj / "okter.txt"
self.okter.write_text("s1 step8\ns1 step9\n", encoding="utf-8")
def tearDown(self) -> None:
self._tmp.cleanup()
def test_four_categories_turns_startup_and_attribution(self) -> None:
got = cost.harvest(self.proj, cost.sessions(self.okter))
self.assertEqual(got["sessions_read"], ["s1"])
runs = {r["unit"]: r for r in got["runs"]}
k = runs["pack1"]
self.assertEqual((k["variant"], k["input"], k["cache_read"], k["cache_write"], k["output"]),
("K", 4, 42_000, 2_500, 950))
self.assertEqual(k["turns"], 2)
self.assertEqual(k["startup"], 3 + 20_000 + 2_000)
self.assertEqual(runs["run1"]["variant"], "K+")
self.assertEqual(runs["run1"]["stop_reason"], None) # a stopped agent is listed, not dropped
self.assertEqual([u["description"] for u in got["unattributed"]], ["load test"])
self.assertEqual(got["main"]["s1"]["cache_read"], 100)
self.assertNotIn("pack2", runs)
def test_output_never_recorded_is_derived_from_the_next_messages_input_growth(self) -> None:
# Found in session 2: a long streamed message keeps only its first usage record (output 2, no stop
# reason). The next message's input side grew by that output plus the text injected between them (tool
# results and rendered attachments), counted high at 2.5 characters per token so the output is a lower bound.
def result(text: str) -> dict[str, Any]:
return {"type": "user", "message": {"content": [{"type": "tool_result", "content": text}]}}
sub = self.proj / "s1" / "subagents"
jsonl(sub / "agent-a4.jsonl", [{"type": "user", "message": {"content": "task"}},
msg("w1", 2, 0, 10_000, 300, "tool_use"), result("x" * 400),
msg("w2", 2, 10_000, 50_000, 2, None), msg("w2", 2, 10_000, 50_000, 2, None),
result("y" * 40), msg("w3", 2, 60_000, 5_010, 20)])
(sub / "agent-a4.meta.json").write_text(json.dumps({"agentType": "p1-compiler", "description": "P1-K-pack3"}))
runs = {r["unit"]: r for r in cost.harvest(self.proj, cost.sessions(self.okter))["runs"]}
w = runs["pack3"]
self.assertEqual(w["output_recorded"], 300 + 2 + 20)
self.assertEqual(w["output"], 300 + (5_010 - 40 * 2 // 5) + 20)
self.assertEqual((w["output_derived_messages"], w["output_unrecorded"]), (1, 0))
self.assertEqual(w["total"], 2 + 2 + 2 + 70_000 + 65_010 + w["output"])
# the last message of a run has no next message: its output stays as recorded and is flagged
self.assertEqual((runs["run1"]["output"], runs["run1"]["output_unrecorded"]), (1, 1))
self.assertEqual(runs["pack1"]["output_derived_messages"], 0)
def test_growth_from_injected_attachments_is_not_counted_as_output(self) -> None:
# The load test's pattern: a short call, then 5133 characters of rendered attachments before the next turn.
sub = self.proj / "s1" / "subagents"
jsonl(sub / "agent-a5.jsonl", [{"type": "user", "message": {"content": "task"}},
msg("v1", 2, 0, 18_602, 16, None),
{"type": "attachment", "rendered": "z" * 5_133},
{"type": "user", "message": {"content": [{"type": "tool_result",
"content": "w" * 151}]}},
msg("v2", 2, 18_602, 1_894, 24, None)])
(sub / "agent-a5.meta.json").write_text(json.dumps({"agentType": "p1-compiler", "description": "P1-LOAD-x"}))
runs = {r["unit"]: r for r in cost.harvest(self.proj, cost.sessions(self.okter))["runs"]}
self.assertEqual((runs["x"]["output"], runs["x"]["output_derived_messages"]), (16 + 24, 0))
class ProjectionTest(unittest.TestCase):
def test_least_squares_on_cost_per_block_against_blocks_read(self) -> None:
p = cost.project([(600_000, 600, 0), (220_000, 200, 600)], total_blocks=3000)
self.assertAlmostEqual(p["fit"]["a"], 1000)
self.assertAlmostEqual(p["fit"]["b"], 1 / 6)
self.assertAlmostEqual(p["fit"]["value"], 3_750_000)
self.assertAlmostEqual(p["flat"], 3_075_000)
def test_one_point_has_no_growth_term(self) -> None:
p = cost.project([(600_000, 600, 0)], total_blocks=3000)
self.assertEqual(p["fit"]["b"], 0)
self.assertEqual(p["fit"]["note"], "one point, no growth term")
self.assertAlmostEqual(p["fit"]["value"], 3_000_000)
def test_stop_per_block_against_naive_per_source(self) -> None:
s = cost.stop_k(pack1_cost=600_000, pack1_blocks=600, cut_blocks=1200, pack1_sources=5, cut_sources=19)
self.assertEqual(s["per_block"], 1_200_000)
self.assertEqual(s["per_source"], 2_280_000)
self.assertTrue(s["stop"])
self.assertFalse(cost.stop_k(500_000, 600, 1200, 5, 19)["stop"]) # exactly 1 M does not stop
self.assertTrue(cost.stop_k(500_001, 600, 1200, 5, 19)["stop"])
def test_kplus_stop(self) -> None:
self.assertFalse(cost.stop_kplus(run1_cost=200_000, run1_pages=8, pages_total=40)["stop"])
self.assertTrue(cost.stop_kplus(run1_cost=200_001, run1_pages=8, pages_total=40)["stop"])
class AllocationTest(unittest.TestCase):
def test_tokens_per_source_by_block_share_and_the_two_means(self) -> None:
names = [f"s{i:02d}.md" for i in range(1, 20)]
blocks_doc = {"order": names, "blocks": [{"file": n, "start": k, "end": k} for n in names for k in (1, 2)]}
packs = [names[i:i + 5] for i in range(0, 19, 5)]
costs = {1: 1000, 2: 500, 3: 500, 4: 800}
per = cost.allocate(costs, packs, blocks_doc)
self.assertEqual(per["s01.md"], 200)
self.assertEqual(per["s16.md"], 200)
means = cost.means(per, names)
self.assertEqual(means["1-5"], 200)
self.assertEqual(means["15-19"], 180) # source 15 in pack 3 (100), 16-19 in pack 4 (200 each)
# ---------------------------------------------------------------------------------------------- step 7: side by side
class SideBySideTest(Fixture):
def setUp(self) -> None:
super().setUp()
self.begreper = self.tmp / "begreper.txt"
self.begreper.write_text("Beds\tBeds;Raised beds\nLabels\tLabels\nCompost\tCompost\n", encoding="utf-8")
self.d = self.tmp / "D"
write_page(self.d, "r-beds.md", "Raised beds", "- [Alpha](k.md)", [src("o1", "docs-en-a.md", 5, "Raised beds")])
self.l4 = self.tmp / "L4"
write_page(self.l4, "x.md", "THE raised-beds, explained!", "Beds drain.[^b1]",
[src("b1", "docs-en-a.md", 5, "Raised beds drain well.")])
write_page(self.l4, "y.md", "Compost", "Compost helps.[^b1]",
[src("b1", "docs-en-a.md", 99, "Compost")]) # a link past the end of the file
def columns(self) -> list[tuple[str, Path | None]]:
return [("K", self.bundle), ("K+", None), ("D", self.d), ("L4", self.l4)]
def test_three_concepts_in_four_columns_with_the_empty_states(self) -> None:
out = self.tmp / "side.html"
sidebyside.render(sidebyside.read_begreper(self.begreper), self.columns(), self.root, out)
html = out.read_text(encoding="utf-8")
self.assertEqual(html.count('<tr class="concept">'), 3)
self.assertEqual(html.count('<td class="cell'), 12)
self.assertIn("ikke bygget", html) # K+ stopped
self.assertIn("ikke i registeret", html) # D has no Labels register page
self.assertIn("ingen side", html) # K is built and has no Compost page
self.assertNotIn("http", html)
def test_l4_finds_a_title_with_other_case_and_punctuation(self) -> None:
found = sidebyside.find(self.l4, ("Raised beds", ("Raised beds",)), l4=True)
self.assertEqual(found, ["x.md"])
def test_the_link_check_flags_a_line_past_the_end(self) -> None:
out = self.tmp / "side.html"
sidebyside.render(sidebyside.read_begreper(self.begreper), self.columns(), self.root, out)
resolved, total = sidebyside.check(out, self.root)
self.assertEqual(total - resolved, 1)
self.assertEqual(sidebyside.main(["check", "--html", str(out), "--kilde-root", str(self.root)]), 1)
class ScriptsTest(unittest.TestCase):
SCRIPTS = (ROOT / "tests" / "p1_repo.sh", ROOT / "tests" / "p1_trial.sh")
def test_bash_syntax_and_ascii(self) -> None:
for script in self.SCRIPTS:
self.assertEqual(subprocess.run(["bash", "-n", str(script)]).returncode, 0, script.name)
script.read_bytes().decode("ascii")
def test_the_trial_check_fails_every_check_when_nothing_was_built(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
env = {**os.environ, "P1_ROOT": tmp}
res = subprocess.run(["bash", str(self.SCRIPTS[1])], env=env, capture_output=True, text=True, cwd=ROOT)
self.assertEqual(res.returncode, 1)
lines = [x for x in res.stdout.splitlines() if x.strip()]
self.assertGreaterEqual(len(lines), 8)
self.assertTrue(all(x.startswith("FAIL") for x in lines), res.stdout)
class NoModelTest(unittest.TestCase):
def test_trial_modules_import_no_network_or_model_client(self) -> None:
for path in sorted((ROOT / "evals" / "p1").glob("*.py")):
source = path.read_text(encoding="utf-8")
for banned in BANNED:
self.assertNotIn(f"import {banned}", source, path.name)
self.assertNotIn(f"from {banned}", source, path.name)
if __name__ == "__main__":
unittest.main()