okf-wiki 1.1.1

First public release, published as one snapshot. What changed between
versions is in CHANGELOG.md.

Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
This commit is contained in:
Kjell Tore Guttormsen 2026-11-14 21:03:20 +01:00
commit 0fd86a23a3
Signed by: ktg
SSH key fingerprint: SHA256:JakMjO6FTBBzN0Bhfj9saOoEjaFxlSdYuZQQpM/lF9Q
227 changed files with 96077 additions and 0 deletions

View file

@ -0,0 +1,355 @@
"""Tests for evals/direction_probe.py: the direction probe R (PLAN.md § Leveranser, row R).
Three arms answer the same questions from the same sources: file search alone, the okf-wiki plus file search, and a
wiki built with Astro-Han's karpathy-llm-wiki skill. The module sets up each arm's basis and task files, collects the
answers into the gates' format, harvests tokens and isolation breaches from the subagent transcripts, and prints the
table. It never calls a model. The fixtures are synthetic text about gardening (CLAUDE.md rule 3).
Run: python3 -m unittest discover -s evals/tests
"""
from __future__ import annotations
import json
import sys
import tempfile
import unittest
from pathlib import Path
from typing import Any
ROOT = Path(__file__).resolve().parents[2]
sys.path.insert(0, str(ROOT / "evals"))
import direction_probe as dp # noqa: E402
TOMATO = "# Tomatoes\nTomatoes need full sun.\nWater them every morning.\n"
BASIL = "# Basil\nBasil likes warm soil.\n"
MINT = "# Mint\nMint spreads through runners.\n"
NOT_IN_CUT = "# Cactus\nCacti need little water.\n"
MD_SET: dict[str, Any] = {
"schema": "okf-wiki/multidoc-set/1",
"questions": [
{"id": "d01", "question": "What do tomatoes and basil need?",
"answer_keys": [{"file": "tomato.md", "text": "SECRETKEYSUN"}],
"evidence": [{"file": "tomato.md", "span": "SECRETSPANTOMATO"}]},
{"id": "d02", "question": "A question outside the probe.",
"answer_keys": [{"file": "mint.md", "text": "OTHERKEY"}],
"evidence": [{"file": "mint.md", "span": "OTHERSPAN"}]},
],
}
WD_SET: dict[str, Any] = {
"schema": "okf-wiki/wholedoc-set/1",
"questions": [
{"id": "d01", "type": "whole", "file": "tomato.md", "question": "What does the tomato page say overall?",
"answer_keys": [{"text": "SECRETWHOLEKEY"}], "evidence": [{"span": "SECRETWHOLESPAN"}]},
{"id": "d02", "type": "count", "file": "basil.md", "question": "How many basil lines are there?",
"count": 424242, "item_pattern": "SECRETPATTERN", "evidence": [{"span": "SECRETCOUNTSPAN"}]},
],
}
SECRETS = ("SECRETKEYSUN", "SECRETSPANTOMATO", "SECRETWHOLEKEY", "SECRETWHOLESPAN", "424242", "SECRETPATTERN",
"SECRETCOUNTSPAN", "OTHERKEY", "A question outside the probe")
class Fixture:
def __init__(self, root: Path) -> None:
self.root = root
self.sources = root / "kilde"
self.sources.mkdir()
for name, text in (("tomato.md", TOMATO), ("basil.md", BASIL), ("mint.md", MINT), ("cactus.md", NOT_IN_CUT)):
(self.sources / name).write_text(text, encoding="utf-8")
self.names = ["basil.md", "mint.md", "tomato.md"]
self.md_set = root / "md.json"
self.md_set.write_text(json.dumps(MD_SET), encoding="utf-8")
self.wd_set = root / "wd.json"
self.wd_set.write_text(json.dumps(WD_SET), encoding="utf-8")
self.wiki = root / "wiki-src"
self.wiki.mkdir()
(self.wiki / "index.md").write_text("# Index\n* [Tomatoes](tomatoes.md)\n", encoding="utf-8")
(self.wiki / "tomatoes.md").write_text("Tomatoes need full sun.\n", encoding="utf-8")
self.run = root / "run"
def setup(self, arm: str, **extra: Path) -> list[Path]:
return dp.setup_arm(run=self.run, arm=arm, sources=self.sources, names=self.names, md_set=self.md_set,
md_ids=["d01"], wd_set=self.wd_set, **extra)
class FetchAstroTest(unittest.TestCase):
def test_raw_file_keeps_the_name_and_the_body_byte_exact_under_the_template_header(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
fx = Fixture(Path(tmp))
project = Path(tmp) / "astro"
written = dp.fetch_astro(sources=fx.sources, names=fx.names, project=project, topic="garden",
collected="2026-09-27")
self.assertEqual([p.name for p in written], fx.names)
raw = (project / "raw" / "garden" / "tomato.md").read_text(encoding="utf-8")
self.assertTrue(raw.startswith("# tomato\n\n> Source: kilde/tomato.md\n> Collected: 2026-09-27\n"
"> Published: Unknown\n\n"))
self.assertTrue(raw.endswith(TOMATO))
self.assertFalse((project / "raw" / "garden" / "cactus.md").exists())
def test_an_existing_raw_file_is_never_overwritten(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
fx = Fixture(Path(tmp))
project = Path(tmp) / "astro"
dp.fetch_astro(sources=fx.sources, names=fx.names, project=project, topic="garden", collected="2026-09-27")
with self.assertRaises(dp.ProbeError):
dp.fetch_astro(sources=fx.sources, names=fx.names, project=project, topic="garden",
collected="2026-09-27")
class SetupTest(unittest.TestCase):
def test_file_search_basis_holds_exactly_the_cut(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
fx = Fixture(Path(tmp))
fx.setup("filsok")
basis = fx.run / "arms" / "filsok"
self.assertEqual(sorted(p.name for p in basis.iterdir()), fx.names)
self.assertEqual((basis / "tomato.md").read_text(encoding="utf-8"), TOMATO)
def test_one_task_per_probe_question_and_none_for_the_rest(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
fx = Fixture(Path(tmp))
tasks = fx.setup("filsok")
self.assertEqual(sorted(p.name for p in tasks), ["md-d01.md", "wd-d01.md", "wd-d02.md"])
self.assertTrue(all(p.parent == fx.run / "tasks" / "filsok" for p in tasks))
def test_a_task_carries_the_question_the_basis_and_the_answer_path_but_nothing_from_the_key(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
fx = Fixture(Path(tmp))
tasks = {p.name: p.read_text(encoding="utf-8") for p in fx.setup("filsok")}
md = tasks["md-d01.md"]
self.assertIn("What do tomatoes and basil need?", md)
self.assertIn(str(fx.run / "arms" / "filsok"), md)
self.assertIn(str(fx.run / "answers" / "filsok" / "md-d01.json"), md)
self.assertIn('"count"', tasks["wd-d02.md"])
self.assertNotIn('"count"', md)
for text in tasks.values():
for secret in SECRETS:
self.assertNotIn(secret, text)
def test_the_wiki_arm_gets_the_wiki_and_the_cut_and_the_router_order(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
fx = Fixture(Path(tmp))
tasks = fx.setup("wiki", wiki=fx.wiki)
basis = fx.run / "arms" / "wiki"
self.assertEqual(sorted(p.name for p in (basis / "kilde").iterdir()), fx.names)
self.assertTrue((basis / "wiki" / "index.md").is_file())
text = tasks[0].read_text(encoding="utf-8")
self.assertLess(text.index("index.md"), text.index("x-locator"))
def test_the_astro_arm_gets_raw_and_wiki_from_the_built_project(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
fx = Fixture(Path(tmp))
project = Path(tmp) / "astro"
dp.fetch_astro(sources=fx.sources, names=fx.names, project=project, topic="garden", collected="2026-09-27")
(project / "wiki").mkdir()
(project / "wiki" / "index.md").write_text("# Knowledge Base Index\n", encoding="utf-8")
tasks = fx.setup("astro", astro=project)
basis = fx.run / "arms" / "astro"
self.assertTrue((basis / "raw" / "garden" / "basil.md").is_file())
self.assertTrue((basis / "wiki" / "index.md").is_file())
self.assertIn("Raw", tasks[0].read_text(encoding="utf-8"))
def test_an_arm_without_its_basis_or_an_existing_arm_is_refused(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
fx = Fixture(Path(tmp))
with self.assertRaises(dp.ProbeError):
fx.setup("wiki")
fx.setup("filsok")
with self.assertRaises(dp.ProbeError):
fx.setup("filsok")
with self.assertRaises(dp.ProbeError):
fx.setup("grep-only")
def test_a_probe_id_missing_from_the_set_is_refused(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
fx = Fixture(Path(tmp))
with self.assertRaises(dp.ProbeError):
dp.setup_arm(run=fx.run, arm="filsok", sources=fx.sources, names=fx.names, md_set=fx.md_set,
md_ids=["d09"], wd_set=fx.wd_set)
class CollectTest(unittest.TestCase):
def test_answers_are_gathered_per_gate_with_ids_from_the_file_names(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
run = Path(tmp)
out = run / "answers" / "wiki"
out.mkdir(parents=True)
(out / "md-d01.json").write_text(json.dumps({"id": "wrong", "answer": "a", "citations": []}))
(out / "wd-d02.json").write_text(json.dumps({"answer": "7", "count": 7, "citations": []}))
(out / "wd-d03.json").write_text("not json")
md_path, wd_path, problems = dp.collect(run, "wiki")
md = json.loads(md_path.read_text(encoding="utf-8"))
wd = json.loads(wd_path.read_text(encoding="utf-8"))
self.assertEqual(md, {"arm": "wiki", "answers": [{"id": "d01", "answer": "a", "citations": []}]})
self.assertEqual(wd["answers"], [{"id": "d02", "answer": "7", "count": 7, "citations": []}])
self.assertEqual(problems, ["wd-d03.json: not a JSON object"])
def transcript(lines: list[dict[str, Any]]) -> str:
return "".join(json.dumps(line) + "\n" for line in lines)
def assistant(msg_id: str, tokens: tuple[int, int, int, int], *tools: tuple[str, dict[str, Any]]) -> dict[str, Any]:
i, cc, cr, o = tokens
content = [{"type": "tool_use", "name": name, "input": inp} for name, inp in tools]
return {"type": "assistant", "message": {"id": msg_id, "content": content, "usage": {
"input_tokens": i, "cache_creation_input_tokens": cc, "cache_read_input_tokens": cr, "output_tokens": o}}}
class HarvestTest(unittest.TestCase):
def write(self, folder: Path, name: str, lines: list[dict[str, Any]]) -> None:
(folder / name).write_text(transcript(lines), encoding="utf-8")
def test_tokens_count_each_message_once_and_breaches_are_named(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
run, folder = Path(tmp) / "run", Path(tmp) / "subagents"
folder.mkdir()
task = run / "tasks" / "wiki" / "md-d01.md"
basis = run / "arms" / "wiki"
answer = run / "answers" / "wiki" / "md-d01.json"
first = {"type": "user", "message": {"content": f"Read {task} and follow it."}}
self.write(folder, "agent-a.jsonl", [
first,
assistant("m1", (2, 100, 1000, 5), ("Read", {"file_path": str(task)})),
assistant("m1", (2, 100, 1000, 5)),
assistant("m2", (1, 10, 2000, 50), ("Grep", {"pattern": "sun", "path": str(basis / "kilde")}),
("Glob", {"pattern": "*.md", "path": str(basis / "wiki")})),
assistant("m3", (1, 0, 3000, 80), ("Write", {"file_path": str(answer), "content": "{}"}),
("SubagentHandback", {"message": "done"})),
])
self.write(folder, "agent-b.jsonl", [
{"type": "user", "message": {"content": [{"type": "text", "text": f"Read {run}/tasks/filsok/wd-d02.md"}]}},
assistant("n1", (1, 1, 1, 1), ("Read", {"file_path": str(run / "arms" / "wiki" / "wiki" / "x.md")}),
("Grep", {"pattern": "sun"}), ("Bash", {"command": "ls"}),
("Read", {"file_path": str(run / "arms" / "filsok" / ".." / "wiki" / "y.md")})),
])
self.write(folder, "agent-c.jsonl", [{"type": "user", "message": {"content": "Something else."}}])
rows = dp.harvest(folder, run)
self.assertEqual(sorted(rows), ["filsok/wd-d02", "wiki/md-d01"])
wiki = rows["wiki/md-d01"]
self.assertEqual(wiki["tokens"], 1107 + 2061 + 3081)
self.assertEqual(wiki["breaches"], [])
self.assertEqual(wiki["transcript"], "agent-a.jsonl")
breaches = rows["filsok/wd-d02"]["breaches"]
self.assertEqual(len(breaches), 4)
self.assertTrue(any(b.startswith("Bash") for b in breaches))
self.assertTrue(any(b.startswith("Grep without a path") for b in breaches))
def test_bash_is_read_only_search_that_starts_in_the_basis(self) -> None:
basis = Path("/runs/r/arms/filsok")
task, answer = Path("/runs/r/tasks/filsok/md-d01.md"), Path("/runs/r/answers/filsok/md-d01.json")
allowed = [
f"cd {basis} && grep -rn 'full sun' .",
f"cd {basis} && grep -n sun tomato.md | head -5",
f"cd {basis} && find . -name '*.md' | sort && wc -l tomato.md 2>/dev/null",
f"cd '{basis}' && sed -n '1,40p' tomato.md",
f"cd {basis}/kilde && ls",
]
refused = [
"grep -rn sun /runs/r/arms/filsok",
f"cd {basis} && cat ../wiki/x.md",
f"cd {basis} && grep sun /runs/r/tasks/filsok/other.md",
f"cd {basis} && rm tomato.md",
f"cd {basis} && grep sun tomato.md > copy.txt",
f"cd {basis} && find . -name '*.md' -exec cat {{}} ;",
f"cd {basis} && sed -i 's/a/b/' tomato.md",
f"cd {basis} && cat $(echo /etc/hosts)",
f"cd {basis} && cd / && ls",
f"cd {basis} && ls ~",
"cd /runs/r/arms/wiki && ls",
]
for command in allowed:
self.assertEqual(dp.breaches_of([("Bash", {"command": command})], basis, task, answer), [], command)
for command in refused:
self.assertEqual(len(dp.breaches_of([("Bash", {"command": command})], basis, task, answer)), 1, command)
def test_the_task_names_bash_as_the_search_tool_and_its_rule(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
fx = Fixture(Path(tmp))
text = fx.setup("filsok")[0].read_text(encoding="utf-8")
self.assertIn(f"cd {fx.run / 'arms' / 'filsok'} && ", text)
self.assertIn("grep", text)
def test_two_transcripts_for_one_task_keep_the_latest_and_report_the_rerun(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
run, folder = Path(tmp) / "run", Path(tmp) / "subagents"
folder.mkdir()
task = run / "tasks" / "filsok" / "md-d01.md"
first = {"type": "user", "timestamp": "2026-09-27T10:00:00Z", "message": {"content": f"Read {task}"}}
later = {"type": "user", "timestamp": "2026-09-27T11:00:00Z", "message": {"content": f"Read {task}"}}
self.write(folder, "agent-old.jsonl", [first, assistant("o", (1, 1, 1, 1))])
self.write(folder, "agent-new.jsonl", [later, assistant("n", (2, 2, 2, 2))])
rows = dp.harvest(folder, run)
self.assertEqual(rows["filsok/md-d01"]["transcript"], "agent-new.jsonl")
self.assertEqual(rows["filsok/md-d01"]["tokens"], 8)
self.assertEqual(rows["filsok/md-d01"]["runs"], 2)
def results(arm: str, correct: dict[str, bool], kind: str) -> dict[str, Any]:
rows = []
for qid, ok in correct.items():
row: dict[str, Any] = {"id": qid, "answered": True, "answer_ok": ok, "source_ok": True, "correct": ok}
if kind == "wd":
row["type"] = "whole" if qid in ("d01", "d02") else "count"
rows.append(row)
return {"arm": arm, "set_sha256": "x", "results": rows}
class ScoreTest(unittest.TestCase):
def test_groups_are_counted_from_the_gate_results_and_pairs_compared(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
run = Path(tmp)
res = run / "results"
res.mkdir()
md = {"filsok": {"d01": True, "d02": False, "d99": True}, "wiki": {"d01": True, "d02": True, "d99": False}}
wd = {"filsok": {"d01": True, "d02": False, "d03": True}, "wiki": {"d01": False, "d02": True, "d03": True}}
for arm in ("filsok", "wiki"):
(res / f"{arm}.md.json").write_text(json.dumps(results(arm, md[arm], "md")))
(res / f"{arm}.wd.json").write_text(json.dumps(results(arm, wd[arm], "wd")))
(run / "tokens.json").write_text(json.dumps({
"filsok/md-d01": {"tokens": 100, "breaches": [], "transcript": "a", "runs": 1},
"wiki/md-d01": {"tokens": 300, "breaches": [], "transcript": "b", "runs": 1}}))
table = dp.score(run, ["filsok", "wiki"], md_ids=["d01", "d02"])
self.assertEqual(table["arms"]["filsok"]["groups"]["flerdokument"], [1, 2])
self.assertEqual(table["arms"]["wiki"]["groups"]["flerdokument"], [2, 2])
self.assertEqual(table["arms"]["wiki"]["groups"]["helt dokument"], [1, 2])
self.assertEqual(table["arms"]["filsok"]["groups"]["telling"], [1, 1])
self.assertEqual(table["arms"]["wiki"]["total"], [4, 5])
self.assertEqual(table["arms"]["wiki"]["tokens"], {"sum": 300, "questions": 1})
pair = table["pairs"]["wiki vs filsok"]["flerdokument"]
self.assertEqual(pair, {"both": 1, "only wiki": 1, "only filsok": 0, "neither": 0})
self.assertEqual(table["pairs"]["wiki vs filsok"]["helt dokument"]["only filsok"], 1)
self.assertIn("| wiki |", dp.render_table(table))
def test_exact_mcnemar_is_the_two_sided_binomial_over_the_discordant_pairs(self) -> None:
self.assertEqual(dp.mcnemar_exact(0, 0), 1.0)
self.assertEqual(dp.mcnemar_exact(2, 0), 0.5)
self.assertAlmostEqual(dp.mcnemar_exact(6, 0), 0.03125)
self.assertAlmostEqual(dp.mcnemar_exact(1, 5), 0.21875)
self.assertEqual(dp.mcnemar_exact(3, 3), 1.0)
def test_every_pair_carries_its_mcnemar_p_over_all_groups(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
run = Path(tmp)
res = run / "results"
res.mkdir()
md = {"filsok": {"d01": False, "d02": False}, "wiki": {"d01": True, "d02": True}}
for arm in ("filsok", "wiki"):
(res / f"{arm}.md.json").write_text(json.dumps(results(arm, md[arm], "md")))
(res / f"{arm}.wd.json").write_text(json.dumps(results(arm, {}, "wd")))
table = dp.score(run, ["filsok", "wiki"], md_ids=["d01", "d02"])
self.assertEqual(table["mcnemar"]["wiki vs filsok"], {"only wiki": 2, "only filsok": 0, "p": 0.5})
def test_a_missing_result_file_is_a_failure_not_a_zero(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
run = Path(tmp)
(run / "results").mkdir()
with self.assertRaises(dp.ProbeError):
dp.score(run, ["filsok"], md_ids=["d01"])
if __name__ == "__main__":
unittest.main()

View file

@ -0,0 +1,609 @@
"""Tests for the P1 method trial's build side (TRIAL, branch p1-metodeprove, not v1 code).
Covers the two trial agent types, the splitter (evals/p1/split.py), the D map and register (dmap.py), K packing
(kpack.py) and assembly with K+ (assemble.py). None of these call a model. The fixtures are synthetic text about
gardening and bread (CLAUDE.md rule 3: no text from the Claude Code documentation is committed).
Run: python3 -m unittest discover -s evals/tests -p 'test_p1_build.py'
"""
from __future__ import annotations
import hashlib
import json
import re
import sys
import tempfile
import unittest
from pathlib import Path
ROOT = Path(__file__).resolve().parents[2]
sys.path.insert(0, str(ROOT))
sys.path.insert(0, str(ROOT / "evals"))
from okfwiki import audit, claims, view # noqa: E402
from p1 import assemble, dmap, kpack, split # noqa: E402
READER_SHA256 = "13a93bc5a9e87a7282752316ac670664cbdca321b0dac9cdaf92d9e74c383445"
BANNED = ("anthropic", "openai", "urllib", "http.client", "requests", "subprocess", "socket")
DOC_TOMATO = """# Growing tomatoes
Source: https://example.org/garden/tomatoes
How to grow tomatoes in a small garden.
## Soil
Tomatoes need deep, loose soil with `compost` mixed in.
Water the soil every morning.
* **Sun**: full sun for eight hours.
* **Space**: forty centimetres between plants,
more in a wet summer.
```bash
# not a heading
mix compost
```
| Month | Task |
|---|---|
| May | plant |
## Overview
The Kitchen Garden keeps tomatoes and basil together.
### Pruning
Remove side shoots once a week.
"""
DOC_BASIL = """# Growing basil
Source: https://example.org/garden/basil
## Soil
Basil likes warm soil and the Kitchen Garden.
Plant Sweet Genovese early.
## Watering
Water basil with `compost` tea when the leaves droop.
"""
REL_1 = """## 1.2.3
- Added the `compost` command to the Kitchen Garden planner.
- Fixed watering reminders.
"""
REL_2 = """## 1.10.0
- The Kitchen Garden planner now shows `compost` levels.
"""
REL_3 = """## 1.9.2
- Basil seedlings get their own tray.
"""
def runs8(text: str) -> set[tuple[str, ...]]:
words = re.findall(r"[a-z0-9]+", text.lower())
return {tuple(words[i:i + 8]) for i in range(len(words) - 7)}
def before_link_rule(found: list[audit.Finding]) -> list[audit.Finding]:
"""The P1 trial predates criterion 6 as structure (operator 04.10.2026), so its link-rule findings are left out."""
return [f for f in found if f.check != "link-rule"]
class Fixture(unittest.TestCase):
def setUp(self) -> None:
self._tmp = tempfile.TemporaryDirectory()
self.tmp = Path(self._tmp.name)
self.kilde = self.tmp / "wiki" / "kilde"
self.kilde.mkdir(parents=True)
files = {
"docs-en-tomatoes.md": DOC_TOMATO,
"docs-en-basil.md": DOC_BASIL,
"release-v1.2.3.md": REL_1,
"release-v1.10.0.md": REL_2,
"release-v1.9.2.md": REL_3,
}
for name, text in files.items():
(self.kilde / name).write_text(text, encoding="utf-8")
def tearDown(self) -> None:
self._tmp.cleanup()
# ---------------------------------------------------------------------------------------------- step 1: agents
class AgentFilesTest(unittest.TestCase):
def frontmatter(self, text: str) -> dict[str, str]:
self.assertTrue(text.startswith("---\n"))
head = text[4:text.index("\n---\n", 4)]
return dict(line.split(": ", 1) for line in head.splitlines())
def test_two_trial_agents_have_narrow_tools_and_a_trial_marker(self) -> None:
for name in ("p1-compiler", "p1-consolidator"):
text = (ROOT / ".claude" / "agents" / f"{name}.md").read_text(encoding="utf-8")
fm = self.frontmatter(text)
self.assertEqual(fm["name"], name)
self.assertEqual(fm["tools"], "Read, Write")
self.assertEqual(fm["model"], "opus")
body = text[text.index("\n---\n", 4) + 5:].strip()
self.assertTrue(body.startswith("TRIAL"), name)
fixture = DOC_TOMATO + DOC_BASIL + REL_1 + REL_2 + REL_3
self.assertEqual(runs8(body) & runs8(fixture), set(), name)
def test_the_reader_is_unchanged(self) -> None:
shipped = (ROOT / "agents" / "okf-wiki-reader.md").read_bytes()
self.assertEqual(hashlib.sha256(shipped).hexdigest(), READER_SHA256)
self.assertEqual(shipped, (ROOT / ".claude" / "agents" / "okf-wiki-reader.md").read_bytes())
def test_compiler_names_the_rules_it_must_keep(self) -> None:
body = (ROOT / ".claude" / "agents" / "p1-compiler.md").read_text(encoding="utf-8").lower()
for idea in ("first turn", "never read", "release note", "four tests", "json"):
self.assertIn(idea, body, idea)
def test_consolidator_keeps_every_label(self) -> None:
body = (ROOT / ".claude" / "agents" / "p1-consolidator.md").read_text(encoding="utf-8").lower()
for idea in ("every footnote label", "no new label", "json"):
self.assertIn(idea, body, idea)
# ---------------------------------------------------------------------------------------------- step 1: split
class OrderTest(unittest.TestCase):
def test_docs_first_then_release_notes_by_version_tuple(self) -> None:
names = ["release-v0.2.34.md", "docs-en-b.md", "release-v0.2.9.md", "docs-en-a.md", "release-v1.0.0.md"]
self.assertEqual(split.order(names), ["docs-en-a.md", "docs-en-b.md", "release-v0.2.9.md",
"release-v0.2.34.md", "release-v1.0.0.md"])
def test_an_unknown_name_is_refused(self) -> None:
with self.assertRaises(split.RunError):
split.order(["notes.md"])
class BlocksTest(Fixture):
def test_fence_aware_blocks_with_spans(self) -> None:
got = split.blocks(self.kilde / "docs-en-tomatoes.md")
kinds = [(b.kind, b.start, b.end) for b in got]
# line 2 "Source:" and line 4 are paragraphs; headings are not blocks
self.assertIn(("paragraph", 2, 2), kinds)
self.assertIn(("paragraph", 8, 9), kinds)
self.assertIn(("item", 11, 11), kinds)
self.assertIn(("item", 12, 13), kinds) # a continuation line belongs to its item
self.assertIn(("code", 15, 18), kinds) # the "#" inside the fence is not a heading
self.assertIn(("table", 20, 22), kinds)
heads = {b.section for b in got}
self.assertIn("Growing tomatoes › Soil", heads)
self.assertIn("Growing tomatoes › Overview › Pruning", heads)
self.assertNotIn("not a heading", " ".join(heads))
def test_every_non_blank_non_heading_line_is_in_exactly_one_block(self) -> None:
path = self.kilde / "docs-en-tomatoes.md"
lines = path.read_text(encoding="utf-8").split("\n")
got = split.blocks(path)
covered: dict[int, int] = {}
for b in got:
for n in range(b.start, b.end + 1):
covered[n] = covered.get(n, 0) + 1
for n, raw in enumerate(lines, 1):
heading = raw.startswith("#") and not (15 <= n <= 18)
if raw.strip() and not heading:
self.assertEqual(covered.get(n), 1, (n, raw))
if heading:
self.assertNotIn(n, covered, (n, raw))
def test_release_note_blocks_carry_the_version(self) -> None:
got = split.blocks(self.kilde / "release-v1.10.0.md")
self.assertEqual({b.version for b in got}, {"1.10.0"})
self.assertEqual(split.blocks(self.kilde / "docs-en-basil.md")[0].version, None)
class DocsHeadingsTest(Fixture):
def test_titles_and_second_and_third_level_headings_of_docs_pages_only(self) -> None:
got = split.docs_headings(self.kilde)
texts = [(h.file, h.level, h.text) for h in got]
self.assertIn(("docs-en-tomatoes.md", 1, "Growing tomatoes"), texts)
self.assertIn(("docs-en-tomatoes.md", 3, "Pruning"), texts)
self.assertFalse(any(f.startswith("release-") for f, _, _ in texts))
self.assertFalse(any(t == "not a heading" for _, _, t in texts))
class PickTest(Fixture):
def blocks_doc(self, order: list[str]) -> dict[str, object]:
return split.blocks_document(self.kilde, order)
def test_three_concepts_by_the_stated_rules(self) -> None:
order = split.order([p.name for p in self.kilde.iterdir()])
picked = split.pick(self.blocks_doc(order), self.kilde, focus="docs-en-tomatoes.md")
names = [p.name for p in picked]
self.assertEqual(len(names), 3)
self.assertEqual(len(set(names)), 3)
self.assertEqual(names[0], "Soil") # the focus page's heading with the most blocks
# "Kitchen Garden" and "compost" are both in 2 release notes; the tie is broken by name
self.assertEqual(names[1], "Kitchen Garden")
self.assertEqual(names[2], "compost") # in 2 docs pages and 2 release notes, distinct from 1 and 2
def test_a_document_counts_for_a_term_only_where_it_stands_as_a_term(self) -> None:
# Found in session 2: the word "and", backticked once, was counted in every document where it is prose.
(self.kilde / "docs-en-zz.md").write_text("# Joining\n\n## Words\n\nUse `and` to join two beds.\n",
encoding="utf-8")
for v in ("2.0.0", "2.0.1", "2.0.2"):
(self.kilde / f"release-v{v}.md").write_text(f"## {v}\n\n- Tomatoes and basil and mint.\n",
encoding="utf-8")
order = split.order([p.name for p in self.kilde.iterdir()])
names = [p.name for p in split.pick(self.blocks_doc(order), self.kilde, focus="docs-en-tomatoes.md")]
self.assertNotIn("and", names)
self.assertEqual(names[1:], ["Kitchen Garden", "compost"])
def test_a_backticked_term_counts_its_length_after_trimming(self) -> None:
# Found in session 2: "` or `" is four characters with its spaces, and became the two-letter term "or".
got = split.terms("Use `compost` ` or ` ` ab ` ` mulch ` here.")
self.assertEqual(got, ["compost", "mulch"])
def test_the_focus_page_must_be_in_the_order(self) -> None:
with self.assertRaises(split.RunError):
split.pick(self.blocks_doc(["docs-en-basil.md"]), self.kilde, focus="docs-en-tomatoes.md")
class CliTest(Fixture):
def test_order_refuses_names_that_differ_from_the_l4_cut(self) -> None:
manifest = self.tmp / "m.json"
manifest.write_text(json.dumps({"cut": {"names": ["docs-en-basil.md", "release-v1.2.3.md"]}}), encoding="utf-8")
l4 = self.tmp / "l4"
l4.mkdir()
(l4 / "a.md").write_text("---\ntype: Concept\nsources:\n - id: a\n resource: /kilde/docs-en-basil.md\n"
"---\n\nx\n", encoding="utf-8")
out = self.tmp / "rekkefolge.txt"
self.assertEqual(split.main(["order", "--manifest", str(manifest), "--l4", str(l4), "--out", str(out)]), 2)
self.assertFalse(out.exists())
(l4 / "b.md").write_text("---\ntype: Concept\nsources:\n - id: b\n resource: /kilde/release-v1.2.3.md\n"
"---\n\nx\n", encoding="utf-8")
self.assertEqual(split.main(["order", "--manifest", str(manifest), "--l4", str(l4), "--out", str(out)]), 0)
self.assertEqual(out.read_text(encoding="utf-8").split(), ["docs-en-basil.md", "release-v1.2.3.md"])
def test_blocks_and_pick_refuse_to_overwrite(self) -> None:
order = self.tmp / "order.txt"
order.write_text("\n".join(split.order([p.name for p in self.kilde.iterdir()])) + "\n", encoding="utf-8")
out = self.tmp / "blocks.json"
self.assertEqual(split.main(["blocks", "--kilde", str(self.kilde), "--order", str(order), "--out", str(out)]), 0)
data = json.loads(out.read_text(encoding="utf-8"))
self.assertEqual(data["schema"], "okf-wiki/p1-blocks/1")
self.assertEqual(split.main(["blocks", "--kilde", str(self.kilde), "--order", str(order), "--out", str(out)]), 2)
picked = self.tmp / "begreper.txt"
args = ["pick", "--blocks", str(out), "--kilde", str(self.kilde), "--focus", "docs-en-tomatoes.md",
"--out", str(picked)]
self.assertEqual(split.main(args), 0)
self.assertEqual(len(picked.read_text(encoding="utf-8").strip().splitlines()), 3)
self.assertEqual(split.main(args), 2)
def test_total_counts_blocks_in_every_file(self) -> None:
expected = sum(len(split.blocks(p)) for p in self.kilde.iterdir())
self.assertEqual(split.blocks_total(self.kilde), expected)
# ---------------------------------------------------------------------------------------------- step 2: D
class DMapTest(Fixture):
def build(self) -> Path:
order = split.order([p.name for p in self.kilde.iterdir()])
out = self.tmp / "D"
dmap.build(split.blocks_document(self.kilde, order), self.kilde.parent, out, at="2026-10-02T12:00:00Z")
return out
def test_d_passes_the_auditor_with_no_claims(self) -> None:
out = self.build()
self.assertEqual(before_link_rule(audit.audit(out, self.kilde.parent)), [])
found = claims.extract(out, self.kilde.parent)
self.assertEqual(found.claims, [])
self.assertTrue(found.exceptions)
def test_register_links_point_to_map_sections(self) -> None:
out = self.build()
page = (out / "r-kitchen-garden.md").read_text(encoding="utf-8")
self.assertIn("(k-docs-en-basil.md#soil)", page)
self.assertIn("(k-release-v1-2-3.md#1-2-3)", page)
def test_register_keeps_docs_headings_and_terms_in_two_documents_only(self) -> None:
out = self.build()
names = {p.name for p in out.glob("r-*.md")}
self.assertIn("r-pruning.md", names) # a docs heading (i), in one document
self.assertIn("r-compost.md", names) # a backticked term in several documents
self.assertNotIn("r-sweet-genovese.md", names) # a capitalised phrase in one document only
self.assertNotIn("r-overview.md", names) # a stop-listed heading fails test 2
def test_quotes_stand_on_their_lines_and_stay_strings(self) -> None:
out = self.build()
doc = audit.read_document(out / "r-compost.md", "r-compost.md")
assert doc.frontmatter is not None
for s in doc.frontmatter["sources"]:
self.assertIsInstance(s["x-quote"], str)
line = (self.kilde / Path(s["resource"]).name).read_text(encoding="utf-8").split("\n")[s["x-locator"]["line"] - 1]
self.assertIn(s["x-quote"], line)
self.assertIsInstance(doc.frontmatter["generated"]["at"], str)
def test_a_quote_like_no_survives_a_yaml_round_trip(self) -> None:
text = dmap.frontmatter({"type": "Concept", "sources": [{"id": "a", "x-quote": "no"}, {"id": "b", "x-quote": "1.10"}]})
path = self.tmp / "x.md"
path.write_text(text + "\n# x\n", encoding="utf-8")
fm = audit.read_document(path, "x.md").frontmatter
assert fm is not None
self.assertEqual([s["x-quote"] for s in fm["sources"]], ["no", "1.10"])
def test_the_log_records_the_thresholds_and_build_refuses_an_existing_target(self) -> None:
out = self.build()
log = (out / "log.md").read_text(encoding="utf-8")
self.assertIn("2 of the", log)
with self.assertRaises(dmap.RunError):
dmap.build({"order": [], "blocks": []}, self.kilde.parent, out, at="2026-10-02T12:00:00Z")
# ---------------------------------------------------------------------------------------------- step 3: K packing
def entry(eid: str, title: str, aliases: tuple[str, ...] = ()) -> dict[str, object]:
return {"id": eid, "title": title, "aliases": [title, *aliases], "kind": "skeleton", "merge_of": None,
"origin": None}
class KSkeletonTest(Fixture):
def test_the_skeleton_is_an_index_of_docs_headings_that_pass_test_2(self) -> None:
index = kpack.skeleton(self.kilde)
titles = {e["title"] for e in index["entries"]}
self.assertIn("Growing tomatoes", titles)
self.assertIn("Pruning", titles)
self.assertIn("Soil", titles)
self.assertNotIn("Overview", titles)
soil = [e for e in index["entries"] if e["title"] == "Soil"]
self.assertEqual(len(soil), 1) # the same heading in two pages is one entry
self.assertEqual(soil[0]["origin"]["file"], "docs-en-basil.md") # pages in name order, first one wins
self.assertFalse(any(p.suffix == ".md" for p in self.tmp.rglob("*") if p.parent.name != "kilde"))
def test_norm(self) -> None:
self.assertEqual(kpack.norm("The `Compost` bins"), "compost bin")
self.assertEqual(kpack.norm("settings.json"), "settings.json")
self.assertEqual(kpack.norm(" Kitchen Garden! "), "kitchen garden")
self.assertEqual(kpack.norm("CLI flags"), "cli flag")
class KIdentityTest(unittest.TestCase):
def setUp(self) -> None:
self.index = {"entries": [entry("kitchen-garden-planner", "Kitchen Garden planner"),
entry("compost", "compost", ("compost heap",))]}
def frag(self, concept: str, status: str = "new", file: str = "docs-en-tomatoes.md", eid: str | None = None
) -> dict[str, object]:
f: dict[str, object] = {"concept": concept, "status": status, "anchor": {"file": file, "start": 8, "end": 9}}
if eid:
f["id"] = eid
return f
def test_outcomes(self) -> None:
r = kpack.resolve
self.assertEqual(r(self.frag("x", "existing", eid="compost"), self.index), ("existing", "compost"))
self.assertEqual(r(self.frag("Kitchen Garden planners"), self.index), ("existing", "kitchen-garden-planner"))
self.assertEqual(r(self.frag("Compost heap"), self.index), ("existing", "compost"))
self.assertEqual(r(self.frag("Kitchen Garden planner tool"), self.index),
("merge-candidate", "kitchen-garden-planner"))
self.assertEqual(r(self.frag("Garden hose"), self.index), ("new", None))
self.assertEqual(r(self.frag("Garden hose", file="release-v1.2.3.md"), self.index), ("rejected", "release-note-new"))
self.assertEqual(r(self.frag("Kitchen Garden planner tool", file="release-v1.2.3.md"), self.index),
("rejected", "release-note-new"))
self.assertEqual(r(self.frag("Troubleshooting"), self.index), ("rejected", "meta"))
def test_relevant_lines_by_phrase(self) -> None:
got = kpack.relevant_lines(self.index, "We keep the compost heaps behind the shed.")
self.assertEqual([e["id"] for e in got], ["compost"])
self.assertEqual(kpack.relevant_lines(self.index, "Nothing about gardens here."), [])
class KPackTest(Fixture):
def blocks_doc(self) -> dict[str, object]:
return split.blocks_document(self.kilde, split.order([p.name for p in self.kilde.iterdir()]))
def test_packs_of_five(self) -> None:
self.assertEqual([len(p) for p in kpack.packs([f"n{i}" for i in range(19)], 5)], [5, 5, 5, 4])
def test_a_large_pack_is_split_into_parts_under_the_read_cap(self) -> None:
big = "".join(f"Line {i} says the tomatoes are fine today and tomorrow, and the basil too.\n\n"
for i in range(1110))
(self.kilde / "docs-en-big.md").write_text("# Big\n\n" + big, encoding="utf-8")
doc = split.blocks_document(self.kilde, ["docs-en-big.md"])
parts = kpack.render_input(1, ["docs-en-big.md"], doc, [], self.kilde)
self.assertGreater(sum(len(p) for p in parts), 130_000)
self.assertEqual(len(parts), 3)
self.assertTrue(all(len(p) <= kpack.PART_CHARS for p in parts))
self.assertIn("docs-en-big.md › Big › 3–3", parts[0])
def test_validate(self) -> None:
doc = self.blocks_doc()
pack = ["docs-en-tomatoes.md", "release-v1.2.3.md"]
good = {"concept": "Soil", "status": "existing", "id": "soil", "text": "Tomatoes need deep, loose soil.",
"anchor": {"file": "docs-en-tomatoes.md", "start": 8, "end": 9}}
cases = {
"anchor-crosses-blocks": {**good, "anchor": {"file": "docs-en-tomatoes.md", "start": 8, "end": 11}},
"quote-missing": {**good, "text": "Plants stand 40 cm apart."},
"quote-not-in-span": {**good, "text": "Keep 40 cm.", "quote": "forty centimetres"},
"sentence-count": {**good, "text": "One. Two. Three. Four."},
"tests-missing": {**good, "status": "new", "concept": "Loose soil"},
"anchor-outside-pack": {**good, "anchor": {"file": "docs-en-basil.md", "start": 7, "end": 7}},
"malformed": {"concept": "Soil"},
}
no_text = {**good, "text": ""}
accepted, rejected = kpack.validate([good, no_text, *cases.values()], pack, doc, self.kilde)
self.assertEqual(len(accepted), 1)
self.assertEqual(rejected["sentence-count"], 2) # the empty text and the four sentences
for reason in cases:
self.assertGreaterEqual(rejected[reason], 1, reason)
ok = {**good, "text": "Plants stand forty centimetres apart.", "anchor": {"file": "docs-en-tomatoes.md",
"start": 12, "end": 13}, "quote": "forty centimetres"}
accepted, rejected = kpack.validate([ok], pack, doc, self.kilde)
self.assertEqual((len(accepted), sum(rejected.values())), (1, 0))
class KCliTest(Fixture):
def test_skeleton_pack_and_accept(self) -> None:
work = self.tmp / "work"
blocks = self.tmp / "blocks.json"
order = split.order([p.name for p in self.kilde.iterdir()])
blocks.write_text(json.dumps(split.blocks_document(self.kilde, order)), encoding="utf-8")
index = work / "K-work" / "index.json"
self.assertEqual(kpack.main(["skeleton", "--kilde", str(self.kilde), "--out", str(index)]), 0)
self.assertEqual(kpack.main(["skeleton", "--kilde", str(self.kilde), "--out", str(index)]), 2)
root = str(self.kilde.parent)
self.assertEqual(kpack.main(["pack", "1", "--blocks", str(blocks), "--index", str(index), "--kilde-root", root,
"--work", str(work)]), 0)
self.assertTrue((work / "K-oppgaver" / "pack-1-part-1.md").is_file())
fragments = {"fragments": [
{"concept": "Soil", "status": "existing", "id": "soil", "text": "Tomatoes need deep, loose soil.",
"anchor": {"file": "docs-en-tomatoes.md", "start": 8, "end": 9}},
{"concept": "Kitchen Garden", "status": "new", "text": "The Kitchen Garden keeps tomatoes and basil.",
"anchor": {"file": "docs-en-tomatoes.md", "start": 26, "end": 26},
"tests": {"topic_shape": "a named planner", "citation": "See the Kitchen Garden reference"}},
{"concept": "Kitchen Garden", "status": "existing", "text": "The planner gained a command.",
"anchor": {"file": "release-v1.2.3.md", "start": 3, "end": 3}},
{"concept": "Watering reminders", "status": "new", "text": "Reminders were fixed.",
"anchor": {"file": "release-v1.2.3.md", "start": 4, "end": 4},
"tests": {"topic_shape": "x", "citation": "y"}},
]}
out = work / "K-fragments" / "pack-1.json"
out.parent.mkdir(parents=True)
out.write_text(json.dumps(fragments), encoding="utf-8")
self.assertEqual(kpack.main(["accept", "1", "--work", str(work), "--blocks", str(blocks), "--kilde-root", root]), 0)
acc = json.loads((work / "K-work" / "accepted.json").read_text(encoding="utf-8"))["fragments"]
self.assertEqual([a["outcome"] for a in acc], ["existing", "new", "existing"])
self.assertEqual(acc[2]["concept_id"], acc[1]["concept_id"]) # the release note enriches the new concept
self.assertEqual(acc[2]["version"], "1.2.3")
log = json.loads((work / "K-work" / "packs.json").read_text(encoding="utf-8"))
self.assertEqual(log["packs"][0]["rejected"], {"release-note-new": 1})
self.assertEqual(log["packs"][0]["proposed"], 4)
# PM 02.10: the ids of the index lines sent, and the accepted fragments whose concept was not among them
sent = log["packs"][0]["index_ids"]
self.assertIn("soil", sent)
self.assertNotIn(acc[1]["concept_id"], sent)
self.assertEqual(log["packs"][0]["not_sent"], {"total": 2, "by_outcome": {"existing": 1, "new": 1}})
# ---------------------------------------------------------------------------------------------- step 4: assembly
def accepted(cid: str, file: str, start: int, end: int, text: str, pack: int, version: str | None = None,
quote: str | None = None, quote_line: int | None = None) -> dict[str, object]:
return {"pack": pack, "position": 1, "file": file, "start": start, "end": end, "section": "-", "text": text,
"quote": quote, "quote_line": quote_line, "version": version, "concept_id": cid,
"outcome": "existing"}
class AssembleTest(Fixture):
def setUp(self) -> None:
super().setUp()
self.work = self.tmp / "work"
(self.work / "K-work").mkdir(parents=True)
index = {"entries": [
{**entry("soil", "Soil"), "origin": {"file": "docs-en-basil.md", "line": 4}},
{**entry("pruning", "Pruning"), "origin": {"file": "docs-en-tomatoes.md", "line": 28}},
{**entry("kitchen-garden", "Kitchen Garden"), "kind": "new"},
{**entry("kitchen-garden-planner", "Kitchen Garden planner"), "kind": "merge-candidate",
"merge_of": "kitchen-garden"},
]}
frags = [
accepted("soil", "docs-en-tomatoes.md", 8, 9, "Tomatoes need deep, loose soil.", 1),
accepted("kitchen-garden", "docs-en-tomatoes.md", 26, 26,
"The Kitchen Garden keeps tomatoes and basil together.", 1, quote="Kitchen Garden", quote_line=26),
accepted("kitchen-garden", "release-v1.2.3.md", 3, 3, "The planner gained a compost command.", 1, "1.2.3"),
accepted("soil", "docs-en-basil.md", 6, 7, "Basil likes warm soil near the Kitchen Garden.", 1),
accepted("kitchen-garden", "release-v1.10.0.md", 3, 3, "The planner shows compost levels.", 2, "1.10.0"),
accepted("kitchen-garden-planner", "release-v1.9.2.md", 3, 3, "Seedlings get their own tray.", 2, "1.9.2"),
]
(self.work / "K-work" / "index.json").write_text(json.dumps(index), encoding="utf-8")
(self.work / "K-work" / "accepted.json").write_text(json.dumps({"fragments": frags}), encoding="utf-8")
self.out = self.tmp / "K"
assemble.assemble(self.work, self.kilde.parent, self.out, at="2026-10-02T12:00:00Z")
def page(self, name: str) -> audit.Document:
return audit.read_document(self.out / name, name)
def test_the_bundle_passes_the_auditor(self) -> None:
self.assertEqual(before_link_rule(audit.audit(self.out, self.kilde.parent)), [])
found = claims.extract(self.out, self.kilde.parent)
self.assertEqual(claims.criterion_1(found)["total"], (6, 6))
def test_only_enriched_concepts_get_pages(self) -> None:
names = {p.name for p in self.out.glob("*.md")} - {"index.md", "log.md"}
self.assertEqual(names, {"soil.md", "kitchen-garden.md", "kitchen-garden-planner.md"})
fm = self.page("kitchen-garden-planner.md").frontmatter
assert fm is not None
self.assertEqual(fm["x-merge-candidate"], "kitchen-garden")
def test_history_in_version_order_without_version_numbers_in_the_text(self) -> None:
doc = self.page("kitchen-garden.md")
body = doc.body
self.assertLess(body.index("gained a compost command"), body.index("shows compost levels"))
prose = " ".join(b.text for b in audit.prose_blocks(doc))
self.assertEqual(view.version_numbers(audit.FOOTNOTE_REF.sub("", prose)), [])
fm = doc.frontmatter
assert fm is not None
versions = [s for s in fm["sources"] if view.is_version_heading(s)]
self.assertEqual(sorted(s["x-quote"] for s in versions), ["1.10.0", "1.2.3"])
self.assertEqual(fm["x-fragments"], 3)
def test_code_quotes_skip_structural_lines_and_are_marked(self) -> None:
fm = self.page("soil.md").frontmatter
assert fm is not None
first = fm["sources"][0]
self.assertEqual(first["x-locator"], {"line": 8})
self.assertEqual(first["x-quote-by"], "code")
self.assertEqual(first["x-span"], [8, 9])
kg = self.page("kitchen-garden.md").frontmatter
assert kg is not None
self.assertNotIn("x-quote-by", kg["sources"][0])
self.assertEqual(assemble.code_quote(["```bash", "<Note>", "|---|---|", "", " real line "], 10),
("real line", 14))
def test_a_concept_title_in_another_page_becomes_one_link(self) -> None:
self.assertIn("[Kitchen Garden](kitchen-garden.md)", self.page("soil.md").body)
class KPlusTest(AssembleTest):
def test_copy_tasks_and_apply(self) -> None:
kplus = self.tmp / "K+"
assemble.copy(self.out, kplus)
with self.assertRaises(assemble.RunError):
assemble.copy(self.out, kplus)
runs = assemble.consolidation_tasks(kplus, self.work, min_fragments=3, per_run=8)
self.assertEqual([r.pages for r in runs], [["kitchen-garden"]])
self.assertTrue(runs[0].task.is_file())
old = self.page("kitchen-garden.md").body
labels = sorted(set(audit.FOOTNOTE_REF.findall(old)))
good = "The Kitchen Garden keeps tomatoes and basil together, and its planner tracks compost " + "".join(
f"[^{x}]" for x in labels) + "."
dropped = "The Kitchen Garden keeps tomatoes and basil together[^" + labels[0] + "]."
added = good.replace(".", "[^zz].", 1)
bodies = self.tmp / "bodies.json"
bodies.write_text(json.dumps({"kitchen-garden": good, "soil": dropped}), encoding="utf-8")
self.assertEqual(assemble.apply(kplus, bodies), (1, 1))
self.assertIn("planner tracks compost", (kplus / "kitchen-garden.md").read_text(encoding="utf-8"))
self.assertEqual(before_link_rule(audit.audit(kplus, self.kilde.parent)), [])
bodies.write_text(json.dumps({"kitchen-garden": added}), encoding="utf-8")
self.assertEqual(assemble.apply(kplus, bodies), (0, 1))
class NoModelTest(unittest.TestCase):
def test_trial_modules_import_no_network_or_model_client(self) -> None:
for path in sorted((ROOT / "evals" / "p1").glob("*.py")):
source = path.read_text(encoding="utf-8")
for banned in BANNED:
self.assertNotIn(f"import {banned}", source, path.name)
self.assertNotIn(f"from {banned}", source, path.name)
if __name__ == "__main__":
unittest.main()

View file

@ -0,0 +1,366 @@
"""Tests for the P1 method trial's report side (TRIAL, branch p1-metodeprove, not v1 code).
Covers the measures (evals/p1/measure.py), the cost harvester (cost.py), the side-by-side page (sidebyside.py) and the
two verification scripts. None of these call a model. The fixtures are synthetic text about gardening (CLAUDE.md
rule 3: no text from the Claude Code documentation is committed).
Run: python3 -m unittest discover -s evals/tests -p 'test_p1_report.py'
"""
from __future__ import annotations
import json
import os
import subprocess
import sys
import tempfile
import unittest
from pathlib import Path
from typing import Any
import yaml
ROOT = Path(__file__).resolve().parents[2]
sys.path.insert(0, str(ROOT))
sys.path.insert(0, str(ROOT / "evals"))
from okfwiki import audit, view # noqa: E402
from p1 import cost, measure, sidebyside, split # noqa: E402
BANNED = ("anthropic", "openai", "urllib", "http.client", "requests", "subprocess", "socket")
DOC_A = """# Alpha
## Beds
Raised beds drain well.
They warm early in spring.
- Line one item.
- Line two item.
## Paths
Gravel paths stay dry.
"""
REL = """## 2.0.0
- Beds got labels.
"""
def src(sid: str, file: str, line: int, quote: str, span: tuple[int, int] | None = None) -> dict[str, Any]:
s: dict[str, Any] = {"id": sid, "resource": f"/kilde/{file}", "title": file, "x-locator": {"line": line},
"x-quote": quote}
if span:
s["x-span"] = list(span)
return s
def write_page(folder: Path, name: str, title: str, body: str, sources: list[dict[str, Any]],
**extra: Any) -> None:
fm = {"type": "Concept", "title": title, "description": title, "sources": sources, **extra,
"generated": {"by": "test", "at": "2026-10-02T12:00:00Z"}}
text = "---\n" + yaml.safe_dump(fm, sort_keys=False) + "---\n\n# " + title + "\n\n" + body + "\n"
folder.mkdir(parents=True, exist_ok=True)
(folder / name).write_text(text, encoding="utf-8")
class Fixture(unittest.TestCase):
def setUp(self) -> None:
self._tmp = tempfile.TemporaryDirectory()
self.tmp = Path(self._tmp.name)
self.root = self.tmp / "wiki"
self.kilde = self.root / "kilde"
self.kilde.mkdir(parents=True)
(self.kilde / "docs-en-a.md").write_text(DOC_A, encoding="utf-8")
(self.kilde / "release-v2.0.0.md").write_text(REL, encoding="utf-8")
self.order = ["docs-en-a.md", "release-v2.0.0.md"]
self.blocks = split.blocks_document(self.kilde, self.order)
self.bundle = self.tmp / "B"
write_page(self.bundle, "beds.md", "Beds",
"Raised beds drain well.[^b1] The beds have a heading.[^b2] Items begin.[^b3] Two items.[^b4]\n\n"
"## History\n\n- Beds got labels.[^r1][^v1]\n\n| a | b |",
[src("b1", "docs-en-a.md", 5, "Raised beds drain well.", (5, 6)),
src("b2", "docs-en-a.md", 3, "Beds", (3, 3)),
src("b3", "docs-en-a.md", 8, "Line one item.", (8, 13)),
src("b4", "docs-en-a.md", 9, "Line two item.", (8, 9)),
src("v1", "release-v2.0.0.md", 1, "2.0.0"),
src("r1", "release-v2.0.0.md", 3, "Beds got labels.", (3, 3)),
src("u1", "docs-en-a.md", 13, "Gravel paths stay dry.", (13, 13))],
**{"x-aliases": ["Beds", "Raised beds"]})
write_page(self.bundle, "labels.md", "Labels", "Beds got labels.[^r1][^v1] See also: [Beds](beds.md)",
[src("v1", "release-v2.0.0.md", 1, "2.0.0"), src("r1", "release-v2.0.0.md", 3, "Beds got labels.")])
(self.bundle / "index.md").write_text('---\nokf_version: "0.2"\n---\n\n# Concepts\n\n- [Beds](beds.md)\n'
'- [Labels](labels.md)\n', encoding="utf-8")
(self.bundle / "log.md").write_text("# Log\n\n## 2026-10-02\n* built\n", encoding="utf-8")
def tearDown(self) -> None:
self._tmp.cleanup()
# ---------------------------------------------------------------------------------------------- step 5: measures
class CoverageTest(Fixture):
def test_cited_entries_by_locator_line_only(self) -> None:
got = measure.coverage(self.bundle, self.root, self.blocks, self.order)
# covered: 5-6 (b1), 8 (b3's locator; its wide span is ignored), 9 (b4), release 3 (r1); not 13 (u1 uncited)
self.assertEqual((got.covered, got.total), (4, 5))
def test_a_stopped_variant_counts_only_the_sources_it_built(self) -> None:
got = measure.coverage(self.bundle, self.root, self.blocks, self.order[:1])
self.assertEqual((got.covered, got.total), (3, 4))
def test_per_pack_position(self) -> None:
got = measure.coverage(self.bundle, self.root, self.blocks, self.order, pack_size=5)
self.assertEqual(got.per_position, {1: (3, 4, 1), 2: (1, 1, 1)})
class StructureTest(Fixture):
def test_shares_precision_and_concepts_per_source(self) -> None:
s = measure.structure(self.bundle, self.root, self.blocks, self.order)
self.assertEqual(s["pages"], 2)
self.assertEqual(s["concepts_per_source"], {"docs-en-a.md": 1, "release-v2.0.0.md": 1})
# version sources are left out of the quote and precision counts
self.assertEqual(s["quotes"]["entries"], 6)
self.assertEqual(s["quotes"]["structural"], 1) # b2 quotes a heading line
self.assertEqual(s["precision"], {"line": 3, "block": 1, "section": 1, "document": 1})
self.assertEqual(s["islands"]["total"], sum(not v for v in view.related(
audit.load_bundle(self.bundle, self.root).concepts).values()))
def test_duplicates_by_norm_alias_and_jaccard_in_any_bundle(self) -> None:
b = self.tmp / "dups"
for name, title in (("a.md", "Beds"), ("b.md", "Bed"), ("c.md", "Raised beds"), ("d.md", "Gravel path"),
("e.md", "Gravel path rules"), ("f.md", "Tools")):
write_page(b, name, title, "x[^s].", [src("s", "docs-en-a.md", 5, "Raised")],
**({"x-aliases": ["Beds", "Raised beds"]} if name == "a.md" else {}))
pairs = measure.duplicates(audit.load_bundle(b, self.root).concepts)
self.assertIn(("a.md", "b.md", "norm"), pairs)
self.assertIn(("a.md", "c.md", "alias"), pairs)
self.assertIn(("d.md", "e.md", "jaccard"), pairs)
self.assertFalse(any("f.md" in p for p in pairs))
def test_a_wrong_root_raises(self) -> None:
with self.assertRaises(measure.RunError):
measure.structure(self.bundle, self.tmp / "nowhere", self.blocks, self.order)
class AuditRecordTest(Fixture):
def test_uncited_sentences_are_classed_by_cause_and_yaml_form_apart(self) -> None:
write_page(self.bundle, "odd.md", "Odd", "```bash\n\nPlain words here.\n\nSee also: [Beds](beds.md)",
[{"id": "y", "resource": "/kilde/docs-en-a.md", "x-locator": {"line": 5}, "x-quote": False}])
out = self.tmp / "resultat.json"
rec = measure.audit_record(self.bundle, self.root, "K", out)
self.assertEqual(rec["exit"], 1)
# labels.md in the shared fixture holds one more link lead-in ("See also: [Beds](beds.md)")
self.assertEqual(rec["uncited"], {"fence line": 1, "link lead-in": 2, "prose": 1, "table row": 1})
self.assertEqual(rec["yaml_form"], 1)
self.assertEqual(json.loads(out.read_text(encoding="utf-8"))["variants"]["K"]["exit"], 1)
with self.assertRaises(measure.RunError):
measure.audit_record(self.bundle, self.root, "K", out) # one record per variant
# ---------------------------------------------------------------------------------------------- step 6: cost
def msg(mid: str, inp: int, cr: int, cw: int, out: int, stop: str | None = "end_turn") -> dict[str, Any]:
return {"type": "assistant", "message": {"id": mid, "stop_reason": stop, "usage": {
"input_tokens": inp, "cache_read_input_tokens": cr, "cache_creation_input_tokens": cw, "output_tokens": out}}}
def jsonl(path: Path, lines: list[dict[str, Any]]) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text("\n".join(json.dumps(x) for x in lines) + "\n", encoding="utf-8")
class HarvestTest(unittest.TestCase):
def setUp(self) -> None:
self._tmp = tempfile.TemporaryDirectory()
self.proj = Path(self._tmp.name)
user = {"type": "user", "message": {"content": "task"}}
jsonl(self.proj / "s1.jsonl", [user, msg("m0", 5, 100, 10, 7)])
sub = self.proj / "s1" / "subagents"
# a streamed message id appears twice with growing output: counted once, at its max
jsonl(sub / "agent-a1.jsonl", [user, msg("x1", 3, 20_000, 2_000, 10, None), msg("x1", 3, 20_000, 2_000, 50),
msg("x2", 1, 22_000, 500, 900)])
(sub / "agent-a1.meta.json").write_text(json.dumps({"agentType": "p1-compiler", "description": "P1-K-pack1"}))
jsonl(sub / "agent-a2.jsonl", [user, msg("y1", 2, 19_000, 1_000, 5)])
(sub / "agent-a2.meta.json").write_text(json.dumps({"agentType": "p1-compiler", "description": "load test"}))
jsonl(sub / "agent-a3.jsonl", [user, msg("z1", 1, 1, 1, 1, None)])
(sub / "agent-a3.meta.json").write_text(json.dumps({"agentType": "p1-consolidator",
"description": "P1-K+-run1"}))
# a session not listed in okter.txt: its transcript is never opened
jsonl(self.proj / "s9" / "subagents" / "agent-b1.jsonl", [user, msg("q", 1, 1, 1, 1)])
(self.proj / "s9" / "subagents" / "agent-b1.meta.json").write_text(
json.dumps({"agentType": "p1-compiler", "description": "P1-K-pack2"}))
self.okter = self.proj / "okter.txt"
self.okter.write_text("s1 step8\ns1 step9\n", encoding="utf-8")
def tearDown(self) -> None:
self._tmp.cleanup()
def test_four_categories_turns_startup_and_attribution(self) -> None:
got = cost.harvest(self.proj, cost.sessions(self.okter))
self.assertEqual(got["sessions_read"], ["s1"])
runs = {r["unit"]: r for r in got["runs"]}
k = runs["pack1"]
self.assertEqual((k["variant"], k["input"], k["cache_read"], k["cache_write"], k["output"]),
("K", 4, 42_000, 2_500, 950))
self.assertEqual(k["turns"], 2)
self.assertEqual(k["startup"], 3 + 20_000 + 2_000)
self.assertEqual(runs["run1"]["variant"], "K+")
self.assertEqual(runs["run1"]["stop_reason"], None) # a stopped agent is listed, not dropped
self.assertEqual([u["description"] for u in got["unattributed"]], ["load test"])
self.assertEqual(got["main"]["s1"]["cache_read"], 100)
self.assertNotIn("pack2", runs)
def test_output_never_recorded_is_derived_from_the_next_messages_input_growth(self) -> None:
# Found in session 2: a long streamed message keeps only its first usage record (output 2, no stop
# reason). The next message's input side grew by that output plus the text injected between them (tool
# results and rendered attachments), counted high at 2.5 characters per token so the output is a lower bound.
def result(text: str) -> dict[str, Any]:
return {"type": "user", "message": {"content": [{"type": "tool_result", "content": text}]}}
sub = self.proj / "s1" / "subagents"
jsonl(sub / "agent-a4.jsonl", [{"type": "user", "message": {"content": "task"}},
msg("w1", 2, 0, 10_000, 300, "tool_use"), result("x" * 400),
msg("w2", 2, 10_000, 50_000, 2, None), msg("w2", 2, 10_000, 50_000, 2, None),
result("y" * 40), msg("w3", 2, 60_000, 5_010, 20)])
(sub / "agent-a4.meta.json").write_text(json.dumps({"agentType": "p1-compiler", "description": "P1-K-pack3"}))
runs = {r["unit"]: r for r in cost.harvest(self.proj, cost.sessions(self.okter))["runs"]}
w = runs["pack3"]
self.assertEqual(w["output_recorded"], 300 + 2 + 20)
self.assertEqual(w["output"], 300 + (5_010 - 40 * 2 // 5) + 20)
self.assertEqual((w["output_derived_messages"], w["output_unrecorded"]), (1, 0))
self.assertEqual(w["total"], 2 + 2 + 2 + 70_000 + 65_010 + w["output"])
# the last message of a run has no next message: its output stays as recorded and is flagged
self.assertEqual((runs["run1"]["output"], runs["run1"]["output_unrecorded"]), (1, 1))
self.assertEqual(runs["pack1"]["output_derived_messages"], 0)
def test_growth_from_injected_attachments_is_not_counted_as_output(self) -> None:
# The load test's pattern: a short call, then 5133 characters of rendered attachments before the next turn.
sub = self.proj / "s1" / "subagents"
jsonl(sub / "agent-a5.jsonl", [{"type": "user", "message": {"content": "task"}},
msg("v1", 2, 0, 18_602, 16, None),
{"type": "attachment", "rendered": "z" * 5_133},
{"type": "user", "message": {"content": [{"type": "tool_result",
"content": "w" * 151}]}},
msg("v2", 2, 18_602, 1_894, 24, None)])
(sub / "agent-a5.meta.json").write_text(json.dumps({"agentType": "p1-compiler", "description": "P1-LOAD-x"}))
runs = {r["unit"]: r for r in cost.harvest(self.proj, cost.sessions(self.okter))["runs"]}
self.assertEqual((runs["x"]["output"], runs["x"]["output_derived_messages"]), (16 + 24, 0))
class ProjectionTest(unittest.TestCase):
def test_least_squares_on_cost_per_block_against_blocks_read(self) -> None:
p = cost.project([(600_000, 600, 0), (220_000, 200, 600)], total_blocks=3000)
self.assertAlmostEqual(p["fit"]["a"], 1000)
self.assertAlmostEqual(p["fit"]["b"], 1 / 6)
self.assertAlmostEqual(p["fit"]["value"], 3_750_000)
self.assertAlmostEqual(p["flat"], 3_075_000)
def test_one_point_has_no_growth_term(self) -> None:
p = cost.project([(600_000, 600, 0)], total_blocks=3000)
self.assertEqual(p["fit"]["b"], 0)
self.assertEqual(p["fit"]["note"], "one point, no growth term")
self.assertAlmostEqual(p["fit"]["value"], 3_000_000)
def test_stop_per_block_against_naive_per_source(self) -> None:
s = cost.stop_k(pack1_cost=600_000, pack1_blocks=600, cut_blocks=1200, pack1_sources=5, cut_sources=19)
self.assertEqual(s["per_block"], 1_200_000)
self.assertEqual(s["per_source"], 2_280_000)
self.assertTrue(s["stop"])
self.assertFalse(cost.stop_k(500_000, 600, 1200, 5, 19)["stop"]) # exactly 1 M does not stop
self.assertTrue(cost.stop_k(500_001, 600, 1200, 5, 19)["stop"])
def test_kplus_stop(self) -> None:
self.assertFalse(cost.stop_kplus(run1_cost=200_000, run1_pages=8, pages_total=40)["stop"])
self.assertTrue(cost.stop_kplus(run1_cost=200_001, run1_pages=8, pages_total=40)["stop"])
class AllocationTest(unittest.TestCase):
def test_tokens_per_source_by_block_share_and_the_two_means(self) -> None:
names = [f"s{i:02d}.md" for i in range(1, 20)]
blocks_doc = {"order": names, "blocks": [{"file": n, "start": k, "end": k} for n in names for k in (1, 2)]}
packs = [names[i:i + 5] for i in range(0, 19, 5)]
costs = {1: 1000, 2: 500, 3: 500, 4: 800}
per = cost.allocate(costs, packs, blocks_doc)
self.assertEqual(per["s01.md"], 200)
self.assertEqual(per["s16.md"], 200)
means = cost.means(per, names)
self.assertEqual(means["1-5"], 200)
self.assertEqual(means["15-19"], 180) # source 15 in pack 3 (100), 16-19 in pack 4 (200 each)
# ---------------------------------------------------------------------------------------------- step 7: side by side
class SideBySideTest(Fixture):
def setUp(self) -> None:
super().setUp()
self.begreper = self.tmp / "begreper.txt"
self.begreper.write_text("Beds\tBeds;Raised beds\nLabels\tLabels\nCompost\tCompost\n", encoding="utf-8")
self.d = self.tmp / "D"
write_page(self.d, "r-beds.md", "Raised beds", "- [Alpha](k.md)", [src("o1", "docs-en-a.md", 5, "Raised beds")])
self.l4 = self.tmp / "L4"
write_page(self.l4, "x.md", "THE raised-beds, explained!", "Beds drain.[^b1]",
[src("b1", "docs-en-a.md", 5, "Raised beds drain well.")])
write_page(self.l4, "y.md", "Compost", "Compost helps.[^b1]",
[src("b1", "docs-en-a.md", 99, "Compost")]) # a link past the end of the file
def columns(self) -> list[tuple[str, Path | None]]:
return [("K", self.bundle), ("K+", None), ("D", self.d), ("L4", self.l4)]
def test_three_concepts_in_four_columns_with_the_empty_states(self) -> None:
out = self.tmp / "side.html"
sidebyside.render(sidebyside.read_begreper(self.begreper), self.columns(), self.root, out)
html = out.read_text(encoding="utf-8")
self.assertEqual(html.count('<tr class="concept">'), 3)
self.assertEqual(html.count('<td class="cell'), 12)
self.assertIn("ikke bygget", html) # K+ stopped
self.assertIn("ikke i registeret", html) # D has no Labels register page
self.assertIn("ingen side", html) # K is built and has no Compost page
self.assertNotIn("http", html)
def test_l4_finds_a_title_with_other_case_and_punctuation(self) -> None:
found = sidebyside.find(self.l4, ("Raised beds", ("Raised beds",)), l4=True)
self.assertEqual(found, ["x.md"])
def test_the_link_check_flags_a_line_past_the_end(self) -> None:
out = self.tmp / "side.html"
sidebyside.render(sidebyside.read_begreper(self.begreper), self.columns(), self.root, out)
resolved, total = sidebyside.check(out, self.root)
self.assertEqual(total - resolved, 1)
self.assertEqual(sidebyside.main(["check", "--html", str(out), "--kilde-root", str(self.root)]), 1)
class ScriptsTest(unittest.TestCase):
SCRIPTS = (ROOT / "tests" / "p1_repo.sh", ROOT / "tests" / "p1_trial.sh")
def test_bash_syntax_and_ascii(self) -> None:
for script in self.SCRIPTS:
self.assertEqual(subprocess.run(["bash", "-n", str(script)]).returncode, 0, script.name)
script.read_bytes().decode("ascii")
def test_the_trial_check_fails_every_check_when_nothing_was_built(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
env = {**os.environ, "P1_ROOT": tmp}
res = subprocess.run(["bash", str(self.SCRIPTS[1])], env=env, capture_output=True, text=True, cwd=ROOT)
self.assertEqual(res.returncode, 1)
lines = [x for x in res.stdout.splitlines() if x.strip()]
self.assertGreaterEqual(len(lines), 8)
self.assertTrue(all(x.startswith("FAIL") for x in lines), res.stdout)
class NoModelTest(unittest.TestCase):
def test_trial_modules_import_no_network_or_model_client(self) -> None:
for path in sorted((ROOT / "evals" / "p1").glob("*.py")):
source = path.read_text(encoding="utf-8")
for banned in BANNED:
self.assertNotIn(f"import {banned}", source, path.name)
self.assertNotIn(f"from {banned}", source, path.name)
if __name__ == "__main__":
unittest.main()