okf-wiki 1.1.1
First public release, published as one snapshot. What changed between versions is in CHANGELOG.md. Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
This commit is contained in:
commit
0fd86a23a3
227 changed files with 96077 additions and 0 deletions
355
evals/tests/test_direction_probe.py
Normal file
355
evals/tests/test_direction_probe.py
Normal file
|
|
@ -0,0 +1,355 @@
|
|||
"""Tests for evals/direction_probe.py: the direction probe R (PLAN.md § Leveranser, row R).
|
||||
|
||||
Three arms answer the same questions from the same sources: file search alone, the okf-wiki plus file search, and a
|
||||
wiki built with Astro-Han's karpathy-llm-wiki skill. The module sets up each arm's basis and task files, collects the
|
||||
answers into the gates' format, harvests tokens and isolation breaches from the subagent transcripts, and prints the
|
||||
table. It never calls a model. The fixtures are synthetic text about gardening (CLAUDE.md rule 3).
|
||||
Run: python3 -m unittest discover -s evals/tests
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
sys.path.insert(0, str(ROOT / "evals"))
|
||||
|
||||
import direction_probe as dp # noqa: E402
|
||||
|
||||
TOMATO = "# Tomatoes\nTomatoes need full sun.\nWater them every morning.\n"
|
||||
BASIL = "# Basil\nBasil likes warm soil.\n"
|
||||
MINT = "# Mint\nMint spreads through runners.\n"
|
||||
NOT_IN_CUT = "# Cactus\nCacti need little water.\n"
|
||||
|
||||
MD_SET: dict[str, Any] = {
|
||||
"schema": "okf-wiki/multidoc-set/1",
|
||||
"questions": [
|
||||
{"id": "d01", "question": "What do tomatoes and basil need?",
|
||||
"answer_keys": [{"file": "tomato.md", "text": "SECRETKEYSUN"}],
|
||||
"evidence": [{"file": "tomato.md", "span": "SECRETSPANTOMATO"}]},
|
||||
{"id": "d02", "question": "A question outside the probe.",
|
||||
"answer_keys": [{"file": "mint.md", "text": "OTHERKEY"}],
|
||||
"evidence": [{"file": "mint.md", "span": "OTHERSPAN"}]},
|
||||
],
|
||||
}
|
||||
|
||||
WD_SET: dict[str, Any] = {
|
||||
"schema": "okf-wiki/wholedoc-set/1",
|
||||
"questions": [
|
||||
{"id": "d01", "type": "whole", "file": "tomato.md", "question": "What does the tomato page say overall?",
|
||||
"answer_keys": [{"text": "SECRETWHOLEKEY"}], "evidence": [{"span": "SECRETWHOLESPAN"}]},
|
||||
{"id": "d02", "type": "count", "file": "basil.md", "question": "How many basil lines are there?",
|
||||
"count": 424242, "item_pattern": "SECRETPATTERN", "evidence": [{"span": "SECRETCOUNTSPAN"}]},
|
||||
],
|
||||
}
|
||||
|
||||
SECRETS = ("SECRETKEYSUN", "SECRETSPANTOMATO", "SECRETWHOLEKEY", "SECRETWHOLESPAN", "424242", "SECRETPATTERN",
|
||||
"SECRETCOUNTSPAN", "OTHERKEY", "A question outside the probe")
|
||||
|
||||
|
||||
class Fixture:
|
||||
def __init__(self, root: Path) -> None:
|
||||
self.root = root
|
||||
self.sources = root / "kilde"
|
||||
self.sources.mkdir()
|
||||
for name, text in (("tomato.md", TOMATO), ("basil.md", BASIL), ("mint.md", MINT), ("cactus.md", NOT_IN_CUT)):
|
||||
(self.sources / name).write_text(text, encoding="utf-8")
|
||||
self.names = ["basil.md", "mint.md", "tomato.md"]
|
||||
self.md_set = root / "md.json"
|
||||
self.md_set.write_text(json.dumps(MD_SET), encoding="utf-8")
|
||||
self.wd_set = root / "wd.json"
|
||||
self.wd_set.write_text(json.dumps(WD_SET), encoding="utf-8")
|
||||
self.wiki = root / "wiki-src"
|
||||
self.wiki.mkdir()
|
||||
(self.wiki / "index.md").write_text("# Index\n* [Tomatoes](tomatoes.md)\n", encoding="utf-8")
|
||||
(self.wiki / "tomatoes.md").write_text("Tomatoes need full sun.\n", encoding="utf-8")
|
||||
self.run = root / "run"
|
||||
|
||||
def setup(self, arm: str, **extra: Path) -> list[Path]:
|
||||
return dp.setup_arm(run=self.run, arm=arm, sources=self.sources, names=self.names, md_set=self.md_set,
|
||||
md_ids=["d01"], wd_set=self.wd_set, **extra)
|
||||
|
||||
|
||||
class FetchAstroTest(unittest.TestCase):
|
||||
def test_raw_file_keeps_the_name_and_the_body_byte_exact_under_the_template_header(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
fx = Fixture(Path(tmp))
|
||||
project = Path(tmp) / "astro"
|
||||
written = dp.fetch_astro(sources=fx.sources, names=fx.names, project=project, topic="garden",
|
||||
collected="2026-09-27")
|
||||
self.assertEqual([p.name for p in written], fx.names)
|
||||
raw = (project / "raw" / "garden" / "tomato.md").read_text(encoding="utf-8")
|
||||
self.assertTrue(raw.startswith("# tomato\n\n> Source: kilde/tomato.md\n> Collected: 2026-09-27\n"
|
||||
"> Published: Unknown\n\n"))
|
||||
self.assertTrue(raw.endswith(TOMATO))
|
||||
self.assertFalse((project / "raw" / "garden" / "cactus.md").exists())
|
||||
|
||||
def test_an_existing_raw_file_is_never_overwritten(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
fx = Fixture(Path(tmp))
|
||||
project = Path(tmp) / "astro"
|
||||
dp.fetch_astro(sources=fx.sources, names=fx.names, project=project, topic="garden", collected="2026-09-27")
|
||||
with self.assertRaises(dp.ProbeError):
|
||||
dp.fetch_astro(sources=fx.sources, names=fx.names, project=project, topic="garden",
|
||||
collected="2026-09-27")
|
||||
|
||||
|
||||
class SetupTest(unittest.TestCase):
|
||||
def test_file_search_basis_holds_exactly_the_cut(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
fx = Fixture(Path(tmp))
|
||||
fx.setup("filsok")
|
||||
basis = fx.run / "arms" / "filsok"
|
||||
self.assertEqual(sorted(p.name for p in basis.iterdir()), fx.names)
|
||||
self.assertEqual((basis / "tomato.md").read_text(encoding="utf-8"), TOMATO)
|
||||
|
||||
def test_one_task_per_probe_question_and_none_for_the_rest(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
fx = Fixture(Path(tmp))
|
||||
tasks = fx.setup("filsok")
|
||||
self.assertEqual(sorted(p.name for p in tasks), ["md-d01.md", "wd-d01.md", "wd-d02.md"])
|
||||
self.assertTrue(all(p.parent == fx.run / "tasks" / "filsok" for p in tasks))
|
||||
|
||||
def test_a_task_carries_the_question_the_basis_and_the_answer_path_but_nothing_from_the_key(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
fx = Fixture(Path(tmp))
|
||||
tasks = {p.name: p.read_text(encoding="utf-8") for p in fx.setup("filsok")}
|
||||
md = tasks["md-d01.md"]
|
||||
self.assertIn("What do tomatoes and basil need?", md)
|
||||
self.assertIn(str(fx.run / "arms" / "filsok"), md)
|
||||
self.assertIn(str(fx.run / "answers" / "filsok" / "md-d01.json"), md)
|
||||
self.assertIn('"count"', tasks["wd-d02.md"])
|
||||
self.assertNotIn('"count"', md)
|
||||
for text in tasks.values():
|
||||
for secret in SECRETS:
|
||||
self.assertNotIn(secret, text)
|
||||
|
||||
def test_the_wiki_arm_gets_the_wiki_and_the_cut_and_the_router_order(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
fx = Fixture(Path(tmp))
|
||||
tasks = fx.setup("wiki", wiki=fx.wiki)
|
||||
basis = fx.run / "arms" / "wiki"
|
||||
self.assertEqual(sorted(p.name for p in (basis / "kilde").iterdir()), fx.names)
|
||||
self.assertTrue((basis / "wiki" / "index.md").is_file())
|
||||
text = tasks[0].read_text(encoding="utf-8")
|
||||
self.assertLess(text.index("index.md"), text.index("x-locator"))
|
||||
|
||||
def test_the_astro_arm_gets_raw_and_wiki_from_the_built_project(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
fx = Fixture(Path(tmp))
|
||||
project = Path(tmp) / "astro"
|
||||
dp.fetch_astro(sources=fx.sources, names=fx.names, project=project, topic="garden", collected="2026-09-27")
|
||||
(project / "wiki").mkdir()
|
||||
(project / "wiki" / "index.md").write_text("# Knowledge Base Index\n", encoding="utf-8")
|
||||
tasks = fx.setup("astro", astro=project)
|
||||
basis = fx.run / "arms" / "astro"
|
||||
self.assertTrue((basis / "raw" / "garden" / "basil.md").is_file())
|
||||
self.assertTrue((basis / "wiki" / "index.md").is_file())
|
||||
self.assertIn("Raw", tasks[0].read_text(encoding="utf-8"))
|
||||
|
||||
def test_an_arm_without_its_basis_or_an_existing_arm_is_refused(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
fx = Fixture(Path(tmp))
|
||||
with self.assertRaises(dp.ProbeError):
|
||||
fx.setup("wiki")
|
||||
fx.setup("filsok")
|
||||
with self.assertRaises(dp.ProbeError):
|
||||
fx.setup("filsok")
|
||||
with self.assertRaises(dp.ProbeError):
|
||||
fx.setup("grep-only")
|
||||
|
||||
def test_a_probe_id_missing_from_the_set_is_refused(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
fx = Fixture(Path(tmp))
|
||||
with self.assertRaises(dp.ProbeError):
|
||||
dp.setup_arm(run=fx.run, arm="filsok", sources=fx.sources, names=fx.names, md_set=fx.md_set,
|
||||
md_ids=["d09"], wd_set=fx.wd_set)
|
||||
|
||||
|
||||
class CollectTest(unittest.TestCase):
|
||||
def test_answers_are_gathered_per_gate_with_ids_from_the_file_names(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
run = Path(tmp)
|
||||
out = run / "answers" / "wiki"
|
||||
out.mkdir(parents=True)
|
||||
(out / "md-d01.json").write_text(json.dumps({"id": "wrong", "answer": "a", "citations": []}))
|
||||
(out / "wd-d02.json").write_text(json.dumps({"answer": "7", "count": 7, "citations": []}))
|
||||
(out / "wd-d03.json").write_text("not json")
|
||||
md_path, wd_path, problems = dp.collect(run, "wiki")
|
||||
md = json.loads(md_path.read_text(encoding="utf-8"))
|
||||
wd = json.loads(wd_path.read_text(encoding="utf-8"))
|
||||
self.assertEqual(md, {"arm": "wiki", "answers": [{"id": "d01", "answer": "a", "citations": []}]})
|
||||
self.assertEqual(wd["answers"], [{"id": "d02", "answer": "7", "count": 7, "citations": []}])
|
||||
self.assertEqual(problems, ["wd-d03.json: not a JSON object"])
|
||||
|
||||
|
||||
def transcript(lines: list[dict[str, Any]]) -> str:
|
||||
return "".join(json.dumps(line) + "\n" for line in lines)
|
||||
|
||||
|
||||
def assistant(msg_id: str, tokens: tuple[int, int, int, int], *tools: tuple[str, dict[str, Any]]) -> dict[str, Any]:
|
||||
i, cc, cr, o = tokens
|
||||
content = [{"type": "tool_use", "name": name, "input": inp} for name, inp in tools]
|
||||
return {"type": "assistant", "message": {"id": msg_id, "content": content, "usage": {
|
||||
"input_tokens": i, "cache_creation_input_tokens": cc, "cache_read_input_tokens": cr, "output_tokens": o}}}
|
||||
|
||||
|
||||
class HarvestTest(unittest.TestCase):
|
||||
def write(self, folder: Path, name: str, lines: list[dict[str, Any]]) -> None:
|
||||
(folder / name).write_text(transcript(lines), encoding="utf-8")
|
||||
|
||||
def test_tokens_count_each_message_once_and_breaches_are_named(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
run, folder = Path(tmp) / "run", Path(tmp) / "subagents"
|
||||
folder.mkdir()
|
||||
task = run / "tasks" / "wiki" / "md-d01.md"
|
||||
basis = run / "arms" / "wiki"
|
||||
answer = run / "answers" / "wiki" / "md-d01.json"
|
||||
first = {"type": "user", "message": {"content": f"Read {task} and follow it."}}
|
||||
self.write(folder, "agent-a.jsonl", [
|
||||
first,
|
||||
assistant("m1", (2, 100, 1000, 5), ("Read", {"file_path": str(task)})),
|
||||
assistant("m1", (2, 100, 1000, 5)),
|
||||
assistant("m2", (1, 10, 2000, 50), ("Grep", {"pattern": "sun", "path": str(basis / "kilde")}),
|
||||
("Glob", {"pattern": "*.md", "path": str(basis / "wiki")})),
|
||||
assistant("m3", (1, 0, 3000, 80), ("Write", {"file_path": str(answer), "content": "{}"}),
|
||||
("SubagentHandback", {"message": "done"})),
|
||||
])
|
||||
self.write(folder, "agent-b.jsonl", [
|
||||
{"type": "user", "message": {"content": [{"type": "text", "text": f"Read {run}/tasks/filsok/wd-d02.md"}]}},
|
||||
assistant("n1", (1, 1, 1, 1), ("Read", {"file_path": str(run / "arms" / "wiki" / "wiki" / "x.md")}),
|
||||
("Grep", {"pattern": "sun"}), ("Bash", {"command": "ls"}),
|
||||
("Read", {"file_path": str(run / "arms" / "filsok" / ".." / "wiki" / "y.md")})),
|
||||
])
|
||||
self.write(folder, "agent-c.jsonl", [{"type": "user", "message": {"content": "Something else."}}])
|
||||
rows = dp.harvest(folder, run)
|
||||
self.assertEqual(sorted(rows), ["filsok/wd-d02", "wiki/md-d01"])
|
||||
wiki = rows["wiki/md-d01"]
|
||||
self.assertEqual(wiki["tokens"], 1107 + 2061 + 3081)
|
||||
self.assertEqual(wiki["breaches"], [])
|
||||
self.assertEqual(wiki["transcript"], "agent-a.jsonl")
|
||||
breaches = rows["filsok/wd-d02"]["breaches"]
|
||||
self.assertEqual(len(breaches), 4)
|
||||
self.assertTrue(any(b.startswith("Bash") for b in breaches))
|
||||
self.assertTrue(any(b.startswith("Grep without a path") for b in breaches))
|
||||
|
||||
def test_bash_is_read_only_search_that_starts_in_the_basis(self) -> None:
|
||||
basis = Path("/runs/r/arms/filsok")
|
||||
task, answer = Path("/runs/r/tasks/filsok/md-d01.md"), Path("/runs/r/answers/filsok/md-d01.json")
|
||||
allowed = [
|
||||
f"cd {basis} && grep -rn 'full sun' .",
|
||||
f"cd {basis} && grep -n sun tomato.md | head -5",
|
||||
f"cd {basis} && find . -name '*.md' | sort && wc -l tomato.md 2>/dev/null",
|
||||
f"cd '{basis}' && sed -n '1,40p' tomato.md",
|
||||
f"cd {basis}/kilde && ls",
|
||||
]
|
||||
refused = [
|
||||
"grep -rn sun /runs/r/arms/filsok",
|
||||
f"cd {basis} && cat ../wiki/x.md",
|
||||
f"cd {basis} && grep sun /runs/r/tasks/filsok/other.md",
|
||||
f"cd {basis} && rm tomato.md",
|
||||
f"cd {basis} && grep sun tomato.md > copy.txt",
|
||||
f"cd {basis} && find . -name '*.md' -exec cat {{}} ;",
|
||||
f"cd {basis} && sed -i 's/a/b/' tomato.md",
|
||||
f"cd {basis} && cat $(echo /etc/hosts)",
|
||||
f"cd {basis} && cd / && ls",
|
||||
f"cd {basis} && ls ~",
|
||||
"cd /runs/r/arms/wiki && ls",
|
||||
]
|
||||
for command in allowed:
|
||||
self.assertEqual(dp.breaches_of([("Bash", {"command": command})], basis, task, answer), [], command)
|
||||
for command in refused:
|
||||
self.assertEqual(len(dp.breaches_of([("Bash", {"command": command})], basis, task, answer)), 1, command)
|
||||
|
||||
def test_the_task_names_bash_as_the_search_tool_and_its_rule(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
fx = Fixture(Path(tmp))
|
||||
text = fx.setup("filsok")[0].read_text(encoding="utf-8")
|
||||
self.assertIn(f"cd {fx.run / 'arms' / 'filsok'} && ", text)
|
||||
self.assertIn("grep", text)
|
||||
|
||||
def test_two_transcripts_for_one_task_keep_the_latest_and_report_the_rerun(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
run, folder = Path(tmp) / "run", Path(tmp) / "subagents"
|
||||
folder.mkdir()
|
||||
task = run / "tasks" / "filsok" / "md-d01.md"
|
||||
first = {"type": "user", "timestamp": "2026-09-27T10:00:00Z", "message": {"content": f"Read {task}"}}
|
||||
later = {"type": "user", "timestamp": "2026-09-27T11:00:00Z", "message": {"content": f"Read {task}"}}
|
||||
self.write(folder, "agent-old.jsonl", [first, assistant("o", (1, 1, 1, 1))])
|
||||
self.write(folder, "agent-new.jsonl", [later, assistant("n", (2, 2, 2, 2))])
|
||||
rows = dp.harvest(folder, run)
|
||||
self.assertEqual(rows["filsok/md-d01"]["transcript"], "agent-new.jsonl")
|
||||
self.assertEqual(rows["filsok/md-d01"]["tokens"], 8)
|
||||
self.assertEqual(rows["filsok/md-d01"]["runs"], 2)
|
||||
|
||||
|
||||
def results(arm: str, correct: dict[str, bool], kind: str) -> dict[str, Any]:
|
||||
rows = []
|
||||
for qid, ok in correct.items():
|
||||
row: dict[str, Any] = {"id": qid, "answered": True, "answer_ok": ok, "source_ok": True, "correct": ok}
|
||||
if kind == "wd":
|
||||
row["type"] = "whole" if qid in ("d01", "d02") else "count"
|
||||
rows.append(row)
|
||||
return {"arm": arm, "set_sha256": "x", "results": rows}
|
||||
|
||||
|
||||
class ScoreTest(unittest.TestCase):
|
||||
def test_groups_are_counted_from_the_gate_results_and_pairs_compared(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
run = Path(tmp)
|
||||
res = run / "results"
|
||||
res.mkdir()
|
||||
md = {"filsok": {"d01": True, "d02": False, "d99": True}, "wiki": {"d01": True, "d02": True, "d99": False}}
|
||||
wd = {"filsok": {"d01": True, "d02": False, "d03": True}, "wiki": {"d01": False, "d02": True, "d03": True}}
|
||||
for arm in ("filsok", "wiki"):
|
||||
(res / f"{arm}.md.json").write_text(json.dumps(results(arm, md[arm], "md")))
|
||||
(res / f"{arm}.wd.json").write_text(json.dumps(results(arm, wd[arm], "wd")))
|
||||
(run / "tokens.json").write_text(json.dumps({
|
||||
"filsok/md-d01": {"tokens": 100, "breaches": [], "transcript": "a", "runs": 1},
|
||||
"wiki/md-d01": {"tokens": 300, "breaches": [], "transcript": "b", "runs": 1}}))
|
||||
table = dp.score(run, ["filsok", "wiki"], md_ids=["d01", "d02"])
|
||||
self.assertEqual(table["arms"]["filsok"]["groups"]["flerdokument"], [1, 2])
|
||||
self.assertEqual(table["arms"]["wiki"]["groups"]["flerdokument"], [2, 2])
|
||||
self.assertEqual(table["arms"]["wiki"]["groups"]["helt dokument"], [1, 2])
|
||||
self.assertEqual(table["arms"]["filsok"]["groups"]["telling"], [1, 1])
|
||||
self.assertEqual(table["arms"]["wiki"]["total"], [4, 5])
|
||||
self.assertEqual(table["arms"]["wiki"]["tokens"], {"sum": 300, "questions": 1})
|
||||
pair = table["pairs"]["wiki vs filsok"]["flerdokument"]
|
||||
self.assertEqual(pair, {"both": 1, "only wiki": 1, "only filsok": 0, "neither": 0})
|
||||
self.assertEqual(table["pairs"]["wiki vs filsok"]["helt dokument"]["only filsok"], 1)
|
||||
self.assertIn("| wiki |", dp.render_table(table))
|
||||
|
||||
def test_exact_mcnemar_is_the_two_sided_binomial_over_the_discordant_pairs(self) -> None:
|
||||
self.assertEqual(dp.mcnemar_exact(0, 0), 1.0)
|
||||
self.assertEqual(dp.mcnemar_exact(2, 0), 0.5)
|
||||
self.assertAlmostEqual(dp.mcnemar_exact(6, 0), 0.03125)
|
||||
self.assertAlmostEqual(dp.mcnemar_exact(1, 5), 0.21875)
|
||||
self.assertEqual(dp.mcnemar_exact(3, 3), 1.0)
|
||||
|
||||
def test_every_pair_carries_its_mcnemar_p_over_all_groups(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
run = Path(tmp)
|
||||
res = run / "results"
|
||||
res.mkdir()
|
||||
md = {"filsok": {"d01": False, "d02": False}, "wiki": {"d01": True, "d02": True}}
|
||||
for arm in ("filsok", "wiki"):
|
||||
(res / f"{arm}.md.json").write_text(json.dumps(results(arm, md[arm], "md")))
|
||||
(res / f"{arm}.wd.json").write_text(json.dumps(results(arm, {}, "wd")))
|
||||
table = dp.score(run, ["filsok", "wiki"], md_ids=["d01", "d02"])
|
||||
self.assertEqual(table["mcnemar"]["wiki vs filsok"], {"only wiki": 2, "only filsok": 0, "p": 0.5})
|
||||
|
||||
def test_a_missing_result_file_is_a_failure_not_a_zero(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
run = Path(tmp)
|
||||
(run / "results").mkdir()
|
||||
with self.assertRaises(dp.ProbeError):
|
||||
dp.score(run, ["filsok"], md_ids=["d01"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
609
evals/tests/test_p1_build.py
Normal file
609
evals/tests/test_p1_build.py
Normal file
|
|
@ -0,0 +1,609 @@
|
|||
"""Tests for the P1 method trial's build side (TRIAL, branch p1-metodeprove, not v1 code).
|
||||
|
||||
Covers the two trial agent types, the splitter (evals/p1/split.py), the D map and register (dmap.py), K packing
|
||||
(kpack.py) and assembly with K+ (assemble.py). None of these call a model. The fixtures are synthetic text about
|
||||
gardening and bread (CLAUDE.md rule 3: no text from the Claude Code documentation is committed).
|
||||
Run: python3 -m unittest discover -s evals/tests -p 'test_p1_build.py'
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
sys.path.insert(0, str(ROOT))
|
||||
sys.path.insert(0, str(ROOT / "evals"))
|
||||
|
||||
from okfwiki import audit, claims, view # noqa: E402
|
||||
from p1 import assemble, dmap, kpack, split # noqa: E402
|
||||
|
||||
READER_SHA256 = "13a93bc5a9e87a7282752316ac670664cbdca321b0dac9cdaf92d9e74c383445"
|
||||
BANNED = ("anthropic", "openai", "urllib", "http.client", "requests", "subprocess", "socket")
|
||||
|
||||
DOC_TOMATO = """# Growing tomatoes
|
||||
Source: https://example.org/garden/tomatoes
|
||||
|
||||
How to grow tomatoes in a small garden.
|
||||
|
||||
## Soil
|
||||
|
||||
Tomatoes need deep, loose soil with `compost` mixed in.
|
||||
Water the soil every morning.
|
||||
|
||||
* **Sun**: full sun for eight hours.
|
||||
* **Space**: forty centimetres between plants,
|
||||
more in a wet summer.
|
||||
|
||||
```bash
|
||||
# not a heading
|
||||
mix compost
|
||||
```
|
||||
|
||||
| Month | Task |
|
||||
|---|---|
|
||||
| May | plant |
|
||||
|
||||
## Overview
|
||||
|
||||
The Kitchen Garden keeps tomatoes and basil together.
|
||||
|
||||
### Pruning
|
||||
|
||||
Remove side shoots once a week.
|
||||
"""
|
||||
|
||||
DOC_BASIL = """# Growing basil
|
||||
Source: https://example.org/garden/basil
|
||||
|
||||
## Soil
|
||||
|
||||
Basil likes warm soil and the Kitchen Garden.
|
||||
Plant Sweet Genovese early.
|
||||
|
||||
## Watering
|
||||
|
||||
Water basil with `compost` tea when the leaves droop.
|
||||
"""
|
||||
|
||||
REL_1 = """## 1.2.3
|
||||
|
||||
- Added the `compost` command to the Kitchen Garden planner.
|
||||
- Fixed watering reminders.
|
||||
"""
|
||||
|
||||
REL_2 = """## 1.10.0
|
||||
|
||||
- The Kitchen Garden planner now shows `compost` levels.
|
||||
"""
|
||||
|
||||
REL_3 = """## 1.9.2
|
||||
|
||||
- Basil seedlings get their own tray.
|
||||
"""
|
||||
|
||||
|
||||
def runs8(text: str) -> set[tuple[str, ...]]:
|
||||
words = re.findall(r"[a-z0-9]+", text.lower())
|
||||
return {tuple(words[i:i + 8]) for i in range(len(words) - 7)}
|
||||
|
||||
|
||||
def before_link_rule(found: list[audit.Finding]) -> list[audit.Finding]:
|
||||
"""The P1 trial predates criterion 6 as structure (operator 04.10.2026), so its link-rule findings are left out."""
|
||||
return [f for f in found if f.check != "link-rule"]
|
||||
|
||||
|
||||
class Fixture(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
self._tmp = tempfile.TemporaryDirectory()
|
||||
self.tmp = Path(self._tmp.name)
|
||||
self.kilde = self.tmp / "wiki" / "kilde"
|
||||
self.kilde.mkdir(parents=True)
|
||||
files = {
|
||||
"docs-en-tomatoes.md": DOC_TOMATO,
|
||||
"docs-en-basil.md": DOC_BASIL,
|
||||
"release-v1.2.3.md": REL_1,
|
||||
"release-v1.10.0.md": REL_2,
|
||||
"release-v1.9.2.md": REL_3,
|
||||
}
|
||||
for name, text in files.items():
|
||||
(self.kilde / name).write_text(text, encoding="utf-8")
|
||||
|
||||
def tearDown(self) -> None:
|
||||
self._tmp.cleanup()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------------------------- step 1: agents
|
||||
|
||||
|
||||
class AgentFilesTest(unittest.TestCase):
|
||||
def frontmatter(self, text: str) -> dict[str, str]:
|
||||
self.assertTrue(text.startswith("---\n"))
|
||||
head = text[4:text.index("\n---\n", 4)]
|
||||
return dict(line.split(": ", 1) for line in head.splitlines())
|
||||
|
||||
def test_two_trial_agents_have_narrow_tools_and_a_trial_marker(self) -> None:
|
||||
for name in ("p1-compiler", "p1-consolidator"):
|
||||
text = (ROOT / ".claude" / "agents" / f"{name}.md").read_text(encoding="utf-8")
|
||||
fm = self.frontmatter(text)
|
||||
self.assertEqual(fm["name"], name)
|
||||
self.assertEqual(fm["tools"], "Read, Write")
|
||||
self.assertEqual(fm["model"], "opus")
|
||||
body = text[text.index("\n---\n", 4) + 5:].strip()
|
||||
self.assertTrue(body.startswith("TRIAL"), name)
|
||||
fixture = DOC_TOMATO + DOC_BASIL + REL_1 + REL_2 + REL_3
|
||||
self.assertEqual(runs8(body) & runs8(fixture), set(), name)
|
||||
|
||||
def test_the_reader_is_unchanged(self) -> None:
|
||||
shipped = (ROOT / "agents" / "okf-wiki-reader.md").read_bytes()
|
||||
self.assertEqual(hashlib.sha256(shipped).hexdigest(), READER_SHA256)
|
||||
self.assertEqual(shipped, (ROOT / ".claude" / "agents" / "okf-wiki-reader.md").read_bytes())
|
||||
|
||||
def test_compiler_names_the_rules_it_must_keep(self) -> None:
|
||||
body = (ROOT / ".claude" / "agents" / "p1-compiler.md").read_text(encoding="utf-8").lower()
|
||||
for idea in ("first turn", "never read", "release note", "four tests", "json"):
|
||||
self.assertIn(idea, body, idea)
|
||||
|
||||
def test_consolidator_keeps_every_label(self) -> None:
|
||||
body = (ROOT / ".claude" / "agents" / "p1-consolidator.md").read_text(encoding="utf-8").lower()
|
||||
for idea in ("every footnote label", "no new label", "json"):
|
||||
self.assertIn(idea, body, idea)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------------------------- step 1: split
|
||||
|
||||
|
||||
class OrderTest(unittest.TestCase):
|
||||
def test_docs_first_then_release_notes_by_version_tuple(self) -> None:
|
||||
names = ["release-v0.2.34.md", "docs-en-b.md", "release-v0.2.9.md", "docs-en-a.md", "release-v1.0.0.md"]
|
||||
self.assertEqual(split.order(names), ["docs-en-a.md", "docs-en-b.md", "release-v0.2.9.md",
|
||||
"release-v0.2.34.md", "release-v1.0.0.md"])
|
||||
|
||||
def test_an_unknown_name_is_refused(self) -> None:
|
||||
with self.assertRaises(split.RunError):
|
||||
split.order(["notes.md"])
|
||||
|
||||
|
||||
class BlocksTest(Fixture):
|
||||
def test_fence_aware_blocks_with_spans(self) -> None:
|
||||
got = split.blocks(self.kilde / "docs-en-tomatoes.md")
|
||||
kinds = [(b.kind, b.start, b.end) for b in got]
|
||||
# line 2 "Source:" and line 4 are paragraphs; headings are not blocks
|
||||
self.assertIn(("paragraph", 2, 2), kinds)
|
||||
self.assertIn(("paragraph", 8, 9), kinds)
|
||||
self.assertIn(("item", 11, 11), kinds)
|
||||
self.assertIn(("item", 12, 13), kinds) # a continuation line belongs to its item
|
||||
self.assertIn(("code", 15, 18), kinds) # the "#" inside the fence is not a heading
|
||||
self.assertIn(("table", 20, 22), kinds)
|
||||
heads = {b.section for b in got}
|
||||
self.assertIn("Growing tomatoes › Soil", heads)
|
||||
self.assertIn("Growing tomatoes › Overview › Pruning", heads)
|
||||
self.assertNotIn("not a heading", " ".join(heads))
|
||||
|
||||
def test_every_non_blank_non_heading_line_is_in_exactly_one_block(self) -> None:
|
||||
path = self.kilde / "docs-en-tomatoes.md"
|
||||
lines = path.read_text(encoding="utf-8").split("\n")
|
||||
got = split.blocks(path)
|
||||
covered: dict[int, int] = {}
|
||||
for b in got:
|
||||
for n in range(b.start, b.end + 1):
|
||||
covered[n] = covered.get(n, 0) + 1
|
||||
for n, raw in enumerate(lines, 1):
|
||||
heading = raw.startswith("#") and not (15 <= n <= 18)
|
||||
if raw.strip() and not heading:
|
||||
self.assertEqual(covered.get(n), 1, (n, raw))
|
||||
if heading:
|
||||
self.assertNotIn(n, covered, (n, raw))
|
||||
|
||||
def test_release_note_blocks_carry_the_version(self) -> None:
|
||||
got = split.blocks(self.kilde / "release-v1.10.0.md")
|
||||
self.assertEqual({b.version for b in got}, {"1.10.0"})
|
||||
self.assertEqual(split.blocks(self.kilde / "docs-en-basil.md")[0].version, None)
|
||||
|
||||
|
||||
class DocsHeadingsTest(Fixture):
|
||||
def test_titles_and_second_and_third_level_headings_of_docs_pages_only(self) -> None:
|
||||
got = split.docs_headings(self.kilde)
|
||||
texts = [(h.file, h.level, h.text) for h in got]
|
||||
self.assertIn(("docs-en-tomatoes.md", 1, "Growing tomatoes"), texts)
|
||||
self.assertIn(("docs-en-tomatoes.md", 3, "Pruning"), texts)
|
||||
self.assertFalse(any(f.startswith("release-") for f, _, _ in texts))
|
||||
self.assertFalse(any(t == "not a heading" for _, _, t in texts))
|
||||
|
||||
|
||||
class PickTest(Fixture):
|
||||
def blocks_doc(self, order: list[str]) -> dict[str, object]:
|
||||
return split.blocks_document(self.kilde, order)
|
||||
|
||||
def test_three_concepts_by_the_stated_rules(self) -> None:
|
||||
order = split.order([p.name for p in self.kilde.iterdir()])
|
||||
picked = split.pick(self.blocks_doc(order), self.kilde, focus="docs-en-tomatoes.md")
|
||||
names = [p.name for p in picked]
|
||||
self.assertEqual(len(names), 3)
|
||||
self.assertEqual(len(set(names)), 3)
|
||||
self.assertEqual(names[0], "Soil") # the focus page's heading with the most blocks
|
||||
# "Kitchen Garden" and "compost" are both in 2 release notes; the tie is broken by name
|
||||
self.assertEqual(names[1], "Kitchen Garden")
|
||||
self.assertEqual(names[2], "compost") # in 2 docs pages and 2 release notes, distinct from 1 and 2
|
||||
|
||||
def test_a_document_counts_for_a_term_only_where_it_stands_as_a_term(self) -> None:
|
||||
# Found in session 2: the word "and", backticked once, was counted in every document where it is prose.
|
||||
(self.kilde / "docs-en-zz.md").write_text("# Joining\n\n## Words\n\nUse `and` to join two beds.\n",
|
||||
encoding="utf-8")
|
||||
for v in ("2.0.0", "2.0.1", "2.0.2"):
|
||||
(self.kilde / f"release-v{v}.md").write_text(f"## {v}\n\n- Tomatoes and basil and mint.\n",
|
||||
encoding="utf-8")
|
||||
order = split.order([p.name for p in self.kilde.iterdir()])
|
||||
names = [p.name for p in split.pick(self.blocks_doc(order), self.kilde, focus="docs-en-tomatoes.md")]
|
||||
self.assertNotIn("and", names)
|
||||
self.assertEqual(names[1:], ["Kitchen Garden", "compost"])
|
||||
|
||||
def test_a_backticked_term_counts_its_length_after_trimming(self) -> None:
|
||||
# Found in session 2: "` or `" is four characters with its spaces, and became the two-letter term "or".
|
||||
got = split.terms("Use `compost` ` or ` ` ab ` ` mulch ` here.")
|
||||
self.assertEqual(got, ["compost", "mulch"])
|
||||
|
||||
def test_the_focus_page_must_be_in_the_order(self) -> None:
|
||||
with self.assertRaises(split.RunError):
|
||||
split.pick(self.blocks_doc(["docs-en-basil.md"]), self.kilde, focus="docs-en-tomatoes.md")
|
||||
|
||||
|
||||
class CliTest(Fixture):
|
||||
def test_order_refuses_names_that_differ_from_the_l4_cut(self) -> None:
|
||||
manifest = self.tmp / "m.json"
|
||||
manifest.write_text(json.dumps({"cut": {"names": ["docs-en-basil.md", "release-v1.2.3.md"]}}), encoding="utf-8")
|
||||
l4 = self.tmp / "l4"
|
||||
l4.mkdir()
|
||||
(l4 / "a.md").write_text("---\ntype: Concept\nsources:\n - id: a\n resource: /kilde/docs-en-basil.md\n"
|
||||
"---\n\nx\n", encoding="utf-8")
|
||||
out = self.tmp / "rekkefolge.txt"
|
||||
self.assertEqual(split.main(["order", "--manifest", str(manifest), "--l4", str(l4), "--out", str(out)]), 2)
|
||||
self.assertFalse(out.exists())
|
||||
(l4 / "b.md").write_text("---\ntype: Concept\nsources:\n - id: b\n resource: /kilde/release-v1.2.3.md\n"
|
||||
"---\n\nx\n", encoding="utf-8")
|
||||
self.assertEqual(split.main(["order", "--manifest", str(manifest), "--l4", str(l4), "--out", str(out)]), 0)
|
||||
self.assertEqual(out.read_text(encoding="utf-8").split(), ["docs-en-basil.md", "release-v1.2.3.md"])
|
||||
|
||||
def test_blocks_and_pick_refuse_to_overwrite(self) -> None:
|
||||
order = self.tmp / "order.txt"
|
||||
order.write_text("\n".join(split.order([p.name for p in self.kilde.iterdir()])) + "\n", encoding="utf-8")
|
||||
out = self.tmp / "blocks.json"
|
||||
self.assertEqual(split.main(["blocks", "--kilde", str(self.kilde), "--order", str(order), "--out", str(out)]), 0)
|
||||
data = json.loads(out.read_text(encoding="utf-8"))
|
||||
self.assertEqual(data["schema"], "okf-wiki/p1-blocks/1")
|
||||
self.assertEqual(split.main(["blocks", "--kilde", str(self.kilde), "--order", str(order), "--out", str(out)]), 2)
|
||||
picked = self.tmp / "begreper.txt"
|
||||
args = ["pick", "--blocks", str(out), "--kilde", str(self.kilde), "--focus", "docs-en-tomatoes.md",
|
||||
"--out", str(picked)]
|
||||
self.assertEqual(split.main(args), 0)
|
||||
self.assertEqual(len(picked.read_text(encoding="utf-8").strip().splitlines()), 3)
|
||||
self.assertEqual(split.main(args), 2)
|
||||
|
||||
def test_total_counts_blocks_in_every_file(self) -> None:
|
||||
expected = sum(len(split.blocks(p)) for p in self.kilde.iterdir())
|
||||
self.assertEqual(split.blocks_total(self.kilde), expected)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------------------------- step 2: D
|
||||
|
||||
|
||||
class DMapTest(Fixture):
|
||||
def build(self) -> Path:
|
||||
order = split.order([p.name for p in self.kilde.iterdir()])
|
||||
out = self.tmp / "D"
|
||||
dmap.build(split.blocks_document(self.kilde, order), self.kilde.parent, out, at="2026-10-02T12:00:00Z")
|
||||
return out
|
||||
|
||||
def test_d_passes_the_auditor_with_no_claims(self) -> None:
|
||||
out = self.build()
|
||||
self.assertEqual(before_link_rule(audit.audit(out, self.kilde.parent)), [])
|
||||
found = claims.extract(out, self.kilde.parent)
|
||||
self.assertEqual(found.claims, [])
|
||||
self.assertTrue(found.exceptions)
|
||||
|
||||
def test_register_links_point_to_map_sections(self) -> None:
|
||||
out = self.build()
|
||||
page = (out / "r-kitchen-garden.md").read_text(encoding="utf-8")
|
||||
self.assertIn("(k-docs-en-basil.md#soil)", page)
|
||||
self.assertIn("(k-release-v1-2-3.md#1-2-3)", page)
|
||||
|
||||
def test_register_keeps_docs_headings_and_terms_in_two_documents_only(self) -> None:
|
||||
out = self.build()
|
||||
names = {p.name for p in out.glob("r-*.md")}
|
||||
self.assertIn("r-pruning.md", names) # a docs heading (i), in one document
|
||||
self.assertIn("r-compost.md", names) # a backticked term in several documents
|
||||
self.assertNotIn("r-sweet-genovese.md", names) # a capitalised phrase in one document only
|
||||
self.assertNotIn("r-overview.md", names) # a stop-listed heading fails test 2
|
||||
|
||||
def test_quotes_stand_on_their_lines_and_stay_strings(self) -> None:
|
||||
out = self.build()
|
||||
doc = audit.read_document(out / "r-compost.md", "r-compost.md")
|
||||
assert doc.frontmatter is not None
|
||||
for s in doc.frontmatter["sources"]:
|
||||
self.assertIsInstance(s["x-quote"], str)
|
||||
line = (self.kilde / Path(s["resource"]).name).read_text(encoding="utf-8").split("\n")[s["x-locator"]["line"] - 1]
|
||||
self.assertIn(s["x-quote"], line)
|
||||
self.assertIsInstance(doc.frontmatter["generated"]["at"], str)
|
||||
|
||||
def test_a_quote_like_no_survives_a_yaml_round_trip(self) -> None:
|
||||
text = dmap.frontmatter({"type": "Concept", "sources": [{"id": "a", "x-quote": "no"}, {"id": "b", "x-quote": "1.10"}]})
|
||||
path = self.tmp / "x.md"
|
||||
path.write_text(text + "\n# x\n", encoding="utf-8")
|
||||
fm = audit.read_document(path, "x.md").frontmatter
|
||||
assert fm is not None
|
||||
self.assertEqual([s["x-quote"] for s in fm["sources"]], ["no", "1.10"])
|
||||
|
||||
def test_the_log_records_the_thresholds_and_build_refuses_an_existing_target(self) -> None:
|
||||
out = self.build()
|
||||
log = (out / "log.md").read_text(encoding="utf-8")
|
||||
self.assertIn("2 of the", log)
|
||||
with self.assertRaises(dmap.RunError):
|
||||
dmap.build({"order": [], "blocks": []}, self.kilde.parent, out, at="2026-10-02T12:00:00Z")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------------------------- step 3: K packing
|
||||
|
||||
|
||||
def entry(eid: str, title: str, aliases: tuple[str, ...] = ()) -> dict[str, object]:
|
||||
return {"id": eid, "title": title, "aliases": [title, *aliases], "kind": "skeleton", "merge_of": None,
|
||||
"origin": None}
|
||||
|
||||
|
||||
class KSkeletonTest(Fixture):
|
||||
def test_the_skeleton_is_an_index_of_docs_headings_that_pass_test_2(self) -> None:
|
||||
index = kpack.skeleton(self.kilde)
|
||||
titles = {e["title"] for e in index["entries"]}
|
||||
self.assertIn("Growing tomatoes", titles)
|
||||
self.assertIn("Pruning", titles)
|
||||
self.assertIn("Soil", titles)
|
||||
self.assertNotIn("Overview", titles)
|
||||
soil = [e for e in index["entries"] if e["title"] == "Soil"]
|
||||
self.assertEqual(len(soil), 1) # the same heading in two pages is one entry
|
||||
self.assertEqual(soil[0]["origin"]["file"], "docs-en-basil.md") # pages in name order, first one wins
|
||||
self.assertFalse(any(p.suffix == ".md" for p in self.tmp.rglob("*") if p.parent.name != "kilde"))
|
||||
|
||||
def test_norm(self) -> None:
|
||||
self.assertEqual(kpack.norm("The `Compost` bins"), "compost bin")
|
||||
self.assertEqual(kpack.norm("settings.json"), "settings.json")
|
||||
self.assertEqual(kpack.norm(" Kitchen Garden! "), "kitchen garden")
|
||||
self.assertEqual(kpack.norm("CLI flags"), "cli flag")
|
||||
|
||||
|
||||
class KIdentityTest(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
self.index = {"entries": [entry("kitchen-garden-planner", "Kitchen Garden planner"),
|
||||
entry("compost", "compost", ("compost heap",))]}
|
||||
|
||||
def frag(self, concept: str, status: str = "new", file: str = "docs-en-tomatoes.md", eid: str | None = None
|
||||
) -> dict[str, object]:
|
||||
f: dict[str, object] = {"concept": concept, "status": status, "anchor": {"file": file, "start": 8, "end": 9}}
|
||||
if eid:
|
||||
f["id"] = eid
|
||||
return f
|
||||
|
||||
def test_outcomes(self) -> None:
|
||||
r = kpack.resolve
|
||||
self.assertEqual(r(self.frag("x", "existing", eid="compost"), self.index), ("existing", "compost"))
|
||||
self.assertEqual(r(self.frag("Kitchen Garden planners"), self.index), ("existing", "kitchen-garden-planner"))
|
||||
self.assertEqual(r(self.frag("Compost heap"), self.index), ("existing", "compost"))
|
||||
self.assertEqual(r(self.frag("Kitchen Garden planner tool"), self.index),
|
||||
("merge-candidate", "kitchen-garden-planner"))
|
||||
self.assertEqual(r(self.frag("Garden hose"), self.index), ("new", None))
|
||||
self.assertEqual(r(self.frag("Garden hose", file="release-v1.2.3.md"), self.index), ("rejected", "release-note-new"))
|
||||
self.assertEqual(r(self.frag("Kitchen Garden planner tool", file="release-v1.2.3.md"), self.index),
|
||||
("rejected", "release-note-new"))
|
||||
self.assertEqual(r(self.frag("Troubleshooting"), self.index), ("rejected", "meta"))
|
||||
|
||||
def test_relevant_lines_by_phrase(self) -> None:
|
||||
got = kpack.relevant_lines(self.index, "We keep the compost heaps behind the shed.")
|
||||
self.assertEqual([e["id"] for e in got], ["compost"])
|
||||
self.assertEqual(kpack.relevant_lines(self.index, "Nothing about gardens here."), [])
|
||||
|
||||
|
||||
class KPackTest(Fixture):
|
||||
def blocks_doc(self) -> dict[str, object]:
|
||||
return split.blocks_document(self.kilde, split.order([p.name for p in self.kilde.iterdir()]))
|
||||
|
||||
def test_packs_of_five(self) -> None:
|
||||
self.assertEqual([len(p) for p in kpack.packs([f"n{i}" for i in range(19)], 5)], [5, 5, 5, 4])
|
||||
|
||||
def test_a_large_pack_is_split_into_parts_under_the_read_cap(self) -> None:
|
||||
big = "".join(f"Line {i} says the tomatoes are fine today and tomorrow, and the basil too.\n\n"
|
||||
for i in range(1110))
|
||||
(self.kilde / "docs-en-big.md").write_text("# Big\n\n" + big, encoding="utf-8")
|
||||
doc = split.blocks_document(self.kilde, ["docs-en-big.md"])
|
||||
parts = kpack.render_input(1, ["docs-en-big.md"], doc, [], self.kilde)
|
||||
self.assertGreater(sum(len(p) for p in parts), 130_000)
|
||||
self.assertEqual(len(parts), 3)
|
||||
self.assertTrue(all(len(p) <= kpack.PART_CHARS for p in parts))
|
||||
self.assertIn("docs-en-big.md › Big › 3–3", parts[0])
|
||||
|
||||
def test_validate(self) -> None:
|
||||
doc = self.blocks_doc()
|
||||
pack = ["docs-en-tomatoes.md", "release-v1.2.3.md"]
|
||||
good = {"concept": "Soil", "status": "existing", "id": "soil", "text": "Tomatoes need deep, loose soil.",
|
||||
"anchor": {"file": "docs-en-tomatoes.md", "start": 8, "end": 9}}
|
||||
cases = {
|
||||
"anchor-crosses-blocks": {**good, "anchor": {"file": "docs-en-tomatoes.md", "start": 8, "end": 11}},
|
||||
"quote-missing": {**good, "text": "Plants stand 40 cm apart."},
|
||||
"quote-not-in-span": {**good, "text": "Keep 40 cm.", "quote": "forty centimetres"},
|
||||
"sentence-count": {**good, "text": "One. Two. Three. Four."},
|
||||
"tests-missing": {**good, "status": "new", "concept": "Loose soil"},
|
||||
"anchor-outside-pack": {**good, "anchor": {"file": "docs-en-basil.md", "start": 7, "end": 7}},
|
||||
"malformed": {"concept": "Soil"},
|
||||
}
|
||||
no_text = {**good, "text": ""}
|
||||
accepted, rejected = kpack.validate([good, no_text, *cases.values()], pack, doc, self.kilde)
|
||||
self.assertEqual(len(accepted), 1)
|
||||
self.assertEqual(rejected["sentence-count"], 2) # the empty text and the four sentences
|
||||
for reason in cases:
|
||||
self.assertGreaterEqual(rejected[reason], 1, reason)
|
||||
ok = {**good, "text": "Plants stand forty centimetres apart.", "anchor": {"file": "docs-en-tomatoes.md",
|
||||
"start": 12, "end": 13}, "quote": "forty centimetres"}
|
||||
accepted, rejected = kpack.validate([ok], pack, doc, self.kilde)
|
||||
self.assertEqual((len(accepted), sum(rejected.values())), (1, 0))
|
||||
|
||||
|
||||
class KCliTest(Fixture):
|
||||
def test_skeleton_pack_and_accept(self) -> None:
|
||||
work = self.tmp / "work"
|
||||
blocks = self.tmp / "blocks.json"
|
||||
order = split.order([p.name for p in self.kilde.iterdir()])
|
||||
blocks.write_text(json.dumps(split.blocks_document(self.kilde, order)), encoding="utf-8")
|
||||
index = work / "K-work" / "index.json"
|
||||
self.assertEqual(kpack.main(["skeleton", "--kilde", str(self.kilde), "--out", str(index)]), 0)
|
||||
self.assertEqual(kpack.main(["skeleton", "--kilde", str(self.kilde), "--out", str(index)]), 2)
|
||||
root = str(self.kilde.parent)
|
||||
self.assertEqual(kpack.main(["pack", "1", "--blocks", str(blocks), "--index", str(index), "--kilde-root", root,
|
||||
"--work", str(work)]), 0)
|
||||
self.assertTrue((work / "K-oppgaver" / "pack-1-part-1.md").is_file())
|
||||
fragments = {"fragments": [
|
||||
{"concept": "Soil", "status": "existing", "id": "soil", "text": "Tomatoes need deep, loose soil.",
|
||||
"anchor": {"file": "docs-en-tomatoes.md", "start": 8, "end": 9}},
|
||||
{"concept": "Kitchen Garden", "status": "new", "text": "The Kitchen Garden keeps tomatoes and basil.",
|
||||
"anchor": {"file": "docs-en-tomatoes.md", "start": 26, "end": 26},
|
||||
"tests": {"topic_shape": "a named planner", "citation": "See the Kitchen Garden reference"}},
|
||||
{"concept": "Kitchen Garden", "status": "existing", "text": "The planner gained a command.",
|
||||
"anchor": {"file": "release-v1.2.3.md", "start": 3, "end": 3}},
|
||||
{"concept": "Watering reminders", "status": "new", "text": "Reminders were fixed.",
|
||||
"anchor": {"file": "release-v1.2.3.md", "start": 4, "end": 4},
|
||||
"tests": {"topic_shape": "x", "citation": "y"}},
|
||||
]}
|
||||
out = work / "K-fragments" / "pack-1.json"
|
||||
out.parent.mkdir(parents=True)
|
||||
out.write_text(json.dumps(fragments), encoding="utf-8")
|
||||
self.assertEqual(kpack.main(["accept", "1", "--work", str(work), "--blocks", str(blocks), "--kilde-root", root]), 0)
|
||||
acc = json.loads((work / "K-work" / "accepted.json").read_text(encoding="utf-8"))["fragments"]
|
||||
self.assertEqual([a["outcome"] for a in acc], ["existing", "new", "existing"])
|
||||
self.assertEqual(acc[2]["concept_id"], acc[1]["concept_id"]) # the release note enriches the new concept
|
||||
self.assertEqual(acc[2]["version"], "1.2.3")
|
||||
log = json.loads((work / "K-work" / "packs.json").read_text(encoding="utf-8"))
|
||||
self.assertEqual(log["packs"][0]["rejected"], {"release-note-new": 1})
|
||||
self.assertEqual(log["packs"][0]["proposed"], 4)
|
||||
# PM 02.10: the ids of the index lines sent, and the accepted fragments whose concept was not among them
|
||||
sent = log["packs"][0]["index_ids"]
|
||||
self.assertIn("soil", sent)
|
||||
self.assertNotIn(acc[1]["concept_id"], sent)
|
||||
self.assertEqual(log["packs"][0]["not_sent"], {"total": 2, "by_outcome": {"existing": 1, "new": 1}})
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------------------------- step 4: assembly
|
||||
|
||||
|
||||
def accepted(cid: str, file: str, start: int, end: int, text: str, pack: int, version: str | None = None,
|
||||
quote: str | None = None, quote_line: int | None = None) -> dict[str, object]:
|
||||
return {"pack": pack, "position": 1, "file": file, "start": start, "end": end, "section": "-", "text": text,
|
||||
"quote": quote, "quote_line": quote_line, "version": version, "concept_id": cid,
|
||||
"outcome": "existing"}
|
||||
|
||||
|
||||
class AssembleTest(Fixture):
|
||||
def setUp(self) -> None:
|
||||
super().setUp()
|
||||
self.work = self.tmp / "work"
|
||||
(self.work / "K-work").mkdir(parents=True)
|
||||
index = {"entries": [
|
||||
{**entry("soil", "Soil"), "origin": {"file": "docs-en-basil.md", "line": 4}},
|
||||
{**entry("pruning", "Pruning"), "origin": {"file": "docs-en-tomatoes.md", "line": 28}},
|
||||
{**entry("kitchen-garden", "Kitchen Garden"), "kind": "new"},
|
||||
{**entry("kitchen-garden-planner", "Kitchen Garden planner"), "kind": "merge-candidate",
|
||||
"merge_of": "kitchen-garden"},
|
||||
]}
|
||||
frags = [
|
||||
accepted("soil", "docs-en-tomatoes.md", 8, 9, "Tomatoes need deep, loose soil.", 1),
|
||||
accepted("kitchen-garden", "docs-en-tomatoes.md", 26, 26,
|
||||
"The Kitchen Garden keeps tomatoes and basil together.", 1, quote="Kitchen Garden", quote_line=26),
|
||||
accepted("kitchen-garden", "release-v1.2.3.md", 3, 3, "The planner gained a compost command.", 1, "1.2.3"),
|
||||
accepted("soil", "docs-en-basil.md", 6, 7, "Basil likes warm soil near the Kitchen Garden.", 1),
|
||||
accepted("kitchen-garden", "release-v1.10.0.md", 3, 3, "The planner shows compost levels.", 2, "1.10.0"),
|
||||
accepted("kitchen-garden-planner", "release-v1.9.2.md", 3, 3, "Seedlings get their own tray.", 2, "1.9.2"),
|
||||
]
|
||||
(self.work / "K-work" / "index.json").write_text(json.dumps(index), encoding="utf-8")
|
||||
(self.work / "K-work" / "accepted.json").write_text(json.dumps({"fragments": frags}), encoding="utf-8")
|
||||
self.out = self.tmp / "K"
|
||||
assemble.assemble(self.work, self.kilde.parent, self.out, at="2026-10-02T12:00:00Z")
|
||||
|
||||
def page(self, name: str) -> audit.Document:
|
||||
return audit.read_document(self.out / name, name)
|
||||
|
||||
def test_the_bundle_passes_the_auditor(self) -> None:
|
||||
self.assertEqual(before_link_rule(audit.audit(self.out, self.kilde.parent)), [])
|
||||
found = claims.extract(self.out, self.kilde.parent)
|
||||
self.assertEqual(claims.criterion_1(found)["total"], (6, 6))
|
||||
|
||||
def test_only_enriched_concepts_get_pages(self) -> None:
|
||||
names = {p.name for p in self.out.glob("*.md")} - {"index.md", "log.md"}
|
||||
self.assertEqual(names, {"soil.md", "kitchen-garden.md", "kitchen-garden-planner.md"})
|
||||
fm = self.page("kitchen-garden-planner.md").frontmatter
|
||||
assert fm is not None
|
||||
self.assertEqual(fm["x-merge-candidate"], "kitchen-garden")
|
||||
|
||||
def test_history_in_version_order_without_version_numbers_in_the_text(self) -> None:
|
||||
doc = self.page("kitchen-garden.md")
|
||||
body = doc.body
|
||||
self.assertLess(body.index("gained a compost command"), body.index("shows compost levels"))
|
||||
prose = " ".join(b.text for b in audit.prose_blocks(doc))
|
||||
self.assertEqual(view.version_numbers(audit.FOOTNOTE_REF.sub("", prose)), [])
|
||||
fm = doc.frontmatter
|
||||
assert fm is not None
|
||||
versions = [s for s in fm["sources"] if view.is_version_heading(s)]
|
||||
self.assertEqual(sorted(s["x-quote"] for s in versions), ["1.10.0", "1.2.3"])
|
||||
self.assertEqual(fm["x-fragments"], 3)
|
||||
|
||||
def test_code_quotes_skip_structural_lines_and_are_marked(self) -> None:
|
||||
fm = self.page("soil.md").frontmatter
|
||||
assert fm is not None
|
||||
first = fm["sources"][0]
|
||||
self.assertEqual(first["x-locator"], {"line": 8})
|
||||
self.assertEqual(first["x-quote-by"], "code")
|
||||
self.assertEqual(first["x-span"], [8, 9])
|
||||
kg = self.page("kitchen-garden.md").frontmatter
|
||||
assert kg is not None
|
||||
self.assertNotIn("x-quote-by", kg["sources"][0])
|
||||
self.assertEqual(assemble.code_quote(["```bash", "<Note>", "|---|---|", "", " real line "], 10),
|
||||
("real line", 14))
|
||||
|
||||
def test_a_concept_title_in_another_page_becomes_one_link(self) -> None:
|
||||
self.assertIn("[Kitchen Garden](kitchen-garden.md)", self.page("soil.md").body)
|
||||
|
||||
|
||||
class KPlusTest(AssembleTest):
|
||||
def test_copy_tasks_and_apply(self) -> None:
|
||||
kplus = self.tmp / "K+"
|
||||
assemble.copy(self.out, kplus)
|
||||
with self.assertRaises(assemble.RunError):
|
||||
assemble.copy(self.out, kplus)
|
||||
runs = assemble.consolidation_tasks(kplus, self.work, min_fragments=3, per_run=8)
|
||||
self.assertEqual([r.pages for r in runs], [["kitchen-garden"]])
|
||||
self.assertTrue(runs[0].task.is_file())
|
||||
old = self.page("kitchen-garden.md").body
|
||||
labels = sorted(set(audit.FOOTNOTE_REF.findall(old)))
|
||||
good = "The Kitchen Garden keeps tomatoes and basil together, and its planner tracks compost " + "".join(
|
||||
f"[^{x}]" for x in labels) + "."
|
||||
dropped = "The Kitchen Garden keeps tomatoes and basil together[^" + labels[0] + "]."
|
||||
added = good.replace(".", "[^zz].", 1)
|
||||
bodies = self.tmp / "bodies.json"
|
||||
bodies.write_text(json.dumps({"kitchen-garden": good, "soil": dropped}), encoding="utf-8")
|
||||
self.assertEqual(assemble.apply(kplus, bodies), (1, 1))
|
||||
self.assertIn("planner tracks compost", (kplus / "kitchen-garden.md").read_text(encoding="utf-8"))
|
||||
self.assertEqual(before_link_rule(audit.audit(kplus, self.kilde.parent)), [])
|
||||
bodies.write_text(json.dumps({"kitchen-garden": added}), encoding="utf-8")
|
||||
self.assertEqual(assemble.apply(kplus, bodies), (0, 1))
|
||||
|
||||
|
||||
class NoModelTest(unittest.TestCase):
|
||||
def test_trial_modules_import_no_network_or_model_client(self) -> None:
|
||||
for path in sorted((ROOT / "evals" / "p1").glob("*.py")):
|
||||
source = path.read_text(encoding="utf-8")
|
||||
for banned in BANNED:
|
||||
self.assertNotIn(f"import {banned}", source, path.name)
|
||||
self.assertNotIn(f"from {banned}", source, path.name)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
366
evals/tests/test_p1_report.py
Normal file
366
evals/tests/test_p1_report.py
Normal file
|
|
@ -0,0 +1,366 @@
|
|||
"""Tests for the P1 method trial's report side (TRIAL, branch p1-metodeprove, not v1 code).
|
||||
|
||||
Covers the measures (evals/p1/measure.py), the cost harvester (cost.py), the side-by-side page (sidebyside.py) and the
|
||||
two verification scripts. None of these call a model. The fixtures are synthetic text about gardening (CLAUDE.md
|
||||
rule 3: no text from the Claude Code documentation is committed).
|
||||
Run: python3 -m unittest discover -s evals/tests -p 'test_p1_report.py'
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import yaml
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
sys.path.insert(0, str(ROOT))
|
||||
sys.path.insert(0, str(ROOT / "evals"))
|
||||
|
||||
from okfwiki import audit, view # noqa: E402
|
||||
from p1 import cost, measure, sidebyside, split # noqa: E402
|
||||
|
||||
BANNED = ("anthropic", "openai", "urllib", "http.client", "requests", "subprocess", "socket")
|
||||
|
||||
DOC_A = """# Alpha
|
||||
|
||||
## Beds
|
||||
|
||||
Raised beds drain well.
|
||||
They warm early in spring.
|
||||
|
||||
- Line one item.
|
||||
- Line two item.
|
||||
|
||||
## Paths
|
||||
|
||||
Gravel paths stay dry.
|
||||
"""
|
||||
|
||||
REL = """## 2.0.0
|
||||
|
||||
- Beds got labels.
|
||||
"""
|
||||
|
||||
|
||||
def src(sid: str, file: str, line: int, quote: str, span: tuple[int, int] | None = None) -> dict[str, Any]:
|
||||
s: dict[str, Any] = {"id": sid, "resource": f"/kilde/{file}", "title": file, "x-locator": {"line": line},
|
||||
"x-quote": quote}
|
||||
if span:
|
||||
s["x-span"] = list(span)
|
||||
return s
|
||||
|
||||
|
||||
def write_page(folder: Path, name: str, title: str, body: str, sources: list[dict[str, Any]],
|
||||
**extra: Any) -> None:
|
||||
fm = {"type": "Concept", "title": title, "description": title, "sources": sources, **extra,
|
||||
"generated": {"by": "test", "at": "2026-10-02T12:00:00Z"}}
|
||||
text = "---\n" + yaml.safe_dump(fm, sort_keys=False) + "---\n\n# " + title + "\n\n" + body + "\n"
|
||||
folder.mkdir(parents=True, exist_ok=True)
|
||||
(folder / name).write_text(text, encoding="utf-8")
|
||||
|
||||
|
||||
class Fixture(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
self._tmp = tempfile.TemporaryDirectory()
|
||||
self.tmp = Path(self._tmp.name)
|
||||
self.root = self.tmp / "wiki"
|
||||
self.kilde = self.root / "kilde"
|
||||
self.kilde.mkdir(parents=True)
|
||||
(self.kilde / "docs-en-a.md").write_text(DOC_A, encoding="utf-8")
|
||||
(self.kilde / "release-v2.0.0.md").write_text(REL, encoding="utf-8")
|
||||
self.order = ["docs-en-a.md", "release-v2.0.0.md"]
|
||||
self.blocks = split.blocks_document(self.kilde, self.order)
|
||||
self.bundle = self.tmp / "B"
|
||||
write_page(self.bundle, "beds.md", "Beds",
|
||||
"Raised beds drain well.[^b1] The beds have a heading.[^b2] Items begin.[^b3] Two items.[^b4]\n\n"
|
||||
"## History\n\n- Beds got labels.[^r1][^v1]\n\n| a | b |",
|
||||
[src("b1", "docs-en-a.md", 5, "Raised beds drain well.", (5, 6)),
|
||||
src("b2", "docs-en-a.md", 3, "Beds", (3, 3)),
|
||||
src("b3", "docs-en-a.md", 8, "Line one item.", (8, 13)),
|
||||
src("b4", "docs-en-a.md", 9, "Line two item.", (8, 9)),
|
||||
src("v1", "release-v2.0.0.md", 1, "2.0.0"),
|
||||
src("r1", "release-v2.0.0.md", 3, "Beds got labels.", (3, 3)),
|
||||
src("u1", "docs-en-a.md", 13, "Gravel paths stay dry.", (13, 13))],
|
||||
**{"x-aliases": ["Beds", "Raised beds"]})
|
||||
write_page(self.bundle, "labels.md", "Labels", "Beds got labels.[^r1][^v1] See also: [Beds](beds.md)",
|
||||
[src("v1", "release-v2.0.0.md", 1, "2.0.0"), src("r1", "release-v2.0.0.md", 3, "Beds got labels.")])
|
||||
(self.bundle / "index.md").write_text('---\nokf_version: "0.2"\n---\n\n# Concepts\n\n- [Beds](beds.md)\n'
|
||||
'- [Labels](labels.md)\n', encoding="utf-8")
|
||||
(self.bundle / "log.md").write_text("# Log\n\n## 2026-10-02\n* built\n", encoding="utf-8")
|
||||
|
||||
def tearDown(self) -> None:
|
||||
self._tmp.cleanup()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------------------------- step 5: measures
|
||||
|
||||
|
||||
class CoverageTest(Fixture):
|
||||
def test_cited_entries_by_locator_line_only(self) -> None:
|
||||
got = measure.coverage(self.bundle, self.root, self.blocks, self.order)
|
||||
# covered: 5-6 (b1), 8 (b3's locator; its wide span is ignored), 9 (b4), release 3 (r1); not 13 (u1 uncited)
|
||||
self.assertEqual((got.covered, got.total), (4, 5))
|
||||
|
||||
def test_a_stopped_variant_counts_only_the_sources_it_built(self) -> None:
|
||||
got = measure.coverage(self.bundle, self.root, self.blocks, self.order[:1])
|
||||
self.assertEqual((got.covered, got.total), (3, 4))
|
||||
|
||||
def test_per_pack_position(self) -> None:
|
||||
got = measure.coverage(self.bundle, self.root, self.blocks, self.order, pack_size=5)
|
||||
self.assertEqual(got.per_position, {1: (3, 4, 1), 2: (1, 1, 1)})
|
||||
|
||||
|
||||
class StructureTest(Fixture):
|
||||
def test_shares_precision_and_concepts_per_source(self) -> None:
|
||||
s = measure.structure(self.bundle, self.root, self.blocks, self.order)
|
||||
self.assertEqual(s["pages"], 2)
|
||||
self.assertEqual(s["concepts_per_source"], {"docs-en-a.md": 1, "release-v2.0.0.md": 1})
|
||||
# version sources are left out of the quote and precision counts
|
||||
self.assertEqual(s["quotes"]["entries"], 6)
|
||||
self.assertEqual(s["quotes"]["structural"], 1) # b2 quotes a heading line
|
||||
self.assertEqual(s["precision"], {"line": 3, "block": 1, "section": 1, "document": 1})
|
||||
self.assertEqual(s["islands"]["total"], sum(not v for v in view.related(
|
||||
audit.load_bundle(self.bundle, self.root).concepts).values()))
|
||||
|
||||
def test_duplicates_by_norm_alias_and_jaccard_in_any_bundle(self) -> None:
|
||||
b = self.tmp / "dups"
|
||||
for name, title in (("a.md", "Beds"), ("b.md", "Bed"), ("c.md", "Raised beds"), ("d.md", "Gravel path"),
|
||||
("e.md", "Gravel path rules"), ("f.md", "Tools")):
|
||||
write_page(b, name, title, "x[^s].", [src("s", "docs-en-a.md", 5, "Raised")],
|
||||
**({"x-aliases": ["Beds", "Raised beds"]} if name == "a.md" else {}))
|
||||
pairs = measure.duplicates(audit.load_bundle(b, self.root).concepts)
|
||||
self.assertIn(("a.md", "b.md", "norm"), pairs)
|
||||
self.assertIn(("a.md", "c.md", "alias"), pairs)
|
||||
self.assertIn(("d.md", "e.md", "jaccard"), pairs)
|
||||
self.assertFalse(any("f.md" in p for p in pairs))
|
||||
|
||||
def test_a_wrong_root_raises(self) -> None:
|
||||
with self.assertRaises(measure.RunError):
|
||||
measure.structure(self.bundle, self.tmp / "nowhere", self.blocks, self.order)
|
||||
|
||||
|
||||
class AuditRecordTest(Fixture):
|
||||
def test_uncited_sentences_are_classed_by_cause_and_yaml_form_apart(self) -> None:
|
||||
write_page(self.bundle, "odd.md", "Odd", "```bash\n\nPlain words here.\n\nSee also: [Beds](beds.md)",
|
||||
[{"id": "y", "resource": "/kilde/docs-en-a.md", "x-locator": {"line": 5}, "x-quote": False}])
|
||||
out = self.tmp / "resultat.json"
|
||||
rec = measure.audit_record(self.bundle, self.root, "K", out)
|
||||
self.assertEqual(rec["exit"], 1)
|
||||
# labels.md in the shared fixture holds one more link lead-in ("See also: [Beds](beds.md)")
|
||||
self.assertEqual(rec["uncited"], {"fence line": 1, "link lead-in": 2, "prose": 1, "table row": 1})
|
||||
self.assertEqual(rec["yaml_form"], 1)
|
||||
self.assertEqual(json.loads(out.read_text(encoding="utf-8"))["variants"]["K"]["exit"], 1)
|
||||
with self.assertRaises(measure.RunError):
|
||||
measure.audit_record(self.bundle, self.root, "K", out) # one record per variant
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------------------------- step 6: cost
|
||||
|
||||
|
||||
def msg(mid: str, inp: int, cr: int, cw: int, out: int, stop: str | None = "end_turn") -> dict[str, Any]:
|
||||
return {"type": "assistant", "message": {"id": mid, "stop_reason": stop, "usage": {
|
||||
"input_tokens": inp, "cache_read_input_tokens": cr, "cache_creation_input_tokens": cw, "output_tokens": out}}}
|
||||
|
||||
|
||||
def jsonl(path: Path, lines: list[dict[str, Any]]) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text("\n".join(json.dumps(x) for x in lines) + "\n", encoding="utf-8")
|
||||
|
||||
|
||||
class HarvestTest(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
self._tmp = tempfile.TemporaryDirectory()
|
||||
self.proj = Path(self._tmp.name)
|
||||
user = {"type": "user", "message": {"content": "task"}}
|
||||
jsonl(self.proj / "s1.jsonl", [user, msg("m0", 5, 100, 10, 7)])
|
||||
sub = self.proj / "s1" / "subagents"
|
||||
# a streamed message id appears twice with growing output: counted once, at its max
|
||||
jsonl(sub / "agent-a1.jsonl", [user, msg("x1", 3, 20_000, 2_000, 10, None), msg("x1", 3, 20_000, 2_000, 50),
|
||||
msg("x2", 1, 22_000, 500, 900)])
|
||||
(sub / "agent-a1.meta.json").write_text(json.dumps({"agentType": "p1-compiler", "description": "P1-K-pack1"}))
|
||||
jsonl(sub / "agent-a2.jsonl", [user, msg("y1", 2, 19_000, 1_000, 5)])
|
||||
(sub / "agent-a2.meta.json").write_text(json.dumps({"agentType": "p1-compiler", "description": "load test"}))
|
||||
jsonl(sub / "agent-a3.jsonl", [user, msg("z1", 1, 1, 1, 1, None)])
|
||||
(sub / "agent-a3.meta.json").write_text(json.dumps({"agentType": "p1-consolidator",
|
||||
"description": "P1-K+-run1"}))
|
||||
# a session not listed in okter.txt: its transcript is never opened
|
||||
jsonl(self.proj / "s9" / "subagents" / "agent-b1.jsonl", [user, msg("q", 1, 1, 1, 1)])
|
||||
(self.proj / "s9" / "subagents" / "agent-b1.meta.json").write_text(
|
||||
json.dumps({"agentType": "p1-compiler", "description": "P1-K-pack2"}))
|
||||
self.okter = self.proj / "okter.txt"
|
||||
self.okter.write_text("s1 step8\ns1 step9\n", encoding="utf-8")
|
||||
|
||||
def tearDown(self) -> None:
|
||||
self._tmp.cleanup()
|
||||
|
||||
def test_four_categories_turns_startup_and_attribution(self) -> None:
|
||||
got = cost.harvest(self.proj, cost.sessions(self.okter))
|
||||
self.assertEqual(got["sessions_read"], ["s1"])
|
||||
runs = {r["unit"]: r for r in got["runs"]}
|
||||
k = runs["pack1"]
|
||||
self.assertEqual((k["variant"], k["input"], k["cache_read"], k["cache_write"], k["output"]),
|
||||
("K", 4, 42_000, 2_500, 950))
|
||||
self.assertEqual(k["turns"], 2)
|
||||
self.assertEqual(k["startup"], 3 + 20_000 + 2_000)
|
||||
self.assertEqual(runs["run1"]["variant"], "K+")
|
||||
self.assertEqual(runs["run1"]["stop_reason"], None) # a stopped agent is listed, not dropped
|
||||
self.assertEqual([u["description"] for u in got["unattributed"]], ["load test"])
|
||||
self.assertEqual(got["main"]["s1"]["cache_read"], 100)
|
||||
self.assertNotIn("pack2", runs)
|
||||
|
||||
def test_output_never_recorded_is_derived_from_the_next_messages_input_growth(self) -> None:
|
||||
# Found in session 2: a long streamed message keeps only its first usage record (output 2, no stop
|
||||
# reason). The next message's input side grew by that output plus the text injected between them (tool
|
||||
# results and rendered attachments), counted high at 2.5 characters per token so the output is a lower bound.
|
||||
def result(text: str) -> dict[str, Any]:
|
||||
return {"type": "user", "message": {"content": [{"type": "tool_result", "content": text}]}}
|
||||
|
||||
sub = self.proj / "s1" / "subagents"
|
||||
jsonl(sub / "agent-a4.jsonl", [{"type": "user", "message": {"content": "task"}},
|
||||
msg("w1", 2, 0, 10_000, 300, "tool_use"), result("x" * 400),
|
||||
msg("w2", 2, 10_000, 50_000, 2, None), msg("w2", 2, 10_000, 50_000, 2, None),
|
||||
result("y" * 40), msg("w3", 2, 60_000, 5_010, 20)])
|
||||
(sub / "agent-a4.meta.json").write_text(json.dumps({"agentType": "p1-compiler", "description": "P1-K-pack3"}))
|
||||
runs = {r["unit"]: r for r in cost.harvest(self.proj, cost.sessions(self.okter))["runs"]}
|
||||
w = runs["pack3"]
|
||||
self.assertEqual(w["output_recorded"], 300 + 2 + 20)
|
||||
self.assertEqual(w["output"], 300 + (5_010 - 40 * 2 // 5) + 20)
|
||||
self.assertEqual((w["output_derived_messages"], w["output_unrecorded"]), (1, 0))
|
||||
self.assertEqual(w["total"], 2 + 2 + 2 + 70_000 + 65_010 + w["output"])
|
||||
# the last message of a run has no next message: its output stays as recorded and is flagged
|
||||
self.assertEqual((runs["run1"]["output"], runs["run1"]["output_unrecorded"]), (1, 1))
|
||||
self.assertEqual(runs["pack1"]["output_derived_messages"], 0)
|
||||
|
||||
def test_growth_from_injected_attachments_is_not_counted_as_output(self) -> None:
|
||||
# The load test's pattern: a short call, then 5133 characters of rendered attachments before the next turn.
|
||||
sub = self.proj / "s1" / "subagents"
|
||||
jsonl(sub / "agent-a5.jsonl", [{"type": "user", "message": {"content": "task"}},
|
||||
msg("v1", 2, 0, 18_602, 16, None),
|
||||
{"type": "attachment", "rendered": "z" * 5_133},
|
||||
{"type": "user", "message": {"content": [{"type": "tool_result",
|
||||
"content": "w" * 151}]}},
|
||||
msg("v2", 2, 18_602, 1_894, 24, None)])
|
||||
(sub / "agent-a5.meta.json").write_text(json.dumps({"agentType": "p1-compiler", "description": "P1-LOAD-x"}))
|
||||
runs = {r["unit"]: r for r in cost.harvest(self.proj, cost.sessions(self.okter))["runs"]}
|
||||
self.assertEqual((runs["x"]["output"], runs["x"]["output_derived_messages"]), (16 + 24, 0))
|
||||
|
||||
|
||||
class ProjectionTest(unittest.TestCase):
|
||||
def test_least_squares_on_cost_per_block_against_blocks_read(self) -> None:
|
||||
p = cost.project([(600_000, 600, 0), (220_000, 200, 600)], total_blocks=3000)
|
||||
self.assertAlmostEqual(p["fit"]["a"], 1000)
|
||||
self.assertAlmostEqual(p["fit"]["b"], 1 / 6)
|
||||
self.assertAlmostEqual(p["fit"]["value"], 3_750_000)
|
||||
self.assertAlmostEqual(p["flat"], 3_075_000)
|
||||
|
||||
def test_one_point_has_no_growth_term(self) -> None:
|
||||
p = cost.project([(600_000, 600, 0)], total_blocks=3000)
|
||||
self.assertEqual(p["fit"]["b"], 0)
|
||||
self.assertEqual(p["fit"]["note"], "one point, no growth term")
|
||||
self.assertAlmostEqual(p["fit"]["value"], 3_000_000)
|
||||
|
||||
def test_stop_per_block_against_naive_per_source(self) -> None:
|
||||
s = cost.stop_k(pack1_cost=600_000, pack1_blocks=600, cut_blocks=1200, pack1_sources=5, cut_sources=19)
|
||||
self.assertEqual(s["per_block"], 1_200_000)
|
||||
self.assertEqual(s["per_source"], 2_280_000)
|
||||
self.assertTrue(s["stop"])
|
||||
self.assertFalse(cost.stop_k(500_000, 600, 1200, 5, 19)["stop"]) # exactly 1 M does not stop
|
||||
self.assertTrue(cost.stop_k(500_001, 600, 1200, 5, 19)["stop"])
|
||||
|
||||
def test_kplus_stop(self) -> None:
|
||||
self.assertFalse(cost.stop_kplus(run1_cost=200_000, run1_pages=8, pages_total=40)["stop"])
|
||||
self.assertTrue(cost.stop_kplus(run1_cost=200_001, run1_pages=8, pages_total=40)["stop"])
|
||||
|
||||
|
||||
class AllocationTest(unittest.TestCase):
|
||||
def test_tokens_per_source_by_block_share_and_the_two_means(self) -> None:
|
||||
names = [f"s{i:02d}.md" for i in range(1, 20)]
|
||||
blocks_doc = {"order": names, "blocks": [{"file": n, "start": k, "end": k} for n in names for k in (1, 2)]}
|
||||
packs = [names[i:i + 5] for i in range(0, 19, 5)]
|
||||
costs = {1: 1000, 2: 500, 3: 500, 4: 800}
|
||||
per = cost.allocate(costs, packs, blocks_doc)
|
||||
self.assertEqual(per["s01.md"], 200)
|
||||
self.assertEqual(per["s16.md"], 200)
|
||||
means = cost.means(per, names)
|
||||
self.assertEqual(means["1-5"], 200)
|
||||
self.assertEqual(means["15-19"], 180) # source 15 in pack 3 (100), 16-19 in pack 4 (200 each)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------------------------- step 7: side by side
|
||||
|
||||
|
||||
class SideBySideTest(Fixture):
|
||||
def setUp(self) -> None:
|
||||
super().setUp()
|
||||
self.begreper = self.tmp / "begreper.txt"
|
||||
self.begreper.write_text("Beds\tBeds;Raised beds\nLabels\tLabels\nCompost\tCompost\n", encoding="utf-8")
|
||||
self.d = self.tmp / "D"
|
||||
write_page(self.d, "r-beds.md", "Raised beds", "- [Alpha](k.md)", [src("o1", "docs-en-a.md", 5, "Raised beds")])
|
||||
self.l4 = self.tmp / "L4"
|
||||
write_page(self.l4, "x.md", "THE raised-beds, explained!", "Beds drain.[^b1]",
|
||||
[src("b1", "docs-en-a.md", 5, "Raised beds drain well.")])
|
||||
write_page(self.l4, "y.md", "Compost", "Compost helps.[^b1]",
|
||||
[src("b1", "docs-en-a.md", 99, "Compost")]) # a link past the end of the file
|
||||
|
||||
def columns(self) -> list[tuple[str, Path | None]]:
|
||||
return [("K", self.bundle), ("K+", None), ("D", self.d), ("L4", self.l4)]
|
||||
|
||||
def test_three_concepts_in_four_columns_with_the_empty_states(self) -> None:
|
||||
out = self.tmp / "side.html"
|
||||
sidebyside.render(sidebyside.read_begreper(self.begreper), self.columns(), self.root, out)
|
||||
html = out.read_text(encoding="utf-8")
|
||||
self.assertEqual(html.count('<tr class="concept">'), 3)
|
||||
self.assertEqual(html.count('<td class="cell'), 12)
|
||||
self.assertIn("ikke bygget", html) # K+ stopped
|
||||
self.assertIn("ikke i registeret", html) # D has no Labels register page
|
||||
self.assertIn("ingen side", html) # K is built and has no Compost page
|
||||
self.assertNotIn("http", html)
|
||||
|
||||
def test_l4_finds_a_title_with_other_case_and_punctuation(self) -> None:
|
||||
found = sidebyside.find(self.l4, ("Raised beds", ("Raised beds",)), l4=True)
|
||||
self.assertEqual(found, ["x.md"])
|
||||
|
||||
def test_the_link_check_flags_a_line_past_the_end(self) -> None:
|
||||
out = self.tmp / "side.html"
|
||||
sidebyside.render(sidebyside.read_begreper(self.begreper), self.columns(), self.root, out)
|
||||
resolved, total = sidebyside.check(out, self.root)
|
||||
self.assertEqual(total - resolved, 1)
|
||||
self.assertEqual(sidebyside.main(["check", "--html", str(out), "--kilde-root", str(self.root)]), 1)
|
||||
|
||||
|
||||
class ScriptsTest(unittest.TestCase):
|
||||
SCRIPTS = (ROOT / "tests" / "p1_repo.sh", ROOT / "tests" / "p1_trial.sh")
|
||||
|
||||
def test_bash_syntax_and_ascii(self) -> None:
|
||||
for script in self.SCRIPTS:
|
||||
self.assertEqual(subprocess.run(["bash", "-n", str(script)]).returncode, 0, script.name)
|
||||
script.read_bytes().decode("ascii")
|
||||
|
||||
def test_the_trial_check_fails_every_check_when_nothing_was_built(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
env = {**os.environ, "P1_ROOT": tmp}
|
||||
res = subprocess.run(["bash", str(self.SCRIPTS[1])], env=env, capture_output=True, text=True, cwd=ROOT)
|
||||
self.assertEqual(res.returncode, 1)
|
||||
lines = [x for x in res.stdout.splitlines() if x.strip()]
|
||||
self.assertGreaterEqual(len(lines), 8)
|
||||
self.assertTrue(all(x.startswith("FAIL") for x in lines), res.stdout)
|
||||
|
||||
|
||||
class NoModelTest(unittest.TestCase):
|
||||
def test_trial_modules_import_no_network_or_model_client(self) -> None:
|
||||
for path in sorted((ROOT / "evals" / "p1").glob("*.py")):
|
||||
source = path.read_text(encoding="utf-8")
|
||||
for banned in BANNED:
|
||||
self.assertNotIn(f"import {banned}", source, path.name)
|
||||
self.assertNotIn(f"from {banned}", source, path.name)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Loading…
Add table
Add a link
Reference in a new issue