okf-wiki/tests/test_part_metrics.py
Kjell Tore Guttormsen 0fd86a23a3
okf-wiki 1.1.1
First public release, published as one snapshot. What changed between
versions is in CHANGELOG.md.

Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
2026-11-14 21:03:20 +01:00

290 lines
17 KiB
Python

"""Tests for tools/part_metrics.py: the numbers per pack and per source of a build folder (contract K7), without a model
(wave 3 order 11, plan revision 5 Step 11: cost, catalogue size, duplicate concepts, pages without a home before and
after the guard, main-page out-degree, with a growth fit).
The fixture is a build folder of two packs of two sources each, written by hand; every expected number below is worked
out by hand from it. The text is synthetic, about bread and gardening (CLAUDE.md rule 3).
Run: python3 -m unittest discover -s tests -p 'test_part_metrics.py'
"""
from __future__ import annotations
import contextlib
import importlib.util
import io
import json
import sys
import tempfile
import unittest
from pathlib import Path
from typing import Any
from okfwiki import cost, kpack
ROOT = Path(__file__).resolve().parents[1]
# Loaded from its file, as tests/test_x_fields.py loads its tool; mypy checks the tool when it is named.
_SPEC = importlib.util.spec_from_file_location("part_metrics", ROOT / "tools" / "part_metrics.py")
assert _SPEC is not None and _SPEC.loader is not None
pm: Any = importlib.util.module_from_spec(_SPEC)
sys.modules["part_metrics"] = pm
_SPEC.loader.exec_module(pm)
SOURCES = [["bread-a.md", "bread-b.md"], ["garden-a.md", "garden-b.md"]]
BLOCKS_PER_FILE = {"bread-a.md": 3, "bread-b.md": 1, "garden-a.md": 2, "garden-b.md": 2}
def write(path: Path, text: str) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(text, encoding="utf-8")
def write_json(path: Path, doc: Any) -> None:
write(path, json.dumps(doc, ensure_ascii=False, indent=1) + "\n")
def run(name: str, line: str, unit: str, total: int) -> dict[str, Any]:
m = cost.PREFIX.match(f"V1-fx-{name}-{unit}")
assert m is not None
return {"description": f"V1-fx-{name}-{unit}", "run": "fx", "line": line, "unit": unit,
"pack": cost.pack_of(unit), "total": total}
def page(title: str, mains: list[str], body: str) -> str:
head = f"---\ntype: Concept\ntitle: {title}\n"
if mains:
head += "x-main-for:\n" + "".join(f" - /kilde/{m}\n" for m in mains)
return head + "---\n\n" + body + "\n\n[^a]: A note.\n"
def make_root(root: Path) -> None:
work = root / "work"
blocks = []
for name in [n for pack in SOURCES for n in pack]:
for i in range(BLOCKS_PER_FILE[name]):
blocks.append({"file": name, "section": "Part", "start": 2 * i + 1, "end": 2 * i + 2})
write_json(work / "blocks.json", {"schema": "okf-wiki/blocks/1", "order": [n for p in SOURCES for n in p],
"blocks": blocks})
write_json(work / "K-work" / "packs.json", {"schema": kpack.PACKS_SCHEMA, "size": 2, "packs": [
{"pack": 1, "sources": SOURCES[0], "blocks": 4, "index_total": 3, "main_pages_sent": 0, "parts": 1,
"accepted": 3},
{"pack": 2, "sources": SOURCES[1], "blocks": 4, "index_total": 4, "main_pages_sent": 2, "parts": 2,
"attempts": [{"proposed": 5, "rejected": {"quote-missing": 1}}], "accepted": 5,
"guard": {"asked": {"pages": ["sourdoughs"], "sources": ["kilde/garden-b.md"], "catalogue": []},
"parts": 1, "answered": {"homes": 1, "main_pages": 1}}},
]})
def entry(cid: str, title: str, kind: str, file: str) -> dict[str, Any]:
return {"id": cid, "title": title, "aliases": [title], "kind": kind, "merge_of": None,
"origin": {"file": file, "line": 1}}
write_json(work / "K-work" / "index.json", {"schema": kpack.INDEX_SCHEMA, "entries": [
entry("sourdough", "Sourdough", "skeleton", "bread-a.md"),
entry("starter", "Starter", "skeleton", "bread-a.md"), # in the index, never a page
entry("compost", "Compost", "skeleton", "garden-a.md"),
entry("crust", "Crust", "new", "bread-b.md"),
entry("mulch", "Mulch", "new", "garden-b.md"),
entry("sourdoughs", "Sourdoughs", "new", "garden-a.md"), # normalises to "sourdough": a duplicate
]})
def frag(pack: int, file: str, cid: str) -> dict[str, Any]:
return {"pack": pack, "position": SOURCES[pack - 1].index(file), "file": file, "start": 1, "end": 2,
"concept_id": cid, "provenance": "documented", "outcome": "existing"}
write_json(work / "K-work" / "accepted.json", {"schema": kpack.ACCEPTED_SCHEMA, "fragments": [
frag(1, "bread-a.md", "sourdough"), frag(1, "bread-b.md", "crust"), frag(1, "bread-b.md", "sourdough"),
frag(2, "garden-a.md", "compost"), frag(2, "garden-a.md", "sourdoughs"), frag(2, "garden-b.md", "mulch"),
frag(2, "garden-b.md", "crust"), frag(2, "garden-b.md", "compost"),
], "main_pages": [
{"source": "bread-a.md", "concept_id": "sourdough", "pack": 1, "by": "compiler"},
{"source": "bread-b.md", "concept_id": "crust", "pack": 1, "by": "compiler"},
{"source": "garden-a.md", "concept_id": "compost", "pack": 2, "by": "compiler"},
{"source": "garden-b.md", "concept_id": "mulch", "pack": 2, "by": "guard"},
], "homes": [
{"source": "garden-a.md", "concept_id": "sourdoughs", "home": "compost", "pack": 2, "by": "guard"},
]})
# The compiler's task parts as kpack writes them: pack 1 has no main page yet; pack 2's catalogue runs into part 2.
write(work / "K-oppgaver" / "pack-1-part-1.md",
"<!-- part 1 of 1 -->\n# K pack 1: 2 sources\n\n## Index (concepts named in these sources)\n\n"
"- id: sourdough | Sourdough\n\n## Main pages (homes you may choose)\n\n"
"(none yet: the main pages you name in this pack are the homes you may choose)\n")
write(work / "K-oppgaver" / "pack-2-part-1.md",
"<!-- part 1 of 2 -->\n# K pack 2: 2 sources\n\n## Index (concepts named in these sources)\n\n"
"- id: sourdough | Sourdough | main page\n- id: compost | Compost\n\n## Main pages (homes you may choose)\n\n")
write(work / "K-oppgaver" / "pack-2-part-2.md", "<!-- part 2 of 2 -->\n- sourdough | Sourdough\n- crust | Crust\n")
# The auditor's JSON after each run-after: pack 2 needed the guard (audit 1 before it, audit 2 after it).
def audit_doc(findings: list[tuple[str, str]]) -> dict[str, Any]:
return {"findings": [{"check": c, "group": "links", "file": f, "line": None, "message": "m"}
for c, f in findings], "summary": {}, "link_rule": {}}
write_json(work / "K-audit" / "p1-audit-1.json", audit_doc([]))
write_json(work / "K-audit" / "p2-audit-1.json", audit_doc([
("no-main-page-link", "sourdoughs.md"), ("main-page-missing", "kilde/garden-b.md"),
("broken-link", "crust.md")]))
write_json(work / "K-audit" / "p2-audit-2.json", audit_doc([]))
# The link passes: which sentences each pass asked about, and the links it gave.
write_json(work / "K-links" / "p1" / "sentences.json", {"schema": "okf-wiki/link-sentences/1", "pack": 1,
"pages": {"sourdough.md": ["s1-1"]}})
write_json(work / "K-links" / "p2" / "sentences.json", {"schema": "okf-wiki/link-sentences/1", "pack": 2,
"pages": {"crust.md": ["c2-1"], "compost.md": ["k2-1"],
"sourdoughs.md": ["d2-1"]}})
def link(p: str, s: str, target: str) -> dict[str, Any]:
return {"page": p, "sentence": s, "anchor": "x", "at": 0, "target": target}
write_json(work / "links.json", {"schema": "okf-wiki/links/1", "links": [
link("sourdough.md", "s1-1", "crust.md"),
link("crust.md", "c2-1", "sourdough.md"),
link("compost.md", "k2-1", "mulch.md"),
link("compost.md", "k2-1", "sourdoughs.md"), # also its home: one page, counted once
link("sourdoughs.md", "d2-1", "mulch.md"), # not a main page: no out-degree
], "none": [], "none_count": 0, "accepted": 5, "rejected": {}})
wiki = root / "wiki"
write(wiki / "index.md", "# Index\n")
write(wiki / "log.md", "# Log\n")
write(wiki / "sourdough.md", page("Sourdough", ["bread-a.md"], "Sourdough has a [crust](crust.md).[^a]"))
write(wiki / "crust.md", page("Crust", ["bread-b.md"], "A crust forms on [sourdough](sourdough.md).[^a]"))
write(wiki / "compost.md", page("Compost", ["garden-a.md"],
"Compost feeds [mulch](mulch.md) and [old dough](sourdoughs.md).[^a]\n\n"
"## Related concepts\n\n- [Sourdoughs](sourdoughs.md)"))
write(wiki / "mulch.md", page("Mulch", ["garden-b.md"], "Mulch covers the bed.[^a]"))
write(wiki / "sourdoughs.md", page("Sourdoughs", [], "Old dough goes on the [mulch](mulch.md).[^a]"))
write_json(root / "kost.json", {"schema": cost.SCHEMA, "lines": {
"build": {"total": 1000}, "link": {"total": 200}, "orchestration-build": {"total": 300},
"calib": {"total": 999}}, "runs": [
run("pack", "build", "p1", 400), run("link", "link", "p1", 50),
run("pack", "build", "p2", 500), run("pack", "build", "p2-retry", 100),
run("link", "link", "p2-guard", 50), run("link", "link", "p2", 100),
run("calib", "calib", "probe-1", 999)]})
write(root / "logg" / "p1.txt", "run-before\nexit 0\nrun-after\nexit 0\npass-before\nexit 0\npass-after\nexit 0\n")
write(root / "logg" / "p2.txt", "run-before\nexit 0\nrun-after\nexit 3\nrun-after\nexit 4\nrun-after\nexit 0\n")
class PartMetricsTest(unittest.TestCase):
def setUp(self) -> None:
self.tmp = tempfile.TemporaryDirectory()
self.root = Path(self.tmp.name) / "del1"
make_root(self.root)
def tearDown(self) -> None:
self.tmp.cleanup()
def metrics(self) -> dict[str, Any]:
doc: dict[str, Any] = pm.metrics(self.root)
return doc
def pack(self, n: int) -> dict[str, Any]:
found: dict[str, Any] = next(p for p in self.metrics()["packs"] if p["pack"] == n)
return found
def test_blocks_and_cost_per_pack_by_hand(self) -> None:
# measured 1500 (calib left out); attributed p1 400+50, p2 500+100+50+100 = 1200; scale 1500/1200 = 1.25
p1, p2 = self.pack(1), self.pack(2)
self.assertEqual((p1["sources"], p1["blocks"], p1["blocks_so_far"]), (SOURCES[0], 4, 4))
self.assertEqual((p2["blocks"], p2["blocks_so_far"]), (4, 8))
self.assertEqual(p1["cost"], {"build": 400, "link": 50, "orchestration": 112.5, "total": 562.5,
"so_far": 562.5, "per_block": 140.625})
self.assertEqual(p2["cost"], {"build": 600, "link": 150, "orchestration": 187.5, "total": 937.5,
"so_far": 1500.0, "per_block": 234.375})
self.assertEqual(self.metrics()["totals"]["cost"], {"build": 1000, "link": 200, "orchestration-build": 300,
"measured": 1500})
def test_runs_retries_guard_rounds_and_exits(self) -> None:
p1, p2 = self.pack(1), self.pack(2)
self.assertEqual((p1["runs"], p1["retries"], p1["guard_rounds"]), ({"build": 1, "link": 1}, 0, 0))
self.assertEqual((p2["runs"], p2["retries"], p2["guard_rounds"]), ({"build": 2, "link": 2}, 1, 1))
self.assertEqual(p2["exits"], [0, 3, 4, 0])
def test_the_exits_of_the_skills_chain_are_read_with_the_command_they_name(self) -> None:
# order 22, cost cut 1: the chain prints "exit N: <module> <command>" for each command it runs
write(self.root / "logg" / "p2.txt", "exit 0: okfwiki.klink pass-after\nexit 3: okfwiki.kpack run-after\n"
"exit 0: okfwiki.kpack run-after\nexit 0: okfwiki.klink pass-before\n"
"exit 0: okfwiki.kpack run-before\n")
self.assertEqual(self.pack(2)["exits"], [0, 3, 0, 0, 0])
def test_index_pages_and_main_pages_after_each_pack(self) -> None:
# after pack 1: three skeleton entries + crust; after pack 2: + mulch, sourdoughs. Starter is never a page.
p1, p2 = self.pack(1), self.pack(2)
self.assertEqual((p1["index_concepts"], p1["pages"], p1["main_pages"]), (4, 2, 2))
self.assertEqual((p2["index_concepts"], p2["pages"], p2["main_pages"]), (6, 5, 4))
def test_catalogue_size_from_the_task_parts(self) -> None:
self.assertEqual(self.pack(1)["catalogue"], {"lines": 0, "chars": 0, "limit": 32_000})
chars = len(kpack.catalogue_line("sourdough", "Sourdough")) + len(kpack.catalogue_line("crust", "Crust"))
self.assertEqual(chars, 40)
self.assertEqual(self.pack(2)["catalogue"], {"lines": 2, "chars": 40, "limit": 32_000})
def test_catalogue_read_as_kpack_writes_it(self) -> None:
kilde = Path(self.tmp.name) / "kilde"
write(kilde / "bread-a.md", "# Sourdough\nA starter makes it rise.\n")
doc = {"blocks": [{"file": "bread-a.md", "section": "Sourdough", "start": 1, "end": 2}]}
cat = [{"id": "crust", "title": "Crust"}, {"id": "mulch", "title": "Mulch"}]
parts = kpack.render_input(3, ["bread-a.md"], doc, [], kilde, catalogue=cat)
want = sum(len(kpack.catalogue_line(e["id"], e["title"])) for e in cat)
self.assertEqual(pm.catalogue_of(parts), (2, want))
self.assertEqual(pm.catalogue_of(kpack.render_input(3, ["bread-a.md"], doc, [], kilde)), (0, 0))
def test_two_like_concepts_are_one_group(self) -> None:
self.assertEqual(self.pack(1)["duplicates"], {"groups": [], "count": 0, "pages": 0})
self.assertEqual(self.pack(2)["duplicates"], {"groups": [["sourdough", "sourdoughs"]], "count": 1,
"pages": 2})
def test_pages_without_a_home_before_and_after_the_guard(self) -> None:
self.assertEqual(self.pack(1)["homeless"], {
"audits": 1, "before": {"pages": [], "sources": []}, "after": {"pages": [], "sources": []}})
self.assertEqual(self.pack(2)["homeless"], {
"audits": 2, "before": {"pages": ["sourdoughs.md"], "sources": ["kilde/garden-b.md"]},
"after": {"pages": [], "sources": []}})
def test_out_degree_counts_main_pages_only(self) -> None:
# after pack 1: sourdough -> crust; crust 0. After pack 2: sourdough 1, crust 1, compost 2 (mulch and
# sourdoughs, by text and by home: one page once), mulch 0. Sourdoughs links to mulch but is no main page.
self.assertEqual(self.pack(1)["out_degree"], {"main_pages": 2, "min": 0, "median": 0.5, "max": 1,
"per_main": {"crust": 0, "sourdough": 1}})
want = {"main_pages": 4, "min": 0, "median": 1.0, "max": 2,
"per_main": {"compost": 2, "crust": 1, "mulch": 0, "sourdough": 1}}
self.assertEqual(self.pack(2)["out_degree"], want)
self.assertEqual(self.metrics()["out_degree_wiki"], want)
def test_cost_per_source_sums_to_the_pack(self) -> None:
sources = self.metrics()["sources"]
self.assertEqual(sources["bread-a.md"], {"pack": 1, "blocks": 3, "build": 300.0, "total": 421.875})
self.assertEqual(sources["garden-b.md"], {"pack": 2, "blocks": 2, "build": 300.0, "total": 468.75})
for p in self.metrics()["packs"]:
mine = [s for s in sources.values() if s["pack"] == p["pack"]]
self.assertAlmostEqual(sum(s["build"] for s in mine), p["cost"]["build"])
self.assertAlmostEqual(sum(s["total"] for s in mine), p["cost"]["total"])
def test_fit_reuses_the_stop_line_projection(self) -> None:
fit = self.metrics()["fit"]
self.assertEqual(fit["cost_per_block"], cost.project([(562.5, 4, 0), (937.5, 4, 4)], 8))
self.assertEqual(fit["cost_per_block"]["fit"]["value"], 1875.0)
self.assertEqual((fit["catalogue_chars"]["a"], fit["catalogue_chars"]["b"]), (0.0, 10.0))
def test_two_runs_are_byte_identical(self) -> None:
outs = [self.root / "m1.json", self.root / "m2.json"]
for out in outs:
with contextlib.redirect_stdout(io.StringIO()):
self.assertEqual(pm.main([str(self.root), "--out", str(out)]), 0)
first = outs[0].read_bytes()
self.assertEqual(first, outs[1].read_bytes())
doc = json.loads(first)
self.assertEqual(first.decode("utf-8"),
json.dumps(doc, indent=1, sort_keys=True, ensure_ascii=False) + "\n")
def test_a_missing_input_cannot_run(self) -> None:
(self.root / "kost.json").unlink()
err = io.StringIO()
with contextlib.redirect_stderr(err):
self.assertEqual(pm.main([str(self.root), "--out", str(self.root / "m.json")]), 2)
self.assertIn("kost.json", err.getvalue())
self.assertFalse((self.root / "m.json").exists())
if __name__ == "__main__":
unittest.main()