First public release, published as one snapshot. What changed between versions is in CHANGELOG.md. Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
290 lines
17 KiB
Python
290 lines
17 KiB
Python
"""Tests for tools/part_metrics.py: the numbers per pack and per source of a build folder (contract K7), without a model
|
|
(wave 3 order 11, plan revision 5 Step 11: cost, catalogue size, duplicate concepts, pages without a home before and
|
|
after the guard, main-page out-degree, with a growth fit).
|
|
|
|
The fixture is a build folder of two packs of two sources each, written by hand; every expected number below is worked
|
|
out by hand from it. The text is synthetic, about bread and gardening (CLAUDE.md rule 3).
|
|
Run: python3 -m unittest discover -s tests -p 'test_part_metrics.py'
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import contextlib
|
|
import importlib.util
|
|
import io
|
|
import json
|
|
import sys
|
|
import tempfile
|
|
import unittest
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
from okfwiki import cost, kpack
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
|
|
# Loaded from its file, as tests/test_x_fields.py loads its tool; mypy checks the tool when it is named.
|
|
_SPEC = importlib.util.spec_from_file_location("part_metrics", ROOT / "tools" / "part_metrics.py")
|
|
assert _SPEC is not None and _SPEC.loader is not None
|
|
pm: Any = importlib.util.module_from_spec(_SPEC)
|
|
sys.modules["part_metrics"] = pm
|
|
_SPEC.loader.exec_module(pm)
|
|
|
|
SOURCES = [["bread-a.md", "bread-b.md"], ["garden-a.md", "garden-b.md"]]
|
|
BLOCKS_PER_FILE = {"bread-a.md": 3, "bread-b.md": 1, "garden-a.md": 2, "garden-b.md": 2}
|
|
|
|
|
|
def write(path: Path, text: str) -> None:
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
path.write_text(text, encoding="utf-8")
|
|
|
|
|
|
def write_json(path: Path, doc: Any) -> None:
|
|
write(path, json.dumps(doc, ensure_ascii=False, indent=1) + "\n")
|
|
|
|
|
|
def run(name: str, line: str, unit: str, total: int) -> dict[str, Any]:
|
|
m = cost.PREFIX.match(f"V1-fx-{name}-{unit}")
|
|
assert m is not None
|
|
return {"description": f"V1-fx-{name}-{unit}", "run": "fx", "line": line, "unit": unit,
|
|
"pack": cost.pack_of(unit), "total": total}
|
|
|
|
|
|
def page(title: str, mains: list[str], body: str) -> str:
|
|
head = f"---\ntype: Concept\ntitle: {title}\n"
|
|
if mains:
|
|
head += "x-main-for:\n" + "".join(f" - /kilde/{m}\n" for m in mains)
|
|
return head + "---\n\n" + body + "\n\n[^a]: A note.\n"
|
|
|
|
|
|
def make_root(root: Path) -> None:
|
|
work = root / "work"
|
|
blocks = []
|
|
for name in [n for pack in SOURCES for n in pack]:
|
|
for i in range(BLOCKS_PER_FILE[name]):
|
|
blocks.append({"file": name, "section": "Part", "start": 2 * i + 1, "end": 2 * i + 2})
|
|
write_json(work / "blocks.json", {"schema": "okf-wiki/blocks/1", "order": [n for p in SOURCES for n in p],
|
|
"blocks": blocks})
|
|
write_json(work / "K-work" / "packs.json", {"schema": kpack.PACKS_SCHEMA, "size": 2, "packs": [
|
|
{"pack": 1, "sources": SOURCES[0], "blocks": 4, "index_total": 3, "main_pages_sent": 0, "parts": 1,
|
|
"accepted": 3},
|
|
{"pack": 2, "sources": SOURCES[1], "blocks": 4, "index_total": 4, "main_pages_sent": 2, "parts": 2,
|
|
"attempts": [{"proposed": 5, "rejected": {"quote-missing": 1}}], "accepted": 5,
|
|
"guard": {"asked": {"pages": ["sourdoughs"], "sources": ["kilde/garden-b.md"], "catalogue": []},
|
|
"parts": 1, "answered": {"homes": 1, "main_pages": 1}}},
|
|
]})
|
|
|
|
def entry(cid: str, title: str, kind: str, file: str) -> dict[str, Any]:
|
|
return {"id": cid, "title": title, "aliases": [title], "kind": kind, "merge_of": None,
|
|
"origin": {"file": file, "line": 1}}
|
|
|
|
write_json(work / "K-work" / "index.json", {"schema": kpack.INDEX_SCHEMA, "entries": [
|
|
entry("sourdough", "Sourdough", "skeleton", "bread-a.md"),
|
|
entry("starter", "Starter", "skeleton", "bread-a.md"), # in the index, never a page
|
|
entry("compost", "Compost", "skeleton", "garden-a.md"),
|
|
entry("crust", "Crust", "new", "bread-b.md"),
|
|
entry("mulch", "Mulch", "new", "garden-b.md"),
|
|
entry("sourdoughs", "Sourdoughs", "new", "garden-a.md"), # normalises to "sourdough": a duplicate
|
|
]})
|
|
|
|
def frag(pack: int, file: str, cid: str) -> dict[str, Any]:
|
|
return {"pack": pack, "position": SOURCES[pack - 1].index(file), "file": file, "start": 1, "end": 2,
|
|
"concept_id": cid, "provenance": "documented", "outcome": "existing"}
|
|
|
|
write_json(work / "K-work" / "accepted.json", {"schema": kpack.ACCEPTED_SCHEMA, "fragments": [
|
|
frag(1, "bread-a.md", "sourdough"), frag(1, "bread-b.md", "crust"), frag(1, "bread-b.md", "sourdough"),
|
|
frag(2, "garden-a.md", "compost"), frag(2, "garden-a.md", "sourdoughs"), frag(2, "garden-b.md", "mulch"),
|
|
frag(2, "garden-b.md", "crust"), frag(2, "garden-b.md", "compost"),
|
|
], "main_pages": [
|
|
{"source": "bread-a.md", "concept_id": "sourdough", "pack": 1, "by": "compiler"},
|
|
{"source": "bread-b.md", "concept_id": "crust", "pack": 1, "by": "compiler"},
|
|
{"source": "garden-a.md", "concept_id": "compost", "pack": 2, "by": "compiler"},
|
|
{"source": "garden-b.md", "concept_id": "mulch", "pack": 2, "by": "guard"},
|
|
], "homes": [
|
|
{"source": "garden-a.md", "concept_id": "sourdoughs", "home": "compost", "pack": 2, "by": "guard"},
|
|
]})
|
|
|
|
# The compiler's task parts as kpack writes them: pack 1 has no main page yet; pack 2's catalogue runs into part 2.
|
|
write(work / "K-oppgaver" / "pack-1-part-1.md",
|
|
"<!-- part 1 of 1 -->\n# K pack 1: 2 sources\n\n## Index (concepts named in these sources)\n\n"
|
|
"- id: sourdough | Sourdough\n\n## Main pages (homes you may choose)\n\n"
|
|
"(none yet: the main pages you name in this pack are the homes you may choose)\n")
|
|
write(work / "K-oppgaver" / "pack-2-part-1.md",
|
|
"<!-- part 1 of 2 -->\n# K pack 2: 2 sources\n\n## Index (concepts named in these sources)\n\n"
|
|
"- id: sourdough | Sourdough | main page\n- id: compost | Compost\n\n## Main pages (homes you may choose)\n\n")
|
|
write(work / "K-oppgaver" / "pack-2-part-2.md", "<!-- part 2 of 2 -->\n- sourdough | Sourdough\n- crust | Crust\n")
|
|
|
|
# The auditor's JSON after each run-after: pack 2 needed the guard (audit 1 before it, audit 2 after it).
|
|
def audit_doc(findings: list[tuple[str, str]]) -> dict[str, Any]:
|
|
return {"findings": [{"check": c, "group": "links", "file": f, "line": None, "message": "m"}
|
|
for c, f in findings], "summary": {}, "link_rule": {}}
|
|
|
|
write_json(work / "K-audit" / "p1-audit-1.json", audit_doc([]))
|
|
write_json(work / "K-audit" / "p2-audit-1.json", audit_doc([
|
|
("no-main-page-link", "sourdoughs.md"), ("main-page-missing", "kilde/garden-b.md"),
|
|
("broken-link", "crust.md")]))
|
|
write_json(work / "K-audit" / "p2-audit-2.json", audit_doc([]))
|
|
|
|
# The link passes: which sentences each pass asked about, and the links it gave.
|
|
write_json(work / "K-links" / "p1" / "sentences.json", {"schema": "okf-wiki/link-sentences/1", "pack": 1,
|
|
"pages": {"sourdough.md": ["s1-1"]}})
|
|
write_json(work / "K-links" / "p2" / "sentences.json", {"schema": "okf-wiki/link-sentences/1", "pack": 2,
|
|
"pages": {"crust.md": ["c2-1"], "compost.md": ["k2-1"],
|
|
"sourdoughs.md": ["d2-1"]}})
|
|
|
|
def link(p: str, s: str, target: str) -> dict[str, Any]:
|
|
return {"page": p, "sentence": s, "anchor": "x", "at": 0, "target": target}
|
|
|
|
write_json(work / "links.json", {"schema": "okf-wiki/links/1", "links": [
|
|
link("sourdough.md", "s1-1", "crust.md"),
|
|
link("crust.md", "c2-1", "sourdough.md"),
|
|
link("compost.md", "k2-1", "mulch.md"),
|
|
link("compost.md", "k2-1", "sourdoughs.md"), # also its home: one page, counted once
|
|
link("sourdoughs.md", "d2-1", "mulch.md"), # not a main page: no out-degree
|
|
], "none": [], "none_count": 0, "accepted": 5, "rejected": {}})
|
|
|
|
wiki = root / "wiki"
|
|
write(wiki / "index.md", "# Index\n")
|
|
write(wiki / "log.md", "# Log\n")
|
|
write(wiki / "sourdough.md", page("Sourdough", ["bread-a.md"], "Sourdough has a [crust](crust.md).[^a]"))
|
|
write(wiki / "crust.md", page("Crust", ["bread-b.md"], "A crust forms on [sourdough](sourdough.md).[^a]"))
|
|
write(wiki / "compost.md", page("Compost", ["garden-a.md"],
|
|
"Compost feeds [mulch](mulch.md) and [old dough](sourdoughs.md).[^a]\n\n"
|
|
"## Related concepts\n\n- [Sourdoughs](sourdoughs.md)"))
|
|
write(wiki / "mulch.md", page("Mulch", ["garden-b.md"], "Mulch covers the bed.[^a]"))
|
|
write(wiki / "sourdoughs.md", page("Sourdoughs", [], "Old dough goes on the [mulch](mulch.md).[^a]"))
|
|
|
|
write_json(root / "kost.json", {"schema": cost.SCHEMA, "lines": {
|
|
"build": {"total": 1000}, "link": {"total": 200}, "orchestration-build": {"total": 300},
|
|
"calib": {"total": 999}}, "runs": [
|
|
run("pack", "build", "p1", 400), run("link", "link", "p1", 50),
|
|
run("pack", "build", "p2", 500), run("pack", "build", "p2-retry", 100),
|
|
run("link", "link", "p2-guard", 50), run("link", "link", "p2", 100),
|
|
run("calib", "calib", "probe-1", 999)]})
|
|
write(root / "logg" / "p1.txt", "run-before\nexit 0\nrun-after\nexit 0\npass-before\nexit 0\npass-after\nexit 0\n")
|
|
write(root / "logg" / "p2.txt", "run-before\nexit 0\nrun-after\nexit 3\nrun-after\nexit 4\nrun-after\nexit 0\n")
|
|
|
|
|
|
class PartMetricsTest(unittest.TestCase):
|
|
def setUp(self) -> None:
|
|
self.tmp = tempfile.TemporaryDirectory()
|
|
self.root = Path(self.tmp.name) / "del1"
|
|
make_root(self.root)
|
|
|
|
def tearDown(self) -> None:
|
|
self.tmp.cleanup()
|
|
|
|
def metrics(self) -> dict[str, Any]:
|
|
doc: dict[str, Any] = pm.metrics(self.root)
|
|
return doc
|
|
|
|
def pack(self, n: int) -> dict[str, Any]:
|
|
found: dict[str, Any] = next(p for p in self.metrics()["packs"] if p["pack"] == n)
|
|
return found
|
|
|
|
def test_blocks_and_cost_per_pack_by_hand(self) -> None:
|
|
# measured 1500 (calib left out); attributed p1 400+50, p2 500+100+50+100 = 1200; scale 1500/1200 = 1.25
|
|
p1, p2 = self.pack(1), self.pack(2)
|
|
self.assertEqual((p1["sources"], p1["blocks"], p1["blocks_so_far"]), (SOURCES[0], 4, 4))
|
|
self.assertEqual((p2["blocks"], p2["blocks_so_far"]), (4, 8))
|
|
self.assertEqual(p1["cost"], {"build": 400, "link": 50, "orchestration": 112.5, "total": 562.5,
|
|
"so_far": 562.5, "per_block": 140.625})
|
|
self.assertEqual(p2["cost"], {"build": 600, "link": 150, "orchestration": 187.5, "total": 937.5,
|
|
"so_far": 1500.0, "per_block": 234.375})
|
|
self.assertEqual(self.metrics()["totals"]["cost"], {"build": 1000, "link": 200, "orchestration-build": 300,
|
|
"measured": 1500})
|
|
|
|
def test_runs_retries_guard_rounds_and_exits(self) -> None:
|
|
p1, p2 = self.pack(1), self.pack(2)
|
|
self.assertEqual((p1["runs"], p1["retries"], p1["guard_rounds"]), ({"build": 1, "link": 1}, 0, 0))
|
|
self.assertEqual((p2["runs"], p2["retries"], p2["guard_rounds"]), ({"build": 2, "link": 2}, 1, 1))
|
|
self.assertEqual(p2["exits"], [0, 3, 4, 0])
|
|
|
|
def test_the_exits_of_the_skills_chain_are_read_with_the_command_they_name(self) -> None:
|
|
# order 22, cost cut 1: the chain prints "exit N: <module> <command>" for each command it runs
|
|
write(self.root / "logg" / "p2.txt", "exit 0: okfwiki.klink pass-after\nexit 3: okfwiki.kpack run-after\n"
|
|
"exit 0: okfwiki.kpack run-after\nexit 0: okfwiki.klink pass-before\n"
|
|
"exit 0: okfwiki.kpack run-before\n")
|
|
self.assertEqual(self.pack(2)["exits"], [0, 3, 0, 0, 0])
|
|
|
|
def test_index_pages_and_main_pages_after_each_pack(self) -> None:
|
|
# after pack 1: three skeleton entries + crust; after pack 2: + mulch, sourdoughs. Starter is never a page.
|
|
p1, p2 = self.pack(1), self.pack(2)
|
|
self.assertEqual((p1["index_concepts"], p1["pages"], p1["main_pages"]), (4, 2, 2))
|
|
self.assertEqual((p2["index_concepts"], p2["pages"], p2["main_pages"]), (6, 5, 4))
|
|
|
|
def test_catalogue_size_from_the_task_parts(self) -> None:
|
|
self.assertEqual(self.pack(1)["catalogue"], {"lines": 0, "chars": 0, "limit": 32_000})
|
|
chars = len(kpack.catalogue_line("sourdough", "Sourdough")) + len(kpack.catalogue_line("crust", "Crust"))
|
|
self.assertEqual(chars, 40)
|
|
self.assertEqual(self.pack(2)["catalogue"], {"lines": 2, "chars": 40, "limit": 32_000})
|
|
|
|
def test_catalogue_read_as_kpack_writes_it(self) -> None:
|
|
kilde = Path(self.tmp.name) / "kilde"
|
|
write(kilde / "bread-a.md", "# Sourdough\nA starter makes it rise.\n")
|
|
doc = {"blocks": [{"file": "bread-a.md", "section": "Sourdough", "start": 1, "end": 2}]}
|
|
cat = [{"id": "crust", "title": "Crust"}, {"id": "mulch", "title": "Mulch"}]
|
|
parts = kpack.render_input(3, ["bread-a.md"], doc, [], kilde, catalogue=cat)
|
|
want = sum(len(kpack.catalogue_line(e["id"], e["title"])) for e in cat)
|
|
self.assertEqual(pm.catalogue_of(parts), (2, want))
|
|
self.assertEqual(pm.catalogue_of(kpack.render_input(3, ["bread-a.md"], doc, [], kilde)), (0, 0))
|
|
|
|
def test_two_like_concepts_are_one_group(self) -> None:
|
|
self.assertEqual(self.pack(1)["duplicates"], {"groups": [], "count": 0, "pages": 0})
|
|
self.assertEqual(self.pack(2)["duplicates"], {"groups": [["sourdough", "sourdoughs"]], "count": 1,
|
|
"pages": 2})
|
|
|
|
def test_pages_without_a_home_before_and_after_the_guard(self) -> None:
|
|
self.assertEqual(self.pack(1)["homeless"], {
|
|
"audits": 1, "before": {"pages": [], "sources": []}, "after": {"pages": [], "sources": []}})
|
|
self.assertEqual(self.pack(2)["homeless"], {
|
|
"audits": 2, "before": {"pages": ["sourdoughs.md"], "sources": ["kilde/garden-b.md"]},
|
|
"after": {"pages": [], "sources": []}})
|
|
|
|
def test_out_degree_counts_main_pages_only(self) -> None:
|
|
# after pack 1: sourdough -> crust; crust 0. After pack 2: sourdough 1, crust 1, compost 2 (mulch and
|
|
# sourdoughs, by text and by home: one page once), mulch 0. Sourdoughs links to mulch but is no main page.
|
|
self.assertEqual(self.pack(1)["out_degree"], {"main_pages": 2, "min": 0, "median": 0.5, "max": 1,
|
|
"per_main": {"crust": 0, "sourdough": 1}})
|
|
want = {"main_pages": 4, "min": 0, "median": 1.0, "max": 2,
|
|
"per_main": {"compost": 2, "crust": 1, "mulch": 0, "sourdough": 1}}
|
|
self.assertEqual(self.pack(2)["out_degree"], want)
|
|
self.assertEqual(self.metrics()["out_degree_wiki"], want)
|
|
|
|
def test_cost_per_source_sums_to_the_pack(self) -> None:
|
|
sources = self.metrics()["sources"]
|
|
self.assertEqual(sources["bread-a.md"], {"pack": 1, "blocks": 3, "build": 300.0, "total": 421.875})
|
|
self.assertEqual(sources["garden-b.md"], {"pack": 2, "blocks": 2, "build": 300.0, "total": 468.75})
|
|
for p in self.metrics()["packs"]:
|
|
mine = [s for s in sources.values() if s["pack"] == p["pack"]]
|
|
self.assertAlmostEqual(sum(s["build"] for s in mine), p["cost"]["build"])
|
|
self.assertAlmostEqual(sum(s["total"] for s in mine), p["cost"]["total"])
|
|
|
|
def test_fit_reuses_the_stop_line_projection(self) -> None:
|
|
fit = self.metrics()["fit"]
|
|
self.assertEqual(fit["cost_per_block"], cost.project([(562.5, 4, 0), (937.5, 4, 4)], 8))
|
|
self.assertEqual(fit["cost_per_block"]["fit"]["value"], 1875.0)
|
|
self.assertEqual((fit["catalogue_chars"]["a"], fit["catalogue_chars"]["b"]), (0.0, 10.0))
|
|
|
|
def test_two_runs_are_byte_identical(self) -> None:
|
|
outs = [self.root / "m1.json", self.root / "m2.json"]
|
|
for out in outs:
|
|
with contextlib.redirect_stdout(io.StringIO()):
|
|
self.assertEqual(pm.main([str(self.root), "--out", str(out)]), 0)
|
|
first = outs[0].read_bytes()
|
|
self.assertEqual(first, outs[1].read_bytes())
|
|
doc = json.loads(first)
|
|
self.assertEqual(first.decode("utf-8"),
|
|
json.dumps(doc, indent=1, sort_keys=True, ensure_ascii=False) + "\n")
|
|
|
|
def test_a_missing_input_cannot_run(self) -> None:
|
|
(self.root / "kost.json").unlink()
|
|
err = io.StringIO()
|
|
with contextlib.redirect_stderr(err):
|
|
self.assertEqual(pm.main([str(self.root), "--out", str(self.root / "m.json")]), 2)
|
|
self.assertIn("kost.json", err.getvalue())
|
|
self.assertFalse((self.root / "m.json").exists())
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|