First public release, published as one snapshot. What changed between versions is in CHANGELOG.md. Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
314 lines
16 KiB
Python
314 lines
16 KiB
Python
"""The numbers per pack and per source of a build folder (contract K7), read without a model (CLAUDE.md rule 1).
|
|
|
|
Usage:
|
|
python3 tools/part_metrics.py ROOT --out FILE
|
|
|
|
ROOT is a build folder: work/ (the skill's WORK), wiki/ (WIKI), logg/pN.txt (what the build commands of pack N wrote,
|
|
with "exit N" or "exit N: <module> <command>" lines) and kost.json (okfwiki.cost harvest). It writes one JSON object (indent 1, sorted keys), the same
|
|
bytes on every run over the same folder. Plan revision 5 Step 11 asks for these numbers per pack, with a growth fit,
|
|
so the price of the rest and the growth of the collection can be read from the first part (PLAN.md § Retning point 4).
|
|
|
|
Per accepted pack (work/K-work/packs.json):
|
|
- sources, blocks in the pack and so far;
|
|
- cost per line: build and link as attributed by description, orchestration spread over the packs in proportion to
|
|
their attributed cost (as okfwiki.cost stop spreads it), the pack's total, the total so far and per block; the
|
|
number of build and link runs, the compiler's new runs after a rejection at writing (exit 3) and the guard rounds,
|
|
because they explain the cost; the exit codes in logg/pN.txt in order;
|
|
- the concepts in the index after the pack (work/K-work/index.json: the skeleton's entries and those whose origin is
|
|
a source of the packs so far), the pages (concepts with an accepted fragment) and the main pages;
|
|
- the catalogue of main pages the pack's compiler got, in lines and characters, read from its task parts in
|
|
work/K-oppgaver/ as kpack wrote them, beside the limit klink.CATALOGUE_CHARS;
|
|
- like concepts: groups of pages that share a normalised title or alias (kpack.norm, its concept identity);
|
|
- pages without a home and docs sources without a main page before and after the guard, from the auditor's JSON
|
|
(no-main-page-link and main-page-missing) in work/K-audit/pN-audit-1.json and in the last pN-audit-k.json; the
|
|
auditor is the only judge (K2 point 17), nothing here counts it again;
|
|
- main-page out-degree: per main page the concept pages it links to, in a sentence or in its list of related
|
|
concepts, each page once; min, median and max. The wiki is assembled again after every pack and keeps no earlier
|
|
state, so the out-degree after pack N is counted from the records that place the links: a sentence link counts from
|
|
the pack whose link pass asked for its sentence (work/K-links/**/sentences.json), a home from the pack its record
|
|
names. Only records that still stand count. Beside it, out_degree_wiki counts the same on wiki/ with the auditor's
|
|
own reading of links from main pages (audit.main_page_links); after the last pack the two agree when every link is
|
|
attributed (links_unattributed counts those that are not).
|
|
Per source: the compiler's cost (the build line) and the pack's total, split over the pack's sources by their share of
|
|
its blocks (okfwiki.cost.allocate), the price of a later repair of one source.
|
|
fit: cost per block against blocks read before the pack (okfwiki.cost.project, as the stop line projects it, to the
|
|
blocks in work/blocks.json), and the catalogue's characters against blocks read before the pack (okfwiki.cost.fit_line),
|
|
with the blocks read at which the fitted line reaches the limit.
|
|
|
|
Exit 0 written, 2 cannot run (an input file missing or not readable; nothing is written).
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import os
|
|
import re
|
|
import statistics
|
|
import sys
|
|
from fractions import Fraction
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
|
|
|
from okfwiki import audit, cost, klink, kpack # noqa: E402
|
|
from okfwiki.common import RunError # noqa: E402
|
|
|
|
SCHEMA = "okf-wiki/part-metrics/1"
|
|
LINES = kpack.STOP_LINES.split(",") # build, link, orchestration-build: the lines the build's stop line caps
|
|
CATALOGUE_HEADER = "## Main pages (homes you may choose)\n\n" # as kpack.render_input writes it
|
|
PART_HEAD = re.compile(r"^<!-- part \d+ of \d+ -->\n")
|
|
PART_FILE = re.compile(r"^pack-(\d+)-part-(\d+)\.md$")
|
|
EXIT_LINE = re.compile(r"^exit (\d+)(?:: .*)?$") # bare, or as the skill's chain prints it
|
|
|
|
|
|
def read_json(path: Path) -> Any:
|
|
try:
|
|
return json.loads(path.read_text(encoding="utf-8"))
|
|
except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc:
|
|
raise RunError(f"cannot read {path}: {exc}") from exc
|
|
|
|
|
|
def catalogue_of(parts: list[str]) -> tuple[int, int] | None:
|
|
"""(lines, characters) of the catalogue of main pages in a compiler task, its parts in order; each line counted
|
|
with its newline, as kpack.main_catalogue counts against its budget. None when the task has no catalogue section."""
|
|
text = "".join(PART_HEAD.sub("", p, count=1) for p in parts)
|
|
at = text.rfind(CATALOGUE_HEADER)
|
|
if at < 0:
|
|
return None
|
|
lines = [line for line in text[at + len(CATALOGUE_HEADER):].split("\n") if line.startswith("- ")]
|
|
return len(lines), sum(len(line) + 1 for line in lines)
|
|
|
|
|
|
def task_parts(work: Path, n: int) -> list[str]:
|
|
found = []
|
|
for p in (work / "K-oppgaver").glob(f"pack-{n}-part-*.md"):
|
|
m = PART_FILE.match(p.name)
|
|
if m and int(m.group(1)) == n:
|
|
found.append((int(m.group(2)), p.read_text(encoding="utf-8")))
|
|
return [text for _, text in sorted(found)]
|
|
|
|
|
|
def pack_costs(kost: dict[str, Any], numbers: list[int]) -> tuple[dict[int, dict[str, Any]], dict[str, int]]:
|
|
"""Per pack: build and link as attributed, and the pack's total with orchestration spread over the packs in
|
|
proportion to their attributed cost, as okfwiki.cost stop does (exact fractions); and the line totals."""
|
|
totals = {name: int(kost["lines"][name]["total"]) for name in LINES}
|
|
measured = sum(totals.values())
|
|
per: dict[int, dict[str, Any]] = {n: {"build": 0, "link": 0, "build_runs": 0, "link_runs": 0} for n in numbers}
|
|
for r in kost.get("runs", []):
|
|
line, n = r.get("line"), r.get("pack")
|
|
if line in ("build", "link") and n is not None and int(n) in per:
|
|
per[int(n)][line] += int(r["total"])
|
|
per[int(n)][f"{line}_runs"] += 1
|
|
attributed = sum(p["build"] + p["link"] for p in per.values())
|
|
scale = Fraction(measured, attributed) if attributed else None
|
|
for p in per.values():
|
|
p["total"] = (p["build"] + p["link"]) * scale if scale is not None else None
|
|
return per, {**totals, "measured": measured}
|
|
|
|
|
|
def concept_names(entry: dict[str, Any]) -> set[str]:
|
|
return {kpack.norm(str(a)) for a in [entry.get("title", ""), *entry.get("aliases", [])]} - {""}
|
|
|
|
|
|
def duplicate_groups(index: dict[str, Any], pages: set[str]) -> list[list[str]]:
|
|
"""Groups of two or more pages that share a normalised title or alias, each group sorted, the groups sorted."""
|
|
names = {str(e["id"]): concept_names(e) for e in index["entries"] if str(e["id"]) in pages}
|
|
parent = {cid: cid for cid in names}
|
|
|
|
def find(c: str) -> str:
|
|
while parent[c] != c:
|
|
c = parent[c]
|
|
return c
|
|
|
|
first: dict[str, str] = {}
|
|
for cid in sorted(names):
|
|
for name in sorted(names[cid]):
|
|
if name in first:
|
|
a, b = find(first[name]), find(cid)
|
|
if a != b:
|
|
parent[max(a, b)] = min(a, b)
|
|
else:
|
|
first[name] = cid
|
|
groups: dict[str, list[str]] = {}
|
|
for cid in sorted(names):
|
|
groups.setdefault(find(cid), []).append(cid)
|
|
return sorted(g for g in groups.values() if len(g) > 1)
|
|
|
|
|
|
def homeless(doc: dict[str, Any]) -> dict[str, list[str]]:
|
|
findings = [f for f in doc.get("findings", []) if isinstance(f, dict)]
|
|
return {"pages": sorted(str(f.get("file")) for f in findings if f.get("check") == "no-main-page-link"),
|
|
"sources": sorted(str(f.get("file")) for f in findings if f.get("check") == "main-page-missing")}
|
|
|
|
|
|
def audits(work: Path, n: int) -> dict[str, Any] | None:
|
|
found = []
|
|
for p in (work / "K-audit").glob(f"p{n}-audit-*.json"):
|
|
m = re.fullmatch(rf"p{n}-audit-(\d+)\.json", p.name)
|
|
if m:
|
|
found.append((int(m.group(1)), p))
|
|
if not found:
|
|
return None
|
|
found.sort()
|
|
return {"audits": len(found), "before": homeless(read_json(found[0][1])), "after": homeless(read_json(found[-1][1]))}
|
|
|
|
|
|
def exits(root: Path, n: int) -> list[int] | None:
|
|
path = root / "logg" / f"p{n}.txt"
|
|
if not path.is_file():
|
|
return None
|
|
return [int(m.group(1)) for line in path.read_text(encoding="utf-8").splitlines()
|
|
if (m := EXIT_LINE.match(line.strip()))]
|
|
|
|
|
|
def link_pairs(work: Path, acc: dict[str, Any]) -> tuple[dict[tuple[str, str], int], int]:
|
|
"""(main page id, linked page id) -> the first pack that placed the link, from the link records and the homes;
|
|
and the number of link records no link pass asked for."""
|
|
asked: dict[tuple[str, str], int] = {}
|
|
for path in sorted((work / "K-links").rglob("sentences.json")) if (work / "K-links").is_dir() else []:
|
|
doc = read_json(path)
|
|
for rel, keys in doc.get("pages", {}).items():
|
|
for key in keys:
|
|
at = (str(rel), str(key))
|
|
asked[at] = min(asked.get(at, int(doc["pack"])), int(doc["pack"]))
|
|
pairs: dict[tuple[str, str], int] = {}
|
|
unattributed = 0
|
|
|
|
def add(a: str, b: str, n: int) -> None:
|
|
pairs[(a, b)] = min(pairs.get((a, b), n), n)
|
|
|
|
links_path = work / "links.json"
|
|
records = read_json(links_path).get("links", []) if links_path.is_file() else []
|
|
for r in records:
|
|
n = asked.get((str(r["page"]), str(r["sentence"])))
|
|
if n is None:
|
|
unattributed += 1
|
|
continue
|
|
add(klink.concept_id(str(r["page"])), klink.concept_id(str(r["target"])), n)
|
|
for h in acc.get("homes", []):
|
|
add(str(h["home"]), str(h["concept_id"]), int(h["pack"]))
|
|
return pairs, unattributed
|
|
|
|
|
|
def degree_stats(per_main: dict[str, int]) -> dict[str, Any]:
|
|
values = list(per_main.values())
|
|
return {"main_pages": len(values), "min": min(values, default=None), "max": max(values, default=None),
|
|
"median": statistics.median(values) if values else None, "per_main": dict(sorted(per_main.items()))}
|
|
|
|
|
|
def wiki_out_degree(wiki: Path) -> dict[str, Any] | None:
|
|
if not wiki.is_dir():
|
|
return None
|
|
b = audit.load_bundle(wiki, wiki)
|
|
mains = set()
|
|
for d in b.concepts:
|
|
value = d.frontmatter.get(audit.MAIN_FOR) if d.frontmatter else None
|
|
if isinstance(value, list) and value:
|
|
mains.add(d.rel)
|
|
linked = audit.main_page_links(b, mains)
|
|
return degree_stats({klink.concept_id(m): sum(m in held for held in linked.values()) for m in mains})
|
|
|
|
|
|
def metrics(root: Path) -> dict[str, Any]:
|
|
work = root / "work"
|
|
blocks_doc = read_json(work / "blocks.json")
|
|
log = read_json(work / "K-work" / "packs.json")
|
|
index = read_json(work / "K-work" / "index.json")
|
|
acc = read_json(work / "K-work" / "accepted.json")
|
|
kost = read_json(root / "kost.json")
|
|
records = sorted(log["packs"], key=lambda r: int(r["pack"]))
|
|
done = [r for r in records if "accepted" in r]
|
|
numbers = [int(r["pack"]) for r in done]
|
|
costs, line_totals = pack_costs(kost, numbers)
|
|
pairs, unattributed = link_pairs(work, acc)
|
|
total_blocks = len(blocks_doc["blocks"])
|
|
|
|
packs: list[dict[str, Any]] = []
|
|
files: set[str] = set()
|
|
read = 0
|
|
so_far = Fraction(0)
|
|
points: list[tuple[float, int, int]] = []
|
|
catalogue_points: list[tuple[float, float]] = []
|
|
for r in done:
|
|
n, blocks = int(r["pack"]), int(r["blocks"])
|
|
files |= {str(s) for s in r.get("sources", [])}
|
|
c = costs[n]
|
|
row: dict[str, Any] = {"pack": n, "sources": list(r.get("sources", [])), "blocks": blocks,
|
|
"blocks_so_far": read + blocks, "runs": {"build": c["build_runs"],
|
|
"link": c["link_runs"]},
|
|
"retries": len(r.get("attempts", [])), "guard_rounds": int("guard" in r),
|
|
"exits": exits(root, n)}
|
|
if c["total"] is None:
|
|
row["cost"] = None
|
|
else:
|
|
so_far += c["total"]
|
|
row["cost"] = {"build": c["build"], "link": c["link"],
|
|
"orchestration": float(c["total"] - c["build"] - c["link"]), "total": float(c["total"]),
|
|
"so_far": float(so_far), "per_block": float(c["total"] / blocks) if blocks else None}
|
|
points.append((float(c["total"]), blocks, read))
|
|
pages = {str(f["concept_id"]) for f in acc["fragments"] if int(f["pack"]) <= n}
|
|
mains = [m for m in kpack.main_ids({**acc, "main_pages": [m for m in acc.get("main_pages", [])
|
|
if int(m["pack"]) <= n]}) if m in pages]
|
|
row["index_concepts"] = sum(1 for e in index["entries"]
|
|
if e.get("kind") == "skeleton" or str(e.get("origin", {}).get("file")) in files)
|
|
row["pages"], row["main_pages"] = len(pages), len(mains)
|
|
cat = catalogue_of(task_parts(work, n))
|
|
row["catalogue"] = None if cat is None else {"lines": cat[0], "chars": cat[1], "limit": klink.CATALOGUE_CHARS}
|
|
if cat is not None:
|
|
catalogue_points.append((float(read), float(cat[1])))
|
|
groups = duplicate_groups(index, pages)
|
|
row["duplicates"] = {"groups": groups, "count": len(groups), "pages": sum(len(g) for g in groups)}
|
|
row["homeless"] = audits(work, n)
|
|
row["out_degree"] = degree_stats({m: sum(1 for (a, b), p in pairs.items()
|
|
if a == m and p <= n and b in pages and b != m) for m in mains})
|
|
packs.append(row)
|
|
read += blocks
|
|
|
|
per_source: dict[str, dict[str, Any]] = {}
|
|
lists = {int(r["pack"]): [str(s) for s in r.get("sources", [])] for r in records}
|
|
ordered = [lists.get(i, []) for i in range(1, max(lists, default=0) + 1)]
|
|
builds = cost.allocate({n: float(costs[n]["build"]) for n in numbers}, ordered, blocks_doc)
|
|
totals = cost.allocate({n: float(costs[n]["total"]) for n in numbers if costs[n]["total"] is not None}, ordered,
|
|
blocks_doc)
|
|
count: dict[str, int] = {}
|
|
for b in blocks_doc["blocks"]:
|
|
count[str(b["file"])] = count.get(str(b["file"]), 0) + 1
|
|
for n in numbers:
|
|
for s in lists[n]:
|
|
per_source[s] = {"pack": n, "blocks": count.get(s, 0), "build": builds.get(s), "total": totals.get(s)}
|
|
|
|
fit: dict[str, Any] = {"cost_per_block": cost.project(points, total_blocks), "catalogue_chars": None}
|
|
if catalogue_points:
|
|
a, b = cost.fit_line(catalogue_points)
|
|
fit["catalogue_chars"] = {"a": a, "b": b, "points": len(catalogue_points), "limit": klink.CATALOGUE_CHARS,
|
|
"reaches_limit_at_blocks_read": (klink.CATALOGUE_CHARS - a) / b if b > 0 else None}
|
|
return {"schema": SCHEMA, "packs": packs, "sources": per_source, "fit": fit,
|
|
"pending": [int(r["pack"]) for r in records if "accepted" not in r],
|
|
"totals": {"blocks": total_blocks, "blocks_done": read, "cost": line_totals},
|
|
"out_degree_wiki": wiki_out_degree(root / "wiki"), "links_unattributed": unattributed}
|
|
|
|
|
|
def main(argv: list[str] | None = None) -> int:
|
|
p = argparse.ArgumentParser(prog="python3 tools/part_metrics.py", description=__doc__,
|
|
formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
p.add_argument("root")
|
|
p.add_argument("--out", required=True)
|
|
args = p.parse_args(argv)
|
|
root, out = Path(args.root), Path(args.out)
|
|
try:
|
|
doc = metrics(root)
|
|
except (RunError, OSError, KeyError, TypeError, ValueError) as exc:
|
|
print(f"part_metrics cannot run: {exc}", file=sys.stderr)
|
|
return 2
|
|
tmp = out.with_name(f".{out.name}.tmp")
|
|
tmp.write_text(json.dumps(doc, indent=1, sort_keys=True, ensure_ascii=False) + "\n", encoding="utf-8")
|
|
os.replace(tmp, out)
|
|
print(f"part_metrics: {len(doc['packs'])} packs, {doc['totals']['blocks_done']} of {doc['totals']['blocks']} "
|
|
f"blocks, measured {doc['totals']['cost']['measured']} tokens -> {out}")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|