okf-wiki 1.1.1

First public release, published as one snapshot. What changed between
versions is in CHANGELOG.md.

Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
This commit is contained in:
Kjell Tore Guttormsen 2026-11-14 21:03:20 +01:00
commit 0fd86a23a3
Signed by: ktg
SSH key fingerprint: SHA256:JakMjO6FTBBzN0Bhfj9saOoEjaFxlSdYuZQQpM/lF9Q
227 changed files with 96077 additions and 0 deletions

94
tools/check-excluded-terms.sh Executable file
View file

@ -0,0 +1,94 @@
#!/bin/bash
# check-excluded-terms.sh - blocks any tracked text, path or commit message in
# this repository that matches the operator's local excluded-terms list.
#
# The list is READ WHERE IT LIVES and never copied into this repository:
# ${OKF_WIKI_EXCLUDED_TERMS:-$HOME/.claude/runtime/excluded-terms.local.txt}
# One extended regex per line, case-insensitive; '#' lines and blank lines are
# skipped. A missing or empty list is a FAIL, never a silent pass: a check that
# cannot run has measured nothing.
#
# Known-positive: the combined pattern must match at least one line of the list
# itself, or the query is dead and a zero would be meaningless.
#
# Usage:
# tools/check-excluded-terms.sh scan every tracked file (path + content)
# tools/check-excluded-terms.sh --staged scan staged files (pre-commit)
# tools/check-excluded-terms.sh --message F scan a commit message file (commit-msg)
# add -v as the LAST argument to list the matching paths (never the terms)
#
# Exit 0 = no match; 1 = match or the check could not run.
# bash 3.2 safe, ASCII only.
set -u
MODE="tree"
MSG=""
VERBOSE=0
case "${1:-}" in
--staged) MODE="staged" ;;
--message) MODE="message"; MSG="${2:-}" ;;
-v) VERBOSE=1 ;;
esac
for a in "$@"; do [ "$a" = "-v" ] && VERBOSE=1; done
LIST="${OKF_WIKI_EXCLUDED_TERMS:-$HOME/.claude/runtime/excluded-terms.local.txt}"
if [ ! -f "$LIST" ]; then
echo "FAIL excluded-terms: no term list at $LIST - nothing measured (set OKF_WIKI_EXCLUDED_TERMS)" >&2
exit 1
fi
PAT=$(grep -v '^#' "$LIST" | grep -v '^[[:space:]]*$' | paste -sd'|' -)
if [ -z "$PAT" ]; then
echo "FAIL excluded-terms: term list is empty - nothing measured" >&2
exit 1
fi
KP=$(grep -ciE "$PAT" "$LIST")
if [ "$?" -eq 2 ] || [ "$KP" -eq 0 ]; then
echo "FAIL excluded-terms: pattern does not compile or matches nothing in the list itself - query is dead" >&2
exit 1
fi
ROOT=$(git rev-parse --show-toplevel 2>/dev/null) || { echo "FAIL excluded-terms: not inside a git repository" >&2; exit 1; }
TMP=$(mktemp -d "${TMPDIR:-/tmp}/okf-wiki-terms.XXXXXX") || exit 1
trap 'rm -rf "$TMP"' EXIT
: > "$TMP/hits"
if [ "$MODE" = "message" ]; then
if [ -z "$MSG" ] || [ ! -f "$MSG" ]; then
echo "FAIL excluded-terms: no commit message file given" >&2
exit 1
fi
if grep -v '^#' "$MSG" | grep -qiE "$PAT"; then
echo "FAIL excluded-terms: the commit message matches the excluded-terms list" >&2
exit 1
fi
exit 0
fi
if [ "$MODE" = "staged" ]; then
git -C "$ROOT" diff --cached --name-only -z --diff-filter=ACMR > "$TMP/files"
else
git -C "$ROOT" ls-files -z > "$TMP/files"
fi
TOTAL=$(tr -cd '\0' < "$TMP/files" | wc -c | tr -d ' ')
while IFS= read -r -d '' f; do
if printf '%s\n' "$f" | grep -qiE "$PAT"; then
echo "$f" >> "$TMP/hits"
continue
fi
if [ "$MODE" = "staged" ]; then
git -C "$ROOT" show ":$f" 2>/dev/null | grep -IqiE "$PAT" && echo "$f" >> "$TMP/hits"
else
[ -f "$ROOT/$f" ] || continue
grep -IqiE "$PAT" "$ROOT/$f" 2>/dev/null && echo "$f" >> "$TMP/hits"
fi
done < "$TMP/files"
HITS=$(wc -l < "$TMP/hits" | tr -d ' ')
[ "$VERBOSE" -eq 1 ] && sed 's/^/ hit: /' "$TMP/hits"
if [ "$HITS" -eq 0 ]; then
echo "PASS excluded-terms ($MODE): 0 of $TOTAL files match (known-positive: $KP list lines)"
exit 0
fi
echo "FAIL excluded-terms ($MODE): $HITS of $TOTAL files match (add -v to list the paths)" >&2
exit 1

226
tools/claim_features.py Normal file
View file

@ -0,0 +1,226 @@
"""What each claim says beyond the quote it cites, counted by code (wave 4 order 20, contract K9). No model call.
Usage:
python3 tools/claim_features.py --claims claims.json --sources SOURCES --wiki WIKI --out features.json
[--verdicts verdicts.json] [--summary]
It reads the claim list (okfwiki.claims list), the reader's verdicts when given, the sources (SOURCES/kilde/) and the
pages (WIKI/<page>, for the page title in the frontmatter), and writes per claim, in the claim list's order:
quotes, quote_chars, claim_chars the number of excerpts, the characters of their texts summed, of the claim
content_words the claim's content words (the rule below), repeats counted
outside, outside_count the claim's content words whose first five letters begin no content word of
its excerpts, in claim order, repeats counted
outside_title, outside_headings of those, the ones whose first five letters begin a content word of the page
title, and of the heading path above each quoted line in its source
headings that heading path (the headings above the first excerpt's line, outermost
first), taken from okfwiki.kblocks.headings_of, so the tool counts a heading
by the same rule as the build, also after order 22 changes that rule
files, provenance, verdict the excerpts' source files, the claim's provenance mark, and the reader's
verdict (null without --verdicts, or when it gave the claim none)
The word rule (phase 0 of wave 4, 11.10.2026): lower-case the text, take every run of [a-z0-9], drop the words in
STOPWORDS, and compare two words on their first five letters (a shorter word is compared whole). It describes; it
judges nothing. A code check by word overlap was ruled out as a gate (PLAN.md § Retning, choice 5: 8 of 664 near
verbatim), and nothing here is a verdict.
--summary prints the phase 0 table: the share rejected per number of words outside the quote (0, 1, 2, 3-4, 5+),
per provenance mark and per source file, the medians per verdict, and the share with a word from the page title or
from the heading path outside the quote. Without verdicts it prints the counts only. A claim the reader gave no
verdict is left out of the share and counted apart as not judged (order 39), so a partial reading does not lower it.
Output: json.dumps(doc, indent=1, sort_keys=True, ensure_ascii=False) + newline; two runs on the same inputs are
byte-identical, and an existing output is never overwritten. Exit 0 written, 2 cannot run (nothing is written).
"""
from __future__ import annotations
import argparse
import hashlib
import json
import re
import statistics
import sys
from pathlib import Path
from typing import Any
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from okfwiki import kblocks # noqa: E402
from okfwiki.audit import read_document # noqa: E402
from okfwiki.common import RunError, write_new # noqa: E402
SCHEMA = "okf-wiki/claim-features/1"
WORD = re.compile(r"[a-z0-9]+")
STOPWORDS = frozenset(
"a an the of to in on for and or is are be by with from as at it this that its can you your when if not no into "
"use uses using".split())
STEM = 5
BUCKETS = (("0", 0, 0), ("1", 1, 1), ("2", 2, 2), ("3-4", 3, 4), ("5+", 5, None))
UNSUPPORTED = "unsupported"
def words(text: str) -> list[str]:
"""The content words of a text, in order, repeats kept."""
return [w for w in WORD.findall(text.lower()) if w not in STOPWORDS]
def stems(text: str) -> set[str]:
return {w[:STEM] for w in words(text)}
def heading_path(lines: list[str], line: int) -> list[str]:
"""The headings above a line, outermost first: each heading closes every open one at its level or deeper."""
path: list[tuple[int, str]] = []
for n, level, text in kblocks.headings_of(lines):
if n >= line:
break
path = [(lv, t) for lv, t in path if lv < level] + [(level, text)]
return [t for _, t in path]
def read_json(path: Path) -> Any:
try:
return json.loads(path.read_text(encoding="utf-8"))
except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc:
raise RunError(f"cannot read {path}: {exc}") from exc
def sha256(path: Path) -> str:
return hashlib.sha256(path.read_bytes()).hexdigest()
def page_title(wiki: Path, page: str) -> str:
path = wiki / page
if not path.is_file():
raise RunError(f"the page {path} is not there")
fm = read_document(path, page).frontmatter
return str((fm or {}).get("title") or "")
def features(claims_doc: dict[str, Any], verdicts: dict[str, str] | None, sources: Path,
wiki: Path) -> list[dict[str, Any]]:
kilde = kblocks.kilde_of(sources)
lines_of: dict[str, list[str]] = {}
titles: dict[str, str] = {}
out: list[dict[str, Any]] = []
for c in claims_doc["claims"]:
excerpts = list(c["excerpts"])
quote_stems: set[str] = set()
heading_stems: set[str] = set()
headings: list[str] = []
for i, e in enumerate(excerpts):
name = str(e["file"])
if name not in lines_of:
path = kilde / name
if not path.is_file():
raise RunError(f"the source {path} is not there")
lines_of[name] = kblocks.read_lines(path)
path_here = heading_path(lines_of[name], int(e["line"]))
headings = path_here if i == 0 else headings
heading_stems |= {s for h in path_here for s in stems(h)}
quote_stems |= stems(str(e["text"]))
page = str(c["page"])
if page not in titles:
titles[page] = page_title(wiki, page)
title_stems = stems(titles[page])
content = words(str(c["claim"]))
outside = [w for w in content if w[:STEM] not in quote_stems]
out.append({
"id": c["id"], "page": page, "provenance": c.get("provenance"),
"files": sorted({str(e["file"]) for e in excerpts}),
"verdict": None if verdicts is None else verdicts.get(str(c["id"])),
"quotes": len(excerpts), "quote_chars": sum(len(str(e["text"])) for e in excerpts),
"claim_chars": len(str(c["claim"])), "content_words": len(content),
"outside": outside, "outside_count": len(outside),
"outside_title": [w for w in outside if w[:STEM] in title_stems],
"outside_headings": [w for w in outside if w[:STEM] in heading_stems],
"headings": headings,
})
return out
def _share(rows: list[dict[str, Any]], judged: bool) -> str:
if not judged:
return f"{len(rows)} claims, no verdicts"
bad = sum(1 for r in rows if r["verdict"] == UNSUPPORTED)
n_judged = sum(1 for r in rows if r["verdict"] is not None)
if n_judged == len(rows):
pct = f"{100 * bad / n_judged:.1f} %" if n_judged else "-"
return f"{len(rows)} claims, {bad} unsupported ({pct})"
pct = f"{100 * bad / n_judged:.1f} % of {n_judged} judged" if n_judged else "- of 0 judged"
return f"{len(rows)} claims, {bad} unsupported ({pct}), {len(rows) - n_judged} not judged"
def _median(xs: list[int]) -> str:
return f"{statistics.median(xs):g}" if xs else "-"
def summary(rows: list[dict[str, Any]]) -> str:
judged = any(r["verdict"] is not None for r in rows)
out = [f"claims: {len(rows)}"]
for label, lo, hi in BUCKETS:
mine = [r for r in rows if r["outside_count"] >= lo and (hi is None or r["outside_count"] <= hi)]
out.append(f"words outside the quote {label}: {_share(mine, judged)}")
for key, label in (("provenance", "provenance"), ("files", "source")):
values = sorted({v for r in rows for v in (r[key] if isinstance(r[key], list) else [r[key]])}, key=str)
for v in values:
mine = [r for r in rows if v in (r[key] if isinstance(r[key], list) else [r[key]])]
out.append(f"{label} {v}: {_share(mine, judged)}")
if not judged:
return "\n".join(out) + "\n"
groups = {name: [r for r in rows if (r["verdict"] == UNSUPPORTED) == (name == UNSUPPORTED)
and r["verdict"] is not None] for name in ("supported", UNSUPPORTED)}
for field, label in (("outside_count", "words outside"), ("quote_chars", "quote characters"),
("claim_chars", "claim characters"), ("content_words", "content words")):
parts = [f"{name} {_median([int(r[field]) for r in g])}" for name, g in groups.items()]
out.append(f"median {label}: {', '.join(parts)}")
for field, label in (("outside_title", "the page title"), ("outside_headings", "the heading path")):
parts = []
for name, g in groups.items():
k = sum(1 for r in g if r[field])
pct = f"{100 * k / len(g):.1f} %" if g else "-"
parts.append(f"{name} {k} of {len(g)} ({pct})")
out.append(f"with a word from {label} outside the quote: {', '.join(parts)}")
return "\n".join(out) + "\n"
def build_parser() -> argparse.ArgumentParser:
p = argparse.ArgumentParser(prog="python3 tools/claim_features.py", description=__doc__,
formatter_class=argparse.RawDescriptionHelpFormatter)
p.add_argument("--claims", required=True)
p.add_argument("--sources", required=True)
p.add_argument("--wiki", required=True)
p.add_argument("--out", required=True)
p.add_argument("--verdicts")
p.add_argument("--summary", action="store_true")
return p
def main(argv: list[str] | None = None) -> int:
args = build_parser().parse_args(argv)
try:
out = Path(args.out)
if out.exists():
raise RunError(f"{out} exists; an output is never overwritten")
claims_path = Path(args.claims)
claims_doc = read_json(claims_path)
verdicts: dict[str, str] | None = None
inputs: dict[str, Any] = {"claims_sha256": sha256(claims_path), "verdicts_sha256": None}
if args.verdicts:
vpath = Path(args.verdicts)
verdicts = {str(v["id"]): str(v["verdict"]) for v in read_json(vpath)["verdicts"]}
inputs["verdicts_sha256"] = sha256(vpath)
rows = features(claims_doc, verdicts, Path(args.sources), Path(args.wiki))
doc = {"schema": SCHEMA, "inputs": inputs, "claims": rows}
write_new(out, json.dumps(doc, indent=1, sort_keys=True, ensure_ascii=False) + "\n")
except (RunError, OSError, KeyError, TypeError, ValueError) as exc:
print(f"claim_features cannot run: {exc}", file=sys.stderr)
return 2
print(f"claim_features: {len(rows)} claims -> {out}")
if args.summary:
print(summary(rows), end="")
return 0
if __name__ == "__main__":
sys.exit(main())

421
tools/diagnose.py Normal file
View file

@ -0,0 +1,421 @@
"""Why a claim is or is not carried by its excerpt: the tool around the diagnosis of wave 4 order 20 (contract K9).
No model call. The diagnosis explains; the reader judges. It is not calibrated and never counts for criterion 2.
Usage:
python3 tools/diagnose.py select --claims claims.json --verdicts verdicts.json --out utvalg.json [--sample 100]
python3 tools/diagnose.py render --utvalg utvalg.json --claims claims.json --sources SOURCES --wiki WIKI
--out tasks/ [--size 10]
python3 tools/diagnose.py collect --transcripts DIR --run R [--run R2]... --tasks tasks/ --out diagnoses.json
python3 tools/diagnose.py tally --diagnoses diagnoses.json --verdicts verdicts.json --claims claims.json
--out tally.json
select every claim the reader did not accept, and a sample of the accepted ones:
random.Random(SEED).sample(sorted(ids), --sample), then the two lists (each sorted) shuffled with the same
generator. The selection holds ids only, never a verdict. A claim without a verdict, or fewer accepted claims
than the sample, cannot run.
render one task per call of at most --size claims, in the selection's order, named d001, d002, ...; and
calls.json (the ids per call). Per claim: the page title, the claim, each excerpt it cites, and per excerpt a
window in its source: the heading path above the quoted line (okfwiki.kblocks.headings_of, as
tools/claim_features.py takes it) and WINDOW lines before and after it, cut at the file's ends. Each window
line is "<line number> <q><c>| <text>", q ">" on the quoted line, c "c" on a line of fenced code (the fence
lines too). No id, verdict or provenance mark reaches the task. A task folder that holds anything is never
written into.
collect the reply of every diagnosis agent (agent type okf-wiki-diagnoser, description "V1-<run>-diagnose-<call>",
started from the session or from a workflow, the two places okfwiki.cost harvest reads): the message it
handed back, or else its last text (okfwiki.reader.final_reply). For each call the last --run that answered
it counts; two transcripts of that run for one call give no diagnosis for any of its claims. A line
"<number>: <code> | <words>" gives the claim of that number its code (one of CODES) and the words that are
missing; S needs no words, every other code does, and A has at most twelve. A line that starts with a number
but breaks the form, or a number given twice, gives that claim no diagnosis: it is "missing", never guessed.
Other lines are ignored.
tally counts per code for the claims the reader did not accept and for the sample it accepted, each apart; how
often the diagnosis and the reader disagree (S on a rejected claim; any other code on an accepted one) with a
two-sided 95 % Clopper-Pearson interval; the same counts per source file and per provenance mark; and the M1
rule of order 20: S for at least a third of the rejected claims, or any claim missing, is stop and present.
Codes (locked in phase 0 before any diagnosis was read): S the excerpt alone carries the claim; O what is missing
stands in the heading path or the page title; L it stands in another line of the window; B the claim says more than
the source (wider scope, stronger words); T a number, version, order, cause or purpose that is not in the window; F a
property given to the wrong thing, or a wrong connection; U what is missing is not in the window at all; A other.
Output: json.dumps(doc, indent=1, sort_keys=True, ensure_ascii=False) + newline; an existing output is never
overwritten. Exit 0 written; 1 written with claims missing (collect) or the M1 rule says stop (tally); 2 cannot run.
"""
from __future__ import annotations
import argparse
import hashlib
import json
import math
import random
import re
import sys
from pathlib import Path
from typing import Any
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
sys.path.insert(0, str(Path(__file__).resolve().parent))
import claim_features # noqa: E402
from okfwiki import kblocks # noqa: E402
from okfwiki.common import RunError, write_new # noqa: E402
from okfwiki.reader import final_reply # noqa: E402
from okfwiki.transcript import is_agent_type # noqa: E402
SELECTION_SCHEMA = "okf-wiki/diagnose-selection/1"
CALLS_SCHEMA = "okf-wiki/diagnose-calls/1"
DIAGNOSES_SCHEMA = "okf-wiki/diagnoses/1"
TALLY_SCHEMA = "okf-wiki/diagnose-tally/1"
SEED = 20261012
SAMPLE = 100
CALL_SIZE = 10
WINDOW = 8
CODES = ("S", "O", "L", "B", "T", "F", "U", "A")
OTHER_WORDS_MAX = 12
AGENT_TYPE = "okf-wiki-diagnoser"
FENCE = re.compile(r"^\s*(```|~~~)") # the fence rule of okfwiki.kblocks.headings_of
NUMBERED = re.compile(r"^\s*(\d+)\s*:(.*)$")
FORM = re.compile(r"^\s*([A-Z])\s*(?:\|\s*(.*?))?\s*$")
UNSUPPORTED = "unsupported"
SUPPORTED = "supported"
GROUPS = ("unsupported", "supported_sample")
def read_json(path: Path) -> Any:
try:
return json.loads(path.read_text(encoding="utf-8"))
except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc:
raise RunError(f"cannot read {path}: {exc}") from exc
def dump(doc: Any) -> str:
return json.dumps(doc, indent=1, sort_keys=True, ensure_ascii=False) + "\n"
def sha256(path: Path) -> str:
return hashlib.sha256(path.read_bytes()).hexdigest()
def verdict_map(path: Path) -> dict[str, str]:
return {str(v["id"]): str(v["verdict"]) for v in read_json(path)["verdicts"]}
# --- select ----------------------------------------------------------------------------------------------------
def select(claim_ids: list[str], verdicts: dict[str, str], sample: int) -> list[str]:
rejected: list[str] = []
accepted: list[str] = []
for cid in claim_ids:
v = verdicts.get(cid)
if v not in (SUPPORTED, UNSUPPORTED):
raise RunError(f"claim {cid} has no verdict of {SUPPORTED} or {UNSUPPORTED}")
(rejected if v == UNSUPPORTED else accepted).append(cid)
if len(accepted) < sample:
raise RunError(f"{len(accepted)} accepted claims, fewer than the sample of {sample}")
rng = random.Random(SEED)
picked = rng.sample(sorted(accepted), sample)
ids = sorted(rejected) + sorted(picked)
rng.shuffle(ids)
return ids
# --- render ----------------------------------------------------------------------------------------------------
def code_lines(lines: list[str]) -> set[int]:
out: set[int] = set()
fence: str | None = None
for n, raw in enumerate(lines, 1):
f = FENCE.match(raw)
if f:
out.add(n)
fence = None if fence == f.group(1) else (fence or f.group(1))
continue
if fence is not None:
out.add(n)
return out
def window(lines: list[str], name: str, line: int) -> str:
first, last = max(1, line - WINDOW), min(len(lines), line + WINDOW)
code = code_lines(lines)
path = " › ".join(claim_features.heading_path(lines, line)).replace('"', "'")
rows = [f"{n:>5} {'>' if n == line else ' '}{'c' if n in code else ' '}| {lines[n - 1]}".rstrip()
for n in range(first, last + 1)]
return "\n".join([f'<window file="{name}" lines="{first}-{last}" headings="{path}">', *rows, "</window>"])
def render_task(claims: list[dict[str, Any]], titles: dict[str, str], lines_of: dict[str, list[str]]) -> str:
n = len(claims)
parts = ["# Diagnosis task", "", f"This task has {n} claim{'s' if n != 1 else ''}, numbered 1 to {n}."]
for number, c in enumerate(claims, 1):
parts += ["", f"## Claim {number}", "", f"Page title: {titles[str(c['page'])]}", "",
f"<claim>{c['claim']}</claim>"]
for e in c["excerpts"]:
parts += ["", f'<excerpt file="{e["file"]}" line="{e["line"]}">', str(e["text"]), "</excerpt>"]
for e in c["excerpts"]:
parts += ["", window(lines_of[str(e["file"])], str(e["file"]), int(e["line"]))]
return "\n".join(parts) + "\n"
def render(ids: list[str], claims_doc: dict[str, Any], sources: Path, wiki: Path, size: int,
out: Path) -> list[dict[str, Any]]:
if size < 1:
raise RunError("--size must be at least 1")
if out.exists() and any(out.iterdir()):
raise RunError(f"{out} holds files; a task folder is never written into")
by_id = {str(c["id"]): c for c in claims_doc["claims"]}
missing = [i for i in ids if i not in by_id]
if missing:
raise RunError(f"selected ids not in the claim list: {missing[:5]}")
kilde = kblocks.kilde_of(sources)
lines_of: dict[str, list[str]] = {}
titles: dict[str, str] = {}
for cid in ids:
c = by_id[cid]
page = str(c["page"])
if page not in titles:
titles[page] = claim_features.page_title(wiki, page)
for e in c["excerpts"]:
name = str(e["file"])
if name not in lines_of:
if not (kilde / name).is_file():
raise RunError(f"the source {kilde / name} is not there")
lines_of[name] = kblocks.read_lines(kilde / name)
chunks = [(f"d{k + 1:03d}", ids[i:i + size]) for k, i in enumerate(range(0, len(ids), size))]
texts = {call: render_task([by_id[i] for i in chunk], titles, lines_of) for call, chunk in chunks}
for call, text in texts.items():
write_new(out / f"{call}.md", text)
return [{"name": call, "ids": chunk} for call, chunk in chunks]
# --- collect ---------------------------------------------------------------------------------------------------
def agent_replies(transcripts: Path, run: str) -> dict[str, list[Path]]:
prefix = re.compile(rf"^V1-{re.escape(run)}-diagnose-(\S+)$")
found: dict[str, list[Path]] = {}
metas = [*transcripts.glob("*/subagents/agent-*.meta.json"),
*transcripts.glob("*/subagents/workflows/*/agent-*.meta.json")]
for meta_path in sorted(metas):
try:
meta = json.loads(meta_path.read_text(encoding="utf-8"))
except (OSError, ValueError):
continue
if not isinstance(meta, dict) or not is_agent_type(meta.get("agentType"), AGENT_TYPE):
continue
m = prefix.match(str(meta.get("description", "")))
if m is not None:
found.setdefault(m.group(1), []).append(meta_path.with_name(
meta_path.name.removesuffix(".meta.json") + ".jsonl"))
return found
def parse_reply(reply: str, n: int) -> dict[int, tuple[str, str]]:
"""The code and missing words per claim number; a broken line or a number given twice leaves that claim out."""
given: dict[int, tuple[str, str]] = {}
bad: set[int] = set()
for raw in reply.splitlines():
m = NUMBERED.match(raw)
if m is None:
continue
number = int(m.group(1))
if not 1 <= number <= n:
continue
if number in given or number in bad:
given.pop(number, None)
bad.add(number)
continue
f = FORM.match(m.group(2))
code, words = (f.group(1), (f.group(2) or "").strip()) if f else ("", "")
ok = (code in CODES and (code == "S" or bool(words))
and (code != "A" or len(words.split()) <= OTHER_WORDS_MAX))
if ok:
given[number] = (code, words)
else:
bad.add(number)
return given
def collect(calls: list[dict[str, Any]], runs: list[str], transcripts: Path) -> dict[str, Any]:
found = {r: agent_replies(transcripts, r) for r in runs}
rows: list[dict[str, Any]] = []
for call in calls:
name, ids = str(call["name"]), [str(i) for i in call["ids"]]
answering = [r for r in runs if name in found[r]]
run = answering[-1] if answering else None
paths = found[run][name] if run else []
given = parse_reply(final_reply(paths[0]) if paths[0].is_file() else "", len(ids)) if len(paths) == 1 else {}
for number, cid in enumerate(ids, 1):
code, words = given.get(number, (None, ""))
rows.append({"id": cid, "call": name, "number": number, "code": code, "words": words, "run": run})
return {"schema": DIAGNOSES_SCHEMA, "runs": runs, "diagnoses": rows,
"missing": [r["id"] for r in rows if r["code"] is None]}
# --- tally -----------------------------------------------------------------------------------------------------
def _binom_tail(k_from: int, k_to: int, n: int, p: float) -> float:
return sum(math.exp(math.lgamma(n + 1) - math.lgamma(i + 1) - math.lgamma(n - i + 1)
+ i * math.log(p) + (n - i) * math.log1p(-p)) for i in range(k_from, k_to + 1))
def _solve(f: Any, rising: bool) -> float:
lo, hi = 0.0, 1.0
for _ in range(200):
mid = (lo + hi) / 2
if mid <= 0.0 or mid >= 1.0:
break
if (f(mid) < 0.025) == rising:
lo = mid
else:
hi = mid
return (lo + hi) / 2
def clopper_pearson(k: int, n: int) -> tuple[float, float]:
"""The two-sided 95 % Clopper-Pearson interval for k events in n."""
if n <= 0:
return 0.0, 1.0
lower = 0.0 if k == 0 else _solve(lambda p: _binom_tail(k, n, n, p), rising=True)
upper = 1.0 if k >= n else _solve(lambda p: _binom_tail(0, k, n, p), rising=False)
return lower, upper
def counts(rows: list[dict[str, Any]]) -> dict[str, int]:
out = {code: 0 for code in CODES}
out["missing"] = 0
for r in rows:
out[r["code"] if r["code"] in CODES else "missing"] += 1
out["n"] = len(rows)
return out
def tally(diagnoses: dict[str, Any], verdicts: dict[str, str], claims_doc: dict[str, Any]) -> dict[str, Any]:
by_id = {str(c["id"]): c for c in claims_doc["claims"]}
grouped: dict[str, list[dict[str, Any]]] = {g: [] for g in GROUPS}
for row in diagnoses["diagnoses"]:
v = verdicts.get(str(row["id"]))
if v not in (SUPPORTED, UNSUPPORTED) or str(row["id"]) not in by_id:
raise RunError(f"diagnosed claim {row['id']} has no verdict or is not in the claim list")
grouped["unsupported" if v == UNSUPPORTED else "supported_sample"].append(row)
def split(key_of: Any) -> dict[str, dict[str, dict[str, int]]]:
keys = sorted({k for g in grouped.values() for r in g for k in key_of(by_id[str(r["id"])])})
return {k: {g: counts([r for r in grouped[g] if k in key_of(by_id[str(r["id"])])]) for g in GROUPS}
for k in keys}
rej, acc = grouped["unsupported"], grouped["supported_sample"]
rej_done = [r for r in rej if r["code"] in CODES]
acc_done = [r for r in acc if r["code"] in CODES]
called_s = sum(1 for r in rej_done if r["code"] == "S")
not_carried = sum(1 for r in acc_done if r["code"] != "S")
missing = sum(1 for g in grouped.values() for r in g if r["code"] not in CODES)
return {
"schema": TALLY_SCHEMA,
"unsupported": counts(rej), "supported_sample": counts(acc),
"agreement": {"unsupported_called_s": called_s, "unsupported_diagnosed": len(rej_done),
"unsupported_called_s_ci95": list(clopper_pearson(called_s, len(rej_done))),
"supported_not_carried": not_carried, "supported_diagnosed": len(acc_done),
"supported_not_carried_ci95": list(clopper_pearson(not_carried, len(acc_done)))},
"per_source": split(lambda c: sorted({str(e["file"]) for e in c["excerpts"]})),
"per_provenance": split(lambda c: [str(c.get("provenance"))]),
"m1": {"s": called_s, "of": len(rej), "missing": missing, "stop": 3 * called_s >= len(rej) or missing > 0},
}
def tally_text(doc: dict[str, Any]) -> str:
out = []
for g in GROUPS:
c = doc[g]
out.append(f"{g}: n {c['n']}; " + ", ".join(f"{k} {c[k]}" for k in (*CODES, "missing")))
a = doc["agreement"]
lo, hi = a["supported_not_carried_ci95"]
out.append(f"accepted by the reader, not carried by the excerpt per the diagnosis: {a['supported_not_carried']} "
f"of {a['supported_diagnosed']} (95 % CI {100 * lo:.1f}-{100 * hi:.1f} %)")
lo, hi = a["unsupported_called_s_ci95"]
out.append(f"rejected by the reader, carried per the diagnosis (S): {a['unsupported_called_s']} of "
f"{a['unsupported_diagnosed']} (95 % CI {100 * lo:.1f}-{100 * hi:.1f} %)")
m = doc["m1"]
verdict = "STOP and present" if m["stop"] else "continue"
out.append(f"M1 rule: S for {m['s']} of {m['of']} rejected, {m['missing']} missing: {verdict}")
return "\n".join(out) + "\n"
# --- command line ----------------------------------------------------------------------------------------------
def build_parser() -> argparse.ArgumentParser:
p = argparse.ArgumentParser(prog="python3 tools/diagnose.py", description=__doc__,
formatter_class=argparse.RawDescriptionHelpFormatter)
sub = p.add_subparsers(dest="command", required=True)
s = sub.add_parser("select")
s.add_argument("--claims", required=True)
s.add_argument("--verdicts", required=True)
s.add_argument("--out", required=True)
s.add_argument("--sample", type=int, default=SAMPLE)
r = sub.add_parser("render")
r.add_argument("--utvalg", required=True)
r.add_argument("--claims", required=True)
r.add_argument("--sources", required=True)
r.add_argument("--wiki", required=True)
r.add_argument("--out", required=True)
r.add_argument("--size", type=int, default=CALL_SIZE)
c = sub.add_parser("collect")
c.add_argument("--transcripts", required=True)
c.add_argument("--run", action="append", required=True)
c.add_argument("--tasks", required=True)
c.add_argument("--out", required=True)
t = sub.add_parser("tally")
t.add_argument("--diagnoses", required=True)
t.add_argument("--verdicts", required=True)
t.add_argument("--claims", required=True)
t.add_argument("--out", required=True)
return p
def main(argv: list[str] | None = None) -> int:
args = build_parser().parse_args(argv)
try:
out = Path(args.out)
if args.command == "select":
if out.exists():
raise RunError(f"{out} exists; an output is never overwritten")
claims_doc = read_json(Path(args.claims))
ids = select([str(c["id"]) for c in claims_doc["claims"]], verdict_map(Path(args.verdicts)), args.sample)
write_new(out, dump({"schema": SELECTION_SCHEMA, "seed": SEED, "sample": args.sample, "ids": ids}))
print(f"select: {len(ids)} claims ({args.sample} of them a sample) -> {out}")
return 0
if args.command == "render":
utvalg = Path(args.utvalg)
ids = [str(i) for i in read_json(utvalg)["ids"]]
calls = render(ids, read_json(Path(args.claims)), Path(args.sources), Path(args.wiki), args.size, out)
write_new(out / "calls.json", dump({"schema": CALLS_SCHEMA, "utvalg_sha256": sha256(utvalg),
"size": args.size, "calls": calls}))
print(f"render: {len(ids)} claims in {len(calls)} calls -> {out}")
return 0
if args.command == "collect":
if out.exists():
raise RunError(f"{out} exists; an output is never overwritten")
calls = read_json(Path(args.tasks) / "calls.json")["calls"]
doc = collect(calls, list(args.run), Path(args.transcripts))
write_new(out, dump(doc))
print(f"collect: {len(doc['diagnoses']) - len(doc['missing'])} of {len(doc['diagnoses'])} claims "
f"diagnosed, {len(doc['missing'])} missing -> {out}")
return 1 if doc["missing"] else 0
if out.exists():
raise RunError(f"{out} exists; an output is never overwritten")
doc = tally(read_json(Path(args.diagnoses)), verdict_map(Path(args.verdicts)), read_json(Path(args.claims)))
write_new(out, dump(doc))
print(tally_text(doc), end="")
return 1 if doc["m1"]["stop"] else 0
except (RunError, OSError, KeyError, TypeError, ValueError) as exc:
print(f"diagnose cannot run: {exc}", file=sys.stderr)
return 2
if __name__ == "__main__":
sys.exit(main())

482
tools/fetch_docs.py Normal file
View file

@ -0,0 +1,482 @@
"""Fetch the Claude Code documentation tree into a staging folder that `okfwiki.sources freeze` reads. No model.
Usage:
python3 tools/fetch_docs.py --staging DIR --meta FILE --report FILE --preamble FILE
[--base URL] [--index URL] [--changelog URL] [--changelog-page URL]
[--pause SECONDS] [--cafile FILE] [--collection NAME]
The index (llms.txt) lists the pages. Every unique address under --base is fetched as .md, one at a time with a pause;
"\\r\\n" becomes "\\n". Addresses outside --base (the indexes of other languages) are listed, not fetched.
Preamble Each page starts with a block of "> " lines whose first is a level-2 heading, then one blank line. It is
removed when its number of lines and the sha256 of its lines ("\\n"-joined) equal the --preamble file
({"lines": N, "sha256": "<hex>"}, kept outside git). Any other start is an error, never a guess.
Moved out A page whose final address after redirects lies outside --base is no longer a page of this tree: it is
not a source, and it is listed in the report (the operator's decision of 08.10.2026 on such a page).
History The weekly digest pages (whats-new/YYYY-wNN.md under --base) are docs sources with main_topic false: they
add history to the concepts and get no main page of their own (the operator's decision of 08.10.2026). The
overview whats-new/index.md is an ordinary docs page.
Changelog The release-note page is not a docs source. After its preamble it is split into one file per release,
the text untouched, named release-vX.Y.Z.md, with the url --changelog-page#X-Y-Z; every anchor is checked
on that page. Preamble + releases + the rest = the page. Two forms, never mixed on one page:
component one <Update label="X.Y.Z"> ... </Update> block per release (the copy of 08.10);
bold a line that is only "**X.Y.Z**", a blank line and the date ("May 1, 2026"); the release runs to
the next such line or the end of the page (the page's form after 20.10, 437 of 437 on 07.11).
Complete docs sources = unique index addresses under --base - the changelog - the pages moved out, and release-note
sources = release_count(page), counted apart: the "<Update label=" on the page, or, in the bold form, every
bold line that starts with a version. A difference is a FAIL.
Exit 0: staging folder and meta written. Exit 1: a page broke a rule or the completeness check failed; the report
lists why and nothing else is written. Exit 2: cannot run (staging, meta or report exists, the index cannot be
fetched, the network fails).
"""
from __future__ import annotations
import argparse
import hashlib
import json
import os
import re
import ssl
import sys
import time
import urllib.error
import urllib.request
from collections.abc import Callable
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from okfwiki.common import RunError, write_new # noqa: E402
from okfwiki.sources import META_SCHEMA, NAME, dump # noqa: E402
BASE = "https://code.claude.com/docs/en/"
INDEX_URL = "https://code.claude.com/docs/llms.txt"
CHANGELOG = BASE + "changelog.md"
CHANGELOG_PAGE = BASE + "changelog"
SYSTEM_CA = "/etc/ssl/cert.pem"
USER_AGENT = "okf-wiki-fetch-docs/1"
LINK = re.compile(r"^\s*[-*]\s+\[([^\]]*)\]\((\S+?)\)")
HEADING = re.compile(r"^(#{2,4})\s+(.*?)\s*$")
OPEN = re.compile(r'^<Update label="([^"]*)"[^>]*>$')
CLOSE = "</Update>"
VERSION = re.compile(r"^\d+\.\d+\.\d+$")
BOLD = re.compile(r"^\*\*(\d+\.\d+\.\d+)\*\*$")
BOLD_LIKE = re.compile(r"^\*\*v?\d+\.\d[^*]*\*\*\s*$")
DATE = re.compile(r"^[A-Z][a-z]+ \d{1,2}, \d{4}$")
HISTORY = re.compile(r"^whats-new/\d{4}-w\d{2}\.md$")
HEX64 = re.compile(r"^[0-9a-f]{64}$")
class FetchError(Exception):
"""A page or the index breaks a rule of the fetch."""
@dataclass(frozen=True)
class Fetched:
status: int
url: str # the final address after redirects
content_type: str
body: bytes
Fetcher = Callable[[str], Fetched]
@dataclass(frozen=True)
class Preamble:
lines: int
sha256: str
@dataclass(frozen=True)
class IndexPage:
url: str
title: str
sections: tuple[str, ...]
@dataclass(frozen=True)
class Index:
items: int
pages: list[IndexPage]
other_links: list[str]
@dataclass(frozen=True)
class Block:
version: str
text: str
@dataclass(frozen=True)
class ChangelogSplit:
blocks: list[Block]
preamble_bytes: int
block_bytes: int
rest_bytes: int
@dataclass
class Collected:
files: dict[str, bytes] = field(default_factory=dict)
meta: dict[str, Any] = field(default_factory=dict)
report: dict[str, Any] = field(default_factory=dict)
problems: list[str] = field(default_factory=list)
def parse_index(text: str, base: str) -> Index:
"""The list items of the index: unique addresses under base in order, each with its ##/###/#### headings."""
items = 0
stack: list[str] = []
pages: dict[str, IndexPage] = {}
other: list[str] = []
for line in text.split("\n"):
h = HEADING.match(line)
if h:
stack = stack[: len(h.group(1)) - 2] + [h.group(2)]
continue
m = LINK.match(line)
if not m:
continue
items += 1
title, url = m.group(1), m.group(2)
if url.startswith(base):
pages.setdefault(url, IndexPage(url, title, tuple(stack)))
elif url not in other:
other.append(url)
return Index(items, list(pages.values()), other)
def source_name(url: str, base: str) -> str:
"""https://…/docs/en/a/b.md -> docs-en-a-b.md (the pattern of the copy of 21.09)."""
if not url.startswith(base) or not url.endswith(".md"):
raise FetchError(f"{url} is not a .md address under {base}")
rel = url[len(base): -len(".md")]
name = f"docs-{base.rstrip('/').rsplit('/', 1)[-1]}-{rel.replace('/', '-')}.md"
if not rel or not NAME.match(name):
raise FetchError(f"{url} gives the name {name!r}, not [a-z0-9._-]")
return name
def is_history(rel: str) -> bool:
"""A weekly digest page, by its path under the docs base: history, not a main topic (operator 08.10.2026)."""
return bool(HISTORY.match(rel))
def load_preamble(path: Path) -> Preamble:
try:
doc = json.loads(path.read_text(encoding="utf-8"))
except (OSError, json.JSONDecodeError) as exc:
raise FetchError(f"the preamble file {path} cannot be read: {exc}") from None
lines, digest = (doc.get("lines"), doc.get("sha256")) if isinstance(doc, dict) else (None, None)
if not (isinstance(lines, int) and not isinstance(lines, bool) and lines >= 1
and isinstance(digest, str) and HEX64.match(digest)):
raise FetchError(f"the preamble file {path} is not {{\"lines\": N, \"sha256\": \"<hex>\"}}")
return Preamble(lines, digest)
def strip_preamble(text: str, spec: Preamble) -> tuple[str, int]:
"""The page without its preamble and the blank line after it, and the number of bytes removed."""
lines = text.split("\n")
n = 0
while n < len(lines) and lines[n].startswith("> "):
n += 1
pre = lines[:n]
if n != spec.lines or not pre[0].startswith("> ## ") or n >= len(lines) or lines[n] != "":
raise FetchError(f"the page does not start with {spec.lines} '> ' lines, a '> ## ' heading first, "
f"and a blank line (found {n} '> ' lines)")
if hashlib.sha256("\n".join(pre).encode("utf-8")).hexdigest() != spec.sha256:
raise FetchError("the preamble has the known shape but not the known sha256")
rest = "\n".join(lines[n + 1:])
return rest, len(text.encode("utf-8")) - len(rest.encode("utf-8"))
def release_count(text: str) -> int:
"""The releases on the page, counted apart from the split: its "<Update label=", or its bold version-like lines."""
if "<Update label=" in text:
return text.count("<Update label=")
return sum(1 for line in text.split("\n") if BOLD_LIKE.match(line))
def split_bold(body: str, pre_bytes: int, total: int) -> ChangelogSplit:
"""One block per "**X.Y.Z**" line with a blank line and a date after it, running to the next one."""
lines = [line + "\n" for line in body.split("\n")]
lines[-1] = lines[-1][:-1]
bare = [line.rstrip("\n") for line in lines]
starts: list[tuple[int, str]] = []
for i, line in enumerate(bare):
if not BOLD_LIKE.match(line):
continue
m = BOLD.match(line)
if not m:
raise FetchError(f"line {i + 1}: {line!r} is not **X.Y.Z**")
if not (i + 2 < len(bare) and bare[i + 1] == "" and DATE.match(bare[i + 2])):
raise FetchError(f"line {i + 1}: version {m.group(1)} is not followed by a blank line and a date")
if any(v == m.group(1) for _, v in starts):
raise FetchError(f"line {i + 1}: version {m.group(1)} appears twice")
starts.append((i, m.group(1)))
if not starts:
raise FetchError("the page has no <Update label=…> block and no **X.Y.Z** release line")
ends = [i for i, _ in starts[1:]] + [len(lines)]
blocks = [Block(v, "".join(lines[i:end])) for (i, v), end in zip(starts, ends)]
rest_bytes = sum(len(line.encode("utf-8")) for line in lines[:starts[0][0]])
block_bytes = sum(len(b.text.encode("utf-8")) for b in blocks)
if pre_bytes + block_bytes + rest_bytes != total:
raise FetchError("preamble + blocks + rest is not the length of the page")
return ChangelogSplit(blocks, pre_bytes, block_bytes, rest_bytes)
def split_changelog(text: str, spec: Preamble) -> ChangelogSplit:
"""One block per release, the text untouched, in the component or the bold form; every byte is counted."""
body, pre_bytes = strip_preamble(text, spec)
if "<Update label=" not in body:
return split_bold(body, pre_bytes, len(text.encode("utf-8")))
if any(BOLD_LIKE.match(line) for line in body.split("\n")):
raise FetchError("the page mixes <Update label=…> blocks and **X.Y.Z** release lines")
lines = [line + "\n" for line in body.split("\n")]
lines[-1] = lines[-1][:-1]
blocks: list[Block] = []
rest_bytes = 0
current: list[str] | None = None
version = ""
for n, line in enumerate(lines, 1):
bare = line.rstrip("\n")
opened = OPEN.match(bare)
if current is None:
if opened:
version = opened.group(1)
if not VERSION.match(version):
raise FetchError(f"line {n}: the label {version!r} is not X.Y.Z")
if any(b.version == version for b in blocks):
raise FetchError(f"line {n}: version {version} appears twice")
current = [line]
elif bare == CLOSE:
raise FetchError(f"line {n}: {CLOSE} without an open block")
else:
rest_bytes += len(line.encode("utf-8"))
else:
if opened:
raise FetchError(f"line {n}: a block opens inside version {version}")
current.append(line)
if bare == CLOSE:
blocks.append(Block(version, "".join(current)))
current = None
if current is not None:
raise FetchError(f"version {version} is never closed")
if not blocks:
raise FetchError("the page has no <Update label=…> block")
if body.count("<Update label=") != len(blocks):
raise FetchError(f"the page has {body.count('<Update label=')} '<Update label=', but {len(blocks)} blocks "
"start a line")
block_bytes = sum(len(b.text.encode("utf-8")) for b in blocks)
if pre_bytes + block_bytes + rest_bytes != len(text.encode("utf-8")):
raise FetchError("preamble + blocks + rest is not the length of the page")
return ChangelogSplit(blocks, pre_bytes, block_bytes, rest_bytes)
def anchor_url(page: str, version: str) -> str:
return f"{page}#{version.replace('.', '-')}"
def missing_anchors(html: str, versions: list[str]) -> list[str]:
return [v for v in versions if f'id="{v.replace(".", "-")}"' not in html]
def collect(fetcher: Fetcher, preamble: Preamble, *, base: str = BASE, index_url: str = INDEX_URL,
changelog: str = CHANGELOG, changelog_page: str = CHANGELOG_PAGE, pause: float = 0.5,
collection: str = "claude-code") -> Collected:
"""Fetch and split everything in memory; nothing is written here."""
got = Collected()
problems = got.problems
def fetch(url: str) -> Fetched:
f = fetcher(url)
if pause:
time.sleep(pause)
return f
raw_index = fetch(index_url)
if raw_index.status != 200:
raise RunError(f"the index {index_url} gave status {raw_index.status}")
index_text = raw_index.body.decode("utf-8").replace("\r\n", "\n")
idx = parse_index(index_text, base)
urls = [p.url for p in idx.pages]
if changelog not in urls:
problems.append(f"the changelog {changelog} is not in the index")
sections = {p.url: p.sections for p in idx.pages}
names: dict[str, str] = {}
by_name: dict[str, str] = {}
for url in urls:
try:
name = source_name(url, base)
except FetchError as exc:
problems.append(str(exc))
continue
if name in by_name:
problems.append(f"{name}: same path for {by_name[name]} and {url}")
continue
by_name[name] = url
names[url] = name
sources: dict[str, dict[str, Any]] = {}
moved_out: list[dict[str, str]] = []
crlf: list[str] = []
preamble_pages = preamble_bytes = raw_updates = 0
changelog_bytes: dict[str, int] = {}
docs_sections: dict[str, list[str]] = {}
versions: list[str] = []
history_pages: list[str] = []
for url in urls:
if url not in names:
continue
name = names[url]
f = fetch(url)
if f.status != 200:
problems.append(f"{url}: status {f.status}")
continue
if not f.url.startswith(base):
moved_out.append({"content_type": f.content_type, "final_url": f.url, "url": url})
continue
if not f.content_type.startswith("text/markdown"):
problems.append(f"{url} ({name}): {f.content_type or 'no content type'}, not markdown")
continue
try:
text = f.body.decode("utf-8")
except UnicodeDecodeError:
problems.append(f"{url} ({name}): not UTF-8")
continue
if "\r\n" in text:
crlf.append(name)
text = text.replace("\r\n", "\n")
try:
if url == changelog:
raw_updates = release_count(text)
split = split_changelog(text, preamble)
changelog_bytes = {"page": len(text.encode("utf-8")), "preamble": split.preamble_bytes,
"blocks": split.block_bytes, "rest": split.rest_bytes}
preamble_pages += 1
preamble_bytes += split.preamble_bytes
for b in split.blocks:
rn = f"release-v{b.version}.md"
got.files[rn] = b.text.encode("utf-8")
sources[rn] = {"kind": "release-notes", "main_topic": False,
"url": anchor_url(changelog_page, b.version), "version": b.version}
versions.append(b.version)
else:
body, removed = strip_preamble(text, preamble)
preamble_pages += 1
preamble_bytes += removed
got.files[name] = body.encode("utf-8")
history = is_history(url[len(base):])
sources[name] = {"kind": "docs", "main_topic": not history, "url": url}
if history:
history_pages.append(name)
docs_sections[name] = list(sections[url])
except FetchError as exc:
problems.append(f"{url} ({name}): {exc}")
if versions:
page = fetch(changelog_page)
if page.status != 200:
problems.append(f"{changelog_page}: status {page.status}; the version anchors cannot be checked")
else:
for v in missing_anchors(page.body.decode("utf-8", errors="replace"), versions):
problems.append(f"{v}: no anchor id=\"{v.replace('.', '-')}\" on {changelog_page}")
docs = sorted(n for n, s in sources.items() if s["kind"] == "docs")
notes = sorted(n for n, s in sources.items() if s["kind"] == "release-notes")
has_changelog = int(changelog in urls)
complete = {"index_pages": len(urls), "changelog": has_changelog, "moved_out": len(moved_out),
"docs_expected": len(urls) - has_changelog - len(moved_out), "docs": len(docs),
"release_notes_expected": raw_updates, "release_notes": len(notes)}
complete["pass"] = (complete["docs"] == complete["docs_expected"]
and complete["release_notes"] == complete["release_notes_expected"] and raw_updates > 0)
if not complete["pass"]:
problems.append(f"completeness FAIL: docs {complete['docs']} of {complete['docs_expected']} expected, "
f"release notes {complete['release_notes']} of {complete['release_notes_expected']}")
got.meta = {"collection": collection, "schema": META_SCHEMA,
"index": {"pages": len(urls), "sha256": hashlib.sha256(raw_index.body).hexdigest(), "url": index_url},
"sources": dict(sorted(sources.items()))}
got.report = {
"index": {"url": index_url, "sha256": got.meta["index"]["sha256"], "bytes": len(raw_index.body),
"items": idx.items, "pages": len(urls)},
"other_links": idx.other_links, "moved_out": moved_out, "completeness": complete, "crlf_pages": crlf,
"history_pages": sorted(history_pages),
"preamble": {"pages": preamble_pages, "bytes": preamble_bytes}, "changelog_bytes": changelog_bytes,
"docs": {"count": len(docs), "bytes": sum(len(got.files[n]) for n in docs)},
"release_notes": {"count": len(notes), "bytes": sum(len(got.files[n]) for n in notes)},
"sections": docs_sections, "problems": problems}
return got
def net_fetcher(cafile: str | None, timeout: float = 60.0) -> Fetcher:
"""urllib with certificate checks on. The python.org build here has no CA store, so the system's file is used."""
if cafile is None and not os.environ.get("SSL_CERT_FILE") and os.path.isfile(SYSTEM_CA):
cafile = SYSTEM_CA
context = ssl.create_default_context(cafile=cafile)
def fetch(url: str) -> Fetched:
req = urllib.request.Request(url, headers={"User-Agent": USER_AGENT})
try:
with urllib.request.urlopen(req, timeout=timeout, context=context) as r:
return Fetched(r.status, r.geturl(), r.headers.get("Content-Type", ""), r.read())
except urllib.error.HTTPError as exc:
return Fetched(exc.code, exc.geturl() or url, exc.headers.get("Content-Type", "") if exc.headers else "",
b"")
return fetch
def build_parser() -> argparse.ArgumentParser:
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
p.add_argument("--staging", required=True)
p.add_argument("--meta", required=True)
p.add_argument("--report", required=True)
p.add_argument("--preamble", required=True)
p.add_argument("--base", default=BASE)
p.add_argument("--index", default=INDEX_URL)
p.add_argument("--changelog", default=CHANGELOG)
p.add_argument("--changelog-page", default=CHANGELOG_PAGE)
p.add_argument("--pause", type=float, default=0.5)
p.add_argument("--cafile")
p.add_argument("--collection", default="claude-code")
return p
def main(argv: list[str] | None = None, fetcher: Fetcher | None = None) -> int:
args = build_parser().parse_args(argv)
staging, meta, report = Path(args.staging), Path(args.meta), Path(args.report)
try:
for p in (staging, meta, report):
if p.exists():
raise RunError(f"{p} exists; refusing to overwrite")
got = collect(fetcher or net_fetcher(args.cafile), load_preamble(Path(args.preamble)), base=args.base,
index_url=args.index, changelog=args.changelog, changelog_page=args.changelog_page,
pause=args.pause, collection=args.collection)
write_new(report, dump(got.report))
c = got.report["completeness"]
summary = (f"fetch: {c['docs']} docs of {c['docs_expected']} expected, {c['release_notes']} release notes of "
f"{c['release_notes_expected']}, {c['moved_out']} moved out, {len(got.report['other_links'])} "
f"other links not fetched")
if got.problems:
print(f"{summary}; FAIL with {len(got.problems)} problems, nothing staged (report: {report})",
file=sys.stderr)
for problem in got.problems:
print(f" {problem}", file=sys.stderr)
return 1
staging.mkdir(parents=True)
for name, data in got.files.items():
(staging / name).write_bytes(data)
write_new(meta, dump(got.meta))
print(f"{summary}; completeness pass -> {staging}")
for m in got.report["moved_out"]:
print(f" moved out: {m['url']} -> {m['final_url']}")
return 0
except (RunError, FetchError, OSError, UnicodeDecodeError) as exc:
print(f"fetch cannot run: {exc}", file=sys.stderr)
return 2
if __name__ == "__main__":
sys.exit(main())

View file

@ -0,0 +1,124 @@
"""Google's viewer on a bundle (PLAN.md criterion 5: "Googles visning leser hver samling uten feil"; brief SC5). It calls
no model and reads no network.
Usage:
python3 tools/google_viewer_check.py BUNDLE --upstream DIR [--out FILE]
DIR is an upstream checkout of Google's Open Knowledge Format repository (a `git archive` of the pinned commit, never
the mirror itself): the tool runs `generate_visualization` from DIR/src/reference_agent/viewer/generator.py with
`python3 -I -B` (isolated, no bytecode written into DIR) from a fresh temporary folder, on an unchanged copy of
BUNDLE, so the bundle itself is never written. The viewer at ad30107 skips only index.md (`_walk_concepts`), so it draws
each log.md as a node too, although SPEC 3.1 reserves the name; those nodes (id "log" or ".../log") are counted apart.
PASS when the run exits 0, its output holds no "Traceback", it writes the graph page, and the graph in that page
(the line "window.BUNDLE = <json>;") has as many concept nodes as the auditor counts concept pages, at least one, and
one log node per log.md. --out keeps the graph page (for `okfwiki.view render --graph`; it shows the log nodes). Exit 0 = PASS, 1 = FAIL, 2 = could not run (no bundle directory,
or no generator under DIR).
"""
from __future__ import annotations
import argparse
import json
import shutil
import subprocess
import sys
import tempfile
from pathlib import Path
from typing import Any
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from okfwiki.audit import load_bundle # noqa: E402
GENERATOR = Path("src") / "reference_agent" / "viewer" / "generator.py"
GRAPH_LINE = "window.BUNDLE = "
TIMEOUT = 900
RUNNER = """
import json, sys
from pathlib import Path
sys.path.insert(0, sys.argv[1])
from reference_agent.viewer.generator import generate_visualization
print(json.dumps(generate_visualization(Path(sys.argv[2]), Path(sys.argv[3]), bundle_name=sys.argv[4])))
"""
class RunError(Exception):
"""The check cannot run; exit 2."""
def graph_nodes(page: Path) -> tuple[list[str], int] | None:
"""(the node ids, the number of edges) of the graph embedded in the viewer's page, or None when it cannot be read."""
try:
lines = page.read_text(encoding="utf-8").splitlines()
except (OSError, UnicodeDecodeError):
return None
line = next((x for x in lines if x.startswith(GRAPH_LINE) and x.endswith(";")), None)
if line is None:
return None
try:
graph: Any = json.loads(line[len(GRAPH_LINE):-1])
except ValueError:
return None
if not isinstance(graph, dict) or not isinstance(graph.get("nodes"), list) or not isinstance(graph.get("edges"), list):
return None
data = [n.get("data") if isinstance(n, dict) else None for n in graph["nodes"]]
return [str(d.get("id")) if isinstance(d, dict) else "" for d in data], len(graph["edges"])
def check(bundle: Path, upstream: Path, out: Path | None) -> tuple[bool, list[str]]:
"""(passed, the lines to print)."""
if not bundle.is_dir():
raise RunError(f"the bundle {bundle} is not a directory")
if not (upstream / GENERATOR).is_file():
raise RunError(f"no viewer generator at {upstream / GENERATOR}")
pages = len(load_bundle(bundle, bundle).concepts)
logs = sum(1 for p in bundle.rglob("log.md") if p.is_file())
with tempfile.TemporaryDirectory(prefix="google-viewer-") as tmp:
work = Path(tmp)
copy, run_in, graph = work / "bundle" / bundle.resolve().name, work / "cwd", work / "graph.html"
shutil.copytree(bundle, copy)
run_in.mkdir()
cmd = [sys.executable, "-I", "-B", "-c", RUNNER, str((upstream / "src").resolve()), str(copy), str(graph),
bundle.resolve().name]
try:
proc = subprocess.run(cmd, cwd=run_in, capture_output=True, text=True, timeout=TIMEOUT, check=False)
code, output = proc.returncode, proc.stdout + proc.stderr
except subprocess.TimeoutExpired:
code, output = -1, f"the viewer did not finish in {TIMEOUT} seconds"
found = graph_nodes(graph)
if out is not None and graph.is_file():
out.parent.mkdir(parents=True, exist_ok=True)
shutil.copyfile(graph, out)
ids, edges = found if found is not None else ([], -1)
log_nodes = sum(1 for i in ids if i.rsplit("/", 1)[-1] == "log")
nodes = len(ids) - log_nodes
traceback = "Traceback" in output
passed = code == 0 and not traceback and found is not None and pages > 0 and nodes == pages and log_nodes == logs
lines = [f"google viewer: exit {code} · traceback {'yes' if traceback else 'no'} · "
f"concept nodes {nodes if found is not None else 'unreadable'} · concept pages {pages} · "
f"log nodes {log_nodes} of {logs} · edges {edges}"]
if code != 0 or traceback:
lines += [" " + x for x in output.strip().splitlines()[-20:]]
lines.append(f"{'PASS' if passed else 'FAIL'} Google's viewer reads the bundle: one node per concept page and per log.md")
return passed, lines
def main(argv: list[str] | None = None) -> int:
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
p.add_argument("bundle")
p.add_argument("--upstream", required=True, help="an upstream checkout (git archive of the pinned commit)")
p.add_argument("--out", help="keep the viewer's graph page here")
args = p.parse_args(argv)
try:
passed, lines = check(Path(args.bundle), Path(args.upstream), Path(args.out) if args.out else None)
except RunError as exc:
print(f"google viewer cannot run: {exc}", file=sys.stderr)
return 2
for line in lines:
print(line)
return 0 if passed else 1
if __name__ == "__main__":
sys.exit(main())

13
tools/install-hooks.sh Executable file
View file

@ -0,0 +1,13 @@
#!/bin/bash
# install-hooks.sh - installs the repository-local git hooks that run the
# excluded-terms check before every commit (staged files) and on every commit
# message. The machine's global hook dispatcher runs these after its own rules.
# Run once per clone. bash 3.2 safe, ASCII only.
set -eu
ROOT=$(git rev-parse --show-toplevel)
HOOKS="$(git rev-parse --git-common-dir)/hooks"
mkdir -p "$HOOKS"
printf '#!/bin/bash\nexec "%s/tools/check-excluded-terms.sh" --staged\n' "$ROOT" > "$HOOKS/pre-commit"
printf '#!/bin/bash\nexec "%s/tools/check-excluded-terms.sh" --message "$1"\n' "$ROOT" > "$HOOKS/commit-msg"
chmod +x "$HOOKS/pre-commit" "$HOOKS/commit-msg"
echo "installed: $HOOKS/pre-commit, $HOOKS/commit-msg"

314
tools/part_metrics.py Normal file
View file

@ -0,0 +1,314 @@
"""The numbers per pack and per source of a build folder (contract K7), read without a model (CLAUDE.md rule 1).
Usage:
python3 tools/part_metrics.py ROOT --out FILE
ROOT is a build folder: work/ (the skill's WORK), wiki/ (WIKI), logg/pN.txt (what the build commands of pack N wrote,
with "exit N" or "exit N: <module> <command>" lines) and kost.json (okfwiki.cost harvest). It writes one JSON object (indent 1, sorted keys), the same
bytes on every run over the same folder. Plan revision 5 Step 11 asks for these numbers per pack, with a growth fit,
so the price of the rest and the growth of the collection can be read from the first part (PLAN.md § Retning point 4).
Per accepted pack (work/K-work/packs.json):
- sources, blocks in the pack and so far;
- cost per line: build and link as attributed by description, orchestration spread over the packs in proportion to
their attributed cost (as okfwiki.cost stop spreads it), the pack's total, the total so far and per block; the
number of build and link runs, the compiler's new runs after a rejection at writing (exit 3) and the guard rounds,
because they explain the cost; the exit codes in logg/pN.txt in order;
- the concepts in the index after the pack (work/K-work/index.json: the skeleton's entries and those whose origin is
a source of the packs so far), the pages (concepts with an accepted fragment) and the main pages;
- the catalogue of main pages the pack's compiler got, in lines and characters, read from its task parts in
work/K-oppgaver/ as kpack wrote them, beside the limit klink.CATALOGUE_CHARS;
- like concepts: groups of pages that share a normalised title or alias (kpack.norm, its concept identity);
- pages without a home and docs sources without a main page before and after the guard, from the auditor's JSON
(no-main-page-link and main-page-missing) in work/K-audit/pN-audit-1.json and in the last pN-audit-k.json; the
auditor is the only judge (K2 point 17), nothing here counts it again;
- main-page out-degree: per main page the concept pages it links to, in a sentence or in its list of related
concepts, each page once; min, median and max. The wiki is assembled again after every pack and keeps no earlier
state, so the out-degree after pack N is counted from the records that place the links: a sentence link counts from
the pack whose link pass asked for its sentence (work/K-links/**/sentences.json), a home from the pack its record
names. Only records that still stand count. Beside it, out_degree_wiki counts the same on wiki/ with the auditor's
own reading of links from main pages (audit.main_page_links); after the last pack the two agree when every link is
attributed (links_unattributed counts those that are not).
Per source: the compiler's cost (the build line) and the pack's total, split over the pack's sources by their share of
its blocks (okfwiki.cost.allocate), the price of a later repair of one source.
fit: cost per block against blocks read before the pack (okfwiki.cost.project, as the stop line projects it, to the
blocks in work/blocks.json), and the catalogue's characters against blocks read before the pack (okfwiki.cost.fit_line),
with the blocks read at which the fitted line reaches the limit.
Exit 0 written, 2 cannot run (an input file missing or not readable; nothing is written).
"""
from __future__ import annotations
import argparse
import json
import os
import re
import statistics
import sys
from fractions import Fraction
from pathlib import Path
from typing import Any
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from okfwiki import audit, cost, klink, kpack # noqa: E402
from okfwiki.common import RunError # noqa: E402
SCHEMA = "okf-wiki/part-metrics/1"
LINES = kpack.STOP_LINES.split(",") # build, link, orchestration-build: the lines the build's stop line caps
CATALOGUE_HEADER = "## Main pages (homes you may choose)\n\n" # as kpack.render_input writes it
PART_HEAD = re.compile(r"^<!-- part \d+ of \d+ -->\n")
PART_FILE = re.compile(r"^pack-(\d+)-part-(\d+)\.md$")
EXIT_LINE = re.compile(r"^exit (\d+)(?:: .*)?$") # bare, or as the skill's chain prints it
def read_json(path: Path) -> Any:
try:
return json.loads(path.read_text(encoding="utf-8"))
except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc:
raise RunError(f"cannot read {path}: {exc}") from exc
def catalogue_of(parts: list[str]) -> tuple[int, int] | None:
"""(lines, characters) of the catalogue of main pages in a compiler task, its parts in order; each line counted
with its newline, as kpack.main_catalogue counts against its budget. None when the task has no catalogue section."""
text = "".join(PART_HEAD.sub("", p, count=1) for p in parts)
at = text.rfind(CATALOGUE_HEADER)
if at < 0:
return None
lines = [line for line in text[at + len(CATALOGUE_HEADER):].split("\n") if line.startswith("- ")]
return len(lines), sum(len(line) + 1 for line in lines)
def task_parts(work: Path, n: int) -> list[str]:
found = []
for p in (work / "K-oppgaver").glob(f"pack-{n}-part-*.md"):
m = PART_FILE.match(p.name)
if m and int(m.group(1)) == n:
found.append((int(m.group(2)), p.read_text(encoding="utf-8")))
return [text for _, text in sorted(found)]
def pack_costs(kost: dict[str, Any], numbers: list[int]) -> tuple[dict[int, dict[str, Any]], dict[str, int]]:
"""Per pack: build and link as attributed, and the pack's total with orchestration spread over the packs in
proportion to their attributed cost, as okfwiki.cost stop does (exact fractions); and the line totals."""
totals = {name: int(kost["lines"][name]["total"]) for name in LINES}
measured = sum(totals.values())
per: dict[int, dict[str, Any]] = {n: {"build": 0, "link": 0, "build_runs": 0, "link_runs": 0} for n in numbers}
for r in kost.get("runs", []):
line, n = r.get("line"), r.get("pack")
if line in ("build", "link") and n is not None and int(n) in per:
per[int(n)][line] += int(r["total"])
per[int(n)][f"{line}_runs"] += 1
attributed = sum(p["build"] + p["link"] for p in per.values())
scale = Fraction(measured, attributed) if attributed else None
for p in per.values():
p["total"] = (p["build"] + p["link"]) * scale if scale is not None else None
return per, {**totals, "measured": measured}
def concept_names(entry: dict[str, Any]) -> set[str]:
return {kpack.norm(str(a)) for a in [entry.get("title", ""), *entry.get("aliases", [])]} - {""}
def duplicate_groups(index: dict[str, Any], pages: set[str]) -> list[list[str]]:
"""Groups of two or more pages that share a normalised title or alias, each group sorted, the groups sorted."""
names = {str(e["id"]): concept_names(e) for e in index["entries"] if str(e["id"]) in pages}
parent = {cid: cid for cid in names}
def find(c: str) -> str:
while parent[c] != c:
c = parent[c]
return c
first: dict[str, str] = {}
for cid in sorted(names):
for name in sorted(names[cid]):
if name in first:
a, b = find(first[name]), find(cid)
if a != b:
parent[max(a, b)] = min(a, b)
else:
first[name] = cid
groups: dict[str, list[str]] = {}
for cid in sorted(names):
groups.setdefault(find(cid), []).append(cid)
return sorted(g for g in groups.values() if len(g) > 1)
def homeless(doc: dict[str, Any]) -> dict[str, list[str]]:
findings = [f for f in doc.get("findings", []) if isinstance(f, dict)]
return {"pages": sorted(str(f.get("file")) for f in findings if f.get("check") == "no-main-page-link"),
"sources": sorted(str(f.get("file")) for f in findings if f.get("check") == "main-page-missing")}
def audits(work: Path, n: int) -> dict[str, Any] | None:
found = []
for p in (work / "K-audit").glob(f"p{n}-audit-*.json"):
m = re.fullmatch(rf"p{n}-audit-(\d+)\.json", p.name)
if m:
found.append((int(m.group(1)), p))
if not found:
return None
found.sort()
return {"audits": len(found), "before": homeless(read_json(found[0][1])), "after": homeless(read_json(found[-1][1]))}
def exits(root: Path, n: int) -> list[int] | None:
path = root / "logg" / f"p{n}.txt"
if not path.is_file():
return None
return [int(m.group(1)) for line in path.read_text(encoding="utf-8").splitlines()
if (m := EXIT_LINE.match(line.strip()))]
def link_pairs(work: Path, acc: dict[str, Any]) -> tuple[dict[tuple[str, str], int], int]:
"""(main page id, linked page id) -> the first pack that placed the link, from the link records and the homes;
and the number of link records no link pass asked for."""
asked: dict[tuple[str, str], int] = {}
for path in sorted((work / "K-links").rglob("sentences.json")) if (work / "K-links").is_dir() else []:
doc = read_json(path)
for rel, keys in doc.get("pages", {}).items():
for key in keys:
at = (str(rel), str(key))
asked[at] = min(asked.get(at, int(doc["pack"])), int(doc["pack"]))
pairs: dict[tuple[str, str], int] = {}
unattributed = 0
def add(a: str, b: str, n: int) -> None:
pairs[(a, b)] = min(pairs.get((a, b), n), n)
links_path = work / "links.json"
records = read_json(links_path).get("links", []) if links_path.is_file() else []
for r in records:
n = asked.get((str(r["page"]), str(r["sentence"])))
if n is None:
unattributed += 1
continue
add(klink.concept_id(str(r["page"])), klink.concept_id(str(r["target"])), n)
for h in acc.get("homes", []):
add(str(h["home"]), str(h["concept_id"]), int(h["pack"]))
return pairs, unattributed
def degree_stats(per_main: dict[str, int]) -> dict[str, Any]:
values = list(per_main.values())
return {"main_pages": len(values), "min": min(values, default=None), "max": max(values, default=None),
"median": statistics.median(values) if values else None, "per_main": dict(sorted(per_main.items()))}
def wiki_out_degree(wiki: Path) -> dict[str, Any] | None:
if not wiki.is_dir():
return None
b = audit.load_bundle(wiki, wiki)
mains = set()
for d in b.concepts:
value = d.frontmatter.get(audit.MAIN_FOR) if d.frontmatter else None
if isinstance(value, list) and value:
mains.add(d.rel)
linked = audit.main_page_links(b, mains)
return degree_stats({klink.concept_id(m): sum(m in held for held in linked.values()) for m in mains})
def metrics(root: Path) -> dict[str, Any]:
work = root / "work"
blocks_doc = read_json(work / "blocks.json")
log = read_json(work / "K-work" / "packs.json")
index = read_json(work / "K-work" / "index.json")
acc = read_json(work / "K-work" / "accepted.json")
kost = read_json(root / "kost.json")
records = sorted(log["packs"], key=lambda r: int(r["pack"]))
done = [r for r in records if "accepted" in r]
numbers = [int(r["pack"]) for r in done]
costs, line_totals = pack_costs(kost, numbers)
pairs, unattributed = link_pairs(work, acc)
total_blocks = len(blocks_doc["blocks"])
packs: list[dict[str, Any]] = []
files: set[str] = set()
read = 0
so_far = Fraction(0)
points: list[tuple[float, int, int]] = []
catalogue_points: list[tuple[float, float]] = []
for r in done:
n, blocks = int(r["pack"]), int(r["blocks"])
files |= {str(s) for s in r.get("sources", [])}
c = costs[n]
row: dict[str, Any] = {"pack": n, "sources": list(r.get("sources", [])), "blocks": blocks,
"blocks_so_far": read + blocks, "runs": {"build": c["build_runs"],
"link": c["link_runs"]},
"retries": len(r.get("attempts", [])), "guard_rounds": int("guard" in r),
"exits": exits(root, n)}
if c["total"] is None:
row["cost"] = None
else:
so_far += c["total"]
row["cost"] = {"build": c["build"], "link": c["link"],
"orchestration": float(c["total"] - c["build"] - c["link"]), "total": float(c["total"]),
"so_far": float(so_far), "per_block": float(c["total"] / blocks) if blocks else None}
points.append((float(c["total"]), blocks, read))
pages = {str(f["concept_id"]) for f in acc["fragments"] if int(f["pack"]) <= n}
mains = [m for m in kpack.main_ids({**acc, "main_pages": [m for m in acc.get("main_pages", [])
if int(m["pack"]) <= n]}) if m in pages]
row["index_concepts"] = sum(1 for e in index["entries"]
if e.get("kind") == "skeleton" or str(e.get("origin", {}).get("file")) in files)
row["pages"], row["main_pages"] = len(pages), len(mains)
cat = catalogue_of(task_parts(work, n))
row["catalogue"] = None if cat is None else {"lines": cat[0], "chars": cat[1], "limit": klink.CATALOGUE_CHARS}
if cat is not None:
catalogue_points.append((float(read), float(cat[1])))
groups = duplicate_groups(index, pages)
row["duplicates"] = {"groups": groups, "count": len(groups), "pages": sum(len(g) for g in groups)}
row["homeless"] = audits(work, n)
row["out_degree"] = degree_stats({m: sum(1 for (a, b), p in pairs.items()
if a == m and p <= n and b in pages and b != m) for m in mains})
packs.append(row)
read += blocks
per_source: dict[str, dict[str, Any]] = {}
lists = {int(r["pack"]): [str(s) for s in r.get("sources", [])] for r in records}
ordered = [lists.get(i, []) for i in range(1, max(lists, default=0) + 1)]
builds = cost.allocate({n: float(costs[n]["build"]) for n in numbers}, ordered, blocks_doc)
totals = cost.allocate({n: float(costs[n]["total"]) for n in numbers if costs[n]["total"] is not None}, ordered,
blocks_doc)
count: dict[str, int] = {}
for b in blocks_doc["blocks"]:
count[str(b["file"])] = count.get(str(b["file"]), 0) + 1
for n in numbers:
for s in lists[n]:
per_source[s] = {"pack": n, "blocks": count.get(s, 0), "build": builds.get(s), "total": totals.get(s)}
fit: dict[str, Any] = {"cost_per_block": cost.project(points, total_blocks), "catalogue_chars": None}
if catalogue_points:
a, b = cost.fit_line(catalogue_points)
fit["catalogue_chars"] = {"a": a, "b": b, "points": len(catalogue_points), "limit": klink.CATALOGUE_CHARS,
"reaches_limit_at_blocks_read": (klink.CATALOGUE_CHARS - a) / b if b > 0 else None}
return {"schema": SCHEMA, "packs": packs, "sources": per_source, "fit": fit,
"pending": [int(r["pack"]) for r in records if "accepted" not in r],
"totals": {"blocks": total_blocks, "blocks_done": read, "cost": line_totals},
"out_degree_wiki": wiki_out_degree(root / "wiki"), "links_unattributed": unattributed}
def main(argv: list[str] | None = None) -> int:
p = argparse.ArgumentParser(prog="python3 tools/part_metrics.py", description=__doc__,
formatter_class=argparse.RawDescriptionHelpFormatter)
p.add_argument("root")
p.add_argument("--out", required=True)
args = p.parse_args(argv)
root, out = Path(args.root), Path(args.out)
try:
doc = metrics(root)
except (RunError, OSError, KeyError, TypeError, ValueError) as exc:
print(f"part_metrics cannot run: {exc}", file=sys.stderr)
return 2
tmp = out.with_name(f".{out.name}.tmp")
tmp.write_text(json.dumps(doc, indent=1, sort_keys=True, ensure_ascii=False) + "\n", encoding="utf-8")
os.replace(tmp, out)
print(f"part_metrics: {len(doc['packs'])} packs, {doc['totals']['blocks_done']} of {doc['totals']['blocks']} "
f"blocks, measured {doc['totals']['cost']['measured']} tokens -> {out}")
return 0
if __name__ == "__main__":
sys.exit(main())

1153
tools/upstream_watch.py Normal file

File diff suppressed because it is too large Load diff

173
tools/x_fields_check.py Normal file
View file

@ -0,0 +1,173 @@
"""The x- field check (format choice g, PLAN.md criterion 5): every x- field on a page has a row with a measured
reason in docs/x-fields.md. It calls no model and reads no network.
Usage:
python3 tools/x_fields_check.py BUNDLE [--table docs/x-fields.md]
The table is the first markdown table in TABLE whose header begins "| Field |", with eight columns: Field, Where
(`page` or `sources[]`), Status (`v1` or `not in v1`), Written by, Read by, Meaning, Measured reason, and Standard
field if Google adopts one. A row fails when it has another number of cells, when its field is not an x- name, when
its Where or Status is not one of those values, when a field has two rows, and when its reason is empty or holds no
number besides references (section, issue, point, criterion, step, wave and order numbers, names such as K2 or SC5,
dates, commit ids): a measured reason states the measurement and its number.
Then every concept page of BUNDLE (every .md except index.md and log.md, read as the auditor reads them): each x- key
at the top of the frontmatter and each x- key on a sources[] entry must have a row with that place and status v1. A
field whose row says "not in v1" fails on a page (x-outdated, CLAUDE.md). A page whose frontmatter does not parse
cannot be checked and fails; so does a bundle with no concept page.
Prints the counts, one line per problem (a field missing from many pages is one line) and a last line PASS or FAIL.
Exit 0 = PASS, 1 = FAIL, 2 = could not run (no bundle directory, no table file, or no table in it).
"""
from __future__ import annotations
import argparse
import re
import sys
from collections import Counter
from dataclasses import dataclass
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from okfwiki.audit import load_bundle # noqa: E402
DEFAULT_TABLE = Path(__file__).resolve().parents[1] / "docs" / "x-fields.md"
COLUMNS = 8
PLACES = ("page", "sources[]")
STATUSES = ("v1", "not in v1")
NAME = re.compile(r"x-[a-z][a-z0-9-]*")
# Numbers that refer to something rather than measure it: section, issue and point numbers, names such as K2, SC5
# or L4, dates and commit ids.
REFERENCE = re.compile(r"§\s*\d+(?:\.\d+)*|#\d+|\b[A-Z]{1,3}\d+[a-z]?\b|\b(?:points?|criteri(?:on|a)|steps?|orders?|waves?|rule)\s+\d+"
r"(?:\s*(?:,|and|-)\s*\d+)*|\b\d{1,2}\.\d{1,2}\.\d{4}\b|\b\d{4}-\d{2}-\d{2}\b"
r"|\b(?=[0-9a-f]*[a-f])(?=[0-9a-f]*\d)[0-9a-f]{7,40}\b", re.IGNORECASE)
class RunError(Exception):
"""The check cannot run; exit 2."""
@dataclass(frozen=True)
class Row:
field: str
where: str
status: str
written: str
read: str
meaning: str
reason: str
standard: str
def cells(line: str) -> list[str]:
return [c.strip() for c in line.strip().removeprefix("|").removesuffix("|").split("|")]
def measured(reason: str) -> bool:
"""A reason is measured when it states a number that is not a reference (a section, issue or point number, a
date or a commit id)."""
return re.search(r"\d", REFERENCE.sub(" ", reason)) is not None
def read_table(path: Path) -> tuple[dict[str, Row], list[str]]:
"""(the rows by field, one problem per bad row). RunError when the file or its table is missing."""
try:
lines = path.read_text(encoding="utf-8").splitlines()
except (OSError, UnicodeDecodeError) as exc:
raise RunError(f"cannot read the table file {path}: {exc}") from exc
start = next((i for i, line in enumerate(lines) if line.strip().startswith("| Field |")), None)
if start is None:
raise RunError(f"{path} has no table whose header begins | Field |")
rows: dict[str, Row] = {}
problems: list[str] = []
for line in lines[start + 2:]:
if not line.strip().startswith("|"):
break
c = cells(line)
if len(c) != COLUMNS:
problems.append(f"table: a row has {len(c)} cells, not {COLUMNS}: {line.strip()[:80]}")
continue
row = Row(c[0].strip("`"), *c[1:])
if not NAME.fullmatch(row.field):
problems.append(f"table: {c[0]!r} is not an x- field name")
continue
if row.field in rows:
problems.append(f"table: {row.field} has two rows")
continue
if row.where not in PLACES or row.status not in STATUSES:
problems.append(f"table: {row.field}: Where must be one of {PLACES} and Status one of {STATUSES}")
if not measured(row.reason):
problems.append(f"table: {row.field} has no measured reason (the reason is empty or states no number)")
rows[row.field] = row # a bad row still names its field, so its pages are not reported a second time
return rows, problems
def page_fields(bundle: Path) -> tuple[int, Counter[tuple[str, str]], dict[tuple[str, str], list[str]], list[str]]:
"""(concept pages, count per (field, place), the pages per (field, place), pages that cannot be read)."""
counts: Counter[tuple[str, str]] = Counter()
pages: dict[tuple[str, str], list[str]] = {}
unreadable: list[str] = []
concepts = load_bundle(bundle, bundle).concepts
for d in concepts:
if d.frontmatter is None:
unreadable.append(d.rel)
continue
found = [(str(k), "page") for k in d.frontmatter if str(k).startswith("x-")]
srcs = d.frontmatter.get("sources")
for s in srcs if isinstance(srcs, list) else [srcs]:
if isinstance(s, dict):
found += [(str(k), "sources[]") for k in s if str(k).startswith("x-")]
for key in found:
counts[key] += 1
if d.rel not in pages.setdefault(key, []):
pages[key].append(d.rel)
return len(concepts), counts, pages, unreadable
def check(bundle: Path, table: Path) -> tuple[list[str], list[str]]:
"""(the count lines, the problems)."""
if not bundle.is_dir():
raise RunError(f"the bundle {bundle} is not a directory")
rows, problems = read_table(table)
n, counts, pages, unreadable = page_fields(bundle)
info = [f"table: {len(rows)} rows in {table}",
f"pages: {n} concept pages; x- fields: "
+ (", ".join(f"{f} ({p}) {k}" for (f, p), k in sorted(counts.items())) or "none")]
if n == 0:
problems.append("pages: the bundle has no concept page, so nothing was checked")
problems += [f"{rel}: the frontmatter does not parse, so its x- fields cannot be checked" for rel in unreadable]
for (name, place), on in sorted(pages.items()):
row = rows.get(name)
where = f"{name} ({place}) on {len(on)} page(s), first {on[0]}"
if row is None:
problems.append(f"{where}: no row in the table")
elif row.where != place:
problems.append(f"{where}: the table says it stands at {row.where}")
elif row.status != "v1":
problems.append(f"{where}: the table says {row.status}")
return info, problems
def main(argv: list[str] | None = None) -> int:
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
p.add_argument("bundle")
p.add_argument("--table", default=str(DEFAULT_TABLE))
args = p.parse_args(argv)
try:
info, problems = check(Path(args.bundle), Path(args.table))
except RunError as exc:
print(f"x-fields cannot run: {exc}", file=sys.stderr)
return 2
for line in info + problems:
print(line)
if problems:
print(f"FAIL x- fields: {len(problems)} problem(s)")
return 1
print("PASS x- fields: every x- field on every concept page has a row with a measured reason")
return 0
if __name__ == "__main__":
sys.exit(main())