okf-wiki 1.1.1
First public release, published as one snapshot. What changed between versions is in CHANGELOG.md. Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
This commit is contained in:
commit
0fd86a23a3
227 changed files with 96077 additions and 0 deletions
94
tools/check-excluded-terms.sh
Executable file
94
tools/check-excluded-terms.sh
Executable file
|
|
@ -0,0 +1,94 @@
|
|||
#!/bin/bash
|
||||
# check-excluded-terms.sh - blocks any tracked text, path or commit message in
|
||||
# this repository that matches the operator's local excluded-terms list.
|
||||
#
|
||||
# The list is READ WHERE IT LIVES and never copied into this repository:
|
||||
# ${OKF_WIKI_EXCLUDED_TERMS:-$HOME/.claude/runtime/excluded-terms.local.txt}
|
||||
# One extended regex per line, case-insensitive; '#' lines and blank lines are
|
||||
# skipped. A missing or empty list is a FAIL, never a silent pass: a check that
|
||||
# cannot run has measured nothing.
|
||||
#
|
||||
# Known-positive: the combined pattern must match at least one line of the list
|
||||
# itself, or the query is dead and a zero would be meaningless.
|
||||
#
|
||||
# Usage:
|
||||
# tools/check-excluded-terms.sh scan every tracked file (path + content)
|
||||
# tools/check-excluded-terms.sh --staged scan staged files (pre-commit)
|
||||
# tools/check-excluded-terms.sh --message F scan a commit message file (commit-msg)
|
||||
# add -v as the LAST argument to list the matching paths (never the terms)
|
||||
#
|
||||
# Exit 0 = no match; 1 = match or the check could not run.
|
||||
# bash 3.2 safe, ASCII only.
|
||||
set -u
|
||||
|
||||
MODE="tree"
|
||||
MSG=""
|
||||
VERBOSE=0
|
||||
case "${1:-}" in
|
||||
--staged) MODE="staged" ;;
|
||||
--message) MODE="message"; MSG="${2:-}" ;;
|
||||
-v) VERBOSE=1 ;;
|
||||
esac
|
||||
for a in "$@"; do [ "$a" = "-v" ] && VERBOSE=1; done
|
||||
|
||||
LIST="${OKF_WIKI_EXCLUDED_TERMS:-$HOME/.claude/runtime/excluded-terms.local.txt}"
|
||||
if [ ! -f "$LIST" ]; then
|
||||
echo "FAIL excluded-terms: no term list at $LIST - nothing measured (set OKF_WIKI_EXCLUDED_TERMS)" >&2
|
||||
exit 1
|
||||
fi
|
||||
PAT=$(grep -v '^#' "$LIST" | grep -v '^[[:space:]]*$' | paste -sd'|' -)
|
||||
if [ -z "$PAT" ]; then
|
||||
echo "FAIL excluded-terms: term list is empty - nothing measured" >&2
|
||||
exit 1
|
||||
fi
|
||||
KP=$(grep -ciE "$PAT" "$LIST")
|
||||
if [ "$?" -eq 2 ] || [ "$KP" -eq 0 ]; then
|
||||
echo "FAIL excluded-terms: pattern does not compile or matches nothing in the list itself - query is dead" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
ROOT=$(git rev-parse --show-toplevel 2>/dev/null) || { echo "FAIL excluded-terms: not inside a git repository" >&2; exit 1; }
|
||||
TMP=$(mktemp -d "${TMPDIR:-/tmp}/okf-wiki-terms.XXXXXX") || exit 1
|
||||
trap 'rm -rf "$TMP"' EXIT
|
||||
: > "$TMP/hits"
|
||||
|
||||
if [ "$MODE" = "message" ]; then
|
||||
if [ -z "$MSG" ] || [ ! -f "$MSG" ]; then
|
||||
echo "FAIL excluded-terms: no commit message file given" >&2
|
||||
exit 1
|
||||
fi
|
||||
if grep -v '^#' "$MSG" | grep -qiE "$PAT"; then
|
||||
echo "FAIL excluded-terms: the commit message matches the excluded-terms list" >&2
|
||||
exit 1
|
||||
fi
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [ "$MODE" = "staged" ]; then
|
||||
git -C "$ROOT" diff --cached --name-only -z --diff-filter=ACMR > "$TMP/files"
|
||||
else
|
||||
git -C "$ROOT" ls-files -z > "$TMP/files"
|
||||
fi
|
||||
TOTAL=$(tr -cd '\0' < "$TMP/files" | wc -c | tr -d ' ')
|
||||
|
||||
while IFS= read -r -d '' f; do
|
||||
if printf '%s\n' "$f" | grep -qiE "$PAT"; then
|
||||
echo "$f" >> "$TMP/hits"
|
||||
continue
|
||||
fi
|
||||
if [ "$MODE" = "staged" ]; then
|
||||
git -C "$ROOT" show ":$f" 2>/dev/null | grep -IqiE "$PAT" && echo "$f" >> "$TMP/hits"
|
||||
else
|
||||
[ -f "$ROOT/$f" ] || continue
|
||||
grep -IqiE "$PAT" "$ROOT/$f" 2>/dev/null && echo "$f" >> "$TMP/hits"
|
||||
fi
|
||||
done < "$TMP/files"
|
||||
|
||||
HITS=$(wc -l < "$TMP/hits" | tr -d ' ')
|
||||
[ "$VERBOSE" -eq 1 ] && sed 's/^/ hit: /' "$TMP/hits"
|
||||
if [ "$HITS" -eq 0 ]; then
|
||||
echo "PASS excluded-terms ($MODE): 0 of $TOTAL files match (known-positive: $KP list lines)"
|
||||
exit 0
|
||||
fi
|
||||
echo "FAIL excluded-terms ($MODE): $HITS of $TOTAL files match (add -v to list the paths)" >&2
|
||||
exit 1
|
||||
226
tools/claim_features.py
Normal file
226
tools/claim_features.py
Normal file
|
|
@ -0,0 +1,226 @@
|
|||
"""What each claim says beyond the quote it cites, counted by code (wave 4 order 20, contract K9). No model call.
|
||||
|
||||
Usage:
|
||||
python3 tools/claim_features.py --claims claims.json --sources SOURCES --wiki WIKI --out features.json
|
||||
[--verdicts verdicts.json] [--summary]
|
||||
|
||||
It reads the claim list (okfwiki.claims list), the reader's verdicts when given, the sources (SOURCES/kilde/) and the
|
||||
pages (WIKI/<page>, for the page title in the frontmatter), and writes per claim, in the claim list's order:
|
||||
quotes, quote_chars, claim_chars the number of excerpts, the characters of their texts summed, of the claim
|
||||
content_words the claim's content words (the rule below), repeats counted
|
||||
outside, outside_count the claim's content words whose first five letters begin no content word of
|
||||
its excerpts, in claim order, repeats counted
|
||||
outside_title, outside_headings of those, the ones whose first five letters begin a content word of the page
|
||||
title, and of the heading path above each quoted line in its source
|
||||
headings that heading path (the headings above the first excerpt's line, outermost
|
||||
first), taken from okfwiki.kblocks.headings_of, so the tool counts a heading
|
||||
by the same rule as the build, also after order 22 changes that rule
|
||||
files, provenance, verdict the excerpts' source files, the claim's provenance mark, and the reader's
|
||||
verdict (null without --verdicts, or when it gave the claim none)
|
||||
|
||||
The word rule (phase 0 of wave 4, 11.10.2026): lower-case the text, take every run of [a-z0-9], drop the words in
|
||||
STOPWORDS, and compare two words on their first five letters (a shorter word is compared whole). It describes; it
|
||||
judges nothing. A code check by word overlap was ruled out as a gate (PLAN.md § Retning, choice 5: 8 of 664 near
|
||||
verbatim), and nothing here is a verdict.
|
||||
|
||||
--summary prints the phase 0 table: the share rejected per number of words outside the quote (0, 1, 2, 3-4, 5+),
|
||||
per provenance mark and per source file, the medians per verdict, and the share with a word from the page title or
|
||||
from the heading path outside the quote. Without verdicts it prints the counts only. A claim the reader gave no
|
||||
verdict is left out of the share and counted apart as not judged (order 39), so a partial reading does not lower it.
|
||||
|
||||
Output: json.dumps(doc, indent=1, sort_keys=True, ensure_ascii=False) + newline; two runs on the same inputs are
|
||||
byte-identical, and an existing output is never overwritten. Exit 0 written, 2 cannot run (nothing is written).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
import statistics
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
|
||||
from okfwiki import kblocks # noqa: E402
|
||||
from okfwiki.audit import read_document # noqa: E402
|
||||
from okfwiki.common import RunError, write_new # noqa: E402
|
||||
|
||||
SCHEMA = "okf-wiki/claim-features/1"
|
||||
WORD = re.compile(r"[a-z0-9]+")
|
||||
STOPWORDS = frozenset(
|
||||
"a an the of to in on for and or is are be by with from as at it this that its can you your when if not no into "
|
||||
"use uses using".split())
|
||||
STEM = 5
|
||||
BUCKETS = (("0", 0, 0), ("1", 1, 1), ("2", 2, 2), ("3-4", 3, 4), ("5+", 5, None))
|
||||
UNSUPPORTED = "unsupported"
|
||||
|
||||
|
||||
def words(text: str) -> list[str]:
|
||||
"""The content words of a text, in order, repeats kept."""
|
||||
return [w for w in WORD.findall(text.lower()) if w not in STOPWORDS]
|
||||
|
||||
|
||||
def stems(text: str) -> set[str]:
|
||||
return {w[:STEM] for w in words(text)}
|
||||
|
||||
|
||||
def heading_path(lines: list[str], line: int) -> list[str]:
|
||||
"""The headings above a line, outermost first: each heading closes every open one at its level or deeper."""
|
||||
path: list[tuple[int, str]] = []
|
||||
for n, level, text in kblocks.headings_of(lines):
|
||||
if n >= line:
|
||||
break
|
||||
path = [(lv, t) for lv, t in path if lv < level] + [(level, text)]
|
||||
return [t for _, t in path]
|
||||
|
||||
|
||||
def read_json(path: Path) -> Any:
|
||||
try:
|
||||
return json.loads(path.read_text(encoding="utf-8"))
|
||||
except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc:
|
||||
raise RunError(f"cannot read {path}: {exc}") from exc
|
||||
|
||||
|
||||
def sha256(path: Path) -> str:
|
||||
return hashlib.sha256(path.read_bytes()).hexdigest()
|
||||
|
||||
|
||||
def page_title(wiki: Path, page: str) -> str:
|
||||
path = wiki / page
|
||||
if not path.is_file():
|
||||
raise RunError(f"the page {path} is not there")
|
||||
fm = read_document(path, page).frontmatter
|
||||
return str((fm or {}).get("title") or "")
|
||||
|
||||
|
||||
def features(claims_doc: dict[str, Any], verdicts: dict[str, str] | None, sources: Path,
|
||||
wiki: Path) -> list[dict[str, Any]]:
|
||||
kilde = kblocks.kilde_of(sources)
|
||||
lines_of: dict[str, list[str]] = {}
|
||||
titles: dict[str, str] = {}
|
||||
out: list[dict[str, Any]] = []
|
||||
for c in claims_doc["claims"]:
|
||||
excerpts = list(c["excerpts"])
|
||||
quote_stems: set[str] = set()
|
||||
heading_stems: set[str] = set()
|
||||
headings: list[str] = []
|
||||
for i, e in enumerate(excerpts):
|
||||
name = str(e["file"])
|
||||
if name not in lines_of:
|
||||
path = kilde / name
|
||||
if not path.is_file():
|
||||
raise RunError(f"the source {path} is not there")
|
||||
lines_of[name] = kblocks.read_lines(path)
|
||||
path_here = heading_path(lines_of[name], int(e["line"]))
|
||||
headings = path_here if i == 0 else headings
|
||||
heading_stems |= {s for h in path_here for s in stems(h)}
|
||||
quote_stems |= stems(str(e["text"]))
|
||||
page = str(c["page"])
|
||||
if page not in titles:
|
||||
titles[page] = page_title(wiki, page)
|
||||
title_stems = stems(titles[page])
|
||||
content = words(str(c["claim"]))
|
||||
outside = [w for w in content if w[:STEM] not in quote_stems]
|
||||
out.append({
|
||||
"id": c["id"], "page": page, "provenance": c.get("provenance"),
|
||||
"files": sorted({str(e["file"]) for e in excerpts}),
|
||||
"verdict": None if verdicts is None else verdicts.get(str(c["id"])),
|
||||
"quotes": len(excerpts), "quote_chars": sum(len(str(e["text"])) for e in excerpts),
|
||||
"claim_chars": len(str(c["claim"])), "content_words": len(content),
|
||||
"outside": outside, "outside_count": len(outside),
|
||||
"outside_title": [w for w in outside if w[:STEM] in title_stems],
|
||||
"outside_headings": [w for w in outside if w[:STEM] in heading_stems],
|
||||
"headings": headings,
|
||||
})
|
||||
return out
|
||||
|
||||
|
||||
def _share(rows: list[dict[str, Any]], judged: bool) -> str:
|
||||
if not judged:
|
||||
return f"{len(rows)} claims, no verdicts"
|
||||
bad = sum(1 for r in rows if r["verdict"] == UNSUPPORTED)
|
||||
n_judged = sum(1 for r in rows if r["verdict"] is not None)
|
||||
if n_judged == len(rows):
|
||||
pct = f"{100 * bad / n_judged:.1f} %" if n_judged else "-"
|
||||
return f"{len(rows)} claims, {bad} unsupported ({pct})"
|
||||
pct = f"{100 * bad / n_judged:.1f} % of {n_judged} judged" if n_judged else "- of 0 judged"
|
||||
return f"{len(rows)} claims, {bad} unsupported ({pct}), {len(rows) - n_judged} not judged"
|
||||
|
||||
|
||||
def _median(xs: list[int]) -> str:
|
||||
return f"{statistics.median(xs):g}" if xs else "-"
|
||||
|
||||
|
||||
def summary(rows: list[dict[str, Any]]) -> str:
|
||||
judged = any(r["verdict"] is not None for r in rows)
|
||||
out = [f"claims: {len(rows)}"]
|
||||
for label, lo, hi in BUCKETS:
|
||||
mine = [r for r in rows if r["outside_count"] >= lo and (hi is None or r["outside_count"] <= hi)]
|
||||
out.append(f"words outside the quote {label}: {_share(mine, judged)}")
|
||||
for key, label in (("provenance", "provenance"), ("files", "source")):
|
||||
values = sorted({v for r in rows for v in (r[key] if isinstance(r[key], list) else [r[key]])}, key=str)
|
||||
for v in values:
|
||||
mine = [r for r in rows if v in (r[key] if isinstance(r[key], list) else [r[key]])]
|
||||
out.append(f"{label} {v}: {_share(mine, judged)}")
|
||||
if not judged:
|
||||
return "\n".join(out) + "\n"
|
||||
groups = {name: [r for r in rows if (r["verdict"] == UNSUPPORTED) == (name == UNSUPPORTED)
|
||||
and r["verdict"] is not None] for name in ("supported", UNSUPPORTED)}
|
||||
for field, label in (("outside_count", "words outside"), ("quote_chars", "quote characters"),
|
||||
("claim_chars", "claim characters"), ("content_words", "content words")):
|
||||
parts = [f"{name} {_median([int(r[field]) for r in g])}" for name, g in groups.items()]
|
||||
out.append(f"median {label}: {', '.join(parts)}")
|
||||
for field, label in (("outside_title", "the page title"), ("outside_headings", "the heading path")):
|
||||
parts = []
|
||||
for name, g in groups.items():
|
||||
k = sum(1 for r in g if r[field])
|
||||
pct = f"{100 * k / len(g):.1f} %" if g else "-"
|
||||
parts.append(f"{name} {k} of {len(g)} ({pct})")
|
||||
out.append(f"with a word from {label} outside the quote: {', '.join(parts)}")
|
||||
return "\n".join(out) + "\n"
|
||||
|
||||
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
p = argparse.ArgumentParser(prog="python3 tools/claim_features.py", description=__doc__,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
p.add_argument("--claims", required=True)
|
||||
p.add_argument("--sources", required=True)
|
||||
p.add_argument("--wiki", required=True)
|
||||
p.add_argument("--out", required=True)
|
||||
p.add_argument("--verdicts")
|
||||
p.add_argument("--summary", action="store_true")
|
||||
return p
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
args = build_parser().parse_args(argv)
|
||||
try:
|
||||
out = Path(args.out)
|
||||
if out.exists():
|
||||
raise RunError(f"{out} exists; an output is never overwritten")
|
||||
claims_path = Path(args.claims)
|
||||
claims_doc = read_json(claims_path)
|
||||
verdicts: dict[str, str] | None = None
|
||||
inputs: dict[str, Any] = {"claims_sha256": sha256(claims_path), "verdicts_sha256": None}
|
||||
if args.verdicts:
|
||||
vpath = Path(args.verdicts)
|
||||
verdicts = {str(v["id"]): str(v["verdict"]) for v in read_json(vpath)["verdicts"]}
|
||||
inputs["verdicts_sha256"] = sha256(vpath)
|
||||
rows = features(claims_doc, verdicts, Path(args.sources), Path(args.wiki))
|
||||
doc = {"schema": SCHEMA, "inputs": inputs, "claims": rows}
|
||||
write_new(out, json.dumps(doc, indent=1, sort_keys=True, ensure_ascii=False) + "\n")
|
||||
except (RunError, OSError, KeyError, TypeError, ValueError) as exc:
|
||||
print(f"claim_features cannot run: {exc}", file=sys.stderr)
|
||||
return 2
|
||||
print(f"claim_features: {len(rows)} claims -> {out}")
|
||||
if args.summary:
|
||||
print(summary(rows), end="")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
421
tools/diagnose.py
Normal file
421
tools/diagnose.py
Normal file
|
|
@ -0,0 +1,421 @@
|
|||
"""Why a claim is or is not carried by its excerpt: the tool around the diagnosis of wave 4 order 20 (contract K9).
|
||||
No model call. The diagnosis explains; the reader judges. It is not calibrated and never counts for criterion 2.
|
||||
|
||||
Usage:
|
||||
python3 tools/diagnose.py select --claims claims.json --verdicts verdicts.json --out utvalg.json [--sample 100]
|
||||
python3 tools/diagnose.py render --utvalg utvalg.json --claims claims.json --sources SOURCES --wiki WIKI
|
||||
--out tasks/ [--size 10]
|
||||
python3 tools/diagnose.py collect --transcripts DIR --run R [--run R2]... --tasks tasks/ --out diagnoses.json
|
||||
python3 tools/diagnose.py tally --diagnoses diagnoses.json --verdicts verdicts.json --claims claims.json
|
||||
--out tally.json
|
||||
|
||||
select every claim the reader did not accept, and a sample of the accepted ones:
|
||||
random.Random(SEED).sample(sorted(ids), --sample), then the two lists (each sorted) shuffled with the same
|
||||
generator. The selection holds ids only, never a verdict. A claim without a verdict, or fewer accepted claims
|
||||
than the sample, cannot run.
|
||||
render one task per call of at most --size claims, in the selection's order, named d001, d002, ...; and
|
||||
calls.json (the ids per call). Per claim: the page title, the claim, each excerpt it cites, and per excerpt a
|
||||
window in its source: the heading path above the quoted line (okfwiki.kblocks.headings_of, as
|
||||
tools/claim_features.py takes it) and WINDOW lines before and after it, cut at the file's ends. Each window
|
||||
line is "<line number> <q><c>| <text>", q ">" on the quoted line, c "c" on a line of fenced code (the fence
|
||||
lines too). No id, verdict or provenance mark reaches the task. A task folder that holds anything is never
|
||||
written into.
|
||||
collect the reply of every diagnosis agent (agent type okf-wiki-diagnoser, description "V1-<run>-diagnose-<call>",
|
||||
started from the session or from a workflow, the two places okfwiki.cost harvest reads): the message it
|
||||
handed back, or else its last text (okfwiki.reader.final_reply). For each call the last --run that answered
|
||||
it counts; two transcripts of that run for one call give no diagnosis for any of its claims. A line
|
||||
"<number>: <code> | <words>" gives the claim of that number its code (one of CODES) and the words that are
|
||||
missing; S needs no words, every other code does, and A has at most twelve. A line that starts with a number
|
||||
but breaks the form, or a number given twice, gives that claim no diagnosis: it is "missing", never guessed.
|
||||
Other lines are ignored.
|
||||
tally counts per code for the claims the reader did not accept and for the sample it accepted, each apart; how
|
||||
often the diagnosis and the reader disagree (S on a rejected claim; any other code on an accepted one) with a
|
||||
two-sided 95 % Clopper-Pearson interval; the same counts per source file and per provenance mark; and the M1
|
||||
rule of order 20: S for at least a third of the rejected claims, or any claim missing, is stop and present.
|
||||
|
||||
Codes (locked in phase 0 before any diagnosis was read): S the excerpt alone carries the claim; O what is missing
|
||||
stands in the heading path or the page title; L it stands in another line of the window; B the claim says more than
|
||||
the source (wider scope, stronger words); T a number, version, order, cause or purpose that is not in the window; F a
|
||||
property given to the wrong thing, or a wrong connection; U what is missing is not in the window at all; A other.
|
||||
|
||||
Output: json.dumps(doc, indent=1, sort_keys=True, ensure_ascii=False) + newline; an existing output is never
|
||||
overwritten. Exit 0 written; 1 written with claims missing (collect) or the M1 rule says stop (tally); 2 cannot run.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
import random
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
|
||||
import claim_features # noqa: E402
|
||||
from okfwiki import kblocks # noqa: E402
|
||||
from okfwiki.common import RunError, write_new # noqa: E402
|
||||
from okfwiki.reader import final_reply # noqa: E402
|
||||
from okfwiki.transcript import is_agent_type # noqa: E402
|
||||
|
||||
SELECTION_SCHEMA = "okf-wiki/diagnose-selection/1"
|
||||
CALLS_SCHEMA = "okf-wiki/diagnose-calls/1"
|
||||
DIAGNOSES_SCHEMA = "okf-wiki/diagnoses/1"
|
||||
TALLY_SCHEMA = "okf-wiki/diagnose-tally/1"
|
||||
SEED = 20261012
|
||||
SAMPLE = 100
|
||||
CALL_SIZE = 10
|
||||
WINDOW = 8
|
||||
CODES = ("S", "O", "L", "B", "T", "F", "U", "A")
|
||||
OTHER_WORDS_MAX = 12
|
||||
AGENT_TYPE = "okf-wiki-diagnoser"
|
||||
FENCE = re.compile(r"^\s*(```|~~~)") # the fence rule of okfwiki.kblocks.headings_of
|
||||
NUMBERED = re.compile(r"^\s*(\d+)\s*:(.*)$")
|
||||
FORM = re.compile(r"^\s*([A-Z])\s*(?:\|\s*(.*?))?\s*$")
|
||||
UNSUPPORTED = "unsupported"
|
||||
SUPPORTED = "supported"
|
||||
GROUPS = ("unsupported", "supported_sample")
|
||||
|
||||
|
||||
def read_json(path: Path) -> Any:
|
||||
try:
|
||||
return json.loads(path.read_text(encoding="utf-8"))
|
||||
except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc:
|
||||
raise RunError(f"cannot read {path}: {exc}") from exc
|
||||
|
||||
|
||||
def dump(doc: Any) -> str:
|
||||
return json.dumps(doc, indent=1, sort_keys=True, ensure_ascii=False) + "\n"
|
||||
|
||||
|
||||
def sha256(path: Path) -> str:
|
||||
return hashlib.sha256(path.read_bytes()).hexdigest()
|
||||
|
||||
|
||||
def verdict_map(path: Path) -> dict[str, str]:
|
||||
return {str(v["id"]): str(v["verdict"]) for v in read_json(path)["verdicts"]}
|
||||
|
||||
|
||||
# --- select ----------------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def select(claim_ids: list[str], verdicts: dict[str, str], sample: int) -> list[str]:
|
||||
rejected: list[str] = []
|
||||
accepted: list[str] = []
|
||||
for cid in claim_ids:
|
||||
v = verdicts.get(cid)
|
||||
if v not in (SUPPORTED, UNSUPPORTED):
|
||||
raise RunError(f"claim {cid} has no verdict of {SUPPORTED} or {UNSUPPORTED}")
|
||||
(rejected if v == UNSUPPORTED else accepted).append(cid)
|
||||
if len(accepted) < sample:
|
||||
raise RunError(f"{len(accepted)} accepted claims, fewer than the sample of {sample}")
|
||||
rng = random.Random(SEED)
|
||||
picked = rng.sample(sorted(accepted), sample)
|
||||
ids = sorted(rejected) + sorted(picked)
|
||||
rng.shuffle(ids)
|
||||
return ids
|
||||
|
||||
|
||||
# --- render ----------------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def code_lines(lines: list[str]) -> set[int]:
|
||||
out: set[int] = set()
|
||||
fence: str | None = None
|
||||
for n, raw in enumerate(lines, 1):
|
||||
f = FENCE.match(raw)
|
||||
if f:
|
||||
out.add(n)
|
||||
fence = None if fence == f.group(1) else (fence or f.group(1))
|
||||
continue
|
||||
if fence is not None:
|
||||
out.add(n)
|
||||
return out
|
||||
|
||||
|
||||
def window(lines: list[str], name: str, line: int) -> str:
|
||||
first, last = max(1, line - WINDOW), min(len(lines), line + WINDOW)
|
||||
code = code_lines(lines)
|
||||
path = " › ".join(claim_features.heading_path(lines, line)).replace('"', "'")
|
||||
rows = [f"{n:>5} {'>' if n == line else ' '}{'c' if n in code else ' '}| {lines[n - 1]}".rstrip()
|
||||
for n in range(first, last + 1)]
|
||||
return "\n".join([f'<window file="{name}" lines="{first}-{last}" headings="{path}">', *rows, "</window>"])
|
||||
|
||||
|
||||
def render_task(claims: list[dict[str, Any]], titles: dict[str, str], lines_of: dict[str, list[str]]) -> str:
|
||||
n = len(claims)
|
||||
parts = ["# Diagnosis task", "", f"This task has {n} claim{'s' if n != 1 else ''}, numbered 1 to {n}."]
|
||||
for number, c in enumerate(claims, 1):
|
||||
parts += ["", f"## Claim {number}", "", f"Page title: {titles[str(c['page'])]}", "",
|
||||
f"<claim>{c['claim']}</claim>"]
|
||||
for e in c["excerpts"]:
|
||||
parts += ["", f'<excerpt file="{e["file"]}" line="{e["line"]}">', str(e["text"]), "</excerpt>"]
|
||||
for e in c["excerpts"]:
|
||||
parts += ["", window(lines_of[str(e["file"])], str(e["file"]), int(e["line"]))]
|
||||
return "\n".join(parts) + "\n"
|
||||
|
||||
|
||||
def render(ids: list[str], claims_doc: dict[str, Any], sources: Path, wiki: Path, size: int,
|
||||
out: Path) -> list[dict[str, Any]]:
|
||||
if size < 1:
|
||||
raise RunError("--size must be at least 1")
|
||||
if out.exists() and any(out.iterdir()):
|
||||
raise RunError(f"{out} holds files; a task folder is never written into")
|
||||
by_id = {str(c["id"]): c for c in claims_doc["claims"]}
|
||||
missing = [i for i in ids if i not in by_id]
|
||||
if missing:
|
||||
raise RunError(f"selected ids not in the claim list: {missing[:5]}")
|
||||
kilde = kblocks.kilde_of(sources)
|
||||
lines_of: dict[str, list[str]] = {}
|
||||
titles: dict[str, str] = {}
|
||||
for cid in ids:
|
||||
c = by_id[cid]
|
||||
page = str(c["page"])
|
||||
if page not in titles:
|
||||
titles[page] = claim_features.page_title(wiki, page)
|
||||
for e in c["excerpts"]:
|
||||
name = str(e["file"])
|
||||
if name not in lines_of:
|
||||
if not (kilde / name).is_file():
|
||||
raise RunError(f"the source {kilde / name} is not there")
|
||||
lines_of[name] = kblocks.read_lines(kilde / name)
|
||||
chunks = [(f"d{k + 1:03d}", ids[i:i + size]) for k, i in enumerate(range(0, len(ids), size))]
|
||||
texts = {call: render_task([by_id[i] for i in chunk], titles, lines_of) for call, chunk in chunks}
|
||||
for call, text in texts.items():
|
||||
write_new(out / f"{call}.md", text)
|
||||
return [{"name": call, "ids": chunk} for call, chunk in chunks]
|
||||
|
||||
|
||||
# --- collect ---------------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def agent_replies(transcripts: Path, run: str) -> dict[str, list[Path]]:
|
||||
prefix = re.compile(rf"^V1-{re.escape(run)}-diagnose-(\S+)$")
|
||||
found: dict[str, list[Path]] = {}
|
||||
metas = [*transcripts.glob("*/subagents/agent-*.meta.json"),
|
||||
*transcripts.glob("*/subagents/workflows/*/agent-*.meta.json")]
|
||||
for meta_path in sorted(metas):
|
||||
try:
|
||||
meta = json.loads(meta_path.read_text(encoding="utf-8"))
|
||||
except (OSError, ValueError):
|
||||
continue
|
||||
if not isinstance(meta, dict) or not is_agent_type(meta.get("agentType"), AGENT_TYPE):
|
||||
continue
|
||||
m = prefix.match(str(meta.get("description", "")))
|
||||
if m is not None:
|
||||
found.setdefault(m.group(1), []).append(meta_path.with_name(
|
||||
meta_path.name.removesuffix(".meta.json") + ".jsonl"))
|
||||
return found
|
||||
|
||||
|
||||
def parse_reply(reply: str, n: int) -> dict[int, tuple[str, str]]:
|
||||
"""The code and missing words per claim number; a broken line or a number given twice leaves that claim out."""
|
||||
given: dict[int, tuple[str, str]] = {}
|
||||
bad: set[int] = set()
|
||||
for raw in reply.splitlines():
|
||||
m = NUMBERED.match(raw)
|
||||
if m is None:
|
||||
continue
|
||||
number = int(m.group(1))
|
||||
if not 1 <= number <= n:
|
||||
continue
|
||||
if number in given or number in bad:
|
||||
given.pop(number, None)
|
||||
bad.add(number)
|
||||
continue
|
||||
f = FORM.match(m.group(2))
|
||||
code, words = (f.group(1), (f.group(2) or "").strip()) if f else ("", "")
|
||||
ok = (code in CODES and (code == "S" or bool(words))
|
||||
and (code != "A" or len(words.split()) <= OTHER_WORDS_MAX))
|
||||
if ok:
|
||||
given[number] = (code, words)
|
||||
else:
|
||||
bad.add(number)
|
||||
return given
|
||||
|
||||
|
||||
def collect(calls: list[dict[str, Any]], runs: list[str], transcripts: Path) -> dict[str, Any]:
|
||||
found = {r: agent_replies(transcripts, r) for r in runs}
|
||||
rows: list[dict[str, Any]] = []
|
||||
for call in calls:
|
||||
name, ids = str(call["name"]), [str(i) for i in call["ids"]]
|
||||
answering = [r for r in runs if name in found[r]]
|
||||
run = answering[-1] if answering else None
|
||||
paths = found[run][name] if run else []
|
||||
given = parse_reply(final_reply(paths[0]) if paths[0].is_file() else "", len(ids)) if len(paths) == 1 else {}
|
||||
for number, cid in enumerate(ids, 1):
|
||||
code, words = given.get(number, (None, ""))
|
||||
rows.append({"id": cid, "call": name, "number": number, "code": code, "words": words, "run": run})
|
||||
return {"schema": DIAGNOSES_SCHEMA, "runs": runs, "diagnoses": rows,
|
||||
"missing": [r["id"] for r in rows if r["code"] is None]}
|
||||
|
||||
|
||||
# --- tally -----------------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _binom_tail(k_from: int, k_to: int, n: int, p: float) -> float:
|
||||
return sum(math.exp(math.lgamma(n + 1) - math.lgamma(i + 1) - math.lgamma(n - i + 1)
|
||||
+ i * math.log(p) + (n - i) * math.log1p(-p)) for i in range(k_from, k_to + 1))
|
||||
|
||||
|
||||
def _solve(f: Any, rising: bool) -> float:
|
||||
lo, hi = 0.0, 1.0
|
||||
for _ in range(200):
|
||||
mid = (lo + hi) / 2
|
||||
if mid <= 0.0 or mid >= 1.0:
|
||||
break
|
||||
if (f(mid) < 0.025) == rising:
|
||||
lo = mid
|
||||
else:
|
||||
hi = mid
|
||||
return (lo + hi) / 2
|
||||
|
||||
|
||||
def clopper_pearson(k: int, n: int) -> tuple[float, float]:
|
||||
"""The two-sided 95 % Clopper-Pearson interval for k events in n."""
|
||||
if n <= 0:
|
||||
return 0.0, 1.0
|
||||
lower = 0.0 if k == 0 else _solve(lambda p: _binom_tail(k, n, n, p), rising=True)
|
||||
upper = 1.0 if k >= n else _solve(lambda p: _binom_tail(0, k, n, p), rising=False)
|
||||
return lower, upper
|
||||
|
||||
|
||||
def counts(rows: list[dict[str, Any]]) -> dict[str, int]:
|
||||
out = {code: 0 for code in CODES}
|
||||
out["missing"] = 0
|
||||
for r in rows:
|
||||
out[r["code"] if r["code"] in CODES else "missing"] += 1
|
||||
out["n"] = len(rows)
|
||||
return out
|
||||
|
||||
|
||||
def tally(diagnoses: dict[str, Any], verdicts: dict[str, str], claims_doc: dict[str, Any]) -> dict[str, Any]:
|
||||
by_id = {str(c["id"]): c for c in claims_doc["claims"]}
|
||||
grouped: dict[str, list[dict[str, Any]]] = {g: [] for g in GROUPS}
|
||||
for row in diagnoses["diagnoses"]:
|
||||
v = verdicts.get(str(row["id"]))
|
||||
if v not in (SUPPORTED, UNSUPPORTED) or str(row["id"]) not in by_id:
|
||||
raise RunError(f"diagnosed claim {row['id']} has no verdict or is not in the claim list")
|
||||
grouped["unsupported" if v == UNSUPPORTED else "supported_sample"].append(row)
|
||||
|
||||
def split(key_of: Any) -> dict[str, dict[str, dict[str, int]]]:
|
||||
keys = sorted({k for g in grouped.values() for r in g for k in key_of(by_id[str(r["id"])])})
|
||||
return {k: {g: counts([r for r in grouped[g] if k in key_of(by_id[str(r["id"])])]) for g in GROUPS}
|
||||
for k in keys}
|
||||
|
||||
rej, acc = grouped["unsupported"], grouped["supported_sample"]
|
||||
rej_done = [r for r in rej if r["code"] in CODES]
|
||||
acc_done = [r for r in acc if r["code"] in CODES]
|
||||
called_s = sum(1 for r in rej_done if r["code"] == "S")
|
||||
not_carried = sum(1 for r in acc_done if r["code"] != "S")
|
||||
missing = sum(1 for g in grouped.values() for r in g if r["code"] not in CODES)
|
||||
return {
|
||||
"schema": TALLY_SCHEMA,
|
||||
"unsupported": counts(rej), "supported_sample": counts(acc),
|
||||
"agreement": {"unsupported_called_s": called_s, "unsupported_diagnosed": len(rej_done),
|
||||
"unsupported_called_s_ci95": list(clopper_pearson(called_s, len(rej_done))),
|
||||
"supported_not_carried": not_carried, "supported_diagnosed": len(acc_done),
|
||||
"supported_not_carried_ci95": list(clopper_pearson(not_carried, len(acc_done)))},
|
||||
"per_source": split(lambda c: sorted({str(e["file"]) for e in c["excerpts"]})),
|
||||
"per_provenance": split(lambda c: [str(c.get("provenance"))]),
|
||||
"m1": {"s": called_s, "of": len(rej), "missing": missing, "stop": 3 * called_s >= len(rej) or missing > 0},
|
||||
}
|
||||
|
||||
|
||||
def tally_text(doc: dict[str, Any]) -> str:
|
||||
out = []
|
||||
for g in GROUPS:
|
||||
c = doc[g]
|
||||
out.append(f"{g}: n {c['n']}; " + ", ".join(f"{k} {c[k]}" for k in (*CODES, "missing")))
|
||||
a = doc["agreement"]
|
||||
lo, hi = a["supported_not_carried_ci95"]
|
||||
out.append(f"accepted by the reader, not carried by the excerpt per the diagnosis: {a['supported_not_carried']} "
|
||||
f"of {a['supported_diagnosed']} (95 % CI {100 * lo:.1f}-{100 * hi:.1f} %)")
|
||||
lo, hi = a["unsupported_called_s_ci95"]
|
||||
out.append(f"rejected by the reader, carried per the diagnosis (S): {a['unsupported_called_s']} of "
|
||||
f"{a['unsupported_diagnosed']} (95 % CI {100 * lo:.1f}-{100 * hi:.1f} %)")
|
||||
m = doc["m1"]
|
||||
verdict = "STOP and present" if m["stop"] else "continue"
|
||||
out.append(f"M1 rule: S for {m['s']} of {m['of']} rejected, {m['missing']} missing: {verdict}")
|
||||
return "\n".join(out) + "\n"
|
||||
|
||||
|
||||
# --- command line ----------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
p = argparse.ArgumentParser(prog="python3 tools/diagnose.py", description=__doc__,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
sub = p.add_subparsers(dest="command", required=True)
|
||||
s = sub.add_parser("select")
|
||||
s.add_argument("--claims", required=True)
|
||||
s.add_argument("--verdicts", required=True)
|
||||
s.add_argument("--out", required=True)
|
||||
s.add_argument("--sample", type=int, default=SAMPLE)
|
||||
r = sub.add_parser("render")
|
||||
r.add_argument("--utvalg", required=True)
|
||||
r.add_argument("--claims", required=True)
|
||||
r.add_argument("--sources", required=True)
|
||||
r.add_argument("--wiki", required=True)
|
||||
r.add_argument("--out", required=True)
|
||||
r.add_argument("--size", type=int, default=CALL_SIZE)
|
||||
c = sub.add_parser("collect")
|
||||
c.add_argument("--transcripts", required=True)
|
||||
c.add_argument("--run", action="append", required=True)
|
||||
c.add_argument("--tasks", required=True)
|
||||
c.add_argument("--out", required=True)
|
||||
t = sub.add_parser("tally")
|
||||
t.add_argument("--diagnoses", required=True)
|
||||
t.add_argument("--verdicts", required=True)
|
||||
t.add_argument("--claims", required=True)
|
||||
t.add_argument("--out", required=True)
|
||||
return p
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
args = build_parser().parse_args(argv)
|
||||
try:
|
||||
out = Path(args.out)
|
||||
if args.command == "select":
|
||||
if out.exists():
|
||||
raise RunError(f"{out} exists; an output is never overwritten")
|
||||
claims_doc = read_json(Path(args.claims))
|
||||
ids = select([str(c["id"]) for c in claims_doc["claims"]], verdict_map(Path(args.verdicts)), args.sample)
|
||||
write_new(out, dump({"schema": SELECTION_SCHEMA, "seed": SEED, "sample": args.sample, "ids": ids}))
|
||||
print(f"select: {len(ids)} claims ({args.sample} of them a sample) -> {out}")
|
||||
return 0
|
||||
if args.command == "render":
|
||||
utvalg = Path(args.utvalg)
|
||||
ids = [str(i) for i in read_json(utvalg)["ids"]]
|
||||
calls = render(ids, read_json(Path(args.claims)), Path(args.sources), Path(args.wiki), args.size, out)
|
||||
write_new(out / "calls.json", dump({"schema": CALLS_SCHEMA, "utvalg_sha256": sha256(utvalg),
|
||||
"size": args.size, "calls": calls}))
|
||||
print(f"render: {len(ids)} claims in {len(calls)} calls -> {out}")
|
||||
return 0
|
||||
if args.command == "collect":
|
||||
if out.exists():
|
||||
raise RunError(f"{out} exists; an output is never overwritten")
|
||||
calls = read_json(Path(args.tasks) / "calls.json")["calls"]
|
||||
doc = collect(calls, list(args.run), Path(args.transcripts))
|
||||
write_new(out, dump(doc))
|
||||
print(f"collect: {len(doc['diagnoses']) - len(doc['missing'])} of {len(doc['diagnoses'])} claims "
|
||||
f"diagnosed, {len(doc['missing'])} missing -> {out}")
|
||||
return 1 if doc["missing"] else 0
|
||||
if out.exists():
|
||||
raise RunError(f"{out} exists; an output is never overwritten")
|
||||
doc = tally(read_json(Path(args.diagnoses)), verdict_map(Path(args.verdicts)), read_json(Path(args.claims)))
|
||||
write_new(out, dump(doc))
|
||||
print(tally_text(doc), end="")
|
||||
return 1 if doc["m1"]["stop"] else 0
|
||||
except (RunError, OSError, KeyError, TypeError, ValueError) as exc:
|
||||
print(f"diagnose cannot run: {exc}", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
482
tools/fetch_docs.py
Normal file
482
tools/fetch_docs.py
Normal file
|
|
@ -0,0 +1,482 @@
|
|||
"""Fetch the Claude Code documentation tree into a staging folder that `okfwiki.sources freeze` reads. No model.
|
||||
|
||||
Usage:
|
||||
python3 tools/fetch_docs.py --staging DIR --meta FILE --report FILE --preamble FILE
|
||||
[--base URL] [--index URL] [--changelog URL] [--changelog-page URL]
|
||||
[--pause SECONDS] [--cafile FILE] [--collection NAME]
|
||||
|
||||
The index (llms.txt) lists the pages. Every unique address under --base is fetched as .md, one at a time with a pause;
|
||||
"\\r\\n" becomes "\\n". Addresses outside --base (the indexes of other languages) are listed, not fetched.
|
||||
|
||||
Preamble Each page starts with a block of "> " lines whose first is a level-2 heading, then one blank line. It is
|
||||
removed when its number of lines and the sha256 of its lines ("\\n"-joined) equal the --preamble file
|
||||
({"lines": N, "sha256": "<hex>"}, kept outside git). Any other start is an error, never a guess.
|
||||
Moved out A page whose final address after redirects lies outside --base is no longer a page of this tree: it is
|
||||
not a source, and it is listed in the report (the operator's decision of 08.10.2026 on such a page).
|
||||
History The weekly digest pages (whats-new/YYYY-wNN.md under --base) are docs sources with main_topic false: they
|
||||
add history to the concepts and get no main page of their own (the operator's decision of 08.10.2026). The
|
||||
overview whats-new/index.md is an ordinary docs page.
|
||||
Changelog The release-note page is not a docs source. After its preamble it is split into one file per release,
|
||||
the text untouched, named release-vX.Y.Z.md, with the url --changelog-page#X-Y-Z; every anchor is checked
|
||||
on that page. Preamble + releases + the rest = the page. Two forms, never mixed on one page:
|
||||
component one <Update label="X.Y.Z"> ... </Update> block per release (the copy of 08.10);
|
||||
bold a line that is only "**X.Y.Z**", a blank line and the date ("May 1, 2026"); the release runs to
|
||||
the next such line or the end of the page (the page's form after 20.10, 437 of 437 on 07.11).
|
||||
Complete docs sources = unique index addresses under --base - the changelog - the pages moved out, and release-note
|
||||
sources = release_count(page), counted apart: the "<Update label=" on the page, or, in the bold form, every
|
||||
bold line that starts with a version. A difference is a FAIL.
|
||||
|
||||
Exit 0: staging folder and meta written. Exit 1: a page broke a rule or the completeness check failed; the report
|
||||
lists why and nothing else is written. Exit 2: cannot run (staging, meta or report exists, the index cannot be
|
||||
fetched, the network fails).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import ssl
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
|
||||
from okfwiki.common import RunError, write_new # noqa: E402
|
||||
from okfwiki.sources import META_SCHEMA, NAME, dump # noqa: E402
|
||||
|
||||
BASE = "https://code.claude.com/docs/en/"
|
||||
INDEX_URL = "https://code.claude.com/docs/llms.txt"
|
||||
CHANGELOG = BASE + "changelog.md"
|
||||
CHANGELOG_PAGE = BASE + "changelog"
|
||||
SYSTEM_CA = "/etc/ssl/cert.pem"
|
||||
USER_AGENT = "okf-wiki-fetch-docs/1"
|
||||
LINK = re.compile(r"^\s*[-*]\s+\[([^\]]*)\]\((\S+?)\)")
|
||||
HEADING = re.compile(r"^(#{2,4})\s+(.*?)\s*$")
|
||||
OPEN = re.compile(r'^<Update label="([^"]*)"[^>]*>$')
|
||||
CLOSE = "</Update>"
|
||||
VERSION = re.compile(r"^\d+\.\d+\.\d+$")
|
||||
BOLD = re.compile(r"^\*\*(\d+\.\d+\.\d+)\*\*$")
|
||||
BOLD_LIKE = re.compile(r"^\*\*v?\d+\.\d[^*]*\*\*\s*$")
|
||||
DATE = re.compile(r"^[A-Z][a-z]+ \d{1,2}, \d{4}$")
|
||||
HISTORY = re.compile(r"^whats-new/\d{4}-w\d{2}\.md$")
|
||||
HEX64 = re.compile(r"^[0-9a-f]{64}$")
|
||||
|
||||
|
||||
class FetchError(Exception):
|
||||
"""A page or the index breaks a rule of the fetch."""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Fetched:
|
||||
status: int
|
||||
url: str # the final address after redirects
|
||||
content_type: str
|
||||
body: bytes
|
||||
|
||||
|
||||
Fetcher = Callable[[str], Fetched]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Preamble:
|
||||
lines: int
|
||||
sha256: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class IndexPage:
|
||||
url: str
|
||||
title: str
|
||||
sections: tuple[str, ...]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Index:
|
||||
items: int
|
||||
pages: list[IndexPage]
|
||||
other_links: list[str]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Block:
|
||||
version: str
|
||||
text: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ChangelogSplit:
|
||||
blocks: list[Block]
|
||||
preamble_bytes: int
|
||||
block_bytes: int
|
||||
rest_bytes: int
|
||||
|
||||
|
||||
@dataclass
|
||||
class Collected:
|
||||
files: dict[str, bytes] = field(default_factory=dict)
|
||||
meta: dict[str, Any] = field(default_factory=dict)
|
||||
report: dict[str, Any] = field(default_factory=dict)
|
||||
problems: list[str] = field(default_factory=list)
|
||||
|
||||
|
||||
def parse_index(text: str, base: str) -> Index:
|
||||
"""The list items of the index: unique addresses under base in order, each with its ##/###/#### headings."""
|
||||
items = 0
|
||||
stack: list[str] = []
|
||||
pages: dict[str, IndexPage] = {}
|
||||
other: list[str] = []
|
||||
for line in text.split("\n"):
|
||||
h = HEADING.match(line)
|
||||
if h:
|
||||
stack = stack[: len(h.group(1)) - 2] + [h.group(2)]
|
||||
continue
|
||||
m = LINK.match(line)
|
||||
if not m:
|
||||
continue
|
||||
items += 1
|
||||
title, url = m.group(1), m.group(2)
|
||||
if url.startswith(base):
|
||||
pages.setdefault(url, IndexPage(url, title, tuple(stack)))
|
||||
elif url not in other:
|
||||
other.append(url)
|
||||
return Index(items, list(pages.values()), other)
|
||||
|
||||
|
||||
def source_name(url: str, base: str) -> str:
|
||||
"""https://…/docs/en/a/b.md -> docs-en-a-b.md (the pattern of the copy of 21.09)."""
|
||||
if not url.startswith(base) or not url.endswith(".md"):
|
||||
raise FetchError(f"{url} is not a .md address under {base}")
|
||||
rel = url[len(base): -len(".md")]
|
||||
name = f"docs-{base.rstrip('/').rsplit('/', 1)[-1]}-{rel.replace('/', '-')}.md"
|
||||
if not rel or not NAME.match(name):
|
||||
raise FetchError(f"{url} gives the name {name!r}, not [a-z0-9._-]")
|
||||
return name
|
||||
|
||||
|
||||
def is_history(rel: str) -> bool:
|
||||
"""A weekly digest page, by its path under the docs base: history, not a main topic (operator 08.10.2026)."""
|
||||
return bool(HISTORY.match(rel))
|
||||
|
||||
|
||||
def load_preamble(path: Path) -> Preamble:
|
||||
try:
|
||||
doc = json.loads(path.read_text(encoding="utf-8"))
|
||||
except (OSError, json.JSONDecodeError) as exc:
|
||||
raise FetchError(f"the preamble file {path} cannot be read: {exc}") from None
|
||||
lines, digest = (doc.get("lines"), doc.get("sha256")) if isinstance(doc, dict) else (None, None)
|
||||
if not (isinstance(lines, int) and not isinstance(lines, bool) and lines >= 1
|
||||
and isinstance(digest, str) and HEX64.match(digest)):
|
||||
raise FetchError(f"the preamble file {path} is not {{\"lines\": N, \"sha256\": \"<hex>\"}}")
|
||||
return Preamble(lines, digest)
|
||||
|
||||
|
||||
def strip_preamble(text: str, spec: Preamble) -> tuple[str, int]:
|
||||
"""The page without its preamble and the blank line after it, and the number of bytes removed."""
|
||||
lines = text.split("\n")
|
||||
n = 0
|
||||
while n < len(lines) and lines[n].startswith("> "):
|
||||
n += 1
|
||||
pre = lines[:n]
|
||||
if n != spec.lines or not pre[0].startswith("> ## ") or n >= len(lines) or lines[n] != "":
|
||||
raise FetchError(f"the page does not start with {spec.lines} '> ' lines, a '> ## ' heading first, "
|
||||
f"and a blank line (found {n} '> ' lines)")
|
||||
if hashlib.sha256("\n".join(pre).encode("utf-8")).hexdigest() != spec.sha256:
|
||||
raise FetchError("the preamble has the known shape but not the known sha256")
|
||||
rest = "\n".join(lines[n + 1:])
|
||||
return rest, len(text.encode("utf-8")) - len(rest.encode("utf-8"))
|
||||
|
||||
|
||||
def release_count(text: str) -> int:
|
||||
"""The releases on the page, counted apart from the split: its "<Update label=", or its bold version-like lines."""
|
||||
if "<Update label=" in text:
|
||||
return text.count("<Update label=")
|
||||
return sum(1 for line in text.split("\n") if BOLD_LIKE.match(line))
|
||||
|
||||
|
||||
def split_bold(body: str, pre_bytes: int, total: int) -> ChangelogSplit:
|
||||
"""One block per "**X.Y.Z**" line with a blank line and a date after it, running to the next one."""
|
||||
lines = [line + "\n" for line in body.split("\n")]
|
||||
lines[-1] = lines[-1][:-1]
|
||||
bare = [line.rstrip("\n") for line in lines]
|
||||
starts: list[tuple[int, str]] = []
|
||||
for i, line in enumerate(bare):
|
||||
if not BOLD_LIKE.match(line):
|
||||
continue
|
||||
m = BOLD.match(line)
|
||||
if not m:
|
||||
raise FetchError(f"line {i + 1}: {line!r} is not **X.Y.Z**")
|
||||
if not (i + 2 < len(bare) and bare[i + 1] == "" and DATE.match(bare[i + 2])):
|
||||
raise FetchError(f"line {i + 1}: version {m.group(1)} is not followed by a blank line and a date")
|
||||
if any(v == m.group(1) for _, v in starts):
|
||||
raise FetchError(f"line {i + 1}: version {m.group(1)} appears twice")
|
||||
starts.append((i, m.group(1)))
|
||||
if not starts:
|
||||
raise FetchError("the page has no <Update label=…> block and no **X.Y.Z** release line")
|
||||
ends = [i for i, _ in starts[1:]] + [len(lines)]
|
||||
blocks = [Block(v, "".join(lines[i:end])) for (i, v), end in zip(starts, ends)]
|
||||
rest_bytes = sum(len(line.encode("utf-8")) for line in lines[:starts[0][0]])
|
||||
block_bytes = sum(len(b.text.encode("utf-8")) for b in blocks)
|
||||
if pre_bytes + block_bytes + rest_bytes != total:
|
||||
raise FetchError("preamble + blocks + rest is not the length of the page")
|
||||
return ChangelogSplit(blocks, pre_bytes, block_bytes, rest_bytes)
|
||||
|
||||
|
||||
def split_changelog(text: str, spec: Preamble) -> ChangelogSplit:
|
||||
"""One block per release, the text untouched, in the component or the bold form; every byte is counted."""
|
||||
body, pre_bytes = strip_preamble(text, spec)
|
||||
if "<Update label=" not in body:
|
||||
return split_bold(body, pre_bytes, len(text.encode("utf-8")))
|
||||
if any(BOLD_LIKE.match(line) for line in body.split("\n")):
|
||||
raise FetchError("the page mixes <Update label=…> blocks and **X.Y.Z** release lines")
|
||||
lines = [line + "\n" for line in body.split("\n")]
|
||||
lines[-1] = lines[-1][:-1]
|
||||
blocks: list[Block] = []
|
||||
rest_bytes = 0
|
||||
current: list[str] | None = None
|
||||
version = ""
|
||||
for n, line in enumerate(lines, 1):
|
||||
bare = line.rstrip("\n")
|
||||
opened = OPEN.match(bare)
|
||||
if current is None:
|
||||
if opened:
|
||||
version = opened.group(1)
|
||||
if not VERSION.match(version):
|
||||
raise FetchError(f"line {n}: the label {version!r} is not X.Y.Z")
|
||||
if any(b.version == version for b in blocks):
|
||||
raise FetchError(f"line {n}: version {version} appears twice")
|
||||
current = [line]
|
||||
elif bare == CLOSE:
|
||||
raise FetchError(f"line {n}: {CLOSE} without an open block")
|
||||
else:
|
||||
rest_bytes += len(line.encode("utf-8"))
|
||||
else:
|
||||
if opened:
|
||||
raise FetchError(f"line {n}: a block opens inside version {version}")
|
||||
current.append(line)
|
||||
if bare == CLOSE:
|
||||
blocks.append(Block(version, "".join(current)))
|
||||
current = None
|
||||
if current is not None:
|
||||
raise FetchError(f"version {version} is never closed")
|
||||
if not blocks:
|
||||
raise FetchError("the page has no <Update label=…> block")
|
||||
if body.count("<Update label=") != len(blocks):
|
||||
raise FetchError(f"the page has {body.count('<Update label=')} '<Update label=', but {len(blocks)} blocks "
|
||||
"start a line")
|
||||
block_bytes = sum(len(b.text.encode("utf-8")) for b in blocks)
|
||||
if pre_bytes + block_bytes + rest_bytes != len(text.encode("utf-8")):
|
||||
raise FetchError("preamble + blocks + rest is not the length of the page")
|
||||
return ChangelogSplit(blocks, pre_bytes, block_bytes, rest_bytes)
|
||||
|
||||
|
||||
def anchor_url(page: str, version: str) -> str:
|
||||
return f"{page}#{version.replace('.', '-')}"
|
||||
|
||||
|
||||
def missing_anchors(html: str, versions: list[str]) -> list[str]:
|
||||
return [v for v in versions if f'id="{v.replace(".", "-")}"' not in html]
|
||||
|
||||
|
||||
def collect(fetcher: Fetcher, preamble: Preamble, *, base: str = BASE, index_url: str = INDEX_URL,
|
||||
changelog: str = CHANGELOG, changelog_page: str = CHANGELOG_PAGE, pause: float = 0.5,
|
||||
collection: str = "claude-code") -> Collected:
|
||||
"""Fetch and split everything in memory; nothing is written here."""
|
||||
got = Collected()
|
||||
problems = got.problems
|
||||
|
||||
def fetch(url: str) -> Fetched:
|
||||
f = fetcher(url)
|
||||
if pause:
|
||||
time.sleep(pause)
|
||||
return f
|
||||
|
||||
raw_index = fetch(index_url)
|
||||
if raw_index.status != 200:
|
||||
raise RunError(f"the index {index_url} gave status {raw_index.status}")
|
||||
index_text = raw_index.body.decode("utf-8").replace("\r\n", "\n")
|
||||
idx = parse_index(index_text, base)
|
||||
urls = [p.url for p in idx.pages]
|
||||
if changelog not in urls:
|
||||
problems.append(f"the changelog {changelog} is not in the index")
|
||||
sections = {p.url: p.sections for p in idx.pages}
|
||||
names: dict[str, str] = {}
|
||||
by_name: dict[str, str] = {}
|
||||
for url in urls:
|
||||
try:
|
||||
name = source_name(url, base)
|
||||
except FetchError as exc:
|
||||
problems.append(str(exc))
|
||||
continue
|
||||
if name in by_name:
|
||||
problems.append(f"{name}: same path for {by_name[name]} and {url}")
|
||||
continue
|
||||
by_name[name] = url
|
||||
names[url] = name
|
||||
sources: dict[str, dict[str, Any]] = {}
|
||||
moved_out: list[dict[str, str]] = []
|
||||
crlf: list[str] = []
|
||||
preamble_pages = preamble_bytes = raw_updates = 0
|
||||
changelog_bytes: dict[str, int] = {}
|
||||
docs_sections: dict[str, list[str]] = {}
|
||||
versions: list[str] = []
|
||||
history_pages: list[str] = []
|
||||
for url in urls:
|
||||
if url not in names:
|
||||
continue
|
||||
name = names[url]
|
||||
f = fetch(url)
|
||||
if f.status != 200:
|
||||
problems.append(f"{url}: status {f.status}")
|
||||
continue
|
||||
if not f.url.startswith(base):
|
||||
moved_out.append({"content_type": f.content_type, "final_url": f.url, "url": url})
|
||||
continue
|
||||
if not f.content_type.startswith("text/markdown"):
|
||||
problems.append(f"{url} ({name}): {f.content_type or 'no content type'}, not markdown")
|
||||
continue
|
||||
try:
|
||||
text = f.body.decode("utf-8")
|
||||
except UnicodeDecodeError:
|
||||
problems.append(f"{url} ({name}): not UTF-8")
|
||||
continue
|
||||
if "\r\n" in text:
|
||||
crlf.append(name)
|
||||
text = text.replace("\r\n", "\n")
|
||||
try:
|
||||
if url == changelog:
|
||||
raw_updates = release_count(text)
|
||||
split = split_changelog(text, preamble)
|
||||
changelog_bytes = {"page": len(text.encode("utf-8")), "preamble": split.preamble_bytes,
|
||||
"blocks": split.block_bytes, "rest": split.rest_bytes}
|
||||
preamble_pages += 1
|
||||
preamble_bytes += split.preamble_bytes
|
||||
for b in split.blocks:
|
||||
rn = f"release-v{b.version}.md"
|
||||
got.files[rn] = b.text.encode("utf-8")
|
||||
sources[rn] = {"kind": "release-notes", "main_topic": False,
|
||||
"url": anchor_url(changelog_page, b.version), "version": b.version}
|
||||
versions.append(b.version)
|
||||
else:
|
||||
body, removed = strip_preamble(text, preamble)
|
||||
preamble_pages += 1
|
||||
preamble_bytes += removed
|
||||
got.files[name] = body.encode("utf-8")
|
||||
history = is_history(url[len(base):])
|
||||
sources[name] = {"kind": "docs", "main_topic": not history, "url": url}
|
||||
if history:
|
||||
history_pages.append(name)
|
||||
docs_sections[name] = list(sections[url])
|
||||
except FetchError as exc:
|
||||
problems.append(f"{url} ({name}): {exc}")
|
||||
if versions:
|
||||
page = fetch(changelog_page)
|
||||
if page.status != 200:
|
||||
problems.append(f"{changelog_page}: status {page.status}; the version anchors cannot be checked")
|
||||
else:
|
||||
for v in missing_anchors(page.body.decode("utf-8", errors="replace"), versions):
|
||||
problems.append(f"{v}: no anchor id=\"{v.replace('.', '-')}\" on {changelog_page}")
|
||||
docs = sorted(n for n, s in sources.items() if s["kind"] == "docs")
|
||||
notes = sorted(n for n, s in sources.items() if s["kind"] == "release-notes")
|
||||
has_changelog = int(changelog in urls)
|
||||
complete = {"index_pages": len(urls), "changelog": has_changelog, "moved_out": len(moved_out),
|
||||
"docs_expected": len(urls) - has_changelog - len(moved_out), "docs": len(docs),
|
||||
"release_notes_expected": raw_updates, "release_notes": len(notes)}
|
||||
complete["pass"] = (complete["docs"] == complete["docs_expected"]
|
||||
and complete["release_notes"] == complete["release_notes_expected"] and raw_updates > 0)
|
||||
if not complete["pass"]:
|
||||
problems.append(f"completeness FAIL: docs {complete['docs']} of {complete['docs_expected']} expected, "
|
||||
f"release notes {complete['release_notes']} of {complete['release_notes_expected']}")
|
||||
got.meta = {"collection": collection, "schema": META_SCHEMA,
|
||||
"index": {"pages": len(urls), "sha256": hashlib.sha256(raw_index.body).hexdigest(), "url": index_url},
|
||||
"sources": dict(sorted(sources.items()))}
|
||||
got.report = {
|
||||
"index": {"url": index_url, "sha256": got.meta["index"]["sha256"], "bytes": len(raw_index.body),
|
||||
"items": idx.items, "pages": len(urls)},
|
||||
"other_links": idx.other_links, "moved_out": moved_out, "completeness": complete, "crlf_pages": crlf,
|
||||
"history_pages": sorted(history_pages),
|
||||
"preamble": {"pages": preamble_pages, "bytes": preamble_bytes}, "changelog_bytes": changelog_bytes,
|
||||
"docs": {"count": len(docs), "bytes": sum(len(got.files[n]) for n in docs)},
|
||||
"release_notes": {"count": len(notes), "bytes": sum(len(got.files[n]) for n in notes)},
|
||||
"sections": docs_sections, "problems": problems}
|
||||
return got
|
||||
|
||||
|
||||
def net_fetcher(cafile: str | None, timeout: float = 60.0) -> Fetcher:
|
||||
"""urllib with certificate checks on. The python.org build here has no CA store, so the system's file is used."""
|
||||
if cafile is None and not os.environ.get("SSL_CERT_FILE") and os.path.isfile(SYSTEM_CA):
|
||||
cafile = SYSTEM_CA
|
||||
context = ssl.create_default_context(cafile=cafile)
|
||||
|
||||
def fetch(url: str) -> Fetched:
|
||||
req = urllib.request.Request(url, headers={"User-Agent": USER_AGENT})
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=timeout, context=context) as r:
|
||||
return Fetched(r.status, r.geturl(), r.headers.get("Content-Type", ""), r.read())
|
||||
except urllib.error.HTTPError as exc:
|
||||
return Fetched(exc.code, exc.geturl() or url, exc.headers.get("Content-Type", "") if exc.headers else "",
|
||||
b"")
|
||||
|
||||
return fetch
|
||||
|
||||
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
p.add_argument("--staging", required=True)
|
||||
p.add_argument("--meta", required=True)
|
||||
p.add_argument("--report", required=True)
|
||||
p.add_argument("--preamble", required=True)
|
||||
p.add_argument("--base", default=BASE)
|
||||
p.add_argument("--index", default=INDEX_URL)
|
||||
p.add_argument("--changelog", default=CHANGELOG)
|
||||
p.add_argument("--changelog-page", default=CHANGELOG_PAGE)
|
||||
p.add_argument("--pause", type=float, default=0.5)
|
||||
p.add_argument("--cafile")
|
||||
p.add_argument("--collection", default="claude-code")
|
||||
return p
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None, fetcher: Fetcher | None = None) -> int:
|
||||
args = build_parser().parse_args(argv)
|
||||
staging, meta, report = Path(args.staging), Path(args.meta), Path(args.report)
|
||||
try:
|
||||
for p in (staging, meta, report):
|
||||
if p.exists():
|
||||
raise RunError(f"{p} exists; refusing to overwrite")
|
||||
got = collect(fetcher or net_fetcher(args.cafile), load_preamble(Path(args.preamble)), base=args.base,
|
||||
index_url=args.index, changelog=args.changelog, changelog_page=args.changelog_page,
|
||||
pause=args.pause, collection=args.collection)
|
||||
write_new(report, dump(got.report))
|
||||
c = got.report["completeness"]
|
||||
summary = (f"fetch: {c['docs']} docs of {c['docs_expected']} expected, {c['release_notes']} release notes of "
|
||||
f"{c['release_notes_expected']}, {c['moved_out']} moved out, {len(got.report['other_links'])} "
|
||||
f"other links not fetched")
|
||||
if got.problems:
|
||||
print(f"{summary}; FAIL with {len(got.problems)} problems, nothing staged (report: {report})",
|
||||
file=sys.stderr)
|
||||
for problem in got.problems:
|
||||
print(f" {problem}", file=sys.stderr)
|
||||
return 1
|
||||
staging.mkdir(parents=True)
|
||||
for name, data in got.files.items():
|
||||
(staging / name).write_bytes(data)
|
||||
write_new(meta, dump(got.meta))
|
||||
print(f"{summary}; completeness pass -> {staging}")
|
||||
for m in got.report["moved_out"]:
|
||||
print(f" moved out: {m['url']} -> {m['final_url']}")
|
||||
return 0
|
||||
except (RunError, FetchError, OSError, UnicodeDecodeError) as exc:
|
||||
print(f"fetch cannot run: {exc}", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
124
tools/google_viewer_check.py
Normal file
124
tools/google_viewer_check.py
Normal file
|
|
@ -0,0 +1,124 @@
|
|||
"""Google's viewer on a bundle (PLAN.md criterion 5: "Googles visning leser hver samling uten feil"; brief SC5). It calls
|
||||
no model and reads no network.
|
||||
|
||||
Usage:
|
||||
python3 tools/google_viewer_check.py BUNDLE --upstream DIR [--out FILE]
|
||||
|
||||
DIR is an upstream checkout of Google's Open Knowledge Format repository (a `git archive` of the pinned commit, never
|
||||
the mirror itself): the tool runs `generate_visualization` from DIR/src/reference_agent/viewer/generator.py with
|
||||
`python3 -I -B` (isolated, no bytecode written into DIR) from a fresh temporary folder, on an unchanged copy of
|
||||
BUNDLE, so the bundle itself is never written. The viewer at ad30107 skips only index.md (`_walk_concepts`), so it draws
|
||||
each log.md as a node too, although SPEC 3.1 reserves the name; those nodes (id "log" or ".../log") are counted apart.
|
||||
|
||||
PASS when the run exits 0, its output holds no "Traceback", it writes the graph page, and the graph in that page
|
||||
(the line "window.BUNDLE = <json>;") has as many concept nodes as the auditor counts concept pages, at least one, and
|
||||
one log node per log.md. --out keeps the graph page (for `okfwiki.view render --graph`; it shows the log nodes). Exit 0 = PASS, 1 = FAIL, 2 = could not run (no bundle directory,
|
||||
or no generator under DIR).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
|
||||
from okfwiki.audit import load_bundle # noqa: E402
|
||||
|
||||
GENERATOR = Path("src") / "reference_agent" / "viewer" / "generator.py"
|
||||
GRAPH_LINE = "window.BUNDLE = "
|
||||
TIMEOUT = 900
|
||||
RUNNER = """
|
||||
import json, sys
|
||||
from pathlib import Path
|
||||
sys.path.insert(0, sys.argv[1])
|
||||
from reference_agent.viewer.generator import generate_visualization
|
||||
print(json.dumps(generate_visualization(Path(sys.argv[2]), Path(sys.argv[3]), bundle_name=sys.argv[4])))
|
||||
"""
|
||||
|
||||
|
||||
class RunError(Exception):
|
||||
"""The check cannot run; exit 2."""
|
||||
|
||||
|
||||
def graph_nodes(page: Path) -> tuple[list[str], int] | None:
|
||||
"""(the node ids, the number of edges) of the graph embedded in the viewer's page, or None when it cannot be read."""
|
||||
try:
|
||||
lines = page.read_text(encoding="utf-8").splitlines()
|
||||
except (OSError, UnicodeDecodeError):
|
||||
return None
|
||||
line = next((x for x in lines if x.startswith(GRAPH_LINE) and x.endswith(";")), None)
|
||||
if line is None:
|
||||
return None
|
||||
try:
|
||||
graph: Any = json.loads(line[len(GRAPH_LINE):-1])
|
||||
except ValueError:
|
||||
return None
|
||||
if not isinstance(graph, dict) or not isinstance(graph.get("nodes"), list) or not isinstance(graph.get("edges"), list):
|
||||
return None
|
||||
data = [n.get("data") if isinstance(n, dict) else None for n in graph["nodes"]]
|
||||
return [str(d.get("id")) if isinstance(d, dict) else "" for d in data], len(graph["edges"])
|
||||
|
||||
|
||||
def check(bundle: Path, upstream: Path, out: Path | None) -> tuple[bool, list[str]]:
|
||||
"""(passed, the lines to print)."""
|
||||
if not bundle.is_dir():
|
||||
raise RunError(f"the bundle {bundle} is not a directory")
|
||||
if not (upstream / GENERATOR).is_file():
|
||||
raise RunError(f"no viewer generator at {upstream / GENERATOR}")
|
||||
pages = len(load_bundle(bundle, bundle).concepts)
|
||||
logs = sum(1 for p in bundle.rglob("log.md") if p.is_file())
|
||||
with tempfile.TemporaryDirectory(prefix="google-viewer-") as tmp:
|
||||
work = Path(tmp)
|
||||
copy, run_in, graph = work / "bundle" / bundle.resolve().name, work / "cwd", work / "graph.html"
|
||||
shutil.copytree(bundle, copy)
|
||||
run_in.mkdir()
|
||||
cmd = [sys.executable, "-I", "-B", "-c", RUNNER, str((upstream / "src").resolve()), str(copy), str(graph),
|
||||
bundle.resolve().name]
|
||||
try:
|
||||
proc = subprocess.run(cmd, cwd=run_in, capture_output=True, text=True, timeout=TIMEOUT, check=False)
|
||||
code, output = proc.returncode, proc.stdout + proc.stderr
|
||||
except subprocess.TimeoutExpired:
|
||||
code, output = -1, f"the viewer did not finish in {TIMEOUT} seconds"
|
||||
found = graph_nodes(graph)
|
||||
if out is not None and graph.is_file():
|
||||
out.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copyfile(graph, out)
|
||||
ids, edges = found if found is not None else ([], -1)
|
||||
log_nodes = sum(1 for i in ids if i.rsplit("/", 1)[-1] == "log")
|
||||
nodes = len(ids) - log_nodes
|
||||
traceback = "Traceback" in output
|
||||
passed = code == 0 and not traceback and found is not None and pages > 0 and nodes == pages and log_nodes == logs
|
||||
lines = [f"google viewer: exit {code} · traceback {'yes' if traceback else 'no'} · "
|
||||
f"concept nodes {nodes if found is not None else 'unreadable'} · concept pages {pages} · "
|
||||
f"log nodes {log_nodes} of {logs} · edges {edges}"]
|
||||
if code != 0 or traceback:
|
||||
lines += [" " + x for x in output.strip().splitlines()[-20:]]
|
||||
lines.append(f"{'PASS' if passed else 'FAIL'} Google's viewer reads the bundle: one node per concept page and per log.md")
|
||||
return passed, lines
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
p.add_argument("bundle")
|
||||
p.add_argument("--upstream", required=True, help="an upstream checkout (git archive of the pinned commit)")
|
||||
p.add_argument("--out", help="keep the viewer's graph page here")
|
||||
args = p.parse_args(argv)
|
||||
try:
|
||||
passed, lines = check(Path(args.bundle), Path(args.upstream), Path(args.out) if args.out else None)
|
||||
except RunError as exc:
|
||||
print(f"google viewer cannot run: {exc}", file=sys.stderr)
|
||||
return 2
|
||||
for line in lines:
|
||||
print(line)
|
||||
return 0 if passed else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
13
tools/install-hooks.sh
Executable file
13
tools/install-hooks.sh
Executable file
|
|
@ -0,0 +1,13 @@
|
|||
#!/bin/bash
|
||||
# install-hooks.sh - installs the repository-local git hooks that run the
|
||||
# excluded-terms check before every commit (staged files) and on every commit
|
||||
# message. The machine's global hook dispatcher runs these after its own rules.
|
||||
# Run once per clone. bash 3.2 safe, ASCII only.
|
||||
set -eu
|
||||
ROOT=$(git rev-parse --show-toplevel)
|
||||
HOOKS="$(git rev-parse --git-common-dir)/hooks"
|
||||
mkdir -p "$HOOKS"
|
||||
printf '#!/bin/bash\nexec "%s/tools/check-excluded-terms.sh" --staged\n' "$ROOT" > "$HOOKS/pre-commit"
|
||||
printf '#!/bin/bash\nexec "%s/tools/check-excluded-terms.sh" --message "$1"\n' "$ROOT" > "$HOOKS/commit-msg"
|
||||
chmod +x "$HOOKS/pre-commit" "$HOOKS/commit-msg"
|
||||
echo "installed: $HOOKS/pre-commit, $HOOKS/commit-msg"
|
||||
314
tools/part_metrics.py
Normal file
314
tools/part_metrics.py
Normal file
|
|
@ -0,0 +1,314 @@
|
|||
"""The numbers per pack and per source of a build folder (contract K7), read without a model (CLAUDE.md rule 1).
|
||||
|
||||
Usage:
|
||||
python3 tools/part_metrics.py ROOT --out FILE
|
||||
|
||||
ROOT is a build folder: work/ (the skill's WORK), wiki/ (WIKI), logg/pN.txt (what the build commands of pack N wrote,
|
||||
with "exit N" or "exit N: <module> <command>" lines) and kost.json (okfwiki.cost harvest). It writes one JSON object (indent 1, sorted keys), the same
|
||||
bytes on every run over the same folder. Plan revision 5 Step 11 asks for these numbers per pack, with a growth fit,
|
||||
so the price of the rest and the growth of the collection can be read from the first part (PLAN.md § Retning point 4).
|
||||
|
||||
Per accepted pack (work/K-work/packs.json):
|
||||
- sources, blocks in the pack and so far;
|
||||
- cost per line: build and link as attributed by description, orchestration spread over the packs in proportion to
|
||||
their attributed cost (as okfwiki.cost stop spreads it), the pack's total, the total so far and per block; the
|
||||
number of build and link runs, the compiler's new runs after a rejection at writing (exit 3) and the guard rounds,
|
||||
because they explain the cost; the exit codes in logg/pN.txt in order;
|
||||
- the concepts in the index after the pack (work/K-work/index.json: the skeleton's entries and those whose origin is
|
||||
a source of the packs so far), the pages (concepts with an accepted fragment) and the main pages;
|
||||
- the catalogue of main pages the pack's compiler got, in lines and characters, read from its task parts in
|
||||
work/K-oppgaver/ as kpack wrote them, beside the limit klink.CATALOGUE_CHARS;
|
||||
- like concepts: groups of pages that share a normalised title or alias (kpack.norm, its concept identity);
|
||||
- pages without a home and docs sources without a main page before and after the guard, from the auditor's JSON
|
||||
(no-main-page-link and main-page-missing) in work/K-audit/pN-audit-1.json and in the last pN-audit-k.json; the
|
||||
auditor is the only judge (K2 point 17), nothing here counts it again;
|
||||
- main-page out-degree: per main page the concept pages it links to, in a sentence or in its list of related
|
||||
concepts, each page once; min, median and max. The wiki is assembled again after every pack and keeps no earlier
|
||||
state, so the out-degree after pack N is counted from the records that place the links: a sentence link counts from
|
||||
the pack whose link pass asked for its sentence (work/K-links/**/sentences.json), a home from the pack its record
|
||||
names. Only records that still stand count. Beside it, out_degree_wiki counts the same on wiki/ with the auditor's
|
||||
own reading of links from main pages (audit.main_page_links); after the last pack the two agree when every link is
|
||||
attributed (links_unattributed counts those that are not).
|
||||
Per source: the compiler's cost (the build line) and the pack's total, split over the pack's sources by their share of
|
||||
its blocks (okfwiki.cost.allocate), the price of a later repair of one source.
|
||||
fit: cost per block against blocks read before the pack (okfwiki.cost.project, as the stop line projects it, to the
|
||||
blocks in work/blocks.json), and the catalogue's characters against blocks read before the pack (okfwiki.cost.fit_line),
|
||||
with the blocks read at which the fitted line reaches the limit.
|
||||
|
||||
Exit 0 written, 2 cannot run (an input file missing or not readable; nothing is written).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import statistics
|
||||
import sys
|
||||
from fractions import Fraction
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
|
||||
from okfwiki import audit, cost, klink, kpack # noqa: E402
|
||||
from okfwiki.common import RunError # noqa: E402
|
||||
|
||||
SCHEMA = "okf-wiki/part-metrics/1"
|
||||
LINES = kpack.STOP_LINES.split(",") # build, link, orchestration-build: the lines the build's stop line caps
|
||||
CATALOGUE_HEADER = "## Main pages (homes you may choose)\n\n" # as kpack.render_input writes it
|
||||
PART_HEAD = re.compile(r"^<!-- part \d+ of \d+ -->\n")
|
||||
PART_FILE = re.compile(r"^pack-(\d+)-part-(\d+)\.md$")
|
||||
EXIT_LINE = re.compile(r"^exit (\d+)(?:: .*)?$") # bare, or as the skill's chain prints it
|
||||
|
||||
|
||||
def read_json(path: Path) -> Any:
|
||||
try:
|
||||
return json.loads(path.read_text(encoding="utf-8"))
|
||||
except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc:
|
||||
raise RunError(f"cannot read {path}: {exc}") from exc
|
||||
|
||||
|
||||
def catalogue_of(parts: list[str]) -> tuple[int, int] | None:
|
||||
"""(lines, characters) of the catalogue of main pages in a compiler task, its parts in order; each line counted
|
||||
with its newline, as kpack.main_catalogue counts against its budget. None when the task has no catalogue section."""
|
||||
text = "".join(PART_HEAD.sub("", p, count=1) for p in parts)
|
||||
at = text.rfind(CATALOGUE_HEADER)
|
||||
if at < 0:
|
||||
return None
|
||||
lines = [line for line in text[at + len(CATALOGUE_HEADER):].split("\n") if line.startswith("- ")]
|
||||
return len(lines), sum(len(line) + 1 for line in lines)
|
||||
|
||||
|
||||
def task_parts(work: Path, n: int) -> list[str]:
|
||||
found = []
|
||||
for p in (work / "K-oppgaver").glob(f"pack-{n}-part-*.md"):
|
||||
m = PART_FILE.match(p.name)
|
||||
if m and int(m.group(1)) == n:
|
||||
found.append((int(m.group(2)), p.read_text(encoding="utf-8")))
|
||||
return [text for _, text in sorted(found)]
|
||||
|
||||
|
||||
def pack_costs(kost: dict[str, Any], numbers: list[int]) -> tuple[dict[int, dict[str, Any]], dict[str, int]]:
|
||||
"""Per pack: build and link as attributed, and the pack's total with orchestration spread over the packs in
|
||||
proportion to their attributed cost, as okfwiki.cost stop does (exact fractions); and the line totals."""
|
||||
totals = {name: int(kost["lines"][name]["total"]) for name in LINES}
|
||||
measured = sum(totals.values())
|
||||
per: dict[int, dict[str, Any]] = {n: {"build": 0, "link": 0, "build_runs": 0, "link_runs": 0} for n in numbers}
|
||||
for r in kost.get("runs", []):
|
||||
line, n = r.get("line"), r.get("pack")
|
||||
if line in ("build", "link") and n is not None and int(n) in per:
|
||||
per[int(n)][line] += int(r["total"])
|
||||
per[int(n)][f"{line}_runs"] += 1
|
||||
attributed = sum(p["build"] + p["link"] for p in per.values())
|
||||
scale = Fraction(measured, attributed) if attributed else None
|
||||
for p in per.values():
|
||||
p["total"] = (p["build"] + p["link"]) * scale if scale is not None else None
|
||||
return per, {**totals, "measured": measured}
|
||||
|
||||
|
||||
def concept_names(entry: dict[str, Any]) -> set[str]:
|
||||
return {kpack.norm(str(a)) for a in [entry.get("title", ""), *entry.get("aliases", [])]} - {""}
|
||||
|
||||
|
||||
def duplicate_groups(index: dict[str, Any], pages: set[str]) -> list[list[str]]:
|
||||
"""Groups of two or more pages that share a normalised title or alias, each group sorted, the groups sorted."""
|
||||
names = {str(e["id"]): concept_names(e) for e in index["entries"] if str(e["id"]) in pages}
|
||||
parent = {cid: cid for cid in names}
|
||||
|
||||
def find(c: str) -> str:
|
||||
while parent[c] != c:
|
||||
c = parent[c]
|
||||
return c
|
||||
|
||||
first: dict[str, str] = {}
|
||||
for cid in sorted(names):
|
||||
for name in sorted(names[cid]):
|
||||
if name in first:
|
||||
a, b = find(first[name]), find(cid)
|
||||
if a != b:
|
||||
parent[max(a, b)] = min(a, b)
|
||||
else:
|
||||
first[name] = cid
|
||||
groups: dict[str, list[str]] = {}
|
||||
for cid in sorted(names):
|
||||
groups.setdefault(find(cid), []).append(cid)
|
||||
return sorted(g for g in groups.values() if len(g) > 1)
|
||||
|
||||
|
||||
def homeless(doc: dict[str, Any]) -> dict[str, list[str]]:
|
||||
findings = [f for f in doc.get("findings", []) if isinstance(f, dict)]
|
||||
return {"pages": sorted(str(f.get("file")) for f in findings if f.get("check") == "no-main-page-link"),
|
||||
"sources": sorted(str(f.get("file")) for f in findings if f.get("check") == "main-page-missing")}
|
||||
|
||||
|
||||
def audits(work: Path, n: int) -> dict[str, Any] | None:
|
||||
found = []
|
||||
for p in (work / "K-audit").glob(f"p{n}-audit-*.json"):
|
||||
m = re.fullmatch(rf"p{n}-audit-(\d+)\.json", p.name)
|
||||
if m:
|
||||
found.append((int(m.group(1)), p))
|
||||
if not found:
|
||||
return None
|
||||
found.sort()
|
||||
return {"audits": len(found), "before": homeless(read_json(found[0][1])), "after": homeless(read_json(found[-1][1]))}
|
||||
|
||||
|
||||
def exits(root: Path, n: int) -> list[int] | None:
|
||||
path = root / "logg" / f"p{n}.txt"
|
||||
if not path.is_file():
|
||||
return None
|
||||
return [int(m.group(1)) for line in path.read_text(encoding="utf-8").splitlines()
|
||||
if (m := EXIT_LINE.match(line.strip()))]
|
||||
|
||||
|
||||
def link_pairs(work: Path, acc: dict[str, Any]) -> tuple[dict[tuple[str, str], int], int]:
|
||||
"""(main page id, linked page id) -> the first pack that placed the link, from the link records and the homes;
|
||||
and the number of link records no link pass asked for."""
|
||||
asked: dict[tuple[str, str], int] = {}
|
||||
for path in sorted((work / "K-links").rglob("sentences.json")) if (work / "K-links").is_dir() else []:
|
||||
doc = read_json(path)
|
||||
for rel, keys in doc.get("pages", {}).items():
|
||||
for key in keys:
|
||||
at = (str(rel), str(key))
|
||||
asked[at] = min(asked.get(at, int(doc["pack"])), int(doc["pack"]))
|
||||
pairs: dict[tuple[str, str], int] = {}
|
||||
unattributed = 0
|
||||
|
||||
def add(a: str, b: str, n: int) -> None:
|
||||
pairs[(a, b)] = min(pairs.get((a, b), n), n)
|
||||
|
||||
links_path = work / "links.json"
|
||||
records = read_json(links_path).get("links", []) if links_path.is_file() else []
|
||||
for r in records:
|
||||
n = asked.get((str(r["page"]), str(r["sentence"])))
|
||||
if n is None:
|
||||
unattributed += 1
|
||||
continue
|
||||
add(klink.concept_id(str(r["page"])), klink.concept_id(str(r["target"])), n)
|
||||
for h in acc.get("homes", []):
|
||||
add(str(h["home"]), str(h["concept_id"]), int(h["pack"]))
|
||||
return pairs, unattributed
|
||||
|
||||
|
||||
def degree_stats(per_main: dict[str, int]) -> dict[str, Any]:
|
||||
values = list(per_main.values())
|
||||
return {"main_pages": len(values), "min": min(values, default=None), "max": max(values, default=None),
|
||||
"median": statistics.median(values) if values else None, "per_main": dict(sorted(per_main.items()))}
|
||||
|
||||
|
||||
def wiki_out_degree(wiki: Path) -> dict[str, Any] | None:
|
||||
if not wiki.is_dir():
|
||||
return None
|
||||
b = audit.load_bundle(wiki, wiki)
|
||||
mains = set()
|
||||
for d in b.concepts:
|
||||
value = d.frontmatter.get(audit.MAIN_FOR) if d.frontmatter else None
|
||||
if isinstance(value, list) and value:
|
||||
mains.add(d.rel)
|
||||
linked = audit.main_page_links(b, mains)
|
||||
return degree_stats({klink.concept_id(m): sum(m in held for held in linked.values()) for m in mains})
|
||||
|
||||
|
||||
def metrics(root: Path) -> dict[str, Any]:
|
||||
work = root / "work"
|
||||
blocks_doc = read_json(work / "blocks.json")
|
||||
log = read_json(work / "K-work" / "packs.json")
|
||||
index = read_json(work / "K-work" / "index.json")
|
||||
acc = read_json(work / "K-work" / "accepted.json")
|
||||
kost = read_json(root / "kost.json")
|
||||
records = sorted(log["packs"], key=lambda r: int(r["pack"]))
|
||||
done = [r for r in records if "accepted" in r]
|
||||
numbers = [int(r["pack"]) for r in done]
|
||||
costs, line_totals = pack_costs(kost, numbers)
|
||||
pairs, unattributed = link_pairs(work, acc)
|
||||
total_blocks = len(blocks_doc["blocks"])
|
||||
|
||||
packs: list[dict[str, Any]] = []
|
||||
files: set[str] = set()
|
||||
read = 0
|
||||
so_far = Fraction(0)
|
||||
points: list[tuple[float, int, int]] = []
|
||||
catalogue_points: list[tuple[float, float]] = []
|
||||
for r in done:
|
||||
n, blocks = int(r["pack"]), int(r["blocks"])
|
||||
files |= {str(s) for s in r.get("sources", [])}
|
||||
c = costs[n]
|
||||
row: dict[str, Any] = {"pack": n, "sources": list(r.get("sources", [])), "blocks": blocks,
|
||||
"blocks_so_far": read + blocks, "runs": {"build": c["build_runs"],
|
||||
"link": c["link_runs"]},
|
||||
"retries": len(r.get("attempts", [])), "guard_rounds": int("guard" in r),
|
||||
"exits": exits(root, n)}
|
||||
if c["total"] is None:
|
||||
row["cost"] = None
|
||||
else:
|
||||
so_far += c["total"]
|
||||
row["cost"] = {"build": c["build"], "link": c["link"],
|
||||
"orchestration": float(c["total"] - c["build"] - c["link"]), "total": float(c["total"]),
|
||||
"so_far": float(so_far), "per_block": float(c["total"] / blocks) if blocks else None}
|
||||
points.append((float(c["total"]), blocks, read))
|
||||
pages = {str(f["concept_id"]) for f in acc["fragments"] if int(f["pack"]) <= n}
|
||||
mains = [m for m in kpack.main_ids({**acc, "main_pages": [m for m in acc.get("main_pages", [])
|
||||
if int(m["pack"]) <= n]}) if m in pages]
|
||||
row["index_concepts"] = sum(1 for e in index["entries"]
|
||||
if e.get("kind") == "skeleton" or str(e.get("origin", {}).get("file")) in files)
|
||||
row["pages"], row["main_pages"] = len(pages), len(mains)
|
||||
cat = catalogue_of(task_parts(work, n))
|
||||
row["catalogue"] = None if cat is None else {"lines": cat[0], "chars": cat[1], "limit": klink.CATALOGUE_CHARS}
|
||||
if cat is not None:
|
||||
catalogue_points.append((float(read), float(cat[1])))
|
||||
groups = duplicate_groups(index, pages)
|
||||
row["duplicates"] = {"groups": groups, "count": len(groups), "pages": sum(len(g) for g in groups)}
|
||||
row["homeless"] = audits(work, n)
|
||||
row["out_degree"] = degree_stats({m: sum(1 for (a, b), p in pairs.items()
|
||||
if a == m and p <= n and b in pages and b != m) for m in mains})
|
||||
packs.append(row)
|
||||
read += blocks
|
||||
|
||||
per_source: dict[str, dict[str, Any]] = {}
|
||||
lists = {int(r["pack"]): [str(s) for s in r.get("sources", [])] for r in records}
|
||||
ordered = [lists.get(i, []) for i in range(1, max(lists, default=0) + 1)]
|
||||
builds = cost.allocate({n: float(costs[n]["build"]) for n in numbers}, ordered, blocks_doc)
|
||||
totals = cost.allocate({n: float(costs[n]["total"]) for n in numbers if costs[n]["total"] is not None}, ordered,
|
||||
blocks_doc)
|
||||
count: dict[str, int] = {}
|
||||
for b in blocks_doc["blocks"]:
|
||||
count[str(b["file"])] = count.get(str(b["file"]), 0) + 1
|
||||
for n in numbers:
|
||||
for s in lists[n]:
|
||||
per_source[s] = {"pack": n, "blocks": count.get(s, 0), "build": builds.get(s), "total": totals.get(s)}
|
||||
|
||||
fit: dict[str, Any] = {"cost_per_block": cost.project(points, total_blocks), "catalogue_chars": None}
|
||||
if catalogue_points:
|
||||
a, b = cost.fit_line(catalogue_points)
|
||||
fit["catalogue_chars"] = {"a": a, "b": b, "points": len(catalogue_points), "limit": klink.CATALOGUE_CHARS,
|
||||
"reaches_limit_at_blocks_read": (klink.CATALOGUE_CHARS - a) / b if b > 0 else None}
|
||||
return {"schema": SCHEMA, "packs": packs, "sources": per_source, "fit": fit,
|
||||
"pending": [int(r["pack"]) for r in records if "accepted" not in r],
|
||||
"totals": {"blocks": total_blocks, "blocks_done": read, "cost": line_totals},
|
||||
"out_degree_wiki": wiki_out_degree(root / "wiki"), "links_unattributed": unattributed}
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
p = argparse.ArgumentParser(prog="python3 tools/part_metrics.py", description=__doc__,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
p.add_argument("root")
|
||||
p.add_argument("--out", required=True)
|
||||
args = p.parse_args(argv)
|
||||
root, out = Path(args.root), Path(args.out)
|
||||
try:
|
||||
doc = metrics(root)
|
||||
except (RunError, OSError, KeyError, TypeError, ValueError) as exc:
|
||||
print(f"part_metrics cannot run: {exc}", file=sys.stderr)
|
||||
return 2
|
||||
tmp = out.with_name(f".{out.name}.tmp")
|
||||
tmp.write_text(json.dumps(doc, indent=1, sort_keys=True, ensure_ascii=False) + "\n", encoding="utf-8")
|
||||
os.replace(tmp, out)
|
||||
print(f"part_metrics: {len(doc['packs'])} packs, {doc['totals']['blocks_done']} of {doc['totals']['blocks']} "
|
||||
f"blocks, measured {doc['totals']['cost']['measured']} tokens -> {out}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
1153
tools/upstream_watch.py
Normal file
1153
tools/upstream_watch.py
Normal file
File diff suppressed because it is too large
Load diff
173
tools/x_fields_check.py
Normal file
173
tools/x_fields_check.py
Normal file
|
|
@ -0,0 +1,173 @@
|
|||
"""The x- field check (format choice g, PLAN.md criterion 5): every x- field on a page has a row with a measured
|
||||
reason in docs/x-fields.md. It calls no model and reads no network.
|
||||
|
||||
Usage:
|
||||
python3 tools/x_fields_check.py BUNDLE [--table docs/x-fields.md]
|
||||
|
||||
The table is the first markdown table in TABLE whose header begins "| Field |", with eight columns: Field, Where
|
||||
(`page` or `sources[]`), Status (`v1` or `not in v1`), Written by, Read by, Meaning, Measured reason, and Standard
|
||||
field if Google adopts one. A row fails when it has another number of cells, when its field is not an x- name, when
|
||||
its Where or Status is not one of those values, when a field has two rows, and when its reason is empty or holds no
|
||||
number besides references (section, issue, point, criterion, step, wave and order numbers, names such as K2 or SC5,
|
||||
dates, commit ids): a measured reason states the measurement and its number.
|
||||
|
||||
Then every concept page of BUNDLE (every .md except index.md and log.md, read as the auditor reads them): each x- key
|
||||
at the top of the frontmatter and each x- key on a sources[] entry must have a row with that place and status v1. A
|
||||
field whose row says "not in v1" fails on a page (x-outdated, CLAUDE.md). A page whose frontmatter does not parse
|
||||
cannot be checked and fails; so does a bundle with no concept page.
|
||||
|
||||
Prints the counts, one line per problem (a field missing from many pages is one line) and a last line PASS or FAIL.
|
||||
Exit 0 = PASS, 1 = FAIL, 2 = could not run (no bundle directory, no table file, or no table in it).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import re
|
||||
import sys
|
||||
from collections import Counter
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
|
||||
from okfwiki.audit import load_bundle # noqa: E402
|
||||
|
||||
DEFAULT_TABLE = Path(__file__).resolve().parents[1] / "docs" / "x-fields.md"
|
||||
COLUMNS = 8
|
||||
PLACES = ("page", "sources[]")
|
||||
STATUSES = ("v1", "not in v1")
|
||||
NAME = re.compile(r"x-[a-z][a-z0-9-]*")
|
||||
# Numbers that refer to something rather than measure it: section, issue and point numbers, names such as K2, SC5
|
||||
# or L4, dates and commit ids.
|
||||
REFERENCE = re.compile(r"§\s*\d+(?:\.\d+)*|#\d+|\b[A-Z]{1,3}\d+[a-z]?\b|\b(?:points?|criteri(?:on|a)|steps?|orders?|waves?|rule)\s+\d+"
|
||||
r"(?:\s*(?:,|and|-)\s*\d+)*|\b\d{1,2}\.\d{1,2}\.\d{4}\b|\b\d{4}-\d{2}-\d{2}\b"
|
||||
r"|\b(?=[0-9a-f]*[a-f])(?=[0-9a-f]*\d)[0-9a-f]{7,40}\b", re.IGNORECASE)
|
||||
|
||||
|
||||
class RunError(Exception):
|
||||
"""The check cannot run; exit 2."""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Row:
|
||||
field: str
|
||||
where: str
|
||||
status: str
|
||||
written: str
|
||||
read: str
|
||||
meaning: str
|
||||
reason: str
|
||||
standard: str
|
||||
|
||||
|
||||
def cells(line: str) -> list[str]:
|
||||
return [c.strip() for c in line.strip().removeprefix("|").removesuffix("|").split("|")]
|
||||
|
||||
|
||||
def measured(reason: str) -> bool:
|
||||
"""A reason is measured when it states a number that is not a reference (a section, issue or point number, a
|
||||
date or a commit id)."""
|
||||
return re.search(r"\d", REFERENCE.sub(" ", reason)) is not None
|
||||
|
||||
|
||||
def read_table(path: Path) -> tuple[dict[str, Row], list[str]]:
|
||||
"""(the rows by field, one problem per bad row). RunError when the file or its table is missing."""
|
||||
try:
|
||||
lines = path.read_text(encoding="utf-8").splitlines()
|
||||
except (OSError, UnicodeDecodeError) as exc:
|
||||
raise RunError(f"cannot read the table file {path}: {exc}") from exc
|
||||
start = next((i for i, line in enumerate(lines) if line.strip().startswith("| Field |")), None)
|
||||
if start is None:
|
||||
raise RunError(f"{path} has no table whose header begins | Field |")
|
||||
rows: dict[str, Row] = {}
|
||||
problems: list[str] = []
|
||||
for line in lines[start + 2:]:
|
||||
if not line.strip().startswith("|"):
|
||||
break
|
||||
c = cells(line)
|
||||
if len(c) != COLUMNS:
|
||||
problems.append(f"table: a row has {len(c)} cells, not {COLUMNS}: {line.strip()[:80]}")
|
||||
continue
|
||||
row = Row(c[0].strip("`"), *c[1:])
|
||||
if not NAME.fullmatch(row.field):
|
||||
problems.append(f"table: {c[0]!r} is not an x- field name")
|
||||
continue
|
||||
if row.field in rows:
|
||||
problems.append(f"table: {row.field} has two rows")
|
||||
continue
|
||||
if row.where not in PLACES or row.status not in STATUSES:
|
||||
problems.append(f"table: {row.field}: Where must be one of {PLACES} and Status one of {STATUSES}")
|
||||
if not measured(row.reason):
|
||||
problems.append(f"table: {row.field} has no measured reason (the reason is empty or states no number)")
|
||||
rows[row.field] = row # a bad row still names its field, so its pages are not reported a second time
|
||||
return rows, problems
|
||||
|
||||
|
||||
def page_fields(bundle: Path) -> tuple[int, Counter[tuple[str, str]], dict[tuple[str, str], list[str]], list[str]]:
|
||||
"""(concept pages, count per (field, place), the pages per (field, place), pages that cannot be read)."""
|
||||
counts: Counter[tuple[str, str]] = Counter()
|
||||
pages: dict[tuple[str, str], list[str]] = {}
|
||||
unreadable: list[str] = []
|
||||
concepts = load_bundle(bundle, bundle).concepts
|
||||
for d in concepts:
|
||||
if d.frontmatter is None:
|
||||
unreadable.append(d.rel)
|
||||
continue
|
||||
found = [(str(k), "page") for k in d.frontmatter if str(k).startswith("x-")]
|
||||
srcs = d.frontmatter.get("sources")
|
||||
for s in srcs if isinstance(srcs, list) else [srcs]:
|
||||
if isinstance(s, dict):
|
||||
found += [(str(k), "sources[]") for k in s if str(k).startswith("x-")]
|
||||
for key in found:
|
||||
counts[key] += 1
|
||||
if d.rel not in pages.setdefault(key, []):
|
||||
pages[key].append(d.rel)
|
||||
return len(concepts), counts, pages, unreadable
|
||||
|
||||
|
||||
def check(bundle: Path, table: Path) -> tuple[list[str], list[str]]:
|
||||
"""(the count lines, the problems)."""
|
||||
if not bundle.is_dir():
|
||||
raise RunError(f"the bundle {bundle} is not a directory")
|
||||
rows, problems = read_table(table)
|
||||
n, counts, pages, unreadable = page_fields(bundle)
|
||||
info = [f"table: {len(rows)} rows in {table}",
|
||||
f"pages: {n} concept pages; x- fields: "
|
||||
+ (", ".join(f"{f} ({p}) {k}" for (f, p), k in sorted(counts.items())) or "none")]
|
||||
if n == 0:
|
||||
problems.append("pages: the bundle has no concept page, so nothing was checked")
|
||||
problems += [f"{rel}: the frontmatter does not parse, so its x- fields cannot be checked" for rel in unreadable]
|
||||
for (name, place), on in sorted(pages.items()):
|
||||
row = rows.get(name)
|
||||
where = f"{name} ({place}) on {len(on)} page(s), first {on[0]}"
|
||||
if row is None:
|
||||
problems.append(f"{where}: no row in the table")
|
||||
elif row.where != place:
|
||||
problems.append(f"{where}: the table says it stands at {row.where}")
|
||||
elif row.status != "v1":
|
||||
problems.append(f"{where}: the table says {row.status}")
|
||||
return info, problems
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
p.add_argument("bundle")
|
||||
p.add_argument("--table", default=str(DEFAULT_TABLE))
|
||||
args = p.parse_args(argv)
|
||||
try:
|
||||
info, problems = check(Path(args.bundle), Path(args.table))
|
||||
except RunError as exc:
|
||||
print(f"x-fields cannot run: {exc}", file=sys.stderr)
|
||||
return 2
|
||||
for line in info + problems:
|
||||
print(line)
|
||||
if problems:
|
||||
print(f"FAIL x- fields: {len(problems)} problem(s)")
|
||||
return 1
|
||||
print("PASS x- fields: every x- field on every concept page has a row with a measured reason")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Loading…
Add table
Add a link
Reference in a new issue