okf-wiki/tools/x_fields_check.py
Kjell Tore Guttormsen 0fd86a23a3
okf-wiki 1.1.1
First public release, published as one snapshot. What changed between
versions is in CHANGELOG.md.

Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
2026-11-14 21:03:20 +01:00

173 lines
7.6 KiB
Python

"""The x- field check (format choice g, PLAN.md criterion 5): every x- field on a page has a row with a measured
reason in docs/x-fields.md. It calls no model and reads no network.
Usage:
python3 tools/x_fields_check.py BUNDLE [--table docs/x-fields.md]
The table is the first markdown table in TABLE whose header begins "| Field |", with eight columns: Field, Where
(`page` or `sources[]`), Status (`v1` or `not in v1`), Written by, Read by, Meaning, Measured reason, and Standard
field if Google adopts one. A row fails when it has another number of cells, when its field is not an x- name, when
its Where or Status is not one of those values, when a field has two rows, and when its reason is empty or holds no
number besides references (section, issue, point, criterion, step, wave and order numbers, names such as K2 or SC5,
dates, commit ids): a measured reason states the measurement and its number.
Then every concept page of BUNDLE (every .md except index.md and log.md, read as the auditor reads them): each x- key
at the top of the frontmatter and each x- key on a sources[] entry must have a row with that place and status v1. A
field whose row says "not in v1" fails on a page (x-outdated, CLAUDE.md). A page whose frontmatter does not parse
cannot be checked and fails; so does a bundle with no concept page.
Prints the counts, one line per problem (a field missing from many pages is one line) and a last line PASS or FAIL.
Exit 0 = PASS, 1 = FAIL, 2 = could not run (no bundle directory, no table file, or no table in it).
"""
from __future__ import annotations
import argparse
import re
import sys
from collections import Counter
from dataclasses import dataclass
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from okfwiki.audit import load_bundle # noqa: E402
DEFAULT_TABLE = Path(__file__).resolve().parents[1] / "docs" / "x-fields.md"
COLUMNS = 8
PLACES = ("page", "sources[]")
STATUSES = ("v1", "not in v1")
NAME = re.compile(r"x-[a-z][a-z0-9-]*")
# Numbers that refer to something rather than measure it: section, issue and point numbers, names such as K2, SC5
# or L4, dates and commit ids.
REFERENCE = re.compile(r"§\s*\d+(?:\.\d+)*|#\d+|\b[A-Z]{1,3}\d+[a-z]?\b|\b(?:points?|criteri(?:on|a)|steps?|orders?|waves?|rule)\s+\d+"
r"(?:\s*(?:,|and|-)\s*\d+)*|\b\d{1,2}\.\d{1,2}\.\d{4}\b|\b\d{4}-\d{2}-\d{2}\b"
r"|\b(?=[0-9a-f]*[a-f])(?=[0-9a-f]*\d)[0-9a-f]{7,40}\b", re.IGNORECASE)
class RunError(Exception):
"""The check cannot run; exit 2."""
@dataclass(frozen=True)
class Row:
field: str
where: str
status: str
written: str
read: str
meaning: str
reason: str
standard: str
def cells(line: str) -> list[str]:
return [c.strip() for c in line.strip().removeprefix("|").removesuffix("|").split("|")]
def measured(reason: str) -> bool:
"""A reason is measured when it states a number that is not a reference (a section, issue or point number, a
date or a commit id)."""
return re.search(r"\d", REFERENCE.sub(" ", reason)) is not None
def read_table(path: Path) -> tuple[dict[str, Row], list[str]]:
"""(the rows by field, one problem per bad row). RunError when the file or its table is missing."""
try:
lines = path.read_text(encoding="utf-8").splitlines()
except (OSError, UnicodeDecodeError) as exc:
raise RunError(f"cannot read the table file {path}: {exc}") from exc
start = next((i for i, line in enumerate(lines) if line.strip().startswith("| Field |")), None)
if start is None:
raise RunError(f"{path} has no table whose header begins | Field |")
rows: dict[str, Row] = {}
problems: list[str] = []
for line in lines[start + 2:]:
if not line.strip().startswith("|"):
break
c = cells(line)
if len(c) != COLUMNS:
problems.append(f"table: a row has {len(c)} cells, not {COLUMNS}: {line.strip()[:80]}")
continue
row = Row(c[0].strip("`"), *c[1:])
if not NAME.fullmatch(row.field):
problems.append(f"table: {c[0]!r} is not an x- field name")
continue
if row.field in rows:
problems.append(f"table: {row.field} has two rows")
continue
if row.where not in PLACES or row.status not in STATUSES:
problems.append(f"table: {row.field}: Where must be one of {PLACES} and Status one of {STATUSES}")
if not measured(row.reason):
problems.append(f"table: {row.field} has no measured reason (the reason is empty or states no number)")
rows[row.field] = row # a bad row still names its field, so its pages are not reported a second time
return rows, problems
def page_fields(bundle: Path) -> tuple[int, Counter[tuple[str, str]], dict[tuple[str, str], list[str]], list[str]]:
"""(concept pages, count per (field, place), the pages per (field, place), pages that cannot be read)."""
counts: Counter[tuple[str, str]] = Counter()
pages: dict[tuple[str, str], list[str]] = {}
unreadable: list[str] = []
concepts = load_bundle(bundle, bundle).concepts
for d in concepts:
if d.frontmatter is None:
unreadable.append(d.rel)
continue
found = [(str(k), "page") for k in d.frontmatter if str(k).startswith("x-")]
srcs = d.frontmatter.get("sources")
for s in srcs if isinstance(srcs, list) else [srcs]:
if isinstance(s, dict):
found += [(str(k), "sources[]") for k in s if str(k).startswith("x-")]
for key in found:
counts[key] += 1
if d.rel not in pages.setdefault(key, []):
pages[key].append(d.rel)
return len(concepts), counts, pages, unreadable
def check(bundle: Path, table: Path) -> tuple[list[str], list[str]]:
"""(the count lines, the problems)."""
if not bundle.is_dir():
raise RunError(f"the bundle {bundle} is not a directory")
rows, problems = read_table(table)
n, counts, pages, unreadable = page_fields(bundle)
info = [f"table: {len(rows)} rows in {table}",
f"pages: {n} concept pages; x- fields: "
+ (", ".join(f"{f} ({p}) {k}" for (f, p), k in sorted(counts.items())) or "none")]
if n == 0:
problems.append("pages: the bundle has no concept page, so nothing was checked")
problems += [f"{rel}: the frontmatter does not parse, so its x- fields cannot be checked" for rel in unreadable]
for (name, place), on in sorted(pages.items()):
row = rows.get(name)
where = f"{name} ({place}) on {len(on)} page(s), first {on[0]}"
if row is None:
problems.append(f"{where}: no row in the table")
elif row.where != place:
problems.append(f"{where}: the table says it stands at {row.where}")
elif row.status != "v1":
problems.append(f"{where}: the table says {row.status}")
return info, problems
def main(argv: list[str] | None = None) -> int:
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
p.add_argument("bundle")
p.add_argument("--table", default=str(DEFAULT_TABLE))
args = p.parse_args(argv)
try:
info, problems = check(Path(args.bundle), Path(args.table))
except RunError as exc:
print(f"x-fields cannot run: {exc}", file=sys.stderr)
return 2
for line in info + problems:
print(line)
if problems:
print(f"FAIL x- fields: {len(problems)} problem(s)")
return 1
print("PASS x- fields: every x- field on every concept page has a row with a measured reason")
return 0
if __name__ == "__main__":
sys.exit(main())