"""The x- field check (format choice g, PLAN.md criterion 5): every x- field on a page has a row with a measured reason in docs/x-fields.md. It calls no model and reads no network. Usage: python3 tools/x_fields_check.py BUNDLE [--table docs/x-fields.md] The table is the first markdown table in TABLE whose header begins "| Field |", with eight columns: Field, Where (`page` or `sources[]`), Status (`v1` or `not in v1`), Written by, Read by, Meaning, Measured reason, and Standard field if Google adopts one. A row fails when it has another number of cells, when its field is not an x- name, when its Where or Status is not one of those values, when a field has two rows, and when its reason is empty or holds no number besides references (section, issue, point, criterion, step, wave and order numbers, names such as K2 or SC5, dates, commit ids): a measured reason states the measurement and its number. Then every concept page of BUNDLE (every .md except index.md and log.md, read as the auditor reads them): each x- key at the top of the frontmatter and each x- key on a sources[] entry must have a row with that place and status v1. A field whose row says "not in v1" fails on a page (x-outdated, CLAUDE.md). A page whose frontmatter does not parse cannot be checked and fails; so does a bundle with no concept page. Prints the counts, one line per problem (a field missing from many pages is one line) and a last line PASS or FAIL. Exit 0 = PASS, 1 = FAIL, 2 = could not run (no bundle directory, no table file, or no table in it). """ from __future__ import annotations import argparse import re import sys from collections import Counter from dataclasses import dataclass from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parents[1])) from okfwiki.audit import load_bundle # noqa: E402 DEFAULT_TABLE = Path(__file__).resolve().parents[1] / "docs" / "x-fields.md" COLUMNS = 8 PLACES = ("page", "sources[]") STATUSES = ("v1", "not in v1") NAME = re.compile(r"x-[a-z][a-z0-9-]*") # Numbers that refer to something rather than measure it: section, issue and point numbers, names such as K2, SC5 # or L4, dates and commit ids. REFERENCE = re.compile(r"ยง\s*\d+(?:\.\d+)*|#\d+|\b[A-Z]{1,3}\d+[a-z]?\b|\b(?:points?|criteri(?:on|a)|steps?|orders?|waves?|rule)\s+\d+" r"(?:\s*(?:,|and|-)\s*\d+)*|\b\d{1,2}\.\d{1,2}\.\d{4}\b|\b\d{4}-\d{2}-\d{2}\b" r"|\b(?=[0-9a-f]*[a-f])(?=[0-9a-f]*\d)[0-9a-f]{7,40}\b", re.IGNORECASE) class RunError(Exception): """The check cannot run; exit 2.""" @dataclass(frozen=True) class Row: field: str where: str status: str written: str read: str meaning: str reason: str standard: str def cells(line: str) -> list[str]: return [c.strip() for c in line.strip().removeprefix("|").removesuffix("|").split("|")] def measured(reason: str) -> bool: """A reason is measured when it states a number that is not a reference (a section, issue or point number, a date or a commit id).""" return re.search(r"\d", REFERENCE.sub(" ", reason)) is not None def read_table(path: Path) -> tuple[dict[str, Row], list[str]]: """(the rows by field, one problem per bad row). RunError when the file or its table is missing.""" try: lines = path.read_text(encoding="utf-8").splitlines() except (OSError, UnicodeDecodeError) as exc: raise RunError(f"cannot read the table file {path}: {exc}") from exc start = next((i for i, line in enumerate(lines) if line.strip().startswith("| Field |")), None) if start is None: raise RunError(f"{path} has no table whose header begins | Field |") rows: dict[str, Row] = {} problems: list[str] = [] for line in lines[start + 2:]: if not line.strip().startswith("|"): break c = cells(line) if len(c) != COLUMNS: problems.append(f"table: a row has {len(c)} cells, not {COLUMNS}: {line.strip()[:80]}") continue row = Row(c[0].strip("`"), *c[1:]) if not NAME.fullmatch(row.field): problems.append(f"table: {c[0]!r} is not an x- field name") continue if row.field in rows: problems.append(f"table: {row.field} has two rows") continue if row.where not in PLACES or row.status not in STATUSES: problems.append(f"table: {row.field}: Where must be one of {PLACES} and Status one of {STATUSES}") if not measured(row.reason): problems.append(f"table: {row.field} has no measured reason (the reason is empty or states no number)") rows[row.field] = row # a bad row still names its field, so its pages are not reported a second time return rows, problems def page_fields(bundle: Path) -> tuple[int, Counter[tuple[str, str]], dict[tuple[str, str], list[str]], list[str]]: """(concept pages, count per (field, place), the pages per (field, place), pages that cannot be read).""" counts: Counter[tuple[str, str]] = Counter() pages: dict[tuple[str, str], list[str]] = {} unreadable: list[str] = [] concepts = load_bundle(bundle, bundle).concepts for d in concepts: if d.frontmatter is None: unreadable.append(d.rel) continue found = [(str(k), "page") for k in d.frontmatter if str(k).startswith("x-")] srcs = d.frontmatter.get("sources") for s in srcs if isinstance(srcs, list) else [srcs]: if isinstance(s, dict): found += [(str(k), "sources[]") for k in s if str(k).startswith("x-")] for key in found: counts[key] += 1 if d.rel not in pages.setdefault(key, []): pages[key].append(d.rel) return len(concepts), counts, pages, unreadable def check(bundle: Path, table: Path) -> tuple[list[str], list[str]]: """(the count lines, the problems).""" if not bundle.is_dir(): raise RunError(f"the bundle {bundle} is not a directory") rows, problems = read_table(table) n, counts, pages, unreadable = page_fields(bundle) info = [f"table: {len(rows)} rows in {table}", f"pages: {n} concept pages; x- fields: " + (", ".join(f"{f} ({p}) {k}" for (f, p), k in sorted(counts.items())) or "none")] if n == 0: problems.append("pages: the bundle has no concept page, so nothing was checked") problems += [f"{rel}: the frontmatter does not parse, so its x- fields cannot be checked" for rel in unreadable] for (name, place), on in sorted(pages.items()): row = rows.get(name) where = f"{name} ({place}) on {len(on)} page(s), first {on[0]}" if row is None: problems.append(f"{where}: no row in the table") elif row.where != place: problems.append(f"{where}: the table says it stands at {row.where}") elif row.status != "v1": problems.append(f"{where}: the table says {row.status}") return info, problems def main(argv: list[str] | None = None) -> int: p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) p.add_argument("bundle") p.add_argument("--table", default=str(DEFAULT_TABLE)) args = p.parse_args(argv) try: info, problems = check(Path(args.bundle), Path(args.table)) except RunError as exc: print(f"x-fields cannot run: {exc}", file=sys.stderr) return 2 for line in info + problems: print(line) if problems: print(f"FAIL x- fields: {len(problems)} problem(s)") return 1 print("PASS x- fields: every x- field on every concept page has a row with a measured reason") return 0 if __name__ == "__main__": sys.exit(main())