"""K's sources in a fixed order, split into blocks with line spans. No model call (CLAUDE.md rule 1). Usage: python3 -m okfwiki.kblocks order SOURCES --out FILE python3 -m okfwiki.kblocks blocks SOURCES --order FILE --out FILE python3 -m okfwiki.kblocks total SOURCES SOURCES is the folder that holds kilde/, the same raw-source root the auditor takes. order every file under SOURCES/kilde/ in K's fixed order, one name per line: pages by name, then release notes (release-vX.Y.Z.md) by version. Any other name is a page: a docs page (docs-*), or a document of a folder (sources folder), .md or .txt alike. blocks every block of every source in the order (schema okf-wiki/blocks/1): a paragraph, one list item with its continuation lines, one table or one fenced code block, with its 1-based inclusive line span, its section (the heading path), its characters (`chars`, every line with its newline) and, for a release note, its version. Heading lines are section markers, not blocks, and a line that is only an HTML or MDX tag (, ) never starts a block, so no block is only a tag. A "#" line inside a component property's example string (example: `# Title ... `) is example text, not a heading. total the number of blocks in every file under SOURCES/kilde/. Lines are counted on the text split at "\\n", as the pack and the link pass count them. Exit 0 = done, 2 = could not run; an existing output is never overwritten. """ from __future__ import annotations import argparse import json import re import sys from dataclasses import asdict, dataclass from pathlib import Path from typing import Any from okfwiki.audit import HEADING, LIST_ITEM, closes_fence, fence_marker from okfwiki.common import HTML_TAG_ONLY, VERSION_HEADING, RunError, write_new SCHEMA = "okf-wiki/blocks/1" RELEASE = re.compile(r"^release-v(\d+(?:\.\d+)*)\.md$") VERSION_LABEL = re.compile(r'^\s*<[A-Za-z][A-Za-z0-9._-]*\s[^<>]*?\blabel="v?(\d+(?:\.\d+){2,})"') VERSION_BOLD = re.compile(r"^\*\*v?(\d+(?:\.\d+){2,})\*\*$") # A component property whose value is a backtick string left open on its line (example: `# Title): the lines after it, # up to the first with a backtick no backslash escapes, are example text, never a heading (order 14 findings C and D). STRING_OPEN = re.compile(r"^\s*[A-Za-z_$][\w$]*\s*:\s*`") TICK = re.compile(r"(? bool: return title.strip().lower() in META_STOP def version_key(name: str) -> tuple[int, ...]: m = RELEASE.match(name) if m is None: raise RunError(f"{name} is not a release note name") return tuple(int(p) for p in m.group(1).split(".")) def order(names: list[str]) -> list[str]: releases = [n for n in names if RELEASE.match(n)] return sorted(n for n in names if not RELEASE.match(n)) + sorted(releases, key=version_key) def source_files(kilde: Path) -> list[Path]: """Every file under kilde, in name order: .md and .txt alike, nothing dropped by its suffix.""" return sorted(p for p in kilde.iterdir() if p.is_file()) def read_lines(path: Path) -> list[str]: return path.read_text(encoding="utf-8").split("\n") def string_lines(lines: list[str]) -> set[int]: """The 1-based lines inside a component property's example string (STRING_OPEN), outside fenced code: from the line after the one that opens it to the line with the backtick that closes it, both included.""" out: set[int] = set() fence = "" # the open fence's marker run, or "" outside fenced code (the auditor's rule, CommonMark 4.5) inside = False for n, raw in enumerate(lines, 1): if inside: out.add(n) inside = TICK.search(raw) is None continue f = fence_marker(raw) if fence: if closes_fence(f, raw, fence): fence = "" elif f: fence = f.group(1) elif STRING_OPEN.match(raw): inside = len(TICK.findall(raw)) % 2 == 1 return out def headings_of(lines: list[str]) -> list[tuple[int, int, str]]: """(line, level, text) of every heading outside fenced code and outside a component's example string.""" out: list[tuple[int, int, str]] = [] fence = "" text_of_string = string_lines(lines) for n, raw in enumerate(lines, 1): if n in text_of_string: continue f = fence_marker(raw) if fence: if closes_fence(f, raw, fence): fence = "" continue if f: fence = f.group(1) continue m = HEADING.match(raw) if m: out.append((n, len(m.group(1)), (m.group(2) or "").strip())) return out def stated_versions(lines: list[str]) -> list[tuple[int, str]]: """(line, version) for every line that states a release note's version: a heading that is only the version, then a component whose label is the version (: every release note of the copy of 08.10, 416 of 416, has this form and no heading; order 27 A6), then a line that is only the bold version (**X.Y.Z**: the page's form after 20.10, 437 of 437 on 07.11).""" found = [(n, m.group(1)) for n, _level, text in headings_of(lines) if (m := VERSION_HEADING.match(text))] found += [(n, m.group(1)) for n, raw in enumerate(lines, 1) if (m := VERSION_LABEL.match(raw))] return found + [(n, m.group(1)) for n, raw in enumerate(lines, 1) if (m := VERSION_BOLD.match(raw))] def version_line(lines: list[str], version: str) -> int | None: """The first line that states `version`, or None.""" return next((n for n, v in stated_versions(lines) if v == version), None) def blocks(path: Path) -> list[Block]: lines = read_lines(path) name = path.name version: str | None = None if RELEASE.match(name): stated = stated_versions(lines) version = stated[0][1] if stated else None out: list[Block] = [] stack: list[tuple[int, str]] = [] kind: str | None = None start = 0 fence = "" text_of_string = string_lines(lines) def close(end: int) -> None: nonlocal kind if kind is not None: out.append(Block(name, start, end, kind, " › ".join(t for _, t in stack), version, sum(len(line) + 1 for line in lines[start - 1:end]))) kind = None for n, raw in enumerate(lines, 1): f = fence_marker(raw) if fence: if closes_fence(f, raw, fence): fence = "" close(n) continue if f: close(n - 1) fence, kind, start = f.group(1), "code", n continue if not raw.strip(): close(n - 1) continue m = None if n in text_of_string else HEADING.match(raw) if m: close(n - 1) level = len(m.group(1)) stack = [(lv, t) for lv, t in stack if lv < level] + [(level, (m.group(2) or "").strip())] continue if raw.lstrip().startswith("|"): if kind != "table": close(n - 1) kind, start = "table", n continue if LIST_ITEM.match(raw): close(n - 1) kind, start = "item", n continue if kind in (None, "table"): close(n - 1) if HTML_TAG_ONLY.match(raw): continue # a tag alone is layout; the block starts at the first line of content kind, start = "paragraph", n if fence is not None: close(len(lines)) else: last = len(lines) while last > 0 and not lines[last - 1].strip(): last -= 1 close(last) return out def docs_headings(kilde: Path) -> list[Heading]: """The title and level 2-3 headings of every page under kilde (any file but a release note), in name order.""" out: list[Heading] = [] for path in (p for p in source_files(kilde) if not RELEASE.match(p.name)): for n, level, text in headings_of(read_lines(path)): if level <= 3 and text: out.append(Heading(path.name, n, level, text)) return out def blocks_document(kilde: Path, names: list[str]) -> dict[str, Any]: found: list[dict[str, Any]] = [] for name in names: path = kilde / name if not path.is_file(): raise RunError(f"{path} is not a file") found.extend(asdict(b) for b in blocks(path)) return {"schema": SCHEMA, "order": names, "blocks": found} def kilde_of(sources: Path) -> Path: kilde = sources / "kilde" if not kilde.is_dir(): raise RunError(f"{kilde} is not a directory") return kilde def build_parser() -> argparse.ArgumentParser: p = argparse.ArgumentParser(prog="python3 -m okfwiki.kblocks", description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) sub = p.add_subparsers(dest="command", required=True) o = sub.add_parser("order") o.add_argument("sources") o.add_argument("--out", required=True) b = sub.add_parser("blocks") b.add_argument("sources") b.add_argument("--order", required=True) b.add_argument("--out", required=True) t = sub.add_parser("total") t.add_argument("sources") return p def main(argv: list[str] | None = None) -> int: args = build_parser().parse_args(argv) try: kilde = kilde_of(Path(args.sources)) if args.command == "order": fixed = order([p.name for p in source_files(kilde)]) write_new(Path(args.out), "".join(n + "\n" for n in fixed)) print(f"order: {len(fixed)} sources -> {args.out}") elif args.command == "blocks": names = Path(args.order).read_text(encoding="utf-8").split() if Path(args.out).exists(): raise RunError(f"{args.out} exists; refusing to overwrite") doc = blocks_document(kilde, names) write_new(Path(args.out), json.dumps(doc, ensure_ascii=False, indent=1) + "\n") print(f"blocks: {len(doc['blocks'])} in {len(names)} sources -> {args.out}") else: files = source_files(kilde) print(f"total: {sum(len(blocks(p)) for p in files)} blocks in {len(files)} files of {kilde}") except (RunError, OSError, UnicodeDecodeError) as exc: print(f"kblocks cannot run: {exc}", file=sys.stderr) return 2 return 0 if __name__ == "__main__": sys.exit(main())