"""Split the trial's sources into a fixed order and into blocks with line spans (TRIAL, P1). It calls no model. order --manifest M --l4 L4DIR --out FILE The 19 names of the cut (gates/wholedoc.manifest.json cut.names) in the trial's fixed order: docs pages by name, then release notes by version. Refuses when the names differ from the files the L4 cut cites. blocks --kilde DIR --order FILE --out FILE Every block of every source in the order: a paragraph, one list item with its continuation lines, one table or one fenced code block, with its 1-based inclusive line span, its section (the heading path) and, for a release note, its version. Heading lines are section markers, not blocks. total --kilde DIR The number of blocks in every file under DIR (the denominator of the 599-source projection). pick --blocks FILE --kilde DIR [--focus NAME] --out FILE The three concepts of the side-by-side page, picked from the source text before any build: the focus page's heading with the most blocks, the term in the most release notes, and the term in at least two docs pages and two release notes with the highest document count; a document counts for a term only where the term stands as a term (backticked or a capitalised phrase), never as a word in prose. One line per concept: name, a tab, aliases joined by ;. Exit 0 = written, 2 = could not run (an existing output is never overwritten). """ from __future__ import annotations import argparse import json import re import sys from dataclasses import asdict, dataclass from pathlib import Path from typing import Any from okfwiki.audit import HEADING, LIST_ITEM, read_document SCHEMA = "okf-wiki/p1-blocks/1" RELEASE = re.compile(r"^release-v(\d+(?:\.\d+)*)\.md$") VERSION_HEADING = re.compile(r"^v?(\d+(?:\.\d+){2,})$") FENCE = re.compile(r"^\s*(```|~~~)") # Concept test 2 of the brief (Google's reference agent), plus the generic section names of docs pages. META_STOP = frozenset({ "overview", "introduction", "getting started", "quickstart", "tutorial", "walkthrough", "release notes", "changelog", "roadmap", "faq", "prerequisites", "next steps", "before you begin", "troubleshooting", "see also", "related resources", "learn more", }) CODE_TERM = re.compile(r"`([^`\n]{3,40})`") CAPITAL_PHRASE = re.compile(r"\b[A-Z][\w-]*(?:[ \t]+[A-Z][\w-]*){1,3}\b") SENTENCE_START = re.compile(r"(?:^|[.!?:]\s+|^\s*(?:[-*+]|\d+[.)])\s+)$") class RunError(Exception): """The command cannot run as asked; nothing was written.""" @dataclass(frozen=True) class SBlock: file: str start: int end: int kind: str # paragraph | item | table | code section: str version: str | None @dataclass(frozen=True) class Heading: file: str line: int level: int text: str @dataclass(frozen=True) class Picked: name: str aliases: tuple[str, ...] def is_meta(title: str) -> bool: return title.strip().lower() in META_STOP def version_key(name: str) -> tuple[int, ...]: m = RELEASE.match(name) if m is None: raise RunError(f"{name} is not a release note name") return tuple(int(p) for p in m.group(1).split(".")) def order(names: list[str]) -> list[str]: docs = sorted(n for n in names if n.startswith("docs-")) releases = [n for n in names if n.startswith("release-")] other = [n for n in names if n not in docs and n not in releases] if other: raise RunError(f"names that are neither docs pages nor release notes: {other}") return docs + sorted(releases, key=version_key) def headings_of(lines: list[str]) -> list[tuple[int, int, str]]: """(line, level, text) of every heading outside fenced code.""" out: list[tuple[int, int, str]] = [] fence: str | None = None for n, raw in enumerate(lines, 1): f = FENCE.match(raw) if f: fence = None if fence == f.group(1) else (fence or f.group(1)) continue if fence is None: m = HEADING.match(raw) if m: out.append((n, len(m.group(1)), (m.group(2) or "").strip())) return out def blocks(path: Path) -> list[SBlock]: lines = path.read_text(encoding="utf-8").split("\n") name = path.name version: str | None = None if name.startswith("release-"): for _n, _level, text in headings_of(lines): m = VERSION_HEADING.match(text) if m: version = m.group(1) break out: list[SBlock] = [] stack: list[tuple[int, str]] = [] kind: str | None = None start = 0 fence: str | None = None def close(end: int) -> None: nonlocal kind if kind is not None: out.append(SBlock(name, start, end, kind, " › ".join(t for _, t in stack), version)) kind = None for n, raw in enumerate(lines, 1): f = FENCE.match(raw) if fence is not None: if f and f.group(1) == fence: fence = None close(n) continue if f: close(n - 1) fence, kind, start = f.group(1), "code", n continue if not raw.strip(): close(n - 1) continue m = HEADING.match(raw) if m: close(n - 1) level = len(m.group(1)) stack = [(lv, t) for lv, t in stack if lv < level] + [(level, (m.group(2) or "").strip())] continue if raw.lstrip().startswith("|"): if kind != "table": close(n - 1) kind, start = "table", n continue if LIST_ITEM.match(raw): close(n - 1) kind, start = "item", n continue if kind in (None, "table"): close(n - 1) kind, start = "paragraph", n if fence is not None: close(len(lines)) else: last = len(lines) while last > 0 and not lines[last - 1].strip(): last -= 1 close(last) return out def docs_headings(kilde: Path) -> list[Heading]: out: list[Heading] = [] for path in sorted(kilde.glob("docs-*.md")): for n, level, text in headings_of(path.read_text(encoding="utf-8").split("\n")): if level <= 3 and text: out.append(Heading(path.name, n, level, text)) return out def blocks_document(kilde: Path, names: list[str]) -> dict[str, Any]: found: list[dict[str, Any]] = [] for name in names: path = kilde / name if not path.is_file(): raise RunError(f"{path} is not a file") found.extend(asdict(b) for b in blocks(path)) return {"schema": SCHEMA, "order": names, "blocks": found} def blocks_total(kilde: Path) -> int: return sum(len(blocks(p)) for p in sorted(kilde.glob("*.md"))) def terms(text: str) -> list[str]: """Backticked spans of 3-40 characters and capitalised phrases of 2-4 words, a sentence-initial word dropped.""" found: list[str] = [t for t in (m.group(1).strip() for m in CODE_TERM.finditer(text)) if 3 <= len(t) <= 40] masked = CODE_TERM.sub(lambda m: " " * len(m.group(0)), text) for raw in masked.split("\n"): for m in CAPITAL_PHRASE.finditer(raw): words = m.group(0).split() if SENTENCE_START.search(raw[:m.start()]): words = words[1:] if len(words) >= 2: found.append(" ".join(words)) return list(dict.fromkeys(t for t in found if t)) def occurs(term: str, text: str) -> bool: return re.search(r"(? list[Picked]: names = [str(n) for n in blocks_doc["order"]] if focus not in names: raise RunError(f"the focus page {focus} is not in the order") texts = {n: (kilde / n).read_text(encoding="utf-8") for n in names} per_section: dict[str, int] = {} for b in blocks_doc["blocks"]: if b["file"] == focus and b["section"].count(" › ") >= 1: per_section[b["section"]] = per_section.get(b["section"], 0) + 1 candidates = [(count, section.split(" › ")[-1]) for section, count in per_section.items() if not is_meta(section.split(" › ")[-1])] if not candidates: raise RunError(f"{focus} has no heading below its title that passes test 2") first = sorted(candidates, key=lambda c: (-c[0], c[1]))[0][1] pool = list(dict.fromkeys(t for n in names for t in terms(texts[n]))) docs = [n for n in names if n.startswith("docs-")] releases = [n for n in names if n.startswith("release-")] as_terms = {n: set(terms(texts[n])) for n in names} def count(term: str, files: list[str]) -> int: return sum(term in as_terms[f] for f in files) # where it stands as a term, not as a word in prose taken = {first.lower()} second_pool = [t for t in pool if t.lower() not in taken] if not second_pool: raise RunError("no term for the second concept") second = sorted(second_pool, key=lambda t: (-count(t, releases), t))[0] taken.add(second.lower()) third_pool = [t for t in pool if t.lower() not in taken and count(t, docs) >= 2 and count(t, releases) >= 2] if not third_pool: raise RunError("no term in two docs pages and two release notes for the third concept") third = sorted(third_pool, key=lambda t: (-count(t, names), t))[0] return [Picked(first, (first,)), Picked(second, (second,)), Picked(third, (third,))] def _write_new(path: Path, text: str) -> None: if path.exists(): raise RunError(f"{path} exists; refusing to overwrite") path.parent.mkdir(parents=True, exist_ok=True) path.write_text(text, encoding="utf-8") def l4_cited(l4: Path) -> set[str]: cited: set[str] = set() for page in sorted(l4.glob("*.md")): fm = read_document(page, page.name).frontmatter or {} srcs = fm.get("sources") for s in srcs if isinstance(srcs, list) else []: res = s.get("resource") if isinstance(s, dict) else None if isinstance(res, str) and res: cited.add(Path(res).name) return cited def build_parser() -> argparse.ArgumentParser: p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) sub = p.add_subparsers(dest="command", required=True) o = sub.add_parser("order") o.add_argument("--manifest", required=True) o.add_argument("--l4", required=True) o.add_argument("--out", required=True) b = sub.add_parser("blocks") b.add_argument("--kilde", required=True) b.add_argument("--order", required=True) b.add_argument("--out", required=True) t = sub.add_parser("total") t.add_argument("--kilde", required=True) k = sub.add_parser("pick") k.add_argument("--blocks", required=True) k.add_argument("--kilde", required=True) k.add_argument("--focus", default="docs-en-permission-modes.md") k.add_argument("--out", required=True) return p def main(argv: list[str] | None = None) -> int: args = build_parser().parse_args(argv) try: if args.command == "order": manifest = json.loads(Path(args.manifest).read_text(encoding="utf-8")) names = manifest.get("cut", {}).get("names") if not isinstance(names, list) or not names: raise RunError(f"{args.manifest} records no cut names") cited = l4_cited(Path(args.l4)) if set(map(str, names)) != cited: raise RunError(f"the manifest's {len(names)} names differ from the {len(cited)} files the L4 cut cites: " f"only in manifest {sorted(set(names) - cited)}, only in L4 {sorted(cited - set(names))}") fixed = order([str(n) for n in names]) _write_new(Path(args.out), "\n".join(fixed) + "\n") print(f"order: {len(fixed)} sources -> {args.out}") elif args.command == "blocks": names = Path(args.order).read_text(encoding="utf-8").split() doc = blocks_document(Path(args.kilde), names) _write_new(Path(args.out), json.dumps(doc, ensure_ascii=False, indent=1) + "\n") print(f"blocks: {len(doc['blocks'])} in {len(names)} sources -> {args.out}") elif args.command == "total": print(f"blocks in every file of {args.kilde}: {blocks_total(Path(args.kilde))}") else: doc = json.loads(Path(args.blocks).read_text(encoding="utf-8")) picked = pick(doc, Path(args.kilde), args.focus) _write_new(Path(args.out), "".join(f"{p.name}\t{';'.join(p.aliases)}\n" for p in picked)) print(f"pick: {len(picked)} concepts -> {args.out}") except (RunError, OSError, json.JSONDecodeError) as exc: print(f"split cannot run: {exc}", file=sys.stderr) return 2 return 0 if __name__ == "__main__": sys.exit(main())