First public release, published as one snapshot. What changed between versions is in CHANGELOG.md. Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
328 lines
13 KiB
Python
328 lines
13 KiB
Python
"""Split the trial's sources into a fixed order and into blocks with line spans (TRIAL, P1). It calls no model.
|
||
|
||
order --manifest M --l4 L4DIR --out FILE
|
||
The 19 names of the cut (gates/wholedoc.manifest.json cut.names) in the trial's fixed order: docs pages by
|
||
name, then release notes by version. Refuses when the names differ from the files the L4 cut cites.
|
||
blocks --kilde DIR --order FILE --out FILE
|
||
Every block of every source in the order: a paragraph, one list item with its continuation lines, one
|
||
table or one fenced code block, with its 1-based inclusive line span, its section (the heading path) and,
|
||
for a release note, its version. Heading lines are section markers, not blocks.
|
||
total --kilde DIR
|
||
The number of blocks in every file under DIR (the denominator of the 599-source projection).
|
||
pick --blocks FILE --kilde DIR [--focus NAME] --out FILE
|
||
The three concepts of the side-by-side page, picked from the source text before any build: the focus page's
|
||
heading with the most blocks, the term in the most release notes, and the term in at least two docs pages and
|
||
two release notes with the highest document count; a document counts for a term only where the term stands
|
||
as a term (backticked or a capitalised phrase), never as a word in prose. One line per concept: name, a tab,
|
||
aliases joined by ;.
|
||
|
||
Exit 0 = written, 2 = could not run (an existing output is never overwritten).
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import json
|
||
import re
|
||
import sys
|
||
from dataclasses import asdict, dataclass
|
||
from pathlib import Path
|
||
from typing import Any
|
||
|
||
from okfwiki.audit import HEADING, LIST_ITEM, read_document
|
||
|
||
SCHEMA = "okf-wiki/p1-blocks/1"
|
||
RELEASE = re.compile(r"^release-v(\d+(?:\.\d+)*)\.md$")
|
||
VERSION_HEADING = re.compile(r"^v?(\d+(?:\.\d+){2,})$")
|
||
FENCE = re.compile(r"^\s*(```|~~~)")
|
||
# Concept test 2 of the brief (Google's reference agent), plus the generic section names of docs pages.
|
||
META_STOP = frozenset({
|
||
"overview", "introduction", "getting started", "quickstart", "tutorial", "walkthrough", "release notes",
|
||
"changelog", "roadmap", "faq", "prerequisites", "next steps", "before you begin", "troubleshooting", "see also",
|
||
"related resources", "learn more",
|
||
})
|
||
CODE_TERM = re.compile(r"`([^`\n]{3,40})`")
|
||
CAPITAL_PHRASE = re.compile(r"\b[A-Z][\w-]*(?:[ \t]+[A-Z][\w-]*){1,3}\b")
|
||
SENTENCE_START = re.compile(r"(?:^|[.!?:]\s+|^\s*(?:[-*+]|\d+[.)])\s+)$")
|
||
|
||
|
||
class RunError(Exception):
|
||
"""The command cannot run as asked; nothing was written."""
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class SBlock:
|
||
file: str
|
||
start: int
|
||
end: int
|
||
kind: str # paragraph | item | table | code
|
||
section: str
|
||
version: str | None
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class Heading:
|
||
file: str
|
||
line: int
|
||
level: int
|
||
text: str
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class Picked:
|
||
name: str
|
||
aliases: tuple[str, ...]
|
||
|
||
|
||
def is_meta(title: str) -> bool:
|
||
return title.strip().lower() in META_STOP
|
||
|
||
|
||
def version_key(name: str) -> tuple[int, ...]:
|
||
m = RELEASE.match(name)
|
||
if m is None:
|
||
raise RunError(f"{name} is not a release note name")
|
||
return tuple(int(p) for p in m.group(1).split("."))
|
||
|
||
|
||
def order(names: list[str]) -> list[str]:
|
||
docs = sorted(n for n in names if n.startswith("docs-"))
|
||
releases = [n for n in names if n.startswith("release-")]
|
||
other = [n for n in names if n not in docs and n not in releases]
|
||
if other:
|
||
raise RunError(f"names that are neither docs pages nor release notes: {other}")
|
||
return docs + sorted(releases, key=version_key)
|
||
|
||
|
||
def headings_of(lines: list[str]) -> list[tuple[int, int, str]]:
|
||
"""(line, level, text) of every heading outside fenced code."""
|
||
out: list[tuple[int, int, str]] = []
|
||
fence: str | None = None
|
||
for n, raw in enumerate(lines, 1):
|
||
f = FENCE.match(raw)
|
||
if f:
|
||
fence = None if fence == f.group(1) else (fence or f.group(1))
|
||
continue
|
||
if fence is None:
|
||
m = HEADING.match(raw)
|
||
if m:
|
||
out.append((n, len(m.group(1)), (m.group(2) or "").strip()))
|
||
return out
|
||
|
||
|
||
def blocks(path: Path) -> list[SBlock]:
|
||
lines = path.read_text(encoding="utf-8").split("\n")
|
||
name = path.name
|
||
version: str | None = None
|
||
if name.startswith("release-"):
|
||
for _n, _level, text in headings_of(lines):
|
||
m = VERSION_HEADING.match(text)
|
||
if m:
|
||
version = m.group(1)
|
||
break
|
||
out: list[SBlock] = []
|
||
stack: list[tuple[int, str]] = []
|
||
kind: str | None = None
|
||
start = 0
|
||
fence: str | None = None
|
||
|
||
def close(end: int) -> None:
|
||
nonlocal kind
|
||
if kind is not None:
|
||
out.append(SBlock(name, start, end, kind, " › ".join(t for _, t in stack), version))
|
||
kind = None
|
||
|
||
for n, raw in enumerate(lines, 1):
|
||
f = FENCE.match(raw)
|
||
if fence is not None:
|
||
if f and f.group(1) == fence:
|
||
fence = None
|
||
close(n)
|
||
continue
|
||
if f:
|
||
close(n - 1)
|
||
fence, kind, start = f.group(1), "code", n
|
||
continue
|
||
if not raw.strip():
|
||
close(n - 1)
|
||
continue
|
||
m = HEADING.match(raw)
|
||
if m:
|
||
close(n - 1)
|
||
level = len(m.group(1))
|
||
stack = [(lv, t) for lv, t in stack if lv < level] + [(level, (m.group(2) or "").strip())]
|
||
continue
|
||
if raw.lstrip().startswith("|"):
|
||
if kind != "table":
|
||
close(n - 1)
|
||
kind, start = "table", n
|
||
continue
|
||
if LIST_ITEM.match(raw):
|
||
close(n - 1)
|
||
kind, start = "item", n
|
||
continue
|
||
if kind in (None, "table"):
|
||
close(n - 1)
|
||
kind, start = "paragraph", n
|
||
if fence is not None:
|
||
close(len(lines))
|
||
else:
|
||
last = len(lines)
|
||
while last > 0 and not lines[last - 1].strip():
|
||
last -= 1
|
||
close(last)
|
||
return out
|
||
|
||
|
||
def docs_headings(kilde: Path) -> list[Heading]:
|
||
out: list[Heading] = []
|
||
for path in sorted(kilde.glob("docs-*.md")):
|
||
for n, level, text in headings_of(path.read_text(encoding="utf-8").split("\n")):
|
||
if level <= 3 and text:
|
||
out.append(Heading(path.name, n, level, text))
|
||
return out
|
||
|
||
|
||
def blocks_document(kilde: Path, names: list[str]) -> dict[str, Any]:
|
||
found: list[dict[str, Any]] = []
|
||
for name in names:
|
||
path = kilde / name
|
||
if not path.is_file():
|
||
raise RunError(f"{path} is not a file")
|
||
found.extend(asdict(b) for b in blocks(path))
|
||
return {"schema": SCHEMA, "order": names, "blocks": found}
|
||
|
||
|
||
def blocks_total(kilde: Path) -> int:
|
||
return sum(len(blocks(p)) for p in sorted(kilde.glob("*.md")))
|
||
|
||
|
||
def terms(text: str) -> list[str]:
|
||
"""Backticked spans of 3-40 characters and capitalised phrases of 2-4 words, a sentence-initial word dropped."""
|
||
found: list[str] = [t for t in (m.group(1).strip() for m in CODE_TERM.finditer(text)) if 3 <= len(t) <= 40]
|
||
masked = CODE_TERM.sub(lambda m: " " * len(m.group(0)), text)
|
||
for raw in masked.split("\n"):
|
||
for m in CAPITAL_PHRASE.finditer(raw):
|
||
words = m.group(0).split()
|
||
if SENTENCE_START.search(raw[:m.start()]):
|
||
words = words[1:]
|
||
if len(words) >= 2:
|
||
found.append(" ".join(words))
|
||
return list(dict.fromkeys(t for t in found if t))
|
||
|
||
|
||
def occurs(term: str, text: str) -> bool:
|
||
return re.search(r"(?<!\w)" + re.escape(term) + r"(?!\w)", text, re.IGNORECASE) is not None
|
||
|
||
|
||
def pick(blocks_doc: dict[str, Any], kilde: Path, focus: str = "docs-en-permission-modes.md") -> list[Picked]:
|
||
names = [str(n) for n in blocks_doc["order"]]
|
||
if focus not in names:
|
||
raise RunError(f"the focus page {focus} is not in the order")
|
||
texts = {n: (kilde / n).read_text(encoding="utf-8") for n in names}
|
||
per_section: dict[str, int] = {}
|
||
for b in blocks_doc["blocks"]:
|
||
if b["file"] == focus and b["section"].count(" › ") >= 1:
|
||
per_section[b["section"]] = per_section.get(b["section"], 0) + 1
|
||
candidates = [(count, section.split(" › ")[-1]) for section, count in per_section.items()
|
||
if not is_meta(section.split(" › ")[-1])]
|
||
if not candidates:
|
||
raise RunError(f"{focus} has no heading below its title that passes test 2")
|
||
first = sorted(candidates, key=lambda c: (-c[0], c[1]))[0][1]
|
||
pool = list(dict.fromkeys(t for n in names for t in terms(texts[n])))
|
||
docs = [n for n in names if n.startswith("docs-")]
|
||
releases = [n for n in names if n.startswith("release-")]
|
||
|
||
as_terms = {n: set(terms(texts[n])) for n in names}
|
||
|
||
def count(term: str, files: list[str]) -> int:
|
||
return sum(term in as_terms[f] for f in files) # where it stands as a term, not as a word in prose
|
||
|
||
taken = {first.lower()}
|
||
second_pool = [t for t in pool if t.lower() not in taken]
|
||
if not second_pool:
|
||
raise RunError("no term for the second concept")
|
||
second = sorted(second_pool, key=lambda t: (-count(t, releases), t))[0]
|
||
taken.add(second.lower())
|
||
third_pool = [t for t in pool if t.lower() not in taken and count(t, docs) >= 2 and count(t, releases) >= 2]
|
||
if not third_pool:
|
||
raise RunError("no term in two docs pages and two release notes for the third concept")
|
||
third = sorted(third_pool, key=lambda t: (-count(t, names), t))[0]
|
||
return [Picked(first, (first,)), Picked(second, (second,)), Picked(third, (third,))]
|
||
|
||
|
||
def _write_new(path: Path, text: str) -> None:
|
||
if path.exists():
|
||
raise RunError(f"{path} exists; refusing to overwrite")
|
||
path.parent.mkdir(parents=True, exist_ok=True)
|
||
path.write_text(text, encoding="utf-8")
|
||
|
||
|
||
def l4_cited(l4: Path) -> set[str]:
|
||
cited: set[str] = set()
|
||
for page in sorted(l4.glob("*.md")):
|
||
fm = read_document(page, page.name).frontmatter or {}
|
||
srcs = fm.get("sources")
|
||
for s in srcs if isinstance(srcs, list) else []:
|
||
res = s.get("resource") if isinstance(s, dict) else None
|
||
if isinstance(res, str) and res:
|
||
cited.add(Path(res).name)
|
||
return cited
|
||
|
||
|
||
def build_parser() -> argparse.ArgumentParser:
|
||
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||
sub = p.add_subparsers(dest="command", required=True)
|
||
o = sub.add_parser("order")
|
||
o.add_argument("--manifest", required=True)
|
||
o.add_argument("--l4", required=True)
|
||
o.add_argument("--out", required=True)
|
||
b = sub.add_parser("blocks")
|
||
b.add_argument("--kilde", required=True)
|
||
b.add_argument("--order", required=True)
|
||
b.add_argument("--out", required=True)
|
||
t = sub.add_parser("total")
|
||
t.add_argument("--kilde", required=True)
|
||
k = sub.add_parser("pick")
|
||
k.add_argument("--blocks", required=True)
|
||
k.add_argument("--kilde", required=True)
|
||
k.add_argument("--focus", default="docs-en-permission-modes.md")
|
||
k.add_argument("--out", required=True)
|
||
return p
|
||
|
||
|
||
def main(argv: list[str] | None = None) -> int:
|
||
args = build_parser().parse_args(argv)
|
||
try:
|
||
if args.command == "order":
|
||
manifest = json.loads(Path(args.manifest).read_text(encoding="utf-8"))
|
||
names = manifest.get("cut", {}).get("names")
|
||
if not isinstance(names, list) or not names:
|
||
raise RunError(f"{args.manifest} records no cut names")
|
||
cited = l4_cited(Path(args.l4))
|
||
if set(map(str, names)) != cited:
|
||
raise RunError(f"the manifest's {len(names)} names differ from the {len(cited)} files the L4 cut cites: "
|
||
f"only in manifest {sorted(set(names) - cited)}, only in L4 {sorted(cited - set(names))}")
|
||
fixed = order([str(n) for n in names])
|
||
_write_new(Path(args.out), "\n".join(fixed) + "\n")
|
||
print(f"order: {len(fixed)} sources -> {args.out}")
|
||
elif args.command == "blocks":
|
||
names = Path(args.order).read_text(encoding="utf-8").split()
|
||
doc = blocks_document(Path(args.kilde), names)
|
||
_write_new(Path(args.out), json.dumps(doc, ensure_ascii=False, indent=1) + "\n")
|
||
print(f"blocks: {len(doc['blocks'])} in {len(names)} sources -> {args.out}")
|
||
elif args.command == "total":
|
||
print(f"blocks in every file of {args.kilde}: {blocks_total(Path(args.kilde))}")
|
||
else:
|
||
doc = json.loads(Path(args.blocks).read_text(encoding="utf-8"))
|
||
picked = pick(doc, Path(args.kilde), args.focus)
|
||
_write_new(Path(args.out), "".join(f"{p.name}\t{';'.join(p.aliases)}\n" for p in picked))
|
||
print(f"pick: {len(picked)} concepts -> {args.out}")
|
||
except (RunError, OSError, json.JSONDecodeError) as exc:
|
||
print(f"split cannot run: {exc}", file=sys.stderr)
|
||
return 2
|
||
return 0
|
||
|
||
|
||
if __name__ == "__main__":
|
||
sys.exit(main())
|