okf-wiki/evals/p1/split.py
Kjell Tore Guttormsen 0fd86a23a3
okf-wiki 1.1.1
First public release, published as one snapshot. What changed between
versions is in CHANGELOG.md.

Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
2026-11-14 21:03:20 +01:00

328 lines
13 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

"""Split the trial's sources into a fixed order and into blocks with line spans (TRIAL, P1). It calls no model.
order --manifest M --l4 L4DIR --out FILE
The 19 names of the cut (gates/wholedoc.manifest.json cut.names) in the trial's fixed order: docs pages by
name, then release notes by version. Refuses when the names differ from the files the L4 cut cites.
blocks --kilde DIR --order FILE --out FILE
Every block of every source in the order: a paragraph, one list item with its continuation lines, one
table or one fenced code block, with its 1-based inclusive line span, its section (the heading path) and,
for a release note, its version. Heading lines are section markers, not blocks.
total --kilde DIR
The number of blocks in every file under DIR (the denominator of the 599-source projection).
pick --blocks FILE --kilde DIR [--focus NAME] --out FILE
The three concepts of the side-by-side page, picked from the source text before any build: the focus page's
heading with the most blocks, the term in the most release notes, and the term in at least two docs pages and
two release notes with the highest document count; a document counts for a term only where the term stands
as a term (backticked or a capitalised phrase), never as a word in prose. One line per concept: name, a tab,
aliases joined by ;.
Exit 0 = written, 2 = could not run (an existing output is never overwritten).
"""
from __future__ import annotations
import argparse
import json
import re
import sys
from dataclasses import asdict, dataclass
from pathlib import Path
from typing import Any
from okfwiki.audit import HEADING, LIST_ITEM, read_document
SCHEMA = "okf-wiki/p1-blocks/1"
RELEASE = re.compile(r"^release-v(\d+(?:\.\d+)*)\.md$")
VERSION_HEADING = re.compile(r"^v?(\d+(?:\.\d+){2,})$")
FENCE = re.compile(r"^\s*(```|~~~)")
# Concept test 2 of the brief (Google's reference agent), plus the generic section names of docs pages.
META_STOP = frozenset({
"overview", "introduction", "getting started", "quickstart", "tutorial", "walkthrough", "release notes",
"changelog", "roadmap", "faq", "prerequisites", "next steps", "before you begin", "troubleshooting", "see also",
"related resources", "learn more",
})
CODE_TERM = re.compile(r"`([^`\n]{3,40})`")
CAPITAL_PHRASE = re.compile(r"\b[A-Z][\w-]*(?:[ \t]+[A-Z][\w-]*){1,3}\b")
SENTENCE_START = re.compile(r"(?:^|[.!?:]\s+|^\s*(?:[-*+]|\d+[.)])\s+)$")
class RunError(Exception):
"""The command cannot run as asked; nothing was written."""
@dataclass(frozen=True)
class SBlock:
file: str
start: int
end: int
kind: str # paragraph | item | table | code
section: str
version: str | None
@dataclass(frozen=True)
class Heading:
file: str
line: int
level: int
text: str
@dataclass(frozen=True)
class Picked:
name: str
aliases: tuple[str, ...]
def is_meta(title: str) -> bool:
return title.strip().lower() in META_STOP
def version_key(name: str) -> tuple[int, ...]:
m = RELEASE.match(name)
if m is None:
raise RunError(f"{name} is not a release note name")
return tuple(int(p) for p in m.group(1).split("."))
def order(names: list[str]) -> list[str]:
docs = sorted(n for n in names if n.startswith("docs-"))
releases = [n for n in names if n.startswith("release-")]
other = [n for n in names if n not in docs and n not in releases]
if other:
raise RunError(f"names that are neither docs pages nor release notes: {other}")
return docs + sorted(releases, key=version_key)
def headings_of(lines: list[str]) -> list[tuple[int, int, str]]:
"""(line, level, text) of every heading outside fenced code."""
out: list[tuple[int, int, str]] = []
fence: str | None = None
for n, raw in enumerate(lines, 1):
f = FENCE.match(raw)
if f:
fence = None if fence == f.group(1) else (fence or f.group(1))
continue
if fence is None:
m = HEADING.match(raw)
if m:
out.append((n, len(m.group(1)), (m.group(2) or "").strip()))
return out
def blocks(path: Path) -> list[SBlock]:
lines = path.read_text(encoding="utf-8").split("\n")
name = path.name
version: str | None = None
if name.startswith("release-"):
for _n, _level, text in headings_of(lines):
m = VERSION_HEADING.match(text)
if m:
version = m.group(1)
break
out: list[SBlock] = []
stack: list[tuple[int, str]] = []
kind: str | None = None
start = 0
fence: str | None = None
def close(end: int) -> None:
nonlocal kind
if kind is not None:
out.append(SBlock(name, start, end, kind, " › ".join(t for _, t in stack), version))
kind = None
for n, raw in enumerate(lines, 1):
f = FENCE.match(raw)
if fence is not None:
if f and f.group(1) == fence:
fence = None
close(n)
continue
if f:
close(n - 1)
fence, kind, start = f.group(1), "code", n
continue
if not raw.strip():
close(n - 1)
continue
m = HEADING.match(raw)
if m:
close(n - 1)
level = len(m.group(1))
stack = [(lv, t) for lv, t in stack if lv < level] + [(level, (m.group(2) or "").strip())]
continue
if raw.lstrip().startswith("|"):
if kind != "table":
close(n - 1)
kind, start = "table", n
continue
if LIST_ITEM.match(raw):
close(n - 1)
kind, start = "item", n
continue
if kind in (None, "table"):
close(n - 1)
kind, start = "paragraph", n
if fence is not None:
close(len(lines))
else:
last = len(lines)
while last > 0 and not lines[last - 1].strip():
last -= 1
close(last)
return out
def docs_headings(kilde: Path) -> list[Heading]:
out: list[Heading] = []
for path in sorted(kilde.glob("docs-*.md")):
for n, level, text in headings_of(path.read_text(encoding="utf-8").split("\n")):
if level <= 3 and text:
out.append(Heading(path.name, n, level, text))
return out
def blocks_document(kilde: Path, names: list[str]) -> dict[str, Any]:
found: list[dict[str, Any]] = []
for name in names:
path = kilde / name
if not path.is_file():
raise RunError(f"{path} is not a file")
found.extend(asdict(b) for b in blocks(path))
return {"schema": SCHEMA, "order": names, "blocks": found}
def blocks_total(kilde: Path) -> int:
return sum(len(blocks(p)) for p in sorted(kilde.glob("*.md")))
def terms(text: str) -> list[str]:
"""Backticked spans of 3-40 characters and capitalised phrases of 2-4 words, a sentence-initial word dropped."""
found: list[str] = [t for t in (m.group(1).strip() for m in CODE_TERM.finditer(text)) if 3 <= len(t) <= 40]
masked = CODE_TERM.sub(lambda m: " " * len(m.group(0)), text)
for raw in masked.split("\n"):
for m in CAPITAL_PHRASE.finditer(raw):
words = m.group(0).split()
if SENTENCE_START.search(raw[:m.start()]):
words = words[1:]
if len(words) >= 2:
found.append(" ".join(words))
return list(dict.fromkeys(t for t in found if t))
def occurs(term: str, text: str) -> bool:
return re.search(r"(?<!\w)" + re.escape(term) + r"(?!\w)", text, re.IGNORECASE) is not None
def pick(blocks_doc: dict[str, Any], kilde: Path, focus: str = "docs-en-permission-modes.md") -> list[Picked]:
names = [str(n) for n in blocks_doc["order"]]
if focus not in names:
raise RunError(f"the focus page {focus} is not in the order")
texts = {n: (kilde / n).read_text(encoding="utf-8") for n in names}
per_section: dict[str, int] = {}
for b in blocks_doc["blocks"]:
if b["file"] == focus and b["section"].count(" › ") >= 1:
per_section[b["section"]] = per_section.get(b["section"], 0) + 1
candidates = [(count, section.split(" › ")[-1]) for section, count in per_section.items()
if not is_meta(section.split(" › ")[-1])]
if not candidates:
raise RunError(f"{focus} has no heading below its title that passes test 2")
first = sorted(candidates, key=lambda c: (-c[0], c[1]))[0][1]
pool = list(dict.fromkeys(t for n in names for t in terms(texts[n])))
docs = [n for n in names if n.startswith("docs-")]
releases = [n for n in names if n.startswith("release-")]
as_terms = {n: set(terms(texts[n])) for n in names}
def count(term: str, files: list[str]) -> int:
return sum(term in as_terms[f] for f in files) # where it stands as a term, not as a word in prose
taken = {first.lower()}
second_pool = [t for t in pool if t.lower() not in taken]
if not second_pool:
raise RunError("no term for the second concept")
second = sorted(second_pool, key=lambda t: (-count(t, releases), t))[0]
taken.add(second.lower())
third_pool = [t for t in pool if t.lower() not in taken and count(t, docs) >= 2 and count(t, releases) >= 2]
if not third_pool:
raise RunError("no term in two docs pages and two release notes for the third concept")
third = sorted(third_pool, key=lambda t: (-count(t, names), t))[0]
return [Picked(first, (first,)), Picked(second, (second,)), Picked(third, (third,))]
def _write_new(path: Path, text: str) -> None:
if path.exists():
raise RunError(f"{path} exists; refusing to overwrite")
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(text, encoding="utf-8")
def l4_cited(l4: Path) -> set[str]:
cited: set[str] = set()
for page in sorted(l4.glob("*.md")):
fm = read_document(page, page.name).frontmatter or {}
srcs = fm.get("sources")
for s in srcs if isinstance(srcs, list) else []:
res = s.get("resource") if isinstance(s, dict) else None
if isinstance(res, str) and res:
cited.add(Path(res).name)
return cited
def build_parser() -> argparse.ArgumentParser:
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
sub = p.add_subparsers(dest="command", required=True)
o = sub.add_parser("order")
o.add_argument("--manifest", required=True)
o.add_argument("--l4", required=True)
o.add_argument("--out", required=True)
b = sub.add_parser("blocks")
b.add_argument("--kilde", required=True)
b.add_argument("--order", required=True)
b.add_argument("--out", required=True)
t = sub.add_parser("total")
t.add_argument("--kilde", required=True)
k = sub.add_parser("pick")
k.add_argument("--blocks", required=True)
k.add_argument("--kilde", required=True)
k.add_argument("--focus", default="docs-en-permission-modes.md")
k.add_argument("--out", required=True)
return p
def main(argv: list[str] | None = None) -> int:
args = build_parser().parse_args(argv)
try:
if args.command == "order":
manifest = json.loads(Path(args.manifest).read_text(encoding="utf-8"))
names = manifest.get("cut", {}).get("names")
if not isinstance(names, list) or not names:
raise RunError(f"{args.manifest} records no cut names")
cited = l4_cited(Path(args.l4))
if set(map(str, names)) != cited:
raise RunError(f"the manifest's {len(names)} names differ from the {len(cited)} files the L4 cut cites: "
f"only in manifest {sorted(set(names) - cited)}, only in L4 {sorted(cited - set(names))}")
fixed = order([str(n) for n in names])
_write_new(Path(args.out), "\n".join(fixed) + "\n")
print(f"order: {len(fixed)} sources -> {args.out}")
elif args.command == "blocks":
names = Path(args.order).read_text(encoding="utf-8").split()
doc = blocks_document(Path(args.kilde), names)
_write_new(Path(args.out), json.dumps(doc, ensure_ascii=False, indent=1) + "\n")
print(f"blocks: {len(doc['blocks'])} in {len(names)} sources -> {args.out}")
elif args.command == "total":
print(f"blocks in every file of {args.kilde}: {blocks_total(Path(args.kilde))}")
else:
doc = json.loads(Path(args.blocks).read_text(encoding="utf-8"))
picked = pick(doc, Path(args.kilde), args.focus)
_write_new(Path(args.out), "".join(f"{p.name}\t{';'.join(p.aliases)}\n" for p in picked))
print(f"pick: {len(picked)} concepts -> {args.out}")
except (RunError, OSError, json.JSONDecodeError) as exc:
print(f"split cannot run: {exc}", file=sys.stderr)
return 2
return 0
if __name__ == "__main__":
sys.exit(main())