First public release, published as one snapshot. What changed between versions is in CHANGELOG.md. Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
290 lines
11 KiB
Python
290 lines
11 KiB
Python
"""K's sources in a fixed order, split into blocks with line spans. No model call (CLAUDE.md rule 1).
|
||
|
||
Usage:
|
||
python3 -m okfwiki.kblocks order SOURCES --out FILE
|
||
python3 -m okfwiki.kblocks blocks SOURCES --order FILE --out FILE
|
||
python3 -m okfwiki.kblocks total SOURCES
|
||
|
||
SOURCES is the folder that holds kilde/, the same raw-source root the auditor takes.
|
||
|
||
order every file under SOURCES/kilde/ in K's fixed order, one name per line: pages by name, then release notes
|
||
(release-vX.Y.Z.md) by version. Any other name is a page: a docs page (docs-*), or a document of a folder
|
||
(sources folder), .md or .txt alike.
|
||
blocks every block of every source in the order (schema okf-wiki/blocks/1): a paragraph, one list item with its
|
||
continuation lines, one table or one fenced code block, with its 1-based inclusive line span, its section (the
|
||
heading path), its characters (`chars`, every line with its newline) and, for a release note, its version.
|
||
Heading lines are section markers, not blocks, and a line that is only an HTML or MDX tag (<Note>, </Steps>)
|
||
never starts a block, so no block is only a tag. A "#" line
|
||
inside a component property's example string (example: `# Title ... `) is example text, not a heading.
|
||
total the number of blocks in every file under SOURCES/kilde/.
|
||
|
||
Lines are counted on the text split at "\\n", as the pack and the link pass count them. Exit 0 = done, 2 = could not
|
||
run; an existing output is never overwritten.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import json
|
||
import re
|
||
import sys
|
||
from dataclasses import asdict, dataclass
|
||
from pathlib import Path
|
||
from typing import Any
|
||
|
||
from okfwiki.audit import HEADING, LIST_ITEM, closes_fence, fence_marker
|
||
from okfwiki.common import HTML_TAG_ONLY, VERSION_HEADING, RunError, write_new
|
||
|
||
SCHEMA = "okf-wiki/blocks/1"
|
||
RELEASE = re.compile(r"^release-v(\d+(?:\.\d+)*)\.md$")
|
||
VERSION_LABEL = re.compile(r'^\s*<[A-Za-z][A-Za-z0-9._-]*\s[^<>]*?\blabel="v?(\d+(?:\.\d+){2,})"')
|
||
VERSION_BOLD = re.compile(r"^\*\*v?(\d+(?:\.\d+){2,})\*\*$")
|
||
# A component property whose value is a backtick string left open on its line (example: `# Title): the lines after it,
|
||
# up to the first with a backtick no backslash escapes, are example text, never a heading (order 14 findings C and D).
|
||
STRING_OPEN = re.compile(r"^\s*[A-Za-z_$][\w$]*\s*:\s*`")
|
||
TICK = re.compile(r"(?<!\\)`")
|
||
# Concept test 2 of the brief (Google's reference agent), plus the generic section names of docs pages.
|
||
META_STOP = frozenset({
|
||
"overview", "introduction", "getting started", "quickstart", "tutorial", "walkthrough", "release notes",
|
||
"changelog", "roadmap", "faq", "prerequisites", "next steps", "before you begin", "troubleshooting", "see also",
|
||
"related resources", "learn more",
|
||
})
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class Block:
|
||
file: str
|
||
start: int
|
||
end: int
|
||
kind: str # paragraph | item | table | code
|
||
section: str
|
||
version: str | None
|
||
chars: int = 0 # the characters of its lines, each with its newline: the unit the stop line prices (B17)
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class Heading:
|
||
file: str
|
||
line: int
|
||
level: int
|
||
text: str
|
||
|
||
|
||
def is_meta(title: str) -> bool:
|
||
return title.strip().lower() in META_STOP
|
||
|
||
|
||
def version_key(name: str) -> tuple[int, ...]:
|
||
m = RELEASE.match(name)
|
||
if m is None:
|
||
raise RunError(f"{name} is not a release note name")
|
||
return tuple(int(p) for p in m.group(1).split("."))
|
||
|
||
|
||
def order(names: list[str]) -> list[str]:
|
||
releases = [n for n in names if RELEASE.match(n)]
|
||
return sorted(n for n in names if not RELEASE.match(n)) + sorted(releases, key=version_key)
|
||
|
||
|
||
def source_files(kilde: Path) -> list[Path]:
|
||
"""Every file under kilde, in name order: .md and .txt alike, nothing dropped by its suffix."""
|
||
return sorted(p for p in kilde.iterdir() if p.is_file())
|
||
|
||
|
||
def read_lines(path: Path) -> list[str]:
|
||
return path.read_text(encoding="utf-8").split("\n")
|
||
|
||
|
||
def string_lines(lines: list[str]) -> set[int]:
|
||
"""The 1-based lines inside a component property's example string (STRING_OPEN), outside fenced code: from the line
|
||
after the one that opens it to the line with the backtick that closes it, both included."""
|
||
out: set[int] = set()
|
||
fence = "" # the open fence's marker run, or "" outside fenced code (the auditor's rule, CommonMark 4.5)
|
||
inside = False
|
||
for n, raw in enumerate(lines, 1):
|
||
if inside:
|
||
out.add(n)
|
||
inside = TICK.search(raw) is None
|
||
continue
|
||
f = fence_marker(raw)
|
||
if fence:
|
||
if closes_fence(f, raw, fence):
|
||
fence = ""
|
||
elif f:
|
||
fence = f.group(1)
|
||
elif STRING_OPEN.match(raw):
|
||
inside = len(TICK.findall(raw)) % 2 == 1
|
||
return out
|
||
|
||
|
||
def headings_of(lines: list[str]) -> list[tuple[int, int, str]]:
|
||
"""(line, level, text) of every heading outside fenced code and outside a component's example string."""
|
||
out: list[tuple[int, int, str]] = []
|
||
fence = ""
|
||
text_of_string = string_lines(lines)
|
||
for n, raw in enumerate(lines, 1):
|
||
if n in text_of_string:
|
||
continue
|
||
f = fence_marker(raw)
|
||
if fence:
|
||
if closes_fence(f, raw, fence):
|
||
fence = ""
|
||
continue
|
||
if f:
|
||
fence = f.group(1)
|
||
continue
|
||
m = HEADING.match(raw)
|
||
if m:
|
||
out.append((n, len(m.group(1)), (m.group(2) or "").strip()))
|
||
return out
|
||
|
||
|
||
def stated_versions(lines: list[str]) -> list[tuple[int, str]]:
|
||
"""(line, version) for every line that states a release note's version: a heading that is only the version, then
|
||
a component whose label is the version (<Update label="X.Y.Z" ...>: every release note of the copy of 08.10, 416
|
||
of 416, has this form and no heading; order 27 A6), then a line that is only the bold version (**X.Y.Z**: the
|
||
page's form after 20.10, 437 of 437 on 07.11)."""
|
||
found = [(n, m.group(1)) for n, _level, text in headings_of(lines) if (m := VERSION_HEADING.match(text))]
|
||
found += [(n, m.group(1)) for n, raw in enumerate(lines, 1) if (m := VERSION_LABEL.match(raw))]
|
||
return found + [(n, m.group(1)) for n, raw in enumerate(lines, 1) if (m := VERSION_BOLD.match(raw))]
|
||
|
||
|
||
def version_line(lines: list[str], version: str) -> int | None:
|
||
"""The first line that states `version`, or None."""
|
||
return next((n for n, v in stated_versions(lines) if v == version), None)
|
||
|
||
|
||
def blocks(path: Path) -> list[Block]:
|
||
lines = read_lines(path)
|
||
name = path.name
|
||
version: str | None = None
|
||
if RELEASE.match(name):
|
||
stated = stated_versions(lines)
|
||
version = stated[0][1] if stated else None
|
||
out: list[Block] = []
|
||
stack: list[tuple[int, str]] = []
|
||
kind: str | None = None
|
||
start = 0
|
||
fence = ""
|
||
text_of_string = string_lines(lines)
|
||
|
||
def close(end: int) -> None:
|
||
nonlocal kind
|
||
if kind is not None:
|
||
out.append(Block(name, start, end, kind, " › ".join(t for _, t in stack), version,
|
||
sum(len(line) + 1 for line in lines[start - 1:end])))
|
||
kind = None
|
||
|
||
for n, raw in enumerate(lines, 1):
|
||
f = fence_marker(raw)
|
||
if fence:
|
||
if closes_fence(f, raw, fence):
|
||
fence = ""
|
||
close(n)
|
||
continue
|
||
if f:
|
||
close(n - 1)
|
||
fence, kind, start = f.group(1), "code", n
|
||
continue
|
||
if not raw.strip():
|
||
close(n - 1)
|
||
continue
|
||
m = None if n in text_of_string else HEADING.match(raw)
|
||
if m:
|
||
close(n - 1)
|
||
level = len(m.group(1))
|
||
stack = [(lv, t) for lv, t in stack if lv < level] + [(level, (m.group(2) or "").strip())]
|
||
continue
|
||
if raw.lstrip().startswith("|"):
|
||
if kind != "table":
|
||
close(n - 1)
|
||
kind, start = "table", n
|
||
continue
|
||
if LIST_ITEM.match(raw):
|
||
close(n - 1)
|
||
kind, start = "item", n
|
||
continue
|
||
if kind in (None, "table"):
|
||
close(n - 1)
|
||
if HTML_TAG_ONLY.match(raw):
|
||
continue # a tag alone is layout; the block starts at the first line of content
|
||
kind, start = "paragraph", n
|
||
if fence is not None:
|
||
close(len(lines))
|
||
else:
|
||
last = len(lines)
|
||
while last > 0 and not lines[last - 1].strip():
|
||
last -= 1
|
||
close(last)
|
||
return out
|
||
|
||
|
||
def docs_headings(kilde: Path) -> list[Heading]:
|
||
"""The title and level 2-3 headings of every page under kilde (any file but a release note), in name order."""
|
||
out: list[Heading] = []
|
||
for path in (p for p in source_files(kilde) if not RELEASE.match(p.name)):
|
||
for n, level, text in headings_of(read_lines(path)):
|
||
if level <= 3 and text:
|
||
out.append(Heading(path.name, n, level, text))
|
||
return out
|
||
|
||
|
||
def blocks_document(kilde: Path, names: list[str]) -> dict[str, Any]:
|
||
found: list[dict[str, Any]] = []
|
||
for name in names:
|
||
path = kilde / name
|
||
if not path.is_file():
|
||
raise RunError(f"{path} is not a file")
|
||
found.extend(asdict(b) for b in blocks(path))
|
||
return {"schema": SCHEMA, "order": names, "blocks": found}
|
||
|
||
|
||
def kilde_of(sources: Path) -> Path:
|
||
kilde = sources / "kilde"
|
||
if not kilde.is_dir():
|
||
raise RunError(f"{kilde} is not a directory")
|
||
return kilde
|
||
|
||
|
||
def build_parser() -> argparse.ArgumentParser:
|
||
p = argparse.ArgumentParser(prog="python3 -m okfwiki.kblocks", description=__doc__,
|
||
formatter_class=argparse.RawDescriptionHelpFormatter)
|
||
sub = p.add_subparsers(dest="command", required=True)
|
||
o = sub.add_parser("order")
|
||
o.add_argument("sources")
|
||
o.add_argument("--out", required=True)
|
||
b = sub.add_parser("blocks")
|
||
b.add_argument("sources")
|
||
b.add_argument("--order", required=True)
|
||
b.add_argument("--out", required=True)
|
||
t = sub.add_parser("total")
|
||
t.add_argument("sources")
|
||
return p
|
||
|
||
|
||
def main(argv: list[str] | None = None) -> int:
|
||
args = build_parser().parse_args(argv)
|
||
try:
|
||
kilde = kilde_of(Path(args.sources))
|
||
if args.command == "order":
|
||
fixed = order([p.name for p in source_files(kilde)])
|
||
write_new(Path(args.out), "".join(n + "\n" for n in fixed))
|
||
print(f"order: {len(fixed)} sources -> {args.out}")
|
||
elif args.command == "blocks":
|
||
names = Path(args.order).read_text(encoding="utf-8").split()
|
||
if Path(args.out).exists():
|
||
raise RunError(f"{args.out} exists; refusing to overwrite")
|
||
doc = blocks_document(kilde, names)
|
||
write_new(Path(args.out), json.dumps(doc, ensure_ascii=False, indent=1) + "\n")
|
||
print(f"blocks: {len(doc['blocks'])} in {len(names)} sources -> {args.out}")
|
||
else:
|
||
files = source_files(kilde)
|
||
print(f"total: {sum(len(blocks(p)) for p in files)} blocks in {len(files)} files of {kilde}")
|
||
except (RunError, OSError, UnicodeDecodeError) as exc:
|
||
print(f"kblocks cannot run: {exc}", file=sys.stderr)
|
||
return 2
|
||
return 0
|
||
|
||
|
||
if __name__ == "__main__":
|
||
sys.exit(main())
|