okf-wiki/okfwiki/kblocks.py
Kjell Tore Guttormsen 0fd86a23a3
okf-wiki 1.1.1
First public release, published as one snapshot. What changed between
versions is in CHANGELOG.md.

Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
2026-11-14 21:03:20 +01:00

290 lines
11 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

"""K's sources in a fixed order, split into blocks with line spans. No model call (CLAUDE.md rule 1).
Usage:
python3 -m okfwiki.kblocks order SOURCES --out FILE
python3 -m okfwiki.kblocks blocks SOURCES --order FILE --out FILE
python3 -m okfwiki.kblocks total SOURCES
SOURCES is the folder that holds kilde/, the same raw-source root the auditor takes.
order every file under SOURCES/kilde/ in K's fixed order, one name per line: pages by name, then release notes
(release-vX.Y.Z.md) by version. Any other name is a page: a docs page (docs-*), or a document of a folder
(sources folder), .md or .txt alike.
blocks every block of every source in the order (schema okf-wiki/blocks/1): a paragraph, one list item with its
continuation lines, one table or one fenced code block, with its 1-based inclusive line span, its section (the
heading path), its characters (`chars`, every line with its newline) and, for a release note, its version.
Heading lines are section markers, not blocks, and a line that is only an HTML or MDX tag (<Note>, </Steps>)
never starts a block, so no block is only a tag. A "#" line
inside a component property's example string (example: `# Title ... `) is example text, not a heading.
total the number of blocks in every file under SOURCES/kilde/.
Lines are counted on the text split at "\\n", as the pack and the link pass count them. Exit 0 = done, 2 = could not
run; an existing output is never overwritten.
"""
from __future__ import annotations
import argparse
import json
import re
import sys
from dataclasses import asdict, dataclass
from pathlib import Path
from typing import Any
from okfwiki.audit import HEADING, LIST_ITEM, closes_fence, fence_marker
from okfwiki.common import HTML_TAG_ONLY, VERSION_HEADING, RunError, write_new
SCHEMA = "okf-wiki/blocks/1"
RELEASE = re.compile(r"^release-v(\d+(?:\.\d+)*)\.md$")
VERSION_LABEL = re.compile(r'^\s*<[A-Za-z][A-Za-z0-9._-]*\s[^<>]*?\blabel="v?(\d+(?:\.\d+){2,})"')
VERSION_BOLD = re.compile(r"^\*\*v?(\d+(?:\.\d+){2,})\*\*$")
# A component property whose value is a backtick string left open on its line (example: `# Title): the lines after it,
# up to the first with a backtick no backslash escapes, are example text, never a heading (order 14 findings C and D).
STRING_OPEN = re.compile(r"^\s*[A-Za-z_$][\w$]*\s*:\s*`")
TICK = re.compile(r"(?<!\\)`")
# Concept test 2 of the brief (Google's reference agent), plus the generic section names of docs pages.
META_STOP = frozenset({
"overview", "introduction", "getting started", "quickstart", "tutorial", "walkthrough", "release notes",
"changelog", "roadmap", "faq", "prerequisites", "next steps", "before you begin", "troubleshooting", "see also",
"related resources", "learn more",
})
@dataclass(frozen=True)
class Block:
file: str
start: int
end: int
kind: str # paragraph | item | table | code
section: str
version: str | None
chars: int = 0 # the characters of its lines, each with its newline: the unit the stop line prices (B17)
@dataclass(frozen=True)
class Heading:
file: str
line: int
level: int
text: str
def is_meta(title: str) -> bool:
return title.strip().lower() in META_STOP
def version_key(name: str) -> tuple[int, ...]:
m = RELEASE.match(name)
if m is None:
raise RunError(f"{name} is not a release note name")
return tuple(int(p) for p in m.group(1).split("."))
def order(names: list[str]) -> list[str]:
releases = [n for n in names if RELEASE.match(n)]
return sorted(n for n in names if not RELEASE.match(n)) + sorted(releases, key=version_key)
def source_files(kilde: Path) -> list[Path]:
"""Every file under kilde, in name order: .md and .txt alike, nothing dropped by its suffix."""
return sorted(p for p in kilde.iterdir() if p.is_file())
def read_lines(path: Path) -> list[str]:
return path.read_text(encoding="utf-8").split("\n")
def string_lines(lines: list[str]) -> set[int]:
"""The 1-based lines inside a component property's example string (STRING_OPEN), outside fenced code: from the line
after the one that opens it to the line with the backtick that closes it, both included."""
out: set[int] = set()
fence = "" # the open fence's marker run, or "" outside fenced code (the auditor's rule, CommonMark 4.5)
inside = False
for n, raw in enumerate(lines, 1):
if inside:
out.add(n)
inside = TICK.search(raw) is None
continue
f = fence_marker(raw)
if fence:
if closes_fence(f, raw, fence):
fence = ""
elif f:
fence = f.group(1)
elif STRING_OPEN.match(raw):
inside = len(TICK.findall(raw)) % 2 == 1
return out
def headings_of(lines: list[str]) -> list[tuple[int, int, str]]:
"""(line, level, text) of every heading outside fenced code and outside a component's example string."""
out: list[tuple[int, int, str]] = []
fence = ""
text_of_string = string_lines(lines)
for n, raw in enumerate(lines, 1):
if n in text_of_string:
continue
f = fence_marker(raw)
if fence:
if closes_fence(f, raw, fence):
fence = ""
continue
if f:
fence = f.group(1)
continue
m = HEADING.match(raw)
if m:
out.append((n, len(m.group(1)), (m.group(2) or "").strip()))
return out
def stated_versions(lines: list[str]) -> list[tuple[int, str]]:
"""(line, version) for every line that states a release note's version: a heading that is only the version, then
a component whose label is the version (<Update label="X.Y.Z" ...>: every release note of the copy of 08.10, 416
of 416, has this form and no heading; order 27 A6), then a line that is only the bold version (**X.Y.Z**: the
page's form after 20.10, 437 of 437 on 07.11)."""
found = [(n, m.group(1)) for n, _level, text in headings_of(lines) if (m := VERSION_HEADING.match(text))]
found += [(n, m.group(1)) for n, raw in enumerate(lines, 1) if (m := VERSION_LABEL.match(raw))]
return found + [(n, m.group(1)) for n, raw in enumerate(lines, 1) if (m := VERSION_BOLD.match(raw))]
def version_line(lines: list[str], version: str) -> int | None:
"""The first line that states `version`, or None."""
return next((n for n, v in stated_versions(lines) if v == version), None)
def blocks(path: Path) -> list[Block]:
lines = read_lines(path)
name = path.name
version: str | None = None
if RELEASE.match(name):
stated = stated_versions(lines)
version = stated[0][1] if stated else None
out: list[Block] = []
stack: list[tuple[int, str]] = []
kind: str | None = None
start = 0
fence = ""
text_of_string = string_lines(lines)
def close(end: int) -> None:
nonlocal kind
if kind is not None:
out.append(Block(name, start, end, kind, " › ".join(t for _, t in stack), version,
sum(len(line) + 1 for line in lines[start - 1:end])))
kind = None
for n, raw in enumerate(lines, 1):
f = fence_marker(raw)
if fence:
if closes_fence(f, raw, fence):
fence = ""
close(n)
continue
if f:
close(n - 1)
fence, kind, start = f.group(1), "code", n
continue
if not raw.strip():
close(n - 1)
continue
m = None if n in text_of_string else HEADING.match(raw)
if m:
close(n - 1)
level = len(m.group(1))
stack = [(lv, t) for lv, t in stack if lv < level] + [(level, (m.group(2) or "").strip())]
continue
if raw.lstrip().startswith("|"):
if kind != "table":
close(n - 1)
kind, start = "table", n
continue
if LIST_ITEM.match(raw):
close(n - 1)
kind, start = "item", n
continue
if kind in (None, "table"):
close(n - 1)
if HTML_TAG_ONLY.match(raw):
continue # a tag alone is layout; the block starts at the first line of content
kind, start = "paragraph", n
if fence is not None:
close(len(lines))
else:
last = len(lines)
while last > 0 and not lines[last - 1].strip():
last -= 1
close(last)
return out
def docs_headings(kilde: Path) -> list[Heading]:
"""The title and level 2-3 headings of every page under kilde (any file but a release note), in name order."""
out: list[Heading] = []
for path in (p for p in source_files(kilde) if not RELEASE.match(p.name)):
for n, level, text in headings_of(read_lines(path)):
if level <= 3 and text:
out.append(Heading(path.name, n, level, text))
return out
def blocks_document(kilde: Path, names: list[str]) -> dict[str, Any]:
found: list[dict[str, Any]] = []
for name in names:
path = kilde / name
if not path.is_file():
raise RunError(f"{path} is not a file")
found.extend(asdict(b) for b in blocks(path))
return {"schema": SCHEMA, "order": names, "blocks": found}
def kilde_of(sources: Path) -> Path:
kilde = sources / "kilde"
if not kilde.is_dir():
raise RunError(f"{kilde} is not a directory")
return kilde
def build_parser() -> argparse.ArgumentParser:
p = argparse.ArgumentParser(prog="python3 -m okfwiki.kblocks", description=__doc__,
formatter_class=argparse.RawDescriptionHelpFormatter)
sub = p.add_subparsers(dest="command", required=True)
o = sub.add_parser("order")
o.add_argument("sources")
o.add_argument("--out", required=True)
b = sub.add_parser("blocks")
b.add_argument("sources")
b.add_argument("--order", required=True)
b.add_argument("--out", required=True)
t = sub.add_parser("total")
t.add_argument("sources")
return p
def main(argv: list[str] | None = None) -> int:
args = build_parser().parse_args(argv)
try:
kilde = kilde_of(Path(args.sources))
if args.command == "order":
fixed = order([p.name for p in source_files(kilde)])
write_new(Path(args.out), "".join(n + "\n" for n in fixed))
print(f"order: {len(fixed)} sources -> {args.out}")
elif args.command == "blocks":
names = Path(args.order).read_text(encoding="utf-8").split()
if Path(args.out).exists():
raise RunError(f"{args.out} exists; refusing to overwrite")
doc = blocks_document(kilde, names)
write_new(Path(args.out), json.dumps(doc, ensure_ascii=False, indent=1) + "\n")
print(f"blocks: {len(doc['blocks'])} in {len(names)} sources -> {args.out}")
else:
files = source_files(kilde)
print(f"total: {sum(len(blocks(p)) for p in files)} blocks in {len(files)} files of {kilde}")
except (RunError, OSError, UnicodeDecodeError) as exc:
print(f"kblocks cannot run: {exc}", file=sys.stderr)
return 2
return 0
if __name__ == "__main__":
sys.exit(main())