okf-wiki 1.1.1

First public release, published as one snapshot. What changed between
versions is in CHANGELOG.md.

Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
This commit is contained in:
Kjell Tore Guttormsen 2026-11-14 21:03:20 +01:00
commit 0fd86a23a3
Signed by: ktg
SSH key fingerprint: SHA256:JakMjO6FTBBzN0Bhfj9saOoEjaFxlSdYuZQQpM/lF9Q
227 changed files with 96077 additions and 0 deletions

290
okfwiki/kblocks.py Normal file
View file

@ -0,0 +1,290 @@
"""K's sources in a fixed order, split into blocks with line spans. No model call (CLAUDE.md rule 1).
Usage:
python3 -m okfwiki.kblocks order SOURCES --out FILE
python3 -m okfwiki.kblocks blocks SOURCES --order FILE --out FILE
python3 -m okfwiki.kblocks total SOURCES
SOURCES is the folder that holds kilde/, the same raw-source root the auditor takes.
order every file under SOURCES/kilde/ in K's fixed order, one name per line: pages by name, then release notes
(release-vX.Y.Z.md) by version. Any other name is a page: a docs page (docs-*), or a document of a folder
(sources folder), .md or .txt alike.
blocks every block of every source in the order (schema okf-wiki/blocks/1): a paragraph, one list item with its
continuation lines, one table or one fenced code block, with its 1-based inclusive line span, its section (the
heading path), its characters (`chars`, every line with its newline) and, for a release note, its version.
Heading lines are section markers, not blocks, and a line that is only an HTML or MDX tag (<Note>, </Steps>)
never starts a block, so no block is only a tag. A "#" line
inside a component property's example string (example: `# Title ... `) is example text, not a heading.
total the number of blocks in every file under SOURCES/kilde/.
Lines are counted on the text split at "\\n", as the pack and the link pass count them. Exit 0 = done, 2 = could not
run; an existing output is never overwritten.
"""
from __future__ import annotations
import argparse
import json
import re
import sys
from dataclasses import asdict, dataclass
from pathlib import Path
from typing import Any
from okfwiki.audit import HEADING, LIST_ITEM, closes_fence, fence_marker
from okfwiki.common import HTML_TAG_ONLY, VERSION_HEADING, RunError, write_new
SCHEMA = "okf-wiki/blocks/1"
RELEASE = re.compile(r"^release-v(\d+(?:\.\d+)*)\.md$")
VERSION_LABEL = re.compile(r'^\s*<[A-Za-z][A-Za-z0-9._-]*\s[^<>]*?\blabel="v?(\d+(?:\.\d+){2,})"')
VERSION_BOLD = re.compile(r"^\*\*v?(\d+(?:\.\d+){2,})\*\*$")
# A component property whose value is a backtick string left open on its line (example: `# Title): the lines after it,
# up to the first with a backtick no backslash escapes, are example text, never a heading (order 14 findings C and D).
STRING_OPEN = re.compile(r"^\s*[A-Za-z_$][\w$]*\s*:\s*`")
TICK = re.compile(r"(?<!\\)`")
# Concept test 2 of the brief (Google's reference agent), plus the generic section names of docs pages.
META_STOP = frozenset({
"overview", "introduction", "getting started", "quickstart", "tutorial", "walkthrough", "release notes",
"changelog", "roadmap", "faq", "prerequisites", "next steps", "before you begin", "troubleshooting", "see also",
"related resources", "learn more",
})
@dataclass(frozen=True)
class Block:
file: str
start: int
end: int
kind: str # paragraph | item | table | code
section: str
version: str | None
chars: int = 0 # the characters of its lines, each with its newline: the unit the stop line prices (B17)
@dataclass(frozen=True)
class Heading:
file: str
line: int
level: int
text: str
def is_meta(title: str) -> bool:
return title.strip().lower() in META_STOP
def version_key(name: str) -> tuple[int, ...]:
m = RELEASE.match(name)
if m is None:
raise RunError(f"{name} is not a release note name")
return tuple(int(p) for p in m.group(1).split("."))
def order(names: list[str]) -> list[str]:
releases = [n for n in names if RELEASE.match(n)]
return sorted(n for n in names if not RELEASE.match(n)) + sorted(releases, key=version_key)
def source_files(kilde: Path) -> list[Path]:
"""Every file under kilde, in name order: .md and .txt alike, nothing dropped by its suffix."""
return sorted(p for p in kilde.iterdir() if p.is_file())
def read_lines(path: Path) -> list[str]:
return path.read_text(encoding="utf-8").split("\n")
def string_lines(lines: list[str]) -> set[int]:
"""The 1-based lines inside a component property's example string (STRING_OPEN), outside fenced code: from the line
after the one that opens it to the line with the backtick that closes it, both included."""
out: set[int] = set()
fence = "" # the open fence's marker run, or "" outside fenced code (the auditor's rule, CommonMark 4.5)
inside = False
for n, raw in enumerate(lines, 1):
if inside:
out.add(n)
inside = TICK.search(raw) is None
continue
f = fence_marker(raw)
if fence:
if closes_fence(f, raw, fence):
fence = ""
elif f:
fence = f.group(1)
elif STRING_OPEN.match(raw):
inside = len(TICK.findall(raw)) % 2 == 1
return out
def headings_of(lines: list[str]) -> list[tuple[int, int, str]]:
"""(line, level, text) of every heading outside fenced code and outside a component's example string."""
out: list[tuple[int, int, str]] = []
fence = ""
text_of_string = string_lines(lines)
for n, raw in enumerate(lines, 1):
if n in text_of_string:
continue
f = fence_marker(raw)
if fence:
if closes_fence(f, raw, fence):
fence = ""
continue
if f:
fence = f.group(1)
continue
m = HEADING.match(raw)
if m:
out.append((n, len(m.group(1)), (m.group(2) or "").strip()))
return out
def stated_versions(lines: list[str]) -> list[tuple[int, str]]:
"""(line, version) for every line that states a release note's version: a heading that is only the version, then
a component whose label is the version (<Update label="X.Y.Z" ...>: every release note of the copy of 08.10, 416
of 416, has this form and no heading; order 27 A6), then a line that is only the bold version (**X.Y.Z**: the
page's form after 20.10, 437 of 437 on 07.11)."""
found = [(n, m.group(1)) for n, _level, text in headings_of(lines) if (m := VERSION_HEADING.match(text))]
found += [(n, m.group(1)) for n, raw in enumerate(lines, 1) if (m := VERSION_LABEL.match(raw))]
return found + [(n, m.group(1)) for n, raw in enumerate(lines, 1) if (m := VERSION_BOLD.match(raw))]
def version_line(lines: list[str], version: str) -> int | None:
"""The first line that states `version`, or None."""
return next((n for n, v in stated_versions(lines) if v == version), None)
def blocks(path: Path) -> list[Block]:
lines = read_lines(path)
name = path.name
version: str | None = None
if RELEASE.match(name):
stated = stated_versions(lines)
version = stated[0][1] if stated else None
out: list[Block] = []
stack: list[tuple[int, str]] = []
kind: str | None = None
start = 0
fence = ""
text_of_string = string_lines(lines)
def close(end: int) -> None:
nonlocal kind
if kind is not None:
out.append(Block(name, start, end, kind, " › ".join(t for _, t in stack), version,
sum(len(line) + 1 for line in lines[start - 1:end])))
kind = None
for n, raw in enumerate(lines, 1):
f = fence_marker(raw)
if fence:
if closes_fence(f, raw, fence):
fence = ""
close(n)
continue
if f:
close(n - 1)
fence, kind, start = f.group(1), "code", n
continue
if not raw.strip():
close(n - 1)
continue
m = None if n in text_of_string else HEADING.match(raw)
if m:
close(n - 1)
level = len(m.group(1))
stack = [(lv, t) for lv, t in stack if lv < level] + [(level, (m.group(2) or "").strip())]
continue
if raw.lstrip().startswith("|"):
if kind != "table":
close(n - 1)
kind, start = "table", n
continue
if LIST_ITEM.match(raw):
close(n - 1)
kind, start = "item", n
continue
if kind in (None, "table"):
close(n - 1)
if HTML_TAG_ONLY.match(raw):
continue # a tag alone is layout; the block starts at the first line of content
kind, start = "paragraph", n
if fence is not None:
close(len(lines))
else:
last = len(lines)
while last > 0 and not lines[last - 1].strip():
last -= 1
close(last)
return out
def docs_headings(kilde: Path) -> list[Heading]:
"""The title and level 2-3 headings of every page under kilde (any file but a release note), in name order."""
out: list[Heading] = []
for path in (p for p in source_files(kilde) if not RELEASE.match(p.name)):
for n, level, text in headings_of(read_lines(path)):
if level <= 3 and text:
out.append(Heading(path.name, n, level, text))
return out
def blocks_document(kilde: Path, names: list[str]) -> dict[str, Any]:
found: list[dict[str, Any]] = []
for name in names:
path = kilde / name
if not path.is_file():
raise RunError(f"{path} is not a file")
found.extend(asdict(b) for b in blocks(path))
return {"schema": SCHEMA, "order": names, "blocks": found}
def kilde_of(sources: Path) -> Path:
kilde = sources / "kilde"
if not kilde.is_dir():
raise RunError(f"{kilde} is not a directory")
return kilde
def build_parser() -> argparse.ArgumentParser:
p = argparse.ArgumentParser(prog="python3 -m okfwiki.kblocks", description=__doc__,
formatter_class=argparse.RawDescriptionHelpFormatter)
sub = p.add_subparsers(dest="command", required=True)
o = sub.add_parser("order")
o.add_argument("sources")
o.add_argument("--out", required=True)
b = sub.add_parser("blocks")
b.add_argument("sources")
b.add_argument("--order", required=True)
b.add_argument("--out", required=True)
t = sub.add_parser("total")
t.add_argument("sources")
return p
def main(argv: list[str] | None = None) -> int:
args = build_parser().parse_args(argv)
try:
kilde = kilde_of(Path(args.sources))
if args.command == "order":
fixed = order([p.name for p in source_files(kilde)])
write_new(Path(args.out), "".join(n + "\n" for n in fixed))
print(f"order: {len(fixed)} sources -> {args.out}")
elif args.command == "blocks":
names = Path(args.order).read_text(encoding="utf-8").split()
if Path(args.out).exists():
raise RunError(f"{args.out} exists; refusing to overwrite")
doc = blocks_document(kilde, names)
write_new(Path(args.out), json.dumps(doc, ensure_ascii=False, indent=1) + "\n")
print(f"blocks: {len(doc['blocks'])} in {len(names)} sources -> {args.out}")
else:
files = source_files(kilde)
print(f"total: {sum(len(blocks(p)) for p in files)} blocks in {len(files)} files of {kilde}")
except (RunError, OSError, UnicodeDecodeError) as exc:
print(f"kblocks cannot run: {exc}", file=sys.stderr)
return 2
return 0
if __name__ == "__main__":
sys.exit(main())