okf-wiki/okfwiki/common.py
Kjell Tore Guttormsen 0fd86a23a3
okf-wiki 1.1.1
First public release, published as one snapshot. What changed between
versions is in CHANGELOG.md.

Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
2026-11-14 21:03:20 +01:00

121 lines
5.8 KiB
Python

"""Helpers shared by the okf-wiki tools. No model call (CLAUDE.md rule 1)."""
from __future__ import annotations
import re
import unicodedata
from pathlib import Path
from typing import Any
# A release note's version heading: "1.2.3" or "v1.2.3", at least three parts.
VERSION_HEADING = re.compile(r"^v?(\d+(?:\.\d+){2,})$")
# A line that is only one HTML or MDX tag, such as <Note> or </Steps>: layout, not content. The tag name is letters,
# digits, ".", "_" or "-", followed by whitespace, "/" or ">", so a link in angle brackets (<https://...>) is not a tag.
HTML_TAG_ONLY = re.compile(r"^\s*</?[A-Za-z][A-Za-z0-9._-]*(?:\s[^<>]*)?/?>\s*$")
class RunError(Exception):
"""The command cannot run as asked; nothing was written. A command reports it with exit 2."""
def write_new(path: Path, text: str) -> None:
"""Write text to a file that does not exist yet; an existing file is never overwritten."""
if path.exists():
raise RunError(f"{path} exists; refusing to overwrite")
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(text, encoding="utf-8")
def jaccard(a: set[str], b: set[str]) -> float:
return len(a & b) / len(a | b) if a | b else 0.0
# Letters that NFKD does not split into an ASCII letter and a mark, spelled out as Norwegian and German write them.
SPELLED = {"æ": "ae", "ø": "oe", "å": "aa", "ß": "ss", "œ": "oe", "þ": "th", "ð": "d", "ł": "l", "đ": "d"}
def ascii_fold(text: str) -> str:
"""Lower-case text with every letter spelled in ASCII where it can be: æ ø å as ae oe aa, an accent dropped
(é -> e); what has no ASCII spelling is left as it is."""
out = []
for ch in unicodedata.normalize("NFC", text).lower():
if ch in SPELLED:
out.append(SPELLED[ch])
else:
out.append("".join(c for c in unicodedata.normalize("NFKD", ch) if not unicodedata.combining(c)))
return "".join(out)
def slug(text: str) -> str:
"""Lower-case letters and digits joined by single hyphens, at most 60 characters; "x" when nothing is left.
Letters outside ASCII are spelled out first (ascii_fold), so "Årlig" is "aarlig", not "rlig"."""
return re.sub(r"[^a-z0-9]+", "-", ascii_fold(text)).strip("-")[:60] or "x"
def place_words(locator: dict[str, Any]) -> str:
"""Where a quote stands, as a reader names it: page 3, paragraph 4, sheet Plan, cell B7, slide 2 (notes), or line N
when the source is a text file."""
if "page" in locator:
return f"page {locator['page']}"
if "paragraph" in locator:
return f"paragraph {locator['paragraph']}"
if "cell" in locator:
cell = str(locator["cell"])
return f"sheet {locator.get('sheet', '')}, {'cells' if ':' in cell else 'cell'} {cell}"
if "slide" in locator:
return f"slide {locator['slide']}" + (" (notes)" if locator.get("notes") else "")
return f"line {locator.get('line')}"
def place_at(place: dict[str, Any] | None, line: str, quote: str) -> dict[str, Any]:
"""The place in the original of a quote on a line of a frozen copy, from that line's entry in K1 `places`. A
spreadsheet row's entry lists its cells with the offset each starts at, so the quote gets the cell, or the range
of cells, it covers; a quote not on the line gets the whole row. Any other entry is the place as it stands."""
if not place:
return {}
if "cells" not in place:
return dict(place)
cells = [(str(ref), int(at)) for ref, at in place["cells"]]
start = line.find(quote) if quote else -1
end = start + len(quote)
if start < 0:
covered = [ref for ref, _ in cells]
else:
bounds = [at for _, at in cells[1:]] + [len(line) + 1]
covered = [ref for (ref, at), stop in zip(cells, bounds) if at < end and stop > start]
cell = covered[0] if len(covered) == 1 else f"{covered[0]}:{covered[-1]}"
return {"sheet": place["sheet"], "cell": cell}
def place_fits(cited: dict[str, Any], frozen: dict[str, Any] | None) -> bool:
"""Whether a cited place (an x-locator without its line) is the frozen line's place: for a spreadsheet row, the
same sheet and a cell, or a range from one cell to a later one, of that row; otherwise the same place."""
if not frozen or "cells" not in frozen:
return cited == (frozen or {})
refs = [str(ref) for ref, _ in frozen["cells"]]
if set(cited) != {"sheet", "cell"} or cited["sheet"] != frozen["sheet"] or not isinstance(cited["cell"], str):
return False
ends = cited["cell"].split(":")
if not 1 <= len(ends) <= 2 or not all(e in refs for e in ends):
return False
return refs.index(ends[0]) <= refs.index(ends[-1])
# Common words of each language the view can name (no word in both lists): a text's language is the one whose words
# it uses clearly more, as a BCP 47 tag for the HTML lang attribute.
LANGUAGE_WORDS = {
"no": frozenset("og er det som på av til en et ikke med har den de seg var kan om ved fra også eller skal vil"
" når så etter må blir være hvor hva dette disse andre mer bare mellom hver".split()),
"en": frozenset("the and is of to that it not with has they was can about from also or shall will when"
" after must be where what this these other more only between under over each are".split()),
}
LANGUAGE_MIN = 5 # common words a text must have before its language is told
def language(text: str) -> str | None:
"""The language of text ("no" or "en"), or None when it has too few common words or no clear winner."""
words = re.findall(r"[^\W\d_]+", text.lower())
counts = {lang: sum(1 for w in words if w in common) for lang, common in LANGUAGE_WORDS.items()}
best = max(counts, key=lambda k: counts[k])
rest = max(v for k, v in counts.items() if k != best)
return best if counts[best] >= LANGUAGE_MIN and counts[best] >= 2 * rest else None