First public release, published as one snapshot. What changed between versions is in CHANGELOG.md. Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
121 lines
5.8 KiB
Python
121 lines
5.8 KiB
Python
"""Helpers shared by the okf-wiki tools. No model call (CLAUDE.md rule 1)."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import re
|
|
import unicodedata
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
# A release note's version heading: "1.2.3" or "v1.2.3", at least three parts.
|
|
VERSION_HEADING = re.compile(r"^v?(\d+(?:\.\d+){2,})$")
|
|
# A line that is only one HTML or MDX tag, such as <Note> or </Steps>: layout, not content. The tag name is letters,
|
|
# digits, ".", "_" or "-", followed by whitespace, "/" or ">", so a link in angle brackets (<https://...>) is not a tag.
|
|
HTML_TAG_ONLY = re.compile(r"^\s*</?[A-Za-z][A-Za-z0-9._-]*(?:\s[^<>]*)?/?>\s*$")
|
|
|
|
|
|
class RunError(Exception):
|
|
"""The command cannot run as asked; nothing was written. A command reports it with exit 2."""
|
|
|
|
|
|
def write_new(path: Path, text: str) -> None:
|
|
"""Write text to a file that does not exist yet; an existing file is never overwritten."""
|
|
if path.exists():
|
|
raise RunError(f"{path} exists; refusing to overwrite")
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
path.write_text(text, encoding="utf-8")
|
|
|
|
|
|
def jaccard(a: set[str], b: set[str]) -> float:
|
|
return len(a & b) / len(a | b) if a | b else 0.0
|
|
|
|
|
|
# Letters that NFKD does not split into an ASCII letter and a mark, spelled out as Norwegian and German write them.
|
|
SPELLED = {"æ": "ae", "ø": "oe", "å": "aa", "ß": "ss", "œ": "oe", "þ": "th", "ð": "d", "ł": "l", "đ": "d"}
|
|
|
|
|
|
def ascii_fold(text: str) -> str:
|
|
"""Lower-case text with every letter spelled in ASCII where it can be: æ ø å as ae oe aa, an accent dropped
|
|
(é -> e); what has no ASCII spelling is left as it is."""
|
|
out = []
|
|
for ch in unicodedata.normalize("NFC", text).lower():
|
|
if ch in SPELLED:
|
|
out.append(SPELLED[ch])
|
|
else:
|
|
out.append("".join(c for c in unicodedata.normalize("NFKD", ch) if not unicodedata.combining(c)))
|
|
return "".join(out)
|
|
|
|
|
|
def slug(text: str) -> str:
|
|
"""Lower-case letters and digits joined by single hyphens, at most 60 characters; "x" when nothing is left.
|
|
Letters outside ASCII are spelled out first (ascii_fold), so "Årlig" is "aarlig", not "rlig"."""
|
|
return re.sub(r"[^a-z0-9]+", "-", ascii_fold(text)).strip("-")[:60] or "x"
|
|
|
|
|
|
def place_words(locator: dict[str, Any]) -> str:
|
|
"""Where a quote stands, as a reader names it: page 3, paragraph 4, sheet Plan, cell B7, slide 2 (notes), or line N
|
|
when the source is a text file."""
|
|
if "page" in locator:
|
|
return f"page {locator['page']}"
|
|
if "paragraph" in locator:
|
|
return f"paragraph {locator['paragraph']}"
|
|
if "cell" in locator:
|
|
cell = str(locator["cell"])
|
|
return f"sheet {locator.get('sheet', '')}, {'cells' if ':' in cell else 'cell'} {cell}"
|
|
if "slide" in locator:
|
|
return f"slide {locator['slide']}" + (" (notes)" if locator.get("notes") else "")
|
|
return f"line {locator.get('line')}"
|
|
|
|
|
|
def place_at(place: dict[str, Any] | None, line: str, quote: str) -> dict[str, Any]:
|
|
"""The place in the original of a quote on a line of a frozen copy, from that line's entry in K1 `places`. A
|
|
spreadsheet row's entry lists its cells with the offset each starts at, so the quote gets the cell, or the range
|
|
of cells, it covers; a quote not on the line gets the whole row. Any other entry is the place as it stands."""
|
|
if not place:
|
|
return {}
|
|
if "cells" not in place:
|
|
return dict(place)
|
|
cells = [(str(ref), int(at)) for ref, at in place["cells"]]
|
|
start = line.find(quote) if quote else -1
|
|
end = start + len(quote)
|
|
if start < 0:
|
|
covered = [ref for ref, _ in cells]
|
|
else:
|
|
bounds = [at for _, at in cells[1:]] + [len(line) + 1]
|
|
covered = [ref for (ref, at), stop in zip(cells, bounds) if at < end and stop > start]
|
|
cell = covered[0] if len(covered) == 1 else f"{covered[0]}:{covered[-1]}"
|
|
return {"sheet": place["sheet"], "cell": cell}
|
|
|
|
|
|
def place_fits(cited: dict[str, Any], frozen: dict[str, Any] | None) -> bool:
|
|
"""Whether a cited place (an x-locator without its line) is the frozen line's place: for a spreadsheet row, the
|
|
same sheet and a cell, or a range from one cell to a later one, of that row; otherwise the same place."""
|
|
if not frozen or "cells" not in frozen:
|
|
return cited == (frozen or {})
|
|
refs = [str(ref) for ref, _ in frozen["cells"]]
|
|
if set(cited) != {"sheet", "cell"} or cited["sheet"] != frozen["sheet"] or not isinstance(cited["cell"], str):
|
|
return False
|
|
ends = cited["cell"].split(":")
|
|
if not 1 <= len(ends) <= 2 or not all(e in refs for e in ends):
|
|
return False
|
|
return refs.index(ends[0]) <= refs.index(ends[-1])
|
|
|
|
|
|
# Common words of each language the view can name (no word in both lists): a text's language is the one whose words
|
|
# it uses clearly more, as a BCP 47 tag for the HTML lang attribute.
|
|
LANGUAGE_WORDS = {
|
|
"no": frozenset("og er det som på av til en et ikke med har den de seg var kan om ved fra også eller skal vil"
|
|
" når så etter må blir være hvor hva dette disse andre mer bare mellom hver".split()),
|
|
"en": frozenset("the and is of to that it not with has they was can about from also or shall will when"
|
|
" after must be where what this these other more only between under over each are".split()),
|
|
}
|
|
LANGUAGE_MIN = 5 # common words a text must have before its language is told
|
|
|
|
|
|
def language(text: str) -> str | None:
|
|
"""The language of text ("no" or "en"), or None when it has too few common words or no clear winner."""
|
|
words = re.findall(r"[^\W\d_]+", text.lower())
|
|
counts = {lang: sum(1 for w in words if w in common) for lang, common in LANGUAGE_WORDS.items()}
|
|
best = max(counts, key=lambda k: counts[k])
|
|
rest = max(v for k, v in counts.items() if k != best)
|
|
return best if counts[best] >= LANGUAGE_MIN and counts[best] >= 2 * rest else None
|