okf-wiki/okfwiki/view.py
Kjell Tore Guttormsen 0fd86a23a3
okf-wiki 1.1.1
First public release, published as one snapshot. What changed between
versions is in CHANGELOG.md.

Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
2026-11-14 21:03:20 +01:00

1092 lines
51 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

"""The static HTML reading view (plan row L3) behind PLAN.md criterion 3 ("operatørens lesing"). It calls no model.
render BUNDLE SOURCES --out DIR [--graph FILE] [--marks FILE]
One HTML page per concept page, plus index.html, log.html and one raw view per cited source file
(raw/<path>.html, one anchor per line, the cited lines marked). A page's other names (`x-aliases`) stand
under its description as "Also called". Every footnote on a page is a link to
its source entry, which shows the verbatim quote, the locator and a link to the raw source at that
line. --graph copies a finished graph file (Google's viewer output) unchanged to graph.html and links it.
--marks reads a change-marks file (contract K4, okf-wiki/source-marks/1: the pages built on a source that
changed or is gone since the frozen copy) and shows a mark with each source and its change at the top of
every marked page, and the list of marked pages at the top of index.html. Marks that name a page the
bundle lacks, or a source the page does not cite, cannot render. Without --marks the output is unchanged.
check BUNDLE DIR
Counts the footnote references in the markdown pages (the denominator) and resolves each one in the
rendered HTML only: reference -> source entry -> quote and locator -> raw file -> line anchor whose text
holds the quote. Prints "footnotes resolved: X of N". A page, entry, quote, raw file or line that is
missing fails its footnotes (fail-closed).
draw BUNDLE --seed N --count K
K distinct concept pages drawn with random.Random(N).sample over the sorted page list (criterion 3), one
per line, then "sha256 <hex>": the sha256 of the UTF-8 text "seed N\ncount K\n" followed by each drawn page
and a newline, in the order printed, so the draw can be locked in the log.
versions BUNDLE SOURCES
Counts the claim sentences (the denominator, from okfwiki.claims) whose running text holds a version number
that is not the claim (plan row L4). A version number is x.y.z (or longer, optionally with a leading v), or
"version" followed by a dotted number, outside code spans. It is the claim, and not counted, when the
sentence cites a label of an x-outdated pair (dated history), when a quote the sentence cites that is not
only a version holds the same number (the source states it, as in "requires 3.2.1 or later"), or when the
page title holds it (the page is about that version). Prints "... X of N" and every counted sentence.
overlap BUNDLE SOURCES
Counts the source entries that repeat the file and line of another entry on the same page (of all entries),
and the claim sentences that are a prefix or substring of another sentence on the same page (end
punctuation ignored). In the view, entries on the same file and line show once, as the one with the
longest quote; the others stay as zero-height anchors so every reference still reaches its entry.
In the view, the references that end a sentence do not dominate it (plan row L4): a footnote whose quote is only a
version (the version heading of a release note) shows as a small grey version label, and the rest sit behind one small
marker whose popup (hover or focus) shows each quote with a link to its raw line. Every reference is still a link to
its source entry, so the check counts it like any other. A "See also" block lists the pages a page links to or is
linked from; render prints how many pages have neither (islands). The source list is closed by default and grouped
by document and section (the nearest markdown heading at or above the cited line in the raw source, with the span of
cited lines), and "Read more in" links each such section at its heading.
Exit 0 = done (check: N of N, N > 0; versions and overlap: 0 of N, N > 0); 1 = check found a footnote that does not resolve, or
versions or overlap found one; 2 = could not run.
"""
from __future__ import annotations
import argparse
import hashlib
import html
import json
import os
import posixpath
import random
import re
import shutil
import sys
from collections import Counter
from collections.abc import Callable
from dataclasses import dataclass, field
from html.parser import HTMLParser
from pathlib import Path, PurePosixPath
from typing import Any
from okfwiki.audit import (
CODE_SPAN,
FENCE,
FOOTNOTE_DEF,
FOOTNOTE_REF,
HEADING,
LINK,
LIST_ITEM,
SCHEME,
body_links,
is_bundle_link,
Document,
load_bundle,
mask_code,
resolve,
)
from okfwiki.claims import extract
from okfwiki.common import place_words
GRAPH = "graph.html"
BOLD = "\x00"
ORDERED = re.compile(r"^\s*\d+[.)]\s+")
INLINE = re.compile(LINK.pattern + "|" + FOOTNOTE_REF.pattern)
VERSION = re.compile(r"(?<![\w.-])v?(\d+(?:\.\d+){2,})(?![\w-]|\.\d)")
VERSION_WORD = re.compile(r"(?i)\bversions?\s+v?(\d+(?:\.\d+)+)")
VERSION_ONLY = re.compile(r"v?\d+(?:\.\d+){2,}")
STYLE = """
body { font: 16px/1.55 -apple-system, "Segoe UI", Helvetica, Arial, sans-serif; color: #1d1d1f; background: #fff;
max-width: 52rem; margin: 0 auto; padding: 1rem 1rem 4rem; }
nav { font-size: 0.9rem; margin-bottom: 1rem; } nav a { margin-right: 1rem; }
.meta { color: #555; font-size: 0.9rem; border-left: 3px solid #ddd; padding-left: 0.75rem; }
code { background: #f3f3f3; padding: 0 0.2em; border-radius: 3px; }
sup { font-size: 0.7em; line-height: 0; }
a.fnref { text-decoration: none; }
a.fnref.ver { color: #777; background: #f0f0f0; border-radius: 3px; padding: 0 0.3em; margin-left: 0.25em;
font-size: 0.7em; vertical-align: 0.15em; white-space: nowrap; }
.cite { position: relative; cursor: pointer; }
.cite .mark { color: #888; padding: 0 0.15em; }
.cite .pop { display: none; position: absolute; left: 0; top: 1.3em; z-index: 10; width: min(34rem, 85vw);
background: #fff; border: 1px solid #ccc; border-radius: 4px; box-shadow: 0 2px 8px rgba(0,0,0,.15);
padding: 0.4rem 0.6rem; font-size: 0.85rem; line-height: 1.4; cursor: auto; }
.cite:hover .pop, .cite:focus .pop, .cite:focus-within .pop { display: block; }
.pop .pin { display: block; margin: 0.25rem 0; }
.pop q { font-family: ui-monospace, Menlo, monospace; }
ul.see-also li { margin-bottom: 0.3rem; } ul.see-also .desc { color: #555; }
ol.sources li { margin-bottom: 0.9rem; } ol.sources li:target { background: #fff6cc; }
ol.sources li.alias { height: 0; overflow: hidden; margin: 0; list-style: none; }
details.sources-list { margin-top: 1.5rem; } details.sources-list summary { cursor: pointer; font-weight: 600; }
details.sources-list h3 { font-size: 0.95rem; color: #444; margin: 1rem 0 0.3rem; }
ol.sources li.alias:target + li { background: #fff6cc; }
q.quote { display: block; quotes: none; background: #f7f7f7; border-left: 3px solid #999; padding: 0.3rem 0.6rem;
margin: 0.3rem 0; font-family: ui-monospace, Menlo, monospace; font-size: 0.85rem; white-space: pre-wrap; }
.loc { color: #555; font-size: 0.9rem; }
.raw { font-family: ui-monospace, Menlo, monospace; font-size: 0.85rem; }
.line { white-space: pre-wrap; display: grid; grid-template-columns: 4.5rem 1fr; }
.line .ln { color: #999; text-align: right; padding-right: 0.8rem; text-decoration: none; user-select: none; }
.line.cited { background: #eef6ff; } .line:target { background: #fff0a8; outline: 1px solid #e0c200; }
"""
class RunError(Exception):
"""The command cannot run; reported with exit 2."""
@dataclass
class RenderStats:
pages: int = 0
raw_files: int = 0
footnotes: int = 0
islands: int = 0 # concept pages with no link to or from another concept page
@dataclass
class CheckResult:
resolved: int = 0
total: int = 0
per_page: dict[str, tuple[int, int]] = field(default_factory=dict)
problems: list[str] = field(default_factory=list)
# --- markdown to HTML ------------------------------------------------------------------------------------------
def html_name(rel: str) -> str:
return rel[:-3] + ".html" if rel.endswith(".md") else rel + ".html"
def up(rel: str) -> str:
"""The relative prefix from a bundle-relative page back to the output root."""
return "../" * (len(PurePosixPath(rel).parts) - 1)
def esc(text: str) -> str:
return html.escape(text, quote=True)
SAFE_SCHEMES = ("http:", "https:", "mailto:")
def link_target(target: str) -> str | None:
"""A link to another page of the bundle points at its HTML page; an http, https or mailto link is kept; a link
with any other scheme (javascript:, data:, file: ...) gets None, so it renders as plain text."""
if SCHEME.match(target):
return target if target.lower().startswith(SAFE_SCHEMES) else None
if not target or target.startswith("#"):
return target
path, hash_, frag = target.partition("#")
if path.endswith(".md"):
path = path[:-3] + ".html"
return path + hash_ + frag
def plain(text: str) -> str:
"""Escaped text with inline code; bold markers become a sentinel that inline() pairs up."""
out: list[str] = []
pos = 0
for m in CODE_SPAN.finditer(text):
out.append(esc(text[pos:m.start()]).replace("**", BOLD))
out.append(f"<code>{esc(m.group(2))}</code>")
pos = m.end()
out.append(esc(text[pos:]).replace("**", BOLD))
return "".join(out)
def pair_bold(text: str) -> str:
parts = text.split(BOLD)
if len(parts) % 2 == 0: # an odd number of markers: the last one stays literal
parts[-2:] = [parts[-2] + "**" + parts[-1]]
return "".join(p if i % 2 == 0 else f"<strong>{p}</strong>" for i, p in enumerate(parts))
class Footnotes:
"""Display numbers by first use, and the markup of the references that end a sentence."""
def __init__(self, sources: dict[str, dict[str, Any]], page_rel: str = "") -> None:
self.sources = sources
self.page_rel = page_rel
self.canon = canonical(sources)
self.numbers: dict[str, int] = {}
self.seen: Counter[str] = Counter()
def number(self, label: str) -> int:
"""Entries on the same file and line share the number of their canonical entry."""
return self.numbers.setdefault(self.canon.get(label, label), len(self.numbers) + 1)
def ref(self, label: str, extra: str = "") -> str:
"""One link to the source entry: a grey version label for a version heading, else the entry's number."""
n = self.number(label)
self.seen[label] += 1
source = self.sources.get(label, {})
quote = source.get("x-quote", "")
title = f' title="{esc(quote)}"' if isinstance(quote, str) and quote else ""
cls, text = ("fnref ver", esc(str(quote).strip())) if is_version_heading(source) else ("fnref", str(n))
return (f'<a class="{cls}" href="#src-{esc(label)}" id="ref-{esc(label)}-{self.seen[label]}"{title}{extra}>'
f"{text}</a>")
def cite(self, labels: list[str]) -> str:
"""A run of references: the version labels stay visible, the rest sit behind one small marker whose popup
(hover or focus) shows each quote with a link to its raw line. A repeated label gets a hidden link, so the
check still finds one link per reference."""
versions = [lb for lb in labels if is_version_heading(self.sources.get(lb, {}))]
rest = [lb for lb in labels if lb not in versions]
out = "".join(self.ref(lb) for lb in versions)
if not rest:
return out
pins: list[str] = []
shown: set[tuple[str, str]] = set()
for label in rest:
line_of = self.canon.get(label, label)
s = self.sources.get(line_of, {})
quote = str(s["x-quote"]) if isinstance(s.get("x-quote"), str) else ""
own = self.sources.get(label, {}).get("x-quote")
if isinstance(own, str) and own not in quote: # another quote on the same line: show the one that
s, quote = self.sources.get(label, {}), own # supports this claim, not the line's longest
if (line_of, quote) in shown: # the same quote again, from this label or another entry on the line
pins.append(self.ref(label, " hidden"))
continue
shown.add((line_of, quote))
rel, line = raw_rel(self.page_rel, s), locator_line(s)
name = str(s.get("title") or s.get("_original", {}).get("name", ""))
where = esc(", ".join(x for x in (name, place_words(s["x-locator"]) if line else "") if x))
raw = (f' <a class="open" href="{esc(up(self.page_rel))}raw/{esc(rel)}.html#L{line}">{where}</a>'
if rel and line else f" {where}")
pins.append(f'<span class="pin">{self.ref(label)} <q>{esc(str(quote))}</q>{raw}'
f'{original_link(self.page_rel, s)}</span>')
mark = f'<sup class="mark">{self.number(rest[0])}</sup>'
return (f'{out}<span class="cite" tabindex="0">{mark}<span class="pop" role="note">'
+ "".join(pins) + "</span></span>")
def canonical(sources: dict[str, dict[str, Any]]) -> dict[str, str]:
"""Each label to the entry that stands for its file and line: the one with the longest quote (the first on a tie).
An entry without a line locator stands for itself."""
groups: dict[tuple[str, int], list[str]] = {}
out: dict[str, str] = {}
for label, s in sources.items():
line = locator_line(s)
if line is None:
out[label] = label
else:
groups.setdefault((str(s.get("resource", "")), line), []).append(label)
for labels in groups.values():
best = max(labels, key=lambda lb: len(str(sources[lb].get("x-quote", "")))) # max keeps the first on a tie
out.update({lb: best for lb in labels})
return out
def is_version_heading(source: dict[str, Any]) -> bool:
"""A source entry whose quote is only a version number: the version heading of a release note."""
quote = source.get("x-quote")
return isinstance(quote, str) and VERSION_ONLY.fullmatch(quote.strip()) is not None
def inline(text: str, notes: Footnotes | None) -> str:
"""Links, footnote references, inline code and bold; everything else escaped."""
masked = mask_code(text)
out: list[str] = []
pos = 0
run: list[str] = [] # footnote references that follow each other with nothing between them
for m in INLINE.finditer(masked):
note = m.group(3) is not None
if not (note and run and m.start() == pos):
if run and notes is not None:
out.append(notes.cite(run))
run = []
out.append(plain(text[pos:m.start()]))
if note: # a footnote reference
if notes is not None:
run.append(text[m.start(3):m.end(3)])
else:
out.append(esc(text[m.start():m.end()]))
else:
label_text = text[m.start(1):m.end(1)]
target = text[m.start(2):m.end(2)]
href = link_target(target)
out.append(plain(label_text) if href is None else f'<a href="{esc(href)}">{plain(label_text)}</a>')
pos = m.end()
if run and notes is not None:
out.append(notes.cite(run))
out.append(plain(text[pos:]))
return pair_bold("".join(out))
def body_html(doc: Document, notes: Footnotes | None) -> str:
"""Headings, paragraphs and lists of a body; footnote definitions are left out (the sources list replaces them)."""
out: list[str] = []
para: list[str] = []
items: list[str] = []
list_tag = ""
def flush() -> None:
nonlocal list_tag
if para:
out.append(f"<p>{inline(' '.join(para), notes)}</p>")
para.clear()
if items:
out.append(f"<{list_tag}>" + "".join(f"<li>{inline(i, notes)}</li>" for i in items) + f"</{list_tag}>")
items.clear()
list_tag = ""
for raw in doc.body.split("\n"):
stripped = raw.strip()
if not stripped or FOOTNOTE_DEF.match(stripped):
flush()
continue
heading = HEADING.match(raw)
if heading:
flush()
level = len(heading.group(1))
out.append(f"<h{level}>{inline(heading.group(2) or '', notes)}</h{level}>")
elif LIST_ITEM.match(raw):
tag = "ol" if ORDERED.match(raw) else "ul"
if para or (items and tag != list_tag):
flush()
list_tag = tag
items.append(LIST_ITEM.sub("", raw, count=1).strip())
elif items:
items[-1] += " " + stripped
else:
para.append(stripped)
flush()
return "\n".join(out)
def page_shell(title: str, prefix: str, nav_graph: bool, content: str, lang: str = "en") -> str:
graph = f' <a href="{prefix}{GRAPH}">Graph (Google OKF viewer)</a>' if nav_graph else ""
return (f"<!doctype html>\n<html lang=\"{esc(lang)}\">\n<head>\n<meta charset=\"utf-8\">\n"
"<meta name=\"viewport\" content=\"width=device-width, initial-scale=1\">\n"
f"<title>{esc(title)}</title>\n<style>{STYLE}</style>\n</head>\n<body>\n"
f'<nav><a href="{prefix}index.html">Index</a> <a href="{prefix}log.html">Log</a>{graph}</nav>\n'
f"{content}\n</body>\n</html>\n")
# --- render ----------------------------------------------------------------------------------------------------
def source_entries(fm: dict[str, Any]) -> dict[str, dict[str, Any]]:
srcs = fm.get("sources", [])
entries = srcs if isinstance(srcs, list) else [srcs]
return {s["id"]: s for s in entries if isinstance(s, dict) and isinstance(s.get("id"), str)}
def raw_rel(page_rel: str, source: dict[str, Any]) -> str | None:
res = source.get("resource")
if not isinstance(res, str) or not res or SCHEME.match(res):
return None
return resolve(page_rel, res) or None
def locator_line(source: dict[str, Any]) -> int | None:
loc = source.get("x-locator")
if isinstance(loc, dict) and type(loc.get("line")) is int and loc["line"] > 0:
return int(loc["line"])
return None
SectionOf = Callable[[str, int], tuple[int, str] | None]
@dataclass
class Group:
"""The entries of one document section: the nearest heading above the cited lines in the raw source."""
title: str
rel: str
heading_line: int | None
heading: str
labels: list[str] = field(default_factory=list)
lines: list[int] = field(default_factory=list)
@property
def name(self) -> str:
return f"{self.title} › {self.heading}" if self.heading else self.title
def source_groups(page_rel: str, notes: Footnotes, section_of: SectionOf) -> list[Group]:
"""The page's canonical entries grouped by document and section, in the documents' order of first use and then
in the order of the sections in the raw source."""
for label in notes.sources: # entries never referenced are listed after the referenced ones
notes.number(label)
groups: dict[tuple[str, str, int | None], Group] = {}
doc_first: dict[tuple[str, str], int] = {}
for label, n in sorted(notes.numbers.items(), key=lambda kv: kv[1]):
s = notes.sources.get(label, {})
title = str(s.get("title") or s.get("_original", {}).get("name", ""))
rel, line = raw_rel(page_rel, s) or "", locator_line(s)
sec = section_of(rel, line) if rel and line else None
key = (title, rel, sec[0] if sec else None)
g = groups.setdefault(key, Group(title, rel, sec[0] if sec else None, sec[1] if sec else ""))
g.labels.append(label)
if line:
g.lines.append(line)
doc_first.setdefault((title, rel), n)
return sorted(groups.values(), key=lambda g: (doc_first[(g.title, g.rel)], g.heading_line or 0))
def sources_html(page_rel: str, notes: Footnotes, section_of: SectionOf) -> str:
"""The source list, closed by default: one group per document section, the line span of its cited lines."""
parts: list[str] = []
for g in source_groups(page_rel, notes, section_of):
lo, hi = (min(g.lines), max(g.lines)) if g.lines else (0, 0)
span = "" if not g.lines else f" (line {lo})" if lo == hi else f" (lines {lo}–{hi})"
rows: list[str] = []
for label in g.labels:
n = notes.numbers[label]
s = notes.sources.get(label)
if s is None:
rows.append(f'<li id="src-{esc(label)}" value="{n}">No source entry with id <code>{esc(label)}</code>.</li>')
continue
# Other entries on the same file and line come first as zero-height anchors, so the line shows once and
# a link to any of them lands here.
for alias in (lb for lb, c in notes.canon.items() if c == label and lb != label):
rows.append(entry_html(page_rel, alias, notes.sources[alias], n, notes.seen[alias] > 0, ' class="alias"'))
rows.append(entry_html(page_rel, label, s, n, notes.seen[label] > 0))
parts.append(f"<h3>{esc(g.name)}{span}</h3>\n" + '<ol class="sources">\n' + "\n".join(rows) + "\n</ol>")
return (f'<details class="sources-list"><summary>Sources ({len(notes.numbers)})</summary>\n'
+ "\n".join(parts) + "\n</details>")
def read_more_html(page_rel: str, notes: Footnotes, section_of: SectionOf) -> str:
"""One link per document section the page draws on, to the raw source at the section's heading."""
rows = []
for g in source_groups(page_rel, notes, section_of):
if not g.rel or not g.lines:
continue
target = g.heading_line or min(g.lines)
rows.append(f'<li><a href="{esc(up(page_rel))}raw/{esc(g.rel)}.html#L{target}">{esc(g.name)}</a></li>')
return '<h2>Read more in</h2>\n<ul class="read-more">\n' + "\n".join(rows) + "\n</ul>" if rows else ""
def nearest_heading(lines: list[str], line: int) -> tuple[int, str] | None:
"""The nearest markdown heading at or above a line (1-based), as (line, text). A heading inside fenced code is
example text, not a section: fences follow the auditor's rules (backticks or tildes, also in a list item or a
quote block, closed by a run of the same marker at least as long)."""
found: tuple[int, str] | None = None
fence = "" # the open fence's marker run, or "" outside a fenced block
for n, raw in enumerate(lines[:line], 1):
f = FENCE.match(raw)
if f and f.group(1)[0] == "`" and "`" in raw[f.end():]:
f = None # a backtick run with a backtick after it is an inline code span, not a fence (CommonMark 4.5)
if fence:
if f and f.group(1)[0] == fence[0] and len(f.group(1)) >= len(fence) and not raw[f.end():].strip():
fence = ""
continue
if f:
fence = f.group(1)
continue
m = HEADING.match(raw)
if m and m.group(2):
found = (n, m.group(2))
return found
def entry_html(page_rel: str, label: str, s: dict[str, Any], n: int, seen: bool, cls: str = "") -> str:
quote = s.get("x-quote") if isinstance(s.get("x-quote"), str) else ""
line = locator_line(s)
rel = raw_rel(page_rel, s)
res = str(s.get("resource", ""))
loc = f"line {line} of <code>{esc(res)}</code>" if line else f"no line locator, <code>{esc(res)}</code>"
if line and set(s["x-locator"]) != {"line"}:
loc = f"{esc(place_words(s['x-locator']))} of {esc(s.get('_original', {}).get('name', ''))} ({loc})"
raw = (f' <a class="raw" href="{esc(up(page_rel))}raw/{esc(rel)}.html#L{line}">open the raw source at line {line}</a>'
if rel and line else "") + original_link(page_rel, s)
back = f' <a class="back" href="#ref-{esc(label)}-1">back to text</a>' if seen else ""
return (f'<li{cls} id="src-{esc(label)}" value="{n}"><strong>{esc(str(s.get("title", "")))}</strong>'
f'<q class="quote">{esc(str(quote))}</q><span class="loc">{loc}</span>{raw}{back}</li>')
def related(concepts: list[Document]) -> dict[str, list[str]]:
"""For each concept page, the other concept pages it links to or that link to it."""
rels = {d.rel for d in concepts}
links: dict[str, set[str]] = {r: set() for r in rels}
for d in concepts:
for _line, target in body_links(d):
other = resolve(d.rel, target) if is_bundle_link(target) else None
if other in rels and other != d.rel:
links[d.rel].add(other)
links[other].add(d.rel)
return {r: sorted(v) for r, v in links.items()}
def see_also_html(page_rel: str, others: list[str], docs: dict[str, Document]) -> str:
if not others:
return ""
rows = []
for other in others:
fm = docs[other].frontmatter or {}
title = fm.get("title") if isinstance(fm.get("title"), str) else other
desc = f' <span class="desc">- {esc(fm["description"])}</span>' if isinstance(fm.get("description"), str) else ""
href = posixpath.relpath(html_name(other), posixpath.dirname(html_name(page_rel)) or ".")
rows.append(f'<li><a href="{esc(href)}">{esc(str(title))}</a>{desc}</li>')
return '<h2>See also</h2>\n<ul class="see-also">\n' + "\n".join(rows) + "\n</ul>"
def meta_html(doc: Document) -> str:
fm = doc.frontmatter or {}
parts: list[str] = []
if isinstance(fm.get("description"), str):
parts.append(f"<p>{esc(fm['description'])}</p>")
names = fm.get("x-aliases")
if isinstance(names, list) and names:
parts.append("<p>Also called: " + "; ".join(esc(str(n)) for n in names) + "</p>")
tags = fm.get("tags")
if isinstance(tags, list) and tags:
parts.append("<p>Tags: " + ", ".join(esc(str(t)) for t in tags) + "</p>")
gen = fm.get("generated")
if isinstance(gen, dict):
parts.append(f"<p>Generated by {esc(str(gen.get('by', '')))} at {esc(str(gen.get('at', '')))}</p>")
parts.append(f"<p>Page file: <code>{esc(doc.rel)}</code></p>")
return '<div class="meta">' + "".join(parts) + "</div>"
def raw_html(rel: str, text: str, cited: set[int], lang: str = "en") -> str:
rows = []
for n, line in enumerate(text.splitlines(), 1):
cls = "line cited" if n in cited else "line"
rows.append(f'<div class="{cls}" id="L{n}"><a class="ln" href="#L{n}">{n}</a><span class="text">{esc(line)}</span></div>')
digest = hashlib.sha256(text.encode("utf-8")).hexdigest()
content = (f"<h1>Raw source <code>/{esc(rel)}</code></h1>\n"
f'<p class="meta">sha256 {digest} · {len(rows)} lines · cited lines are shaded</p>\n'
f'<div class="raw">\n' + "\n".join(rows) + "\n</div>")
return page_shell(f"/{rel}", up(rel) + "../", False, content, lang)
def write(path: Path, text: str) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(text, encoding="utf-8")
# --- change marks (contract K4) ----------------------------------------------------------------------------------
MARKS_SCHEMA = "okf-wiki/source-marks/1"
CHANGES = ("changed", "gone")
Marks = dict[str, list[tuple[str, str]]] # page -> [(source path, change)], in the file's order
# Inline, so the shared stylesheet, and with it the view without marks, stays as it was.
MARK_STYLE = "border: 2px solid #c25e00; background: #fff4e5; border-radius: 4px; padding: 0.5rem 0.75rem; margin-bottom: 1rem;"
def read_marks(path: Path) -> Marks:
"""The pages and their marked sources from a change-marks file in the form of contract K4 point 2; RunError when the
file cannot be read as one. The K1 module is not imported: the view reads the file itself."""
try:
data = json.loads(path.read_text(encoding="utf-8"))
except (OSError, UnicodeDecodeError, ValueError) as exc:
raise RunError(f"the marks file {path} cannot be read as JSON: {exc}") from exc
if not isinstance(data, dict) or data.get("schema") != MARKS_SCHEMA or not isinstance(data.get("pages"), list):
raise RunError(f"the marks file {path} is not {MARKS_SCHEMA} with a list of pages")
marks: Marks = {}
for entry in data["pages"]:
page = entry.get("page") if isinstance(entry, dict) else None
srcs = entry.get("sources") if isinstance(entry, dict) else None
if not isinstance(page, str) or not isinstance(srcs, list) or not srcs or page in marks:
raise RunError(f"the marks file {path}: an entry is not one page with a list of sources: {entry!r}")
for s in srcs:
if not (isinstance(s, dict) and isinstance(s.get("path"), str) and s.get("change") in CHANGES):
raise RunError(f"the marks file {path}: {page}: a source is not a path with change changed or gone: {s!r}")
marks[page] = [(s["path"], s["change"]) for s in srcs]
return marks
def check_marks(marks: Marks, docs: dict[str, Document]) -> None:
"""RunError unless every marked page is a concept page of the bundle that cites each of its marked sources as
"/" + path in sources[].resource (K4 point 2), so marks from another bundle are never shown."""
for page, entries in marks.items():
if page not in docs:
raise RunError(f"the marks name {page}, which is not a concept page of the bundle")
srcs = (docs[page].frontmatter or {}).get("sources")
resources = {s.get("resource") for s in (srcs if isinstance(srcs, list) else [srcs]) if isinstance(s, dict)}
for path, _change in entries:
if "/" + path not in resources:
raise RunError(f"the marks say {page} builds on /{path}, but the page does not cite it")
def mark_html(entries: list[tuple[str, str]]) -> str:
rows = "".join(f"<li><code>/{esc(path)}</code> {esc(change)}</li>" for path, change in entries)
return (f'<div class="source-mark" role="note" style="{MARK_STYLE}"><strong>This page builds on a source that has '
f"changed or is gone since the frozen copy.</strong>\n<ul>{rows}</ul></div>")
def marked_pages_html(marks: Marks, docs: dict[str, Document]) -> str:
"""The list of marked pages at the top of the root index.html, or a line that says there are none."""
rows = []
for page in sorted(marks):
fm = docs[page].frontmatter or {}
title = fm.get("title") if isinstance(fm.get("title"), str) else page
srcs = ", ".join(f"<code>/{esc(path)}</code> {esc(change)}" for path, change in marks[page])
rows.append(f'<li><a href="{esc(html_name(page))}">{esc(str(title))}</a>: {srcs}</li>')
listing = "<ul>" + "".join(rows) + "</ul>" if rows else "<p>No page builds on a changed or gone source.</p>"
return (f'<div class="marked-pages" style="{MARK_STYLE}"><h2>Pages built on changed or gone sources ({len(rows)})</h2>\n'
f"{listing}</div>\n")
def render(bundle: Path, sources: Path, out: Path, graph: Path | None = None, marks: Marks | None = None) -> RenderStats:
b = load_bundle(bundle, sources)
if marks is not None:
check_marks(marks, {d.rel: d for d in b.concepts})
stats = RenderStats()
cited: dict[str, set[int]] = {}
has_graph = graph is not None
if graph is not None:
out.mkdir(parents=True, exist_ok=True)
shutil.copyfile(graph, out / GRAPH)
docs = {d.rel: d for d in b.concepts}
links = related(b.concepts)
raw_text: dict[str, list[str] | None] = {}
def section_of(rel: str, line: int) -> tuple[int, str] | None:
if rel not in raw_text:
try:
raw_text[rel] = (sources / rel).read_text(encoding="utf-8").splitlines()
except (OSError, UnicodeDecodeError):
raw_text[rel] = None
lines = raw_text[rel]
return nearest_heading(lines, line) if lines is not None else None
stats.islands = sum(1 for others in links.values() if not others)
known = originals(sources)
langs = languages(sources)
counted = Counter(langs.values()).most_common(1)
collection_lang = counted[0][0] if counted else "en"
copies: set[str] = set()
for doc in b.concepts:
fm = doc.frontmatter or {}
entries = source_entries(fm)
for s in entries.values():
rel = raw_rel(doc.rel, s)
if rel in known:
s["_original"] = known[rel] # in memory only: the view names the original and opens its copy
if "copy" in known[rel]:
copies.add(known[rel]["copy"])
notes = Footnotes(entries, doc.rel)
body = body_html(doc, notes)
stats.footnotes += sum(notes.seen.values())
for s in notes.sources.values():
rel, line = raw_rel(doc.rel, s), locator_line(s)
if rel and line:
cited.setdefault(rel, set()).add(line)
title = fm.get("title") if isinstance(fm.get("title"), str) else doc.rel
also = see_also_html(doc.rel, links[doc.rel], docs)
more = read_more_html(doc.rel, notes, section_of)
content = (meta_html(doc) + "\n" + body + "\n" + "".join(x + "\n" for x in (more, also) if x)
+ sources_html(doc.rel, notes, section_of))
if marks is not None and doc.rel in marks:
content = mark_html(marks[doc.rel]) + "\n" + content
cited_langs = Counter(langs[r] for r in (raw_rel(doc.rel, s) for s in entries.values()) if r in langs)
lang = cited_langs.most_common(1)[0][0] if cited_langs else collection_lang
write(out / html_name(doc.rel), page_shell(str(title), up(doc.rel), has_graph, content, lang))
stats.pages += 1
for name in ("index.md", "log.md"):
for doc in b.named(name):
title = "Index" if name == "index.md" else "Log"
listing = marked_pages_html(marks, docs) if marks is not None and doc.rel == "index.md" else ""
write(out / html_name(doc.rel), page_shell(title, up(doc.rel), has_graph, listing + body_html(doc, None),
collection_lang))
for rel, lines in sorted(cited.items()):
try:
text = (sources / rel).read_text(encoding="utf-8")
except (OSError, UnicodeDecodeError):
continue # the check reports its footnotes as unresolved
write(out / "raw" / (rel + ".html"), raw_html(rel, text, lines, langs.get(rel, collection_lang)))
stats.raw_files += 1
for copy in sorted(copies):
if (sources / copy).is_file():
(out / copy).parent.mkdir(parents=True, exist_ok=True)
shutil.copyfile(sources / copy, out / copy)
return stats
def languages(sources: Path) -> dict[str, str]:
"""Per raw source, the language `sources folder` recorded for it (K1 `lang`); none for collection 1."""
try:
doc = json.loads((sources / "source-manifest.json").read_text(encoding="utf-8"))
except (OSError, ValueError):
return {}
return {e["path"]: e["lang"] for e in (doc.get("sources", []) if isinstance(doc, dict) else [])
if isinstance(e, dict) and isinstance(e.get("path"), str) and isinstance(e.get("lang"), str)}
def originals(sources: Path) -> dict[str, dict[str, str]]:
"""Per raw source (kilde/...) of a folder, the original's name, and for PDF and Office its copy under original/."""
try:
doc = json.loads((sources / "source-manifest.json").read_text(encoding="utf-8"))
except (OSError, ValueError):
return {}
found: dict[str, dict[str, str]] = {}
for e in doc.get("sources", []) if isinstance(doc, dict) else []:
if isinstance(e, dict) and isinstance(e.get("path"), str) and isinstance(e.get("original"), str):
found[e["path"]] = {"name": PurePosixPath(e["original"]).name}
if isinstance(e.get("original_copy"), str):
found[e["path"]]["copy"] = e["original_copy"]
return found
def original_link(page_rel: str, s: dict[str, Any]) -> str:
"""A link that opens the original PDF or Office file, a PDF at its page; "" for a text source."""
known, loc = s.get("_original", {}), s.get("x-locator")
if "copy" not in known or not isinstance(loc, dict):
return ""
at = f"#page={loc['page']}" if type(loc.get("page")) is int else ""
words = f"open {known['name']} at {place_words(loc)}" if set(loc) != {"line"} else f"open {known['name']}"
return f' <a class="orig" href="{esc(up(page_rel))}{esc(known["copy"])}{at}">{esc(words)}</a>'
# --- check (reads the rendered HTML only) ----------------------------------------------------------------------
class PageParser(HTMLParser):
"""Footnote references and source entries (quote, locator text, raw links) of one rendered page."""
def __init__(self) -> None:
super().__init__(convert_charrefs=True)
self.refs: list[str] = []
self.entries: dict[str, dict[str, Any]] = {}
self._entry: str | None = None
self._li_depth = 0
self._capture: str | None = None
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
a = {k: v or "" for k, v in attrs}
classes = a.get("class", "").split()
if tag == "a" and "fnref" in classes and a.get("href", "").startswith("#src-"):
self.refs.append(a["href"][len("#src-"):])
if tag == "li":
if self._entry is None and a.get("id", "").startswith("src-"):
self._entry = a["id"][len("src-"):]
self.entries[self._entry] = {"quote": None, "loc": "", "raw": []}
self._li_depth = 0
elif self._entry is not None:
self._li_depth += 1
if self._entry is not None:
e = self.entries[self._entry]
if tag == "q" and "quote" in classes:
self._capture, e["quote"] = "quote", ""
elif tag == "span" and "loc" in classes:
self._capture = "loc"
elif tag == "a" and "raw" in classes:
e["raw"].append(a.get("href", ""))
def handle_endtag(self, tag: str) -> None:
if tag in ("q", "span"):
self._capture = None
if tag == "li" and self._entry is not None:
if self._li_depth == 0:
self._entry = None
else:
self._li_depth -= 1
def handle_data(self, data: str) -> None:
if self._entry is not None and self._capture is not None:
self.entries[self._entry][self._capture] += data
class RawParser(HTMLParser):
"""The text of each line anchor (div id="L<n>" > span.text) of a raw view."""
def __init__(self) -> None:
super().__init__(convert_charrefs=True)
self.lines: dict[str, str] = {}
self._line: str | None = None
self._in_text = False
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
a = {k: v or "" for k, v in attrs}
if tag == "div" and re.fullmatch(r"L\d+", a.get("id", "")):
self._line = a["id"]
self.lines[self._line] = ""
elif tag == "span" and self._line is not None and "text" in a.get("class", "").split():
self._in_text = True
def handle_endtag(self, tag: str) -> None:
if tag == "span":
self._in_text = False
elif tag == "div":
self._line = None
def handle_data(self, data: str) -> None:
if self._in_text and self._line is not None:
self.lines[self._line] += data
def markdown_refs(doc: Document) -> Counter[str]:
"""Footnote references in a page body, outside code spans and footnote definitions (the denominator)."""
found: Counter[str] = Counter()
for raw in mask_code(doc.body).split("\n"):
if not FOOTNOTE_DEF.match(raw.strip()):
found.update(FOOTNOTE_REF.findall(raw))
return found
def check(bundle: Path, out: Path) -> CheckResult:
if not out.is_dir():
raise RunError(f"the view {out} is not a directory")
b = load_bundle(bundle, bundle)
result = CheckResult()
raw_cache: dict[Path, dict[str, str] | None] = {}
root = out.resolve()
def raw_lines(path: Path) -> dict[str, str] | None:
if path not in raw_cache:
try:
parser = RawParser()
parser.feed(path.read_text(encoding="utf-8"))
raw_cache[path] = parser.lines
except (OSError, UnicodeDecodeError):
raw_cache[path] = None
return raw_cache[path]
for doc in b.concepts:
want = markdown_refs(doc)
total = sum(want.values())
page = out / html_name(doc.rel)
ok: Counter[str] = Counter()
try:
parser = PageParser()
parser.feed(page.read_text(encoding="utf-8"))
except (OSError, UnicodeDecodeError):
result.problems.append(f"{doc.rel}: no rendered page {page.name}")
parser = PageParser()
verdict: dict[str, str] = {}
for label in parser.refs:
if label not in verdict:
verdict[label] = resolve_entry(parser.entries.get(label), page, root, raw_lines)
if verdict[label] == "":
ok[label] += 1
for label, why in sorted(verdict.items()):
if why:
result.problems.append(f"{doc.rel}: [^{label}] {why}")
resolved = sum(min(n, ok[label]) for label, n in want.items())
for label, n in sorted(want.items()):
if ok[label] < n and label not in verdict:
result.problems.append(f"{doc.rel}: [^{label}] has no clickable reference in the page")
result.per_page[doc.rel] = (resolved, total)
result.resolved += resolved
result.total += total
return result
def resolve_entry(entry: dict[str, Any] | None, page: Path, root: Path,
raw_lines: Callable[[Path], dict[str, str] | None]) -> str:
""""" when the entry shows a quote, its line and a raw link whose line anchor holds the quote; else why not."""
if entry is None:
return "has no source entry"
quote = entry["quote"]
if not quote:
return "has no quote in its source entry"
for href in entry["raw"]:
path, _, frag = href.partition("#")
m = re.fullmatch(r"L(\d+)", frag)
if not m or f"line {m.group(1)}" not in entry["loc"]:
continue
target = (page.parent / path).resolve()
if not target.is_relative_to(root):
continue
lines = raw_lines(target)
if lines is not None and quote in lines.get(frag, ""):
return ""
return "does not reach a raw line that holds its quote"
# --- draw ------------------------------------------------------------------------------------------------------
def draw(bundle: Path, seed: int, count: int) -> list[str]:
pages = [d.rel for d in load_bundle(bundle, bundle).concepts]
if count > len(pages):
raise RunError(f"cannot draw {count} pages from {len(pages)}")
return random.Random(seed).sample(pages, count)
def draw_sha256(seed: int, count: int, pages: list[str]) -> str:
"""The sha256 that locks a draw: of "seed N\\ncount K\\n" and each page with a newline, in the order printed."""
text = f"seed {seed}\ncount {count}\n" + "".join(page + "\n" for page in pages)
return hashlib.sha256(text.encode("utf-8")).hexdigest()
# --- versions --------------------------------------------------------------------------------------------------
@dataclass(frozen=True)
class Flagged:
page: str
line: int
sentence: str
numbers: tuple[str, ...]
@dataclass
class VersionCount:
total: int = 0
flagged: list[Flagged] = field(default_factory=list)
licensed: dict[str, int] = field(default_factory=lambda: {"outdated": 0, "content-quote": 0, "page-title": 0})
def version_numbers(text: str) -> list[str]:
"""The version numbers in running text (code spans masked), without a leading v, in order of first use."""
masked = mask_code(text)
found = [m.group(1) for m in VERSION.finditer(masked)] + [m.group(1) for m in VERSION_WORD.finditer(masked)]
return list(dict.fromkeys(found))
def holds_number(text: str, number: str) -> bool:
return re.search(r"(?<![\d.])" + re.escape(number) + r"(?!\d|\.\d)", text) is not None
def versions(bundle: Path, sources: Path) -> VersionCount:
docs = {d.rel: d for d in load_bundle(bundle, sources).concepts}
result = VersionCount()
for claim in extract(bundle, sources).claims:
result.total += 1
numbers = version_numbers(claim.claim)
if not numbers:
continue
fm = docs[claim.page].frontmatter or {}
pairs = fm.get("x-outdated")
outdated = {str(p.get(k)) for p in (pairs if isinstance(pairs, list) else []) if isinstance(p, dict)
for k in ("old", "new")}
if outdated & set(claim.footnotes):
result.licensed["outdated"] += 1
continue
entries = source_entries(fm)
quotes = [str(entries[k].get("x-quote", "")) for k in claim.footnotes
if k in entries and not is_version_heading(entries[k])]
title = str(fm.get("title", ""))
left = [n for n in numbers if not any(holds_number(q, n) for q in quotes)]
if not left:
result.licensed["content-quote"] += 1
elif all(holds_number(title, n) for n in left):
result.licensed["page-title"] += 1
else:
result.flagged.append(Flagged(claim.page, claim.line, claim.claim, tuple(left)))
return result
# --- overlap ---------------------------------------------------------------------------------------------------
@dataclass
class OverlapCount:
entries: int = 0
redundant: list[tuple[str, str, str]] = field(default_factory=list) # (page, kept label, redundant label)
claims: int = 0
substring: list[tuple[str, str, str]] = field(default_factory=list) # (page, sentence, the longer sentence)
def _bare(sentence: str) -> str:
return sentence.strip().rstrip(".!?").strip()
def overlap(bundle: Path, sources: Path) -> OverlapCount:
"""Source entries on a page that repeat another entry's file and line, and sentences that are a prefix or
substring of another sentence on the same page (end punctuation ignored)."""
result = OverlapCount()
for doc in load_bundle(bundle, sources).concepts:
entries = source_entries(doc.frontmatter or {})
result.entries += len(entries)
for label, kept in canonical(entries).items():
if label != kept:
result.redundant.append((doc.rel, kept, label))
by_page: dict[str, list[str]] = {}
for claim in extract(bundle, sources).claims:
result.claims += 1
by_page.setdefault(claim.page, []).append(claim.claim)
for page, sentences in by_page.items():
for i, a in enumerate(sentences):
longer = next((b for j, b in enumerate(sentences) if j != i and _bare(a) in _bare(b)), None)
if longer is not None:
result.substring.append((page, a, longer))
return result
# --- command line ----------------------------------------------------------------------------------------------
def build_parser() -> argparse.ArgumentParser:
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
sub = p.add_subparsers(dest="command", required=True)
r = sub.add_parser("render", help="write the HTML view")
r.add_argument("bundle")
r.add_argument("sources")
r.add_argument("--out", required=True)
r.add_argument("--graph", help="a finished graph file (Google's viewer output), copied unchanged")
r.add_argument("--marks", help="a change-marks file (contract K4): mark the pages built on changed or gone sources")
c = sub.add_parser("check", help="count the footnotes that resolve in the HTML view")
c.add_argument("bundle")
c.add_argument("view")
d = sub.add_parser("draw", help="draw pages for the operator's reading")
d.add_argument("bundle")
d.add_argument("--seed", type=int, required=True)
d.add_argument("--count", type=int, required=True)
v = sub.add_parser("versions", help="count version numbers in running text that are not the claim")
v.add_argument("bundle")
v.add_argument("sources")
o = sub.add_parser("overlap", help="count redundant source entries and overlapping sentences")
o.add_argument("bundle")
o.add_argument("sources")
return p
def main(argv: list[str] | None = None) -> int:
args = build_parser().parse_args(argv)
try:
if not Path(args.bundle).is_dir():
raise RunError(f"the bundle {args.bundle} is not a directory")
if args.command == "render":
if not Path(args.sources).is_dir():
raise RunError(f"the raw-source root {args.sources} is not a directory")
graph = Path(args.graph) if args.graph else None
if graph is not None and not graph.is_file():
raise RunError(f"the graph file {graph} does not exist")
marks = read_marks(Path(args.marks)) if args.marks else None
s = render(Path(args.bundle), Path(args.sources), Path(args.out), graph, marks)
print(f"render: {s.pages} pages, {s.raw_files} raw source files, {s.footnotes} footnotes -> {args.out}")
print(f"pages without a link to or from another page (islands): {s.islands} of {s.pages}")
return 0
if args.command == "check":
res = check(Path(args.bundle), Path(args.view))
print(f"footnotes resolved: {res.resolved} of {res.total}")
short = [f"{p}: {r} of {t}" for p, (r, t) in sorted(res.per_page.items()) if r < t]
for line in short + res.problems[:50]:
print(" " + line)
if res.total == 0:
print("FAIL: no footnotes to resolve", file=sys.stderr)
return 1
return 0 if res.resolved == res.total else 1
if args.command in ("versions", "overlap") and not Path(args.sources).is_dir():
raise RunError(f"the raw-source root {args.sources} is not a directory")
if args.command == "overlap":
oc = overlap(Path(args.bundle), Path(args.sources))
print(f"redundant source entries (same file and line): {len(oc.redundant)} of {oc.entries}")
print("sentences that are a prefix or substring of another on the same page: "
f"{len(oc.substring)} of {oc.claims}")
for page, n in sorted(Counter(p for p, _, _ in oc.redundant).items()):
print(f" {page}: {n} redundant")
for page, kept, label in oc.redundant:
print(f" {page}: [^{label}] repeats the file and line of [^{kept}]")
for page, a, b in oc.substring:
print(f" {page}: \"{a}\" is inside \"{b}\"")
if oc.entries == 0 or oc.claims == 0:
print("FAIL: no source entries or claim sentences to count", file=sys.stderr)
return 1
return 0 if not oc.redundant and not oc.substring else 1
if args.command == "versions":
vc = versions(Path(args.bundle), Path(args.sources))
print(f"version numbers in running text outside dated history: {len(vc.flagged)} of {vc.total}")
print(" not counted (the version is the claim): "
+ ", ".join(f"{k} {n}" for k, n in vc.licensed.items()))
for page, n in sorted(Counter(f.page for f in vc.flagged).items()):
print(f" {page}: {n}")
for f in vc.flagged:
print(f" {f.page}:{f.line} [{' '.join(f.numbers)}] {f.sentence}")
if vc.total == 0:
print("FAIL: no claim sentences to count", file=sys.stderr)
return 1
return 0 if not vc.flagged else 1
pages = draw(Path(args.bundle), args.seed, args.count)
for page in pages:
print(page)
print(f"sha256 {draw_sha256(args.seed, args.count, pages)}")
return 0
except RunError as exc:
print(f"FAIL: {exc}", file=sys.stderr)
return 2
if __name__ == "__main__":
sys.exit(main())