First public release, published as one snapshot. What changed between versions is in CHANGELOG.md. Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
1092 lines
51 KiB
Python
1092 lines
51 KiB
Python
"""The static HTML reading view (plan row L3) behind PLAN.md criterion 3 ("operatørens lesing"). It calls no model.
|
||
|
||
render BUNDLE SOURCES --out DIR [--graph FILE] [--marks FILE]
|
||
One HTML page per concept page, plus index.html, log.html and one raw view per cited source file
|
||
(raw/<path>.html, one anchor per line, the cited lines marked). A page's other names (`x-aliases`) stand
|
||
under its description as "Also called". Every footnote on a page is a link to
|
||
its source entry, which shows the verbatim quote, the locator and a link to the raw source at that
|
||
line. --graph copies a finished graph file (Google's viewer output) unchanged to graph.html and links it.
|
||
--marks reads a change-marks file (contract K4, okf-wiki/source-marks/1: the pages built on a source that
|
||
changed or is gone since the frozen copy) and shows a mark with each source and its change at the top of
|
||
every marked page, and the list of marked pages at the top of index.html. Marks that name a page the
|
||
bundle lacks, or a source the page does not cite, cannot render. Without --marks the output is unchanged.
|
||
check BUNDLE DIR
|
||
Counts the footnote references in the markdown pages (the denominator) and resolves each one in the
|
||
rendered HTML only: reference -> source entry -> quote and locator -> raw file -> line anchor whose text
|
||
holds the quote. Prints "footnotes resolved: X of N". A page, entry, quote, raw file or line that is
|
||
missing fails its footnotes (fail-closed).
|
||
draw BUNDLE --seed N --count K
|
||
K distinct concept pages drawn with random.Random(N).sample over the sorted page list (criterion 3), one
|
||
per line, then "sha256 <hex>": the sha256 of the UTF-8 text "seed N\ncount K\n" followed by each drawn page
|
||
and a newline, in the order printed, so the draw can be locked in the log.
|
||
versions BUNDLE SOURCES
|
||
Counts the claim sentences (the denominator, from okfwiki.claims) whose running text holds a version number
|
||
that is not the claim (plan row L4). A version number is x.y.z (or longer, optionally with a leading v), or
|
||
"version" followed by a dotted number, outside code spans. It is the claim, and not counted, when the
|
||
sentence cites a label of an x-outdated pair (dated history), when a quote the sentence cites that is not
|
||
only a version holds the same number (the source states it, as in "requires 3.2.1 or later"), or when the
|
||
page title holds it (the page is about that version). Prints "... X of N" and every counted sentence.
|
||
overlap BUNDLE SOURCES
|
||
Counts the source entries that repeat the file and line of another entry on the same page (of all entries),
|
||
and the claim sentences that are a prefix or substring of another sentence on the same page (end
|
||
punctuation ignored). In the view, entries on the same file and line show once, as the one with the
|
||
longest quote; the others stay as zero-height anchors so every reference still reaches its entry.
|
||
|
||
In the view, the references that end a sentence do not dominate it (plan row L4): a footnote whose quote is only a
|
||
version (the version heading of a release note) shows as a small grey version label, and the rest sit behind one small
|
||
marker whose popup (hover or focus) shows each quote with a link to its raw line. Every reference is still a link to
|
||
its source entry, so the check counts it like any other. A "See also" block lists the pages a page links to or is
|
||
linked from; render prints how many pages have neither (islands). The source list is closed by default and grouped
|
||
by document and section (the nearest markdown heading at or above the cited line in the raw source, with the span of
|
||
cited lines), and "Read more in" links each such section at its heading.
|
||
|
||
Exit 0 = done (check: N of N, N > 0; versions and overlap: 0 of N, N > 0); 1 = check found a footnote that does not resolve, or
|
||
versions or overlap found one; 2 = could not run.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import hashlib
|
||
import html
|
||
import json
|
||
import os
|
||
import posixpath
|
||
import random
|
||
import re
|
||
import shutil
|
||
import sys
|
||
from collections import Counter
|
||
from collections.abc import Callable
|
||
from dataclasses import dataclass, field
|
||
from html.parser import HTMLParser
|
||
from pathlib import Path, PurePosixPath
|
||
from typing import Any
|
||
|
||
from okfwiki.audit import (
|
||
CODE_SPAN,
|
||
FENCE,
|
||
FOOTNOTE_DEF,
|
||
FOOTNOTE_REF,
|
||
HEADING,
|
||
LINK,
|
||
LIST_ITEM,
|
||
SCHEME,
|
||
body_links,
|
||
is_bundle_link,
|
||
Document,
|
||
load_bundle,
|
||
mask_code,
|
||
resolve,
|
||
)
|
||
from okfwiki.claims import extract
|
||
from okfwiki.common import place_words
|
||
|
||
GRAPH = "graph.html"
|
||
BOLD = "\x00"
|
||
ORDERED = re.compile(r"^\s*\d+[.)]\s+")
|
||
INLINE = re.compile(LINK.pattern + "|" + FOOTNOTE_REF.pattern)
|
||
VERSION = re.compile(r"(?<![\w.-])v?(\d+(?:\.\d+){2,})(?![\w-]|\.\d)")
|
||
VERSION_WORD = re.compile(r"(?i)\bversions?\s+v?(\d+(?:\.\d+)+)")
|
||
VERSION_ONLY = re.compile(r"v?\d+(?:\.\d+){2,}")
|
||
|
||
STYLE = """
|
||
body { font: 16px/1.55 -apple-system, "Segoe UI", Helvetica, Arial, sans-serif; color: #1d1d1f; background: #fff;
|
||
max-width: 52rem; margin: 0 auto; padding: 1rem 1rem 4rem; }
|
||
nav { font-size: 0.9rem; margin-bottom: 1rem; } nav a { margin-right: 1rem; }
|
||
.meta { color: #555; font-size: 0.9rem; border-left: 3px solid #ddd; padding-left: 0.75rem; }
|
||
code { background: #f3f3f3; padding: 0 0.2em; border-radius: 3px; }
|
||
sup { font-size: 0.7em; line-height: 0; }
|
||
a.fnref { text-decoration: none; }
|
||
a.fnref.ver { color: #777; background: #f0f0f0; border-radius: 3px; padding: 0 0.3em; margin-left: 0.25em;
|
||
font-size: 0.7em; vertical-align: 0.15em; white-space: nowrap; }
|
||
.cite { position: relative; cursor: pointer; }
|
||
.cite .mark { color: #888; padding: 0 0.15em; }
|
||
.cite .pop { display: none; position: absolute; left: 0; top: 1.3em; z-index: 10; width: min(34rem, 85vw);
|
||
background: #fff; border: 1px solid #ccc; border-radius: 4px; box-shadow: 0 2px 8px rgba(0,0,0,.15);
|
||
padding: 0.4rem 0.6rem; font-size: 0.85rem; line-height: 1.4; cursor: auto; }
|
||
.cite:hover .pop, .cite:focus .pop, .cite:focus-within .pop { display: block; }
|
||
.pop .pin { display: block; margin: 0.25rem 0; }
|
||
.pop q { font-family: ui-monospace, Menlo, monospace; }
|
||
ul.see-also li { margin-bottom: 0.3rem; } ul.see-also .desc { color: #555; }
|
||
ol.sources li { margin-bottom: 0.9rem; } ol.sources li:target { background: #fff6cc; }
|
||
ol.sources li.alias { height: 0; overflow: hidden; margin: 0; list-style: none; }
|
||
details.sources-list { margin-top: 1.5rem; } details.sources-list summary { cursor: pointer; font-weight: 600; }
|
||
details.sources-list h3 { font-size: 0.95rem; color: #444; margin: 1rem 0 0.3rem; }
|
||
ol.sources li.alias:target + li { background: #fff6cc; }
|
||
q.quote { display: block; quotes: none; background: #f7f7f7; border-left: 3px solid #999; padding: 0.3rem 0.6rem;
|
||
margin: 0.3rem 0; font-family: ui-monospace, Menlo, monospace; font-size: 0.85rem; white-space: pre-wrap; }
|
||
.loc { color: #555; font-size: 0.9rem; }
|
||
.raw { font-family: ui-monospace, Menlo, monospace; font-size: 0.85rem; }
|
||
.line { white-space: pre-wrap; display: grid; grid-template-columns: 4.5rem 1fr; }
|
||
.line .ln { color: #999; text-align: right; padding-right: 0.8rem; text-decoration: none; user-select: none; }
|
||
.line.cited { background: #eef6ff; } .line:target { background: #fff0a8; outline: 1px solid #e0c200; }
|
||
"""
|
||
|
||
|
||
class RunError(Exception):
|
||
"""The command cannot run; reported with exit 2."""
|
||
|
||
|
||
@dataclass
|
||
class RenderStats:
|
||
pages: int = 0
|
||
raw_files: int = 0
|
||
footnotes: int = 0
|
||
islands: int = 0 # concept pages with no link to or from another concept page
|
||
|
||
|
||
@dataclass
|
||
class CheckResult:
|
||
resolved: int = 0
|
||
total: int = 0
|
||
per_page: dict[str, tuple[int, int]] = field(default_factory=dict)
|
||
problems: list[str] = field(default_factory=list)
|
||
|
||
|
||
# --- markdown to HTML ------------------------------------------------------------------------------------------
|
||
|
||
|
||
def html_name(rel: str) -> str:
|
||
return rel[:-3] + ".html" if rel.endswith(".md") else rel + ".html"
|
||
|
||
|
||
def up(rel: str) -> str:
|
||
"""The relative prefix from a bundle-relative page back to the output root."""
|
||
return "../" * (len(PurePosixPath(rel).parts) - 1)
|
||
|
||
|
||
def esc(text: str) -> str:
|
||
return html.escape(text, quote=True)
|
||
|
||
|
||
SAFE_SCHEMES = ("http:", "https:", "mailto:")
|
||
|
||
|
||
def link_target(target: str) -> str | None:
|
||
"""A link to another page of the bundle points at its HTML page; an http, https or mailto link is kept; a link
|
||
with any other scheme (javascript:, data:, file: ...) gets None, so it renders as plain text."""
|
||
if SCHEME.match(target):
|
||
return target if target.lower().startswith(SAFE_SCHEMES) else None
|
||
if not target or target.startswith("#"):
|
||
return target
|
||
path, hash_, frag = target.partition("#")
|
||
if path.endswith(".md"):
|
||
path = path[:-3] + ".html"
|
||
return path + hash_ + frag
|
||
|
||
|
||
def plain(text: str) -> str:
|
||
"""Escaped text with inline code; bold markers become a sentinel that inline() pairs up."""
|
||
out: list[str] = []
|
||
pos = 0
|
||
for m in CODE_SPAN.finditer(text):
|
||
out.append(esc(text[pos:m.start()]).replace("**", BOLD))
|
||
out.append(f"<code>{esc(m.group(2))}</code>")
|
||
pos = m.end()
|
||
out.append(esc(text[pos:]).replace("**", BOLD))
|
||
return "".join(out)
|
||
|
||
|
||
def pair_bold(text: str) -> str:
|
||
parts = text.split(BOLD)
|
||
if len(parts) % 2 == 0: # an odd number of markers: the last one stays literal
|
||
parts[-2:] = [parts[-2] + "**" + parts[-1]]
|
||
return "".join(p if i % 2 == 0 else f"<strong>{p}</strong>" for i, p in enumerate(parts))
|
||
|
||
|
||
class Footnotes:
|
||
"""Display numbers by first use, and the markup of the references that end a sentence."""
|
||
|
||
def __init__(self, sources: dict[str, dict[str, Any]], page_rel: str = "") -> None:
|
||
self.sources = sources
|
||
self.page_rel = page_rel
|
||
self.canon = canonical(sources)
|
||
self.numbers: dict[str, int] = {}
|
||
self.seen: Counter[str] = Counter()
|
||
|
||
def number(self, label: str) -> int:
|
||
"""Entries on the same file and line share the number of their canonical entry."""
|
||
return self.numbers.setdefault(self.canon.get(label, label), len(self.numbers) + 1)
|
||
|
||
def ref(self, label: str, extra: str = "") -> str:
|
||
"""One link to the source entry: a grey version label for a version heading, else the entry's number."""
|
||
n = self.number(label)
|
||
self.seen[label] += 1
|
||
source = self.sources.get(label, {})
|
||
quote = source.get("x-quote", "")
|
||
title = f' title="{esc(quote)}"' if isinstance(quote, str) and quote else ""
|
||
cls, text = ("fnref ver", esc(str(quote).strip())) if is_version_heading(source) else ("fnref", str(n))
|
||
return (f'<a class="{cls}" href="#src-{esc(label)}" id="ref-{esc(label)}-{self.seen[label]}"{title}{extra}>'
|
||
f"{text}</a>")
|
||
|
||
def cite(self, labels: list[str]) -> str:
|
||
"""A run of references: the version labels stay visible, the rest sit behind one small marker whose popup
|
||
(hover or focus) shows each quote with a link to its raw line. A repeated label gets a hidden link, so the
|
||
check still finds one link per reference."""
|
||
versions = [lb for lb in labels if is_version_heading(self.sources.get(lb, {}))]
|
||
rest = [lb for lb in labels if lb not in versions]
|
||
out = "".join(self.ref(lb) for lb in versions)
|
||
if not rest:
|
||
return out
|
||
pins: list[str] = []
|
||
shown: set[tuple[str, str]] = set()
|
||
for label in rest:
|
||
line_of = self.canon.get(label, label)
|
||
s = self.sources.get(line_of, {})
|
||
quote = str(s["x-quote"]) if isinstance(s.get("x-quote"), str) else ""
|
||
own = self.sources.get(label, {}).get("x-quote")
|
||
if isinstance(own, str) and own not in quote: # another quote on the same line: show the one that
|
||
s, quote = self.sources.get(label, {}), own # supports this claim, not the line's longest
|
||
if (line_of, quote) in shown: # the same quote again, from this label or another entry on the line
|
||
pins.append(self.ref(label, " hidden"))
|
||
continue
|
||
shown.add((line_of, quote))
|
||
rel, line = raw_rel(self.page_rel, s), locator_line(s)
|
||
name = str(s.get("title") or s.get("_original", {}).get("name", ""))
|
||
where = esc(", ".join(x for x in (name, place_words(s["x-locator"]) if line else "") if x))
|
||
raw = (f' <a class="open" href="{esc(up(self.page_rel))}raw/{esc(rel)}.html#L{line}">{where}</a>'
|
||
if rel and line else f" {where}")
|
||
pins.append(f'<span class="pin">{self.ref(label)} <q>{esc(str(quote))}</q>{raw}'
|
||
f'{original_link(self.page_rel, s)}</span>')
|
||
mark = f'<sup class="mark">{self.number(rest[0])}</sup>'
|
||
return (f'{out}<span class="cite" tabindex="0">{mark}<span class="pop" role="note">'
|
||
+ "".join(pins) + "</span></span>")
|
||
|
||
|
||
def canonical(sources: dict[str, dict[str, Any]]) -> dict[str, str]:
|
||
"""Each label to the entry that stands for its file and line: the one with the longest quote (the first on a tie).
|
||
An entry without a line locator stands for itself."""
|
||
groups: dict[tuple[str, int], list[str]] = {}
|
||
out: dict[str, str] = {}
|
||
for label, s in sources.items():
|
||
line = locator_line(s)
|
||
if line is None:
|
||
out[label] = label
|
||
else:
|
||
groups.setdefault((str(s.get("resource", "")), line), []).append(label)
|
||
for labels in groups.values():
|
||
best = max(labels, key=lambda lb: len(str(sources[lb].get("x-quote", "")))) # max keeps the first on a tie
|
||
out.update({lb: best for lb in labels})
|
||
return out
|
||
|
||
|
||
def is_version_heading(source: dict[str, Any]) -> bool:
|
||
"""A source entry whose quote is only a version number: the version heading of a release note."""
|
||
quote = source.get("x-quote")
|
||
return isinstance(quote, str) and VERSION_ONLY.fullmatch(quote.strip()) is not None
|
||
|
||
|
||
def inline(text: str, notes: Footnotes | None) -> str:
|
||
"""Links, footnote references, inline code and bold; everything else escaped."""
|
||
masked = mask_code(text)
|
||
out: list[str] = []
|
||
pos = 0
|
||
run: list[str] = [] # footnote references that follow each other with nothing between them
|
||
for m in INLINE.finditer(masked):
|
||
note = m.group(3) is not None
|
||
if not (note and run and m.start() == pos):
|
||
if run and notes is not None:
|
||
out.append(notes.cite(run))
|
||
run = []
|
||
out.append(plain(text[pos:m.start()]))
|
||
if note: # a footnote reference
|
||
if notes is not None:
|
||
run.append(text[m.start(3):m.end(3)])
|
||
else:
|
||
out.append(esc(text[m.start():m.end()]))
|
||
else:
|
||
label_text = text[m.start(1):m.end(1)]
|
||
target = text[m.start(2):m.end(2)]
|
||
href = link_target(target)
|
||
out.append(plain(label_text) if href is None else f'<a href="{esc(href)}">{plain(label_text)}</a>')
|
||
pos = m.end()
|
||
if run and notes is not None:
|
||
out.append(notes.cite(run))
|
||
out.append(plain(text[pos:]))
|
||
return pair_bold("".join(out))
|
||
|
||
|
||
def body_html(doc: Document, notes: Footnotes | None) -> str:
|
||
"""Headings, paragraphs and lists of a body; footnote definitions are left out (the sources list replaces them)."""
|
||
out: list[str] = []
|
||
para: list[str] = []
|
||
items: list[str] = []
|
||
list_tag = ""
|
||
|
||
def flush() -> None:
|
||
nonlocal list_tag
|
||
if para:
|
||
out.append(f"<p>{inline(' '.join(para), notes)}</p>")
|
||
para.clear()
|
||
if items:
|
||
out.append(f"<{list_tag}>" + "".join(f"<li>{inline(i, notes)}</li>" for i in items) + f"</{list_tag}>")
|
||
items.clear()
|
||
list_tag = ""
|
||
|
||
for raw in doc.body.split("\n"):
|
||
stripped = raw.strip()
|
||
if not stripped or FOOTNOTE_DEF.match(stripped):
|
||
flush()
|
||
continue
|
||
heading = HEADING.match(raw)
|
||
if heading:
|
||
flush()
|
||
level = len(heading.group(1))
|
||
out.append(f"<h{level}>{inline(heading.group(2) or '', notes)}</h{level}>")
|
||
elif LIST_ITEM.match(raw):
|
||
tag = "ol" if ORDERED.match(raw) else "ul"
|
||
if para or (items and tag != list_tag):
|
||
flush()
|
||
list_tag = tag
|
||
items.append(LIST_ITEM.sub("", raw, count=1).strip())
|
||
elif items:
|
||
items[-1] += " " + stripped
|
||
else:
|
||
para.append(stripped)
|
||
flush()
|
||
return "\n".join(out)
|
||
|
||
|
||
def page_shell(title: str, prefix: str, nav_graph: bool, content: str, lang: str = "en") -> str:
|
||
graph = f' <a href="{prefix}{GRAPH}">Graph (Google OKF viewer)</a>' if nav_graph else ""
|
||
return (f"<!doctype html>\n<html lang=\"{esc(lang)}\">\n<head>\n<meta charset=\"utf-8\">\n"
|
||
"<meta name=\"viewport\" content=\"width=device-width, initial-scale=1\">\n"
|
||
f"<title>{esc(title)}</title>\n<style>{STYLE}</style>\n</head>\n<body>\n"
|
||
f'<nav><a href="{prefix}index.html">Index</a> <a href="{prefix}log.html">Log</a>{graph}</nav>\n'
|
||
f"{content}\n</body>\n</html>\n")
|
||
|
||
|
||
# --- render ----------------------------------------------------------------------------------------------------
|
||
|
||
|
||
def source_entries(fm: dict[str, Any]) -> dict[str, dict[str, Any]]:
|
||
srcs = fm.get("sources", [])
|
||
entries = srcs if isinstance(srcs, list) else [srcs]
|
||
return {s["id"]: s for s in entries if isinstance(s, dict) and isinstance(s.get("id"), str)}
|
||
|
||
|
||
def raw_rel(page_rel: str, source: dict[str, Any]) -> str | None:
|
||
res = source.get("resource")
|
||
if not isinstance(res, str) or not res or SCHEME.match(res):
|
||
return None
|
||
return resolve(page_rel, res) or None
|
||
|
||
|
||
def locator_line(source: dict[str, Any]) -> int | None:
|
||
loc = source.get("x-locator")
|
||
if isinstance(loc, dict) and type(loc.get("line")) is int and loc["line"] > 0:
|
||
return int(loc["line"])
|
||
return None
|
||
|
||
|
||
SectionOf = Callable[[str, int], tuple[int, str] | None]
|
||
|
||
|
||
@dataclass
|
||
class Group:
|
||
"""The entries of one document section: the nearest heading above the cited lines in the raw source."""
|
||
|
||
title: str
|
||
rel: str
|
||
heading_line: int | None
|
||
heading: str
|
||
labels: list[str] = field(default_factory=list)
|
||
lines: list[int] = field(default_factory=list)
|
||
|
||
@property
|
||
def name(self) -> str:
|
||
return f"{self.title} › {self.heading}" if self.heading else self.title
|
||
|
||
|
||
def source_groups(page_rel: str, notes: Footnotes, section_of: SectionOf) -> list[Group]:
|
||
"""The page's canonical entries grouped by document and section, in the documents' order of first use and then
|
||
in the order of the sections in the raw source."""
|
||
for label in notes.sources: # entries never referenced are listed after the referenced ones
|
||
notes.number(label)
|
||
groups: dict[tuple[str, str, int | None], Group] = {}
|
||
doc_first: dict[tuple[str, str], int] = {}
|
||
for label, n in sorted(notes.numbers.items(), key=lambda kv: kv[1]):
|
||
s = notes.sources.get(label, {})
|
||
title = str(s.get("title") or s.get("_original", {}).get("name", ""))
|
||
rel, line = raw_rel(page_rel, s) or "", locator_line(s)
|
||
sec = section_of(rel, line) if rel and line else None
|
||
key = (title, rel, sec[0] if sec else None)
|
||
g = groups.setdefault(key, Group(title, rel, sec[0] if sec else None, sec[1] if sec else ""))
|
||
g.labels.append(label)
|
||
if line:
|
||
g.lines.append(line)
|
||
doc_first.setdefault((title, rel), n)
|
||
return sorted(groups.values(), key=lambda g: (doc_first[(g.title, g.rel)], g.heading_line or 0))
|
||
|
||
|
||
def sources_html(page_rel: str, notes: Footnotes, section_of: SectionOf) -> str:
|
||
"""The source list, closed by default: one group per document section, the line span of its cited lines."""
|
||
parts: list[str] = []
|
||
for g in source_groups(page_rel, notes, section_of):
|
||
lo, hi = (min(g.lines), max(g.lines)) if g.lines else (0, 0)
|
||
span = "" if not g.lines else f" (line {lo})" if lo == hi else f" (lines {lo}–{hi})"
|
||
rows: list[str] = []
|
||
for label in g.labels:
|
||
n = notes.numbers[label]
|
||
s = notes.sources.get(label)
|
||
if s is None:
|
||
rows.append(f'<li id="src-{esc(label)}" value="{n}">No source entry with id <code>{esc(label)}</code>.</li>')
|
||
continue
|
||
# Other entries on the same file and line come first as zero-height anchors, so the line shows once and
|
||
# a link to any of them lands here.
|
||
for alias in (lb for lb, c in notes.canon.items() if c == label and lb != label):
|
||
rows.append(entry_html(page_rel, alias, notes.sources[alias], n, notes.seen[alias] > 0, ' class="alias"'))
|
||
rows.append(entry_html(page_rel, label, s, n, notes.seen[label] > 0))
|
||
parts.append(f"<h3>{esc(g.name)}{span}</h3>\n" + '<ol class="sources">\n' + "\n".join(rows) + "\n</ol>")
|
||
return (f'<details class="sources-list"><summary>Sources ({len(notes.numbers)})</summary>\n'
|
||
+ "\n".join(parts) + "\n</details>")
|
||
|
||
|
||
def read_more_html(page_rel: str, notes: Footnotes, section_of: SectionOf) -> str:
|
||
"""One link per document section the page draws on, to the raw source at the section's heading."""
|
||
rows = []
|
||
for g in source_groups(page_rel, notes, section_of):
|
||
if not g.rel or not g.lines:
|
||
continue
|
||
target = g.heading_line or min(g.lines)
|
||
rows.append(f'<li><a href="{esc(up(page_rel))}raw/{esc(g.rel)}.html#L{target}">{esc(g.name)}</a></li>')
|
||
return '<h2>Read more in</h2>\n<ul class="read-more">\n' + "\n".join(rows) + "\n</ul>" if rows else ""
|
||
|
||
|
||
def nearest_heading(lines: list[str], line: int) -> tuple[int, str] | None:
|
||
"""The nearest markdown heading at or above a line (1-based), as (line, text). A heading inside fenced code is
|
||
example text, not a section: fences follow the auditor's rules (backticks or tildes, also in a list item or a
|
||
quote block, closed by a run of the same marker at least as long)."""
|
||
found: tuple[int, str] | None = None
|
||
fence = "" # the open fence's marker run, or "" outside a fenced block
|
||
for n, raw in enumerate(lines[:line], 1):
|
||
f = FENCE.match(raw)
|
||
if f and f.group(1)[0] == "`" and "`" in raw[f.end():]:
|
||
f = None # a backtick run with a backtick after it is an inline code span, not a fence (CommonMark 4.5)
|
||
if fence:
|
||
if f and f.group(1)[0] == fence[0] and len(f.group(1)) >= len(fence) and not raw[f.end():].strip():
|
||
fence = ""
|
||
continue
|
||
if f:
|
||
fence = f.group(1)
|
||
continue
|
||
m = HEADING.match(raw)
|
||
if m and m.group(2):
|
||
found = (n, m.group(2))
|
||
return found
|
||
|
||
|
||
def entry_html(page_rel: str, label: str, s: dict[str, Any], n: int, seen: bool, cls: str = "") -> str:
|
||
quote = s.get("x-quote") if isinstance(s.get("x-quote"), str) else ""
|
||
line = locator_line(s)
|
||
rel = raw_rel(page_rel, s)
|
||
res = str(s.get("resource", ""))
|
||
loc = f"line {line} of <code>{esc(res)}</code>" if line else f"no line locator, <code>{esc(res)}</code>"
|
||
if line and set(s["x-locator"]) != {"line"}:
|
||
loc = f"{esc(place_words(s['x-locator']))} of {esc(s.get('_original', {}).get('name', ''))} ({loc})"
|
||
raw = (f' <a class="raw" href="{esc(up(page_rel))}raw/{esc(rel)}.html#L{line}">open the raw source at line {line}</a>'
|
||
if rel and line else "") + original_link(page_rel, s)
|
||
back = f' <a class="back" href="#ref-{esc(label)}-1">back to text</a>' if seen else ""
|
||
return (f'<li{cls} id="src-{esc(label)}" value="{n}"><strong>{esc(str(s.get("title", "")))}</strong>'
|
||
f'<q class="quote">{esc(str(quote))}</q><span class="loc">{loc}</span>{raw}{back}</li>')
|
||
|
||
|
||
def related(concepts: list[Document]) -> dict[str, list[str]]:
|
||
"""For each concept page, the other concept pages it links to or that link to it."""
|
||
rels = {d.rel for d in concepts}
|
||
links: dict[str, set[str]] = {r: set() for r in rels}
|
||
for d in concepts:
|
||
for _line, target in body_links(d):
|
||
other = resolve(d.rel, target) if is_bundle_link(target) else None
|
||
if other in rels and other != d.rel:
|
||
links[d.rel].add(other)
|
||
links[other].add(d.rel)
|
||
return {r: sorted(v) for r, v in links.items()}
|
||
|
||
|
||
def see_also_html(page_rel: str, others: list[str], docs: dict[str, Document]) -> str:
|
||
if not others:
|
||
return ""
|
||
rows = []
|
||
for other in others:
|
||
fm = docs[other].frontmatter or {}
|
||
title = fm.get("title") if isinstance(fm.get("title"), str) else other
|
||
desc = f' <span class="desc">- {esc(fm["description"])}</span>' if isinstance(fm.get("description"), str) else ""
|
||
href = posixpath.relpath(html_name(other), posixpath.dirname(html_name(page_rel)) or ".")
|
||
rows.append(f'<li><a href="{esc(href)}">{esc(str(title))}</a>{desc}</li>')
|
||
return '<h2>See also</h2>\n<ul class="see-also">\n' + "\n".join(rows) + "\n</ul>"
|
||
|
||
|
||
def meta_html(doc: Document) -> str:
|
||
fm = doc.frontmatter or {}
|
||
parts: list[str] = []
|
||
if isinstance(fm.get("description"), str):
|
||
parts.append(f"<p>{esc(fm['description'])}</p>")
|
||
names = fm.get("x-aliases")
|
||
if isinstance(names, list) and names:
|
||
parts.append("<p>Also called: " + "; ".join(esc(str(n)) for n in names) + "</p>")
|
||
tags = fm.get("tags")
|
||
if isinstance(tags, list) and tags:
|
||
parts.append("<p>Tags: " + ", ".join(esc(str(t)) for t in tags) + "</p>")
|
||
gen = fm.get("generated")
|
||
if isinstance(gen, dict):
|
||
parts.append(f"<p>Generated by {esc(str(gen.get('by', '')))} at {esc(str(gen.get('at', '')))}</p>")
|
||
parts.append(f"<p>Page file: <code>{esc(doc.rel)}</code></p>")
|
||
return '<div class="meta">' + "".join(parts) + "</div>"
|
||
|
||
|
||
def raw_html(rel: str, text: str, cited: set[int], lang: str = "en") -> str:
|
||
rows = []
|
||
for n, line in enumerate(text.splitlines(), 1):
|
||
cls = "line cited" if n in cited else "line"
|
||
rows.append(f'<div class="{cls}" id="L{n}"><a class="ln" href="#L{n}">{n}</a><span class="text">{esc(line)}</span></div>')
|
||
digest = hashlib.sha256(text.encode("utf-8")).hexdigest()
|
||
content = (f"<h1>Raw source <code>/{esc(rel)}</code></h1>\n"
|
||
f'<p class="meta">sha256 {digest} · {len(rows)} lines · cited lines are shaded</p>\n'
|
||
f'<div class="raw">\n' + "\n".join(rows) + "\n</div>")
|
||
return page_shell(f"/{rel}", up(rel) + "../", False, content, lang)
|
||
|
||
|
||
def write(path: Path, text: str) -> None:
|
||
path.parent.mkdir(parents=True, exist_ok=True)
|
||
path.write_text(text, encoding="utf-8")
|
||
|
||
|
||
# --- change marks (contract K4) ----------------------------------------------------------------------------------
|
||
|
||
MARKS_SCHEMA = "okf-wiki/source-marks/1"
|
||
CHANGES = ("changed", "gone")
|
||
Marks = dict[str, list[tuple[str, str]]] # page -> [(source path, change)], in the file's order
|
||
# Inline, so the shared stylesheet, and with it the view without marks, stays as it was.
|
||
MARK_STYLE = "border: 2px solid #c25e00; background: #fff4e5; border-radius: 4px; padding: 0.5rem 0.75rem; margin-bottom: 1rem;"
|
||
|
||
|
||
def read_marks(path: Path) -> Marks:
|
||
"""The pages and their marked sources from a change-marks file in the form of contract K4 point 2; RunError when the
|
||
file cannot be read as one. The K1 module is not imported: the view reads the file itself."""
|
||
try:
|
||
data = json.loads(path.read_text(encoding="utf-8"))
|
||
except (OSError, UnicodeDecodeError, ValueError) as exc:
|
||
raise RunError(f"the marks file {path} cannot be read as JSON: {exc}") from exc
|
||
if not isinstance(data, dict) or data.get("schema") != MARKS_SCHEMA or not isinstance(data.get("pages"), list):
|
||
raise RunError(f"the marks file {path} is not {MARKS_SCHEMA} with a list of pages")
|
||
marks: Marks = {}
|
||
for entry in data["pages"]:
|
||
page = entry.get("page") if isinstance(entry, dict) else None
|
||
srcs = entry.get("sources") if isinstance(entry, dict) else None
|
||
if not isinstance(page, str) or not isinstance(srcs, list) or not srcs or page in marks:
|
||
raise RunError(f"the marks file {path}: an entry is not one page with a list of sources: {entry!r}")
|
||
for s in srcs:
|
||
if not (isinstance(s, dict) and isinstance(s.get("path"), str) and s.get("change") in CHANGES):
|
||
raise RunError(f"the marks file {path}: {page}: a source is not a path with change changed or gone: {s!r}")
|
||
marks[page] = [(s["path"], s["change"]) for s in srcs]
|
||
return marks
|
||
|
||
|
||
def check_marks(marks: Marks, docs: dict[str, Document]) -> None:
|
||
"""RunError unless every marked page is a concept page of the bundle that cites each of its marked sources as
|
||
"/" + path in sources[].resource (K4 point 2), so marks from another bundle are never shown."""
|
||
for page, entries in marks.items():
|
||
if page not in docs:
|
||
raise RunError(f"the marks name {page}, which is not a concept page of the bundle")
|
||
srcs = (docs[page].frontmatter or {}).get("sources")
|
||
resources = {s.get("resource") for s in (srcs if isinstance(srcs, list) else [srcs]) if isinstance(s, dict)}
|
||
for path, _change in entries:
|
||
if "/" + path not in resources:
|
||
raise RunError(f"the marks say {page} builds on /{path}, but the page does not cite it")
|
||
|
||
|
||
def mark_html(entries: list[tuple[str, str]]) -> str:
|
||
rows = "".join(f"<li><code>/{esc(path)}</code> {esc(change)}</li>" for path, change in entries)
|
||
return (f'<div class="source-mark" role="note" style="{MARK_STYLE}"><strong>This page builds on a source that has '
|
||
f"changed or is gone since the frozen copy.</strong>\n<ul>{rows}</ul></div>")
|
||
|
||
|
||
def marked_pages_html(marks: Marks, docs: dict[str, Document]) -> str:
|
||
"""The list of marked pages at the top of the root index.html, or a line that says there are none."""
|
||
rows = []
|
||
for page in sorted(marks):
|
||
fm = docs[page].frontmatter or {}
|
||
title = fm.get("title") if isinstance(fm.get("title"), str) else page
|
||
srcs = ", ".join(f"<code>/{esc(path)}</code> {esc(change)}" for path, change in marks[page])
|
||
rows.append(f'<li><a href="{esc(html_name(page))}">{esc(str(title))}</a>: {srcs}</li>')
|
||
listing = "<ul>" + "".join(rows) + "</ul>" if rows else "<p>No page builds on a changed or gone source.</p>"
|
||
return (f'<div class="marked-pages" style="{MARK_STYLE}"><h2>Pages built on changed or gone sources ({len(rows)})</h2>\n'
|
||
f"{listing}</div>\n")
|
||
|
||
|
||
def render(bundle: Path, sources: Path, out: Path, graph: Path | None = None, marks: Marks | None = None) -> RenderStats:
|
||
b = load_bundle(bundle, sources)
|
||
if marks is not None:
|
||
check_marks(marks, {d.rel: d for d in b.concepts})
|
||
stats = RenderStats()
|
||
cited: dict[str, set[int]] = {}
|
||
has_graph = graph is not None
|
||
if graph is not None:
|
||
out.mkdir(parents=True, exist_ok=True)
|
||
shutil.copyfile(graph, out / GRAPH)
|
||
docs = {d.rel: d for d in b.concepts}
|
||
links = related(b.concepts)
|
||
raw_text: dict[str, list[str] | None] = {}
|
||
|
||
def section_of(rel: str, line: int) -> tuple[int, str] | None:
|
||
if rel not in raw_text:
|
||
try:
|
||
raw_text[rel] = (sources / rel).read_text(encoding="utf-8").splitlines()
|
||
except (OSError, UnicodeDecodeError):
|
||
raw_text[rel] = None
|
||
lines = raw_text[rel]
|
||
return nearest_heading(lines, line) if lines is not None else None
|
||
|
||
stats.islands = sum(1 for others in links.values() if not others)
|
||
known = originals(sources)
|
||
langs = languages(sources)
|
||
counted = Counter(langs.values()).most_common(1)
|
||
collection_lang = counted[0][0] if counted else "en"
|
||
copies: set[str] = set()
|
||
for doc in b.concepts:
|
||
fm = doc.frontmatter or {}
|
||
entries = source_entries(fm)
|
||
for s in entries.values():
|
||
rel = raw_rel(doc.rel, s)
|
||
if rel in known:
|
||
s["_original"] = known[rel] # in memory only: the view names the original and opens its copy
|
||
if "copy" in known[rel]:
|
||
copies.add(known[rel]["copy"])
|
||
notes = Footnotes(entries, doc.rel)
|
||
body = body_html(doc, notes)
|
||
stats.footnotes += sum(notes.seen.values())
|
||
for s in notes.sources.values():
|
||
rel, line = raw_rel(doc.rel, s), locator_line(s)
|
||
if rel and line:
|
||
cited.setdefault(rel, set()).add(line)
|
||
title = fm.get("title") if isinstance(fm.get("title"), str) else doc.rel
|
||
also = see_also_html(doc.rel, links[doc.rel], docs)
|
||
more = read_more_html(doc.rel, notes, section_of)
|
||
content = (meta_html(doc) + "\n" + body + "\n" + "".join(x + "\n" for x in (more, also) if x)
|
||
+ sources_html(doc.rel, notes, section_of))
|
||
if marks is not None and doc.rel in marks:
|
||
content = mark_html(marks[doc.rel]) + "\n" + content
|
||
cited_langs = Counter(langs[r] for r in (raw_rel(doc.rel, s) for s in entries.values()) if r in langs)
|
||
lang = cited_langs.most_common(1)[0][0] if cited_langs else collection_lang
|
||
write(out / html_name(doc.rel), page_shell(str(title), up(doc.rel), has_graph, content, lang))
|
||
stats.pages += 1
|
||
for name in ("index.md", "log.md"):
|
||
for doc in b.named(name):
|
||
title = "Index" if name == "index.md" else "Log"
|
||
listing = marked_pages_html(marks, docs) if marks is not None and doc.rel == "index.md" else ""
|
||
write(out / html_name(doc.rel), page_shell(title, up(doc.rel), has_graph, listing + body_html(doc, None),
|
||
collection_lang))
|
||
for rel, lines in sorted(cited.items()):
|
||
try:
|
||
text = (sources / rel).read_text(encoding="utf-8")
|
||
except (OSError, UnicodeDecodeError):
|
||
continue # the check reports its footnotes as unresolved
|
||
write(out / "raw" / (rel + ".html"), raw_html(rel, text, lines, langs.get(rel, collection_lang)))
|
||
stats.raw_files += 1
|
||
for copy in sorted(copies):
|
||
if (sources / copy).is_file():
|
||
(out / copy).parent.mkdir(parents=True, exist_ok=True)
|
||
shutil.copyfile(sources / copy, out / copy)
|
||
return stats
|
||
|
||
|
||
def languages(sources: Path) -> dict[str, str]:
|
||
"""Per raw source, the language `sources folder` recorded for it (K1 `lang`); none for collection 1."""
|
||
try:
|
||
doc = json.loads((sources / "source-manifest.json").read_text(encoding="utf-8"))
|
||
except (OSError, ValueError):
|
||
return {}
|
||
return {e["path"]: e["lang"] for e in (doc.get("sources", []) if isinstance(doc, dict) else [])
|
||
if isinstance(e, dict) and isinstance(e.get("path"), str) and isinstance(e.get("lang"), str)}
|
||
|
||
|
||
def originals(sources: Path) -> dict[str, dict[str, str]]:
|
||
"""Per raw source (kilde/...) of a folder, the original's name, and for PDF and Office its copy under original/."""
|
||
try:
|
||
doc = json.loads((sources / "source-manifest.json").read_text(encoding="utf-8"))
|
||
except (OSError, ValueError):
|
||
return {}
|
||
found: dict[str, dict[str, str]] = {}
|
||
for e in doc.get("sources", []) if isinstance(doc, dict) else []:
|
||
if isinstance(e, dict) and isinstance(e.get("path"), str) and isinstance(e.get("original"), str):
|
||
found[e["path"]] = {"name": PurePosixPath(e["original"]).name}
|
||
if isinstance(e.get("original_copy"), str):
|
||
found[e["path"]]["copy"] = e["original_copy"]
|
||
return found
|
||
|
||
|
||
def original_link(page_rel: str, s: dict[str, Any]) -> str:
|
||
"""A link that opens the original PDF or Office file, a PDF at its page; "" for a text source."""
|
||
known, loc = s.get("_original", {}), s.get("x-locator")
|
||
if "copy" not in known or not isinstance(loc, dict):
|
||
return ""
|
||
at = f"#page={loc['page']}" if type(loc.get("page")) is int else ""
|
||
words = f"open {known['name']} at {place_words(loc)}" if set(loc) != {"line"} else f"open {known['name']}"
|
||
return f' <a class="orig" href="{esc(up(page_rel))}{esc(known["copy"])}{at}">{esc(words)}</a>'
|
||
|
||
|
||
# --- check (reads the rendered HTML only) ----------------------------------------------------------------------
|
||
|
||
|
||
class PageParser(HTMLParser):
|
||
"""Footnote references and source entries (quote, locator text, raw links) of one rendered page."""
|
||
|
||
def __init__(self) -> None:
|
||
super().__init__(convert_charrefs=True)
|
||
self.refs: list[str] = []
|
||
self.entries: dict[str, dict[str, Any]] = {}
|
||
self._entry: str | None = None
|
||
self._li_depth = 0
|
||
self._capture: str | None = None
|
||
|
||
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
|
||
a = {k: v or "" for k, v in attrs}
|
||
classes = a.get("class", "").split()
|
||
if tag == "a" and "fnref" in classes and a.get("href", "").startswith("#src-"):
|
||
self.refs.append(a["href"][len("#src-"):])
|
||
if tag == "li":
|
||
if self._entry is None and a.get("id", "").startswith("src-"):
|
||
self._entry = a["id"][len("src-"):]
|
||
self.entries[self._entry] = {"quote": None, "loc": "", "raw": []}
|
||
self._li_depth = 0
|
||
elif self._entry is not None:
|
||
self._li_depth += 1
|
||
if self._entry is not None:
|
||
e = self.entries[self._entry]
|
||
if tag == "q" and "quote" in classes:
|
||
self._capture, e["quote"] = "quote", ""
|
||
elif tag == "span" and "loc" in classes:
|
||
self._capture = "loc"
|
||
elif tag == "a" and "raw" in classes:
|
||
e["raw"].append(a.get("href", ""))
|
||
|
||
def handle_endtag(self, tag: str) -> None:
|
||
if tag in ("q", "span"):
|
||
self._capture = None
|
||
if tag == "li" and self._entry is not None:
|
||
if self._li_depth == 0:
|
||
self._entry = None
|
||
else:
|
||
self._li_depth -= 1
|
||
|
||
def handle_data(self, data: str) -> None:
|
||
if self._entry is not None and self._capture is not None:
|
||
self.entries[self._entry][self._capture] += data
|
||
|
||
|
||
class RawParser(HTMLParser):
|
||
"""The text of each line anchor (div id="L<n>" > span.text) of a raw view."""
|
||
|
||
def __init__(self) -> None:
|
||
super().__init__(convert_charrefs=True)
|
||
self.lines: dict[str, str] = {}
|
||
self._line: str | None = None
|
||
self._in_text = False
|
||
|
||
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
|
||
a = {k: v or "" for k, v in attrs}
|
||
if tag == "div" and re.fullmatch(r"L\d+", a.get("id", "")):
|
||
self._line = a["id"]
|
||
self.lines[self._line] = ""
|
||
elif tag == "span" and self._line is not None and "text" in a.get("class", "").split():
|
||
self._in_text = True
|
||
|
||
def handle_endtag(self, tag: str) -> None:
|
||
if tag == "span":
|
||
self._in_text = False
|
||
elif tag == "div":
|
||
self._line = None
|
||
|
||
def handle_data(self, data: str) -> None:
|
||
if self._in_text and self._line is not None:
|
||
self.lines[self._line] += data
|
||
|
||
|
||
def markdown_refs(doc: Document) -> Counter[str]:
|
||
"""Footnote references in a page body, outside code spans and footnote definitions (the denominator)."""
|
||
found: Counter[str] = Counter()
|
||
for raw in mask_code(doc.body).split("\n"):
|
||
if not FOOTNOTE_DEF.match(raw.strip()):
|
||
found.update(FOOTNOTE_REF.findall(raw))
|
||
return found
|
||
|
||
|
||
def check(bundle: Path, out: Path) -> CheckResult:
|
||
if not out.is_dir():
|
||
raise RunError(f"the view {out} is not a directory")
|
||
b = load_bundle(bundle, bundle)
|
||
result = CheckResult()
|
||
raw_cache: dict[Path, dict[str, str] | None] = {}
|
||
root = out.resolve()
|
||
|
||
def raw_lines(path: Path) -> dict[str, str] | None:
|
||
if path not in raw_cache:
|
||
try:
|
||
parser = RawParser()
|
||
parser.feed(path.read_text(encoding="utf-8"))
|
||
raw_cache[path] = parser.lines
|
||
except (OSError, UnicodeDecodeError):
|
||
raw_cache[path] = None
|
||
return raw_cache[path]
|
||
|
||
for doc in b.concepts:
|
||
want = markdown_refs(doc)
|
||
total = sum(want.values())
|
||
page = out / html_name(doc.rel)
|
||
ok: Counter[str] = Counter()
|
||
try:
|
||
parser = PageParser()
|
||
parser.feed(page.read_text(encoding="utf-8"))
|
||
except (OSError, UnicodeDecodeError):
|
||
result.problems.append(f"{doc.rel}: no rendered page {page.name}")
|
||
parser = PageParser()
|
||
verdict: dict[str, str] = {}
|
||
for label in parser.refs:
|
||
if label not in verdict:
|
||
verdict[label] = resolve_entry(parser.entries.get(label), page, root, raw_lines)
|
||
if verdict[label] == "":
|
||
ok[label] += 1
|
||
for label, why in sorted(verdict.items()):
|
||
if why:
|
||
result.problems.append(f"{doc.rel}: [^{label}] {why}")
|
||
resolved = sum(min(n, ok[label]) for label, n in want.items())
|
||
for label, n in sorted(want.items()):
|
||
if ok[label] < n and label not in verdict:
|
||
result.problems.append(f"{doc.rel}: [^{label}] has no clickable reference in the page")
|
||
result.per_page[doc.rel] = (resolved, total)
|
||
result.resolved += resolved
|
||
result.total += total
|
||
return result
|
||
|
||
|
||
def resolve_entry(entry: dict[str, Any] | None, page: Path, root: Path,
|
||
raw_lines: Callable[[Path], dict[str, str] | None]) -> str:
|
||
""""" when the entry shows a quote, its line and a raw link whose line anchor holds the quote; else why not."""
|
||
if entry is None:
|
||
return "has no source entry"
|
||
quote = entry["quote"]
|
||
if not quote:
|
||
return "has no quote in its source entry"
|
||
for href in entry["raw"]:
|
||
path, _, frag = href.partition("#")
|
||
m = re.fullmatch(r"L(\d+)", frag)
|
||
if not m or f"line {m.group(1)}" not in entry["loc"]:
|
||
continue
|
||
target = (page.parent / path).resolve()
|
||
if not target.is_relative_to(root):
|
||
continue
|
||
lines = raw_lines(target)
|
||
if lines is not None and quote in lines.get(frag, ""):
|
||
return ""
|
||
return "does not reach a raw line that holds its quote"
|
||
|
||
|
||
# --- draw ------------------------------------------------------------------------------------------------------
|
||
|
||
|
||
def draw(bundle: Path, seed: int, count: int) -> list[str]:
|
||
pages = [d.rel for d in load_bundle(bundle, bundle).concepts]
|
||
if count > len(pages):
|
||
raise RunError(f"cannot draw {count} pages from {len(pages)}")
|
||
return random.Random(seed).sample(pages, count)
|
||
|
||
|
||
def draw_sha256(seed: int, count: int, pages: list[str]) -> str:
|
||
"""The sha256 that locks a draw: of "seed N\\ncount K\\n" and each page with a newline, in the order printed."""
|
||
text = f"seed {seed}\ncount {count}\n" + "".join(page + "\n" for page in pages)
|
||
return hashlib.sha256(text.encode("utf-8")).hexdigest()
|
||
|
||
|
||
# --- versions --------------------------------------------------------------------------------------------------
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class Flagged:
|
||
page: str
|
||
line: int
|
||
sentence: str
|
||
numbers: tuple[str, ...]
|
||
|
||
|
||
@dataclass
|
||
class VersionCount:
|
||
total: int = 0
|
||
flagged: list[Flagged] = field(default_factory=list)
|
||
licensed: dict[str, int] = field(default_factory=lambda: {"outdated": 0, "content-quote": 0, "page-title": 0})
|
||
|
||
|
||
def version_numbers(text: str) -> list[str]:
|
||
"""The version numbers in running text (code spans masked), without a leading v, in order of first use."""
|
||
masked = mask_code(text)
|
||
found = [m.group(1) for m in VERSION.finditer(masked)] + [m.group(1) for m in VERSION_WORD.finditer(masked)]
|
||
return list(dict.fromkeys(found))
|
||
|
||
|
||
def holds_number(text: str, number: str) -> bool:
|
||
return re.search(r"(?<![\d.])" + re.escape(number) + r"(?!\d|\.\d)", text) is not None
|
||
|
||
|
||
def versions(bundle: Path, sources: Path) -> VersionCount:
|
||
docs = {d.rel: d for d in load_bundle(bundle, sources).concepts}
|
||
result = VersionCount()
|
||
for claim in extract(bundle, sources).claims:
|
||
result.total += 1
|
||
numbers = version_numbers(claim.claim)
|
||
if not numbers:
|
||
continue
|
||
fm = docs[claim.page].frontmatter or {}
|
||
pairs = fm.get("x-outdated")
|
||
outdated = {str(p.get(k)) for p in (pairs if isinstance(pairs, list) else []) if isinstance(p, dict)
|
||
for k in ("old", "new")}
|
||
if outdated & set(claim.footnotes):
|
||
result.licensed["outdated"] += 1
|
||
continue
|
||
entries = source_entries(fm)
|
||
quotes = [str(entries[k].get("x-quote", "")) for k in claim.footnotes
|
||
if k in entries and not is_version_heading(entries[k])]
|
||
title = str(fm.get("title", ""))
|
||
left = [n for n in numbers if not any(holds_number(q, n) for q in quotes)]
|
||
if not left:
|
||
result.licensed["content-quote"] += 1
|
||
elif all(holds_number(title, n) for n in left):
|
||
result.licensed["page-title"] += 1
|
||
else:
|
||
result.flagged.append(Flagged(claim.page, claim.line, claim.claim, tuple(left)))
|
||
return result
|
||
|
||
|
||
# --- overlap ---------------------------------------------------------------------------------------------------
|
||
|
||
|
||
@dataclass
|
||
class OverlapCount:
|
||
entries: int = 0
|
||
redundant: list[tuple[str, str, str]] = field(default_factory=list) # (page, kept label, redundant label)
|
||
claims: int = 0
|
||
substring: list[tuple[str, str, str]] = field(default_factory=list) # (page, sentence, the longer sentence)
|
||
|
||
|
||
def _bare(sentence: str) -> str:
|
||
return sentence.strip().rstrip(".!?").strip()
|
||
|
||
|
||
def overlap(bundle: Path, sources: Path) -> OverlapCount:
|
||
"""Source entries on a page that repeat another entry's file and line, and sentences that are a prefix or
|
||
substring of another sentence on the same page (end punctuation ignored)."""
|
||
result = OverlapCount()
|
||
for doc in load_bundle(bundle, sources).concepts:
|
||
entries = source_entries(doc.frontmatter or {})
|
||
result.entries += len(entries)
|
||
for label, kept in canonical(entries).items():
|
||
if label != kept:
|
||
result.redundant.append((doc.rel, kept, label))
|
||
by_page: dict[str, list[str]] = {}
|
||
for claim in extract(bundle, sources).claims:
|
||
result.claims += 1
|
||
by_page.setdefault(claim.page, []).append(claim.claim)
|
||
for page, sentences in by_page.items():
|
||
for i, a in enumerate(sentences):
|
||
longer = next((b for j, b in enumerate(sentences) if j != i and _bare(a) in _bare(b)), None)
|
||
if longer is not None:
|
||
result.substring.append((page, a, longer))
|
||
return result
|
||
|
||
|
||
# --- command line ----------------------------------------------------------------------------------------------
|
||
|
||
|
||
def build_parser() -> argparse.ArgumentParser:
|
||
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||
sub = p.add_subparsers(dest="command", required=True)
|
||
r = sub.add_parser("render", help="write the HTML view")
|
||
r.add_argument("bundle")
|
||
r.add_argument("sources")
|
||
r.add_argument("--out", required=True)
|
||
r.add_argument("--graph", help="a finished graph file (Google's viewer output), copied unchanged")
|
||
r.add_argument("--marks", help="a change-marks file (contract K4): mark the pages built on changed or gone sources")
|
||
c = sub.add_parser("check", help="count the footnotes that resolve in the HTML view")
|
||
c.add_argument("bundle")
|
||
c.add_argument("view")
|
||
d = sub.add_parser("draw", help="draw pages for the operator's reading")
|
||
d.add_argument("bundle")
|
||
d.add_argument("--seed", type=int, required=True)
|
||
d.add_argument("--count", type=int, required=True)
|
||
v = sub.add_parser("versions", help="count version numbers in running text that are not the claim")
|
||
v.add_argument("bundle")
|
||
v.add_argument("sources")
|
||
o = sub.add_parser("overlap", help="count redundant source entries and overlapping sentences")
|
||
o.add_argument("bundle")
|
||
o.add_argument("sources")
|
||
return p
|
||
|
||
|
||
def main(argv: list[str] | None = None) -> int:
|
||
args = build_parser().parse_args(argv)
|
||
try:
|
||
if not Path(args.bundle).is_dir():
|
||
raise RunError(f"the bundle {args.bundle} is not a directory")
|
||
if args.command == "render":
|
||
if not Path(args.sources).is_dir():
|
||
raise RunError(f"the raw-source root {args.sources} is not a directory")
|
||
graph = Path(args.graph) if args.graph else None
|
||
if graph is not None and not graph.is_file():
|
||
raise RunError(f"the graph file {graph} does not exist")
|
||
marks = read_marks(Path(args.marks)) if args.marks else None
|
||
s = render(Path(args.bundle), Path(args.sources), Path(args.out), graph, marks)
|
||
print(f"render: {s.pages} pages, {s.raw_files} raw source files, {s.footnotes} footnotes -> {args.out}")
|
||
print(f"pages without a link to or from another page (islands): {s.islands} of {s.pages}")
|
||
return 0
|
||
if args.command == "check":
|
||
res = check(Path(args.bundle), Path(args.view))
|
||
print(f"footnotes resolved: {res.resolved} of {res.total}")
|
||
short = [f"{p}: {r} of {t}" for p, (r, t) in sorted(res.per_page.items()) if r < t]
|
||
for line in short + res.problems[:50]:
|
||
print(" " + line)
|
||
if res.total == 0:
|
||
print("FAIL: no footnotes to resolve", file=sys.stderr)
|
||
return 1
|
||
return 0 if res.resolved == res.total else 1
|
||
if args.command in ("versions", "overlap") and not Path(args.sources).is_dir():
|
||
raise RunError(f"the raw-source root {args.sources} is not a directory")
|
||
if args.command == "overlap":
|
||
oc = overlap(Path(args.bundle), Path(args.sources))
|
||
print(f"redundant source entries (same file and line): {len(oc.redundant)} of {oc.entries}")
|
||
print("sentences that are a prefix or substring of another on the same page: "
|
||
f"{len(oc.substring)} of {oc.claims}")
|
||
for page, n in sorted(Counter(p for p, _, _ in oc.redundant).items()):
|
||
print(f" {page}: {n} redundant")
|
||
for page, kept, label in oc.redundant:
|
||
print(f" {page}: [^{label}] repeats the file and line of [^{kept}]")
|
||
for page, a, b in oc.substring:
|
||
print(f" {page}: \"{a}\" is inside \"{b}\"")
|
||
if oc.entries == 0 or oc.claims == 0:
|
||
print("FAIL: no source entries or claim sentences to count", file=sys.stderr)
|
||
return 1
|
||
return 0 if not oc.redundant and not oc.substring else 1
|
||
if args.command == "versions":
|
||
vc = versions(Path(args.bundle), Path(args.sources))
|
||
print(f"version numbers in running text outside dated history: {len(vc.flagged)} of {vc.total}")
|
||
print(" not counted (the version is the claim): "
|
||
+ ", ".join(f"{k} {n}" for k, n in vc.licensed.items()))
|
||
for page, n in sorted(Counter(f.page for f in vc.flagged).items()):
|
||
print(f" {page}: {n}")
|
||
for f in vc.flagged:
|
||
print(f" {f.page}:{f.line} [{' '.join(f.numbers)}] {f.sentence}")
|
||
if vc.total == 0:
|
||
print("FAIL: no claim sentences to count", file=sys.stderr)
|
||
return 1
|
||
return 0 if not vc.flagged else 1
|
||
pages = draw(Path(args.bundle), args.seed, args.count)
|
||
for page in pages:
|
||
print(page)
|
||
print(f"sha256 {draw_sha256(args.seed, args.count, pages)}")
|
||
return 0
|
||
except RunError as exc:
|
||
print(f"FAIL: {exc}", file=sys.stderr)
|
||
return 2
|
||
|
||
|
||
if __name__ == "__main__":
|
||
sys.exit(main())
|