"""The static HTML reading view (plan row L3) behind PLAN.md criterion 3 ("operatørens lesing"). It calls no model. render BUNDLE SOURCES --out DIR [--graph FILE] [--marks FILE] One HTML page per concept page, plus index.html, log.html and one raw view per cited source file (raw/.html, one anchor per line, the cited lines marked). A page's other names (`x-aliases`) stand under its description as "Also called". Every footnote on a page is a link to its source entry, which shows the verbatim quote, the locator and a link to the raw source at that line. --graph copies a finished graph file (Google's viewer output) unchanged to graph.html and links it. --marks reads a change-marks file (contract K4, okf-wiki/source-marks/1: the pages built on a source that changed or is gone since the frozen copy) and shows a mark with each source and its change at the top of every marked page, and the list of marked pages at the top of index.html. Marks that name a page the bundle lacks, or a source the page does not cite, cannot render. Without --marks the output is unchanged. check BUNDLE DIR Counts the footnote references in the markdown pages (the denominator) and resolves each one in the rendered HTML only: reference -> source entry -> quote and locator -> raw file -> line anchor whose text holds the quote. Prints "footnotes resolved: X of N". A page, entry, quote, raw file or line that is missing fails its footnotes (fail-closed). draw BUNDLE --seed N --count K K distinct concept pages drawn with random.Random(N).sample over the sorted page list (criterion 3), one per line, then "sha256 ": the sha256 of the UTF-8 text "seed N\ncount K\n" followed by each drawn page and a newline, in the order printed, so the draw can be locked in the log. versions BUNDLE SOURCES Counts the claim sentences (the denominator, from okfwiki.claims) whose running text holds a version number that is not the claim (plan row L4). A version number is x.y.z (or longer, optionally with a leading v), or "version" followed by a dotted number, outside code spans. It is the claim, and not counted, when the sentence cites a label of an x-outdated pair (dated history), when a quote the sentence cites that is not only a version holds the same number (the source states it, as in "requires 3.2.1 or later"), or when the page title holds it (the page is about that version). Prints "... X of N" and every counted sentence. overlap BUNDLE SOURCES Counts the source entries that repeat the file and line of another entry on the same page (of all entries), and the claim sentences that are a prefix or substring of another sentence on the same page (end punctuation ignored). In the view, entries on the same file and line show once, as the one with the longest quote; the others stay as zero-height anchors so every reference still reaches its entry. In the view, the references that end a sentence do not dominate it (plan row L4): a footnote whose quote is only a version (the version heading of a release note) shows as a small grey version label, and the rest sit behind one small marker whose popup (hover or focus) shows each quote with a link to its raw line. Every reference is still a link to its source entry, so the check counts it like any other. A "See also" block lists the pages a page links to or is linked from; render prints how many pages have neither (islands). The source list is closed by default and grouped by document and section (the nearest markdown heading at or above the cited line in the raw source, with the span of cited lines), and "Read more in" links each such section at its heading. Exit 0 = done (check: N of N, N > 0; versions and overlap: 0 of N, N > 0); 1 = check found a footnote that does not resolve, or versions or overlap found one; 2 = could not run. """ from __future__ import annotations import argparse import hashlib import html import json import os import posixpath import random import re import shutil import sys from collections import Counter from collections.abc import Callable from dataclasses import dataclass, field from html.parser import HTMLParser from pathlib import Path, PurePosixPath from typing import Any from okfwiki.audit import ( CODE_SPAN, FENCE, FOOTNOTE_DEF, FOOTNOTE_REF, HEADING, LINK, LIST_ITEM, SCHEME, body_links, is_bundle_link, Document, load_bundle, mask_code, resolve, ) from okfwiki.claims import extract from okfwiki.common import place_words GRAPH = "graph.html" BOLD = "\x00" ORDERED = re.compile(r"^\s*\d+[.)]\s+") INLINE = re.compile(LINK.pattern + "|" + FOOTNOTE_REF.pattern) VERSION = re.compile(r"(? str: return rel[:-3] + ".html" if rel.endswith(".md") else rel + ".html" def up(rel: str) -> str: """The relative prefix from a bundle-relative page back to the output root.""" return "../" * (len(PurePosixPath(rel).parts) - 1) def esc(text: str) -> str: return html.escape(text, quote=True) SAFE_SCHEMES = ("http:", "https:", "mailto:") def link_target(target: str) -> str | None: """A link to another page of the bundle points at its HTML page; an http, https or mailto link is kept; a link with any other scheme (javascript:, data:, file: ...) gets None, so it renders as plain text.""" if SCHEME.match(target): return target if target.lower().startswith(SAFE_SCHEMES) else None if not target or target.startswith("#"): return target path, hash_, frag = target.partition("#") if path.endswith(".md"): path = path[:-3] + ".html" return path + hash_ + frag def plain(text: str) -> str: """Escaped text with inline code; bold markers become a sentinel that inline() pairs up.""" out: list[str] = [] pos = 0 for m in CODE_SPAN.finditer(text): out.append(esc(text[pos:m.start()]).replace("**", BOLD)) out.append(f"{esc(m.group(2))}") pos = m.end() out.append(esc(text[pos:]).replace("**", BOLD)) return "".join(out) def pair_bold(text: str) -> str: parts = text.split(BOLD) if len(parts) % 2 == 0: # an odd number of markers: the last one stays literal parts[-2:] = [parts[-2] + "**" + parts[-1]] return "".join(p if i % 2 == 0 else f"{p}" for i, p in enumerate(parts)) class Footnotes: """Display numbers by first use, and the markup of the references that end a sentence.""" def __init__(self, sources: dict[str, dict[str, Any]], page_rel: str = "") -> None: self.sources = sources self.page_rel = page_rel self.canon = canonical(sources) self.numbers: dict[str, int] = {} self.seen: Counter[str] = Counter() def number(self, label: str) -> int: """Entries on the same file and line share the number of their canonical entry.""" return self.numbers.setdefault(self.canon.get(label, label), len(self.numbers) + 1) def ref(self, label: str, extra: str = "") -> str: """One link to the source entry: a grey version label for a version heading, else the entry's number.""" n = self.number(label) self.seen[label] += 1 source = self.sources.get(label, {}) quote = source.get("x-quote", "") title = f' title="{esc(quote)}"' if isinstance(quote, str) and quote else "" cls, text = ("fnref ver", esc(str(quote).strip())) if is_version_heading(source) else ("fnref", str(n)) return (f'' f"{text}") def cite(self, labels: list[str]) -> str: """A run of references: the version labels stay visible, the rest sit behind one small marker whose popup (hover or focus) shows each quote with a link to its raw line. A repeated label gets a hidden link, so the check still finds one link per reference.""" versions = [lb for lb in labels if is_version_heading(self.sources.get(lb, {}))] rest = [lb for lb in labels if lb not in versions] out = "".join(self.ref(lb) for lb in versions) if not rest: return out pins: list[str] = [] shown: set[tuple[str, str]] = set() for label in rest: line_of = self.canon.get(label, label) s = self.sources.get(line_of, {}) quote = str(s["x-quote"]) if isinstance(s.get("x-quote"), str) else "" own = self.sources.get(label, {}).get("x-quote") if isinstance(own, str) and own not in quote: # another quote on the same line: show the one that s, quote = self.sources.get(label, {}), own # supports this claim, not the line's longest if (line_of, quote) in shown: # the same quote again, from this label or another entry on the line pins.append(self.ref(label, " hidden")) continue shown.add((line_of, quote)) rel, line = raw_rel(self.page_rel, s), locator_line(s) name = str(s.get("title") or s.get("_original", {}).get("name", "")) where = esc(", ".join(x for x in (name, place_words(s["x-locator"]) if line else "") if x)) raw = (f' {where}' if rel and line else f" {where}") pins.append(f'{self.ref(label)} {esc(str(quote))}{raw}' f'{original_link(self.page_rel, s)}') mark = f'{self.number(rest[0])}' return (f'{out}{mark}' + "".join(pins) + "") def canonical(sources: dict[str, dict[str, Any]]) -> dict[str, str]: """Each label to the entry that stands for its file and line: the one with the longest quote (the first on a tie). An entry without a line locator stands for itself.""" groups: dict[tuple[str, int], list[str]] = {} out: dict[str, str] = {} for label, s in sources.items(): line = locator_line(s) if line is None: out[label] = label else: groups.setdefault((str(s.get("resource", "")), line), []).append(label) for labels in groups.values(): best = max(labels, key=lambda lb: len(str(sources[lb].get("x-quote", "")))) # max keeps the first on a tie out.update({lb: best for lb in labels}) return out def is_version_heading(source: dict[str, Any]) -> bool: """A source entry whose quote is only a version number: the version heading of a release note.""" quote = source.get("x-quote") return isinstance(quote, str) and VERSION_ONLY.fullmatch(quote.strip()) is not None def inline(text: str, notes: Footnotes | None) -> str: """Links, footnote references, inline code and bold; everything else escaped.""" masked = mask_code(text) out: list[str] = [] pos = 0 run: list[str] = [] # footnote references that follow each other with nothing between them for m in INLINE.finditer(masked): note = m.group(3) is not None if not (note and run and m.start() == pos): if run and notes is not None: out.append(notes.cite(run)) run = [] out.append(plain(text[pos:m.start()])) if note: # a footnote reference if notes is not None: run.append(text[m.start(3):m.end(3)]) else: out.append(esc(text[m.start():m.end()])) else: label_text = text[m.start(1):m.end(1)] target = text[m.start(2):m.end(2)] href = link_target(target) out.append(plain(label_text) if href is None else f'{plain(label_text)}') pos = m.end() if run and notes is not None: out.append(notes.cite(run)) out.append(plain(text[pos:])) return pair_bold("".join(out)) def body_html(doc: Document, notes: Footnotes | None) -> str: """Headings, paragraphs and lists of a body; footnote definitions are left out (the sources list replaces them).""" out: list[str] = [] para: list[str] = [] items: list[str] = [] list_tag = "" def flush() -> None: nonlocal list_tag if para: out.append(f"

{inline(' '.join(para), notes)}

") para.clear() if items: out.append(f"<{list_tag}>" + "".join(f"
  • {inline(i, notes)}
  • " for i in items) + f"") items.clear() list_tag = "" for raw in doc.body.split("\n"): stripped = raw.strip() if not stripped or FOOTNOTE_DEF.match(stripped): flush() continue heading = HEADING.match(raw) if heading: flush() level = len(heading.group(1)) out.append(f"{inline(heading.group(2) or '', notes)}") elif LIST_ITEM.match(raw): tag = "ol" if ORDERED.match(raw) else "ul" if para or (items and tag != list_tag): flush() list_tag = tag items.append(LIST_ITEM.sub("", raw, count=1).strip()) elif items: items[-1] += " " + stripped else: para.append(stripped) flush() return "\n".join(out) def page_shell(title: str, prefix: str, nav_graph: bool, content: str, lang: str = "en") -> str: graph = f' Graph (Google OKF viewer)' if nav_graph else "" return (f"\n\n\n\n" "\n" f"{esc(title)}\n\n\n\n" f'\n' f"{content}\n\n\n") # --- render ---------------------------------------------------------------------------------------------------- def source_entries(fm: dict[str, Any]) -> dict[str, dict[str, Any]]: srcs = fm.get("sources", []) entries = srcs if isinstance(srcs, list) else [srcs] return {s["id"]: s for s in entries if isinstance(s, dict) and isinstance(s.get("id"), str)} def raw_rel(page_rel: str, source: dict[str, Any]) -> str | None: res = source.get("resource") if not isinstance(res, str) or not res or SCHEME.match(res): return None return resolve(page_rel, res) or None def locator_line(source: dict[str, Any]) -> int | None: loc = source.get("x-locator") if isinstance(loc, dict) and type(loc.get("line")) is int and loc["line"] > 0: return int(loc["line"]) return None SectionOf = Callable[[str, int], tuple[int, str] | None] @dataclass class Group: """The entries of one document section: the nearest heading above the cited lines in the raw source.""" title: str rel: str heading_line: int | None heading: str labels: list[str] = field(default_factory=list) lines: list[int] = field(default_factory=list) @property def name(self) -> str: return f"{self.title} › {self.heading}" if self.heading else self.title def source_groups(page_rel: str, notes: Footnotes, section_of: SectionOf) -> list[Group]: """The page's canonical entries grouped by document and section, in the documents' order of first use and then in the order of the sections in the raw source.""" for label in notes.sources: # entries never referenced are listed after the referenced ones notes.number(label) groups: dict[tuple[str, str, int | None], Group] = {} doc_first: dict[tuple[str, str], int] = {} for label, n in sorted(notes.numbers.items(), key=lambda kv: kv[1]): s = notes.sources.get(label, {}) title = str(s.get("title") or s.get("_original", {}).get("name", "")) rel, line = raw_rel(page_rel, s) or "", locator_line(s) sec = section_of(rel, line) if rel and line else None key = (title, rel, sec[0] if sec else None) g = groups.setdefault(key, Group(title, rel, sec[0] if sec else None, sec[1] if sec else "")) g.labels.append(label) if line: g.lines.append(line) doc_first.setdefault((title, rel), n) return sorted(groups.values(), key=lambda g: (doc_first[(g.title, g.rel)], g.heading_line or 0)) def sources_html(page_rel: str, notes: Footnotes, section_of: SectionOf) -> str: """The source list, closed by default: one group per document section, the line span of its cited lines.""" parts: list[str] = [] for g in source_groups(page_rel, notes, section_of): lo, hi = (min(g.lines), max(g.lines)) if g.lines else (0, 0) span = "" if not g.lines else f" (line {lo})" if lo == hi else f" (lines {lo}–{hi})" rows: list[str] = [] for label in g.labels: n = notes.numbers[label] s = notes.sources.get(label) if s is None: rows.append(f'
  • No source entry with id {esc(label)}.
  • ') continue # Other entries on the same file and line come first as zero-height anchors, so the line shows once and # a link to any of them lands here. for alias in (lb for lb, c in notes.canon.items() if c == label and lb != label): rows.append(entry_html(page_rel, alias, notes.sources[alias], n, notes.seen[alias] > 0, ' class="alias"')) rows.append(entry_html(page_rel, label, s, n, notes.seen[label] > 0)) parts.append(f"

    {esc(g.name)}{span}

    \n" + '
      \n' + "\n".join(rows) + "\n
    ") return (f'
    Sources ({len(notes.numbers)})\n' + "\n".join(parts) + "\n
    ") def read_more_html(page_rel: str, notes: Footnotes, section_of: SectionOf) -> str: """One link per document section the page draws on, to the raw source at the section's heading.""" rows = [] for g in source_groups(page_rel, notes, section_of): if not g.rel or not g.lines: continue target = g.heading_line or min(g.lines) rows.append(f'
  • {esc(g.name)}
  • ') return '

    Read more in

    \n" if rows else "" def nearest_heading(lines: list[str], line: int) -> tuple[int, str] | None: """The nearest markdown heading at or above a line (1-based), as (line, text). A heading inside fenced code is example text, not a section: fences follow the auditor's rules (backticks or tildes, also in a list item or a quote block, closed by a run of the same marker at least as long).""" found: tuple[int, str] | None = None fence = "" # the open fence's marker run, or "" outside a fenced block for n, raw in enumerate(lines[:line], 1): f = FENCE.match(raw) if f and f.group(1)[0] == "`" and "`" in raw[f.end():]: f = None # a backtick run with a backtick after it is an inline code span, not a fence (CommonMark 4.5) if fence: if f and f.group(1)[0] == fence[0] and len(f.group(1)) >= len(fence) and not raw[f.end():].strip(): fence = "" continue if f: fence = f.group(1) continue m = HEADING.match(raw) if m and m.group(2): found = (n, m.group(2)) return found def entry_html(page_rel: str, label: str, s: dict[str, Any], n: int, seen: bool, cls: str = "") -> str: quote = s.get("x-quote") if isinstance(s.get("x-quote"), str) else "" line = locator_line(s) rel = raw_rel(page_rel, s) res = str(s.get("resource", "")) loc = f"line {line} of {esc(res)}" if line else f"no line locator, {esc(res)}" if line and set(s["x-locator"]) != {"line"}: loc = f"{esc(place_words(s['x-locator']))} of {esc(s.get('_original', {}).get('name', ''))} ({loc})" raw = (f' open the raw source at line {line}' if rel and line else "") + original_link(page_rel, s) back = f' back to text' if seen else "" return (f'{esc(str(s.get("title", "")))}' f'{esc(str(quote))}{loc}{raw}{back}') def related(concepts: list[Document]) -> dict[str, list[str]]: """For each concept page, the other concept pages it links to or that link to it.""" rels = {d.rel for d in concepts} links: dict[str, set[str]] = {r: set() for r in rels} for d in concepts: for _line, target in body_links(d): other = resolve(d.rel, target) if is_bundle_link(target) else None if other in rels and other != d.rel: links[d.rel].add(other) links[other].add(d.rel) return {r: sorted(v) for r, v in links.items()} def see_also_html(page_rel: str, others: list[str], docs: dict[str, Document]) -> str: if not others: return "" rows = [] for other in others: fm = docs[other].frontmatter or {} title = fm.get("title") if isinstance(fm.get("title"), str) else other desc = f' - {esc(fm["description"])}' if isinstance(fm.get("description"), str) else "" href = posixpath.relpath(html_name(other), posixpath.dirname(html_name(page_rel)) or ".") rows.append(f'
  • {esc(str(title))}{desc}
  • ') return '

    See also

    \n
      \n' + "\n".join(rows) + "\n
    " def meta_html(doc: Document) -> str: fm = doc.frontmatter or {} parts: list[str] = [] if isinstance(fm.get("description"), str): parts.append(f"

    {esc(fm['description'])}

    ") names = fm.get("x-aliases") if isinstance(names, list) and names: parts.append("

    Also called: " + "; ".join(esc(str(n)) for n in names) + "

    ") tags = fm.get("tags") if isinstance(tags, list) and tags: parts.append("

    Tags: " + ", ".join(esc(str(t)) for t in tags) + "

    ") gen = fm.get("generated") if isinstance(gen, dict): parts.append(f"

    Generated by {esc(str(gen.get('by', '')))} at {esc(str(gen.get('at', '')))}

    ") parts.append(f"

    Page file: {esc(doc.rel)}

    ") return '
    ' + "".join(parts) + "
    " def raw_html(rel: str, text: str, cited: set[int], lang: str = "en") -> str: rows = [] for n, line in enumerate(text.splitlines(), 1): cls = "line cited" if n in cited else "line" rows.append(f'
    {n}{esc(line)}
    ') digest = hashlib.sha256(text.encode("utf-8")).hexdigest() content = (f"

    Raw source /{esc(rel)}

    \n" f'

    sha256 {digest} · {len(rows)} lines · cited lines are shaded

    \n' f'
    \n' + "\n".join(rows) + "\n
    ") return page_shell(f"/{rel}", up(rel) + "../", False, content, lang) def write(path: Path, text: str) -> None: path.parent.mkdir(parents=True, exist_ok=True) path.write_text(text, encoding="utf-8") # --- change marks (contract K4) ---------------------------------------------------------------------------------- MARKS_SCHEMA = "okf-wiki/source-marks/1" CHANGES = ("changed", "gone") Marks = dict[str, list[tuple[str, str]]] # page -> [(source path, change)], in the file's order # Inline, so the shared stylesheet, and with it the view without marks, stays as it was. MARK_STYLE = "border: 2px solid #c25e00; background: #fff4e5; border-radius: 4px; padding: 0.5rem 0.75rem; margin-bottom: 1rem;" def read_marks(path: Path) -> Marks: """The pages and their marked sources from a change-marks file in the form of contract K4 point 2; RunError when the file cannot be read as one. The K1 module is not imported: the view reads the file itself.""" try: data = json.loads(path.read_text(encoding="utf-8")) except (OSError, UnicodeDecodeError, ValueError) as exc: raise RunError(f"the marks file {path} cannot be read as JSON: {exc}") from exc if not isinstance(data, dict) or data.get("schema") != MARKS_SCHEMA or not isinstance(data.get("pages"), list): raise RunError(f"the marks file {path} is not {MARKS_SCHEMA} with a list of pages") marks: Marks = {} for entry in data["pages"]: page = entry.get("page") if isinstance(entry, dict) else None srcs = entry.get("sources") if isinstance(entry, dict) else None if not isinstance(page, str) or not isinstance(srcs, list) or not srcs or page in marks: raise RunError(f"the marks file {path}: an entry is not one page with a list of sources: {entry!r}") for s in srcs: if not (isinstance(s, dict) and isinstance(s.get("path"), str) and s.get("change") in CHANGES): raise RunError(f"the marks file {path}: {page}: a source is not a path with change changed or gone: {s!r}") marks[page] = [(s["path"], s["change"]) for s in srcs] return marks def check_marks(marks: Marks, docs: dict[str, Document]) -> None: """RunError unless every marked page is a concept page of the bundle that cites each of its marked sources as "/" + path in sources[].resource (K4 point 2), so marks from another bundle are never shown.""" for page, entries in marks.items(): if page not in docs: raise RunError(f"the marks name {page}, which is not a concept page of the bundle") srcs = (docs[page].frontmatter or {}).get("sources") resources = {s.get("resource") for s in (srcs if isinstance(srcs, list) else [srcs]) if isinstance(s, dict)} for path, _change in entries: if "/" + path not in resources: raise RunError(f"the marks say {page} builds on /{path}, but the page does not cite it") def mark_html(entries: list[tuple[str, str]]) -> str: rows = "".join(f"
  • /{esc(path)} {esc(change)}
  • " for path, change in entries) return (f'
    This page builds on a source that has ' f"changed or is gone since the frozen copy.\n
      {rows}
    ") def marked_pages_html(marks: Marks, docs: dict[str, Document]) -> str: """The list of marked pages at the top of the root index.html, or a line that says there are none.""" rows = [] for page in sorted(marks): fm = docs[page].frontmatter or {} title = fm.get("title") if isinstance(fm.get("title"), str) else page srcs = ", ".join(f"/{esc(path)} {esc(change)}" for path, change in marks[page]) rows.append(f'
  • {esc(str(title))}: {srcs}
  • ') listing = "
      " + "".join(rows) + "
    " if rows else "

    No page builds on a changed or gone source.

    " return (f'

    Pages built on changed or gone sources ({len(rows)})

    \n' f"{listing}
    \n") def render(bundle: Path, sources: Path, out: Path, graph: Path | None = None, marks: Marks | None = None) -> RenderStats: b = load_bundle(bundle, sources) if marks is not None: check_marks(marks, {d.rel: d for d in b.concepts}) stats = RenderStats() cited: dict[str, set[int]] = {} has_graph = graph is not None if graph is not None: out.mkdir(parents=True, exist_ok=True) shutil.copyfile(graph, out / GRAPH) docs = {d.rel: d for d in b.concepts} links = related(b.concepts) raw_text: dict[str, list[str] | None] = {} def section_of(rel: str, line: int) -> tuple[int, str] | None: if rel not in raw_text: try: raw_text[rel] = (sources / rel).read_text(encoding="utf-8").splitlines() except (OSError, UnicodeDecodeError): raw_text[rel] = None lines = raw_text[rel] return nearest_heading(lines, line) if lines is not None else None stats.islands = sum(1 for others in links.values() if not others) known = originals(sources) langs = languages(sources) counted = Counter(langs.values()).most_common(1) collection_lang = counted[0][0] if counted else "en" copies: set[str] = set() for doc in b.concepts: fm = doc.frontmatter or {} entries = source_entries(fm) for s in entries.values(): rel = raw_rel(doc.rel, s) if rel in known: s["_original"] = known[rel] # in memory only: the view names the original and opens its copy if "copy" in known[rel]: copies.add(known[rel]["copy"]) notes = Footnotes(entries, doc.rel) body = body_html(doc, notes) stats.footnotes += sum(notes.seen.values()) for s in notes.sources.values(): rel, line = raw_rel(doc.rel, s), locator_line(s) if rel and line: cited.setdefault(rel, set()).add(line) title = fm.get("title") if isinstance(fm.get("title"), str) else doc.rel also = see_also_html(doc.rel, links[doc.rel], docs) more = read_more_html(doc.rel, notes, section_of) content = (meta_html(doc) + "\n" + body + "\n" + "".join(x + "\n" for x in (more, also) if x) + sources_html(doc.rel, notes, section_of)) if marks is not None and doc.rel in marks: content = mark_html(marks[doc.rel]) + "\n" + content cited_langs = Counter(langs[r] for r in (raw_rel(doc.rel, s) for s in entries.values()) if r in langs) lang = cited_langs.most_common(1)[0][0] if cited_langs else collection_lang write(out / html_name(doc.rel), page_shell(str(title), up(doc.rel), has_graph, content, lang)) stats.pages += 1 for name in ("index.md", "log.md"): for doc in b.named(name): title = "Index" if name == "index.md" else "Log" listing = marked_pages_html(marks, docs) if marks is not None and doc.rel == "index.md" else "" write(out / html_name(doc.rel), page_shell(title, up(doc.rel), has_graph, listing + body_html(doc, None), collection_lang)) for rel, lines in sorted(cited.items()): try: text = (sources / rel).read_text(encoding="utf-8") except (OSError, UnicodeDecodeError): continue # the check reports its footnotes as unresolved write(out / "raw" / (rel + ".html"), raw_html(rel, text, lines, langs.get(rel, collection_lang))) stats.raw_files += 1 for copy in sorted(copies): if (sources / copy).is_file(): (out / copy).parent.mkdir(parents=True, exist_ok=True) shutil.copyfile(sources / copy, out / copy) return stats def languages(sources: Path) -> dict[str, str]: """Per raw source, the language `sources folder` recorded for it (K1 `lang`); none for collection 1.""" try: doc = json.loads((sources / "source-manifest.json").read_text(encoding="utf-8")) except (OSError, ValueError): return {} return {e["path"]: e["lang"] for e in (doc.get("sources", []) if isinstance(doc, dict) else []) if isinstance(e, dict) and isinstance(e.get("path"), str) and isinstance(e.get("lang"), str)} def originals(sources: Path) -> dict[str, dict[str, str]]: """Per raw source (kilde/...) of a folder, the original's name, and for PDF and Office its copy under original/.""" try: doc = json.loads((sources / "source-manifest.json").read_text(encoding="utf-8")) except (OSError, ValueError): return {} found: dict[str, dict[str, str]] = {} for e in doc.get("sources", []) if isinstance(doc, dict) else []: if isinstance(e, dict) and isinstance(e.get("path"), str) and isinstance(e.get("original"), str): found[e["path"]] = {"name": PurePosixPath(e["original"]).name} if isinstance(e.get("original_copy"), str): found[e["path"]]["copy"] = e["original_copy"] return found def original_link(page_rel: str, s: dict[str, Any]) -> str: """A link that opens the original PDF or Office file, a PDF at its page; "" for a text source.""" known, loc = s.get("_original", {}), s.get("x-locator") if "copy" not in known or not isinstance(loc, dict): return "" at = f"#page={loc['page']}" if type(loc.get("page")) is int else "" words = f"open {known['name']} at {place_words(loc)}" if set(loc) != {"line"} else f"open {known['name']}" return f' {esc(words)}' # --- check (reads the rendered HTML only) ---------------------------------------------------------------------- class PageParser(HTMLParser): """Footnote references and source entries (quote, locator text, raw links) of one rendered page.""" def __init__(self) -> None: super().__init__(convert_charrefs=True) self.refs: list[str] = [] self.entries: dict[str, dict[str, Any]] = {} self._entry: str | None = None self._li_depth = 0 self._capture: str | None = None def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: a = {k: v or "" for k, v in attrs} classes = a.get("class", "").split() if tag == "a" and "fnref" in classes and a.get("href", "").startswith("#src-"): self.refs.append(a["href"][len("#src-"):]) if tag == "li": if self._entry is None and a.get("id", "").startswith("src-"): self._entry = a["id"][len("src-"):] self.entries[self._entry] = {"quote": None, "loc": "", "raw": []} self._li_depth = 0 elif self._entry is not None: self._li_depth += 1 if self._entry is not None: e = self.entries[self._entry] if tag == "q" and "quote" in classes: self._capture, e["quote"] = "quote", "" elif tag == "span" and "loc" in classes: self._capture = "loc" elif tag == "a" and "raw" in classes: e["raw"].append(a.get("href", "")) def handle_endtag(self, tag: str) -> None: if tag in ("q", "span"): self._capture = None if tag == "li" and self._entry is not None: if self._li_depth == 0: self._entry = None else: self._li_depth -= 1 def handle_data(self, data: str) -> None: if self._entry is not None and self._capture is not None: self.entries[self._entry][self._capture] += data class RawParser(HTMLParser): """The text of each line anchor (div id="L" > span.text) of a raw view.""" def __init__(self) -> None: super().__init__(convert_charrefs=True) self.lines: dict[str, str] = {} self._line: str | None = None self._in_text = False def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: a = {k: v or "" for k, v in attrs} if tag == "div" and re.fullmatch(r"L\d+", a.get("id", "")): self._line = a["id"] self.lines[self._line] = "" elif tag == "span" and self._line is not None and "text" in a.get("class", "").split(): self._in_text = True def handle_endtag(self, tag: str) -> None: if tag == "span": self._in_text = False elif tag == "div": self._line = None def handle_data(self, data: str) -> None: if self._in_text and self._line is not None: self.lines[self._line] += data def markdown_refs(doc: Document) -> Counter[str]: """Footnote references in a page body, outside code spans and footnote definitions (the denominator).""" found: Counter[str] = Counter() for raw in mask_code(doc.body).split("\n"): if not FOOTNOTE_DEF.match(raw.strip()): found.update(FOOTNOTE_REF.findall(raw)) return found def check(bundle: Path, out: Path) -> CheckResult: if not out.is_dir(): raise RunError(f"the view {out} is not a directory") b = load_bundle(bundle, bundle) result = CheckResult() raw_cache: dict[Path, dict[str, str] | None] = {} root = out.resolve() def raw_lines(path: Path) -> dict[str, str] | None: if path not in raw_cache: try: parser = RawParser() parser.feed(path.read_text(encoding="utf-8")) raw_cache[path] = parser.lines except (OSError, UnicodeDecodeError): raw_cache[path] = None return raw_cache[path] for doc in b.concepts: want = markdown_refs(doc) total = sum(want.values()) page = out / html_name(doc.rel) ok: Counter[str] = Counter() try: parser = PageParser() parser.feed(page.read_text(encoding="utf-8")) except (OSError, UnicodeDecodeError): result.problems.append(f"{doc.rel}: no rendered page {page.name}") parser = PageParser() verdict: dict[str, str] = {} for label in parser.refs: if label not in verdict: verdict[label] = resolve_entry(parser.entries.get(label), page, root, raw_lines) if verdict[label] == "": ok[label] += 1 for label, why in sorted(verdict.items()): if why: result.problems.append(f"{doc.rel}: [^{label}] {why}") resolved = sum(min(n, ok[label]) for label, n in want.items()) for label, n in sorted(want.items()): if ok[label] < n and label not in verdict: result.problems.append(f"{doc.rel}: [^{label}] has no clickable reference in the page") result.per_page[doc.rel] = (resolved, total) result.resolved += resolved result.total += total return result def resolve_entry(entry: dict[str, Any] | None, page: Path, root: Path, raw_lines: Callable[[Path], dict[str, str] | None]) -> str: """"" when the entry shows a quote, its line and a raw link whose line anchor holds the quote; else why not.""" if entry is None: return "has no source entry" quote = entry["quote"] if not quote: return "has no quote in its source entry" for href in entry["raw"]: path, _, frag = href.partition("#") m = re.fullmatch(r"L(\d+)", frag) if not m or f"line {m.group(1)}" not in entry["loc"]: continue target = (page.parent / path).resolve() if not target.is_relative_to(root): continue lines = raw_lines(target) if lines is not None and quote in lines.get(frag, ""): return "" return "does not reach a raw line that holds its quote" # --- draw ------------------------------------------------------------------------------------------------------ def draw(bundle: Path, seed: int, count: int) -> list[str]: pages = [d.rel for d in load_bundle(bundle, bundle).concepts] if count > len(pages): raise RunError(f"cannot draw {count} pages from {len(pages)}") return random.Random(seed).sample(pages, count) def draw_sha256(seed: int, count: int, pages: list[str]) -> str: """The sha256 that locks a draw: of "seed N\\ncount K\\n" and each page with a newline, in the order printed.""" text = f"seed {seed}\ncount {count}\n" + "".join(page + "\n" for page in pages) return hashlib.sha256(text.encode("utf-8")).hexdigest() # --- versions -------------------------------------------------------------------------------------------------- @dataclass(frozen=True) class Flagged: page: str line: int sentence: str numbers: tuple[str, ...] @dataclass class VersionCount: total: int = 0 flagged: list[Flagged] = field(default_factory=list) licensed: dict[str, int] = field(default_factory=lambda: {"outdated": 0, "content-quote": 0, "page-title": 0}) def version_numbers(text: str) -> list[str]: """The version numbers in running text (code spans masked), without a leading v, in order of first use.""" masked = mask_code(text) found = [m.group(1) for m in VERSION.finditer(masked)] + [m.group(1) for m in VERSION_WORD.finditer(masked)] return list(dict.fromkeys(found)) def holds_number(text: str, number: str) -> bool: return re.search(r"(? VersionCount: docs = {d.rel: d for d in load_bundle(bundle, sources).concepts} result = VersionCount() for claim in extract(bundle, sources).claims: result.total += 1 numbers = version_numbers(claim.claim) if not numbers: continue fm = docs[claim.page].frontmatter or {} pairs = fm.get("x-outdated") outdated = {str(p.get(k)) for p in (pairs if isinstance(pairs, list) else []) if isinstance(p, dict) for k in ("old", "new")} if outdated & set(claim.footnotes): result.licensed["outdated"] += 1 continue entries = source_entries(fm) quotes = [str(entries[k].get("x-quote", "")) for k in claim.footnotes if k in entries and not is_version_heading(entries[k])] title = str(fm.get("title", "")) left = [n for n in numbers if not any(holds_number(q, n) for q in quotes)] if not left: result.licensed["content-quote"] += 1 elif all(holds_number(title, n) for n in left): result.licensed["page-title"] += 1 else: result.flagged.append(Flagged(claim.page, claim.line, claim.claim, tuple(left))) return result # --- overlap --------------------------------------------------------------------------------------------------- @dataclass class OverlapCount: entries: int = 0 redundant: list[tuple[str, str, str]] = field(default_factory=list) # (page, kept label, redundant label) claims: int = 0 substring: list[tuple[str, str, str]] = field(default_factory=list) # (page, sentence, the longer sentence) def _bare(sentence: str) -> str: return sentence.strip().rstrip(".!?").strip() def overlap(bundle: Path, sources: Path) -> OverlapCount: """Source entries on a page that repeat another entry's file and line, and sentences that are a prefix or substring of another sentence on the same page (end punctuation ignored).""" result = OverlapCount() for doc in load_bundle(bundle, sources).concepts: entries = source_entries(doc.frontmatter or {}) result.entries += len(entries) for label, kept in canonical(entries).items(): if label != kept: result.redundant.append((doc.rel, kept, label)) by_page: dict[str, list[str]] = {} for claim in extract(bundle, sources).claims: result.claims += 1 by_page.setdefault(claim.page, []).append(claim.claim) for page, sentences in by_page.items(): for i, a in enumerate(sentences): longer = next((b for j, b in enumerate(sentences) if j != i and _bare(a) in _bare(b)), None) if longer is not None: result.substring.append((page, a, longer)) return result # --- command line ---------------------------------------------------------------------------------------------- def build_parser() -> argparse.ArgumentParser: p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) sub = p.add_subparsers(dest="command", required=True) r = sub.add_parser("render", help="write the HTML view") r.add_argument("bundle") r.add_argument("sources") r.add_argument("--out", required=True) r.add_argument("--graph", help="a finished graph file (Google's viewer output), copied unchanged") r.add_argument("--marks", help="a change-marks file (contract K4): mark the pages built on changed or gone sources") c = sub.add_parser("check", help="count the footnotes that resolve in the HTML view") c.add_argument("bundle") c.add_argument("view") d = sub.add_parser("draw", help="draw pages for the operator's reading") d.add_argument("bundle") d.add_argument("--seed", type=int, required=True) d.add_argument("--count", type=int, required=True) v = sub.add_parser("versions", help="count version numbers in running text that are not the claim") v.add_argument("bundle") v.add_argument("sources") o = sub.add_parser("overlap", help="count redundant source entries and overlapping sentences") o.add_argument("bundle") o.add_argument("sources") return p def main(argv: list[str] | None = None) -> int: args = build_parser().parse_args(argv) try: if not Path(args.bundle).is_dir(): raise RunError(f"the bundle {args.bundle} is not a directory") if args.command == "render": if not Path(args.sources).is_dir(): raise RunError(f"the raw-source root {args.sources} is not a directory") graph = Path(args.graph) if args.graph else None if graph is not None and not graph.is_file(): raise RunError(f"the graph file {graph} does not exist") marks = read_marks(Path(args.marks)) if args.marks else None s = render(Path(args.bundle), Path(args.sources), Path(args.out), graph, marks) print(f"render: {s.pages} pages, {s.raw_files} raw source files, {s.footnotes} footnotes -> {args.out}") print(f"pages without a link to or from another page (islands): {s.islands} of {s.pages}") return 0 if args.command == "check": res = check(Path(args.bundle), Path(args.view)) print(f"footnotes resolved: {res.resolved} of {res.total}") short = [f"{p}: {r} of {t}" for p, (r, t) in sorted(res.per_page.items()) if r < t] for line in short + res.problems[:50]: print(" " + line) if res.total == 0: print("FAIL: no footnotes to resolve", file=sys.stderr) return 1 return 0 if res.resolved == res.total else 1 if args.command in ("versions", "overlap") and not Path(args.sources).is_dir(): raise RunError(f"the raw-source root {args.sources} is not a directory") if args.command == "overlap": oc = overlap(Path(args.bundle), Path(args.sources)) print(f"redundant source entries (same file and line): {len(oc.redundant)} of {oc.entries}") print("sentences that are a prefix or substring of another on the same page: " f"{len(oc.substring)} of {oc.claims}") for page, n in sorted(Counter(p for p, _, _ in oc.redundant).items()): print(f" {page}: {n} redundant") for page, kept, label in oc.redundant: print(f" {page}: [^{label}] repeats the file and line of [^{kept}]") for page, a, b in oc.substring: print(f" {page}: \"{a}\" is inside \"{b}\"") if oc.entries == 0 or oc.claims == 0: print("FAIL: no source entries or claim sentences to count", file=sys.stderr) return 1 return 0 if not oc.redundant and not oc.substring else 1 if args.command == "versions": vc = versions(Path(args.bundle), Path(args.sources)) print(f"version numbers in running text outside dated history: {len(vc.flagged)} of {vc.total}") print(" not counted (the version is the claim): " + ", ".join(f"{k} {n}" for k, n in vc.licensed.items())) for page, n in sorted(Counter(f.page for f in vc.flagged).items()): print(f" {page}: {n}") for f in vc.flagged: print(f" {f.page}:{f.line} [{' '.join(f.numbers)}] {f.sentence}") if vc.total == 0: print("FAIL: no claim sentences to count", file=sys.stderr) return 1 return 0 if not vc.flagged else 1 pages = draw(Path(args.bundle), args.seed, args.count) for page in pages: print(page) print(f"sha256 {draw_sha256(args.seed, args.count, pages)}") return 0 except RunError as exc: print(f"FAIL: {exc}", file=sys.stderr) return 2 if __name__ == "__main__": sys.exit(main())