"""The auditor: checks an OKF wiki bundle against its raw sources. It never calls a language model (CLAUDE.md rule 1). Usage: python3 -m okfwiki.audit BUNDLE SOURCES BUNDLE is the wiki directory. SOURCES is the raw-source root that a bundle-absolute `sources[].resource` (`/kilde/x.md`) resolves against, and that holds the source manifest `source-manifest.json` (contract K1). L1a reads raw sources as UTF-8 text with a `{ line: N }` locator only. Output on stdout, one JSON object: {"findings": [{"check", "group", "file", "line", "message"}, ...], "summary": {: n}, "link_rule": {"line", "main_pages": {"pass", "of"}, "linked": {"pass", "of"}, "misplaced", "broken", "not_to_concept", "max_out_degree", "provenance"}}. `link_rule` is criterion 6 as numbers (contract K2 point 7): "main pages: N of N sources · linked from a main page: N of N pages". Exit 0 clean, 1 at least one finding, 2 the audit could not run (bad arguments; no valid K1 file at SOURCES, since the main-page rule cannot run without it, K2 point 6; or no main-topic source in K1 and no concept page, no denominator). Four groups are reported apart, because they answer different questions: - spec-11: OKF v0.2 SPEC section 11 conformance (PLAN.md criterion 5). - links: okf-wiki's own link rules. SPEC 6.1 says consumers MUST tolerate broken links, so these are not section 11. - citations: every body sentence carries a footnote, every footnote a source, every source a verbatim quote at its locator (PLAN.md criterion 1, format choice f). - format: format choices (a), (b), (c) and (e) in PLAN.md section Rammer. The exact definitions of the link classes, including link-misplaced (b) of criterion 6 and the main-page rule (contract K2: main-page-missing, main-page-duplicate, main-page-invalid, no-main-page-link), are in gates/mutants.manifest.json under provenance.link_rules. The rule of 04.10.2026 (an inbound link from any other concept page) is no finding any more; link_rule_report() still counts it, for the linker and the trial. The function audit() reports the main-page rule only when SOURCES holds a K1 file (K2 point 18); without one it gives the other checks, as the trial's tests and the assembler's call it on roots without K1. """ from __future__ import annotations import json import os import re import sys from dataclasses import asdict, dataclass, field from datetime import date, datetime from pathlib import Path, PurePosixPath from typing import Any, Callable from urllib.parse import unquote import yaml from okfwiki.common import place_fits RESERVED = ("index.md", "log.md") # SPEC 3.1 GROUPS = ("spec-11", "links", "citations", "format") GROUP_OF = { "spec11-1-frontmatter": "spec-11", "spec11-2-type": "spec-11", "spec11-3-index": "spec-11", "spec11-3-log": "spec-11", "broken-link": "links", "orphan-page": "links", "index-coverage": "links", "link-misplaced": "links", "main-page-missing": "links", "main-page-duplicate": "links", "main-page-invalid": "links", "no-main-page-link": "links", "uncited-sentence": "citations", "quote-not-verbatim": "citations", "quote-wrong-locator": "citations", "quote-wrong-place": "citations", "quote-missing": "citations", "locator-unsupported": "citations", "source-unreadable": "citations", "source-missing-id": "citations", "source-duplicate-id": "citations", "footnote-unknown-id": "citations", "resource-not-bundle-absolute": "format", "body-link-bundle-absolute": "format", "timestamp-form": "format", "verified-not-human": "format", } class _Loader(yaml.SafeLoader): """SafeLoader without the timestamp resolver, as Google's reference `document.py` does (format choice c).""" _Loader.yaml_implicit_resolvers = { ch: [(tag, rx) for tag, rx in resolvers if tag != "tag:yaml.org,2002:timestamp"] for ch, resolvers in yaml.SafeLoader.yaml_implicit_resolvers.items() } @dataclass(frozen=True) class Finding: check: str group: str file: str line: int | None message: str def finding(check: str, file: str, line: int | None, message: str) -> Finding: return Finding(check=check, group=GROUP_OF[check], file=file, line=line, message=message) # --- documents ------------------------------------------------------------------------------------------------- @dataclass class Document: """One markdown file split as SPEC section 4 says: a frontmatter block, then the body.""" rel: str has_frontmatter: bool frontmatter: dict[str, Any] | None # None when absent or not parseable to a mapping frontmatter_error: str body: str body_start: int # 1-based file line of the body's first line def read_document(path: Path, rel: str) -> Document: text = path.read_text(encoding="utf-8") lines = text.split("\n") if not lines or lines[0].strip() != "---": return Document(rel, False, None, "", text, 1) end = next((i for i in range(1, len(lines)) if lines[i].strip() == "---"), None) if end is None: return Document(rel, True, None, "the frontmatter block is not closed with ---", "", len(lines) + 1) body = "\n".join(lines[end + 1:]) try: data = yaml.load("\n".join(lines[1:end]), Loader=_Loader) except yaml.YAMLError as exc: first = str(exc).splitlines()[0] if str(exc) else "YAML error" return Document(rel, True, None, f"the frontmatter does not parse: {first}", body, end + 2) if data is None: data = {} if not isinstance(data, dict): return Document(rel, True, None, "the frontmatter is not a YAML mapping", body, end + 2) return Document(rel, True, data, "", body, end + 2) # --- markdown body --------------------------------------------------------------------------------------------- # A code span: a backtick run not escaped and not part of a longer run, closed by a run of the same length (6.1). CODE_SPAN = re.compile(r"(?)*[ \t]*(`{3,}|~{3,})") # in a list item or a quote block too LINK = re.compile(r"(?]*)>?(?:\s+\"[^\"]*\")?\s*\)") FOOTNOTE_REF = re.compile(r"\[\^([^\]\s]+)\](?!:)") FOOTNOTE_DEF = re.compile(r"^\[\^([^\]\s]+)\]:") HEADING = re.compile(r"^ {0,3}(#{1,6})(?:\s+(.*?))?\s*#*\s*$") LIST_ITEM = re.compile(r"^\s*(?:[-*+]|\d+[.)])\s+") SCHEME = re.compile(r"^[A-Za-z][A-Za-z0-9+.-]*:") # A sentence ends at . ! or ?, optionally followed by closing quotes or brackets and by footnote references, # then whitespace. sentences() skips an end followed by a lowercase letter, which keeps abbreviations such as # "e.g. overnight" inside their sentence. SENTENCE_END = re.compile(r"[.!?][\"'»”’)\]]*(?:\[\^[^\]\s]+\])*(?=\s+(\S))") # A period after one of these ends no sentence, whatever follows ("kl. 14", "f.eks. Oslo", "e.g. Claude Code"). Only # abbreviations that seldom end a sentence are here: "osv.", "etc." and "m.m." usually close one and still cut. A # single capital letter before the period is an initial and does not cut either. ABBREVIATIONS = frozenset( "f.eks bl.a kl ca nr jf jfr dvs evt ev inkl ekskl mht pga ift iht vedr ang tlf stk pkt kap avd ref hhv mill mrd" " e.g i.e cf vs approx incl mr mrs ms dr prof".split()) ABBREVIATION_BEFORE = re.compile(r"([^\s(\[\"'«]+)\.$") LINK_ONLY = re.compile(r"^(?:\s*\[[^\]\n]*\]\([^()\s]*\)\s*[,;]?)+\s*$") def fence_marker(raw: str) -> re.Match[str] | None: """The fence run that opens or closes a fenced block on this line (CommonMark 4.5), or None. A backtick run with a backtick after it is an inline code span, not a fence.""" f = FENCE.match(raw) if f and f.group(1)[0] == "`" and "`" in raw[f.end():]: return None return f def closes_fence(f: re.Match[str] | None, raw: str, fence: str) -> bool: """A fence closes on a run of the opening character at least as long as the opening run, with nothing after it.""" return bool(f and f.group(1)[0] == fence[0] and len(f.group(1)) >= len(fence) and not raw[f.end():].strip()) def mask_code(text: str) -> str: """Blank out code so nothing inside it reads as a link, footnote or sentence end; keeps offsets and line breaks. A fenced block (backticks or tildes) is masked line by line after its opening run. An inline code span pairs backtick runs of equal length only inside one block: a blank line, a heading, a list item or a footnote definition starts a new one, and a backslash before a backtick makes it literal (CommonMark 6.1).""" out: list[str] = [] block: list[str] = [] def flush() -> None: if block: out.append(CODE_SPAN.sub(lambda m: m.group(1) + "x" * len(m.group(2)) + m.group(1), "\n".join(block))) block.clear() fence = "" for raw in text.split("\n"): f = fence_marker(raw) if fence: if closes_fence(f, raw, fence): fence = "" out.append(raw) else: out.append("x" * len(raw)) elif f: flush() fence = f.group(1) out.append(raw[:f.end()] + "x" * len(raw[f.end():])) elif not raw.strip() or HEADING.match(raw): flush() out.append(CODE_SPAN.sub(lambda m: m.group(1) + "x" * len(m.group(2)) + m.group(1), raw)) else: if LIST_ITEM.match(raw) or FOOTNOTE_DEF.match(raw.strip()): flush() block.append(raw) flush() return "\n".join(out) def link_matches(text: str, masked: str | None = None) -> list[re.Match[str]]: """The inline links of text. A [ inside a code span opens no link (CommonMark 6.1): a match that opens in code and closes outside it is passed over from its bracket, so the link after it is found. A link written wholly inside code is kept, and placed_links calls it code. masked is mask_code(text), or the same slice of it.""" masked = mask_code(text) if masked is None else masked out: list[re.Match[str]] = [] pos = 0 while (m := LINK.search(text, pos)) is not None: if masked[m.start()] != text[m.start()] and masked[m.end() - 1] == text[m.end() - 1]: pos = m.start() + 1 continue out.append(m) pos = m.end() return out def sub_links(repl: Callable[[re.Match[str]], str], text: str) -> str: """LINK.sub over link_matches: a bracket inside a code span is left as it stands.""" parts: list[str] = [] last = 0 for m in link_matches(text): parts += [text[last:m.start()], repl(m)] last = m.end() return "".join(parts) + text[last:] @dataclass class Block: line: int text: str is_list_item: bool def prose_blocks(doc: Document) -> list[Block]: """Paragraphs and list items of the body; headings and footnote definitions are not prose.""" blocks: list[Block] = [] current: Block | None = None for i, raw in enumerate(doc.body.split("\n")): line_no = doc.body_start + i if not raw.strip(): current = None continue if HEADING.match(raw) or FOOTNOTE_DEF.match(raw.strip()): current = None continue if LIST_ITEM.match(raw): current = Block(line_no, LIST_ITEM.sub("", raw, count=1), True) blocks.append(current) elif current is None: current = Block(line_no, raw.strip(), False) blocks.append(current) else: current.text += " " + raw.strip() return blocks def sentence_spans(text: str) -> list[tuple[int, int]]: """(start, end) of every sentence in text, found on the code-masked text, so the offsets fit the original too. A bundle link never moves a cut: the ends are found on the masked text with every bundle link outside code reduced to its anchor text (klink.reduce_links' notion), and each cut is mapped back to the linked text, after the whole link when it falls at the anchor's end. So "U.S. [primary](p.md) was" stays one sentence, as "U.S. primary was" is. Text without a bundle link is split exactly as before: the reduced text is the masked text.""" masked = mask_code(text) plain, after = masked, list(range(1, len(masked) + 1)) # after[i]: offset in masked just after plain[i] links = [m for m in link_matches(text, masked) if masked[m.start()] == "[" and is_bundle_link(m.group(2))] if links: parts: list[str] = [] after = [] last = 0 for m in links: parts += [masked[last:m.start()], masked[m.start(1):m.end(1)]] after += list(range(last + 1, m.start() + 1)) + list(range(m.start(1) + 1, m.end(1) + 1)) if m.end(1) > m.start(1): after[-1] = m.end() last = m.end() parts.append(masked[last:]) after += list(range(last + 1, len(masked) + 1)) plain = "".join(parts) cuts: list[tuple[int, int]] = [] start = 0 for e in sentence_ends(plain): end = after[e - 1] cuts.append((start, end)) start = end cuts.append((start, len(masked))) out: list[tuple[int, int]] = [] for a, b in cuts: piece = masked[a:b] if not re.search(r"\w", FOOTNOTE_REF.sub("", piece)): continue lead = len(piece) - len(piece.lstrip()) out.append((a + lead, a + len(piece.rstrip()))) return out def sentence_ends(masked: str) -> list[int]: """The offset just after each sentence end of a code-masked text.""" ends: list[int] = [] for m in SENTENCE_END.finditer(masked): if m.group(1).islower(): continue word = ABBREVIATION_BEFORE.search(masked, max(0, m.start() - 20), m.start() + 1) w = word.group(1) if word else "" if masked[m.start()] == "." and (w.lower() in ABBREVIATIONS and w in (w.lower(), w.capitalize()) or len(w) == 1 and w.isupper()): continue # an abbreviation, or an initial ("Steven L. Rickman") ends.append(m.end()) return ends def sentences(text: str) -> list[str]: masked = mask_code(text) return [masked[a:b] for a, b in sentence_spans(text)] def body_links(doc: Document) -> list[tuple[int, str]]: """(file line, target) for every inline markdown link in the body, outside code spans.""" out: list[tuple[int, str]] = [] for i, raw in masked_lines(doc.body): for m in LINK.finditer(raw): out.append((doc.body_start + i, m.group(2))) return out def masked_lines(body: str) -> list[tuple[int, str]]: """(index, masked line) per line of the unmasked body. mask_code also masks a fenced block's line breaks, so the masked text is cut at the body's own line offsets: a line after a fenced block keeps its own number.""" masked = mask_code(body) out: list[tuple[int, str]] = [] offset = 0 for i, raw in enumerate(body.split("\n")): out.append((i, masked[offset:offset + len(raw)])) offset += len(raw) + 1 return out def is_bundle_link(target: str) -> bool: return bool(target) and not target.startswith("#") and not SCHEME.match(target) def resolve(rel_file: str, target: str) -> str | None: """Bundle-relative POSIX path a link or path names, or None if it leaves the bundle (SPEC 6.1, 6.2).""" path = unquote(target.split("#", 1)[0].split("?", 1)[0]) if path.startswith("/"): joined = path.lstrip("/") else: joined = str(PurePosixPath(rel_file).parent / path) norm = os.path.normpath(joined).replace(os.sep, "/") if norm == ".": return "" if norm == ".." or norm.startswith("../"): return None return norm # --- the audit ------------------------------------------------------------------------------------------------- @dataclass class Bundle: root: Path sources: Path docs: dict[str, Document] = field(default_factory=dict) _places: dict[str, list[Any]] | None = None def places(self, rel: str) -> list[Any] | None: """The place in the original of each line of the raw source at rel (K1 `places`, PDF and Office), or None.""" if self._places is None: self._places = {} try: doc = json.loads((self.sources / "source-manifest.json").read_text(encoding="utf-8")) for e in doc.get("sources", []) if isinstance(doc, dict) else []: if isinstance(e, dict) and isinstance(e.get("path"), str) and isinstance(e.get("places"), list): self._places[e["path"]] = e["places"] except (OSError, ValueError): pass return self._places.get(rel) @property def concepts(self) -> list[Document]: return [d for rel, d in sorted(self.docs.items()) if PurePosixPath(rel).name not in RESERVED] def named(self, name: str) -> list[Document]: return [d for rel, d in sorted(self.docs.items()) if PurePosixPath(rel).name == name] def is_file(self, rel: str | None) -> bool: return rel is not None and rel != "" and (self.root / rel).is_file() def load_bundle(root: Path, sources: Path) -> Bundle: b = Bundle(root, sources) for p in sorted(root.rglob("*.md")): if p.is_file(): rel = p.relative_to(root).as_posix() b.docs[rel] = read_document(p, rel) return b def check_spec11_concepts(b: Bundle) -> list[Finding]: """SPEC 11.1 and 11.2 on every non-reserved .md file in the tree.""" out: list[Finding] = [] for d in b.concepts: if not d.has_frontmatter: out.append(finding("spec11-1-frontmatter", d.rel, 1, "no YAML frontmatter block (SPEC 11.1)")) elif d.frontmatter is None: out.append(finding("spec11-1-frontmatter", d.rel, 1, d.frontmatter_error + " (SPEC 11.1)")) else: t = d.frontmatter.get("type") if not isinstance(t, str) or not t.strip(): out.append(finding("spec11-2-type", d.rel, 1, "no non-empty type field (SPEC 11.2)")) return out def body_headings(doc: Document) -> list[tuple[int, int, str]]: """(file line, level, text) for every ATX heading in the body.""" out: list[tuple[int, int, str]] = [] for i, raw in enumerate(doc.body.split("\n")): m = HEADING.match(raw) if m: out.append((doc.body_start + i, len(m.group(1)), (m.group(2) or "").strip())) return out def check_spec11_index(b: Bundle) -> list[Finding]: """SPEC 8 via 11.3: no frontmatter except `okf_version` at the bundle root; entries grouped under headings.""" out: list[Finding] = [] for d in b.named("index.md"): if d.has_frontmatter: at_root = d.rel == "index.md" if d.frontmatter is None: out.append(finding("spec11-3-index", d.rel, 1, d.frontmatter_error + " (SPEC 8)")) elif not at_root or set(d.frontmatter) - {"okf_version"}: out.append(finding("spec11-3-index", d.rel, 1, "an index file carries no frontmatter, except okf_version in the bundle-root index (SPEC 8)")) body_lines = d.body.split("\n") headings = body_headings(d) first_heading = headings[0][0] if headings else None before = [d.body_start + i for i, raw in enumerate(body_lines) if raw.strip() and (first_heading is None or d.body_start + i < first_heading)] if not headings: out.append(finding("spec11-3-index", d.rel, d.body_start, "no section heading; entries are grouped under headings (SPEC 8)")) elif before: out.append(finding("spec11-3-index", d.rel, before[0], "content before the first section heading (SPEC 8)")) return out DATE = re.compile(r"^\d{4}-\d{2}-\d{2}$") def check_spec11_log(b: Bundle) -> list[Finding]: """SPEC 9 via 11.3: date headings in YYYY-MM-DD form, entries under a date, newest first.""" out: list[Finding] = [] for d in b.named("log.md"): dates: list[tuple[int, date]] = [] first_date_line: int | None = None for line, level, text in body_headings(d): if level == 1: continue if not DATE.match(text): out.append(finding("spec11-3-log", d.rel, line, f"date heading {text!r} is not ISO 8601 YYYY-MM-DD (SPEC 9)")) continue try: dates.append((line, date.fromisoformat(text))) except ValueError: out.append(finding("spec11-3-log", d.rel, line, f"date heading {text!r} is not a calendar date (SPEC 9)")) continue if first_date_line is None: first_date_line = line for (_, newer), (line, older) in zip(dates, dates[1:]): if older > newer: out.append(finding("spec11-3-log", d.rel, line, "entries are not newest first (SPEC 9)")) for i, raw in enumerate(d.body.split("\n")): line = d.body_start + i if LIST_ITEM.match(raw) and (first_date_line is None or line < first_date_line): out.append(finding("spec11-3-log", d.rel, line, "an entry that is not under a date heading (SPEC 9)")) break return out def check_links(b: Bundle) -> list[Finding]: """broken-link, orphan-page and index-coverage, as defined in the manifest's provenance.link_rules.""" out: list[Finding] = [] incoming: dict[str, set[str]] = {d.rel: set() for d in b.concepts} for d in b.concepts: for line, target in body_links(d): if not is_bundle_link(target): continue rel = resolve(d.rel, target) if not b.is_file(rel): out.append(finding("broken-link", d.rel, line, f"link target {target!r} is not a file in the bundle")) elif rel in incoming and rel != d.rel: incoming[rel].add(d.rel) for idx in b.named("index.md"): listed: set[str] = set() for line, target in body_links(idx): if not is_bundle_link(target): continue rel = resolve(idx.rel, target) if rel is not None and rel != "" and (b.root / rel).is_dir(): continue if not b.is_file(rel): out.append(finding("index-coverage", idx.rel, line, f"index entry {target!r} is not a file")) continue assert rel is not None listed.add(rel) if rel in incoming: incoming[rel].add(idx.rel) directory = PurePosixPath(idx.rel).parent for d in b.concepts: if PurePosixPath(d.rel).parent == directory and d.rel not in listed: out.append(finding("index-coverage", idx.rel, None, f"{d.rel} is in the directory but not in its index (SPEC 8)")) for rel, sources in sorted(incoming.items()): if not sources: out.append(finding("orphan-page", rel, None, "no other concept and no index.md links to this page (log.md does not count)")) return out def placed_links(doc: Document) -> list[tuple[str, int, str]]: """(place, file line, target) for every inline markdown link in the body; place is prose, code, heading or footnote. Code is what mask_code blanks (inline spans, and blocks fenced with backticks) and every line of a fenced block, backticks or tildes, fences included, also when the fence is indented in a list item or stands behind > in a quote block (K2 point 14). body_links() is the prose and heading/footnote part. """ masked = mask_code(doc.body) # same length as the body, so index by offset out: list[tuple[str, int, str]] = [] offset = 0 fence = "" # the open fence's marker run, or "" outside a fenced block for i, raw in enumerate(doc.body.split("\n")): f = fence_marker(raw) in_fence = bool(fence or f) if fence and closes_fence(f, raw, fence): fence = "" elif not fence and f: fence = f.group(1) place = "heading" if HEADING.match(raw) else "footnote" if FOOTNOTE_DEF.match(raw.strip()) else "prose" for m in link_matches(raw, masked[offset:offset + len(raw)]): at = offset + m.start() in_code = in_fence or masked[at] != doc.body[at] # the link opens inside code; code inside the anchor is fine out.append(("code" if in_code else place, doc.body_start + i, m.group(2))) offset += len(raw) + 1 return out def concept_inbound(b: Bundle) -> tuple[dict[str, set[str]], list[Finding]]: """Criterion 6 (b) and (c): inbound concept links per concept page counted from prose only, and misplaced links.""" incoming: dict[str, set[str]] = {d.rel: set() for d in b.concepts} misplaced: list[Finding] = [] for d in b.concepts: for place, line, target in placed_links(d): if not is_bundle_link(target): continue if place != "prose": misplaced.append(finding("link-misplaced", d.rel, line, f"link to {target!r} sits in {place}; a link belongs in a body sentence (criterion 6 b)")) continue rel = resolve(d.rel, target) if rel in incoming and rel != d.rel: incoming[rel].add(d.rel) return incoming, misplaced def check_link_structure(b: Bundle) -> list[Finding]: """link-misplaced (b) of criterion 6; (a) is broken-link in check_links, and the main-page rule is check_main_pages.""" return concept_inbound(b)[1] def link_rule_report(b: Bundle) -> dict[str, Any]: """Criterion 6 as numbers: N of N concept pages with an inbound link, the breaches of (a) and (b), and counts.""" incoming, misplaced = concept_inbound(b) concepts = set(incoming) passed = sum(1 for s in incoming.values() if s) not_to_concept = 0 out_degree: dict[str, set[str]] = {rel: set() for rel in concepts} provenance: dict[str, int] = {} for d in b.concepts: for place, _, target in placed_links(d): if place != "prose" or not is_bundle_link(target): continue rel = resolve(d.rel, target) if rel in concepts and rel != d.rel: out_degree[d.rel].add(rel) elif rel not in concepts and b.is_file(rel): not_to_concept += 1 srcs = d.frontmatter.get("sources") if d.frontmatter else None for s in srcs if isinstance(srcs, list) else []: value = s.get("x-provenance") if isinstance(s, dict) else None key = value if isinstance(value, str) else "missing" provenance[key] = provenance.get(key, 0) + 1 return { "pass": passed, "of": len(concepts), "line": f"link rule: {passed} of {len(concepts)} concept pages have an inbound link", "misplaced": len(misplaced), "broken": sum(f.check == "broken-link" for f in check_links(b)), "not_to_concept": not_to_concept, "max_out_degree": max((len(v) for v in out_degree.values()), default=0), "provenance": dict(sorted(provenance.items())), } # --- the main-page rule (contract K2, read against K1) ------------------------------------------------------------- K1_FILE = "source-manifest.json" K1_SCHEMA = "okf-wiki/source-manifest/1" MAIN_FOR = "x-main-for" class K1Error(Exception): """The K1 file at the sources root is not valid, so the main-page rule cannot run (K2 point 18).""" def read_k1(sources: Path) -> dict[str, bool] | None: """`main_topic` per `path` from /source-manifest.json (contract K1), read by this module itself and not by the K1 module, which wave 2 changes beside it. None when there is no such file. K1Error when it is not valid: not JSON, `schema` not okf-wiki/source-manifest/1, `sources` not a list, or an entry without `path` (a string) or `main_topic` (a boolean). Unknown keys are ignored (K1 point 10).""" path = sources / K1_FILE if not path.exists(): return None try: data = json.loads(path.read_text(encoding="utf-8")) except (OSError, UnicodeDecodeError, ValueError) as exc: raise K1Error(f"{path} is not readable JSON: {exc}") from exc if not isinstance(data, dict) or data.get("schema") != K1_SCHEMA: raise K1Error(f"{path}: schema is not {K1_SCHEMA}") entries = data.get("sources") if not isinstance(entries, list): raise K1Error(f"{path}: sources is not a list") out: dict[str, bool] = {} for i, e in enumerate(entries): if not (isinstance(e, dict) and isinstance(e.get("path"), str) and isinstance(e.get("main_topic"), bool)): raise K1Error(f"{path}: sources[{i}] has no path (a string) or main_topic (a boolean)") out[e["path"]] = e["main_topic"] return out def main_marks(b: Bundle, k1: dict[str, bool]) -> tuple[dict[str, set[str]], list[Finding]]: """The valid x-main-for values per page, and one main-page-invalid finding per invalid value (K2 points 1, 2, 10-13). A valid value is a string that stands, exactly, among the page's own sources[].resource and is "/" + a path with main_topic true in K1. A value that is not a list, or an empty list, is one finding and gives no role; in a list, each item is judged alone, and a value given twice counts once and gives one finding. A page is a main page when at least one value is valid. A page whose frontmatter does not parse has no role (point 16). """ topics = {"/" + p for p, main in k1.items() if main} valid: dict[str, set[str]] = {} out: list[Finding] = [] for d in b.concepts: fm = d.frontmatter if fm is None or MAIN_FOR not in fm: continue value = fm[MAIN_FOR] if not isinstance(value, list): out.append(finding("main-page-invalid", d.rel, None, f"{MAIN_FOR} {value!r} is not a list (K2 point 1)")) continue if not value: out.append(finding("main-page-invalid", d.rel, None, f"{MAIN_FOR} is an empty list; a page that is no main page leaves the key out (K2 point 11)")) continue srcs = fm.get("sources") own = {s["resource"] for s in (srcs if isinstance(srcs, list) else [srcs]) if isinstance(s, dict) and isinstance(s.get("resource"), str)} seen: set[str] = set() for v in value: if not isinstance(v, str): why = "is not a string (K2 point 10)" elif v in seen: why = "is listed twice (K2 point 12)" elif v not in own: why = "is not among the page's own sources[].resource (K2 points 1 and 13)" elif v not in topics: why = "is not / + a path with main_topic true in K1 (K2 points 1 and 13)" else: why = "" valid.setdefault(d.rel, set()).add(v) if isinstance(v, str): seen.add(v) if why: out.append(finding("main-page-invalid", d.rel, None, f"{MAIN_FOR} value {v!r} {why}")) return valid, out def main_holders(valid: dict[str, set[str]], k1: dict[str, bool]) -> dict[str, list[str]]: """For every K1 path with main_topic true, the pages that list it validly in x-main-for (valid: main_marks).""" return {p: sorted(rel for rel, values in valid.items() if "/" + p in values) for p, main in sorted(k1.items()) if main} def main_page_links(b: Bundle, mains: set[str]) -> dict[str, set[str]]: """For every concept page, the main pages whose body links to it (K2 points 3, 14 and 15). Only inline links with a relative target count, in prose: paragraphs, list items, table cells and quote blocks under any heading, outside code (inline and fenced), headings and footnote definitions. A link with #anchor counts for its file; a link starting with / does not; a link from the page itself does not. Reference links and autolinks are no inline links, so they do not count. """ linked: dict[str, set[str]] = {d.rel: set() for d in b.concepts} for d in b.concepts: if d.rel not in mains: continue for place, _, target in placed_links(d): if place != "prose" or not is_bundle_link(target) or target.startswith("/"): continue rel = resolve(d.rel, target) if rel in linked and rel != d.rel: linked[rel].add(d.rel) return linked def check_main_pages(b: Bundle, k1: dict[str, bool]) -> list[Finding]: """K2 point 5: main-page-missing and main-page-duplicate per main-topic source (the finding's file is its K1 path), main-page-invalid per invalid value, and no-main-page-link per page that is not a main page and has no link from one.""" valid, out = main_marks(b, k1) for path, pages in main_holders(valid, k1).items(): if not pages: out.append(finding("main-page-missing", path, None, f"main-topic source /{path} has no main page: no concept page lists it in {MAIN_FOR} (K2 point 2)")) elif len(pages) > 1: out.append(finding("main-page-duplicate", path, None, f"main-topic source /{path} has {len(pages)} main pages: {', '.join(pages)} (K2 point 2)")) for rel, mains in sorted(main_page_links(b, set(valid)).items()): if rel not in valid and not mains: out.append(finding("no-main-page-link", rel, None, "no main page links to this page from its body, in a sentence or a list of related concepts; " "index.md, log.md and the page itself do not count (K2 point 3)")) return out def main_page_report(b: Bundle, k1: dict[str, bool]) -> dict[str, Any]: """Criterion 6 as numbers (K2 point 7): the main-topic sources with exactly one main page, the pages that are not main pages with a link from one, and the counts of link_rule_report beside them.""" valid, _ = main_marks(b, k1) holders = main_holders(valid, k1) sources = sum(1 for pages in holders.values() if len(pages) == 1) linked = main_page_links(b, set(valid)) others = [rel for rel in linked if rel not in valid] passed = sum(1 for rel in others if linked[rel]) counts = link_rule_report(b) return { "line": f"main pages: {sources} of {len(holders)} sources · linked from a main page: {passed} of {len(others)} pages", "main_pages": {"pass": sources, "of": len(holders)}, "linked": {"pass": passed, "of": len(others)}, **{k: counts[k] for k in ("misplaced", "broken", "not_to_concept", "max_out_degree", "provenance")}, } def as_events(value: Any) -> list[Any]: """`verified` as a list of events; a bare mapping is a one-element list (SPEC 5.2).""" if isinstance(value, dict): return [value] if isinstance(value, list): return value return [value] TIMESTAMP = re.compile(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}(?::\d{2}(?:\.\d+)?)?(?:Z|[+-]\d{2}:\d{2})$") def timestamp_ok(value: Any) -> bool: """ISO 8601 datetime with an explicit UTC offset (SPEC 5), kept as the text the author wrote (format choice c).""" if not isinstance(value, str) or not TIMESTAMP.match(value): return False try: datetime.fromisoformat(value.replace("Z", "+00:00")) except ValueError: return False return True def timestamp_values(fm: dict[str, Any]) -> list[tuple[str, Any]]: """Every OKF timestamp-valued key present: generated.at, verified[].at, stale_after, sources[].last_modified, and the from/to of usage_window, shared or per source (SPEC 5.1, 5.2, 5.5).""" out: list[tuple[str, Any]] = [] def window(prefix: str, w: Any) -> None: if isinstance(w, dict): out.extend((f"{prefix}.{k}", w[k]) for k in ("from", "to") if k in w) gen = fm.get("generated") if isinstance(gen, dict) and "at" in gen: out.append(("generated.at", gen["at"])) if "verified" in fm: out.extend(("verified[].at", e["at"]) for e in as_events(fm["verified"]) if isinstance(e, dict) and "at" in e) if "stale_after" in fm: out.append(("stale_after", fm["stale_after"])) window("usage_window", fm.get("usage_window")) srcs = fm.get("sources") if isinstance(srcs, list): for s in srcs: if isinstance(s, dict): if "last_modified" in s: out.append(("sources[].last_modified", s["last_modified"])) window("sources[].usage_window", s.get("usage_window")) return out def check_format(b: Bundle) -> list[Finding]: """Format choices (a) resource with a leading /, (b) relative body links, (c) timestamps, (e) verified by humans.""" out: list[Finding] = [] for d in b.concepts: for line, target in body_links(d): if is_bundle_link(target) and target.startswith("/"): out.append(finding("body-link-bundle-absolute", d.rel, line, f"body link {target!r} starts with /; body links are relative (format choice b)")) fm = d.frontmatter if fm is None: continue srcs = fm.get("sources") for s in srcs if isinstance(srcs, list) else []: res = s.get("resource") if isinstance(s, dict) else None if isinstance(res, str) and res and not SCHEME.match(res) and not res.startswith("/"): out.append(finding("resource-not-bundle-absolute", d.rel, None, f"sources[].resource {res!r} has no leading / (format choice a)")) if "verified" in fm: for e in as_events(fm["verified"]): by = e.get("by") if isinstance(e, dict) else None if not (isinstance(by, str) and by.startswith("human:") and len(by) > len("human:")): out.append(finding("verified-not-human", d.rel, None, f"verified by {by!r}: only a human: may write verified (format choice e)")) for key, value in timestamp_values(fm): if not timestamp_ok(value): out.append(finding("timestamp-form", d.rel, None, f"{key} {value!r} is not an ISO 8601 datetime with an explicit UTC offset (SPEC 5)")) return out def check_citations(b: Bundle) -> list[Finding]: """Criterion 1 and format choice f: sentence -> footnote -> sources[].id -> verbatim quote at its locator.""" out: list[Finding] = [] source_lines: dict[str, list[str] | None] = {} for d in b.concepts: for block in prose_blocks(d): if LINK_ONLY.match(mask_code(block.text)): continue # a pure link list is not a claim for s in sentences(block.text): if not FOOTNOTE_REF.search(s): out.append(finding("uncited-sentence", d.rel, block.line, f"sentence without a footnote: {s[:80]!r}")) fm = d.frontmatter if fm is None: continue # without parseable frontmatter there are no sources to join to (spec11-1 reports it) srcs = fm.get("sources", []) entries = srcs if isinstance(srcs, list) else [srcs] ids: list[str] = [] for s in entries: sid = s.get("id") if isinstance(s, dict) else None if not isinstance(sid, str) or not sid.strip(): out.append(finding("source-missing-id", d.rel, None, "a sources entry has no id (format choice f)")) else: ids.append(sid) for sid in sorted({i for i in ids if ids.count(i) > 1}): out.append(finding("source-duplicate-id", d.rel, None, f"sources[].id {sid!r} is used more than once (format choice f)")) known = set(ids) labels: dict[str, int] = {} for i, raw in masked_lines(d.body): m = FOOTNOTE_DEF.match(raw.strip()) found = [m.group(1)] if m else FOOTNOTE_REF.findall(raw) for label in found: labels.setdefault(label, d.body_start + i) for label, line in sorted(labels.items(), key=lambda kv: kv[1]): if label not in known: out.append(finding("footnote-unknown-id", d.rel, line, f"footnote [^{label}] names no sources[].id")) for s in entries: if isinstance(s, dict): out.extend(check_quote(b, d, s, source_lines)) return out PLACE_KEYS = frozenset({"page", "paragraph", "sheet", "cell", "slide", "notes"}) def check_quote(b: Bundle, d: Document, s: dict[str, Any], cache: dict[str, list[str] | None]) -> list[Finding]: sid = s.get("id", "?") quote, locator, res = s.get("x-quote"), s.get("x-locator"), s.get("resource") if not isinstance(quote, str) or not quote: return [finding("quote-missing", d.rel, None, f"source {sid!r} has no x-quote")] if not (isinstance(locator, dict) and "line" in locator and set(locator) - {"line"} <= PLACE_KEYS and type(locator["line"]) is int and locator["line"] > 0): return [finding("locator-unsupported", d.rel, None, f"source {sid!r}: x-locator {locator!r} is not {{ line: N }}" " with at most a page, paragraph, sheet and cell, or slide and notes")] if not isinstance(res, str) or not res or SCHEME.match(res): return [finding("source-unreadable", d.rel, None, f"source {sid!r}: resource {res!r} is not a raw-source path")] rel = resolve(d.rel, res) if not rel: return [finding("source-unreadable", d.rel, None, f"source {sid!r}: resource {res!r} leaves the raw-source root")] if rel not in cache: try: cache[rel] = (b.sources / rel).read_text(encoding="utf-8").splitlines() except (OSError, UnicodeDecodeError): cache[rel] = None lines = cache[rel] if lines is None: return [finding("source-unreadable", d.rel, None, f"source {sid!r}: cannot read {res!r} as UTF-8 text")] n = locator["line"] if n <= len(lines) and quote in lines[n - 1]: place = {k: v for k, v in locator.items() if k != "line"} places = b.places(rel) frozen = places[n - 1] if places is not None and n <= len(places) else None if not place_fits(place, frozen): return [finding("quote-wrong-place", d.rel, None, f"source {sid!r}: line {n} of {res!r} is at " f"{frozen or 'no place'} in the original, not {place}")] return [] if any(quote in line for line in lines): return [finding("quote-wrong-locator", d.rel, None, f"source {sid!r}: the quote is in {res!r}, but not on line {n}")] return [finding("quote-not-verbatim", d.rel, None, f"source {sid!r}: the quote is not verbatim anywhere in {res!r}")] def audit(bundle: Path, sources: Path) -> list[Finding]: """Every finding on the bundle; the main-page rule only when SOURCES holds a K1 file (K1Error when it is invalid).""" k1 = read_k1(sources) return check_bundle(load_bundle(bundle, sources), k1) def check_bundle(b: Bundle, k1: dict[str, bool] | None) -> list[Finding]: found = (check_spec11_concepts(b) + check_spec11_index(b) + check_spec11_log(b) + check_links(b) + check_link_structure(b) + check_citations(b) + check_format(b)) if k1 is not None: found += check_main_pages(b, k1) return sorted(found, key=lambda f: (GROUPS.index(f.group), f.file, f.line or 0, f.check, f.message)) def main(argv: list[str] | None = None) -> int: args = sys.argv[1:] if argv is None else argv if len(args) != 2: print("usage: python3 -m okfwiki.audit BUNDLE SOURCES", file=sys.stderr) return 2 bundle, sources = Path(args[0]), Path(args[1]) for p, what in ((bundle, "bundle"), (sources, "raw-source root")): if not p.is_dir(): print(f"audit cannot run: {what} {p} is not a directory", file=sys.stderr) return 2 try: k1 = read_k1(sources) except K1Error as exc: print(f"audit cannot run: {exc}; the main-page rule needs a valid K1 file (K2 points 6 and 18)", file=sys.stderr) return 2 if k1 is None: print(f"audit cannot run: no {K1_FILE} at the raw-source root {sources}; the main-page rule needs K1 (K2 point 6)", file=sys.stderr) return 2 b = load_bundle(bundle, sources) if not any(k1.values()) and not b.concepts: print(f"audit cannot run: {K1_FILE} lists no main-topic source and the bundle has no concept page, so the " "main-page rule has no denominator", file=sys.stderr) return 2 found = check_bundle(b, k1) summary = {g: sum(f.group == g for f in found) for g in GROUPS} print(json.dumps({"findings": [asdict(f) for f in found], "summary": summary, "link_rule": main_page_report(b, k1)}, ensure_ascii=False, indent=1)) return 1 if found else 0 if __name__ == "__main__": sys.exit(main())