okf-wiki/okfwiki/kassemble.py
Kjell Tore Guttormsen 0fd86a23a3
okf-wiki 1.1.1
First public release, published as one snapshot. What changed between
versions is in CHANGELOG.md.

Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
2026-11-14 21:03:20 +01:00

573 lines
29 KiB
Python

"""K's assembly: concept pages from accepted fragments, by code. No model call (CLAUDE.md rule 1).
Usage:
python3 -m okfwiki.kassemble build --work DIR --sources DIR --out DIR [--links FILE] [--replace] [--at ISO]
One page per concept that at least one accepted fragment (WORK/K-work/accepted.json) enriches; a concept without a
fragment gets none. Docs fragments form the body in the order of their sources (pack, position, then line); the
fragments of a history source go under "## History": release notes in version order, then the pages K1 marks as
history (main_topic false, the weekly What's New pages) in week order. Every sentence ends in a footnote to its
fragment's source entry and, for a history source, to its label entry: for a release note the entry whose quote is the
bare version on the note's heading line, for a weekly page the entry titled with the week from the file name
(YYYY-wNN), whose quote is the page's first heading. A fragment's source entry quotes the whole line its quote
stands on, trimmed (`x-quote`), so the reader sees all the line says, and each context quote kpack placed (a
heading above it or another line of its section) is a source entry of its own, cited after it. Each fragment's
provenance mark is `x-provenance` on its source entries. SOURCES' K1 manifest (source-manifest.json) says which source is history; without it, release-* files are. A footnote label is keyed by its fragment (file,
span, quote line, quote, provenance), never by its place on the page, so one source's fragments can be replaced
without renumbering the others' (SPEC 5.1: "a stable `id` survives reordering").
A page's description is the first sentence of its first fragment, of the fragments that are its own when kmerge has
moved others in (a fragment marked `merged_from`), so the description follows the page's name. The catalogue's other
names for the concept (the aliases of its entry that are not the title, once, case set aside) stand on the page as
`x-aliases`, left out when there is none, and end its index line as " (also: A; B)".
Main pages and homes (accepted.json main_pages and homes, written by kpack from the compiler's and the guard's
answers): a page named as the main page of a docs source it cites carries `x-main-for`, the sorted list of those
sources as "/" + their K1 path, and the key is left out on every other page (contract K2). A main page lists the
pages whose home it is under "## Related concepts", one inline relative link per item, before the footnote
definitions; a home on a page that is no main page, or on itself, places nothing.
Code makes no other link: every inline link in a fragment's text, to a page or to the web, is reduced to its text, and
in-text links come only from --links, a links file of the link pass, placed by klink.place. A link, in-text or
related, whose target page no longer exists is dropped, and the pages that held one are listed; a record of the
--links file into or on a page that is gone is taken out of that file once the build is in (its accepted count
follows), so no later step places it again and it never counts as stale. `generated.at` is the content's last meaningful change (SPEC 5.2): a page
whose content is the same as in the existing output keeps its old `generated` and its bytes; any other page gets --at
(default now), written in UTC. index.md lists every page; log.md keeps every old line and gains one entry, newest
first (format choice d). Format choices (a)-(h) of PLAN.md section Rammer: sources[].resource starts with /, body links
are relative, no `verified` is written, quote and locator are x- fields.
The output is written to a staging folder beside --out and swapped in; the old --out is set aside beside it until the
new one is in, and comes back if the swap fails. An existing --out is refused unless --replace, and refused even then
when it holds anything this command does not write: a file other than index.md, log.md and pages whose generated.by is
this generator (a file browser's .DS_Store is tolerated and dropped). A marker left in WORK/K-work by an unfinished
kpack write stops the build. Exit 0 = built, 1 = built but some link records are stale (their sentence no longer
stands), 2 = could not run (also for an input the code did not foresee, with its traceback).
"""
from __future__ import annotations
import argparse
import hashlib
import json
import re
import shutil
import sys
import tempfile
import traceback
from collections.abc import Mapping
from dataclasses import dataclass, field
from datetime import datetime, timezone
from pathlib import Path, PurePosixPath
from typing import Any
import yaml
from okfwiki import audit, kblocks, klink
from okfwiki.common import RunError, place_at, place_words
from okfwiki.kblocks import headings_of, kilde_of, read_lines, version_key
from okfwiki.kpack import RESERVED_IDS, Fact, check_pending, read_index, read_json, source_facts, week_of
GENERATOR = "okf-wiki/kassemble"
LABEL_HEX = 10
SAFE_ID = re.compile(r"[a-z0-9][a-z0-9-]*")
RECORD_KEYS = {"pack": int, "position": int, "file": str, "start": int, "end": int, "section": str, "text": str,
"quote": str, "quote_line": int, "provenance": str, "concept_id": str}
LINK_KEYS = {"page": str, "sentence": str, "anchor": str, "at": int, "target": str}
MAIN_KEYS = {"source": str, "concept_id": str}
HOME_KEYS = {"source": str, "concept_id": str, "home": str}
RELATED = "## Related concepts"
TOLERATED = frozenset({".DS_Store"}) # folder metadata a file browser writes; dropped by the swap
@dataclass
class Page:
name: str
fm: dict[str, Any] # every key but `generated`
body: str # everything after the frontmatter block
@dataclass
class Report:
pages: int = 0
fragments: int = 0
new: list[str] = field(default_factory=list)
changed: list[str] = field(default_factory=list)
unchanged: list[str] = field(default_factory=list)
gone: list[str] = field(default_factory=list)
placed: int = 0
stale: int = 0
main_pages: int = 0
related: int = 0
dropped: dict[str, set[str]] = field(default_factory=dict) # page -> targets that are gone
unlinked: int = 0 # link records into a page that is gone, taken out of the links file
def frontmatter(data: dict[str, Any]) -> str:
"""A frontmatter block; PyYAML quotes any string that would load as another type (no, 1.10)."""
return "---\n" + yaml.safe_dump(data, sort_keys=False, allow_unicode=True, width=10_000) + "---\n"
def head(fm: dict[str, Any], at: str) -> str:
return frontmatter({**fm, "generated": {"by": GENERATOR, "at": at}})
def clean(text: str) -> str:
"""A fragment's text as body prose: no link (bundle or external) and no footnote mark of its own, on one line."""
text = audit.FOOTNOTE_REF.sub("", audit.sub_links(lambda m: m.group(1), text))
return re.sub(r"\s*\n\s*", " ", text).strip()
def footnoted(text: str, labels: list[str]) -> str:
if text and text[-1] not in ".!?":
text += "."
marks = "".join(f"[^{x}]" for x in labels)
out, last = [], 0
for _a, b in audit.sentence_spans(text):
out.append(text[last:b] + marks)
last = b
out.append(text[last:])
return "".join(out)
def label(r: dict[str, Any]) -> str:
key = "\0".join([r["file"], str(r["start"]), str(r["end"]), str(r["quote_line"]), r["quote"], r["provenance"]])
return "s" + hashlib.sha256(key.encode("utf-8")).hexdigest()[:LABEL_HEX]
def context_label(file: str, line: int, provenance: str) -> str:
"""A context entry's label, keyed by its line and mark: fragments citing the same line share one entry."""
key = "\0".join(["context", file, str(line), provenance])
return "c" + hashlib.sha256(key.encode("utf-8")).hexdigest()[:LABEL_HEX]
def version_label(version: str) -> str:
return "v" + version.replace(".", "-")
def version_line(lines: list[str], version: str, file: str) -> int:
"""The line that states the version: its heading, or the label of the component it is (kblocks.stated_versions)."""
n = kblocks.version_line(lines, version)
if n is None:
raise RunError(f"{file} has no heading or label for version {version}")
return n
def shown(line: str) -> str:
"""What a source entry quotes from a line: the whole line, trimmed, or a heading's text (order 22: a quote that is
a piece of its line hid from the reader what the claim drew on)."""
m = audit.HEADING.match(line)
return (m.group(2) or "").strip() if m else line.strip()
def one_line(text: str) -> str:
return " ".join(str(text).split())
def history_entry(file: str, lines: list[str], resource: str) -> dict[str, Any]:
"""The label entry of a history page without a version: titled with the week from its file name (or, failing
that, its first heading), quoting its first heading on that line."""
heads = headings_of(lines)
if not heads:
raise RunError(f"{file} is history but has no heading to quote for its label")
line, _level, text = heads[0]
week = week_of(file)
return {"id": "w" + week if week else "h-" + re.sub(r"[^a-z0-9]+", "-", file[:-3].lower()).strip("-"),
"resource": resource, "title": week or text, "x-locator": {"line": line}, "x-quote": text,
"x-provenance": "documented"}
def history_key(r: dict[str, Any]) -> tuple[Any, ...]:
if r.get("version"):
return 0, version_key(f"release-v{r['version']}.md")
week = week_of(str(r["file"]))
return (1, week) if week else (2, str(r["file"]))
def check_roles(raw: Any, key: str, keys: Mapping[str, type]) -> list[dict[str, Any]]:
"""accepted.json's main_pages or homes: absent is empty; a record without its keys cannot run."""
value = raw.get(key, []) if isinstance(raw, dict) else []
if not isinstance(value, list) or not all(isinstance(r, dict) and all(type(r.get(k)) is t for k, t in keys.items())
for r in value):
raise RunError(f"accepted.json: {key} is not a list of records with {', '.join(keys)}")
return value
def check_records(records: Any) -> list[dict[str, Any]]:
if not isinstance(records, list):
raise RunError("accepted.json has no fragments list")
for i, r in enumerate(records):
bad = [k for k, t in RECORD_KEYS.items() if not isinstance(r, dict) or type(r.get(k)) is not t]
if bad or not r["quote"].strip() or not (r.get("version") is None or isinstance(r["version"], str)):
raise RunError(f"accepted record {i} lacks {bad or ['a quote or a version string']}; every fragment "
f"carries a quote and a provenance mark (rule E)")
context = r.get("context", [])
if not (isinstance(context, list) and all(
isinstance(c, dict) and type(c.get("line")) is int and isinstance(c.get("quote"), str)
and c["quote"].strip() and isinstance(c.get("section"), str) for c in context)):
raise RunError(f"accepted record {i}: context is not a list of records with line, quote and section")
return records
def page_models(index: dict[str, Any], records: list[dict[str, Any]], kilde: Path,
facts: dict[str, Fact] | None = None, main_pages: list[dict[str, Any]] | None = None) -> dict[str, Page]:
facts = facts or {}
entries = {str(e["id"]): e for e in index["entries"]}
by_concept: dict[str, list[dict[str, Any]]] = {}
for r in records:
by_concept.setdefault(r["concept_id"], []).append(r)
cache: dict[str, list[str]] = {}
pages: dict[str, Page] = {}
for cid, items in sorted(by_concept.items()):
if cid not in entries:
raise RunError(f"concept {cid!r} has accepted fragments but no index entry")
if cid in RESERVED_IDS or not SAFE_ID.fullmatch(cid):
raise RunError(f"concept id {cid!r} cannot be a page name")
e = entries[cid]
if not (isinstance(e.get("title"), str) and e["title"].strip()):
raise RunError(f"the index entry of {cid!r} has no title")
title = one_line(e["title"])
items = sorted(items, key=lambda r: (r["pack"], r["position"], r["start"], r["end"]))
is_history = [bool(r.get("version")) or (r["file"] in facts and facts[r["file"]].history) for r in items]
body_items = [r for r, h in zip(items, is_history) if not h]
history = sorted((r for r, h in zip(items, is_history) if h), key=history_key)
sources: dict[str, dict[str, Any]] = {}
def add(entry: dict[str, Any]) -> str:
known = sources.setdefault(entry["id"], entry)
if known != entry:
raise RunError(f"two source entries on {cid}.md share the label {entry['id']}")
return str(entry["id"])
def source_lines(file: str) -> list[str]:
if file not in cache:
cache[file] = read_lines(kilde / file)
return cache[file]
def locator(file: str, line: int, text: str, quote: str) -> dict[str, Any]:
"""The line in the frozen copy, and for a PDF or Office source the place in the original (K1 places): in a
spreadsheet row, the cell or cells the quote covers (common.place_at); "" for a quote names the whole row."""
return {"line": line, **(place_at(facts[file].place(line), text, quote) if file in facts else {})}
def footnote(s: dict[str, Any]) -> str:
file = PurePosixPath(s["resource"]).name
original = facts[file].original if file in facts else None
return f"{PurePosixPath(original).name if original else file}, {place_words(s['x-locator'])}"
def cite(r: dict[str, Any]) -> list[str]:
src: dict[str, Any] = {"id": label(r), "resource": f"/kilde/{r['file']}"}
if r["section"]:
src["title"] = r["section"]
lines = source_lines(r["file"])
line = lines[r["quote_line"] - 1] if 0 < r["quote_line"] <= len(lines) else ""
quote = shown(line) if r["quote"] in line else r["quote"]
src.update({"x-locator": locator(r["file"], r["quote_line"], line, r["quote"]), "x-quote": quote, "x-span": [r["start"], r["end"]],
"x-provenance": r["provenance"]})
labels = [add(src)]
for c in r.get("context", []):
at = lines[c["line"] - 1] if 0 < c["line"] <= len(lines) else ""
ctx: dict[str, Any] = {"id": context_label(r["file"], c["line"], r["provenance"]),
"resource": f"/kilde/{r['file']}"}
if c["section"]:
ctx["title"] = c["section"]
ctx.update({"x-locator": locator(r["file"], c["line"], at, ""), "x-quote": shown(at) if c["quote"] in at else c["quote"],
"x-provenance": r["provenance"]})
labels.append(add(ctx))
if r.get("version"):
v = str(r["version"])
labels.append(add({"id": version_label(v), "resource": f"/kilde/{r['file']}", "title": v,
"x-locator": {"line": version_line(lines, v, r["file"])},
"x-quote": v, "x-provenance": "documented"}))
elif r["file"] in facts and facts[r["file"]].history:
labels.append(add(history_entry(r["file"], lines, f"/kilde/{r['file']}")))
return labels
lines = [f"# {title}", ""]
for r in body_items:
lines += [footnoted(clean(r["text"]), cite(r)), ""]
if history:
lines += ["## History", ""]
lines += [f"- {footnoted(clean(r['text']), cite(r))}" for r in history]
lines.append("")
lines += [f"[^{s['id']}]: {footnote(s)}" for s in sources.values()]
own = [r for r in [*body_items, *history] if not r.get("merged_from")] # the description follows the name
first = clean((own or body_items or history)[0]["text"])
spans = audit.sentence_spans(first)
fm: dict[str, Any] = {"type": "Concept", "title": title,
"description": first[spans[0][0]:spans[0][1]] if spans else title,
"sources": list(sources.values())}
names = other_names(title, e.get("aliases"))
if names:
fm["x-aliases"] = names
if e.get("kind") == "merge-candidate" and e.get("merge_of"):
fm["x-merge-candidate"] = str(e["merge_of"])
cited = {r["file"] for r in items}
mains = sorted({f"/kilde/{m['source']}" for m in main_pages or [] if m["concept_id"] == cid
and m["source"] in cited and m["source"] in facts and facts[m["source"]].main_topic})
if mains:
fm["x-main-for"] = mains
pages[f"{cid}.md"] = Page(f"{cid}.md", fm, "\n" + "\n".join(lines) + "\n")
return pages
def other_names(title: str, aliases: Any) -> list[str]:
"""The catalogue's other names for a concept, in its order: each alias that is not the title, once, with case set
aside."""
seen = {title.casefold()}
out: list[str] = []
for a in aliases if isinstance(aliases, list) else []:
name = one_line(str(a))
if name and name.casefold() not in seen:
seen.add(name.casefold())
out.append(name)
return out
def link_text(title: str) -> str:
return str(title).replace("[", "(").replace("]", ")")
def related(pages: dict[str, Page], homes: list[dict[str, Any]]) -> dict[str, list[str]]:
"""Per main page, the list items of the pages whose home it is: the home links the model chose, nothing else."""
out: dict[str, list[str]] = {}
for name, page in pages.items():
if not page.fm.get("x-main-for"):
continue
cid = name[:-3]
held = {str(h["concept_id"]) for h in homes if h["home"] == cid and h["concept_id"] != cid
and f"{h['concept_id']}.md" in pages}
items = sorted(held, key=lambda c: (str(pages[f"{c}.md"].fm["title"]).casefold(), c))
if items:
out[name] = [f"- [{link_text(pages[f'{c}.md'].fm['title'])}]({c}.md)" for c in items]
return out
def with_related(body: str, items: list[str]) -> str:
"""The body with the related-concepts list placed before its footnote definitions."""
if not items:
return body
lines = body.split("\n")
at = next((i for i, line in enumerate(lines) if audit.FOOTNOTE_DEF.match(line)), len(lines))
return "\n".join(lines[:at] + [RELATED, "", *items, ""] + lines[at:])
def gone_targets(old: Path, names: set[str], gone: set[str]) -> set[str]:
"""The pages gone since the last build that the old page `old` linked to."""
doc = audit.read_document(old, old.name)
targets = {PurePosixPath(t.split("#", 1)[0]).name for _, t in audit.body_links(doc) if audit.is_bundle_link(t)}
return targets & gone - names
def index_text(pages: dict[str, Page]) -> str:
"""The bundle-root index: one entry per page with its frontmatter description (SPEC 8), and after it the page's
other names (`x-aliases`) as " (also: A; B)", so a search of the index finds a concept by any of its names."""
out = ["---", 'okf_version: "0.2"', "---", "", "# Concepts", ""]
out += [f"- [{link_text(p.fm['title'])}]({name}) - {link_text(p.fm['description'])}"
+ (f" (also: {'; '.join(link_text(a) for a in p.fm['x-aliases'])})" if p.fm.get("x-aliases") else "")
for name, p in sorted(pages.items())]
return "\n".join(out) + "\n"
def log_text(old: str | None, at: str, entry: str) -> str:
"""log.md with one new entry, newest first under its date (SPEC 9); every old line stays as it was."""
day = at[:10]
if old is None:
return f"# Log\n\n## {day}\n{entry}\n"
lines = old.split("\n")
for i, raw in enumerate(lines):
m = audit.HEADING.match(raw)
text = (m.group(2) or "").strip() if m else ""
if m and len(m.group(1)) >= 2 and audit.DATE.match(text):
if text > day:
raise RunError(f"the build time {at} is older than the log's newest date {text}")
if text == day:
lines.insert(i + 1, entry)
else:
lines[i:i] = [f"## {day}", entry, ""]
return "\n".join(lines)
return old.rstrip("\n") + f"\n\n## {day}\n{entry}\n"
def generated_at(path: Path) -> str | None:
fm = audit.read_document(path, path.name).frontmatter or {}
gen = fm.get("generated")
at = gen.get("at") if isinstance(gen, dict) else None
return at if isinstance(at, str) else None
def written_here(path: Path) -> bool:
"""A concept page this command wrote: its generated.by names the generator."""
fm = audit.read_document(path, path.name).frontmatter or {}
gen = fm.get("generated")
return isinstance(gen, dict) and gen.get("by") == GENERATOR
def aside(out: Path) -> Path:
return out.with_name(f".{out.name}-old")
def check_out(out: Path, replace: bool) -> None:
if aside(out).exists():
raise RunError(f"{aside(out)} exists: an earlier swap did not finish; look at it before building again")
if not out.exists():
return
if not replace:
raise RunError(f"{out} exists; refusing to overwrite without --replace")
if not out.is_dir():
raise RunError(f"{out} is not a directory")
foreign = sorted(p.name for p in out.iterdir() if p.name not in TOLERATED and not (
p.is_file() and p.suffix == ".md" and (p.name in audit.RESERVED or written_here(p))))
if foreign or not (out / "log.md").is_file():
raise RunError(f"{out} holds what this command does not write ({foreign or 'no log.md'}); refusing to replace")
def check_links(path: Path) -> dict[str, Any]:
doc: dict[str, Any] = read_json(path)
records = doc.get("links") if isinstance(doc, dict) else None
if not isinstance(records, list) or not all(
isinstance(r, dict) and all(type(r.get(k)) is t for k, t in LINK_KEYS.items()) for r in records):
raise RunError(f"{path} is not a links file: a links list of records with {', '.join(LINK_KEYS)}")
return doc
def swap_in(new: Path, out: Path) -> None:
"""Put new in out's place. An old out is set aside beside it first and comes back if the swap fails."""
old = aside(out) if out.exists() else None
if old is not None:
out.rename(old)
try:
new.rename(out)
except BaseException:
if old is not None:
old.rename(out)
raise
if old is not None:
shutil.rmtree(old)
def utc(at: str) -> str:
if not audit.timestamp_ok(at):
raise RunError(f"--at {at!r} is not an ISO 8601 datetime with an explicit UTC offset (SPEC 5)")
return datetime.fromisoformat(at.replace("Z", "+00:00")).astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
def build(work: Path, sources: Path, out: Path, links: Path | None = None, replace: bool = False,
at: str | None = None) -> Report:
check_pending(work)
at = utc(at or datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"))
kilde = kilde_of(sources)
accepted = work / "K-work" / "accepted.json"
if not accepted.is_file():
raise RunError(f"{accepted} does not exist; accept a pack first")
raw = read_json(accepted)
records = check_records(raw.get("fragments") if isinstance(raw, dict) else None)
main_pages, homes = check_roles(raw, "main_pages", MAIN_KEYS), check_roles(raw, "homes", HOME_KEYS)
facts = source_facts(sources, sorted({str(r["file"]) for r in records} | {str(m["source"]) for m in main_pages}))
pages = page_models(read_index(work), records, kilde, facts, main_pages)
lists = related(pages, homes)
check_out(out, replace)
link_doc = check_links(links) if links is not None else None
report = Report(pages=len(pages), fragments=len(records), main_pages=sum(bool(p.fm.get("x-main-for"))
for p in pages.values()),
related=sum(len(v) for v in lists.values()))
old_pages = {p.name for p in out.glob("*.md") if klink.is_concept(p.name)} if out.is_dir() else set()
gone = old_pages - set(pages)
out.parent.mkdir(parents=True, exist_ok=True)
stage = Path(tempfile.mkdtemp(prefix=f".{out.name}-", dir=out.parent))
try:
assembled = stage / "pages"
assembled.mkdir()
for name, page in pages.items():
(assembled / name).write_text(head(page.fm, at) + page.body, encoding="utf-8")
if link_doc is not None:
records = [{**r, "target": klink.page_rel(r["target"])} for r in link_doc["links"]]
live = [r for r in records if r["target"] in pages and r["page"] in pages]
for r in records:
if r["target"] not in pages and r["page"] in pages:
report.dropped.setdefault(r["page"], set()).add(r["target"])
report.unlinked = len(records) - len(live) # into or on a page that is gone
(stage / "links.json").write_text(json.dumps({**link_doc, "links": live}), encoding="utf-8")
report.placed, report.stale = klink.place(assembled, stage / "links.json", stage / "linked", replace=False)
assembled = stage / "linked"
final = stage / "final"
final.mkdir()
for name, page in pages.items():
_, body = klink.split_page((assembled / name).read_text(encoding="utf-8"))
body = with_related(body, lists.get(name, []))
old = out / name
old_text = old.read_text(encoding="utf-8") if old.is_file() else None
if old_text is not None:
lost = gone_targets(old, set(pages), gone)
if lost:
report.dropped.setdefault(name, set()).update(lost)
old_at = generated_at(old) if old_text is not None else None
if old_at is not None and head(page.fm, old_at) + body == old_text:
text = old_text
report.unchanged.append(name)
else:
text = head(page.fm, at) + body
(report.changed if old_text is not None else report.new).append(name)
(final / name).write_text(text, encoding="utf-8")
if out.exists():
report.gone = sorted(p.name for p in out.glob("*.md") if klink.is_concept(p.name) and p.name not in pages)
(final / "index.md").write_text(index_text(pages), encoding="utf-8")
dropped = sum(len(t) for t in report.dropped.values())
entry = (f"* **Build**: {GENERATOR}, {report.pages} pages from {report.fragments} accepted fragments: "
f"new {len(report.new)}, changed {len(report.changed)}, unchanged {len(report.unchanged)}, "
f"gone {len(report.gone)}" + (f"; main pages {report.main_pages}, related links {report.related}"
if report.main_pages else "")
+ (f"; links placed {report.placed}, stale {report.stale}" if links is not None else "")
+ (f"; dropped links to pages that are gone {dropped}" if dropped else "") + ".")
old_log = (out / "log.md").read_text(encoding="utf-8") if out.exists() else None
(final / "log.md").write_text(log_text(old_log, at, entry), encoding="utf-8")
swap_in(final, out)
if links is not None and link_doc is not None and report.unlinked:
kept = [r for r in link_doc["links"] if klink.page_rel(r["target"]) in pages and r["page"] in pages]
tmp = links.with_name(f".{links.name}.tmp")
tmp.write_text(klink.dump({**link_doc, "links": kept, "accepted": len(kept)}), encoding="utf-8")
tmp.replace(links) # after the swap, so a build that fails keeps every record
finally:
shutil.rmtree(stage, ignore_errors=True)
return report
def build_parser() -> argparse.ArgumentParser:
p = argparse.ArgumentParser(prog="python3 -m okfwiki.kassemble", description=__doc__,
formatter_class=argparse.RawDescriptionHelpFormatter)
sub = p.add_subparsers(dest="command", required=True)
b = sub.add_parser("build")
b.add_argument("--work", required=True)
b.add_argument("--sources", required=True)
b.add_argument("--out", required=True)
b.add_argument("--links")
b.add_argument("--replace", action="store_true")
b.add_argument("--at")
return p
def main(argv: list[str] | None = None) -> int:
args = build_parser().parse_args(argv)
try:
r = build(Path(args.work), Path(args.sources), Path(args.out), Path(args.links) if args.links else None,
args.replace, args.at)
except (RunError, OSError, UnicodeDecodeError) as exc:
print(f"kassemble cannot run: {exc}", file=sys.stderr)
return 2
except Exception: # an input this code did not foresee: "could not run", never exit 1 (stale links)
traceback.print_exc()
return 2
print(f"kassemble: {r.pages} pages from {r.fragments} accepted fragments -> {args.out}")
print(f" new {len(r.new)}, changed {len(r.changed)}, unchanged {len(r.unchanged)}, gone {len(r.gone)}")
print(f" main pages {r.main_pages}, related links {r.related}")
if args.links:
print(f" links placed {r.placed}, stale {r.stale}")
if r.dropped:
print(f" dropped links to pages that are gone: {sum(len(t) for t in r.dropped.values())} "
f"({', '.join(sorted(r.dropped))})")
if r.unlinked:
print(f" removed from the links file: {r.unlinked} records into or on pages that are gone")
return 1 if r.stale else 0
if __name__ == "__main__":
sys.exit(main())