First public release, published as one snapshot. What changed between versions is in CHANGELOG.md. Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
573 lines
29 KiB
Python
573 lines
29 KiB
Python
"""K's assembly: concept pages from accepted fragments, by code. No model call (CLAUDE.md rule 1).
|
|
|
|
Usage:
|
|
python3 -m okfwiki.kassemble build --work DIR --sources DIR --out DIR [--links FILE] [--replace] [--at ISO]
|
|
|
|
One page per concept that at least one accepted fragment (WORK/K-work/accepted.json) enriches; a concept without a
|
|
fragment gets none. Docs fragments form the body in the order of their sources (pack, position, then line); the
|
|
fragments of a history source go under "## History": release notes in version order, then the pages K1 marks as
|
|
history (main_topic false, the weekly What's New pages) in week order. Every sentence ends in a footnote to its
|
|
fragment's source entry and, for a history source, to its label entry: for a release note the entry whose quote is the
|
|
bare version on the note's heading line, for a weekly page the entry titled with the week from the file name
|
|
(YYYY-wNN), whose quote is the page's first heading. A fragment's source entry quotes the whole line its quote
|
|
stands on, trimmed (`x-quote`), so the reader sees all the line says, and each context quote kpack placed (a
|
|
heading above it or another line of its section) is a source entry of its own, cited after it. Each fragment's
|
|
provenance mark is `x-provenance` on its source entries. SOURCES' K1 manifest (source-manifest.json) says which source is history; without it, release-* files are. A footnote label is keyed by its fragment (file,
|
|
span, quote line, quote, provenance), never by its place on the page, so one source's fragments can be replaced
|
|
without renumbering the others' (SPEC 5.1: "a stable `id` survives reordering").
|
|
|
|
A page's description is the first sentence of its first fragment, of the fragments that are its own when kmerge has
|
|
moved others in (a fragment marked `merged_from`), so the description follows the page's name. The catalogue's other
|
|
names for the concept (the aliases of its entry that are not the title, once, case set aside) stand on the page as
|
|
`x-aliases`, left out when there is none, and end its index line as " (also: A; B)".
|
|
|
|
Main pages and homes (accepted.json main_pages and homes, written by kpack from the compiler's and the guard's
|
|
answers): a page named as the main page of a docs source it cites carries `x-main-for`, the sorted list of those
|
|
sources as "/" + their K1 path, and the key is left out on every other page (contract K2). A main page lists the
|
|
pages whose home it is under "## Related concepts", one inline relative link per item, before the footnote
|
|
definitions; a home on a page that is no main page, or on itself, places nothing.
|
|
|
|
Code makes no other link: every inline link in a fragment's text, to a page or to the web, is reduced to its text, and
|
|
in-text links come only from --links, a links file of the link pass, placed by klink.place. A link, in-text or
|
|
related, whose target page no longer exists is dropped, and the pages that held one are listed; a record of the
|
|
--links file into or on a page that is gone is taken out of that file once the build is in (its accepted count
|
|
follows), so no later step places it again and it never counts as stale. `generated.at` is the content's last meaningful change (SPEC 5.2): a page
|
|
whose content is the same as in the existing output keeps its old `generated` and its bytes; any other page gets --at
|
|
(default now), written in UTC. index.md lists every page; log.md keeps every old line and gains one entry, newest
|
|
first (format choice d). Format choices (a)-(h) of PLAN.md section Rammer: sources[].resource starts with /, body links
|
|
are relative, no `verified` is written, quote and locator are x- fields.
|
|
|
|
The output is written to a staging folder beside --out and swapped in; the old --out is set aside beside it until the
|
|
new one is in, and comes back if the swap fails. An existing --out is refused unless --replace, and refused even then
|
|
when it holds anything this command does not write: a file other than index.md, log.md and pages whose generated.by is
|
|
this generator (a file browser's .DS_Store is tolerated and dropped). A marker left in WORK/K-work by an unfinished
|
|
kpack write stops the build. Exit 0 = built, 1 = built but some link records are stale (their sentence no longer
|
|
stands), 2 = could not run (also for an input the code did not foresee, with its traceback).
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
import re
|
|
import shutil
|
|
import sys
|
|
import tempfile
|
|
import traceback
|
|
from collections.abc import Mapping
|
|
from dataclasses import dataclass, field
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path, PurePosixPath
|
|
from typing import Any
|
|
|
|
import yaml
|
|
|
|
from okfwiki import audit, kblocks, klink
|
|
from okfwiki.common import RunError, place_at, place_words
|
|
from okfwiki.kblocks import headings_of, kilde_of, read_lines, version_key
|
|
from okfwiki.kpack import RESERVED_IDS, Fact, check_pending, read_index, read_json, source_facts, week_of
|
|
|
|
GENERATOR = "okf-wiki/kassemble"
|
|
LABEL_HEX = 10
|
|
SAFE_ID = re.compile(r"[a-z0-9][a-z0-9-]*")
|
|
RECORD_KEYS = {"pack": int, "position": int, "file": str, "start": int, "end": int, "section": str, "text": str,
|
|
"quote": str, "quote_line": int, "provenance": str, "concept_id": str}
|
|
LINK_KEYS = {"page": str, "sentence": str, "anchor": str, "at": int, "target": str}
|
|
MAIN_KEYS = {"source": str, "concept_id": str}
|
|
HOME_KEYS = {"source": str, "concept_id": str, "home": str}
|
|
RELATED = "## Related concepts"
|
|
TOLERATED = frozenset({".DS_Store"}) # folder metadata a file browser writes; dropped by the swap
|
|
|
|
|
|
@dataclass
|
|
class Page:
|
|
name: str
|
|
fm: dict[str, Any] # every key but `generated`
|
|
body: str # everything after the frontmatter block
|
|
|
|
|
|
@dataclass
|
|
class Report:
|
|
pages: int = 0
|
|
fragments: int = 0
|
|
new: list[str] = field(default_factory=list)
|
|
changed: list[str] = field(default_factory=list)
|
|
unchanged: list[str] = field(default_factory=list)
|
|
gone: list[str] = field(default_factory=list)
|
|
placed: int = 0
|
|
stale: int = 0
|
|
main_pages: int = 0
|
|
related: int = 0
|
|
dropped: dict[str, set[str]] = field(default_factory=dict) # page -> targets that are gone
|
|
unlinked: int = 0 # link records into a page that is gone, taken out of the links file
|
|
|
|
|
|
def frontmatter(data: dict[str, Any]) -> str:
|
|
"""A frontmatter block; PyYAML quotes any string that would load as another type (no, 1.10)."""
|
|
return "---\n" + yaml.safe_dump(data, sort_keys=False, allow_unicode=True, width=10_000) + "---\n"
|
|
|
|
|
|
def head(fm: dict[str, Any], at: str) -> str:
|
|
return frontmatter({**fm, "generated": {"by": GENERATOR, "at": at}})
|
|
|
|
|
|
def clean(text: str) -> str:
|
|
"""A fragment's text as body prose: no link (bundle or external) and no footnote mark of its own, on one line."""
|
|
text = audit.FOOTNOTE_REF.sub("", audit.sub_links(lambda m: m.group(1), text))
|
|
return re.sub(r"\s*\n\s*", " ", text).strip()
|
|
|
|
|
|
def footnoted(text: str, labels: list[str]) -> str:
|
|
if text and text[-1] not in ".!?":
|
|
text += "."
|
|
marks = "".join(f"[^{x}]" for x in labels)
|
|
out, last = [], 0
|
|
for _a, b in audit.sentence_spans(text):
|
|
out.append(text[last:b] + marks)
|
|
last = b
|
|
out.append(text[last:])
|
|
return "".join(out)
|
|
|
|
|
|
def label(r: dict[str, Any]) -> str:
|
|
key = "\0".join([r["file"], str(r["start"]), str(r["end"]), str(r["quote_line"]), r["quote"], r["provenance"]])
|
|
return "s" + hashlib.sha256(key.encode("utf-8")).hexdigest()[:LABEL_HEX]
|
|
|
|
|
|
def context_label(file: str, line: int, provenance: str) -> str:
|
|
"""A context entry's label, keyed by its line and mark: fragments citing the same line share one entry."""
|
|
key = "\0".join(["context", file, str(line), provenance])
|
|
return "c" + hashlib.sha256(key.encode("utf-8")).hexdigest()[:LABEL_HEX]
|
|
|
|
|
|
def version_label(version: str) -> str:
|
|
return "v" + version.replace(".", "-")
|
|
|
|
|
|
def version_line(lines: list[str], version: str, file: str) -> int:
|
|
"""The line that states the version: its heading, or the label of the component it is (kblocks.stated_versions)."""
|
|
n = kblocks.version_line(lines, version)
|
|
if n is None:
|
|
raise RunError(f"{file} has no heading or label for version {version}")
|
|
return n
|
|
|
|
|
|
def shown(line: str) -> str:
|
|
"""What a source entry quotes from a line: the whole line, trimmed, or a heading's text (order 22: a quote that is
|
|
a piece of its line hid from the reader what the claim drew on)."""
|
|
m = audit.HEADING.match(line)
|
|
return (m.group(2) or "").strip() if m else line.strip()
|
|
|
|
|
|
def one_line(text: str) -> str:
|
|
return " ".join(str(text).split())
|
|
|
|
|
|
def history_entry(file: str, lines: list[str], resource: str) -> dict[str, Any]:
|
|
"""The label entry of a history page without a version: titled with the week from its file name (or, failing
|
|
that, its first heading), quoting its first heading on that line."""
|
|
heads = headings_of(lines)
|
|
if not heads:
|
|
raise RunError(f"{file} is history but has no heading to quote for its label")
|
|
line, _level, text = heads[0]
|
|
week = week_of(file)
|
|
return {"id": "w" + week if week else "h-" + re.sub(r"[^a-z0-9]+", "-", file[:-3].lower()).strip("-"),
|
|
"resource": resource, "title": week or text, "x-locator": {"line": line}, "x-quote": text,
|
|
"x-provenance": "documented"}
|
|
|
|
|
|
def history_key(r: dict[str, Any]) -> tuple[Any, ...]:
|
|
if r.get("version"):
|
|
return 0, version_key(f"release-v{r['version']}.md")
|
|
week = week_of(str(r["file"]))
|
|
return (1, week) if week else (2, str(r["file"]))
|
|
|
|
|
|
def check_roles(raw: Any, key: str, keys: Mapping[str, type]) -> list[dict[str, Any]]:
|
|
"""accepted.json's main_pages or homes: absent is empty; a record without its keys cannot run."""
|
|
value = raw.get(key, []) if isinstance(raw, dict) else []
|
|
if not isinstance(value, list) or not all(isinstance(r, dict) and all(type(r.get(k)) is t for k, t in keys.items())
|
|
for r in value):
|
|
raise RunError(f"accepted.json: {key} is not a list of records with {', '.join(keys)}")
|
|
return value
|
|
|
|
|
|
def check_records(records: Any) -> list[dict[str, Any]]:
|
|
if not isinstance(records, list):
|
|
raise RunError("accepted.json has no fragments list")
|
|
for i, r in enumerate(records):
|
|
bad = [k for k, t in RECORD_KEYS.items() if not isinstance(r, dict) or type(r.get(k)) is not t]
|
|
if bad or not r["quote"].strip() or not (r.get("version") is None or isinstance(r["version"], str)):
|
|
raise RunError(f"accepted record {i} lacks {bad or ['a quote or a version string']}; every fragment "
|
|
f"carries a quote and a provenance mark (rule E)")
|
|
context = r.get("context", [])
|
|
if not (isinstance(context, list) and all(
|
|
isinstance(c, dict) and type(c.get("line")) is int and isinstance(c.get("quote"), str)
|
|
and c["quote"].strip() and isinstance(c.get("section"), str) for c in context)):
|
|
raise RunError(f"accepted record {i}: context is not a list of records with line, quote and section")
|
|
return records
|
|
|
|
|
|
def page_models(index: dict[str, Any], records: list[dict[str, Any]], kilde: Path,
|
|
facts: dict[str, Fact] | None = None, main_pages: list[dict[str, Any]] | None = None) -> dict[str, Page]:
|
|
facts = facts or {}
|
|
entries = {str(e["id"]): e for e in index["entries"]}
|
|
by_concept: dict[str, list[dict[str, Any]]] = {}
|
|
for r in records:
|
|
by_concept.setdefault(r["concept_id"], []).append(r)
|
|
cache: dict[str, list[str]] = {}
|
|
pages: dict[str, Page] = {}
|
|
for cid, items in sorted(by_concept.items()):
|
|
if cid not in entries:
|
|
raise RunError(f"concept {cid!r} has accepted fragments but no index entry")
|
|
if cid in RESERVED_IDS or not SAFE_ID.fullmatch(cid):
|
|
raise RunError(f"concept id {cid!r} cannot be a page name")
|
|
e = entries[cid]
|
|
if not (isinstance(e.get("title"), str) and e["title"].strip()):
|
|
raise RunError(f"the index entry of {cid!r} has no title")
|
|
title = one_line(e["title"])
|
|
items = sorted(items, key=lambda r: (r["pack"], r["position"], r["start"], r["end"]))
|
|
is_history = [bool(r.get("version")) or (r["file"] in facts and facts[r["file"]].history) for r in items]
|
|
body_items = [r for r, h in zip(items, is_history) if not h]
|
|
history = sorted((r for r, h in zip(items, is_history) if h), key=history_key)
|
|
sources: dict[str, dict[str, Any]] = {}
|
|
|
|
def add(entry: dict[str, Any]) -> str:
|
|
known = sources.setdefault(entry["id"], entry)
|
|
if known != entry:
|
|
raise RunError(f"two source entries on {cid}.md share the label {entry['id']}")
|
|
return str(entry["id"])
|
|
|
|
def source_lines(file: str) -> list[str]:
|
|
if file not in cache:
|
|
cache[file] = read_lines(kilde / file)
|
|
return cache[file]
|
|
|
|
def locator(file: str, line: int, text: str, quote: str) -> dict[str, Any]:
|
|
"""The line in the frozen copy, and for a PDF or Office source the place in the original (K1 places): in a
|
|
spreadsheet row, the cell or cells the quote covers (common.place_at); "" for a quote names the whole row."""
|
|
return {"line": line, **(place_at(facts[file].place(line), text, quote) if file in facts else {})}
|
|
|
|
def footnote(s: dict[str, Any]) -> str:
|
|
file = PurePosixPath(s["resource"]).name
|
|
original = facts[file].original if file in facts else None
|
|
return f"{PurePosixPath(original).name if original else file}, {place_words(s['x-locator'])}"
|
|
|
|
def cite(r: dict[str, Any]) -> list[str]:
|
|
src: dict[str, Any] = {"id": label(r), "resource": f"/kilde/{r['file']}"}
|
|
if r["section"]:
|
|
src["title"] = r["section"]
|
|
lines = source_lines(r["file"])
|
|
line = lines[r["quote_line"] - 1] if 0 < r["quote_line"] <= len(lines) else ""
|
|
quote = shown(line) if r["quote"] in line else r["quote"]
|
|
src.update({"x-locator": locator(r["file"], r["quote_line"], line, r["quote"]), "x-quote": quote, "x-span": [r["start"], r["end"]],
|
|
"x-provenance": r["provenance"]})
|
|
labels = [add(src)]
|
|
for c in r.get("context", []):
|
|
at = lines[c["line"] - 1] if 0 < c["line"] <= len(lines) else ""
|
|
ctx: dict[str, Any] = {"id": context_label(r["file"], c["line"], r["provenance"]),
|
|
"resource": f"/kilde/{r['file']}"}
|
|
if c["section"]:
|
|
ctx["title"] = c["section"]
|
|
ctx.update({"x-locator": locator(r["file"], c["line"], at, ""), "x-quote": shown(at) if c["quote"] in at else c["quote"],
|
|
"x-provenance": r["provenance"]})
|
|
labels.append(add(ctx))
|
|
if r.get("version"):
|
|
v = str(r["version"])
|
|
labels.append(add({"id": version_label(v), "resource": f"/kilde/{r['file']}", "title": v,
|
|
"x-locator": {"line": version_line(lines, v, r["file"])},
|
|
"x-quote": v, "x-provenance": "documented"}))
|
|
elif r["file"] in facts and facts[r["file"]].history:
|
|
labels.append(add(history_entry(r["file"], lines, f"/kilde/{r['file']}")))
|
|
return labels
|
|
|
|
lines = [f"# {title}", ""]
|
|
for r in body_items:
|
|
lines += [footnoted(clean(r["text"]), cite(r)), ""]
|
|
if history:
|
|
lines += ["## History", ""]
|
|
lines += [f"- {footnoted(clean(r['text']), cite(r))}" for r in history]
|
|
lines.append("")
|
|
lines += [f"[^{s['id']}]: {footnote(s)}" for s in sources.values()]
|
|
own = [r for r in [*body_items, *history] if not r.get("merged_from")] # the description follows the name
|
|
first = clean((own or body_items or history)[0]["text"])
|
|
spans = audit.sentence_spans(first)
|
|
fm: dict[str, Any] = {"type": "Concept", "title": title,
|
|
"description": first[spans[0][0]:spans[0][1]] if spans else title,
|
|
"sources": list(sources.values())}
|
|
names = other_names(title, e.get("aliases"))
|
|
if names:
|
|
fm["x-aliases"] = names
|
|
if e.get("kind") == "merge-candidate" and e.get("merge_of"):
|
|
fm["x-merge-candidate"] = str(e["merge_of"])
|
|
cited = {r["file"] for r in items}
|
|
mains = sorted({f"/kilde/{m['source']}" for m in main_pages or [] if m["concept_id"] == cid
|
|
and m["source"] in cited and m["source"] in facts and facts[m["source"]].main_topic})
|
|
if mains:
|
|
fm["x-main-for"] = mains
|
|
pages[f"{cid}.md"] = Page(f"{cid}.md", fm, "\n" + "\n".join(lines) + "\n")
|
|
return pages
|
|
|
|
|
|
def other_names(title: str, aliases: Any) -> list[str]:
|
|
"""The catalogue's other names for a concept, in its order: each alias that is not the title, once, with case set
|
|
aside."""
|
|
seen = {title.casefold()}
|
|
out: list[str] = []
|
|
for a in aliases if isinstance(aliases, list) else []:
|
|
name = one_line(str(a))
|
|
if name and name.casefold() not in seen:
|
|
seen.add(name.casefold())
|
|
out.append(name)
|
|
return out
|
|
|
|
|
|
def link_text(title: str) -> str:
|
|
return str(title).replace("[", "(").replace("]", ")")
|
|
|
|
|
|
def related(pages: dict[str, Page], homes: list[dict[str, Any]]) -> dict[str, list[str]]:
|
|
"""Per main page, the list items of the pages whose home it is: the home links the model chose, nothing else."""
|
|
out: dict[str, list[str]] = {}
|
|
for name, page in pages.items():
|
|
if not page.fm.get("x-main-for"):
|
|
continue
|
|
cid = name[:-3]
|
|
held = {str(h["concept_id"]) for h in homes if h["home"] == cid and h["concept_id"] != cid
|
|
and f"{h['concept_id']}.md" in pages}
|
|
items = sorted(held, key=lambda c: (str(pages[f"{c}.md"].fm["title"]).casefold(), c))
|
|
if items:
|
|
out[name] = [f"- [{link_text(pages[f'{c}.md'].fm['title'])}]({c}.md)" for c in items]
|
|
return out
|
|
|
|
|
|
def with_related(body: str, items: list[str]) -> str:
|
|
"""The body with the related-concepts list placed before its footnote definitions."""
|
|
if not items:
|
|
return body
|
|
lines = body.split("\n")
|
|
at = next((i for i, line in enumerate(lines) if audit.FOOTNOTE_DEF.match(line)), len(lines))
|
|
return "\n".join(lines[:at] + [RELATED, "", *items, ""] + lines[at:])
|
|
|
|
|
|
def gone_targets(old: Path, names: set[str], gone: set[str]) -> set[str]:
|
|
"""The pages gone since the last build that the old page `old` linked to."""
|
|
doc = audit.read_document(old, old.name)
|
|
targets = {PurePosixPath(t.split("#", 1)[0]).name for _, t in audit.body_links(doc) if audit.is_bundle_link(t)}
|
|
return targets & gone - names
|
|
|
|
|
|
def index_text(pages: dict[str, Page]) -> str:
|
|
"""The bundle-root index: one entry per page with its frontmatter description (SPEC 8), and after it the page's
|
|
other names (`x-aliases`) as " (also: A; B)", so a search of the index finds a concept by any of its names."""
|
|
out = ["---", 'okf_version: "0.2"', "---", "", "# Concepts", ""]
|
|
out += [f"- [{link_text(p.fm['title'])}]({name}) - {link_text(p.fm['description'])}"
|
|
+ (f" (also: {'; '.join(link_text(a) for a in p.fm['x-aliases'])})" if p.fm.get("x-aliases") else "")
|
|
for name, p in sorted(pages.items())]
|
|
return "\n".join(out) + "\n"
|
|
|
|
|
|
def log_text(old: str | None, at: str, entry: str) -> str:
|
|
"""log.md with one new entry, newest first under its date (SPEC 9); every old line stays as it was."""
|
|
day = at[:10]
|
|
if old is None:
|
|
return f"# Log\n\n## {day}\n{entry}\n"
|
|
lines = old.split("\n")
|
|
for i, raw in enumerate(lines):
|
|
m = audit.HEADING.match(raw)
|
|
text = (m.group(2) or "").strip() if m else ""
|
|
if m and len(m.group(1)) >= 2 and audit.DATE.match(text):
|
|
if text > day:
|
|
raise RunError(f"the build time {at} is older than the log's newest date {text}")
|
|
if text == day:
|
|
lines.insert(i + 1, entry)
|
|
else:
|
|
lines[i:i] = [f"## {day}", entry, ""]
|
|
return "\n".join(lines)
|
|
return old.rstrip("\n") + f"\n\n## {day}\n{entry}\n"
|
|
|
|
|
|
def generated_at(path: Path) -> str | None:
|
|
fm = audit.read_document(path, path.name).frontmatter or {}
|
|
gen = fm.get("generated")
|
|
at = gen.get("at") if isinstance(gen, dict) else None
|
|
return at if isinstance(at, str) else None
|
|
|
|
|
|
def written_here(path: Path) -> bool:
|
|
"""A concept page this command wrote: its generated.by names the generator."""
|
|
fm = audit.read_document(path, path.name).frontmatter or {}
|
|
gen = fm.get("generated")
|
|
return isinstance(gen, dict) and gen.get("by") == GENERATOR
|
|
|
|
|
|
def aside(out: Path) -> Path:
|
|
return out.with_name(f".{out.name}-old")
|
|
|
|
|
|
def check_out(out: Path, replace: bool) -> None:
|
|
if aside(out).exists():
|
|
raise RunError(f"{aside(out)} exists: an earlier swap did not finish; look at it before building again")
|
|
if not out.exists():
|
|
return
|
|
if not replace:
|
|
raise RunError(f"{out} exists; refusing to overwrite without --replace")
|
|
if not out.is_dir():
|
|
raise RunError(f"{out} is not a directory")
|
|
foreign = sorted(p.name for p in out.iterdir() if p.name not in TOLERATED and not (
|
|
p.is_file() and p.suffix == ".md" and (p.name in audit.RESERVED or written_here(p))))
|
|
if foreign or not (out / "log.md").is_file():
|
|
raise RunError(f"{out} holds what this command does not write ({foreign or 'no log.md'}); refusing to replace")
|
|
|
|
|
|
def check_links(path: Path) -> dict[str, Any]:
|
|
doc: dict[str, Any] = read_json(path)
|
|
records = doc.get("links") if isinstance(doc, dict) else None
|
|
if not isinstance(records, list) or not all(
|
|
isinstance(r, dict) and all(type(r.get(k)) is t for k, t in LINK_KEYS.items()) for r in records):
|
|
raise RunError(f"{path} is not a links file: a links list of records with {', '.join(LINK_KEYS)}")
|
|
return doc
|
|
|
|
|
|
def swap_in(new: Path, out: Path) -> None:
|
|
"""Put new in out's place. An old out is set aside beside it first and comes back if the swap fails."""
|
|
old = aside(out) if out.exists() else None
|
|
if old is not None:
|
|
out.rename(old)
|
|
try:
|
|
new.rename(out)
|
|
except BaseException:
|
|
if old is not None:
|
|
old.rename(out)
|
|
raise
|
|
if old is not None:
|
|
shutil.rmtree(old)
|
|
|
|
|
|
def utc(at: str) -> str:
|
|
if not audit.timestamp_ok(at):
|
|
raise RunError(f"--at {at!r} is not an ISO 8601 datetime with an explicit UTC offset (SPEC 5)")
|
|
return datetime.fromisoformat(at.replace("Z", "+00:00")).astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
|
|
|
|
|
|
def build(work: Path, sources: Path, out: Path, links: Path | None = None, replace: bool = False,
|
|
at: str | None = None) -> Report:
|
|
check_pending(work)
|
|
at = utc(at or datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"))
|
|
kilde = kilde_of(sources)
|
|
accepted = work / "K-work" / "accepted.json"
|
|
if not accepted.is_file():
|
|
raise RunError(f"{accepted} does not exist; accept a pack first")
|
|
raw = read_json(accepted)
|
|
records = check_records(raw.get("fragments") if isinstance(raw, dict) else None)
|
|
main_pages, homes = check_roles(raw, "main_pages", MAIN_KEYS), check_roles(raw, "homes", HOME_KEYS)
|
|
facts = source_facts(sources, sorted({str(r["file"]) for r in records} | {str(m["source"]) for m in main_pages}))
|
|
pages = page_models(read_index(work), records, kilde, facts, main_pages)
|
|
lists = related(pages, homes)
|
|
check_out(out, replace)
|
|
link_doc = check_links(links) if links is not None else None
|
|
report = Report(pages=len(pages), fragments=len(records), main_pages=sum(bool(p.fm.get("x-main-for"))
|
|
for p in pages.values()),
|
|
related=sum(len(v) for v in lists.values()))
|
|
old_pages = {p.name for p in out.glob("*.md") if klink.is_concept(p.name)} if out.is_dir() else set()
|
|
gone = old_pages - set(pages)
|
|
out.parent.mkdir(parents=True, exist_ok=True)
|
|
stage = Path(tempfile.mkdtemp(prefix=f".{out.name}-", dir=out.parent))
|
|
try:
|
|
assembled = stage / "pages"
|
|
assembled.mkdir()
|
|
for name, page in pages.items():
|
|
(assembled / name).write_text(head(page.fm, at) + page.body, encoding="utf-8")
|
|
if link_doc is not None:
|
|
records = [{**r, "target": klink.page_rel(r["target"])} for r in link_doc["links"]]
|
|
live = [r for r in records if r["target"] in pages and r["page"] in pages]
|
|
for r in records:
|
|
if r["target"] not in pages and r["page"] in pages:
|
|
report.dropped.setdefault(r["page"], set()).add(r["target"])
|
|
report.unlinked = len(records) - len(live) # into or on a page that is gone
|
|
(stage / "links.json").write_text(json.dumps({**link_doc, "links": live}), encoding="utf-8")
|
|
report.placed, report.stale = klink.place(assembled, stage / "links.json", stage / "linked", replace=False)
|
|
assembled = stage / "linked"
|
|
final = stage / "final"
|
|
final.mkdir()
|
|
for name, page in pages.items():
|
|
_, body = klink.split_page((assembled / name).read_text(encoding="utf-8"))
|
|
body = with_related(body, lists.get(name, []))
|
|
old = out / name
|
|
old_text = old.read_text(encoding="utf-8") if old.is_file() else None
|
|
if old_text is not None:
|
|
lost = gone_targets(old, set(pages), gone)
|
|
if lost:
|
|
report.dropped.setdefault(name, set()).update(lost)
|
|
old_at = generated_at(old) if old_text is not None else None
|
|
if old_at is not None and head(page.fm, old_at) + body == old_text:
|
|
text = old_text
|
|
report.unchanged.append(name)
|
|
else:
|
|
text = head(page.fm, at) + body
|
|
(report.changed if old_text is not None else report.new).append(name)
|
|
(final / name).write_text(text, encoding="utf-8")
|
|
if out.exists():
|
|
report.gone = sorted(p.name for p in out.glob("*.md") if klink.is_concept(p.name) and p.name not in pages)
|
|
(final / "index.md").write_text(index_text(pages), encoding="utf-8")
|
|
dropped = sum(len(t) for t in report.dropped.values())
|
|
entry = (f"* **Build**: {GENERATOR}, {report.pages} pages from {report.fragments} accepted fragments: "
|
|
f"new {len(report.new)}, changed {len(report.changed)}, unchanged {len(report.unchanged)}, "
|
|
f"gone {len(report.gone)}" + (f"; main pages {report.main_pages}, related links {report.related}"
|
|
if report.main_pages else "")
|
|
+ (f"; links placed {report.placed}, stale {report.stale}" if links is not None else "")
|
|
+ (f"; dropped links to pages that are gone {dropped}" if dropped else "") + ".")
|
|
old_log = (out / "log.md").read_text(encoding="utf-8") if out.exists() else None
|
|
(final / "log.md").write_text(log_text(old_log, at, entry), encoding="utf-8")
|
|
swap_in(final, out)
|
|
if links is not None and link_doc is not None and report.unlinked:
|
|
kept = [r for r in link_doc["links"] if klink.page_rel(r["target"]) in pages and r["page"] in pages]
|
|
tmp = links.with_name(f".{links.name}.tmp")
|
|
tmp.write_text(klink.dump({**link_doc, "links": kept, "accepted": len(kept)}), encoding="utf-8")
|
|
tmp.replace(links) # after the swap, so a build that fails keeps every record
|
|
finally:
|
|
shutil.rmtree(stage, ignore_errors=True)
|
|
return report
|
|
|
|
|
|
def build_parser() -> argparse.ArgumentParser:
|
|
p = argparse.ArgumentParser(prog="python3 -m okfwiki.kassemble", description=__doc__,
|
|
formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
sub = p.add_subparsers(dest="command", required=True)
|
|
b = sub.add_parser("build")
|
|
b.add_argument("--work", required=True)
|
|
b.add_argument("--sources", required=True)
|
|
b.add_argument("--out", required=True)
|
|
b.add_argument("--links")
|
|
b.add_argument("--replace", action="store_true")
|
|
b.add_argument("--at")
|
|
return p
|
|
|
|
|
|
def main(argv: list[str] | None = None) -> int:
|
|
args = build_parser().parse_args(argv)
|
|
try:
|
|
r = build(Path(args.work), Path(args.sources), Path(args.out), Path(args.links) if args.links else None,
|
|
args.replace, args.at)
|
|
except (RunError, OSError, UnicodeDecodeError) as exc:
|
|
print(f"kassemble cannot run: {exc}", file=sys.stderr)
|
|
return 2
|
|
except Exception: # an input this code did not foresee: "could not run", never exit 1 (stale links)
|
|
traceback.print_exc()
|
|
return 2
|
|
print(f"kassemble: {r.pages} pages from {r.fragments} accepted fragments -> {args.out}")
|
|
print(f" new {len(r.new)}, changed {len(r.changed)}, unchanged {len(r.unchanged)}, gone {len(r.gone)}")
|
|
print(f" main pages {r.main_pages}, related links {r.related}")
|
|
if args.links:
|
|
print(f" links placed {r.placed}, stale {r.stale}")
|
|
if r.dropped:
|
|
print(f" dropped links to pages that are gone: {sum(len(t) for t in r.dropped.values())} "
|
|
f"({', '.join(sorted(r.dropped))})")
|
|
if r.unlinked:
|
|
print(f" removed from the links file: {r.unlinked} records into or on pages that are gone")
|
|
return 1 if r.stale else 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|