"""K's assembler and K+'s apply step (TRIAL, P1): pages from accepted fragments, by code. It calls no model. build --work DIR --kilde-root DIR --out DIR [--at ISO] One page per concept that at least one accepted fragment enriches (unused skeleton entries get none). Docs fragments form the body; release-note fragments go under "## History" in version order. Every sentence ends in a content footnote and, for a release note, a version footnote whose quote is the bare version on the note's heading line. A fragment without a model quote gets a code quote: the first line of its span that is not structural, marked x-quote-by: code. A fragment that names another concept's title links to it once. copy --src DIR --out DIR K+ starts as a copy of K. tasks --bundle DIR --work DIR [--min 3] [--per-run 8] One task file per consolidation run (pages with at least --min fragments, --per-run pages each) in /Kplus-oppgaver/, output path /Kplus-bodies/run-N.json. apply --bundle DIR --bodies FILE Replaces a page body only when the new body keeps every old footnote label and adds none; prints "applied X, refused Y". --kilde-root is the folder that contains kilde/. Exit 0 = done, 2 = could not run. """ from __future__ import annotations import argparse import json import re import shutil import sys from dataclasses import dataclass from datetime import datetime, timezone from pathlib import Path from typing import Any from okfwiki.audit import FOOTNOTE_DEF, FOOTNOTE_REF, HEADING, read_document, sentence_spans from p1.dmap import frontmatter from p1.split import FENCE, headings_of, version_key GENERATOR = "okf-wiki/p1-assemble" QUOTE_MAX = 200 HTML_TAG_ONLY = re.compile(r"^\s*]*/?>\s*$") TABLE_RULE = re.compile(r"^\s*\|?\s*:?-+:?\s*(\|\s*:?-+:?\s*)*\|?\s*$") VERSION_HEADING = re.compile(r"^v?(\d+(?:\.\d+){2,})$") class RunError(Exception): """The command cannot run as asked; nothing was written.""" @dataclass class Run: task: Path output: Path pages: list[str] def structural(line: str) -> bool: return (not line.strip() or HEADING.match(line) is not None or FENCE.match(line) is not None or HTML_TAG_ONLY.match(line) is not None or TABLE_RULE.match(line) is not None) def code_quote(span: list[str], start: int) -> tuple[str, int]: """The first non-structural line of a span (stripped, capped), and its file line.""" for i, line in enumerate(span): if not structural(line): return line.strip()[:QUOTE_MAX], start + i return span[0].strip()[:QUOTE_MAX] or span[0], start def footnoted(text: str, labels: list[str]) -> str: text = text.strip() if text and text[-1] not in ".!?": text += "." marks = "".join(f"[^{x}]" for x in labels) out, last = [], 0 for _a, b in sentence_spans(text): out.append(text[last:b] + marks) last = b out.append(text[last:]) return "".join(out) def _link_titles(text: str, titles: dict[str, str], self_id: str, linked: set[str]) -> str: for cid, title in sorted(titles.items(), key=lambda kv: -len(kv[1])): if cid == self_id or cid in linked or len(title) < 4: continue pattern = re.compile(r"(? int: for n, _level, text in headings_of(lines): m = VERSION_HEADING.match(text) if m and m.group(1) == version: return n raise RunError(f"no heading for version {version}") def assemble(work: Path, kilde_root: Path, out: Path, at: str | None = None) -> int: if out.exists(): raise RunError(f"{out} exists; refusing to overwrite") at = at or datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") index = json.loads((work / "K-work" / "index.json").read_text(encoding="utf-8")) frags = json.loads((work / "K-work" / "accepted.json").read_text(encoding="utf-8"))["fragments"] entries = {e["id"]: e for e in index["entries"]} kilde = kilde_root / "kilde" cache: dict[str, list[str]] = {} def lines_of(name: str) -> list[str]: if name not in cache: cache[name] = (kilde / name).read_text(encoding="utf-8").split("\n") return cache[name] by_concept: dict[str, list[dict[str, Any]]] = {} for f in frags: by_concept.setdefault(f["concept_id"], []).append(f) titles = {cid: str(entries[cid]["title"]) for cid in by_concept} pages: dict[str, str] = {} for cid, items in by_concept.items(): e = entries[cid] sources: list[dict[str, Any]] = [] versions: dict[str, str] = {} body_parts: list[str] = [] history: list[tuple[tuple[int, ...], int, str]] = [] linked: set[str] = set() for k, f in enumerate(items, 1): lines = lines_of(f["file"]) src: dict[str, Any] = {"id": f"f{k}", "resource": f"/kilde/{f['file']}", "title": f["section"]} if f.get("quote"): src["x-locator"] = {"line": f["quote_line"]} src["x-quote"] = f["quote"] else: quote, line = code_quote(lines[f["start"] - 1:f["end"]], f["start"]) src["x-locator"] = {"line": line} src["x-quote"] = quote src["x-quote-by"] = "code" src["x-span"] = [f["start"], f["end"]] sources.append(src) labels = [f"f{k}"] if f.get("version"): v = str(f["version"]) if v not in versions: versions[v] = "v" + re.sub(r"\D", "", v) sources.append({"id": versions[v], "resource": f"/kilde/{f['file']}", "title": v, "x-locator": {"line": _version_line(lines, v)}, "x-quote": v}) labels.append(versions[v]) text = _link_titles(footnoted(f["text"], labels), titles, cid, linked) history.append((version_key(f"release-v{v}.md"), k, text)) else: body_parts.append(_link_titles(footnoted(f["text"], labels), titles, cid, linked)) body = [f"# {e['title']}", ""] for p in body_parts: body += [p, ""] if history: body += ["## History", ""] body += [f"- {t}" for _, _, t in sorted(history)] body.append("") body += [f"[^{s['id']}]: {Path(s['resource']).name}, line {s['x-locator']['line']}" for s in sources] desc = FOOTNOTE_REF.sub("", items[0]["text"]).strip()[:200] fm: dict[str, Any] = {"type": "Concept", "title": str(e["title"]), "description": desc, "sources": sources, "x-aliases": [str(a) for a in e.get("aliases", [])], "x-fragments": len(items)} if e.get("kind") == "merge-candidate" and e.get("merge_of"): fm["x-merge-candidate"] = str(e["merge_of"]) fm["generated"] = {"by": GENERATOR, "at": at} pages[f"{cid}.md"] = frontmatter(fm) + "\n" + "\n".join(body) + "\n" out.mkdir(parents=True) for name, text in pages.items(): (out / name).write_text(text, encoding="utf-8") index_md = ["---", 'okf_version: "0.2"', "---", "", "# Concepts", ""] index_md += [f"- [{titles[cid]}]({cid}.md)" for cid in sorted(by_concept)] (out / "index.md").write_text("\n".join(index_md) + "\n", encoding="utf-8") (out / "log.md").write_text(f"# Log\n\n## {at[:10]}\n* **Build**: K assembled by {GENERATOR}: {len(pages)} " f"pages from {len(frags)} accepted fragments.\n", encoding="utf-8") return len(pages) def copy(src: Path, dst: Path) -> None: if dst.exists(): raise RunError(f"{dst} exists; refusing to overwrite") shutil.copytree(src, dst) def _split_page(text: str) -> tuple[str, str, list[str], list[str]]: """(frontmatter block, title line, body lines without title and footnote definitions, definition lines).""" end = text.index("\n---\n", 4) + 5 head, rest = text[:end], text[end:].split("\n") title = next(line for line in rest if line.startswith("# ")) body = [line for line in rest if line != title and not FOOTNOTE_DEF.match(line.strip())] defs = [line for line in rest if FOOTNOTE_DEF.match(line.strip())] return head, title, body, defs def consolidation_tasks(bundle: Path, work: Path, min_fragments: int = 3, per_run: int = 8) -> list[Run]: chosen: list[str] = [] for p in sorted(bundle.glob("*.md")): if p.name in ("index.md", "log.md"): continue fm = read_document(p, p.name).frontmatter or {} if int(fm.get("x-fragments", 0)) >= min_fragments: chosen.append(p.stem) runs: list[Run] = [] for i in range(0, len(chosen), per_run): n = len(runs) + 1 task = work / "Kplus-oppgaver" / f"run-{n}.md" output = work / "Kplus-bodies" / f"run-{n}.json" if task.exists(): raise RunError(f"{task} exists; refusing to overwrite") parts = [f"# K+ consolidation run {n}\n\nRewrite each page body below into flowing text. Output: {output}\n"] for cid in chosen[i:i + per_run]: text = (bundle / f"{cid}.md").read_text(encoding="utf-8") fm = read_document(bundle / f"{cid}.md", f"{cid}.md").frontmatter or {} _head, title, body, _defs = _split_page(text) quotes = [f"- [^{s['id']}]: {s.get('x-quote', '')}" for s in fm.get("sources", [])] parts.append(f"## Page {cid} ({title[2:]})\n\nBody:\n\n" + "\n".join(body).strip() + "\n\nWhat each label quotes:\n\n" + "\n".join(quotes) + "\n") task.parent.mkdir(parents=True, exist_ok=True) task.write_text("\n".join(parts), encoding="utf-8") runs.append(Run(task, output, chosen[i:i + per_run])) return runs def apply(bundle: Path, bodies_path: Path) -> tuple[int, int]: bodies = json.loads(bodies_path.read_text(encoding="utf-8")) applied = refused = 0 for cid, new in bodies.items(): page = bundle / f"{cid}.md" if not page.is_file() or not isinstance(new, str): refused += 1 continue head, title, body, defs = _split_page(page.read_text(encoding="utf-8")) old_labels = set(FOOTNOTE_REF.findall("\n".join(body))) if set(FOOTNOTE_REF.findall(new)) != old_labels or FOOTNOTE_DEF.search(new): refused += 1 continue page.write_text(head + "\n" + title + "\n\n" + new.strip() + "\n\n" + "\n".join(defs) + "\n", encoding="utf-8") applied += 1 return applied, refused def build_parser() -> argparse.ArgumentParser: p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) sub = p.add_subparsers(dest="command", required=True) b = sub.add_parser("build") b.add_argument("--work", required=True) b.add_argument("--kilde-root", required=True) b.add_argument("--out", required=True) b.add_argument("--at") c = sub.add_parser("copy") c.add_argument("--src", required=True) c.add_argument("--out", required=True) t = sub.add_parser("tasks") t.add_argument("--bundle", required=True) t.add_argument("--work", required=True) t.add_argument("--min", type=int, default=3) t.add_argument("--per-run", type=int, default=8) a = sub.add_parser("apply") a.add_argument("--bundle", required=True) a.add_argument("--bodies", required=True) return p def main(argv: list[str] | None = None) -> int: args = build_parser().parse_args(argv) try: if args.command == "build": n = assemble(Path(args.work), Path(args.kilde_root), Path(args.out), args.at) print(f"K: {n} pages -> {args.out}") elif args.command == "copy": copy(Path(args.src), Path(args.out)) print(f"copied {args.src} -> {args.out}") elif args.command == "tasks": for r in consolidation_tasks(Path(args.bundle), Path(args.work), args.min, args.per_run): print(f"task: {r.task} output: {r.output} pages: {len(r.pages)}") else: applied, refused = apply(Path(args.bundle), Path(args.bodies)) print(f"applied {applied}, refused {refused}") except (RunError, OSError, KeyError, ValueError) as exc: print(f"assemble cannot run: {exc}", file=sys.stderr) return 2 return 0 if __name__ == "__main__": sys.exit(main())