First public release, published as one snapshot. What changed between versions is in CHANGELOG.md. Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
287 lines
13 KiB
Python
287 lines
13 KiB
Python
"""K's assembler and K+'s apply step (TRIAL, P1): pages from accepted fragments, by code. It calls no model.
|
|
|
|
build --work DIR --kilde-root DIR --out DIR [--at ISO]
|
|
One page per concept that at least one accepted fragment enriches (unused skeleton entries get none). Docs
|
|
fragments form the body; release-note fragments go under "## History" in version order. Every sentence ends
|
|
in a content footnote and, for a release note, a version footnote whose quote is the bare version on the
|
|
note's heading line. A fragment without a model quote gets a code quote: the first line of its span that is
|
|
not structural, marked x-quote-by: code. A fragment that names another concept's title links to it once.
|
|
copy --src DIR --out DIR
|
|
K+ starts as a copy of K.
|
|
tasks --bundle DIR --work DIR [--min 3] [--per-run 8]
|
|
One task file per consolidation run (pages with at least --min fragments, --per-run pages each) in
|
|
<work>/Kplus-oppgaver/, output path <work>/Kplus-bodies/run-N.json.
|
|
apply --bundle DIR --bodies FILE
|
|
Replaces a page body only when the new body keeps every old footnote label and adds none; prints
|
|
"applied X, refused Y".
|
|
|
|
--kilde-root is the folder that contains kilde/. Exit 0 = done, 2 = could not run.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import re
|
|
import shutil
|
|
import sys
|
|
from dataclasses import dataclass
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
from okfwiki.audit import FOOTNOTE_DEF, FOOTNOTE_REF, HEADING, read_document, sentence_spans
|
|
from p1.dmap import frontmatter
|
|
from p1.split import FENCE, headings_of, version_key
|
|
|
|
GENERATOR = "okf-wiki/p1-assemble"
|
|
QUOTE_MAX = 200
|
|
HTML_TAG_ONLY = re.compile(r"^\s*</?[A-Za-z][^<>]*/?>\s*$")
|
|
TABLE_RULE = re.compile(r"^\s*\|?\s*:?-+:?\s*(\|\s*:?-+:?\s*)*\|?\s*$")
|
|
VERSION_HEADING = re.compile(r"^v?(\d+(?:\.\d+){2,})$")
|
|
|
|
|
|
class RunError(Exception):
|
|
"""The command cannot run as asked; nothing was written."""
|
|
|
|
|
|
@dataclass
|
|
class Run:
|
|
task: Path
|
|
output: Path
|
|
pages: list[str]
|
|
|
|
|
|
def structural(line: str) -> bool:
|
|
return (not line.strip() or HEADING.match(line) is not None or FENCE.match(line) is not None
|
|
or HTML_TAG_ONLY.match(line) is not None or TABLE_RULE.match(line) is not None)
|
|
|
|
|
|
def code_quote(span: list[str], start: int) -> tuple[str, int]:
|
|
"""The first non-structural line of a span (stripped, capped), and its file line."""
|
|
for i, line in enumerate(span):
|
|
if not structural(line):
|
|
return line.strip()[:QUOTE_MAX], start + i
|
|
return span[0].strip()[:QUOTE_MAX] or span[0], start
|
|
|
|
|
|
def footnoted(text: str, labels: list[str]) -> str:
|
|
text = text.strip()
|
|
if text and text[-1] not in ".!?":
|
|
text += "."
|
|
marks = "".join(f"[^{x}]" for x in labels)
|
|
out, last = [], 0
|
|
for _a, b in sentence_spans(text):
|
|
out.append(text[last:b] + marks)
|
|
last = b
|
|
out.append(text[last:])
|
|
return "".join(out)
|
|
|
|
|
|
def _link_titles(text: str, titles: dict[str, str], self_id: str, linked: set[str]) -> str:
|
|
for cid, title in sorted(titles.items(), key=lambda kv: -len(kv[1])):
|
|
if cid == self_id or cid in linked or len(title) < 4:
|
|
continue
|
|
pattern = re.compile(r"(?<![\w`\[])" + re.escape(title) + r"(?![\w`\]])", re.IGNORECASE)
|
|
m = pattern.search(text)
|
|
if m is None:
|
|
continue
|
|
text = text[:m.start()] + f"[{m.group(0)}]({cid}.md)" + text[m.end():]
|
|
linked.add(cid)
|
|
return text
|
|
|
|
|
|
def _version_line(lines: list[str], version: str) -> int:
|
|
for n, _level, text in headings_of(lines):
|
|
m = VERSION_HEADING.match(text)
|
|
if m and m.group(1) == version:
|
|
return n
|
|
raise RunError(f"no heading for version {version}")
|
|
|
|
|
|
def assemble(work: Path, kilde_root: Path, out: Path, at: str | None = None) -> int:
|
|
if out.exists():
|
|
raise RunError(f"{out} exists; refusing to overwrite")
|
|
at = at or datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
|
|
index = json.loads((work / "K-work" / "index.json").read_text(encoding="utf-8"))
|
|
frags = json.loads((work / "K-work" / "accepted.json").read_text(encoding="utf-8"))["fragments"]
|
|
entries = {e["id"]: e for e in index["entries"]}
|
|
kilde = kilde_root / "kilde"
|
|
cache: dict[str, list[str]] = {}
|
|
|
|
def lines_of(name: str) -> list[str]:
|
|
if name not in cache:
|
|
cache[name] = (kilde / name).read_text(encoding="utf-8").split("\n")
|
|
return cache[name]
|
|
|
|
by_concept: dict[str, list[dict[str, Any]]] = {}
|
|
for f in frags:
|
|
by_concept.setdefault(f["concept_id"], []).append(f)
|
|
titles = {cid: str(entries[cid]["title"]) for cid in by_concept}
|
|
pages: dict[str, str] = {}
|
|
for cid, items in by_concept.items():
|
|
e = entries[cid]
|
|
sources: list[dict[str, Any]] = []
|
|
versions: dict[str, str] = {}
|
|
body_parts: list[str] = []
|
|
history: list[tuple[tuple[int, ...], int, str]] = []
|
|
linked: set[str] = set()
|
|
for k, f in enumerate(items, 1):
|
|
lines = lines_of(f["file"])
|
|
src: dict[str, Any] = {"id": f"f{k}", "resource": f"/kilde/{f['file']}", "title": f["section"]}
|
|
if f.get("quote"):
|
|
src["x-locator"] = {"line": f["quote_line"]}
|
|
src["x-quote"] = f["quote"]
|
|
else:
|
|
quote, line = code_quote(lines[f["start"] - 1:f["end"]], f["start"])
|
|
src["x-locator"] = {"line": line}
|
|
src["x-quote"] = quote
|
|
src["x-quote-by"] = "code"
|
|
src["x-span"] = [f["start"], f["end"]]
|
|
sources.append(src)
|
|
labels = [f"f{k}"]
|
|
if f.get("version"):
|
|
v = str(f["version"])
|
|
if v not in versions:
|
|
versions[v] = "v" + re.sub(r"\D", "", v)
|
|
sources.append({"id": versions[v], "resource": f"/kilde/{f['file']}", "title": v,
|
|
"x-locator": {"line": _version_line(lines, v)}, "x-quote": v})
|
|
labels.append(versions[v])
|
|
text = _link_titles(footnoted(f["text"], labels), titles, cid, linked)
|
|
history.append((version_key(f"release-v{v}.md"), k, text))
|
|
else:
|
|
body_parts.append(_link_titles(footnoted(f["text"], labels), titles, cid, linked))
|
|
body = [f"# {e['title']}", ""]
|
|
for p in body_parts:
|
|
body += [p, ""]
|
|
if history:
|
|
body += ["## History", ""]
|
|
body += [f"- {t}" for _, _, t in sorted(history)]
|
|
body.append("")
|
|
body += [f"[^{s['id']}]: {Path(s['resource']).name}, line {s['x-locator']['line']}" for s in sources]
|
|
desc = FOOTNOTE_REF.sub("", items[0]["text"]).strip()[:200]
|
|
fm: dict[str, Any] = {"type": "Concept", "title": str(e["title"]), "description": desc, "sources": sources,
|
|
"x-aliases": [str(a) for a in e.get("aliases", [])], "x-fragments": len(items)}
|
|
if e.get("kind") == "merge-candidate" and e.get("merge_of"):
|
|
fm["x-merge-candidate"] = str(e["merge_of"])
|
|
fm["generated"] = {"by": GENERATOR, "at": at}
|
|
pages[f"{cid}.md"] = frontmatter(fm) + "\n" + "\n".join(body) + "\n"
|
|
out.mkdir(parents=True)
|
|
for name, text in pages.items():
|
|
(out / name).write_text(text, encoding="utf-8")
|
|
index_md = ["---", 'okf_version: "0.2"', "---", "", "# Concepts", ""]
|
|
index_md += [f"- [{titles[cid]}]({cid}.md)" for cid in sorted(by_concept)]
|
|
(out / "index.md").write_text("\n".join(index_md) + "\n", encoding="utf-8")
|
|
(out / "log.md").write_text(f"# Log\n\n## {at[:10]}\n* **Build**: K assembled by {GENERATOR}: {len(pages)} "
|
|
f"pages from {len(frags)} accepted fragments.\n", encoding="utf-8")
|
|
return len(pages)
|
|
|
|
|
|
def copy(src: Path, dst: Path) -> None:
|
|
if dst.exists():
|
|
raise RunError(f"{dst} exists; refusing to overwrite")
|
|
shutil.copytree(src, dst)
|
|
|
|
|
|
def _split_page(text: str) -> tuple[str, str, list[str], list[str]]:
|
|
"""(frontmatter block, title line, body lines without title and footnote definitions, definition lines)."""
|
|
end = text.index("\n---\n", 4) + 5
|
|
head, rest = text[:end], text[end:].split("\n")
|
|
title = next(line for line in rest if line.startswith("# "))
|
|
body = [line for line in rest if line != title and not FOOTNOTE_DEF.match(line.strip())]
|
|
defs = [line for line in rest if FOOTNOTE_DEF.match(line.strip())]
|
|
return head, title, body, defs
|
|
|
|
|
|
def consolidation_tasks(bundle: Path, work: Path, min_fragments: int = 3, per_run: int = 8) -> list[Run]:
|
|
chosen: list[str] = []
|
|
for p in sorted(bundle.glob("*.md")):
|
|
if p.name in ("index.md", "log.md"):
|
|
continue
|
|
fm = read_document(p, p.name).frontmatter or {}
|
|
if int(fm.get("x-fragments", 0)) >= min_fragments:
|
|
chosen.append(p.stem)
|
|
runs: list[Run] = []
|
|
for i in range(0, len(chosen), per_run):
|
|
n = len(runs) + 1
|
|
task = work / "Kplus-oppgaver" / f"run-{n}.md"
|
|
output = work / "Kplus-bodies" / f"run-{n}.json"
|
|
if task.exists():
|
|
raise RunError(f"{task} exists; refusing to overwrite")
|
|
parts = [f"# K+ consolidation run {n}\n\nRewrite each page body below into flowing text. Output: {output}\n"]
|
|
for cid in chosen[i:i + per_run]:
|
|
text = (bundle / f"{cid}.md").read_text(encoding="utf-8")
|
|
fm = read_document(bundle / f"{cid}.md", f"{cid}.md").frontmatter or {}
|
|
_head, title, body, _defs = _split_page(text)
|
|
quotes = [f"- [^{s['id']}]: {s.get('x-quote', '')}" for s in fm.get("sources", [])]
|
|
parts.append(f"## Page {cid} ({title[2:]})\n\nBody:\n\n" + "\n".join(body).strip()
|
|
+ "\n\nWhat each label quotes:\n\n" + "\n".join(quotes) + "\n")
|
|
task.parent.mkdir(parents=True, exist_ok=True)
|
|
task.write_text("\n".join(parts), encoding="utf-8")
|
|
runs.append(Run(task, output, chosen[i:i + per_run]))
|
|
return runs
|
|
|
|
|
|
def apply(bundle: Path, bodies_path: Path) -> tuple[int, int]:
|
|
bodies = json.loads(bodies_path.read_text(encoding="utf-8"))
|
|
applied = refused = 0
|
|
for cid, new in bodies.items():
|
|
page = bundle / f"{cid}.md"
|
|
if not page.is_file() or not isinstance(new, str):
|
|
refused += 1
|
|
continue
|
|
head, title, body, defs = _split_page(page.read_text(encoding="utf-8"))
|
|
old_labels = set(FOOTNOTE_REF.findall("\n".join(body)))
|
|
if set(FOOTNOTE_REF.findall(new)) != old_labels or FOOTNOTE_DEF.search(new):
|
|
refused += 1
|
|
continue
|
|
page.write_text(head + "\n" + title + "\n\n" + new.strip() + "\n\n" + "\n".join(defs) + "\n", encoding="utf-8")
|
|
applied += 1
|
|
return applied, refused
|
|
|
|
|
|
def build_parser() -> argparse.ArgumentParser:
|
|
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
sub = p.add_subparsers(dest="command", required=True)
|
|
b = sub.add_parser("build")
|
|
b.add_argument("--work", required=True)
|
|
b.add_argument("--kilde-root", required=True)
|
|
b.add_argument("--out", required=True)
|
|
b.add_argument("--at")
|
|
c = sub.add_parser("copy")
|
|
c.add_argument("--src", required=True)
|
|
c.add_argument("--out", required=True)
|
|
t = sub.add_parser("tasks")
|
|
t.add_argument("--bundle", required=True)
|
|
t.add_argument("--work", required=True)
|
|
t.add_argument("--min", type=int, default=3)
|
|
t.add_argument("--per-run", type=int, default=8)
|
|
a = sub.add_parser("apply")
|
|
a.add_argument("--bundle", required=True)
|
|
a.add_argument("--bodies", required=True)
|
|
return p
|
|
|
|
|
|
def main(argv: list[str] | None = None) -> int:
|
|
args = build_parser().parse_args(argv)
|
|
try:
|
|
if args.command == "build":
|
|
n = assemble(Path(args.work), Path(args.kilde_root), Path(args.out), args.at)
|
|
print(f"K: {n} pages -> {args.out}")
|
|
elif args.command == "copy":
|
|
copy(Path(args.src), Path(args.out))
|
|
print(f"copied {args.src} -> {args.out}")
|
|
elif args.command == "tasks":
|
|
for r in consolidation_tasks(Path(args.bundle), Path(args.work), args.min, args.per_run):
|
|
print(f"task: {r.task} output: {r.output} pages: {len(r.pages)}")
|
|
else:
|
|
applied, refused = apply(Path(args.bundle), Path(args.bodies))
|
|
print(f"applied {applied}, refused {refused}")
|
|
except (RunError, OSError, KeyError, ValueError) as exc:
|
|
print(f"assemble cannot run: {exc}", file=sys.stderr)
|
|
return 2
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|