okf-wiki/evals/p1/assemble.py
Kjell Tore Guttormsen 0fd86a23a3
okf-wiki 1.1.1
First public release, published as one snapshot. What changed between
versions is in CHANGELOG.md.

Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
2026-11-14 21:03:20 +01:00

287 lines
13 KiB
Python

"""K's assembler and K+'s apply step (TRIAL, P1): pages from accepted fragments, by code. It calls no model.
build --work DIR --kilde-root DIR --out DIR [--at ISO]
One page per concept that at least one accepted fragment enriches (unused skeleton entries get none). Docs
fragments form the body; release-note fragments go under "## History" in version order. Every sentence ends
in a content footnote and, for a release note, a version footnote whose quote is the bare version on the
note's heading line. A fragment without a model quote gets a code quote: the first line of its span that is
not structural, marked x-quote-by: code. A fragment that names another concept's title links to it once.
copy --src DIR --out DIR
K+ starts as a copy of K.
tasks --bundle DIR --work DIR [--min 3] [--per-run 8]
One task file per consolidation run (pages with at least --min fragments, --per-run pages each) in
<work>/Kplus-oppgaver/, output path <work>/Kplus-bodies/run-N.json.
apply --bundle DIR --bodies FILE
Replaces a page body only when the new body keeps every old footnote label and adds none; prints
"applied X, refused Y".
--kilde-root is the folder that contains kilde/. Exit 0 = done, 2 = could not run.
"""
from __future__ import annotations
import argparse
import json
import re
import shutil
import sys
from dataclasses import dataclass
from datetime import datetime, timezone
from pathlib import Path
from typing import Any
from okfwiki.audit import FOOTNOTE_DEF, FOOTNOTE_REF, HEADING, read_document, sentence_spans
from p1.dmap import frontmatter
from p1.split import FENCE, headings_of, version_key
GENERATOR = "okf-wiki/p1-assemble"
QUOTE_MAX = 200
HTML_TAG_ONLY = re.compile(r"^\s*</?[A-Za-z][^<>]*/?>\s*$")
TABLE_RULE = re.compile(r"^\s*\|?\s*:?-+:?\s*(\|\s*:?-+:?\s*)*\|?\s*$")
VERSION_HEADING = re.compile(r"^v?(\d+(?:\.\d+){2,})$")
class RunError(Exception):
"""The command cannot run as asked; nothing was written."""
@dataclass
class Run:
task: Path
output: Path
pages: list[str]
def structural(line: str) -> bool:
return (not line.strip() or HEADING.match(line) is not None or FENCE.match(line) is not None
or HTML_TAG_ONLY.match(line) is not None or TABLE_RULE.match(line) is not None)
def code_quote(span: list[str], start: int) -> tuple[str, int]:
"""The first non-structural line of a span (stripped, capped), and its file line."""
for i, line in enumerate(span):
if not structural(line):
return line.strip()[:QUOTE_MAX], start + i
return span[0].strip()[:QUOTE_MAX] or span[0], start
def footnoted(text: str, labels: list[str]) -> str:
text = text.strip()
if text and text[-1] not in ".!?":
text += "."
marks = "".join(f"[^{x}]" for x in labels)
out, last = [], 0
for _a, b in sentence_spans(text):
out.append(text[last:b] + marks)
last = b
out.append(text[last:])
return "".join(out)
def _link_titles(text: str, titles: dict[str, str], self_id: str, linked: set[str]) -> str:
for cid, title in sorted(titles.items(), key=lambda kv: -len(kv[1])):
if cid == self_id or cid in linked or len(title) < 4:
continue
pattern = re.compile(r"(?<![\w`\[])" + re.escape(title) + r"(?![\w`\]])", re.IGNORECASE)
m = pattern.search(text)
if m is None:
continue
text = text[:m.start()] + f"[{m.group(0)}]({cid}.md)" + text[m.end():]
linked.add(cid)
return text
def _version_line(lines: list[str], version: str) -> int:
for n, _level, text in headings_of(lines):
m = VERSION_HEADING.match(text)
if m and m.group(1) == version:
return n
raise RunError(f"no heading for version {version}")
def assemble(work: Path, kilde_root: Path, out: Path, at: str | None = None) -> int:
if out.exists():
raise RunError(f"{out} exists; refusing to overwrite")
at = at or datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
index = json.loads((work / "K-work" / "index.json").read_text(encoding="utf-8"))
frags = json.loads((work / "K-work" / "accepted.json").read_text(encoding="utf-8"))["fragments"]
entries = {e["id"]: e for e in index["entries"]}
kilde = kilde_root / "kilde"
cache: dict[str, list[str]] = {}
def lines_of(name: str) -> list[str]:
if name not in cache:
cache[name] = (kilde / name).read_text(encoding="utf-8").split("\n")
return cache[name]
by_concept: dict[str, list[dict[str, Any]]] = {}
for f in frags:
by_concept.setdefault(f["concept_id"], []).append(f)
titles = {cid: str(entries[cid]["title"]) for cid in by_concept}
pages: dict[str, str] = {}
for cid, items in by_concept.items():
e = entries[cid]
sources: list[dict[str, Any]] = []
versions: dict[str, str] = {}
body_parts: list[str] = []
history: list[tuple[tuple[int, ...], int, str]] = []
linked: set[str] = set()
for k, f in enumerate(items, 1):
lines = lines_of(f["file"])
src: dict[str, Any] = {"id": f"f{k}", "resource": f"/kilde/{f['file']}", "title": f["section"]}
if f.get("quote"):
src["x-locator"] = {"line": f["quote_line"]}
src["x-quote"] = f["quote"]
else:
quote, line = code_quote(lines[f["start"] - 1:f["end"]], f["start"])
src["x-locator"] = {"line": line}
src["x-quote"] = quote
src["x-quote-by"] = "code"
src["x-span"] = [f["start"], f["end"]]
sources.append(src)
labels = [f"f{k}"]
if f.get("version"):
v = str(f["version"])
if v not in versions:
versions[v] = "v" + re.sub(r"\D", "", v)
sources.append({"id": versions[v], "resource": f"/kilde/{f['file']}", "title": v,
"x-locator": {"line": _version_line(lines, v)}, "x-quote": v})
labels.append(versions[v])
text = _link_titles(footnoted(f["text"], labels), titles, cid, linked)
history.append((version_key(f"release-v{v}.md"), k, text))
else:
body_parts.append(_link_titles(footnoted(f["text"], labels), titles, cid, linked))
body = [f"# {e['title']}", ""]
for p in body_parts:
body += [p, ""]
if history:
body += ["## History", ""]
body += [f"- {t}" for _, _, t in sorted(history)]
body.append("")
body += [f"[^{s['id']}]: {Path(s['resource']).name}, line {s['x-locator']['line']}" for s in sources]
desc = FOOTNOTE_REF.sub("", items[0]["text"]).strip()[:200]
fm: dict[str, Any] = {"type": "Concept", "title": str(e["title"]), "description": desc, "sources": sources,
"x-aliases": [str(a) for a in e.get("aliases", [])], "x-fragments": len(items)}
if e.get("kind") == "merge-candidate" and e.get("merge_of"):
fm["x-merge-candidate"] = str(e["merge_of"])
fm["generated"] = {"by": GENERATOR, "at": at}
pages[f"{cid}.md"] = frontmatter(fm) + "\n" + "\n".join(body) + "\n"
out.mkdir(parents=True)
for name, text in pages.items():
(out / name).write_text(text, encoding="utf-8")
index_md = ["---", 'okf_version: "0.2"', "---", "", "# Concepts", ""]
index_md += [f"- [{titles[cid]}]({cid}.md)" for cid in sorted(by_concept)]
(out / "index.md").write_text("\n".join(index_md) + "\n", encoding="utf-8")
(out / "log.md").write_text(f"# Log\n\n## {at[:10]}\n* **Build**: K assembled by {GENERATOR}: {len(pages)} "
f"pages from {len(frags)} accepted fragments.\n", encoding="utf-8")
return len(pages)
def copy(src: Path, dst: Path) -> None:
if dst.exists():
raise RunError(f"{dst} exists; refusing to overwrite")
shutil.copytree(src, dst)
def _split_page(text: str) -> tuple[str, str, list[str], list[str]]:
"""(frontmatter block, title line, body lines without title and footnote definitions, definition lines)."""
end = text.index("\n---\n", 4) + 5
head, rest = text[:end], text[end:].split("\n")
title = next(line for line in rest if line.startswith("# "))
body = [line for line in rest if line != title and not FOOTNOTE_DEF.match(line.strip())]
defs = [line for line in rest if FOOTNOTE_DEF.match(line.strip())]
return head, title, body, defs
def consolidation_tasks(bundle: Path, work: Path, min_fragments: int = 3, per_run: int = 8) -> list[Run]:
chosen: list[str] = []
for p in sorted(bundle.glob("*.md")):
if p.name in ("index.md", "log.md"):
continue
fm = read_document(p, p.name).frontmatter or {}
if int(fm.get("x-fragments", 0)) >= min_fragments:
chosen.append(p.stem)
runs: list[Run] = []
for i in range(0, len(chosen), per_run):
n = len(runs) + 1
task = work / "Kplus-oppgaver" / f"run-{n}.md"
output = work / "Kplus-bodies" / f"run-{n}.json"
if task.exists():
raise RunError(f"{task} exists; refusing to overwrite")
parts = [f"# K+ consolidation run {n}\n\nRewrite each page body below into flowing text. Output: {output}\n"]
for cid in chosen[i:i + per_run]:
text = (bundle / f"{cid}.md").read_text(encoding="utf-8")
fm = read_document(bundle / f"{cid}.md", f"{cid}.md").frontmatter or {}
_head, title, body, _defs = _split_page(text)
quotes = [f"- [^{s['id']}]: {s.get('x-quote', '')}" for s in fm.get("sources", [])]
parts.append(f"## Page {cid} ({title[2:]})\n\nBody:\n\n" + "\n".join(body).strip()
+ "\n\nWhat each label quotes:\n\n" + "\n".join(quotes) + "\n")
task.parent.mkdir(parents=True, exist_ok=True)
task.write_text("\n".join(parts), encoding="utf-8")
runs.append(Run(task, output, chosen[i:i + per_run]))
return runs
def apply(bundle: Path, bodies_path: Path) -> tuple[int, int]:
bodies = json.loads(bodies_path.read_text(encoding="utf-8"))
applied = refused = 0
for cid, new in bodies.items():
page = bundle / f"{cid}.md"
if not page.is_file() or not isinstance(new, str):
refused += 1
continue
head, title, body, defs = _split_page(page.read_text(encoding="utf-8"))
old_labels = set(FOOTNOTE_REF.findall("\n".join(body)))
if set(FOOTNOTE_REF.findall(new)) != old_labels or FOOTNOTE_DEF.search(new):
refused += 1
continue
page.write_text(head + "\n" + title + "\n\n" + new.strip() + "\n\n" + "\n".join(defs) + "\n", encoding="utf-8")
applied += 1
return applied, refused
def build_parser() -> argparse.ArgumentParser:
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
sub = p.add_subparsers(dest="command", required=True)
b = sub.add_parser("build")
b.add_argument("--work", required=True)
b.add_argument("--kilde-root", required=True)
b.add_argument("--out", required=True)
b.add_argument("--at")
c = sub.add_parser("copy")
c.add_argument("--src", required=True)
c.add_argument("--out", required=True)
t = sub.add_parser("tasks")
t.add_argument("--bundle", required=True)
t.add_argument("--work", required=True)
t.add_argument("--min", type=int, default=3)
t.add_argument("--per-run", type=int, default=8)
a = sub.add_parser("apply")
a.add_argument("--bundle", required=True)
a.add_argument("--bodies", required=True)
return p
def main(argv: list[str] | None = None) -> int:
args = build_parser().parse_args(argv)
try:
if args.command == "build":
n = assemble(Path(args.work), Path(args.kilde_root), Path(args.out), args.at)
print(f"K: {n} pages -> {args.out}")
elif args.command == "copy":
copy(Path(args.src), Path(args.out))
print(f"copied {args.src} -> {args.out}")
elif args.command == "tasks":
for r in consolidation_tasks(Path(args.bundle), Path(args.work), args.min, args.per_run):
print(f"task: {r.task} output: {r.output} pages: {len(r.pages)}")
else:
applied, refused = apply(Path(args.bundle), Path(args.bodies))
print(f"applied {applied}, refused {refused}")
except (RunError, OSError, KeyError, ValueError) as exc:
print(f"assemble cannot run: {exc}", file=sys.stderr)
return 2
return 0
if __name__ == "__main__":
sys.exit(main())