First public release, published as one snapshot. What changed between versions is in CHANGELOG.md. Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
178 lines
7.7 KiB
Python
178 lines
7.7 KiB
Python
"""D, the trial's no-model baseline (TRIAL, P1): a generated map and a code-built concept register. It calls no model.
|
||
|
||
build --blocks FILE --kilde-root DIR --out DIR [--at ISO]
|
||
One map page per source (k-<slug>.md) whose body mirrors the source's headings, with one sources[] entry per
|
||
section; one register page per kept term (r-<slug>.md) whose body is a pure link list to the map sections
|
||
where the term occurs, with one sources[] entry per occurrence. A term is kept when it is a docs-page heading
|
||
that passes concept test 2, or when it occurs in at least 2 of the documents. Bodies hold only headings and
|
||
pure link lists, so the claim count is 0 by construction; the exact locators live in sources[].
|
||
|
||
--kilde-root is the folder that contains kilde/ (the auditor's SOURCES argument). Exit 0 = written, 2 = could not run.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import json
|
||
import re
|
||
import sys
|
||
from dataclasses import dataclass
|
||
from datetime import datetime, timezone
|
||
from pathlib import Path
|
||
from typing import Any
|
||
|
||
import yaml
|
||
|
||
from p1.split import headings_of, is_meta, occurs, terms
|
||
|
||
MIN_DOCUMENTS = 2
|
||
GENERATOR = "okf-wiki/p1-dmap"
|
||
|
||
|
||
class RunError(Exception):
|
||
"""The command cannot run as asked; nothing was written."""
|
||
|
||
|
||
@dataclass
|
||
class DStats:
|
||
maps: int
|
||
registers: int
|
||
|
||
|
||
def slug(text: str) -> str:
|
||
return re.sub(r"[^a-z0-9]+", "-", text.lower()).strip("-")[:60] or "x"
|
||
|
||
|
||
def frontmatter(data: dict[str, Any]) -> str:
|
||
"""A frontmatter block; PyYAML quotes any string that would load as another type (no, 1.10)."""
|
||
return "---\n" + yaml.safe_dump(data, sort_keys=False, allow_unicode=True, width=10_000) + "---\n"
|
||
|
||
|
||
def _anchors(headings: list[tuple[int, int, str]]) -> dict[int, str]:
|
||
seen: dict[str, int] = {}
|
||
out: dict[int, str] = {}
|
||
for line, _level, text in headings:
|
||
base = slug(text)
|
||
n = seen.get(base, 0)
|
||
seen[base] = n + 1
|
||
out[line] = base if n == 0 else f"{base}-{n}"
|
||
return out
|
||
|
||
|
||
def _title(name: str, headings: list[tuple[int, int, str]]) -> str:
|
||
first = next((t for _, level, t in headings if level == 1), None)
|
||
if first:
|
||
return first
|
||
return f"Release notes {headings[0][2]}" if name.startswith("release-") and headings else name
|
||
|
||
|
||
def build(blocks_doc: dict[str, Any], kilde_root: Path, out: Path, at: str | None = None) -> DStats:
|
||
if out.exists():
|
||
raise RunError(f"{out} exists; refusing to overwrite")
|
||
at = at or datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
|
||
names = [str(n) for n in blocks_doc["order"]]
|
||
kilde = kilde_root / "kilde"
|
||
texts = {n: (kilde / n).read_text(encoding="utf-8") for n in names}
|
||
lines = {n: t.split("\n") for n, t in texts.items()}
|
||
heads = {n: headings_of(lines[n]) for n in names}
|
||
anchors = {n: _anchors(heads[n]) for n in names}
|
||
titles = {n: _title(n, heads[n]) for n in names}
|
||
pages: dict[str, str] = {}
|
||
gen = {"by": GENERATOR, "at": at}
|
||
|
||
for n in names:
|
||
sources = []
|
||
body = [f"# {titles[n]}", ""]
|
||
for i, (line, level, text) in enumerate(heads[n], 1):
|
||
sources.append({"id": f"s{i}", "resource": f"/kilde/{n}", "title": text, "x-locator": {"line": line},
|
||
"x-quote": text})
|
||
if level > 1 or text != titles[n]:
|
||
body += [f"{'#' * max(level, 2)} {text}", ""]
|
||
fm = {"type": "Document map", "title": titles[n], "description": f"The sections of {n}.", "sources": sources,
|
||
"generated": gen}
|
||
pages[f"k-{slug(n.removesuffix('.md'))}.md"] = frontmatter(fm) + "\n" + "\n".join(body).rstrip() + "\n"
|
||
|
||
candidates = [t for n in names if n.startswith("docs-") for _, level, t in heads[n] if level in (2, 3)]
|
||
from_headings = {t for t in candidates if not is_meta(t)}
|
||
pool = list(dict.fromkeys(list(from_headings) + [t for n in names for t in terms(texts[n])]))
|
||
registers: dict[str, str] = {}
|
||
for term in pool:
|
||
if is_meta(term):
|
||
continue
|
||
holders = [n for n in names if occurs(term, texts[n])]
|
||
if term not in from_headings and len(holders) < MIN_DOCUMENTS:
|
||
continue
|
||
pattern = re.compile(r"(?<!\w)" + re.escape(term) + r"(?!\w)", re.IGNORECASE)
|
||
sources, links, seen = [], [], set()
|
||
for n in holders:
|
||
for i, raw in enumerate(lines[n], 1):
|
||
m = pattern.search(raw)
|
||
if m is None:
|
||
continue
|
||
section = max((h for h in heads[n] if h[0] <= i), default=None)
|
||
key = (n, section[0] if section else 0)
|
||
if key in seen:
|
||
continue
|
||
seen.add(key)
|
||
sources.append({"id": f"o{len(sources) + 1}", "resource": f"/kilde/{n}", "title": titles[n],
|
||
"x-locator": {"line": i}, "x-quote": m.group(0)})
|
||
target = f"k-{slug(n.removesuffix('.md'))}.md"
|
||
if section is not None:
|
||
target += f"#{anchors[n][section[0]]}"
|
||
where = titles[n] + (f" › {section[2]}" if section and section[2] != titles[n] else "")
|
||
links.append(f"- [{where.replace('[', '(').replace(']', ')')}]({target})")
|
||
if not sources:
|
||
continue
|
||
name = f"r-{slug(term)}.md"
|
||
k = 2
|
||
while name in registers:
|
||
name, k = f"r-{slug(term)}-{k}.md", k + 1
|
||
fm = {"type": "Register entry", "title": term, "description": f"Where {term} occurs in the sources.",
|
||
"sources": sources, "generated": gen}
|
||
registers[name] = frontmatter(fm) + "\n" + f"# {term}\n\n" + "\n".join(links) + "\n"
|
||
|
||
out.mkdir(parents=True)
|
||
for name, text in {**pages, **registers}.items():
|
||
(out / name).write_text(text, encoding="utf-8")
|
||
index = ["---", 'okf_version: "0.2"', "---", "", "# Document maps", ""]
|
||
index += [f"- [{titles[n]}](k-{slug(n.removesuffix('.md'))}.md)" for n in names]
|
||
index += ["", "# Register", ""]
|
||
index += [f"- [{name[2:-3]}]({name})" for name in sorted(registers)]
|
||
(out / "index.md").write_text("\n".join(index) + "\n", encoding="utf-8")
|
||
day = at[:10]
|
||
(out / "log.md").write_text(
|
||
f"# Log\n\n## {day}\n"
|
||
f"* **Build**: D by {GENERATOR}, {len(pages)} map pages and {len(registers)} register pages from {len(names)} "
|
||
f"sources. Thresholds, recorded before any count: a term is kept when it is a docs-page heading of level 2 or "
|
||
f"3 that passes concept test 2, or when it occurs in at least {MIN_DOCUMENTS} of the {len(names)} documents "
|
||
f"(case-insensitive, whole phrase). Terms: backticked spans of 3-40 characters and capitalised phrases of 2-4 "
|
||
f"words, a sentence-initial word dropped.\n",
|
||
encoding="utf-8")
|
||
return DStats(len(pages), len(registers))
|
||
|
||
|
||
def build_parser() -> argparse.ArgumentParser:
|
||
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||
sub = p.add_subparsers(dest="command", required=True)
|
||
b = sub.add_parser("build")
|
||
b.add_argument("--blocks", required=True)
|
||
b.add_argument("--kilde-root", required=True)
|
||
b.add_argument("--out", required=True)
|
||
b.add_argument("--at")
|
||
return p
|
||
|
||
|
||
def main(argv: list[str] | None = None) -> int:
|
||
args = build_parser().parse_args(argv)
|
||
try:
|
||
doc = json.loads(Path(args.blocks).read_text(encoding="utf-8"))
|
||
s = build(doc, Path(args.kilde_root), Path(args.out), args.at)
|
||
except (RunError, OSError, json.JSONDecodeError) as exc:
|
||
print(f"dmap cannot run: {exc}", file=sys.stderr)
|
||
return 2
|
||
print(f"D: {s.maps} map pages, {s.registers} register pages -> {args.out}")
|
||
return 0
|
||
|
||
|
||
if __name__ == "__main__":
|
||
sys.exit(main())
|