okf-wiki/evals/p1/dmap.py
Kjell Tore Guttormsen 0fd86a23a3
okf-wiki 1.1.1
First public release, published as one snapshot. What changed between
versions is in CHANGELOG.md.

Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
2026-11-14 21:03:20 +01:00

178 lines
7.7 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

"""D, the trial's no-model baseline (TRIAL, P1): a generated map and a code-built concept register. It calls no model.
build --blocks FILE --kilde-root DIR --out DIR [--at ISO]
One map page per source (k-<slug>.md) whose body mirrors the source's headings, with one sources[] entry per
section; one register page per kept term (r-<slug>.md) whose body is a pure link list to the map sections
where the term occurs, with one sources[] entry per occurrence. A term is kept when it is a docs-page heading
that passes concept test 2, or when it occurs in at least 2 of the documents. Bodies hold only headings and
pure link lists, so the claim count is 0 by construction; the exact locators live in sources[].
--kilde-root is the folder that contains kilde/ (the auditor's SOURCES argument). Exit 0 = written, 2 = could not run.
"""
from __future__ import annotations
import argparse
import json
import re
import sys
from dataclasses import dataclass
from datetime import datetime, timezone
from pathlib import Path
from typing import Any
import yaml
from p1.split import headings_of, is_meta, occurs, terms
MIN_DOCUMENTS = 2
GENERATOR = "okf-wiki/p1-dmap"
class RunError(Exception):
"""The command cannot run as asked; nothing was written."""
@dataclass
class DStats:
maps: int
registers: int
def slug(text: str) -> str:
return re.sub(r"[^a-z0-9]+", "-", text.lower()).strip("-")[:60] or "x"
def frontmatter(data: dict[str, Any]) -> str:
"""A frontmatter block; PyYAML quotes any string that would load as another type (no, 1.10)."""
return "---\n" + yaml.safe_dump(data, sort_keys=False, allow_unicode=True, width=10_000) + "---\n"
def _anchors(headings: list[tuple[int, int, str]]) -> dict[int, str]:
seen: dict[str, int] = {}
out: dict[int, str] = {}
for line, _level, text in headings:
base = slug(text)
n = seen.get(base, 0)
seen[base] = n + 1
out[line] = base if n == 0 else f"{base}-{n}"
return out
def _title(name: str, headings: list[tuple[int, int, str]]) -> str:
first = next((t for _, level, t in headings if level == 1), None)
if first:
return first
return f"Release notes {headings[0][2]}" if name.startswith("release-") and headings else name
def build(blocks_doc: dict[str, Any], kilde_root: Path, out: Path, at: str | None = None) -> DStats:
if out.exists():
raise RunError(f"{out} exists; refusing to overwrite")
at = at or datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
names = [str(n) for n in blocks_doc["order"]]
kilde = kilde_root / "kilde"
texts = {n: (kilde / n).read_text(encoding="utf-8") for n in names}
lines = {n: t.split("\n") for n, t in texts.items()}
heads = {n: headings_of(lines[n]) for n in names}
anchors = {n: _anchors(heads[n]) for n in names}
titles = {n: _title(n, heads[n]) for n in names}
pages: dict[str, str] = {}
gen = {"by": GENERATOR, "at": at}
for n in names:
sources = []
body = [f"# {titles[n]}", ""]
for i, (line, level, text) in enumerate(heads[n], 1):
sources.append({"id": f"s{i}", "resource": f"/kilde/{n}", "title": text, "x-locator": {"line": line},
"x-quote": text})
if level > 1 or text != titles[n]:
body += [f"{'#' * max(level, 2)} {text}", ""]
fm = {"type": "Document map", "title": titles[n], "description": f"The sections of {n}.", "sources": sources,
"generated": gen}
pages[f"k-{slug(n.removesuffix('.md'))}.md"] = frontmatter(fm) + "\n" + "\n".join(body).rstrip() + "\n"
candidates = [t for n in names if n.startswith("docs-") for _, level, t in heads[n] if level in (2, 3)]
from_headings = {t for t in candidates if not is_meta(t)}
pool = list(dict.fromkeys(list(from_headings) + [t for n in names for t in terms(texts[n])]))
registers: dict[str, str] = {}
for term in pool:
if is_meta(term):
continue
holders = [n for n in names if occurs(term, texts[n])]
if term not in from_headings and len(holders) < MIN_DOCUMENTS:
continue
pattern = re.compile(r"(?<!\w)" + re.escape(term) + r"(?!\w)", re.IGNORECASE)
sources, links, seen = [], [], set()
for n in holders:
for i, raw in enumerate(lines[n], 1):
m = pattern.search(raw)
if m is None:
continue
section = max((h for h in heads[n] if h[0] <= i), default=None)
key = (n, section[0] if section else 0)
if key in seen:
continue
seen.add(key)
sources.append({"id": f"o{len(sources) + 1}", "resource": f"/kilde/{n}", "title": titles[n],
"x-locator": {"line": i}, "x-quote": m.group(0)})
target = f"k-{slug(n.removesuffix('.md'))}.md"
if section is not None:
target += f"#{anchors[n][section[0]]}"
where = titles[n] + (f" › {section[2]}" if section and section[2] != titles[n] else "")
links.append(f"- [{where.replace('[', '(').replace(']', ')')}]({target})")
if not sources:
continue
name = f"r-{slug(term)}.md"
k = 2
while name in registers:
name, k = f"r-{slug(term)}-{k}.md", k + 1
fm = {"type": "Register entry", "title": term, "description": f"Where {term} occurs in the sources.",
"sources": sources, "generated": gen}
registers[name] = frontmatter(fm) + "\n" + f"# {term}\n\n" + "\n".join(links) + "\n"
out.mkdir(parents=True)
for name, text in {**pages, **registers}.items():
(out / name).write_text(text, encoding="utf-8")
index = ["---", 'okf_version: "0.2"', "---", "", "# Document maps", ""]
index += [f"- [{titles[n]}](k-{slug(n.removesuffix('.md'))}.md)" for n in names]
index += ["", "# Register", ""]
index += [f"- [{name[2:-3]}]({name})" for name in sorted(registers)]
(out / "index.md").write_text("\n".join(index) + "\n", encoding="utf-8")
day = at[:10]
(out / "log.md").write_text(
f"# Log\n\n## {day}\n"
f"* **Build**: D by {GENERATOR}, {len(pages)} map pages and {len(registers)} register pages from {len(names)} "
f"sources. Thresholds, recorded before any count: a term is kept when it is a docs-page heading of level 2 or "
f"3 that passes concept test 2, or when it occurs in at least {MIN_DOCUMENTS} of the {len(names)} documents "
f"(case-insensitive, whole phrase). Terms: backticked spans of 3-40 characters and capitalised phrases of 2-4 "
f"words, a sentence-initial word dropped.\n",
encoding="utf-8")
return DStats(len(pages), len(registers))
def build_parser() -> argparse.ArgumentParser:
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
sub = p.add_subparsers(dest="command", required=True)
b = sub.add_parser("build")
b.add_argument("--blocks", required=True)
b.add_argument("--kilde-root", required=True)
b.add_argument("--out", required=True)
b.add_argument("--at")
return p
def main(argv: list[str] | None = None) -> int:
args = build_parser().parse_args(argv)
try:
doc = json.loads(Path(args.blocks).read_text(encoding="utf-8"))
s = build(doc, Path(args.kilde_root), Path(args.out), args.at)
except (RunError, OSError, json.JSONDecodeError) as exc:
print(f"dmap cannot run: {exc}", file=sys.stderr)
return 2
print(f"D: {s.maps} map pages, {s.registers} register pages -> {args.out}")
return 0
if __name__ == "__main__":
sys.exit(main())