okf-wiki 1.1.1
First public release, published as one snapshot. What changed between versions is in CHANGELOG.md. Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
This commit is contained in:
commit
0fd86a23a3
227 changed files with 96077 additions and 0 deletions
178
evals/p1/dmap.py
Normal file
178
evals/p1/dmap.py
Normal file
|
|
@ -0,0 +1,178 @@
|
|||
"""D, the trial's no-model baseline (TRIAL, P1): a generated map and a code-built concept register. It calls no model.
|
||||
|
||||
build --blocks FILE --kilde-root DIR --out DIR [--at ISO]
|
||||
One map page per source (k-<slug>.md) whose body mirrors the source's headings, with one sources[] entry per
|
||||
section; one register page per kept term (r-<slug>.md) whose body is a pure link list to the map sections
|
||||
where the term occurs, with one sources[] entry per occurrence. A term is kept when it is a docs-page heading
|
||||
that passes concept test 2, or when it occurs in at least 2 of the documents. Bodies hold only headings and
|
||||
pure link lists, so the claim count is 0 by construction; the exact locators live in sources[].
|
||||
|
||||
--kilde-root is the folder that contains kilde/ (the auditor's SOURCES argument). Exit 0 = written, 2 = could not run.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import yaml
|
||||
|
||||
from p1.split import headings_of, is_meta, occurs, terms
|
||||
|
||||
MIN_DOCUMENTS = 2
|
||||
GENERATOR = "okf-wiki/p1-dmap"
|
||||
|
||||
|
||||
class RunError(Exception):
|
||||
"""The command cannot run as asked; nothing was written."""
|
||||
|
||||
|
||||
@dataclass
|
||||
class DStats:
|
||||
maps: int
|
||||
registers: int
|
||||
|
||||
|
||||
def slug(text: str) -> str:
|
||||
return re.sub(r"[^a-z0-9]+", "-", text.lower()).strip("-")[:60] or "x"
|
||||
|
||||
|
||||
def frontmatter(data: dict[str, Any]) -> str:
|
||||
"""A frontmatter block; PyYAML quotes any string that would load as another type (no, 1.10)."""
|
||||
return "---\n" + yaml.safe_dump(data, sort_keys=False, allow_unicode=True, width=10_000) + "---\n"
|
||||
|
||||
|
||||
def _anchors(headings: list[tuple[int, int, str]]) -> dict[int, str]:
|
||||
seen: dict[str, int] = {}
|
||||
out: dict[int, str] = {}
|
||||
for line, _level, text in headings:
|
||||
base = slug(text)
|
||||
n = seen.get(base, 0)
|
||||
seen[base] = n + 1
|
||||
out[line] = base if n == 0 else f"{base}-{n}"
|
||||
return out
|
||||
|
||||
|
||||
def _title(name: str, headings: list[tuple[int, int, str]]) -> str:
|
||||
first = next((t for _, level, t in headings if level == 1), None)
|
||||
if first:
|
||||
return first
|
||||
return f"Release notes {headings[0][2]}" if name.startswith("release-") and headings else name
|
||||
|
||||
|
||||
def build(blocks_doc: dict[str, Any], kilde_root: Path, out: Path, at: str | None = None) -> DStats:
|
||||
if out.exists():
|
||||
raise RunError(f"{out} exists; refusing to overwrite")
|
||||
at = at or datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||
names = [str(n) for n in blocks_doc["order"]]
|
||||
kilde = kilde_root / "kilde"
|
||||
texts = {n: (kilde / n).read_text(encoding="utf-8") for n in names}
|
||||
lines = {n: t.split("\n") for n, t in texts.items()}
|
||||
heads = {n: headings_of(lines[n]) for n in names}
|
||||
anchors = {n: _anchors(heads[n]) for n in names}
|
||||
titles = {n: _title(n, heads[n]) for n in names}
|
||||
pages: dict[str, str] = {}
|
||||
gen = {"by": GENERATOR, "at": at}
|
||||
|
||||
for n in names:
|
||||
sources = []
|
||||
body = [f"# {titles[n]}", ""]
|
||||
for i, (line, level, text) in enumerate(heads[n], 1):
|
||||
sources.append({"id": f"s{i}", "resource": f"/kilde/{n}", "title": text, "x-locator": {"line": line},
|
||||
"x-quote": text})
|
||||
if level > 1 or text != titles[n]:
|
||||
body += [f"{'#' * max(level, 2)} {text}", ""]
|
||||
fm = {"type": "Document map", "title": titles[n], "description": f"The sections of {n}.", "sources": sources,
|
||||
"generated": gen}
|
||||
pages[f"k-{slug(n.removesuffix('.md'))}.md"] = frontmatter(fm) + "\n" + "\n".join(body).rstrip() + "\n"
|
||||
|
||||
candidates = [t for n in names if n.startswith("docs-") for _, level, t in heads[n] if level in (2, 3)]
|
||||
from_headings = {t for t in candidates if not is_meta(t)}
|
||||
pool = list(dict.fromkeys(list(from_headings) + [t for n in names for t in terms(texts[n])]))
|
||||
registers: dict[str, str] = {}
|
||||
for term in pool:
|
||||
if is_meta(term):
|
||||
continue
|
||||
holders = [n for n in names if occurs(term, texts[n])]
|
||||
if term not in from_headings and len(holders) < MIN_DOCUMENTS:
|
||||
continue
|
||||
pattern = re.compile(r"(?<!\w)" + re.escape(term) + r"(?!\w)", re.IGNORECASE)
|
||||
sources, links, seen = [], [], set()
|
||||
for n in holders:
|
||||
for i, raw in enumerate(lines[n], 1):
|
||||
m = pattern.search(raw)
|
||||
if m is None:
|
||||
continue
|
||||
section = max((h for h in heads[n] if h[0] <= i), default=None)
|
||||
key = (n, section[0] if section else 0)
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
sources.append({"id": f"o{len(sources) + 1}", "resource": f"/kilde/{n}", "title": titles[n],
|
||||
"x-locator": {"line": i}, "x-quote": m.group(0)})
|
||||
target = f"k-{slug(n.removesuffix('.md'))}.md"
|
||||
if section is not None:
|
||||
target += f"#{anchors[n][section[0]]}"
|
||||
where = titles[n] + (f" › {section[2]}" if section and section[2] != titles[n] else "")
|
||||
links.append(f"- [{where.replace('[', '(').replace(']', ')')}]({target})")
|
||||
if not sources:
|
||||
continue
|
||||
name = f"r-{slug(term)}.md"
|
||||
k = 2
|
||||
while name in registers:
|
||||
name, k = f"r-{slug(term)}-{k}.md", k + 1
|
||||
fm = {"type": "Register entry", "title": term, "description": f"Where {term} occurs in the sources.",
|
||||
"sources": sources, "generated": gen}
|
||||
registers[name] = frontmatter(fm) + "\n" + f"# {term}\n\n" + "\n".join(links) + "\n"
|
||||
|
||||
out.mkdir(parents=True)
|
||||
for name, text in {**pages, **registers}.items():
|
||||
(out / name).write_text(text, encoding="utf-8")
|
||||
index = ["---", 'okf_version: "0.2"', "---", "", "# Document maps", ""]
|
||||
index += [f"- [{titles[n]}](k-{slug(n.removesuffix('.md'))}.md)" for n in names]
|
||||
index += ["", "# Register", ""]
|
||||
index += [f"- [{name[2:-3]}]({name})" for name in sorted(registers)]
|
||||
(out / "index.md").write_text("\n".join(index) + "\n", encoding="utf-8")
|
||||
day = at[:10]
|
||||
(out / "log.md").write_text(
|
||||
f"# Log\n\n## {day}\n"
|
||||
f"* **Build**: D by {GENERATOR}, {len(pages)} map pages and {len(registers)} register pages from {len(names)} "
|
||||
f"sources. Thresholds, recorded before any count: a term is kept when it is a docs-page heading of level 2 or "
|
||||
f"3 that passes concept test 2, or when it occurs in at least {MIN_DOCUMENTS} of the {len(names)} documents "
|
||||
f"(case-insensitive, whole phrase). Terms: backticked spans of 3-40 characters and capitalised phrases of 2-4 "
|
||||
f"words, a sentence-initial word dropped.\n",
|
||||
encoding="utf-8")
|
||||
return DStats(len(pages), len(registers))
|
||||
|
||||
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
sub = p.add_subparsers(dest="command", required=True)
|
||||
b = sub.add_parser("build")
|
||||
b.add_argument("--blocks", required=True)
|
||||
b.add_argument("--kilde-root", required=True)
|
||||
b.add_argument("--out", required=True)
|
||||
b.add_argument("--at")
|
||||
return p
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
args = build_parser().parse_args(argv)
|
||||
try:
|
||||
doc = json.loads(Path(args.blocks).read_text(encoding="utf-8"))
|
||||
s = build(doc, Path(args.kilde_root), Path(args.out), args.at)
|
||||
except (RunError, OSError, json.JSONDecodeError) as exc:
|
||||
print(f"dmap cannot run: {exc}", file=sys.stderr)
|
||||
return 2
|
||||
print(f"D: {s.maps} map pages, {s.registers} register pages -> {args.out}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Loading…
Add table
Add a link
Reference in a new issue