okf-wiki 1.1.1

First public release, published as one snapshot. What changed between
versions is in CHANGELOG.md.

Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
This commit is contained in:
Kjell Tore Guttormsen 2026-11-14 21:03:20 +01:00
commit 0fd86a23a3
Signed by: ktg
SSH key fingerprint: SHA256:JakMjO6FTBBzN0Bhfj9saOoEjaFxlSdYuZQQpM/lF9Q
227 changed files with 96077 additions and 0 deletions

374
evals/p1/kpack.py Normal file
View file

@ -0,0 +1,374 @@
"""K, the fragment compiler's code side (TRIAL, P1): skeleton, packs, input parts, validation, identity. No model.
skeleton --kilde DIR --out FILE
The concept index from the titles and level 2-3 headings of every docs page under DIR that pass concept
test 2. One entry per normalised name (the first page in name order wins); an index, not pages.
pack N --blocks FILE --index FILE --kilde-root DIR --work DIR [--size 5]
Pack N's input for the compiler: every block of its sources with its anchor (file › section › span) and
numbered lines, then the index lines whose name occurs in those blocks, then the output contract, in parts
of at most PART_CHARS characters (the Read tool's cap). Prints the part paths and the output path.
accept N --work DIR --blocks FILE --kilde-root DIR
Validates pack N's fragments, resolves each to a concept by code (existing, merge candidate, new or
rejected), updates the index and appends the accepted fragments. Prints "accepted X of N" and the
rejections by reason, and how many accepted fragments resolved to a concept whose index line was not
sent with the pack (the pack log keeps the ids sent; the risk of the D7 phrase filter).
The work directory holds K-oppgaver/ (task parts), K-fragments/ (the compiler's JSON) and K-work/ (index.json,
accepted.json, packs.json). Exit 0 = done, 2 = could not run.
"""
from __future__ import annotations
import argparse
import json
import re
import sys
from collections import Counter
from pathlib import Path
from typing import Any
from okfwiki.audit import sentence_spans
from p1.split import docs_headings, is_meta
PART_CHARS = 60_000
MERGE_JACCARD = 0.6
NEEDS_QUOTE = re.compile(r"\d|`")
REASONS = ("malformed", "anchor-outside-pack", "anchor-crosses-blocks", "sentence-count", "quote-missing",
"quote-not-in-span", "tests-missing", "release-note-new", "meta")
class RunError(Exception):
"""The command cannot run as asked; nothing was written."""
def norm(name: str) -> str:
text = name.replace("`", "").lower()
text = re.sub(r"[^\w\s/.\-]", " ", text)
words = text.split()
if words and words[0] == "the":
words = words[1:]
words = [w[:-1] if len(w) > 3 and w.endswith("s") and not w.endswith("ss") else w for w in words]
return " ".join(w.strip(".") or w for w in words)
def slug(text: str) -> str:
return re.sub(r"[^a-z0-9]+", "-", text.lower()).strip("-")[:60] or "x"
def _unique_id(index: dict[str, Any], title: str) -> str:
taken = {e["id"] for e in index["entries"]}
base = slug(title)
eid, k = base, 2
while eid in taken:
eid, k = f"{base}-{k}", k + 1
return eid
def skeleton(kilde: Path) -> dict[str, Any]:
index: dict[str, Any] = {"schema": "okf-wiki/p1-index/1", "entries": []}
seen: dict[str, dict[str, Any]] = {}
for h in docs_headings(kilde):
if is_meta(h.text):
continue
n = norm(h.text)
if not n:
continue
if n in seen:
if h.text not in seen[n]["aliases"]:
seen[n]["aliases"].append(h.text)
continue
e = {"id": _unique_id(index, h.text), "title": h.text, "aliases": [h.text], "kind": "skeleton",
"merge_of": None, "origin": {"file": h.file, "line": h.line}}
index["entries"].append(e)
seen[n] = e
return index
def _names(e: dict[str, Any]) -> set[str]:
return {norm(a) for a in [e["title"], *e.get("aliases", [])]} - {""}
def _jaccard(a: str, b: str) -> float:
x, y = set(a.split()), set(b.split())
return len(x & y) / len(x | y) if x | y else 0.0
def resolve(fragment: dict[str, Any], index: dict[str, Any]) -> tuple[str, str | None]:
"""existing <id> | merge-candidate <id> | new | rejected <reason> (D6, D3b)."""
by_id = {e["id"]: e for e in index["entries"]}
if fragment.get("status") == "existing" and fragment.get("id") in by_id:
return "existing", str(fragment["id"])
n = norm(str(fragment.get("concept", "")))
for e in index["entries"]:
if n in _names(e):
return "existing", str(e["id"])
if str(fragment.get("anchor", {}).get("file", "")).startswith("release-"):
return "rejected", "release-note-new"
if is_meta(str(fragment.get("concept", ""))):
return "rejected", "meta"
best = max(((max(_jaccard(n, m) for m in _names(e)), str(e["id"])) for e in index["entries"] if _names(e)),
default=(0.0, ""))
if best[0] >= MERGE_JACCARD:
return "merge-candidate", best[1]
return "new", None
def relevant_lines(index: dict[str, Any], text: str) -> list[dict[str, Any]]:
"""Index entries whose normalised name or alias occurs as a phrase in the normalised text (D7)."""
hay = f" {norm(text)} "
return [e for e in index["entries"] if any(f" {m} " in hay for m in _names(e))]
def packs(names: list[str], size: int = 5) -> list[list[str]]:
return [names[i:i + size] for i in range(0, len(names), size)]
CONTRACT = """## Output contract
Write one JSON object `{"fragments": [...]}` to the output path named in your prompt. Each fragment:
`concept`, `status` ("existing" with the index `id`, or "new" with `tests`: {"topic_shape", "citation"}),
`text` (1-3 sentences), `anchor` {"file", "start", "end"} inside one block above, and `quote` (one piece copied from
one line inside the anchor; required when the text holds a digit or a backticked name). A new concept only from a docs
page, never from a release note.
"""
def render_input(pack_no: int, names: list[str], blocks_doc: dict[str, Any], index_lines: list[dict[str, Any]],
kilde: Path) -> list[str]:
lines = {n: (kilde / n).read_text(encoding="utf-8").split("\n") for n in names}
pieces: list[str] = [f"# K pack {pack_no}: {len(names)} sources\n\nSources in order: {', '.join(names)}\n\n"
+ CONTRACT + "\n## Blocks\n\n"]
for b in blocks_doc["blocks"]:
if b["file"] not in names:
continue
head = f"### {b['file']} › {b['section'] or '-'} › {b['start']}–{b['end']}"
if b.get("version"):
head += f" (version {b['version']})"
body = [f"{n}: {lines[b['file']][n - 1]}" for n in range(b["start"], b["end"] + 1)]
chunk = head + "\n"
for row in body:
if len(chunk) + len(row) + 1 > PART_CHARS // 2:
pieces.append(chunk + "\n")
chunk = head + " (continued)\n"
chunk += row + "\n"
pieces.append(chunk + "\n")
pieces.append("## Index (concepts named in these blocks)\n\n")
for e in index_lines:
aliases = "; ".join(a for a in e["aliases"] if a != e["title"])
pieces.append(f"- id: {e['id']} | {e['title']}" + (f" | aliases: {aliases}" if aliases else "") + "\n")
room = PART_CHARS - len("<!-- part 99 of 99 -->\n")
parts: list[str] = []
current = ""
for piece in pieces:
if current and len(current) + len(piece) > room:
parts.append(current)
current = ""
current += piece
if current:
parts.append(current)
total = len(parts)
return [f"<!-- part {i} of {total} -->\n" + p for i, p in enumerate(parts, 1)]
def validate(fragments: list[Any], names: list[str], blocks_doc: dict[str, Any], kilde: Path
) -> tuple[list[dict[str, Any]], Counter[str]]:
by_file: dict[str, list[dict[str, Any]]] = {}
for b in blocks_doc["blocks"]:
by_file.setdefault(b["file"], []).append(b)
texts: dict[str, list[str]] = {}
accepted: list[dict[str, Any]] = []
rejected: Counter[str] = Counter()
for f in fragments:
a = f.get("anchor") if isinstance(f, dict) else None
if not (isinstance(f, dict) and isinstance(f.get("concept"), str) and f["concept"].strip()
and f.get("status") in ("new", "existing") and isinstance(f.get("text"), str)
and isinstance(a, dict) and isinstance(a.get("file"), str)
and type(a.get("start")) is int and type(a.get("end")) is int and a["start"] <= a["end"]):
rejected["malformed"] += 1
continue
if a["file"] not in names:
rejected["anchor-outside-pack"] += 1
continue
block = next((b for b in by_file.get(a["file"], []) if b["start"] <= a["start"] and a["end"] <= b["end"]), None)
if block is None:
rejected["anchor-crosses-blocks"] += 1
continue
if not 1 <= len(sentence_spans(f["text"].strip())) <= 3:
rejected["sentence-count"] += 1
continue
if a["file"] not in texts:
texts[a["file"]] = (kilde / a["file"]).read_text(encoding="utf-8").split("\n")
span = texts[a["file"]][a["start"] - 1:a["end"]]
quote = f.get("quote")
if not (isinstance(quote, str) and quote.strip()):
if NEEDS_QUOTE.search(f["text"]):
rejected["quote-missing"] += 1
continue
quote, quote_line = None, None
else:
hit = next((a["start"] + i for i, line in enumerate(span) if quote in line), None)
if hit is None:
rejected["quote-not-in-span"] += 1
continue
quote_line = hit
tests = f.get("tests")
if f["status"] == "new" and not (isinstance(tests, dict) and str(tests.get("topic_shape", "")).strip()
and str(tests.get("citation", "")).strip()):
rejected["tests-missing"] += 1
continue
accepted.append({**f, "quote": quote, "quote_line": quote_line, "version": block.get("version"),
"section": block["section"], "position": names.index(a["file"]) + 1})
return accepted, rejected
def _read_json(path: Path) -> Any:
try:
return json.loads(path.read_text(encoding="utf-8"))
except (OSError, json.JSONDecodeError) as exc:
raise RunError(f"cannot read {path}: {exc}") from exc
def _write_json(path: Path, data: Any) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(data, ensure_ascii=False, indent=1) + "\n", encoding="utf-8")
def _pack_log(work: Path) -> dict[str, Any]:
path = work / "K-work" / "packs.json"
return _read_json(path) if path.is_file() else {"schema": "okf-wiki/p1-packs/1", "packs": []}
def _record(log: dict[str, Any], n: int) -> dict[str, Any]:
for r in log["packs"]:
if r["pack"] == n:
return r # type: ignore[no-any-return]
r = {"pack": n}
log["packs"].append(r)
return r
def cmd_pack(args: argparse.Namespace) -> int:
work, n = Path(args.work), args.n
doc = _read_json(Path(args.blocks))
all_packs = packs([str(x) for x in doc["order"]], args.size)
if not 1 <= n <= len(all_packs):
raise RunError(f"pack {n} does not exist; there are {len(all_packs)}")
names = all_packs[n - 1]
kilde = Path(args.kilde_root) / "kilde"
index = _read_json(Path(args.index))
text = "\n".join((kilde / name).read_text(encoding="utf-8") for name in names)
lines = relevant_lines(index, text)
parts = render_input(n, names, doc, lines, kilde)
out_dir = work / "K-oppgaver"
paths = [out_dir / f"pack-{n}-part-{i}.md" for i in range(1, len(parts) + 1)]
if any(p.exists() for p in paths):
raise RunError(f"pack {n}'s task parts exist in {out_dir}; refusing to overwrite")
out_dir.mkdir(parents=True, exist_ok=True)
for p, text_part in zip(paths, parts):
p.write_text(text_part, encoding="utf-8")
log = _pack_log(work)
rec = _record(log, n)
rec.update(sources=names, blocks=sum(b["file"] in names for b in doc["blocks"]), index_lines=len(lines),
index_total=len(index["entries"]), index_ids=[str(e["id"]) for e in lines], parts=len(parts),
chars=sum(len(p) for p in parts))
_write_json(work / "K-work" / "packs.json", log)
for p in paths:
print(f"part: {p}")
print(f"output: {work / 'K-fragments' / f'pack-{n}.json'}")
print(f"index lines sent: {len(lines)} of {len(index['entries'])}")
return 0
def cmd_accept(args: argparse.Namespace) -> int:
work, n = Path(args.work), args.n
doc = _read_json(Path(args.blocks))
names = packs([str(x) for x in doc["order"]], args.size)[n - 1]
kilde = Path(args.kilde_root) / "kilde"
raw = _read_json(work / "K-fragments" / f"pack-{n}.json")
fragments = raw.get("fragments") if isinstance(raw, dict) else None
if not isinstance(fragments, list):
raise RunError(f"pack {n}'s output has no fragments list")
index_path = work / "K-work" / "index.json"
index = _read_json(index_path)
valid, rejected = validate(fragments, names, doc, kilde)
acc_path = work / "K-work" / "accepted.json"
acc: dict[str, Any] = (_read_json(acc_path) if acc_path.is_file()
else {"schema": "okf-wiki/p1-accepted/1", "fragments": []})
if any(a["pack"] == n for a in acc["fragments"]):
raise RunError(f"pack {n} is already accepted")
sent = set(_record(_pack_log(work), n).get("index_ids", []))
not_sent: Counter[str] = Counter()
accepted = 0
for f in valid:
outcome, ref = resolve(f, index)
if outcome == "rejected":
rejected[str(ref)] += 1
continue
if outcome == "existing":
cid = str(ref)
else:
cid = _unique_id(index, f["concept"])
index["entries"].append({"id": cid, "title": f["concept"], "aliases": [f["concept"]], "kind": outcome,
"merge_of": ref, "origin": {"file": f["anchor"]["file"], "line": f["anchor"]["start"]},
"tests": f.get("tests")})
acc["fragments"].append({"pack": n, "position": f["position"], "file": f["anchor"]["file"],
"start": f["anchor"]["start"], "end": f["anchor"]["end"], "section": f["section"],
"text": f["text"].strip(), "quote": f["quote"], "quote_line": f["quote_line"],
"version": f["version"], "concept_id": cid, "outcome": outcome})
if cid not in sent:
not_sent[outcome] += 1
accepted += 1
_write_json(index_path, index)
_write_json(acc_path, acc)
log = _pack_log(work)
rec = _record(log, n)
rec.update(proposed=len(fragments), accepted=accepted, rejected=dict(sorted(rejected.items())),
not_sent={"total": sum(not_sent.values()), "by_outcome": dict(sorted(not_sent.items()))})
_write_json(work / "K-work" / "packs.json", log)
print(f"pack {n}: accepted {accepted} of {len(fragments)}")
print(f" accepted with a concept not among the index lines sent: {sum(not_sent.values())}")
for reason, count in sorted(rejected.items()):
print(f" rejected {reason}: {count}")
return 0
def build_parser() -> argparse.ArgumentParser:
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
sub = p.add_subparsers(dest="command", required=True)
s = sub.add_parser("skeleton")
s.add_argument("--kilde", required=True)
s.add_argument("--out", required=True)
for name in ("pack", "accept"):
c = sub.add_parser(name)
c.add_argument("n", type=int)
c.add_argument("--blocks", required=True)
c.add_argument("--kilde-root", required=True)
c.add_argument("--work", required=True)
c.add_argument("--size", type=int, default=5)
if name == "pack":
c.add_argument("--index", required=True)
return p
def main(argv: list[str] | None = None) -> int:
args = build_parser().parse_args(argv)
try:
if args.command == "skeleton":
out = Path(args.out)
if out.exists():
raise RunError(f"{out} exists; refusing to overwrite")
index = skeleton(Path(args.kilde))
_write_json(out, index)
print(f"skeleton: {len(index['entries'])} entries -> {out}")
return 0
if args.command == "pack":
return cmd_pack(args)
return cmd_accept(args)
except (RunError, OSError) as exc:
print(f"kpack cannot run: {exc}", file=sys.stderr)
return 2
if __name__ == "__main__":
sys.exit(main())