okf-wiki 1.1.1

First public release, published as one snapshot. What changed between
versions is in CHANGELOG.md.

Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
This commit is contained in:
Kjell Tore Guttormsen 2026-11-14 21:03:20 +01:00
commit 0fd86a23a3
Signed by: ktg
SSH key fingerprint: SHA256:JakMjO6FTBBzN0Bhfj9saOoEjaFxlSdYuZQQpM/lF9Q
227 changed files with 96077 additions and 0 deletions

421
tools/diagnose.py Normal file
View file

@ -0,0 +1,421 @@
"""Why a claim is or is not carried by its excerpt: the tool around the diagnosis of wave 4 order 20 (contract K9).
No model call. The diagnosis explains; the reader judges. It is not calibrated and never counts for criterion 2.
Usage:
python3 tools/diagnose.py select --claims claims.json --verdicts verdicts.json --out utvalg.json [--sample 100]
python3 tools/diagnose.py render --utvalg utvalg.json --claims claims.json --sources SOURCES --wiki WIKI
--out tasks/ [--size 10]
python3 tools/diagnose.py collect --transcripts DIR --run R [--run R2]... --tasks tasks/ --out diagnoses.json
python3 tools/diagnose.py tally --diagnoses diagnoses.json --verdicts verdicts.json --claims claims.json
--out tally.json
select every claim the reader did not accept, and a sample of the accepted ones:
random.Random(SEED).sample(sorted(ids), --sample), then the two lists (each sorted) shuffled with the same
generator. The selection holds ids only, never a verdict. A claim without a verdict, or fewer accepted claims
than the sample, cannot run.
render one task per call of at most --size claims, in the selection's order, named d001, d002, ...; and
calls.json (the ids per call). Per claim: the page title, the claim, each excerpt it cites, and per excerpt a
window in its source: the heading path above the quoted line (okfwiki.kblocks.headings_of, as
tools/claim_features.py takes it) and WINDOW lines before and after it, cut at the file's ends. Each window
line is "<line number> <q><c>| <text>", q ">" on the quoted line, c "c" on a line of fenced code (the fence
lines too). No id, verdict or provenance mark reaches the task. A task folder that holds anything is never
written into.
collect the reply of every diagnosis agent (agent type okf-wiki-diagnoser, description "V1-<run>-diagnose-<call>",
started from the session or from a workflow, the two places okfwiki.cost harvest reads): the message it
handed back, or else its last text (okfwiki.reader.final_reply). For each call the last --run that answered
it counts; two transcripts of that run for one call give no diagnosis for any of its claims. A line
"<number>: <code> | <words>" gives the claim of that number its code (one of CODES) and the words that are
missing; S needs no words, every other code does, and A has at most twelve. A line that starts with a number
but breaks the form, or a number given twice, gives that claim no diagnosis: it is "missing", never guessed.
Other lines are ignored.
tally counts per code for the claims the reader did not accept and for the sample it accepted, each apart; how
often the diagnosis and the reader disagree (S on a rejected claim; any other code on an accepted one) with a
two-sided 95 % Clopper-Pearson interval; the same counts per source file and per provenance mark; and the M1
rule of order 20: S for at least a third of the rejected claims, or any claim missing, is stop and present.
Codes (locked in phase 0 before any diagnosis was read): S the excerpt alone carries the claim; O what is missing
stands in the heading path or the page title; L it stands in another line of the window; B the claim says more than
the source (wider scope, stronger words); T a number, version, order, cause or purpose that is not in the window; F a
property given to the wrong thing, or a wrong connection; U what is missing is not in the window at all; A other.
Output: json.dumps(doc, indent=1, sort_keys=True, ensure_ascii=False) + newline; an existing output is never
overwritten. Exit 0 written; 1 written with claims missing (collect) or the M1 rule says stop (tally); 2 cannot run.
"""
from __future__ import annotations
import argparse
import hashlib
import json
import math
import random
import re
import sys
from pathlib import Path
from typing import Any
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
sys.path.insert(0, str(Path(__file__).resolve().parent))
import claim_features # noqa: E402
from okfwiki import kblocks # noqa: E402
from okfwiki.common import RunError, write_new # noqa: E402
from okfwiki.reader import final_reply # noqa: E402
from okfwiki.transcript import is_agent_type # noqa: E402
SELECTION_SCHEMA = "okf-wiki/diagnose-selection/1"
CALLS_SCHEMA = "okf-wiki/diagnose-calls/1"
DIAGNOSES_SCHEMA = "okf-wiki/diagnoses/1"
TALLY_SCHEMA = "okf-wiki/diagnose-tally/1"
SEED = 20261012
SAMPLE = 100
CALL_SIZE = 10
WINDOW = 8
CODES = ("S", "O", "L", "B", "T", "F", "U", "A")
OTHER_WORDS_MAX = 12
AGENT_TYPE = "okf-wiki-diagnoser"
FENCE = re.compile(r"^\s*(```|~~~)") # the fence rule of okfwiki.kblocks.headings_of
NUMBERED = re.compile(r"^\s*(\d+)\s*:(.*)$")
FORM = re.compile(r"^\s*([A-Z])\s*(?:\|\s*(.*?))?\s*$")
UNSUPPORTED = "unsupported"
SUPPORTED = "supported"
GROUPS = ("unsupported", "supported_sample")
def read_json(path: Path) -> Any:
try:
return json.loads(path.read_text(encoding="utf-8"))
except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc:
raise RunError(f"cannot read {path}: {exc}") from exc
def dump(doc: Any) -> str:
return json.dumps(doc, indent=1, sort_keys=True, ensure_ascii=False) + "\n"
def sha256(path: Path) -> str:
return hashlib.sha256(path.read_bytes()).hexdigest()
def verdict_map(path: Path) -> dict[str, str]:
return {str(v["id"]): str(v["verdict"]) for v in read_json(path)["verdicts"]}
# --- select ----------------------------------------------------------------------------------------------------
def select(claim_ids: list[str], verdicts: dict[str, str], sample: int) -> list[str]:
rejected: list[str] = []
accepted: list[str] = []
for cid in claim_ids:
v = verdicts.get(cid)
if v not in (SUPPORTED, UNSUPPORTED):
raise RunError(f"claim {cid} has no verdict of {SUPPORTED} or {UNSUPPORTED}")
(rejected if v == UNSUPPORTED else accepted).append(cid)
if len(accepted) < sample:
raise RunError(f"{len(accepted)} accepted claims, fewer than the sample of {sample}")
rng = random.Random(SEED)
picked = rng.sample(sorted(accepted), sample)
ids = sorted(rejected) + sorted(picked)
rng.shuffle(ids)
return ids
# --- render ----------------------------------------------------------------------------------------------------
def code_lines(lines: list[str]) -> set[int]:
out: set[int] = set()
fence: str | None = None
for n, raw in enumerate(lines, 1):
f = FENCE.match(raw)
if f:
out.add(n)
fence = None if fence == f.group(1) else (fence or f.group(1))
continue
if fence is not None:
out.add(n)
return out
def window(lines: list[str], name: str, line: int) -> str:
first, last = max(1, line - WINDOW), min(len(lines), line + WINDOW)
code = code_lines(lines)
path = " › ".join(claim_features.heading_path(lines, line)).replace('"', "'")
rows = [f"{n:>5} {'>' if n == line else ' '}{'c' if n in code else ' '}| {lines[n - 1]}".rstrip()
for n in range(first, last + 1)]
return "\n".join([f'<window file="{name}" lines="{first}-{last}" headings="{path}">', *rows, "</window>"])
def render_task(claims: list[dict[str, Any]], titles: dict[str, str], lines_of: dict[str, list[str]]) -> str:
n = len(claims)
parts = ["# Diagnosis task", "", f"This task has {n} claim{'s' if n != 1 else ''}, numbered 1 to {n}."]
for number, c in enumerate(claims, 1):
parts += ["", f"## Claim {number}", "", f"Page title: {titles[str(c['page'])]}", "",
f"<claim>{c['claim']}</claim>"]
for e in c["excerpts"]:
parts += ["", f'<excerpt file="{e["file"]}" line="{e["line"]}">', str(e["text"]), "</excerpt>"]
for e in c["excerpts"]:
parts += ["", window(lines_of[str(e["file"])], str(e["file"]), int(e["line"]))]
return "\n".join(parts) + "\n"
def render(ids: list[str], claims_doc: dict[str, Any], sources: Path, wiki: Path, size: int,
out: Path) -> list[dict[str, Any]]:
if size < 1:
raise RunError("--size must be at least 1")
if out.exists() and any(out.iterdir()):
raise RunError(f"{out} holds files; a task folder is never written into")
by_id = {str(c["id"]): c for c in claims_doc["claims"]}
missing = [i for i in ids if i not in by_id]
if missing:
raise RunError(f"selected ids not in the claim list: {missing[:5]}")
kilde = kblocks.kilde_of(sources)
lines_of: dict[str, list[str]] = {}
titles: dict[str, str] = {}
for cid in ids:
c = by_id[cid]
page = str(c["page"])
if page not in titles:
titles[page] = claim_features.page_title(wiki, page)
for e in c["excerpts"]:
name = str(e["file"])
if name not in lines_of:
if not (kilde / name).is_file():
raise RunError(f"the source {kilde / name} is not there")
lines_of[name] = kblocks.read_lines(kilde / name)
chunks = [(f"d{k + 1:03d}", ids[i:i + size]) for k, i in enumerate(range(0, len(ids), size))]
texts = {call: render_task([by_id[i] for i in chunk], titles, lines_of) for call, chunk in chunks}
for call, text in texts.items():
write_new(out / f"{call}.md", text)
return [{"name": call, "ids": chunk} for call, chunk in chunks]
# --- collect ---------------------------------------------------------------------------------------------------
def agent_replies(transcripts: Path, run: str) -> dict[str, list[Path]]:
prefix = re.compile(rf"^V1-{re.escape(run)}-diagnose-(\S+)$")
found: dict[str, list[Path]] = {}
metas = [*transcripts.glob("*/subagents/agent-*.meta.json"),
*transcripts.glob("*/subagents/workflows/*/agent-*.meta.json")]
for meta_path in sorted(metas):
try:
meta = json.loads(meta_path.read_text(encoding="utf-8"))
except (OSError, ValueError):
continue
if not isinstance(meta, dict) or not is_agent_type(meta.get("agentType"), AGENT_TYPE):
continue
m = prefix.match(str(meta.get("description", "")))
if m is not None:
found.setdefault(m.group(1), []).append(meta_path.with_name(
meta_path.name.removesuffix(".meta.json") + ".jsonl"))
return found
def parse_reply(reply: str, n: int) -> dict[int, tuple[str, str]]:
"""The code and missing words per claim number; a broken line or a number given twice leaves that claim out."""
given: dict[int, tuple[str, str]] = {}
bad: set[int] = set()
for raw in reply.splitlines():
m = NUMBERED.match(raw)
if m is None:
continue
number = int(m.group(1))
if not 1 <= number <= n:
continue
if number in given or number in bad:
given.pop(number, None)
bad.add(number)
continue
f = FORM.match(m.group(2))
code, words = (f.group(1), (f.group(2) or "").strip()) if f else ("", "")
ok = (code in CODES and (code == "S" or bool(words))
and (code != "A" or len(words.split()) <= OTHER_WORDS_MAX))
if ok:
given[number] = (code, words)
else:
bad.add(number)
return given
def collect(calls: list[dict[str, Any]], runs: list[str], transcripts: Path) -> dict[str, Any]:
found = {r: agent_replies(transcripts, r) for r in runs}
rows: list[dict[str, Any]] = []
for call in calls:
name, ids = str(call["name"]), [str(i) for i in call["ids"]]
answering = [r for r in runs if name in found[r]]
run = answering[-1] if answering else None
paths = found[run][name] if run else []
given = parse_reply(final_reply(paths[0]) if paths[0].is_file() else "", len(ids)) if len(paths) == 1 else {}
for number, cid in enumerate(ids, 1):
code, words = given.get(number, (None, ""))
rows.append({"id": cid, "call": name, "number": number, "code": code, "words": words, "run": run})
return {"schema": DIAGNOSES_SCHEMA, "runs": runs, "diagnoses": rows,
"missing": [r["id"] for r in rows if r["code"] is None]}
# --- tally -----------------------------------------------------------------------------------------------------
def _binom_tail(k_from: int, k_to: int, n: int, p: float) -> float:
return sum(math.exp(math.lgamma(n + 1) - math.lgamma(i + 1) - math.lgamma(n - i + 1)
+ i * math.log(p) + (n - i) * math.log1p(-p)) for i in range(k_from, k_to + 1))
def _solve(f: Any, rising: bool) -> float:
lo, hi = 0.0, 1.0
for _ in range(200):
mid = (lo + hi) / 2
if mid <= 0.0 or mid >= 1.0:
break
if (f(mid) < 0.025) == rising:
lo = mid
else:
hi = mid
return (lo + hi) / 2
def clopper_pearson(k: int, n: int) -> tuple[float, float]:
"""The two-sided 95 % Clopper-Pearson interval for k events in n."""
if n <= 0:
return 0.0, 1.0
lower = 0.0 if k == 0 else _solve(lambda p: _binom_tail(k, n, n, p), rising=True)
upper = 1.0 if k >= n else _solve(lambda p: _binom_tail(0, k, n, p), rising=False)
return lower, upper
def counts(rows: list[dict[str, Any]]) -> dict[str, int]:
out = {code: 0 for code in CODES}
out["missing"] = 0
for r in rows:
out[r["code"] if r["code"] in CODES else "missing"] += 1
out["n"] = len(rows)
return out
def tally(diagnoses: dict[str, Any], verdicts: dict[str, str], claims_doc: dict[str, Any]) -> dict[str, Any]:
by_id = {str(c["id"]): c for c in claims_doc["claims"]}
grouped: dict[str, list[dict[str, Any]]] = {g: [] for g in GROUPS}
for row in diagnoses["diagnoses"]:
v = verdicts.get(str(row["id"]))
if v not in (SUPPORTED, UNSUPPORTED) or str(row["id"]) not in by_id:
raise RunError(f"diagnosed claim {row['id']} has no verdict or is not in the claim list")
grouped["unsupported" if v == UNSUPPORTED else "supported_sample"].append(row)
def split(key_of: Any) -> dict[str, dict[str, dict[str, int]]]:
keys = sorted({k for g in grouped.values() for r in g for k in key_of(by_id[str(r["id"])])})
return {k: {g: counts([r for r in grouped[g] if k in key_of(by_id[str(r["id"])])]) for g in GROUPS}
for k in keys}
rej, acc = grouped["unsupported"], grouped["supported_sample"]
rej_done = [r for r in rej if r["code"] in CODES]
acc_done = [r for r in acc if r["code"] in CODES]
called_s = sum(1 for r in rej_done if r["code"] == "S")
not_carried = sum(1 for r in acc_done if r["code"] != "S")
missing = sum(1 for g in grouped.values() for r in g if r["code"] not in CODES)
return {
"schema": TALLY_SCHEMA,
"unsupported": counts(rej), "supported_sample": counts(acc),
"agreement": {"unsupported_called_s": called_s, "unsupported_diagnosed": len(rej_done),
"unsupported_called_s_ci95": list(clopper_pearson(called_s, len(rej_done))),
"supported_not_carried": not_carried, "supported_diagnosed": len(acc_done),
"supported_not_carried_ci95": list(clopper_pearson(not_carried, len(acc_done)))},
"per_source": split(lambda c: sorted({str(e["file"]) for e in c["excerpts"]})),
"per_provenance": split(lambda c: [str(c.get("provenance"))]),
"m1": {"s": called_s, "of": len(rej), "missing": missing, "stop": 3 * called_s >= len(rej) or missing > 0},
}
def tally_text(doc: dict[str, Any]) -> str:
out = []
for g in GROUPS:
c = doc[g]
out.append(f"{g}: n {c['n']}; " + ", ".join(f"{k} {c[k]}" for k in (*CODES, "missing")))
a = doc["agreement"]
lo, hi = a["supported_not_carried_ci95"]
out.append(f"accepted by the reader, not carried by the excerpt per the diagnosis: {a['supported_not_carried']} "
f"of {a['supported_diagnosed']} (95 % CI {100 * lo:.1f}-{100 * hi:.1f} %)")
lo, hi = a["unsupported_called_s_ci95"]
out.append(f"rejected by the reader, carried per the diagnosis (S): {a['unsupported_called_s']} of "
f"{a['unsupported_diagnosed']} (95 % CI {100 * lo:.1f}-{100 * hi:.1f} %)")
m = doc["m1"]
verdict = "STOP and present" if m["stop"] else "continue"
out.append(f"M1 rule: S for {m['s']} of {m['of']} rejected, {m['missing']} missing: {verdict}")
return "\n".join(out) + "\n"
# --- command line ----------------------------------------------------------------------------------------------
def build_parser() -> argparse.ArgumentParser:
p = argparse.ArgumentParser(prog="python3 tools/diagnose.py", description=__doc__,
formatter_class=argparse.RawDescriptionHelpFormatter)
sub = p.add_subparsers(dest="command", required=True)
s = sub.add_parser("select")
s.add_argument("--claims", required=True)
s.add_argument("--verdicts", required=True)
s.add_argument("--out", required=True)
s.add_argument("--sample", type=int, default=SAMPLE)
r = sub.add_parser("render")
r.add_argument("--utvalg", required=True)
r.add_argument("--claims", required=True)
r.add_argument("--sources", required=True)
r.add_argument("--wiki", required=True)
r.add_argument("--out", required=True)
r.add_argument("--size", type=int, default=CALL_SIZE)
c = sub.add_parser("collect")
c.add_argument("--transcripts", required=True)
c.add_argument("--run", action="append", required=True)
c.add_argument("--tasks", required=True)
c.add_argument("--out", required=True)
t = sub.add_parser("tally")
t.add_argument("--diagnoses", required=True)
t.add_argument("--verdicts", required=True)
t.add_argument("--claims", required=True)
t.add_argument("--out", required=True)
return p
def main(argv: list[str] | None = None) -> int:
args = build_parser().parse_args(argv)
try:
out = Path(args.out)
if args.command == "select":
if out.exists():
raise RunError(f"{out} exists; an output is never overwritten")
claims_doc = read_json(Path(args.claims))
ids = select([str(c["id"]) for c in claims_doc["claims"]], verdict_map(Path(args.verdicts)), args.sample)
write_new(out, dump({"schema": SELECTION_SCHEMA, "seed": SEED, "sample": args.sample, "ids": ids}))
print(f"select: {len(ids)} claims ({args.sample} of them a sample) -> {out}")
return 0
if args.command == "render":
utvalg = Path(args.utvalg)
ids = [str(i) for i in read_json(utvalg)["ids"]]
calls = render(ids, read_json(Path(args.claims)), Path(args.sources), Path(args.wiki), args.size, out)
write_new(out / "calls.json", dump({"schema": CALLS_SCHEMA, "utvalg_sha256": sha256(utvalg),
"size": args.size, "calls": calls}))
print(f"render: {len(ids)} claims in {len(calls)} calls -> {out}")
return 0
if args.command == "collect":
if out.exists():
raise RunError(f"{out} exists; an output is never overwritten")
calls = read_json(Path(args.tasks) / "calls.json")["calls"]
doc = collect(calls, list(args.run), Path(args.transcripts))
write_new(out, dump(doc))
print(f"collect: {len(doc['diagnoses']) - len(doc['missing'])} of {len(doc['diagnoses'])} claims "
f"diagnosed, {len(doc['missing'])} missing -> {out}")
return 1 if doc["missing"] else 0
if out.exists():
raise RunError(f"{out} exists; an output is never overwritten")
doc = tally(read_json(Path(args.diagnoses)), verdict_map(Path(args.verdicts)), read_json(Path(args.claims)))
write_new(out, dump(doc))
print(tally_text(doc), end="")
return 1 if doc["m1"]["stop"] else 0
except (RunError, OSError, KeyError, TypeError, ValueError) as exc:
print(f"diagnose cannot run: {exc}", file=sys.stderr)
return 2
if __name__ == "__main__":
sys.exit(main())