okf-wiki/tools/diagnose.py
Kjell Tore Guttormsen 0fd86a23a3
okf-wiki 1.1.1
First public release, published as one snapshot. What changed between
versions is in CHANGELOG.md.

Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
2026-11-14 21:03:20 +01:00

421 lines
20 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

"""Why a claim is or is not carried by its excerpt: the tool around the diagnosis of wave 4 order 20 (contract K9).
No model call. The diagnosis explains; the reader judges. It is not calibrated and never counts for criterion 2.
Usage:
python3 tools/diagnose.py select --claims claims.json --verdicts verdicts.json --out utvalg.json [--sample 100]
python3 tools/diagnose.py render --utvalg utvalg.json --claims claims.json --sources SOURCES --wiki WIKI
--out tasks/ [--size 10]
python3 tools/diagnose.py collect --transcripts DIR --run R [--run R2]... --tasks tasks/ --out diagnoses.json
python3 tools/diagnose.py tally --diagnoses diagnoses.json --verdicts verdicts.json --claims claims.json
--out tally.json
select every claim the reader did not accept, and a sample of the accepted ones:
random.Random(SEED).sample(sorted(ids), --sample), then the two lists (each sorted) shuffled with the same
generator. The selection holds ids only, never a verdict. A claim without a verdict, or fewer accepted claims
than the sample, cannot run.
render one task per call of at most --size claims, in the selection's order, named d001, d002, ...; and
calls.json (the ids per call). Per claim: the page title, the claim, each excerpt it cites, and per excerpt a
window in its source: the heading path above the quoted line (okfwiki.kblocks.headings_of, as
tools/claim_features.py takes it) and WINDOW lines before and after it, cut at the file's ends. Each window
line is "<line number> <q><c>| <text>", q ">" on the quoted line, c "c" on a line of fenced code (the fence
lines too). No id, verdict or provenance mark reaches the task. A task folder that holds anything is never
written into.
collect the reply of every diagnosis agent (agent type okf-wiki-diagnoser, description "V1-<run>-diagnose-<call>",
started from the session or from a workflow, the two places okfwiki.cost harvest reads): the message it
handed back, or else its last text (okfwiki.reader.final_reply). For each call the last --run that answered
it counts; two transcripts of that run for one call give no diagnosis for any of its claims. A line
"<number>: <code> | <words>" gives the claim of that number its code (one of CODES) and the words that are
missing; S needs no words, every other code does, and A has at most twelve. A line that starts with a number
but breaks the form, or a number given twice, gives that claim no diagnosis: it is "missing", never guessed.
Other lines are ignored.
tally counts per code for the claims the reader did not accept and for the sample it accepted, each apart; how
often the diagnosis and the reader disagree (S on a rejected claim; any other code on an accepted one) with a
two-sided 95 % Clopper-Pearson interval; the same counts per source file and per provenance mark; and the M1
rule of order 20: S for at least a third of the rejected claims, or any claim missing, is stop and present.
Codes (locked in phase 0 before any diagnosis was read): S the excerpt alone carries the claim; O what is missing
stands in the heading path or the page title; L it stands in another line of the window; B the claim says more than
the source (wider scope, stronger words); T a number, version, order, cause or purpose that is not in the window; F a
property given to the wrong thing, or a wrong connection; U what is missing is not in the window at all; A other.
Output: json.dumps(doc, indent=1, sort_keys=True, ensure_ascii=False) + newline; an existing output is never
overwritten. Exit 0 written; 1 written with claims missing (collect) or the M1 rule says stop (tally); 2 cannot run.
"""
from __future__ import annotations
import argparse
import hashlib
import json
import math
import random
import re
import sys
from pathlib import Path
from typing import Any
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
sys.path.insert(0, str(Path(__file__).resolve().parent))
import claim_features # noqa: E402
from okfwiki import kblocks # noqa: E402
from okfwiki.common import RunError, write_new # noqa: E402
from okfwiki.reader import final_reply # noqa: E402
from okfwiki.transcript import is_agent_type # noqa: E402
SELECTION_SCHEMA = "okf-wiki/diagnose-selection/1"
CALLS_SCHEMA = "okf-wiki/diagnose-calls/1"
DIAGNOSES_SCHEMA = "okf-wiki/diagnoses/1"
TALLY_SCHEMA = "okf-wiki/diagnose-tally/1"
SEED = 20261012
SAMPLE = 100
CALL_SIZE = 10
WINDOW = 8
CODES = ("S", "O", "L", "B", "T", "F", "U", "A")
OTHER_WORDS_MAX = 12
AGENT_TYPE = "okf-wiki-diagnoser"
FENCE = re.compile(r"^\s*(```|~~~)") # the fence rule of okfwiki.kblocks.headings_of
NUMBERED = re.compile(r"^\s*(\d+)\s*:(.*)$")
FORM = re.compile(r"^\s*([A-Z])\s*(?:\|\s*(.*?))?\s*$")
UNSUPPORTED = "unsupported"
SUPPORTED = "supported"
GROUPS = ("unsupported", "supported_sample")
def read_json(path: Path) -> Any:
try:
return json.loads(path.read_text(encoding="utf-8"))
except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc:
raise RunError(f"cannot read {path}: {exc}") from exc
def dump(doc: Any) -> str:
return json.dumps(doc, indent=1, sort_keys=True, ensure_ascii=False) + "\n"
def sha256(path: Path) -> str:
return hashlib.sha256(path.read_bytes()).hexdigest()
def verdict_map(path: Path) -> dict[str, str]:
return {str(v["id"]): str(v["verdict"]) for v in read_json(path)["verdicts"]}
# --- select ----------------------------------------------------------------------------------------------------
def select(claim_ids: list[str], verdicts: dict[str, str], sample: int) -> list[str]:
rejected: list[str] = []
accepted: list[str] = []
for cid in claim_ids:
v = verdicts.get(cid)
if v not in (SUPPORTED, UNSUPPORTED):
raise RunError(f"claim {cid} has no verdict of {SUPPORTED} or {UNSUPPORTED}")
(rejected if v == UNSUPPORTED else accepted).append(cid)
if len(accepted) < sample:
raise RunError(f"{len(accepted)} accepted claims, fewer than the sample of {sample}")
rng = random.Random(SEED)
picked = rng.sample(sorted(accepted), sample)
ids = sorted(rejected) + sorted(picked)
rng.shuffle(ids)
return ids
# --- render ----------------------------------------------------------------------------------------------------
def code_lines(lines: list[str]) -> set[int]:
out: set[int] = set()
fence: str | None = None
for n, raw in enumerate(lines, 1):
f = FENCE.match(raw)
if f:
out.add(n)
fence = None if fence == f.group(1) else (fence or f.group(1))
continue
if fence is not None:
out.add(n)
return out
def window(lines: list[str], name: str, line: int) -> str:
first, last = max(1, line - WINDOW), min(len(lines), line + WINDOW)
code = code_lines(lines)
path = " › ".join(claim_features.heading_path(lines, line)).replace('"', "'")
rows = [f"{n:>5} {'>' if n == line else ' '}{'c' if n in code else ' '}| {lines[n - 1]}".rstrip()
for n in range(first, last + 1)]
return "\n".join([f'<window file="{name}" lines="{first}-{last}" headings="{path}">', *rows, "</window>"])
def render_task(claims: list[dict[str, Any]], titles: dict[str, str], lines_of: dict[str, list[str]]) -> str:
n = len(claims)
parts = ["# Diagnosis task", "", f"This task has {n} claim{'s' if n != 1 else ''}, numbered 1 to {n}."]
for number, c in enumerate(claims, 1):
parts += ["", f"## Claim {number}", "", f"Page title: {titles[str(c['page'])]}", "",
f"<claim>{c['claim']}</claim>"]
for e in c["excerpts"]:
parts += ["", f'<excerpt file="{e["file"]}" line="{e["line"]}">', str(e["text"]), "</excerpt>"]
for e in c["excerpts"]:
parts += ["", window(lines_of[str(e["file"])], str(e["file"]), int(e["line"]))]
return "\n".join(parts) + "\n"
def render(ids: list[str], claims_doc: dict[str, Any], sources: Path, wiki: Path, size: int,
out: Path) -> list[dict[str, Any]]:
if size < 1:
raise RunError("--size must be at least 1")
if out.exists() and any(out.iterdir()):
raise RunError(f"{out} holds files; a task folder is never written into")
by_id = {str(c["id"]): c for c in claims_doc["claims"]}
missing = [i for i in ids if i not in by_id]
if missing:
raise RunError(f"selected ids not in the claim list: {missing[:5]}")
kilde = kblocks.kilde_of(sources)
lines_of: dict[str, list[str]] = {}
titles: dict[str, str] = {}
for cid in ids:
c = by_id[cid]
page = str(c["page"])
if page not in titles:
titles[page] = claim_features.page_title(wiki, page)
for e in c["excerpts"]:
name = str(e["file"])
if name not in lines_of:
if not (kilde / name).is_file():
raise RunError(f"the source {kilde / name} is not there")
lines_of[name] = kblocks.read_lines(kilde / name)
chunks = [(f"d{k + 1:03d}", ids[i:i + size]) for k, i in enumerate(range(0, len(ids), size))]
texts = {call: render_task([by_id[i] for i in chunk], titles, lines_of) for call, chunk in chunks}
for call, text in texts.items():
write_new(out / f"{call}.md", text)
return [{"name": call, "ids": chunk} for call, chunk in chunks]
# --- collect ---------------------------------------------------------------------------------------------------
def agent_replies(transcripts: Path, run: str) -> dict[str, list[Path]]:
prefix = re.compile(rf"^V1-{re.escape(run)}-diagnose-(\S+)$")
found: dict[str, list[Path]] = {}
metas = [*transcripts.glob("*/subagents/agent-*.meta.json"),
*transcripts.glob("*/subagents/workflows/*/agent-*.meta.json")]
for meta_path in sorted(metas):
try:
meta = json.loads(meta_path.read_text(encoding="utf-8"))
except (OSError, ValueError):
continue
if not isinstance(meta, dict) or not is_agent_type(meta.get("agentType"), AGENT_TYPE):
continue
m = prefix.match(str(meta.get("description", "")))
if m is not None:
found.setdefault(m.group(1), []).append(meta_path.with_name(
meta_path.name.removesuffix(".meta.json") + ".jsonl"))
return found
def parse_reply(reply: str, n: int) -> dict[int, tuple[str, str]]:
"""The code and missing words per claim number; a broken line or a number given twice leaves that claim out."""
given: dict[int, tuple[str, str]] = {}
bad: set[int] = set()
for raw in reply.splitlines():
m = NUMBERED.match(raw)
if m is None:
continue
number = int(m.group(1))
if not 1 <= number <= n:
continue
if number in given or number in bad:
given.pop(number, None)
bad.add(number)
continue
f = FORM.match(m.group(2))
code, words = (f.group(1), (f.group(2) or "").strip()) if f else ("", "")
ok = (code in CODES and (code == "S" or bool(words))
and (code != "A" or len(words.split()) <= OTHER_WORDS_MAX))
if ok:
given[number] = (code, words)
else:
bad.add(number)
return given
def collect(calls: list[dict[str, Any]], runs: list[str], transcripts: Path) -> dict[str, Any]:
found = {r: agent_replies(transcripts, r) for r in runs}
rows: list[dict[str, Any]] = []
for call in calls:
name, ids = str(call["name"]), [str(i) for i in call["ids"]]
answering = [r for r in runs if name in found[r]]
run = answering[-1] if answering else None
paths = found[run][name] if run else []
given = parse_reply(final_reply(paths[0]) if paths[0].is_file() else "", len(ids)) if len(paths) == 1 else {}
for number, cid in enumerate(ids, 1):
code, words = given.get(number, (None, ""))
rows.append({"id": cid, "call": name, "number": number, "code": code, "words": words, "run": run})
return {"schema": DIAGNOSES_SCHEMA, "runs": runs, "diagnoses": rows,
"missing": [r["id"] for r in rows if r["code"] is None]}
# --- tally -----------------------------------------------------------------------------------------------------
def _binom_tail(k_from: int, k_to: int, n: int, p: float) -> float:
return sum(math.exp(math.lgamma(n + 1) - math.lgamma(i + 1) - math.lgamma(n - i + 1)
+ i * math.log(p) + (n - i) * math.log1p(-p)) for i in range(k_from, k_to + 1))
def _solve(f: Any, rising: bool) -> float:
lo, hi = 0.0, 1.0
for _ in range(200):
mid = (lo + hi) / 2
if mid <= 0.0 or mid >= 1.0:
break
if (f(mid) < 0.025) == rising:
lo = mid
else:
hi = mid
return (lo + hi) / 2
def clopper_pearson(k: int, n: int) -> tuple[float, float]:
"""The two-sided 95 % Clopper-Pearson interval for k events in n."""
if n <= 0:
return 0.0, 1.0
lower = 0.0 if k == 0 else _solve(lambda p: _binom_tail(k, n, n, p), rising=True)
upper = 1.0 if k >= n else _solve(lambda p: _binom_tail(0, k, n, p), rising=False)
return lower, upper
def counts(rows: list[dict[str, Any]]) -> dict[str, int]:
out = {code: 0 for code in CODES}
out["missing"] = 0
for r in rows:
out[r["code"] if r["code"] in CODES else "missing"] += 1
out["n"] = len(rows)
return out
def tally(diagnoses: dict[str, Any], verdicts: dict[str, str], claims_doc: dict[str, Any]) -> dict[str, Any]:
by_id = {str(c["id"]): c for c in claims_doc["claims"]}
grouped: dict[str, list[dict[str, Any]]] = {g: [] for g in GROUPS}
for row in diagnoses["diagnoses"]:
v = verdicts.get(str(row["id"]))
if v not in (SUPPORTED, UNSUPPORTED) or str(row["id"]) not in by_id:
raise RunError(f"diagnosed claim {row['id']} has no verdict or is not in the claim list")
grouped["unsupported" if v == UNSUPPORTED else "supported_sample"].append(row)
def split(key_of: Any) -> dict[str, dict[str, dict[str, int]]]:
keys = sorted({k for g in grouped.values() for r in g for k in key_of(by_id[str(r["id"])])})
return {k: {g: counts([r for r in grouped[g] if k in key_of(by_id[str(r["id"])])]) for g in GROUPS}
for k in keys}
rej, acc = grouped["unsupported"], grouped["supported_sample"]
rej_done = [r for r in rej if r["code"] in CODES]
acc_done = [r for r in acc if r["code"] in CODES]
called_s = sum(1 for r in rej_done if r["code"] == "S")
not_carried = sum(1 for r in acc_done if r["code"] != "S")
missing = sum(1 for g in grouped.values() for r in g if r["code"] not in CODES)
return {
"schema": TALLY_SCHEMA,
"unsupported": counts(rej), "supported_sample": counts(acc),
"agreement": {"unsupported_called_s": called_s, "unsupported_diagnosed": len(rej_done),
"unsupported_called_s_ci95": list(clopper_pearson(called_s, len(rej_done))),
"supported_not_carried": not_carried, "supported_diagnosed": len(acc_done),
"supported_not_carried_ci95": list(clopper_pearson(not_carried, len(acc_done)))},
"per_source": split(lambda c: sorted({str(e["file"]) for e in c["excerpts"]})),
"per_provenance": split(lambda c: [str(c.get("provenance"))]),
"m1": {"s": called_s, "of": len(rej), "missing": missing, "stop": 3 * called_s >= len(rej) or missing > 0},
}
def tally_text(doc: dict[str, Any]) -> str:
out = []
for g in GROUPS:
c = doc[g]
out.append(f"{g}: n {c['n']}; " + ", ".join(f"{k} {c[k]}" for k in (*CODES, "missing")))
a = doc["agreement"]
lo, hi = a["supported_not_carried_ci95"]
out.append(f"accepted by the reader, not carried by the excerpt per the diagnosis: {a['supported_not_carried']} "
f"of {a['supported_diagnosed']} (95 % CI {100 * lo:.1f}-{100 * hi:.1f} %)")
lo, hi = a["unsupported_called_s_ci95"]
out.append(f"rejected by the reader, carried per the diagnosis (S): {a['unsupported_called_s']} of "
f"{a['unsupported_diagnosed']} (95 % CI {100 * lo:.1f}-{100 * hi:.1f} %)")
m = doc["m1"]
verdict = "STOP and present" if m["stop"] else "continue"
out.append(f"M1 rule: S for {m['s']} of {m['of']} rejected, {m['missing']} missing: {verdict}")
return "\n".join(out) + "\n"
# --- command line ----------------------------------------------------------------------------------------------
def build_parser() -> argparse.ArgumentParser:
p = argparse.ArgumentParser(prog="python3 tools/diagnose.py", description=__doc__,
formatter_class=argparse.RawDescriptionHelpFormatter)
sub = p.add_subparsers(dest="command", required=True)
s = sub.add_parser("select")
s.add_argument("--claims", required=True)
s.add_argument("--verdicts", required=True)
s.add_argument("--out", required=True)
s.add_argument("--sample", type=int, default=SAMPLE)
r = sub.add_parser("render")
r.add_argument("--utvalg", required=True)
r.add_argument("--claims", required=True)
r.add_argument("--sources", required=True)
r.add_argument("--wiki", required=True)
r.add_argument("--out", required=True)
r.add_argument("--size", type=int, default=CALL_SIZE)
c = sub.add_parser("collect")
c.add_argument("--transcripts", required=True)
c.add_argument("--run", action="append", required=True)
c.add_argument("--tasks", required=True)
c.add_argument("--out", required=True)
t = sub.add_parser("tally")
t.add_argument("--diagnoses", required=True)
t.add_argument("--verdicts", required=True)
t.add_argument("--claims", required=True)
t.add_argument("--out", required=True)
return p
def main(argv: list[str] | None = None) -> int:
args = build_parser().parse_args(argv)
try:
out = Path(args.out)
if args.command == "select":
if out.exists():
raise RunError(f"{out} exists; an output is never overwritten")
claims_doc = read_json(Path(args.claims))
ids = select([str(c["id"]) for c in claims_doc["claims"]], verdict_map(Path(args.verdicts)), args.sample)
write_new(out, dump({"schema": SELECTION_SCHEMA, "seed": SEED, "sample": args.sample, "ids": ids}))
print(f"select: {len(ids)} claims ({args.sample} of them a sample) -> {out}")
return 0
if args.command == "render":
utvalg = Path(args.utvalg)
ids = [str(i) for i in read_json(utvalg)["ids"]]
calls = render(ids, read_json(Path(args.claims)), Path(args.sources), Path(args.wiki), args.size, out)
write_new(out / "calls.json", dump({"schema": CALLS_SCHEMA, "utvalg_sha256": sha256(utvalg),
"size": args.size, "calls": calls}))
print(f"render: {len(ids)} claims in {len(calls)} calls -> {out}")
return 0
if args.command == "collect":
if out.exists():
raise RunError(f"{out} exists; an output is never overwritten")
calls = read_json(Path(args.tasks) / "calls.json")["calls"]
doc = collect(calls, list(args.run), Path(args.transcripts))
write_new(out, dump(doc))
print(f"collect: {len(doc['diagnoses']) - len(doc['missing'])} of {len(doc['diagnoses'])} claims "
f"diagnosed, {len(doc['missing'])} missing -> {out}")
return 1 if doc["missing"] else 0
if out.exists():
raise RunError(f"{out} exists; an output is never overwritten")
doc = tally(read_json(Path(args.diagnoses)), verdict_map(Path(args.verdicts)), read_json(Path(args.claims)))
write_new(out, dump(doc))
print(tally_text(doc), end="")
return 1 if doc["m1"]["stop"] else 0
except (RunError, OSError, KeyError, TypeError, ValueError) as exc:
print(f"diagnose cannot run: {exc}", file=sys.stderr)
return 2
if __name__ == "__main__":
sys.exit(main())