"""Why a claim is or is not carried by its excerpt: the tool around the diagnosis of wave 4 order 20 (contract K9). No model call. The diagnosis explains; the reader judges. It is not calibrated and never counts for criterion 2. Usage: python3 tools/diagnose.py select --claims claims.json --verdicts verdicts.json --out utvalg.json [--sample 100] python3 tools/diagnose.py render --utvalg utvalg.json --claims claims.json --sources SOURCES --wiki WIKI --out tasks/ [--size 10] python3 tools/diagnose.py collect --transcripts DIR --run R [--run R2]... --tasks tasks/ --out diagnoses.json python3 tools/diagnose.py tally --diagnoses diagnoses.json --verdicts verdicts.json --claims claims.json --out tally.json select every claim the reader did not accept, and a sample of the accepted ones: random.Random(SEED).sample(sorted(ids), --sample), then the two lists (each sorted) shuffled with the same generator. The selection holds ids only, never a verdict. A claim without a verdict, or fewer accepted claims than the sample, cannot run. render one task per call of at most --size claims, in the selection's order, named d001, d002, ...; and calls.json (the ids per call). Per claim: the page title, the claim, each excerpt it cites, and per excerpt a window in its source: the heading path above the quoted line (okfwiki.kblocks.headings_of, as tools/claim_features.py takes it) and WINDOW lines before and after it, cut at the file's ends. Each window line is " | ", q ">" on the quoted line, c "c" on a line of fenced code (the fence lines too). No id, verdict or provenance mark reaches the task. A task folder that holds anything is never written into. collect the reply of every diagnosis agent (agent type okf-wiki-diagnoser, description "V1--diagnose-", started from the session or from a workflow, the two places okfwiki.cost harvest reads): the message it handed back, or else its last text (okfwiki.reader.final_reply). For each call the last --run that answered it counts; two transcripts of that run for one call give no diagnosis for any of its claims. A line ": | " gives the claim of that number its code (one of CODES) and the words that are missing; S needs no words, every other code does, and A has at most twelve. A line that starts with a number but breaks the form, or a number given twice, gives that claim no diagnosis: it is "missing", never guessed. Other lines are ignored. tally counts per code for the claims the reader did not accept and for the sample it accepted, each apart; how often the diagnosis and the reader disagree (S on a rejected claim; any other code on an accepted one) with a two-sided 95 % Clopper-Pearson interval; the same counts per source file and per provenance mark; and the M1 rule of order 20: S for at least a third of the rejected claims, or any claim missing, is stop and present. Codes (locked in phase 0 before any diagnosis was read): S the excerpt alone carries the claim; O what is missing stands in the heading path or the page title; L it stands in another line of the window; B the claim says more than the source (wider scope, stronger words); T a number, version, order, cause or purpose that is not in the window; F a property given to the wrong thing, or a wrong connection; U what is missing is not in the window at all; A other. Output: json.dumps(doc, indent=1, sort_keys=True, ensure_ascii=False) + newline; an existing output is never overwritten. Exit 0 written; 1 written with claims missing (collect) or the M1 rule says stop (tally); 2 cannot run. """ from __future__ import annotations import argparse import hashlib import json import math import random import re import sys from pathlib import Path from typing import Any sys.path.insert(0, str(Path(__file__).resolve().parents[1])) sys.path.insert(0, str(Path(__file__).resolve().parent)) import claim_features # noqa: E402 from okfwiki import kblocks # noqa: E402 from okfwiki.common import RunError, write_new # noqa: E402 from okfwiki.reader import final_reply # noqa: E402 from okfwiki.transcript import is_agent_type # noqa: E402 SELECTION_SCHEMA = "okf-wiki/diagnose-selection/1" CALLS_SCHEMA = "okf-wiki/diagnose-calls/1" DIAGNOSES_SCHEMA = "okf-wiki/diagnoses/1" TALLY_SCHEMA = "okf-wiki/diagnose-tally/1" SEED = 20261012 SAMPLE = 100 CALL_SIZE = 10 WINDOW = 8 CODES = ("S", "O", "L", "B", "T", "F", "U", "A") OTHER_WORDS_MAX = 12 AGENT_TYPE = "okf-wiki-diagnoser" FENCE = re.compile(r"^\s*(```|~~~)") # the fence rule of okfwiki.kblocks.headings_of NUMBERED = re.compile(r"^\s*(\d+)\s*:(.*)$") FORM = re.compile(r"^\s*([A-Z])\s*(?:\|\s*(.*?))?\s*$") UNSUPPORTED = "unsupported" SUPPORTED = "supported" GROUPS = ("unsupported", "supported_sample") def read_json(path: Path) -> Any: try: return json.loads(path.read_text(encoding="utf-8")) except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc: raise RunError(f"cannot read {path}: {exc}") from exc def dump(doc: Any) -> str: return json.dumps(doc, indent=1, sort_keys=True, ensure_ascii=False) + "\n" def sha256(path: Path) -> str: return hashlib.sha256(path.read_bytes()).hexdigest() def verdict_map(path: Path) -> dict[str, str]: return {str(v["id"]): str(v["verdict"]) for v in read_json(path)["verdicts"]} # --- select ---------------------------------------------------------------------------------------------------- def select(claim_ids: list[str], verdicts: dict[str, str], sample: int) -> list[str]: rejected: list[str] = [] accepted: list[str] = [] for cid in claim_ids: v = verdicts.get(cid) if v not in (SUPPORTED, UNSUPPORTED): raise RunError(f"claim {cid} has no verdict of {SUPPORTED} or {UNSUPPORTED}") (rejected if v == UNSUPPORTED else accepted).append(cid) if len(accepted) < sample: raise RunError(f"{len(accepted)} accepted claims, fewer than the sample of {sample}") rng = random.Random(SEED) picked = rng.sample(sorted(accepted), sample) ids = sorted(rejected) + sorted(picked) rng.shuffle(ids) return ids # --- render ---------------------------------------------------------------------------------------------------- def code_lines(lines: list[str]) -> set[int]: out: set[int] = set() fence: str | None = None for n, raw in enumerate(lines, 1): f = FENCE.match(raw) if f: out.add(n) fence = None if fence == f.group(1) else (fence or f.group(1)) continue if fence is not None: out.add(n) return out def window(lines: list[str], name: str, line: int) -> str: first, last = max(1, line - WINDOW), min(len(lines), line + WINDOW) code = code_lines(lines) path = " › ".join(claim_features.heading_path(lines, line)).replace('"', "'") rows = [f"{n:>5} {'>' if n == line else ' '}{'c' if n in code else ' '}| {lines[n - 1]}".rstrip() for n in range(first, last + 1)] return "\n".join([f'', *rows, ""]) def render_task(claims: list[dict[str, Any]], titles: dict[str, str], lines_of: dict[str, list[str]]) -> str: n = len(claims) parts = ["# Diagnosis task", "", f"This task has {n} claim{'s' if n != 1 else ''}, numbered 1 to {n}."] for number, c in enumerate(claims, 1): parts += ["", f"## Claim {number}", "", f"Page title: {titles[str(c['page'])]}", "", f"{c['claim']}"] for e in c["excerpts"]: parts += ["", f'', str(e["text"]), ""] for e in c["excerpts"]: parts += ["", window(lines_of[str(e["file"])], str(e["file"]), int(e["line"]))] return "\n".join(parts) + "\n" def render(ids: list[str], claims_doc: dict[str, Any], sources: Path, wiki: Path, size: int, out: Path) -> list[dict[str, Any]]: if size < 1: raise RunError("--size must be at least 1") if out.exists() and any(out.iterdir()): raise RunError(f"{out} holds files; a task folder is never written into") by_id = {str(c["id"]): c for c in claims_doc["claims"]} missing = [i for i in ids if i not in by_id] if missing: raise RunError(f"selected ids not in the claim list: {missing[:5]}") kilde = kblocks.kilde_of(sources) lines_of: dict[str, list[str]] = {} titles: dict[str, str] = {} for cid in ids: c = by_id[cid] page = str(c["page"]) if page not in titles: titles[page] = claim_features.page_title(wiki, page) for e in c["excerpts"]: name = str(e["file"]) if name not in lines_of: if not (kilde / name).is_file(): raise RunError(f"the source {kilde / name} is not there") lines_of[name] = kblocks.read_lines(kilde / name) chunks = [(f"d{k + 1:03d}", ids[i:i + size]) for k, i in enumerate(range(0, len(ids), size))] texts = {call: render_task([by_id[i] for i in chunk], titles, lines_of) for call, chunk in chunks} for call, text in texts.items(): write_new(out / f"{call}.md", text) return [{"name": call, "ids": chunk} for call, chunk in chunks] # --- collect --------------------------------------------------------------------------------------------------- def agent_replies(transcripts: Path, run: str) -> dict[str, list[Path]]: prefix = re.compile(rf"^V1-{re.escape(run)}-diagnose-(\S+)$") found: dict[str, list[Path]] = {} metas = [*transcripts.glob("*/subagents/agent-*.meta.json"), *transcripts.glob("*/subagents/workflows/*/agent-*.meta.json")] for meta_path in sorted(metas): try: meta = json.loads(meta_path.read_text(encoding="utf-8")) except (OSError, ValueError): continue if not isinstance(meta, dict) or not is_agent_type(meta.get("agentType"), AGENT_TYPE): continue m = prefix.match(str(meta.get("description", ""))) if m is not None: found.setdefault(m.group(1), []).append(meta_path.with_name( meta_path.name.removesuffix(".meta.json") + ".jsonl")) return found def parse_reply(reply: str, n: int) -> dict[int, tuple[str, str]]: """The code and missing words per claim number; a broken line or a number given twice leaves that claim out.""" given: dict[int, tuple[str, str]] = {} bad: set[int] = set() for raw in reply.splitlines(): m = NUMBERED.match(raw) if m is None: continue number = int(m.group(1)) if not 1 <= number <= n: continue if number in given or number in bad: given.pop(number, None) bad.add(number) continue f = FORM.match(m.group(2)) code, words = (f.group(1), (f.group(2) or "").strip()) if f else ("", "") ok = (code in CODES and (code == "S" or bool(words)) and (code != "A" or len(words.split()) <= OTHER_WORDS_MAX)) if ok: given[number] = (code, words) else: bad.add(number) return given def collect(calls: list[dict[str, Any]], runs: list[str], transcripts: Path) -> dict[str, Any]: found = {r: agent_replies(transcripts, r) for r in runs} rows: list[dict[str, Any]] = [] for call in calls: name, ids = str(call["name"]), [str(i) for i in call["ids"]] answering = [r for r in runs if name in found[r]] run = answering[-1] if answering else None paths = found[run][name] if run else [] given = parse_reply(final_reply(paths[0]) if paths[0].is_file() else "", len(ids)) if len(paths) == 1 else {} for number, cid in enumerate(ids, 1): code, words = given.get(number, (None, "")) rows.append({"id": cid, "call": name, "number": number, "code": code, "words": words, "run": run}) return {"schema": DIAGNOSES_SCHEMA, "runs": runs, "diagnoses": rows, "missing": [r["id"] for r in rows if r["code"] is None]} # --- tally ----------------------------------------------------------------------------------------------------- def _binom_tail(k_from: int, k_to: int, n: int, p: float) -> float: return sum(math.exp(math.lgamma(n + 1) - math.lgamma(i + 1) - math.lgamma(n - i + 1) + i * math.log(p) + (n - i) * math.log1p(-p)) for i in range(k_from, k_to + 1)) def _solve(f: Any, rising: bool) -> float: lo, hi = 0.0, 1.0 for _ in range(200): mid = (lo + hi) / 2 if mid <= 0.0 or mid >= 1.0: break if (f(mid) < 0.025) == rising: lo = mid else: hi = mid return (lo + hi) / 2 def clopper_pearson(k: int, n: int) -> tuple[float, float]: """The two-sided 95 % Clopper-Pearson interval for k events in n.""" if n <= 0: return 0.0, 1.0 lower = 0.0 if k == 0 else _solve(lambda p: _binom_tail(k, n, n, p), rising=True) upper = 1.0 if k >= n else _solve(lambda p: _binom_tail(0, k, n, p), rising=False) return lower, upper def counts(rows: list[dict[str, Any]]) -> dict[str, int]: out = {code: 0 for code in CODES} out["missing"] = 0 for r in rows: out[r["code"] if r["code"] in CODES else "missing"] += 1 out["n"] = len(rows) return out def tally(diagnoses: dict[str, Any], verdicts: dict[str, str], claims_doc: dict[str, Any]) -> dict[str, Any]: by_id = {str(c["id"]): c for c in claims_doc["claims"]} grouped: dict[str, list[dict[str, Any]]] = {g: [] for g in GROUPS} for row in diagnoses["diagnoses"]: v = verdicts.get(str(row["id"])) if v not in (SUPPORTED, UNSUPPORTED) or str(row["id"]) not in by_id: raise RunError(f"diagnosed claim {row['id']} has no verdict or is not in the claim list") grouped["unsupported" if v == UNSUPPORTED else "supported_sample"].append(row) def split(key_of: Any) -> dict[str, dict[str, dict[str, int]]]: keys = sorted({k for g in grouped.values() for r in g for k in key_of(by_id[str(r["id"])])}) return {k: {g: counts([r for r in grouped[g] if k in key_of(by_id[str(r["id"])])]) for g in GROUPS} for k in keys} rej, acc = grouped["unsupported"], grouped["supported_sample"] rej_done = [r for r in rej if r["code"] in CODES] acc_done = [r for r in acc if r["code"] in CODES] called_s = sum(1 for r in rej_done if r["code"] == "S") not_carried = sum(1 for r in acc_done if r["code"] != "S") missing = sum(1 for g in grouped.values() for r in g if r["code"] not in CODES) return { "schema": TALLY_SCHEMA, "unsupported": counts(rej), "supported_sample": counts(acc), "agreement": {"unsupported_called_s": called_s, "unsupported_diagnosed": len(rej_done), "unsupported_called_s_ci95": list(clopper_pearson(called_s, len(rej_done))), "supported_not_carried": not_carried, "supported_diagnosed": len(acc_done), "supported_not_carried_ci95": list(clopper_pearson(not_carried, len(acc_done)))}, "per_source": split(lambda c: sorted({str(e["file"]) for e in c["excerpts"]})), "per_provenance": split(lambda c: [str(c.get("provenance"))]), "m1": {"s": called_s, "of": len(rej), "missing": missing, "stop": 3 * called_s >= len(rej) or missing > 0}, } def tally_text(doc: dict[str, Any]) -> str: out = [] for g in GROUPS: c = doc[g] out.append(f"{g}: n {c['n']}; " + ", ".join(f"{k} {c[k]}" for k in (*CODES, "missing"))) a = doc["agreement"] lo, hi = a["supported_not_carried_ci95"] out.append(f"accepted by the reader, not carried by the excerpt per the diagnosis: {a['supported_not_carried']} " f"of {a['supported_diagnosed']} (95 % CI {100 * lo:.1f}-{100 * hi:.1f} %)") lo, hi = a["unsupported_called_s_ci95"] out.append(f"rejected by the reader, carried per the diagnosis (S): {a['unsupported_called_s']} of " f"{a['unsupported_diagnosed']} (95 % CI {100 * lo:.1f}-{100 * hi:.1f} %)") m = doc["m1"] verdict = "STOP and present" if m["stop"] else "continue" out.append(f"M1 rule: S for {m['s']} of {m['of']} rejected, {m['missing']} missing: {verdict}") return "\n".join(out) + "\n" # --- command line ---------------------------------------------------------------------------------------------- def build_parser() -> argparse.ArgumentParser: p = argparse.ArgumentParser(prog="python3 tools/diagnose.py", description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) sub = p.add_subparsers(dest="command", required=True) s = sub.add_parser("select") s.add_argument("--claims", required=True) s.add_argument("--verdicts", required=True) s.add_argument("--out", required=True) s.add_argument("--sample", type=int, default=SAMPLE) r = sub.add_parser("render") r.add_argument("--utvalg", required=True) r.add_argument("--claims", required=True) r.add_argument("--sources", required=True) r.add_argument("--wiki", required=True) r.add_argument("--out", required=True) r.add_argument("--size", type=int, default=CALL_SIZE) c = sub.add_parser("collect") c.add_argument("--transcripts", required=True) c.add_argument("--run", action="append", required=True) c.add_argument("--tasks", required=True) c.add_argument("--out", required=True) t = sub.add_parser("tally") t.add_argument("--diagnoses", required=True) t.add_argument("--verdicts", required=True) t.add_argument("--claims", required=True) t.add_argument("--out", required=True) return p def main(argv: list[str] | None = None) -> int: args = build_parser().parse_args(argv) try: out = Path(args.out) if args.command == "select": if out.exists(): raise RunError(f"{out} exists; an output is never overwritten") claims_doc = read_json(Path(args.claims)) ids = select([str(c["id"]) for c in claims_doc["claims"]], verdict_map(Path(args.verdicts)), args.sample) write_new(out, dump({"schema": SELECTION_SCHEMA, "seed": SEED, "sample": args.sample, "ids": ids})) print(f"select: {len(ids)} claims ({args.sample} of them a sample) -> {out}") return 0 if args.command == "render": utvalg = Path(args.utvalg) ids = [str(i) for i in read_json(utvalg)["ids"]] calls = render(ids, read_json(Path(args.claims)), Path(args.sources), Path(args.wiki), args.size, out) write_new(out / "calls.json", dump({"schema": CALLS_SCHEMA, "utvalg_sha256": sha256(utvalg), "size": args.size, "calls": calls})) print(f"render: {len(ids)} claims in {len(calls)} calls -> {out}") return 0 if args.command == "collect": if out.exists(): raise RunError(f"{out} exists; an output is never overwritten") calls = read_json(Path(args.tasks) / "calls.json")["calls"] doc = collect(calls, list(args.run), Path(args.transcripts)) write_new(out, dump(doc)) print(f"collect: {len(doc['diagnoses']) - len(doc['missing'])} of {len(doc['diagnoses'])} claims " f"diagnosed, {len(doc['missing'])} missing -> {out}") return 1 if doc["missing"] else 0 if out.exists(): raise RunError(f"{out} exists; an output is never overwritten") doc = tally(read_json(Path(args.diagnoses)), verdict_map(Path(args.verdicts)), read_json(Path(args.claims))) write_new(out, dump(doc)) print(tally_text(doc), end="") return 1 if doc["m1"]["stop"] else 0 except (RunError, OSError, KeyError, TypeError, ValueError) as exc: print(f"diagnose cannot run: {exc}", file=sys.stderr) return 2 if __name__ == "__main__": sys.exit(main())