First public release, published as one snapshot. What changed between versions is in CHANGELOG.md. Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
421 lines
20 KiB
Python
421 lines
20 KiB
Python
"""Why a claim is or is not carried by its excerpt: the tool around the diagnosis of wave 4 order 20 (contract K9).
|
||
No model call. The diagnosis explains; the reader judges. It is not calibrated and never counts for criterion 2.
|
||
|
||
Usage:
|
||
python3 tools/diagnose.py select --claims claims.json --verdicts verdicts.json --out utvalg.json [--sample 100]
|
||
python3 tools/diagnose.py render --utvalg utvalg.json --claims claims.json --sources SOURCES --wiki WIKI
|
||
--out tasks/ [--size 10]
|
||
python3 tools/diagnose.py collect --transcripts DIR --run R [--run R2]... --tasks tasks/ --out diagnoses.json
|
||
python3 tools/diagnose.py tally --diagnoses diagnoses.json --verdicts verdicts.json --claims claims.json
|
||
--out tally.json
|
||
|
||
select every claim the reader did not accept, and a sample of the accepted ones:
|
||
random.Random(SEED).sample(sorted(ids), --sample), then the two lists (each sorted) shuffled with the same
|
||
generator. The selection holds ids only, never a verdict. A claim without a verdict, or fewer accepted claims
|
||
than the sample, cannot run.
|
||
render one task per call of at most --size claims, in the selection's order, named d001, d002, ...; and
|
||
calls.json (the ids per call). Per claim: the page title, the claim, each excerpt it cites, and per excerpt a
|
||
window in its source: the heading path above the quoted line (okfwiki.kblocks.headings_of, as
|
||
tools/claim_features.py takes it) and WINDOW lines before and after it, cut at the file's ends. Each window
|
||
line is "<line number> <q><c>| <text>", q ">" on the quoted line, c "c" on a line of fenced code (the fence
|
||
lines too). No id, verdict or provenance mark reaches the task. A task folder that holds anything is never
|
||
written into.
|
||
collect the reply of every diagnosis agent (agent type okf-wiki-diagnoser, description "V1-<run>-diagnose-<call>",
|
||
started from the session or from a workflow, the two places okfwiki.cost harvest reads): the message it
|
||
handed back, or else its last text (okfwiki.reader.final_reply). For each call the last --run that answered
|
||
it counts; two transcripts of that run for one call give no diagnosis for any of its claims. A line
|
||
"<number>: <code> | <words>" gives the claim of that number its code (one of CODES) and the words that are
|
||
missing; S needs no words, every other code does, and A has at most twelve. A line that starts with a number
|
||
but breaks the form, or a number given twice, gives that claim no diagnosis: it is "missing", never guessed.
|
||
Other lines are ignored.
|
||
tally counts per code for the claims the reader did not accept and for the sample it accepted, each apart; how
|
||
often the diagnosis and the reader disagree (S on a rejected claim; any other code on an accepted one) with a
|
||
two-sided 95 % Clopper-Pearson interval; the same counts per source file and per provenance mark; and the M1
|
||
rule of order 20: S for at least a third of the rejected claims, or any claim missing, is stop and present.
|
||
|
||
Codes (locked in phase 0 before any diagnosis was read): S the excerpt alone carries the claim; O what is missing
|
||
stands in the heading path or the page title; L it stands in another line of the window; B the claim says more than
|
||
the source (wider scope, stronger words); T a number, version, order, cause or purpose that is not in the window; F a
|
||
property given to the wrong thing, or a wrong connection; U what is missing is not in the window at all; A other.
|
||
|
||
Output: json.dumps(doc, indent=1, sort_keys=True, ensure_ascii=False) + newline; an existing output is never
|
||
overwritten. Exit 0 written; 1 written with claims missing (collect) or the M1 rule says stop (tally); 2 cannot run.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import hashlib
|
||
import json
|
||
import math
|
||
import random
|
||
import re
|
||
import sys
|
||
from pathlib import Path
|
||
from typing import Any
|
||
|
||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||
|
||
import claim_features # noqa: E402
|
||
from okfwiki import kblocks # noqa: E402
|
||
from okfwiki.common import RunError, write_new # noqa: E402
|
||
from okfwiki.reader import final_reply # noqa: E402
|
||
from okfwiki.transcript import is_agent_type # noqa: E402
|
||
|
||
SELECTION_SCHEMA = "okf-wiki/diagnose-selection/1"
|
||
CALLS_SCHEMA = "okf-wiki/diagnose-calls/1"
|
||
DIAGNOSES_SCHEMA = "okf-wiki/diagnoses/1"
|
||
TALLY_SCHEMA = "okf-wiki/diagnose-tally/1"
|
||
SEED = 20261012
|
||
SAMPLE = 100
|
||
CALL_SIZE = 10
|
||
WINDOW = 8
|
||
CODES = ("S", "O", "L", "B", "T", "F", "U", "A")
|
||
OTHER_WORDS_MAX = 12
|
||
AGENT_TYPE = "okf-wiki-diagnoser"
|
||
FENCE = re.compile(r"^\s*(```|~~~)") # the fence rule of okfwiki.kblocks.headings_of
|
||
NUMBERED = re.compile(r"^\s*(\d+)\s*:(.*)$")
|
||
FORM = re.compile(r"^\s*([A-Z])\s*(?:\|\s*(.*?))?\s*$")
|
||
UNSUPPORTED = "unsupported"
|
||
SUPPORTED = "supported"
|
||
GROUPS = ("unsupported", "supported_sample")
|
||
|
||
|
||
def read_json(path: Path) -> Any:
|
||
try:
|
||
return json.loads(path.read_text(encoding="utf-8"))
|
||
except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc:
|
||
raise RunError(f"cannot read {path}: {exc}") from exc
|
||
|
||
|
||
def dump(doc: Any) -> str:
|
||
return json.dumps(doc, indent=1, sort_keys=True, ensure_ascii=False) + "\n"
|
||
|
||
|
||
def sha256(path: Path) -> str:
|
||
return hashlib.sha256(path.read_bytes()).hexdigest()
|
||
|
||
|
||
def verdict_map(path: Path) -> dict[str, str]:
|
||
return {str(v["id"]): str(v["verdict"]) for v in read_json(path)["verdicts"]}
|
||
|
||
|
||
# --- select ----------------------------------------------------------------------------------------------------
|
||
|
||
|
||
def select(claim_ids: list[str], verdicts: dict[str, str], sample: int) -> list[str]:
|
||
rejected: list[str] = []
|
||
accepted: list[str] = []
|
||
for cid in claim_ids:
|
||
v = verdicts.get(cid)
|
||
if v not in (SUPPORTED, UNSUPPORTED):
|
||
raise RunError(f"claim {cid} has no verdict of {SUPPORTED} or {UNSUPPORTED}")
|
||
(rejected if v == UNSUPPORTED else accepted).append(cid)
|
||
if len(accepted) < sample:
|
||
raise RunError(f"{len(accepted)} accepted claims, fewer than the sample of {sample}")
|
||
rng = random.Random(SEED)
|
||
picked = rng.sample(sorted(accepted), sample)
|
||
ids = sorted(rejected) + sorted(picked)
|
||
rng.shuffle(ids)
|
||
return ids
|
||
|
||
|
||
# --- render ----------------------------------------------------------------------------------------------------
|
||
|
||
|
||
def code_lines(lines: list[str]) -> set[int]:
|
||
out: set[int] = set()
|
||
fence: str | None = None
|
||
for n, raw in enumerate(lines, 1):
|
||
f = FENCE.match(raw)
|
||
if f:
|
||
out.add(n)
|
||
fence = None if fence == f.group(1) else (fence or f.group(1))
|
||
continue
|
||
if fence is not None:
|
||
out.add(n)
|
||
return out
|
||
|
||
|
||
def window(lines: list[str], name: str, line: int) -> str:
|
||
first, last = max(1, line - WINDOW), min(len(lines), line + WINDOW)
|
||
code = code_lines(lines)
|
||
path = " › ".join(claim_features.heading_path(lines, line)).replace('"', "'")
|
||
rows = [f"{n:>5} {'>' if n == line else ' '}{'c' if n in code else ' '}| {lines[n - 1]}".rstrip()
|
||
for n in range(first, last + 1)]
|
||
return "\n".join([f'<window file="{name}" lines="{first}-{last}" headings="{path}">', *rows, "</window>"])
|
||
|
||
|
||
def render_task(claims: list[dict[str, Any]], titles: dict[str, str], lines_of: dict[str, list[str]]) -> str:
|
||
n = len(claims)
|
||
parts = ["# Diagnosis task", "", f"This task has {n} claim{'s' if n != 1 else ''}, numbered 1 to {n}."]
|
||
for number, c in enumerate(claims, 1):
|
||
parts += ["", f"## Claim {number}", "", f"Page title: {titles[str(c['page'])]}", "",
|
||
f"<claim>{c['claim']}</claim>"]
|
||
for e in c["excerpts"]:
|
||
parts += ["", f'<excerpt file="{e["file"]}" line="{e["line"]}">', str(e["text"]), "</excerpt>"]
|
||
for e in c["excerpts"]:
|
||
parts += ["", window(lines_of[str(e["file"])], str(e["file"]), int(e["line"]))]
|
||
return "\n".join(parts) + "\n"
|
||
|
||
|
||
def render(ids: list[str], claims_doc: dict[str, Any], sources: Path, wiki: Path, size: int,
|
||
out: Path) -> list[dict[str, Any]]:
|
||
if size < 1:
|
||
raise RunError("--size must be at least 1")
|
||
if out.exists() and any(out.iterdir()):
|
||
raise RunError(f"{out} holds files; a task folder is never written into")
|
||
by_id = {str(c["id"]): c for c in claims_doc["claims"]}
|
||
missing = [i for i in ids if i not in by_id]
|
||
if missing:
|
||
raise RunError(f"selected ids not in the claim list: {missing[:5]}")
|
||
kilde = kblocks.kilde_of(sources)
|
||
lines_of: dict[str, list[str]] = {}
|
||
titles: dict[str, str] = {}
|
||
for cid in ids:
|
||
c = by_id[cid]
|
||
page = str(c["page"])
|
||
if page not in titles:
|
||
titles[page] = claim_features.page_title(wiki, page)
|
||
for e in c["excerpts"]:
|
||
name = str(e["file"])
|
||
if name not in lines_of:
|
||
if not (kilde / name).is_file():
|
||
raise RunError(f"the source {kilde / name} is not there")
|
||
lines_of[name] = kblocks.read_lines(kilde / name)
|
||
chunks = [(f"d{k + 1:03d}", ids[i:i + size]) for k, i in enumerate(range(0, len(ids), size))]
|
||
texts = {call: render_task([by_id[i] for i in chunk], titles, lines_of) for call, chunk in chunks}
|
||
for call, text in texts.items():
|
||
write_new(out / f"{call}.md", text)
|
||
return [{"name": call, "ids": chunk} for call, chunk in chunks]
|
||
|
||
|
||
# --- collect ---------------------------------------------------------------------------------------------------
|
||
|
||
|
||
def agent_replies(transcripts: Path, run: str) -> dict[str, list[Path]]:
|
||
prefix = re.compile(rf"^V1-{re.escape(run)}-diagnose-(\S+)$")
|
||
found: dict[str, list[Path]] = {}
|
||
metas = [*transcripts.glob("*/subagents/agent-*.meta.json"),
|
||
*transcripts.glob("*/subagents/workflows/*/agent-*.meta.json")]
|
||
for meta_path in sorted(metas):
|
||
try:
|
||
meta = json.loads(meta_path.read_text(encoding="utf-8"))
|
||
except (OSError, ValueError):
|
||
continue
|
||
if not isinstance(meta, dict) or not is_agent_type(meta.get("agentType"), AGENT_TYPE):
|
||
continue
|
||
m = prefix.match(str(meta.get("description", "")))
|
||
if m is not None:
|
||
found.setdefault(m.group(1), []).append(meta_path.with_name(
|
||
meta_path.name.removesuffix(".meta.json") + ".jsonl"))
|
||
return found
|
||
|
||
|
||
def parse_reply(reply: str, n: int) -> dict[int, tuple[str, str]]:
|
||
"""The code and missing words per claim number; a broken line or a number given twice leaves that claim out."""
|
||
given: dict[int, tuple[str, str]] = {}
|
||
bad: set[int] = set()
|
||
for raw in reply.splitlines():
|
||
m = NUMBERED.match(raw)
|
||
if m is None:
|
||
continue
|
||
number = int(m.group(1))
|
||
if not 1 <= number <= n:
|
||
continue
|
||
if number in given or number in bad:
|
||
given.pop(number, None)
|
||
bad.add(number)
|
||
continue
|
||
f = FORM.match(m.group(2))
|
||
code, words = (f.group(1), (f.group(2) or "").strip()) if f else ("", "")
|
||
ok = (code in CODES and (code == "S" or bool(words))
|
||
and (code != "A" or len(words.split()) <= OTHER_WORDS_MAX))
|
||
if ok:
|
||
given[number] = (code, words)
|
||
else:
|
||
bad.add(number)
|
||
return given
|
||
|
||
|
||
def collect(calls: list[dict[str, Any]], runs: list[str], transcripts: Path) -> dict[str, Any]:
|
||
found = {r: agent_replies(transcripts, r) for r in runs}
|
||
rows: list[dict[str, Any]] = []
|
||
for call in calls:
|
||
name, ids = str(call["name"]), [str(i) for i in call["ids"]]
|
||
answering = [r for r in runs if name in found[r]]
|
||
run = answering[-1] if answering else None
|
||
paths = found[run][name] if run else []
|
||
given = parse_reply(final_reply(paths[0]) if paths[0].is_file() else "", len(ids)) if len(paths) == 1 else {}
|
||
for number, cid in enumerate(ids, 1):
|
||
code, words = given.get(number, (None, ""))
|
||
rows.append({"id": cid, "call": name, "number": number, "code": code, "words": words, "run": run})
|
||
return {"schema": DIAGNOSES_SCHEMA, "runs": runs, "diagnoses": rows,
|
||
"missing": [r["id"] for r in rows if r["code"] is None]}
|
||
|
||
|
||
# --- tally -----------------------------------------------------------------------------------------------------
|
||
|
||
|
||
def _binom_tail(k_from: int, k_to: int, n: int, p: float) -> float:
|
||
return sum(math.exp(math.lgamma(n + 1) - math.lgamma(i + 1) - math.lgamma(n - i + 1)
|
||
+ i * math.log(p) + (n - i) * math.log1p(-p)) for i in range(k_from, k_to + 1))
|
||
|
||
|
||
def _solve(f: Any, rising: bool) -> float:
|
||
lo, hi = 0.0, 1.0
|
||
for _ in range(200):
|
||
mid = (lo + hi) / 2
|
||
if mid <= 0.0 or mid >= 1.0:
|
||
break
|
||
if (f(mid) < 0.025) == rising:
|
||
lo = mid
|
||
else:
|
||
hi = mid
|
||
return (lo + hi) / 2
|
||
|
||
|
||
def clopper_pearson(k: int, n: int) -> tuple[float, float]:
|
||
"""The two-sided 95 % Clopper-Pearson interval for k events in n."""
|
||
if n <= 0:
|
||
return 0.0, 1.0
|
||
lower = 0.0 if k == 0 else _solve(lambda p: _binom_tail(k, n, n, p), rising=True)
|
||
upper = 1.0 if k >= n else _solve(lambda p: _binom_tail(0, k, n, p), rising=False)
|
||
return lower, upper
|
||
|
||
|
||
def counts(rows: list[dict[str, Any]]) -> dict[str, int]:
|
||
out = {code: 0 for code in CODES}
|
||
out["missing"] = 0
|
||
for r in rows:
|
||
out[r["code"] if r["code"] in CODES else "missing"] += 1
|
||
out["n"] = len(rows)
|
||
return out
|
||
|
||
|
||
def tally(diagnoses: dict[str, Any], verdicts: dict[str, str], claims_doc: dict[str, Any]) -> dict[str, Any]:
|
||
by_id = {str(c["id"]): c for c in claims_doc["claims"]}
|
||
grouped: dict[str, list[dict[str, Any]]] = {g: [] for g in GROUPS}
|
||
for row in diagnoses["diagnoses"]:
|
||
v = verdicts.get(str(row["id"]))
|
||
if v not in (SUPPORTED, UNSUPPORTED) or str(row["id"]) not in by_id:
|
||
raise RunError(f"diagnosed claim {row['id']} has no verdict or is not in the claim list")
|
||
grouped["unsupported" if v == UNSUPPORTED else "supported_sample"].append(row)
|
||
|
||
def split(key_of: Any) -> dict[str, dict[str, dict[str, int]]]:
|
||
keys = sorted({k for g in grouped.values() for r in g for k in key_of(by_id[str(r["id"])])})
|
||
return {k: {g: counts([r for r in grouped[g] if k in key_of(by_id[str(r["id"])])]) for g in GROUPS}
|
||
for k in keys}
|
||
|
||
rej, acc = grouped["unsupported"], grouped["supported_sample"]
|
||
rej_done = [r for r in rej if r["code"] in CODES]
|
||
acc_done = [r for r in acc if r["code"] in CODES]
|
||
called_s = sum(1 for r in rej_done if r["code"] == "S")
|
||
not_carried = sum(1 for r in acc_done if r["code"] != "S")
|
||
missing = sum(1 for g in grouped.values() for r in g if r["code"] not in CODES)
|
||
return {
|
||
"schema": TALLY_SCHEMA,
|
||
"unsupported": counts(rej), "supported_sample": counts(acc),
|
||
"agreement": {"unsupported_called_s": called_s, "unsupported_diagnosed": len(rej_done),
|
||
"unsupported_called_s_ci95": list(clopper_pearson(called_s, len(rej_done))),
|
||
"supported_not_carried": not_carried, "supported_diagnosed": len(acc_done),
|
||
"supported_not_carried_ci95": list(clopper_pearson(not_carried, len(acc_done)))},
|
||
"per_source": split(lambda c: sorted({str(e["file"]) for e in c["excerpts"]})),
|
||
"per_provenance": split(lambda c: [str(c.get("provenance"))]),
|
||
"m1": {"s": called_s, "of": len(rej), "missing": missing, "stop": 3 * called_s >= len(rej) or missing > 0},
|
||
}
|
||
|
||
|
||
def tally_text(doc: dict[str, Any]) -> str:
|
||
out = []
|
||
for g in GROUPS:
|
||
c = doc[g]
|
||
out.append(f"{g}: n {c['n']}; " + ", ".join(f"{k} {c[k]}" for k in (*CODES, "missing")))
|
||
a = doc["agreement"]
|
||
lo, hi = a["supported_not_carried_ci95"]
|
||
out.append(f"accepted by the reader, not carried by the excerpt per the diagnosis: {a['supported_not_carried']} "
|
||
f"of {a['supported_diagnosed']} (95 % CI {100 * lo:.1f}-{100 * hi:.1f} %)")
|
||
lo, hi = a["unsupported_called_s_ci95"]
|
||
out.append(f"rejected by the reader, carried per the diagnosis (S): {a['unsupported_called_s']} of "
|
||
f"{a['unsupported_diagnosed']} (95 % CI {100 * lo:.1f}-{100 * hi:.1f} %)")
|
||
m = doc["m1"]
|
||
verdict = "STOP and present" if m["stop"] else "continue"
|
||
out.append(f"M1 rule: S for {m['s']} of {m['of']} rejected, {m['missing']} missing: {verdict}")
|
||
return "\n".join(out) + "\n"
|
||
|
||
|
||
# --- command line ----------------------------------------------------------------------------------------------
|
||
|
||
|
||
def build_parser() -> argparse.ArgumentParser:
|
||
p = argparse.ArgumentParser(prog="python3 tools/diagnose.py", description=__doc__,
|
||
formatter_class=argparse.RawDescriptionHelpFormatter)
|
||
sub = p.add_subparsers(dest="command", required=True)
|
||
s = sub.add_parser("select")
|
||
s.add_argument("--claims", required=True)
|
||
s.add_argument("--verdicts", required=True)
|
||
s.add_argument("--out", required=True)
|
||
s.add_argument("--sample", type=int, default=SAMPLE)
|
||
r = sub.add_parser("render")
|
||
r.add_argument("--utvalg", required=True)
|
||
r.add_argument("--claims", required=True)
|
||
r.add_argument("--sources", required=True)
|
||
r.add_argument("--wiki", required=True)
|
||
r.add_argument("--out", required=True)
|
||
r.add_argument("--size", type=int, default=CALL_SIZE)
|
||
c = sub.add_parser("collect")
|
||
c.add_argument("--transcripts", required=True)
|
||
c.add_argument("--run", action="append", required=True)
|
||
c.add_argument("--tasks", required=True)
|
||
c.add_argument("--out", required=True)
|
||
t = sub.add_parser("tally")
|
||
t.add_argument("--diagnoses", required=True)
|
||
t.add_argument("--verdicts", required=True)
|
||
t.add_argument("--claims", required=True)
|
||
t.add_argument("--out", required=True)
|
||
return p
|
||
|
||
|
||
def main(argv: list[str] | None = None) -> int:
|
||
args = build_parser().parse_args(argv)
|
||
try:
|
||
out = Path(args.out)
|
||
if args.command == "select":
|
||
if out.exists():
|
||
raise RunError(f"{out} exists; an output is never overwritten")
|
||
claims_doc = read_json(Path(args.claims))
|
||
ids = select([str(c["id"]) for c in claims_doc["claims"]], verdict_map(Path(args.verdicts)), args.sample)
|
||
write_new(out, dump({"schema": SELECTION_SCHEMA, "seed": SEED, "sample": args.sample, "ids": ids}))
|
||
print(f"select: {len(ids)} claims ({args.sample} of them a sample) -> {out}")
|
||
return 0
|
||
if args.command == "render":
|
||
utvalg = Path(args.utvalg)
|
||
ids = [str(i) for i in read_json(utvalg)["ids"]]
|
||
calls = render(ids, read_json(Path(args.claims)), Path(args.sources), Path(args.wiki), args.size, out)
|
||
write_new(out / "calls.json", dump({"schema": CALLS_SCHEMA, "utvalg_sha256": sha256(utvalg),
|
||
"size": args.size, "calls": calls}))
|
||
print(f"render: {len(ids)} claims in {len(calls)} calls -> {out}")
|
||
return 0
|
||
if args.command == "collect":
|
||
if out.exists():
|
||
raise RunError(f"{out} exists; an output is never overwritten")
|
||
calls = read_json(Path(args.tasks) / "calls.json")["calls"]
|
||
doc = collect(calls, list(args.run), Path(args.transcripts))
|
||
write_new(out, dump(doc))
|
||
print(f"collect: {len(doc['diagnoses']) - len(doc['missing'])} of {len(doc['diagnoses'])} claims "
|
||
f"diagnosed, {len(doc['missing'])} missing -> {out}")
|
||
return 1 if doc["missing"] else 0
|
||
if out.exists():
|
||
raise RunError(f"{out} exists; an output is never overwritten")
|
||
doc = tally(read_json(Path(args.diagnoses)), verdict_map(Path(args.verdicts)), read_json(Path(args.claims)))
|
||
write_new(out, dump(doc))
|
||
print(tally_text(doc), end="")
|
||
return 1 if doc["m1"]["stop"] else 0
|
||
except (RunError, OSError, KeyError, TypeError, ValueError) as exc:
|
||
print(f"diagnose cannot run: {exc}", file=sys.stderr)
|
||
return 2
|
||
|
||
|
||
if __name__ == "__main__":
|
||
sys.exit(main())
|