okf-wiki/okfwiki/claims.py
Kjell Tore Guttormsen 0fd86a23a3
okf-wiki 1.1.1
First public release, published as one snapshot. What changed between
versions is in CHANGELOG.md.

Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
2026-11-14 21:03:20 +01:00

467 lines
21 KiB
Python

"""The claim list behind PLAN.md criterion 1 ("sitat ved plassering") and 2 ("slutning støttet"), and the reading
layout for pages (K6 in the wave-2 contracts). It calls no model.
The denominator comes from the pages, not from the footnotes: every sentence in the body of a concept page is a claim.
Headings and pure link lists are not claims; they are listed as exceptions so the count can be checked by hand. Every
claim has a stable id "<page>#<key>", the key being klink's sentence key (okfwiki.klink.page_sentences) for the same
sentence at the same place, so the id survives a new assembly of the page; a claim klink does not see is listed as
unkeyed and counts as failing, never as a claim without an id. Every claim carries excerpt_sha256 (the excerpts it shows
the reader, in order) and its provenance mark: the strongest x-provenance among the entries it cites, missing > merged >
inferred > documented, "missing" when an entry has no valid mark or the claim cites none (so it is read, never sampled).
list BUNDLE SOURCES --out CLAIMS [--pairs PAIRS]
Criterion 1: a claim counts when it has at least one footnote and every footnote names a sources[].id whose
x-quote stands verbatim at its x-locator (the auditor's own quote check). With --pairs, the reader set
({"pairs": [{id, claim, excerpts: [{file, line, text}]}]}); a claim with no quote to show is left out.
calls --claims CLAIMS --pairs PAIRS --out CALLS [--ids IDS] [--size N]
The call layout (okf-wiki/reader-calls/1) from the pages: the claims of one page stay together in calls of
at most ten (at most N with --size, 1 to 10), in page order; only claims in PAIRS, and with --ids only
those listed.
sample --claims CLAIMS --judged JUDGED --part-root ROOT --seed N --min 300 --out IDS
The claims to read after the first part: a claim is judged when its id and excerpt_sha256 stand in JUDGED
(the claim list of an earlier reading). Read are every claim not judged that cites a source of the part
(ROOT's source-manifest.json) or is marked inferred, merged or missing, and a seeded draw of at least --min
of the others. Written with the seed, the sha256 of the inputs and of the ids, and the error bound.
sample --claims CLAIMS --of-all --seed N --min 300 --out IDS
The reading after a build changes (the operator 13.11.2026): a seeded draw of --min among every claim,
whatever its mark and whether it was judged before, and nothing added to it.
score --claims CLAIMS --verdicts VERDICTS [--ids IDS]
Criterion 2 over the same claims (or the ids): a claim counts when its verdict is "supported"; a missing or
invalid verdict counts as not supported (fail-closed). Not supported is counted per provenance mark, and for
a sample the upper one-sided 95 % Clopper-Pearson bound is printed (0 errors in 299 is under 1 %).
Exit 0 = N of N (or written), 1 = at least one claim does not count (or nothing to write), 2 = could not run.
"""
from __future__ import annotations
import argparse
import hashlib
import json
import math
import random
import sys
from dataclasses import asdict, dataclass, field
from pathlib import Path, PurePosixPath
from typing import Any
from okfwiki.audit import (
FOOTNOTE_REF,
LINK_ONLY,
Bundle,
Document,
body_headings,
check_quote,
load_bundle,
mask_code,
prose_blocks,
sentence_spans,
)
from okfwiki.klink import key_text, page_sentences
from okfwiki.reader import SUPPORTED, load_pairs
PROVENANCE = ("documented", "inferred", "merged")
MISSING = "missing"
STRENGTH = {"documented": 0, "inferred": 1, "merged": 2, MISSING: 3}
MUST_READ = ("inferred", "merged", MISSING)
CALL_SIZE = 10
SAMPLE_MIN = 300
@dataclass(frozen=True)
class Excerpt:
file: str
line: int
text: str
@dataclass
class Claim:
id: str
page: str
line: int
claim: str
footnotes: list[str]
excerpts: list[Excerpt]
quote_ok: bool
excerpt_sha256: str = ""
provenance: str = MISSING
resources: list[str] = field(default_factory=list)
@dataclass(frozen=True)
class Exempt:
page: str
line: int
kind: str # "heading" or "link-list"
text: str
@dataclass(frozen=True)
class Unkeyed:
page: str
line: int
text: str
@dataclass
class ClaimList:
claims: list[Claim] = field(default_factory=list)
exceptions: list[Exempt] = field(default_factory=list)
unkeyed: list[Unkeyed] = field(default_factory=list)
def excerpt_digest(excerpts: list[Excerpt]) -> str:
"""sha256 of the excerpts a claim shows the reader, in order (file, line, text)."""
rows = [[e.file, e.line, e.text] for e in excerpts]
return hashlib.sha256(json.dumps(rows, ensure_ascii=False, separators=(",", ":")).encode("utf-8")).hexdigest()
def provenance_of(doc: Document, labels: list[str]) -> str:
"""The strongest mark among the cited entries; missing when an entry has no valid mark or nothing is cited."""
by_id = _sources_by_id(doc)
marks = [by_id[x].get("x-provenance") if x in by_id else None for x in labels] or [None]
return max((m if isinstance(m, str) and m in PROVENANCE else MISSING for m in marks), key=STRENGTH.__getitem__)
def _digest(text: str) -> str:
return hashlib.sha256(key_text(text).encode("utf-8")).hexdigest()[:16]
def _sources_by_id(doc: Document) -> dict[str, dict[str, Any]]:
fm = doc.frontmatter or {}
srcs = fm.get("sources", [])
entries = srcs if isinstance(srcs, list) else [srcs]
out: dict[str, dict[str, Any]] = {}
for s in entries:
if isinstance(s, dict) and isinstance(s.get("id"), str):
out.setdefault(s["id"], s)
return out
def _judge(b: Bundle, doc: Document, labels: list[str], cache: dict[str, list[str] | None]) -> tuple[list[Excerpt], bool]:
"""The excerpts a claim cites, and whether every one of them stands verbatim at its locator."""
by_id = _sources_by_id(doc)
excerpts: list[Excerpt] = []
ok = bool(labels)
for label in labels:
s = by_id.get(label)
if s is None:
ok = False
continue
if check_quote(b, doc, s, cache):
ok = False
quote, locator, res = s.get("x-quote"), s.get("x-locator"), s.get("resource")
if isinstance(quote, str) and quote and isinstance(res, str) and isinstance(locator, dict) \
and type(locator.get("line")) is int:
e = Excerpt(PurePosixPath(res).name, locator["line"], quote)
if e not in excerpts:
excerpts.append(e)
return excerpts, ok
def extract(bundle: Path, sources: Path) -> ClaimList:
b = load_bundle(bundle, sources)
out = ClaimList()
cache: dict[str, list[str] | None] = {}
for doc in b.concepts:
keys = [x.key for x in page_sentences((bundle / doc.rel).read_text(encoding="utf-8"))]
at = 0
for line, _level, text in body_headings(doc):
out.exceptions.append(Exempt(doc.rel, line, "heading", text))
for block in prose_blocks(doc):
if LINK_ONLY.match(mask_code(block.text)):
out.exceptions.append(Exempt(doc.rel, block.line, "link-list", block.text))
continue
for a, z in sentence_spans(block.text):
sentence = block.text[a:z]
labels: list[str] = []
for label in FOOTNOTE_REF.findall(mask_code(sentence)):
if label not in labels:
labels.append(label)
text = FOOTNOTE_REF.sub("", sentence).strip()
digest = _digest(sentence)
found = next((i for i in range(at, len(keys)) if keys[i].rsplit("-", 1)[0] == digest), None)
if found is None:
out.unkeyed.append(Unkeyed(doc.rel, block.line, text))
continue
at = found + 1
excerpts, ok = _judge(b, doc, labels, cache)
resources = [str(r) for r in (_sources_by_id(doc).get(x, {}).get("resource") for x in labels)
if isinstance(r, str)]
out.claims.append(Claim(f"{doc.rel}#{keys[found]}", doc.rel, block.line, text, labels, excerpts, ok,
excerpt_digest(excerpts), provenance_of(doc, labels),
list(dict.fromkeys(resources))))
return out
Tally = tuple[int, int]
def _tally(items: list[tuple[list[str], bool]]) -> dict[str, Tally]:
counts: dict[str, list[int]] = {}
for keys, ok in items:
for k in keys:
c = counts.setdefault(k, [0, 0])
c[0] += int(ok)
c[1] += 1
return {k: (v[0], v[1]) for k, v in sorted(counts.items())}
def _report(found: ClaimList, ok: dict[str, bool], cs: list[Claim] | None = None, extra: int = 0) -> dict[str, Any]:
"""Totals per page, per cited file and per provenance mark; `extra` claims that cannot count (unkeyed claims,
ids that are not claims) are added to the denominator as failing."""
cs = found.claims if cs is None else cs
return {
"total": (sum(ok[c.id] for c in cs), len(cs) + extra),
"per_page": _tally([([c.page], ok[c.id]) for c in cs]),
"per_file": _tally([(sorted({e.file for e in c.excerpts}), ok[c.id]) for c in cs]),
"not_supported_per_mark": {k: (b - a, b)
for k, (a, b) in _tally([([c.provenance], ok[c.id]) for c in cs]).items()},
"not_supported": sorted(c.id for c in cs if not ok[c.id]),
}
def criterion_1(found: ClaimList) -> dict[str, Any]:
return _report(found, {c.id: c.quote_ok for c in found.claims}, extra=len(found.unkeyed))
def criterion_2(found: ClaimList, verdicts: dict[str, Any], ids: list[str] | None = None) -> dict[str, Any]:
given: dict[str, list[str]] = {}
for v in verdicts.get("verdicts", []) if isinstance(verdicts, dict) else []:
if isinstance(v, dict) and isinstance(v.get("id"), str):
given.setdefault(v["id"], []).append(str(v.get("verdict")))
ok = {c.id: given.get(c.id) == [SUPPORTED] for c in found.claims}
if ids is None:
return _report(found, ok, extra=len(found.unkeyed))
wanted = set(ids)
chosen = [c for c in found.claims if c.id in wanted]
return _report(found, ok, chosen, extra=len(wanted) - len(chosen))
def error_bound(k: int, n: int) -> float:
"""The upper one-sided 95 % Clopper-Pearson bound on the error rate after k errors in n: the p at which k or fewer
errors in n have probability 0.05. For k = 0 it is 1 - 0.05 ** (1 / n)."""
if n <= 0 or k >= n:
return 1.0
def at_most_k(p: float) -> float:
return sum(math.exp(math.lgamma(n + 1) - math.lgamma(i + 1) - math.lgamma(n - i + 1)
+ i * math.log(p) + (n - i) * math.log1p(-p)) for i in range(k + 1))
lo, hi = 0.0, 1.0
for _ in range(200):
mid = (lo + hi) / 2
if mid <= 0.0 or at_most_k(mid) > 0.05:
lo = mid
else:
hi = mid
return hi
def write_pairs(found: ClaimList, out: Path) -> None:
pairs = [{"id": c.id, "claim": c.claim, "excerpts": [asdict(e) for e in c.excerpts]}
for c in found.claims if c.excerpts]
out.write_text(json.dumps({"schema": "okf-wiki/claim-pairs/1", "pairs": pairs}, indent=1, ensure_ascii=False) + "\n",
encoding="utf-8")
def load_claims(path: Path) -> ClaimList:
data = json.loads(path.read_text(encoding="utf-8"))
found = ClaimList()
for c in data["claims"]:
excerpts = [Excerpt(**e) for e in c["excerpts"]]
found.claims.append(Claim(c["id"], c["page"], c["line"], c["claim"], list(c["footnotes"]), excerpts,
bool(c["quote_ok"]), str(c.get("excerpt_sha256") or excerpt_digest(excerpts)),
str(c.get("provenance", MISSING)), list(c.get("resources", []))))
for e in data.get("exceptions", []):
found.exceptions.append(Exempt(**e))
for u in data.get("unkeyed", []):
found.unkeyed.append(Unkeyed(**u))
return found
_load = load_claims
def _sha(path: Path) -> str:
return hashlib.sha256(path.read_bytes()).hexdigest()
def _dump(doc: Any) -> str:
return json.dumps(doc, indent=1, sort_keys=True, ensure_ascii=False) + "\n"
def _ids(path: Path) -> list[str]:
data = json.loads(path.read_text(encoding="utf-8"))
ids = data.get("ids") if isinstance(data, dict) else None
if not isinstance(ids, list) or not all(isinstance(i, str) for i in ids):
raise ValueError(f"{path} has no list of ids")
return ids
def _print(name: str, report: dict[str, Any]) -> None:
x, n = report["total"]
print(f"{name}: {x} of {n}")
for key in ("per_page", "per_file"):
print(f" {key.replace('_', ' ')}:")
for k, (a, b) in report[key].items():
print(f" {k}: {a} of {b}")
def cmd_list(args: argparse.Namespace) -> int:
bundle, sources = Path(args.bundle), Path(args.sources)
for p, what in ((bundle, "bundle"), (sources, "raw-source root")):
if not p.is_dir():
print(f"claims cannot run: {what} {p} is not a directory", file=sys.stderr)
return 2
found = extract(bundle, sources)
doc = {"schema": "okf-wiki/claims/1",
"claims": [asdict(c) for c in found.claims],
"exceptions": [asdict(e) for e in found.exceptions],
"unkeyed": [asdict(u) for u in found.unkeyed]}
Path(args.out).write_text(json.dumps(doc, indent=1, ensure_ascii=False) + "\n", encoding="utf-8")
if args.pairs:
write_pairs(found, Path(args.pairs))
report = criterion_1(found)
_print("criterion 1, quote at locator", report)
kinds = {k: sum(e.kind == k for e in found.exceptions) for k in ("heading", "link-list")}
print(f"exceptions (not claims): {kinds['heading']} headings, {kinds['link-list']} link lists")
print(f"claims without a sentence key (counted as failing): {len(found.unkeyed)}")
x, n = report["total"]
return 0 if n and x == n else 1
def cmd_calls(args: argparse.Namespace) -> int:
size: int = args.size
if not 1 <= size <= CALL_SIZE:
raise ValueError(f"--size {size} is not a whole number from 1 to {CALL_SIZE}; nothing written")
found = load_claims(Path(args.claims))
pairs_path = Path(args.pairs)
with_pair = {p.id for p in load_pairs(pairs_path)}
wanted = set(_ids(Path(args.ids))) if args.ids else None
chosen = [c for c in found.claims if (wanted is None or c.id in wanted) and c.id in with_pair]
left_out = (len(wanted) if wanted is not None else len(found.claims)) - len(chosen)
pages: dict[str, list[str]] = {}
for c in chosen:
pages.setdefault(c.page, []).append(c.id)
groups = [ids[i : i + size] for ids in pages.values() for i in range(0, len(ids), size)]
if not groups:
print("calls: no claim to lay out", file=sys.stderr)
return 1
layout = {"call_size_max": size, "calls": [{"call": f"k{n:03d}", "ids": g} for n, g in enumerate(groups, 1)],
"schema": "okf-wiki/reader-calls/1", "set_sha256": _sha(pairs_path)}
Path(args.out).write_text(_dump(layout), encoding="utf-8")
print(f"calls: {len(chosen)} claims from {len(pages)} pages in {len(groups)} calls; "
f"{left_out} left out (no excerpt to show, or not a claim)")
return 0
def cmd_sample(args: argparse.Namespace) -> int:
if args.min < SAMPLE_MIN:
raise ValueError(f"--min {args.min} is under {SAMPLE_MIN}, the least sample K6 point 4 allows")
claims_path = Path(args.claims)
found = load_claims(claims_path)
judged_sha: str | None = None
manifest_sha: str | None = None
if args.of_all:
if args.judged or args.part_root:
raise ValueError("--of-all draws among every claim and takes no --judged or --part-root")
open_, must = list(found.claims), []
else:
if not args.judged or not args.part_root:
raise ValueError("sample needs --judged and --part-root, or --of-all to draw among every claim")
judged_path, manifest = Path(args.judged), Path(args.part_root) / "source-manifest.json"
part = {str(s["path"]) for s in json.loads(manifest.read_text(encoding="utf-8"))["sources"]}
done = {(c.id, c.excerpt_sha256) for c in load_claims(judged_path).claims}
open_ = [c for c in found.claims if (c.id, c.excerpt_sha256) not in done]
must = [c.id for c in open_ if c.provenance in MUST_READ or any(r.lstrip("/") in part for r in c.resources)]
judged_sha, manifest_sha = _sha(judged_path), _sha(manifest)
others = [c.id for c in open_ if c.id not in set(must)]
drawn = set(random.Random(args.seed).sample(others, min(len(others), args.min)))
sample = [i for i in others if i in drawn]
keep = set(must) | drawn
ids = [c.id for c in found.claims if c.id in keep]
doc = {"schema": "okf-wiki/claim-sample/1", "seed": args.seed, "min": args.min, "of_all": bool(args.of_all),
"claims_sha256": _sha(claims_path), "judged_sha256": judged_sha,
"part_manifest_sha256": manifest_sha,
"counts": {"claims": len(found.claims), "judged": len(found.claims) - len(open_), "must_read": len(must),
"others": len(others), "sample": len(sample)},
"must_read": must, "sample": sample, "ids": ids,
"ids_sha256": hashlib.sha256("\n".join(ids).encode("utf-8")).hexdigest(),
"error_bound_at_zero": error_bound(0, len(sample))}
Path(args.out).write_text(_dump(doc), encoding="utf-8")
print(f"sample (seed {args.seed}): {len(found.claims)} claims, {len(found.claims) - len(open_)} judged, "
f"{len(must)} must be read, {len(sample)} drawn of {len(others)}; read {len(ids)}")
print(f"0 not supported in the sample would bound the error at {error_bound(0, len(sample)):.2%} "
"(one-sided Clopper-Pearson, 95 %)")
return 0
def cmd_score(args: argparse.Namespace) -> int:
try:
found = load_claims(Path(args.claims))
verdicts = json.loads(Path(args.verdicts).read_text(encoding="utf-8"))
ids = _ids(Path(args.ids)) if args.ids else None
sample = json.loads(Path(args.ids).read_text(encoding="utf-8")).get("sample") if args.ids else None
except (OSError, ValueError, KeyError, TypeError) as exc:
print(f"score cannot run: {exc}", file=sys.stderr)
return 2
report = criterion_2(found, verdicts, ids)
_print("criterion 2, supported", report)
marks = ", ".join(f"{k} {a} of {b}" for k, (a, b) in report["not_supported_per_mark"].items())
print(f"not supported per provenance mark: {marks or 'none'}")
if isinstance(sample, list):
k = len(set(report["not_supported"]) & set(sample)) + len(set(sample) - {c.id for c in found.claims})
print(f"{k} not supported in a sample of {len(sample)}: upper bound {error_bound(k, len(sample)):.2%} "
"(one-sided Clopper-Pearson, 95 %)")
if report["not_supported"]:
print("not supported: " + " ".join(report["not_supported"]))
x, n = report["total"]
return 0 if n and x == n else 1
def build_parser() -> argparse.ArgumentParser:
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
sub = p.add_subparsers(dest="command", required=True)
ls = sub.add_parser("list", help="list the claims and count criterion 1")
ls.add_argument("bundle")
ls.add_argument("sources")
ls.add_argument("--out", required=True)
ls.add_argument("--pairs")
ca = sub.add_parser("calls", help="lay the claims out in calls of at most ten per page")
ca.add_argument("--claims", required=True)
ca.add_argument("--pairs", required=True)
ca.add_argument("--out", required=True)
ca.add_argument("--ids")
ca.add_argument("--size", type=int, default=CALL_SIZE,
help=f"at most this many claims per call, 1 to {CALL_SIZE} (default {CALL_SIZE})")
sa = sub.add_parser("sample", help="the claims to read: the part, inferred, merged and a seeded draw")
sa.add_argument("--claims", required=True)
sa.add_argument("--judged")
sa.add_argument("--part-root")
sa.add_argument("--of-all", action="store_true", help="draw among every claim and add nothing")
sa.add_argument("--seed", type=int, required=True)
sa.add_argument("--min", type=int, required=True)
sa.add_argument("--out", required=True)
sc = sub.add_parser("score", help="count criterion 2 from a reader's verdicts")
sc.add_argument("--claims", required=True)
sc.add_argument("--verdicts", required=True)
sc.add_argument("--ids")
return p
def main(argv: list[str] | None = None) -> int:
args = build_parser().parse_args(argv)
commands = {"list": cmd_list, "calls": cmd_calls, "sample": cmd_sample, "score": cmd_score}
if args.command in ("list", "calls", "sample"):
try:
return commands[args.command](args)
except (OSError, ValueError, KeyError, TypeError) as exc:
print(f"claims cannot run: {exc}", file=sys.stderr)
return 2
return commands[args.command](args)
if __name__ == "__main__":
sys.exit(main())