"""What each claim says beyond the quote it cites, counted by code (wave 4 order 20, contract K9). No model call. Usage: python3 tools/claim_features.py --claims claims.json --sources SOURCES --wiki WIKI --out features.json [--verdicts verdicts.json] [--summary] It reads the claim list (okfwiki.claims list), the reader's verdicts when given, the sources (SOURCES/kilde/) and the pages (WIKI/, for the page title in the frontmatter), and writes per claim, in the claim list's order: quotes, quote_chars, claim_chars the number of excerpts, the characters of their texts summed, of the claim content_words the claim's content words (the rule below), repeats counted outside, outside_count the claim's content words whose first five letters begin no content word of its excerpts, in claim order, repeats counted outside_title, outside_headings of those, the ones whose first five letters begin a content word of the page title, and of the heading path above each quoted line in its source headings that heading path (the headings above the first excerpt's line, outermost first), taken from okfwiki.kblocks.headings_of, so the tool counts a heading by the same rule as the build, also after order 22 changes that rule files, provenance, verdict the excerpts' source files, the claim's provenance mark, and the reader's verdict (null without --verdicts, or when it gave the claim none) The word rule (phase 0 of wave 4, 11.10.2026): lower-case the text, take every run of [a-z0-9], drop the words in STOPWORDS, and compare two words on their first five letters (a shorter word is compared whole). It describes; it judges nothing. A code check by word overlap was ruled out as a gate (PLAN.md ยง Retning, choice 5: 8 of 664 near verbatim), and nothing here is a verdict. --summary prints the phase 0 table: the share rejected per number of words outside the quote (0, 1, 2, 3-4, 5+), per provenance mark and per source file, the medians per verdict, and the share with a word from the page title or from the heading path outside the quote. Without verdicts it prints the counts only. A claim the reader gave no verdict is left out of the share and counted apart as not judged (order 39), so a partial reading does not lower it. Output: json.dumps(doc, indent=1, sort_keys=True, ensure_ascii=False) + newline; two runs on the same inputs are byte-identical, and an existing output is never overwritten. Exit 0 written, 2 cannot run (nothing is written). """ from __future__ import annotations import argparse import hashlib import json import re import statistics import sys from pathlib import Path from typing import Any sys.path.insert(0, str(Path(__file__).resolve().parents[1])) from okfwiki import kblocks # noqa: E402 from okfwiki.audit import read_document # noqa: E402 from okfwiki.common import RunError, write_new # noqa: E402 SCHEMA = "okf-wiki/claim-features/1" WORD = re.compile(r"[a-z0-9]+") STOPWORDS = frozenset( "a an the of to in on for and or is are be by with from as at it this that its can you your when if not no into " "use uses using".split()) STEM = 5 BUCKETS = (("0", 0, 0), ("1", 1, 1), ("2", 2, 2), ("3-4", 3, 4), ("5+", 5, None)) UNSUPPORTED = "unsupported" def words(text: str) -> list[str]: """The content words of a text, in order, repeats kept.""" return [w for w in WORD.findall(text.lower()) if w not in STOPWORDS] def stems(text: str) -> set[str]: return {w[:STEM] for w in words(text)} def heading_path(lines: list[str], line: int) -> list[str]: """The headings above a line, outermost first: each heading closes every open one at its level or deeper.""" path: list[tuple[int, str]] = [] for n, level, text in kblocks.headings_of(lines): if n >= line: break path = [(lv, t) for lv, t in path if lv < level] + [(level, text)] return [t for _, t in path] def read_json(path: Path) -> Any: try: return json.loads(path.read_text(encoding="utf-8")) except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc: raise RunError(f"cannot read {path}: {exc}") from exc def sha256(path: Path) -> str: return hashlib.sha256(path.read_bytes()).hexdigest() def page_title(wiki: Path, page: str) -> str: path = wiki / page if not path.is_file(): raise RunError(f"the page {path} is not there") fm = read_document(path, page).frontmatter return str((fm or {}).get("title") or "") def features(claims_doc: dict[str, Any], verdicts: dict[str, str] | None, sources: Path, wiki: Path) -> list[dict[str, Any]]: kilde = kblocks.kilde_of(sources) lines_of: dict[str, list[str]] = {} titles: dict[str, str] = {} out: list[dict[str, Any]] = [] for c in claims_doc["claims"]: excerpts = list(c["excerpts"]) quote_stems: set[str] = set() heading_stems: set[str] = set() headings: list[str] = [] for i, e in enumerate(excerpts): name = str(e["file"]) if name not in lines_of: path = kilde / name if not path.is_file(): raise RunError(f"the source {path} is not there") lines_of[name] = kblocks.read_lines(path) path_here = heading_path(lines_of[name], int(e["line"])) headings = path_here if i == 0 else headings heading_stems |= {s for h in path_here for s in stems(h)} quote_stems |= stems(str(e["text"])) page = str(c["page"]) if page not in titles: titles[page] = page_title(wiki, page) title_stems = stems(titles[page]) content = words(str(c["claim"])) outside = [w for w in content if w[:STEM] not in quote_stems] out.append({ "id": c["id"], "page": page, "provenance": c.get("provenance"), "files": sorted({str(e["file"]) for e in excerpts}), "verdict": None if verdicts is None else verdicts.get(str(c["id"])), "quotes": len(excerpts), "quote_chars": sum(len(str(e["text"])) for e in excerpts), "claim_chars": len(str(c["claim"])), "content_words": len(content), "outside": outside, "outside_count": len(outside), "outside_title": [w for w in outside if w[:STEM] in title_stems], "outside_headings": [w for w in outside if w[:STEM] in heading_stems], "headings": headings, }) return out def _share(rows: list[dict[str, Any]], judged: bool) -> str: if not judged: return f"{len(rows)} claims, no verdicts" bad = sum(1 for r in rows if r["verdict"] == UNSUPPORTED) n_judged = sum(1 for r in rows if r["verdict"] is not None) if n_judged == len(rows): pct = f"{100 * bad / n_judged:.1f} %" if n_judged else "-" return f"{len(rows)} claims, {bad} unsupported ({pct})" pct = f"{100 * bad / n_judged:.1f} % of {n_judged} judged" if n_judged else "- of 0 judged" return f"{len(rows)} claims, {bad} unsupported ({pct}), {len(rows) - n_judged} not judged" def _median(xs: list[int]) -> str: return f"{statistics.median(xs):g}" if xs else "-" def summary(rows: list[dict[str, Any]]) -> str: judged = any(r["verdict"] is not None for r in rows) out = [f"claims: {len(rows)}"] for label, lo, hi in BUCKETS: mine = [r for r in rows if r["outside_count"] >= lo and (hi is None or r["outside_count"] <= hi)] out.append(f"words outside the quote {label}: {_share(mine, judged)}") for key, label in (("provenance", "provenance"), ("files", "source")): values = sorted({v for r in rows for v in (r[key] if isinstance(r[key], list) else [r[key]])}, key=str) for v in values: mine = [r for r in rows if v in (r[key] if isinstance(r[key], list) else [r[key]])] out.append(f"{label} {v}: {_share(mine, judged)}") if not judged: return "\n".join(out) + "\n" groups = {name: [r for r in rows if (r["verdict"] == UNSUPPORTED) == (name == UNSUPPORTED) and r["verdict"] is not None] for name in ("supported", UNSUPPORTED)} for field, label in (("outside_count", "words outside"), ("quote_chars", "quote characters"), ("claim_chars", "claim characters"), ("content_words", "content words")): parts = [f"{name} {_median([int(r[field]) for r in g])}" for name, g in groups.items()] out.append(f"median {label}: {', '.join(parts)}") for field, label in (("outside_title", "the page title"), ("outside_headings", "the heading path")): parts = [] for name, g in groups.items(): k = sum(1 for r in g if r[field]) pct = f"{100 * k / len(g):.1f} %" if g else "-" parts.append(f"{name} {k} of {len(g)} ({pct})") out.append(f"with a word from {label} outside the quote: {', '.join(parts)}") return "\n".join(out) + "\n" def build_parser() -> argparse.ArgumentParser: p = argparse.ArgumentParser(prog="python3 tools/claim_features.py", description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) p.add_argument("--claims", required=True) p.add_argument("--sources", required=True) p.add_argument("--wiki", required=True) p.add_argument("--out", required=True) p.add_argument("--verdicts") p.add_argument("--summary", action="store_true") return p def main(argv: list[str] | None = None) -> int: args = build_parser().parse_args(argv) try: out = Path(args.out) if out.exists(): raise RunError(f"{out} exists; an output is never overwritten") claims_path = Path(args.claims) claims_doc = read_json(claims_path) verdicts: dict[str, str] | None = None inputs: dict[str, Any] = {"claims_sha256": sha256(claims_path), "verdicts_sha256": None} if args.verdicts: vpath = Path(args.verdicts) verdicts = {str(v["id"]): str(v["verdict"]) for v in read_json(vpath)["verdicts"]} inputs["verdicts_sha256"] = sha256(vpath) rows = features(claims_doc, verdicts, Path(args.sources), Path(args.wiki)) doc = {"schema": SCHEMA, "inputs": inputs, "claims": rows} write_new(out, json.dumps(doc, indent=1, sort_keys=True, ensure_ascii=False) + "\n") except (RunError, OSError, KeyError, TypeError, ValueError) as exc: print(f"claim_features cannot run: {exc}", file=sys.stderr) return 2 print(f"claim_features: {len(rows)} claims -> {out}") if args.summary: print(summary(rows), end="") return 0 if __name__ == "__main__": sys.exit(main())