okf-wiki/tools/claim_features.py
Kjell Tore Guttormsen 0fd86a23a3
okf-wiki 1.1.1
First public release, published as one snapshot. What changed between
versions is in CHANGELOG.md.

Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
2026-11-14 21:03:20 +01:00

226 lines
11 KiB
Python

"""What each claim says beyond the quote it cites, counted by code (wave 4 order 20, contract K9). No model call.
Usage:
python3 tools/claim_features.py --claims claims.json --sources SOURCES --wiki WIKI --out features.json
[--verdicts verdicts.json] [--summary]
It reads the claim list (okfwiki.claims list), the reader's verdicts when given, the sources (SOURCES/kilde/) and the
pages (WIKI/<page>, for the page title in the frontmatter), and writes per claim, in the claim list's order:
quotes, quote_chars, claim_chars the number of excerpts, the characters of their texts summed, of the claim
content_words the claim's content words (the rule below), repeats counted
outside, outside_count the claim's content words whose first five letters begin no content word of
its excerpts, in claim order, repeats counted
outside_title, outside_headings of those, the ones whose first five letters begin a content word of the page
title, and of the heading path above each quoted line in its source
headings that heading path (the headings above the first excerpt's line, outermost
first), taken from okfwiki.kblocks.headings_of, so the tool counts a heading
by the same rule as the build, also after order 22 changes that rule
files, provenance, verdict the excerpts' source files, the claim's provenance mark, and the reader's
verdict (null without --verdicts, or when it gave the claim none)
The word rule (phase 0 of wave 4, 11.10.2026): lower-case the text, take every run of [a-z0-9], drop the words in
STOPWORDS, and compare two words on their first five letters (a shorter word is compared whole). It describes; it
judges nothing. A code check by word overlap was ruled out as a gate (PLAN.md § Retning, choice 5: 8 of 664 near
verbatim), and nothing here is a verdict.
--summary prints the phase 0 table: the share rejected per number of words outside the quote (0, 1, 2, 3-4, 5+),
per provenance mark and per source file, the medians per verdict, and the share with a word from the page title or
from the heading path outside the quote. Without verdicts it prints the counts only. A claim the reader gave no
verdict is left out of the share and counted apart as not judged (order 39), so a partial reading does not lower it.
Output: json.dumps(doc, indent=1, sort_keys=True, ensure_ascii=False) + newline; two runs on the same inputs are
byte-identical, and an existing output is never overwritten. Exit 0 written, 2 cannot run (nothing is written).
"""
from __future__ import annotations
import argparse
import hashlib
import json
import re
import statistics
import sys
from pathlib import Path
from typing import Any
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from okfwiki import kblocks # noqa: E402
from okfwiki.audit import read_document # noqa: E402
from okfwiki.common import RunError, write_new # noqa: E402
SCHEMA = "okf-wiki/claim-features/1"
WORD = re.compile(r"[a-z0-9]+")
STOPWORDS = frozenset(
"a an the of to in on for and or is are be by with from as at it this that its can you your when if not no into "
"use uses using".split())
STEM = 5
BUCKETS = (("0", 0, 0), ("1", 1, 1), ("2", 2, 2), ("3-4", 3, 4), ("5+", 5, None))
UNSUPPORTED = "unsupported"
def words(text: str) -> list[str]:
"""The content words of a text, in order, repeats kept."""
return [w for w in WORD.findall(text.lower()) if w not in STOPWORDS]
def stems(text: str) -> set[str]:
return {w[:STEM] for w in words(text)}
def heading_path(lines: list[str], line: int) -> list[str]:
"""The headings above a line, outermost first: each heading closes every open one at its level or deeper."""
path: list[tuple[int, str]] = []
for n, level, text in kblocks.headings_of(lines):
if n >= line:
break
path = [(lv, t) for lv, t in path if lv < level] + [(level, text)]
return [t for _, t in path]
def read_json(path: Path) -> Any:
try:
return json.loads(path.read_text(encoding="utf-8"))
except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc:
raise RunError(f"cannot read {path}: {exc}") from exc
def sha256(path: Path) -> str:
return hashlib.sha256(path.read_bytes()).hexdigest()
def page_title(wiki: Path, page: str) -> str:
path = wiki / page
if not path.is_file():
raise RunError(f"the page {path} is not there")
fm = read_document(path, page).frontmatter
return str((fm or {}).get("title") or "")
def features(claims_doc: dict[str, Any], verdicts: dict[str, str] | None, sources: Path,
wiki: Path) -> list[dict[str, Any]]:
kilde = kblocks.kilde_of(sources)
lines_of: dict[str, list[str]] = {}
titles: dict[str, str] = {}
out: list[dict[str, Any]] = []
for c in claims_doc["claims"]:
excerpts = list(c["excerpts"])
quote_stems: set[str] = set()
heading_stems: set[str] = set()
headings: list[str] = []
for i, e in enumerate(excerpts):
name = str(e["file"])
if name not in lines_of:
path = kilde / name
if not path.is_file():
raise RunError(f"the source {path} is not there")
lines_of[name] = kblocks.read_lines(path)
path_here = heading_path(lines_of[name], int(e["line"]))
headings = path_here if i == 0 else headings
heading_stems |= {s for h in path_here for s in stems(h)}
quote_stems |= stems(str(e["text"]))
page = str(c["page"])
if page not in titles:
titles[page] = page_title(wiki, page)
title_stems = stems(titles[page])
content = words(str(c["claim"]))
outside = [w for w in content if w[:STEM] not in quote_stems]
out.append({
"id": c["id"], "page": page, "provenance": c.get("provenance"),
"files": sorted({str(e["file"]) for e in excerpts}),
"verdict": None if verdicts is None else verdicts.get(str(c["id"])),
"quotes": len(excerpts), "quote_chars": sum(len(str(e["text"])) for e in excerpts),
"claim_chars": len(str(c["claim"])), "content_words": len(content),
"outside": outside, "outside_count": len(outside),
"outside_title": [w for w in outside if w[:STEM] in title_stems],
"outside_headings": [w for w in outside if w[:STEM] in heading_stems],
"headings": headings,
})
return out
def _share(rows: list[dict[str, Any]], judged: bool) -> str:
if not judged:
return f"{len(rows)} claims, no verdicts"
bad = sum(1 for r in rows if r["verdict"] == UNSUPPORTED)
n_judged = sum(1 for r in rows if r["verdict"] is not None)
if n_judged == len(rows):
pct = f"{100 * bad / n_judged:.1f} %" if n_judged else "-"
return f"{len(rows)} claims, {bad} unsupported ({pct})"
pct = f"{100 * bad / n_judged:.1f} % of {n_judged} judged" if n_judged else "- of 0 judged"
return f"{len(rows)} claims, {bad} unsupported ({pct}), {len(rows) - n_judged} not judged"
def _median(xs: list[int]) -> str:
return f"{statistics.median(xs):g}" if xs else "-"
def summary(rows: list[dict[str, Any]]) -> str:
judged = any(r["verdict"] is not None for r in rows)
out = [f"claims: {len(rows)}"]
for label, lo, hi in BUCKETS:
mine = [r for r in rows if r["outside_count"] >= lo and (hi is None or r["outside_count"] <= hi)]
out.append(f"words outside the quote {label}: {_share(mine, judged)}")
for key, label in (("provenance", "provenance"), ("files", "source")):
values = sorted({v for r in rows for v in (r[key] if isinstance(r[key], list) else [r[key]])}, key=str)
for v in values:
mine = [r for r in rows if v in (r[key] if isinstance(r[key], list) else [r[key]])]
out.append(f"{label} {v}: {_share(mine, judged)}")
if not judged:
return "\n".join(out) + "\n"
groups = {name: [r for r in rows if (r["verdict"] == UNSUPPORTED) == (name == UNSUPPORTED)
and r["verdict"] is not None] for name in ("supported", UNSUPPORTED)}
for field, label in (("outside_count", "words outside"), ("quote_chars", "quote characters"),
("claim_chars", "claim characters"), ("content_words", "content words")):
parts = [f"{name} {_median([int(r[field]) for r in g])}" for name, g in groups.items()]
out.append(f"median {label}: {', '.join(parts)}")
for field, label in (("outside_title", "the page title"), ("outside_headings", "the heading path")):
parts = []
for name, g in groups.items():
k = sum(1 for r in g if r[field])
pct = f"{100 * k / len(g):.1f} %" if g else "-"
parts.append(f"{name} {k} of {len(g)} ({pct})")
out.append(f"with a word from {label} outside the quote: {', '.join(parts)}")
return "\n".join(out) + "\n"
def build_parser() -> argparse.ArgumentParser:
p = argparse.ArgumentParser(prog="python3 tools/claim_features.py", description=__doc__,
formatter_class=argparse.RawDescriptionHelpFormatter)
p.add_argument("--claims", required=True)
p.add_argument("--sources", required=True)
p.add_argument("--wiki", required=True)
p.add_argument("--out", required=True)
p.add_argument("--verdicts")
p.add_argument("--summary", action="store_true")
return p
def main(argv: list[str] | None = None) -> int:
args = build_parser().parse_args(argv)
try:
out = Path(args.out)
if out.exists():
raise RunError(f"{out} exists; an output is never overwritten")
claims_path = Path(args.claims)
claims_doc = read_json(claims_path)
verdicts: dict[str, str] | None = None
inputs: dict[str, Any] = {"claims_sha256": sha256(claims_path), "verdicts_sha256": None}
if args.verdicts:
vpath = Path(args.verdicts)
verdicts = {str(v["id"]): str(v["verdict"]) for v in read_json(vpath)["verdicts"]}
inputs["verdicts_sha256"] = sha256(vpath)
rows = features(claims_doc, verdicts, Path(args.sources), Path(args.wiki))
doc = {"schema": SCHEMA, "inputs": inputs, "claims": rows}
write_new(out, json.dumps(doc, indent=1, sort_keys=True, ensure_ascii=False) + "\n")
except (RunError, OSError, KeyError, TypeError, ValueError) as exc:
print(f"claim_features cannot run: {exc}", file=sys.stderr)
return 2
print(f"claim_features: {len(rows)} claims -> {out}")
if args.summary:
print(summary(rows), end="")
return 0
if __name__ == "__main__":
sys.exit(main())