First public release, published as one snapshot. What changed between versions is in CHANGELOG.md. Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
226 lines
11 KiB
Python
226 lines
11 KiB
Python
"""What each claim says beyond the quote it cites, counted by code (wave 4 order 20, contract K9). No model call.
|
|
|
|
Usage:
|
|
python3 tools/claim_features.py --claims claims.json --sources SOURCES --wiki WIKI --out features.json
|
|
[--verdicts verdicts.json] [--summary]
|
|
|
|
It reads the claim list (okfwiki.claims list), the reader's verdicts when given, the sources (SOURCES/kilde/) and the
|
|
pages (WIKI/<page>, for the page title in the frontmatter), and writes per claim, in the claim list's order:
|
|
quotes, quote_chars, claim_chars the number of excerpts, the characters of their texts summed, of the claim
|
|
content_words the claim's content words (the rule below), repeats counted
|
|
outside, outside_count the claim's content words whose first five letters begin no content word of
|
|
its excerpts, in claim order, repeats counted
|
|
outside_title, outside_headings of those, the ones whose first five letters begin a content word of the page
|
|
title, and of the heading path above each quoted line in its source
|
|
headings that heading path (the headings above the first excerpt's line, outermost
|
|
first), taken from okfwiki.kblocks.headings_of, so the tool counts a heading
|
|
by the same rule as the build, also after order 22 changes that rule
|
|
files, provenance, verdict the excerpts' source files, the claim's provenance mark, and the reader's
|
|
verdict (null without --verdicts, or when it gave the claim none)
|
|
|
|
The word rule (phase 0 of wave 4, 11.10.2026): lower-case the text, take every run of [a-z0-9], drop the words in
|
|
STOPWORDS, and compare two words on their first five letters (a shorter word is compared whole). It describes; it
|
|
judges nothing. A code check by word overlap was ruled out as a gate (PLAN.md § Retning, choice 5: 8 of 664 near
|
|
verbatim), and nothing here is a verdict.
|
|
|
|
--summary prints the phase 0 table: the share rejected per number of words outside the quote (0, 1, 2, 3-4, 5+),
|
|
per provenance mark and per source file, the medians per verdict, and the share with a word from the page title or
|
|
from the heading path outside the quote. Without verdicts it prints the counts only. A claim the reader gave no
|
|
verdict is left out of the share and counted apart as not judged (order 39), so a partial reading does not lower it.
|
|
|
|
Output: json.dumps(doc, indent=1, sort_keys=True, ensure_ascii=False) + newline; two runs on the same inputs are
|
|
byte-identical, and an existing output is never overwritten. Exit 0 written, 2 cannot run (nothing is written).
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
import re
|
|
import statistics
|
|
import sys
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
|
|
|
from okfwiki import kblocks # noqa: E402
|
|
from okfwiki.audit import read_document # noqa: E402
|
|
from okfwiki.common import RunError, write_new # noqa: E402
|
|
|
|
SCHEMA = "okf-wiki/claim-features/1"
|
|
WORD = re.compile(r"[a-z0-9]+")
|
|
STOPWORDS = frozenset(
|
|
"a an the of to in on for and or is are be by with from as at it this that its can you your when if not no into "
|
|
"use uses using".split())
|
|
STEM = 5
|
|
BUCKETS = (("0", 0, 0), ("1", 1, 1), ("2", 2, 2), ("3-4", 3, 4), ("5+", 5, None))
|
|
UNSUPPORTED = "unsupported"
|
|
|
|
|
|
def words(text: str) -> list[str]:
|
|
"""The content words of a text, in order, repeats kept."""
|
|
return [w for w in WORD.findall(text.lower()) if w not in STOPWORDS]
|
|
|
|
|
|
def stems(text: str) -> set[str]:
|
|
return {w[:STEM] for w in words(text)}
|
|
|
|
|
|
def heading_path(lines: list[str], line: int) -> list[str]:
|
|
"""The headings above a line, outermost first: each heading closes every open one at its level or deeper."""
|
|
path: list[tuple[int, str]] = []
|
|
for n, level, text in kblocks.headings_of(lines):
|
|
if n >= line:
|
|
break
|
|
path = [(lv, t) for lv, t in path if lv < level] + [(level, text)]
|
|
return [t for _, t in path]
|
|
|
|
|
|
def read_json(path: Path) -> Any:
|
|
try:
|
|
return json.loads(path.read_text(encoding="utf-8"))
|
|
except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc:
|
|
raise RunError(f"cannot read {path}: {exc}") from exc
|
|
|
|
|
|
def sha256(path: Path) -> str:
|
|
return hashlib.sha256(path.read_bytes()).hexdigest()
|
|
|
|
|
|
def page_title(wiki: Path, page: str) -> str:
|
|
path = wiki / page
|
|
if not path.is_file():
|
|
raise RunError(f"the page {path} is not there")
|
|
fm = read_document(path, page).frontmatter
|
|
return str((fm or {}).get("title") or "")
|
|
|
|
|
|
def features(claims_doc: dict[str, Any], verdicts: dict[str, str] | None, sources: Path,
|
|
wiki: Path) -> list[dict[str, Any]]:
|
|
kilde = kblocks.kilde_of(sources)
|
|
lines_of: dict[str, list[str]] = {}
|
|
titles: dict[str, str] = {}
|
|
out: list[dict[str, Any]] = []
|
|
for c in claims_doc["claims"]:
|
|
excerpts = list(c["excerpts"])
|
|
quote_stems: set[str] = set()
|
|
heading_stems: set[str] = set()
|
|
headings: list[str] = []
|
|
for i, e in enumerate(excerpts):
|
|
name = str(e["file"])
|
|
if name not in lines_of:
|
|
path = kilde / name
|
|
if not path.is_file():
|
|
raise RunError(f"the source {path} is not there")
|
|
lines_of[name] = kblocks.read_lines(path)
|
|
path_here = heading_path(lines_of[name], int(e["line"]))
|
|
headings = path_here if i == 0 else headings
|
|
heading_stems |= {s for h in path_here for s in stems(h)}
|
|
quote_stems |= stems(str(e["text"]))
|
|
page = str(c["page"])
|
|
if page not in titles:
|
|
titles[page] = page_title(wiki, page)
|
|
title_stems = stems(titles[page])
|
|
content = words(str(c["claim"]))
|
|
outside = [w for w in content if w[:STEM] not in quote_stems]
|
|
out.append({
|
|
"id": c["id"], "page": page, "provenance": c.get("provenance"),
|
|
"files": sorted({str(e["file"]) for e in excerpts}),
|
|
"verdict": None if verdicts is None else verdicts.get(str(c["id"])),
|
|
"quotes": len(excerpts), "quote_chars": sum(len(str(e["text"])) for e in excerpts),
|
|
"claim_chars": len(str(c["claim"])), "content_words": len(content),
|
|
"outside": outside, "outside_count": len(outside),
|
|
"outside_title": [w for w in outside if w[:STEM] in title_stems],
|
|
"outside_headings": [w for w in outside if w[:STEM] in heading_stems],
|
|
"headings": headings,
|
|
})
|
|
return out
|
|
|
|
|
|
def _share(rows: list[dict[str, Any]], judged: bool) -> str:
|
|
if not judged:
|
|
return f"{len(rows)} claims, no verdicts"
|
|
bad = sum(1 for r in rows if r["verdict"] == UNSUPPORTED)
|
|
n_judged = sum(1 for r in rows if r["verdict"] is not None)
|
|
if n_judged == len(rows):
|
|
pct = f"{100 * bad / n_judged:.1f} %" if n_judged else "-"
|
|
return f"{len(rows)} claims, {bad} unsupported ({pct})"
|
|
pct = f"{100 * bad / n_judged:.1f} % of {n_judged} judged" if n_judged else "- of 0 judged"
|
|
return f"{len(rows)} claims, {bad} unsupported ({pct}), {len(rows) - n_judged} not judged"
|
|
|
|
|
|
def _median(xs: list[int]) -> str:
|
|
return f"{statistics.median(xs):g}" if xs else "-"
|
|
|
|
|
|
def summary(rows: list[dict[str, Any]]) -> str:
|
|
judged = any(r["verdict"] is not None for r in rows)
|
|
out = [f"claims: {len(rows)}"]
|
|
for label, lo, hi in BUCKETS:
|
|
mine = [r for r in rows if r["outside_count"] >= lo and (hi is None or r["outside_count"] <= hi)]
|
|
out.append(f"words outside the quote {label}: {_share(mine, judged)}")
|
|
for key, label in (("provenance", "provenance"), ("files", "source")):
|
|
values = sorted({v for r in rows for v in (r[key] if isinstance(r[key], list) else [r[key]])}, key=str)
|
|
for v in values:
|
|
mine = [r for r in rows if v in (r[key] if isinstance(r[key], list) else [r[key]])]
|
|
out.append(f"{label} {v}: {_share(mine, judged)}")
|
|
if not judged:
|
|
return "\n".join(out) + "\n"
|
|
groups = {name: [r for r in rows if (r["verdict"] == UNSUPPORTED) == (name == UNSUPPORTED)
|
|
and r["verdict"] is not None] for name in ("supported", UNSUPPORTED)}
|
|
for field, label in (("outside_count", "words outside"), ("quote_chars", "quote characters"),
|
|
("claim_chars", "claim characters"), ("content_words", "content words")):
|
|
parts = [f"{name} {_median([int(r[field]) for r in g])}" for name, g in groups.items()]
|
|
out.append(f"median {label}: {', '.join(parts)}")
|
|
for field, label in (("outside_title", "the page title"), ("outside_headings", "the heading path")):
|
|
parts = []
|
|
for name, g in groups.items():
|
|
k = sum(1 for r in g if r[field])
|
|
pct = f"{100 * k / len(g):.1f} %" if g else "-"
|
|
parts.append(f"{name} {k} of {len(g)} ({pct})")
|
|
out.append(f"with a word from {label} outside the quote: {', '.join(parts)}")
|
|
return "\n".join(out) + "\n"
|
|
|
|
|
|
def build_parser() -> argparse.ArgumentParser:
|
|
p = argparse.ArgumentParser(prog="python3 tools/claim_features.py", description=__doc__,
|
|
formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
p.add_argument("--claims", required=True)
|
|
p.add_argument("--sources", required=True)
|
|
p.add_argument("--wiki", required=True)
|
|
p.add_argument("--out", required=True)
|
|
p.add_argument("--verdicts")
|
|
p.add_argument("--summary", action="store_true")
|
|
return p
|
|
|
|
|
|
def main(argv: list[str] | None = None) -> int:
|
|
args = build_parser().parse_args(argv)
|
|
try:
|
|
out = Path(args.out)
|
|
if out.exists():
|
|
raise RunError(f"{out} exists; an output is never overwritten")
|
|
claims_path = Path(args.claims)
|
|
claims_doc = read_json(claims_path)
|
|
verdicts: dict[str, str] | None = None
|
|
inputs: dict[str, Any] = {"claims_sha256": sha256(claims_path), "verdicts_sha256": None}
|
|
if args.verdicts:
|
|
vpath = Path(args.verdicts)
|
|
verdicts = {str(v["id"]): str(v["verdict"]) for v in read_json(vpath)["verdicts"]}
|
|
inputs["verdicts_sha256"] = sha256(vpath)
|
|
rows = features(claims_doc, verdicts, Path(args.sources), Path(args.wiki))
|
|
doc = {"schema": SCHEMA, "inputs": inputs, "claims": rows}
|
|
write_new(out, json.dumps(doc, indent=1, sort_keys=True, ensure_ascii=False) + "\n")
|
|
except (RunError, OSError, KeyError, TypeError, ValueError) as exc:
|
|
print(f"claim_features cannot run: {exc}", file=sys.stderr)
|
|
return 2
|
|
print(f"claim_features: {len(rows)} claims -> {out}")
|
|
if args.summary:
|
|
print(summary(rows), end="")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|