okf-wiki/evals/p1/measure.py
Kjell Tore Guttormsen 0fd86a23a3
okf-wiki 1.1.1
First public release, published as one snapshot. What changed between
versions is in CHANGELOG.md.

Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
2026-11-14 21:03:20 +01:00

361 lines
15 KiB
Python

"""The trial's measures without a reader (TRIAL, P1): audit record, coverage, structure. It calls no model.
audit --bundle B --root R --variant V --out FILE
Runs the auditor in process and records, per variant, the exit status, findings per check, uncited
sentences by cause (fence line, table row, link lead-in, prose), "yaml form" findings (a quote that YAML
loaded as another type, a timestamp in the wrong form) and criterion 1 (N of N, claims.py's count).
coverage --bundle B --root R --blocks F --built N [--pack-size K] --out FILE
Blocks of the first N sources in the order with at least one body-cited source entry whose locator line
lies inside the block (spans are not used), X of N blocks; per pack position with --pack-size.
structure --bundle B --root R --blocks F [--skeleton I] --out FILE
Pages, concepts per source (the earliest source in the order among a page's body-cited sources),
skeleton entries without a page, duplicates (same name, alias or title Jaccard >= 0.6), merge
candidates, islands, sources per page, version numbers in running text, concept tests 2 and 4,
the share of quotes on a structural line and of code-supplied quotes (version sources left out), and the
precision of each body-cited entry (line, block, section, document).
--root is the folder that contains kilde/. Before reporting, structure and coverage check that at least one quote
stands at its locator under --root, because a wrong root makes every quote unreadable. Exit 0 = written, 2 = could
not run (an existing output, or a variant already in the audit record, is never overwritten).
"""
from __future__ import annotations
import argparse
import json
import sys
from collections import Counter
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any
from okfwiki import claims as claims_mod
from okfwiki.audit import (
FOOTNOTE_REF,
LINK,
LINK_ONLY,
Document,
audit,
body_links,
is_bundle_link,
load_bundle,
mask_code,
prose_blocks,
resolve,
sentences,
)
from okfwiki.view import is_version_heading, related, versions
from p1.assemble import structural
from p1.kpack import MERGE_JACCARD, norm
from p1.split import FENCE, is_meta
STRUCTURE_SCHEMA = "okf-wiki/p1-structure/1"
RESULT_SCHEMA = "okf-wiki/p1-result/1"
class RunError(Exception):
"""The measure cannot run as asked; nothing was written."""
@dataclass
class Coverage:
covered: int
total: int
per_position: dict[int, tuple[int, int, int]] = field(default_factory=dict)
def _entries(doc: Document) -> dict[str, dict[str, Any]]:
srcs = (doc.frontmatter or {}).get("sources")
return {str(s["id"]): s for s in srcs if isinstance(s, dict) and "id" in s} if isinstance(srcs, list) else {}
def cited(doc: Document) -> list[dict[str, Any]]:
"""The source entries a body footnote uses, in label order of first use."""
entries = _entries(doc)
labels: list[str] = []
for block in prose_blocks(doc):
for label in FOOTNOTE_REF.findall(mask_code(block.text)):
if label not in labels:
labels.append(label)
return [entries[x] for x in labels if x in entries]
def _file(s: dict[str, Any]) -> str:
return Path(str(s.get("resource", ""))).name
def _line(s: dict[str, Any]) -> int | None:
loc = s.get("x-locator")
return loc["line"] if isinstance(loc, dict) and type(loc.get("line")) is int else None
class _Lines:
def __init__(self, root: Path) -> None:
self.root = root
self.cache: dict[str, list[str] | None] = {}
def get(self, name: str) -> list[str] | None:
if name not in self.cache:
p = self.root / "kilde" / name
self.cache[name] = p.read_text(encoding="utf-8").split("\n") if p.is_file() else None
return self.cache[name]
def _check_root(concepts: list[Document], root: Path) -> None:
lines = _Lines(root)
seen = 0
for d in concepts:
for s in _entries(d).values():
n, quote, text = _line(s), s.get("x-quote"), lines.get(_file(s))
seen += 1
if n and isinstance(quote, str) and text and n <= len(text) and quote in text[n - 1]:
return
if seen:
raise RunError(f"no quote stands at its locator under {root}; is it the folder that contains kilde/?")
def _block_of(blocks: list[dict[str, Any]], file: str, line: int) -> dict[str, Any] | None:
return next((b for b in blocks if b["file"] == file and b["start"] <= line <= b["end"]), None)
def coverage(bundle: Path, root: Path, blocks_doc: dict[str, Any], built: list[str],
pack_size: int | None = None) -> Coverage:
concepts = load_bundle(bundle, root).concepts
_check_root(concepts, root)
blocks = [b for b in blocks_doc["blocks"] if b["file"] in built]
hit: set[tuple[str, int]] = set()
for d in concepts:
for s in cited(d):
n = _line(s)
b = _block_of(blocks, _file(s), n) if n else None
if b is not None:
hit.add((b["file"], b["start"]))
result = Coverage(len(hit), len(blocks))
if pack_size:
names = [str(x) for x in blocks_doc["order"] if x in built]
for pos in range(1, pack_size + 1):
files = {n for i, n in enumerate(names) if i % pack_size == pos - 1}
if files:
mine = [b for b in blocks if b["file"] in files]
result.per_position[pos] = (sum((b["file"], b["start"]) in hit for b in mine), len(mine), len(files))
return result
def _jaccard(a: str, b: str) -> float:
x, y = set(a.split()), set(b.split())
return len(x & y) / len(x | y) if x | y else 0.0
def duplicates(concepts: list[Document]) -> list[tuple[str, str, str]]:
info = []
for d in concepts:
fm = d.frontmatter or {}
aliases = fm.get("x-aliases")
info.append((d.rel, norm(str(fm.get("title", ""))),
{norm(str(a)) for a in aliases} if isinstance(aliases, list) else set()))
pairs: list[tuple[str, str, str]] = []
for i, (a, na, aa) in enumerate(info):
for b, nb, ab in info[i + 1:]:
if na and na == nb:
pairs.append((a, b, "norm"))
elif na in ab or nb in aa:
pairs.append((a, b, "alias"))
elif _jaccard(na, nb) >= MERGE_JACCARD:
pairs.append((a, b, "jaccard"))
return pairs
def _kind(d: Document) -> str:
fm = d.frontmatter or {}
if fm.get("x-merge-candidate"):
return "merge-candidate"
return {"Document map": "map", "Register entry": "register"}.get(str(fm.get("type", "")), "concept")
def _precision(s: dict[str, Any], blocks: list[dict[str, Any]]) -> str:
span = s.get("x-span")
n = _line(s) or 0
a, z = (span[0], span[1]) if isinstance(span, list) and len(span) == 2 else (n, n)
if a == z:
return "line"
inside = [b for b in blocks if b["file"] == _file(s) and b["end"] >= a and b["start"] <= z]
if len(inside) == 1 and inside[0]["start"] <= a and z <= inside[0]["end"]:
return "block"
if inside and len({b["section"] for b in inside}) == 1:
return "section"
return "document"
def structure(bundle: Path, root: Path, blocks_doc: dict[str, Any], order: list[str],
skeleton: dict[str, Any] | None = None) -> dict[str, Any]:
concepts = load_bundle(bundle, root).concepts
_check_root(concepts, root)
lines = _Lines(root)
blocks = list(blocks_doc["blocks"])
rank = {n: i for i, n in enumerate(order)}
per_source: Counter[str] = Counter()
incoming: Counter[str] = Counter()
rels = {d.rel for d in concepts}
for d in concepts:
for _line_no, target in body_links(d):
other = resolve(d.rel, target) if is_bundle_link(target) else None
if other in rels and other != d.rel:
incoming[str(other)] += 1
quotes: Counter[str] = Counter()
precision: Counter[str] = Counter()
docs_per_page: list[int] = []
test2 = test4 = 0
for d in concepts:
cs = cited(d)
files = [_file(s) for s in cs if _file(s) in rank]
if files:
per_source[min(files, key=lambda f: rank[f])] += 1
docs_per_page.append(len(set(_file(s) for s in cs)))
title = str((d.frontmatter or {}).get("title", ""))
test2 += not is_meta(title)
test4 += incoming[d.rel] >= 2 or len(set(files)) >= 2
for s in cs:
if is_version_heading(s):
continue
quotes["entries"] += 1
quotes["code"] += s.get("x-quote-by") == "code"
text, n = lines.get(_file(s)), _line(s)
quotes["structural"] += bool(text and n and n <= len(text) and structural(text[n - 1]))
precision[_precision(s, blocks)] += 1
rel_map = related(concepts)
kinds = {d.rel: _kind(d) for d in concepts}
islands: dict[str, Any] = {"total": sum(not v for v in rel_map.values())}
for k in sorted(set(kinds.values())):
islands[k] = sum(not rel_map[r] for r, kk in kinds.items() if kk == k)
pages = {d.rel for d in concepts}
unused = None
if skeleton is not None:
unused = sum(e.get("kind") == "skeleton" and f"{e['id']}.md" not in pages for e in skeleton["entries"])
tests13 = None
if skeleton is not None:
new_ids = {e["id"] for e in skeleton["entries"] if e.get("kind") == "new" and e.get("tests")}
tests13 = sum(f"{i}.md" in pages for i in new_ids)
v = versions(bundle, root)
dups = duplicates(concepts)
return {
"schema": STRUCTURE_SCHEMA,
"pages": len(concepts),
"page_kinds": dict(Counter(kinds.values())),
"concepts_per_source": {n: per_source[n] for n in order if per_source[n]},
"skeleton_unused": unused,
"duplicates": {"pairs": len(dups), "by_rule": dict(Counter(r for _, _, r in dups)),
"list": [list(p) for p in dups]},
"merge_candidates": sum(k == "merge-candidate" for k in kinds.values()),
"islands": islands,
"documents_per_page": {"mean": round(sum(docs_per_page) / len(docs_per_page), 2) if docs_per_page else 0,
"max": max(docs_per_page, default=0)},
"version_numbers_in_text": {"flagged": len(v.flagged), "claims": v.total},
"concept_tests": {"test2": test2, "test4": test4, "tests1and3_self_declared": tests13},
"quotes": {"entries": quotes["entries"], "structural": quotes["structural"], "code": quotes["code"]},
"precision": {k: precision[k] for k in ("line", "block", "section", "document")},
}
def _uncited_cause(sentence: str, block_text: str) -> str:
s = sentence.strip()
if FENCE.match(block_text): # the auditor's sentence text masks code spans, so a fence is read on the block
return "fence line"
if s.startswith("|"):
return "table row"
if LINK.search(block_text) and not LINK_ONLY.match(mask_code(block_text)):
return "link lead-in"
return "prose"
def audit_record(bundle: Path, root: Path, variant: str, out: Path) -> dict[str, Any]:
record_all: dict[str, Any] = (json.loads(out.read_text(encoding="utf-8")) if out.is_file()
else {"schema": RESULT_SCHEMA, "variants": {}})
if variant in record_all["variants"]:
raise RunError(f"{out} already records variant {variant}")
b = load_bundle(bundle, root)
found = audit(bundle, root)
uncited: Counter[str] = Counter()
for d in b.concepts:
for block in prose_blocks(d):
if LINK_ONLY.match(mask_code(block.text)):
continue
for s in sentences(block.text):
if not FOOTNOTE_REF.search(s):
uncited[_uncited_cause(s, block.text)] += 1
yaml_form = 0
for d in b.concepts:
for entry in _entries(d).values():
q = entry.get("x-quote")
yaml_form += q is not None and not isinstance(q, str)
yaml_form += sum(f.check == "timestamp-form" for f in found)
listed = claims_mod.extract(bundle, root)
ok, total = claims_mod.criterion_1(listed)["total"]
rec = {"exit": 1 if found else 0, "findings": dict(sorted(Counter(f.check for f in found).items())),
"uncited": dict(sorted(uncited.items())), "yaml_form": yaml_form, "criterion_1": [ok, total]}
record_all["variants"][variant] = rec
out.parent.mkdir(parents=True, exist_ok=True)
out.write_text(json.dumps(record_all, ensure_ascii=False, indent=1) + "\n", encoding="utf-8")
return rec
def _write_new(path: Path, data: Any) -> None:
if path.exists():
raise RunError(f"{path} exists; refusing to overwrite")
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(data, ensure_ascii=False, indent=1) + "\n", encoding="utf-8")
def build_parser() -> argparse.ArgumentParser:
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
sub = p.add_subparsers(dest="command", required=True)
a = sub.add_parser("audit")
a.add_argument("--bundle", required=True)
a.add_argument("--root", required=True)
a.add_argument("--variant", required=True)
a.add_argument("--out", required=True)
c = sub.add_parser("coverage")
s = sub.add_parser("structure")
for x in (c, s):
x.add_argument("--bundle", required=True)
x.add_argument("--root", required=True)
x.add_argument("--blocks", required=True)
x.add_argument("--out", required=True)
c.add_argument("--built", type=int, required=True)
c.add_argument("--pack-size", type=int)
s.add_argument("--skeleton")
return p
def main(argv: list[str] | None = None) -> int:
args = build_parser().parse_args(argv)
try:
bundle, root = Path(args.bundle), Path(args.root)
if not bundle.is_dir():
raise RunError(f"the bundle {bundle} is not a directory")
if args.command == "audit":
rec = audit_record(bundle, root, args.variant, Path(args.out))
print(f"{args.variant}: audit exit {rec['exit']}, criterion 1 {rec['criterion_1'][0]} of "
f"{rec['criterion_1'][1]}, findings {rec['findings']}")
return 0
blocks_doc = json.loads(Path(args.blocks).read_text(encoding="utf-8"))
order = [str(x) for x in blocks_doc["order"]]
if args.command == "coverage":
cov = coverage(bundle, root, blocks_doc, order[:args.built], args.pack_size)
_write_new(Path(args.out), {"schema": "okf-wiki/p1-coverage/1", "covered": cov.covered, "total": cov.total,
"built": args.built,
"per_position": {str(k): list(v) for k, v in cov.per_position.items()}})
print(f"coverage: {cov.covered} of {cov.total} blocks")
return 0
skeleton = json.loads(Path(args.skeleton).read_text(encoding="utf-8")) if args.skeleton else None
data = structure(bundle, root, blocks_doc, order, skeleton)
_write_new(Path(args.out), data)
print(f"structure: {data['pages']} pages, {data['duplicates']['pairs']} duplicate pairs, "
f"{data['islands']['total']} islands")
return 0
except (RunError, OSError, json.JSONDecodeError, KeyError) as exc:
print(f"measure cannot run: {exc}", file=sys.stderr)
return 2
if __name__ == "__main__":
sys.exit(main())