First public release, published as one snapshot. What changed between versions is in CHANGELOG.md. Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
361 lines
15 KiB
Python
361 lines
15 KiB
Python
"""The trial's measures without a reader (TRIAL, P1): audit record, coverage, structure. It calls no model.
|
|
|
|
audit --bundle B --root R --variant V --out FILE
|
|
Runs the auditor in process and records, per variant, the exit status, findings per check, uncited
|
|
sentences by cause (fence line, table row, link lead-in, prose), "yaml form" findings (a quote that YAML
|
|
loaded as another type, a timestamp in the wrong form) and criterion 1 (N of N, claims.py's count).
|
|
coverage --bundle B --root R --blocks F --built N [--pack-size K] --out FILE
|
|
Blocks of the first N sources in the order with at least one body-cited source entry whose locator line
|
|
lies inside the block (spans are not used), X of N blocks; per pack position with --pack-size.
|
|
structure --bundle B --root R --blocks F [--skeleton I] --out FILE
|
|
Pages, concepts per source (the earliest source in the order among a page's body-cited sources),
|
|
skeleton entries without a page, duplicates (same name, alias or title Jaccard >= 0.6), merge
|
|
candidates, islands, sources per page, version numbers in running text, concept tests 2 and 4,
|
|
the share of quotes on a structural line and of code-supplied quotes (version sources left out), and the
|
|
precision of each body-cited entry (line, block, section, document).
|
|
|
|
--root is the folder that contains kilde/. Before reporting, structure and coverage check that at least one quote
|
|
stands at its locator under --root, because a wrong root makes every quote unreadable. Exit 0 = written, 2 = could
|
|
not run (an existing output, or a variant already in the audit record, is never overwritten).
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import sys
|
|
from collections import Counter
|
|
from dataclasses import dataclass, field
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
from okfwiki import claims as claims_mod
|
|
from okfwiki.audit import (
|
|
FOOTNOTE_REF,
|
|
LINK,
|
|
LINK_ONLY,
|
|
Document,
|
|
audit,
|
|
body_links,
|
|
is_bundle_link,
|
|
load_bundle,
|
|
mask_code,
|
|
prose_blocks,
|
|
resolve,
|
|
sentences,
|
|
)
|
|
from okfwiki.view import is_version_heading, related, versions
|
|
from p1.assemble import structural
|
|
from p1.kpack import MERGE_JACCARD, norm
|
|
from p1.split import FENCE, is_meta
|
|
|
|
STRUCTURE_SCHEMA = "okf-wiki/p1-structure/1"
|
|
RESULT_SCHEMA = "okf-wiki/p1-result/1"
|
|
|
|
|
|
class RunError(Exception):
|
|
"""The measure cannot run as asked; nothing was written."""
|
|
|
|
|
|
@dataclass
|
|
class Coverage:
|
|
covered: int
|
|
total: int
|
|
per_position: dict[int, tuple[int, int, int]] = field(default_factory=dict)
|
|
|
|
|
|
def _entries(doc: Document) -> dict[str, dict[str, Any]]:
|
|
srcs = (doc.frontmatter or {}).get("sources")
|
|
return {str(s["id"]): s for s in srcs if isinstance(s, dict) and "id" in s} if isinstance(srcs, list) else {}
|
|
|
|
|
|
def cited(doc: Document) -> list[dict[str, Any]]:
|
|
"""The source entries a body footnote uses, in label order of first use."""
|
|
entries = _entries(doc)
|
|
labels: list[str] = []
|
|
for block in prose_blocks(doc):
|
|
for label in FOOTNOTE_REF.findall(mask_code(block.text)):
|
|
if label not in labels:
|
|
labels.append(label)
|
|
return [entries[x] for x in labels if x in entries]
|
|
|
|
|
|
def _file(s: dict[str, Any]) -> str:
|
|
return Path(str(s.get("resource", ""))).name
|
|
|
|
|
|
def _line(s: dict[str, Any]) -> int | None:
|
|
loc = s.get("x-locator")
|
|
return loc["line"] if isinstance(loc, dict) and type(loc.get("line")) is int else None
|
|
|
|
|
|
class _Lines:
|
|
def __init__(self, root: Path) -> None:
|
|
self.root = root
|
|
self.cache: dict[str, list[str] | None] = {}
|
|
|
|
def get(self, name: str) -> list[str] | None:
|
|
if name not in self.cache:
|
|
p = self.root / "kilde" / name
|
|
self.cache[name] = p.read_text(encoding="utf-8").split("\n") if p.is_file() else None
|
|
return self.cache[name]
|
|
|
|
|
|
def _check_root(concepts: list[Document], root: Path) -> None:
|
|
lines = _Lines(root)
|
|
seen = 0
|
|
for d in concepts:
|
|
for s in _entries(d).values():
|
|
n, quote, text = _line(s), s.get("x-quote"), lines.get(_file(s))
|
|
seen += 1
|
|
if n and isinstance(quote, str) and text and n <= len(text) and quote in text[n - 1]:
|
|
return
|
|
if seen:
|
|
raise RunError(f"no quote stands at its locator under {root}; is it the folder that contains kilde/?")
|
|
|
|
|
|
def _block_of(blocks: list[dict[str, Any]], file: str, line: int) -> dict[str, Any] | None:
|
|
return next((b for b in blocks if b["file"] == file and b["start"] <= line <= b["end"]), None)
|
|
|
|
|
|
def coverage(bundle: Path, root: Path, blocks_doc: dict[str, Any], built: list[str],
|
|
pack_size: int | None = None) -> Coverage:
|
|
concepts = load_bundle(bundle, root).concepts
|
|
_check_root(concepts, root)
|
|
blocks = [b for b in blocks_doc["blocks"] if b["file"] in built]
|
|
hit: set[tuple[str, int]] = set()
|
|
for d in concepts:
|
|
for s in cited(d):
|
|
n = _line(s)
|
|
b = _block_of(blocks, _file(s), n) if n else None
|
|
if b is not None:
|
|
hit.add((b["file"], b["start"]))
|
|
result = Coverage(len(hit), len(blocks))
|
|
if pack_size:
|
|
names = [str(x) for x in blocks_doc["order"] if x in built]
|
|
for pos in range(1, pack_size + 1):
|
|
files = {n for i, n in enumerate(names) if i % pack_size == pos - 1}
|
|
if files:
|
|
mine = [b for b in blocks if b["file"] in files]
|
|
result.per_position[pos] = (sum((b["file"], b["start"]) in hit for b in mine), len(mine), len(files))
|
|
return result
|
|
|
|
|
|
def _jaccard(a: str, b: str) -> float:
|
|
x, y = set(a.split()), set(b.split())
|
|
return len(x & y) / len(x | y) if x | y else 0.0
|
|
|
|
|
|
def duplicates(concepts: list[Document]) -> list[tuple[str, str, str]]:
|
|
info = []
|
|
for d in concepts:
|
|
fm = d.frontmatter or {}
|
|
aliases = fm.get("x-aliases")
|
|
info.append((d.rel, norm(str(fm.get("title", ""))),
|
|
{norm(str(a)) for a in aliases} if isinstance(aliases, list) else set()))
|
|
pairs: list[tuple[str, str, str]] = []
|
|
for i, (a, na, aa) in enumerate(info):
|
|
for b, nb, ab in info[i + 1:]:
|
|
if na and na == nb:
|
|
pairs.append((a, b, "norm"))
|
|
elif na in ab or nb in aa:
|
|
pairs.append((a, b, "alias"))
|
|
elif _jaccard(na, nb) >= MERGE_JACCARD:
|
|
pairs.append((a, b, "jaccard"))
|
|
return pairs
|
|
|
|
|
|
def _kind(d: Document) -> str:
|
|
fm = d.frontmatter or {}
|
|
if fm.get("x-merge-candidate"):
|
|
return "merge-candidate"
|
|
return {"Document map": "map", "Register entry": "register"}.get(str(fm.get("type", "")), "concept")
|
|
|
|
|
|
def _precision(s: dict[str, Any], blocks: list[dict[str, Any]]) -> str:
|
|
span = s.get("x-span")
|
|
n = _line(s) or 0
|
|
a, z = (span[0], span[1]) if isinstance(span, list) and len(span) == 2 else (n, n)
|
|
if a == z:
|
|
return "line"
|
|
inside = [b for b in blocks if b["file"] == _file(s) and b["end"] >= a and b["start"] <= z]
|
|
if len(inside) == 1 and inside[0]["start"] <= a and z <= inside[0]["end"]:
|
|
return "block"
|
|
if inside and len({b["section"] for b in inside}) == 1:
|
|
return "section"
|
|
return "document"
|
|
|
|
|
|
def structure(bundle: Path, root: Path, blocks_doc: dict[str, Any], order: list[str],
|
|
skeleton: dict[str, Any] | None = None) -> dict[str, Any]:
|
|
concepts = load_bundle(bundle, root).concepts
|
|
_check_root(concepts, root)
|
|
lines = _Lines(root)
|
|
blocks = list(blocks_doc["blocks"])
|
|
rank = {n: i for i, n in enumerate(order)}
|
|
per_source: Counter[str] = Counter()
|
|
incoming: Counter[str] = Counter()
|
|
rels = {d.rel for d in concepts}
|
|
for d in concepts:
|
|
for _line_no, target in body_links(d):
|
|
other = resolve(d.rel, target) if is_bundle_link(target) else None
|
|
if other in rels and other != d.rel:
|
|
incoming[str(other)] += 1
|
|
quotes: Counter[str] = Counter()
|
|
precision: Counter[str] = Counter()
|
|
docs_per_page: list[int] = []
|
|
test2 = test4 = 0
|
|
for d in concepts:
|
|
cs = cited(d)
|
|
files = [_file(s) for s in cs if _file(s) in rank]
|
|
if files:
|
|
per_source[min(files, key=lambda f: rank[f])] += 1
|
|
docs_per_page.append(len(set(_file(s) for s in cs)))
|
|
title = str((d.frontmatter or {}).get("title", ""))
|
|
test2 += not is_meta(title)
|
|
test4 += incoming[d.rel] >= 2 or len(set(files)) >= 2
|
|
for s in cs:
|
|
if is_version_heading(s):
|
|
continue
|
|
quotes["entries"] += 1
|
|
quotes["code"] += s.get("x-quote-by") == "code"
|
|
text, n = lines.get(_file(s)), _line(s)
|
|
quotes["structural"] += bool(text and n and n <= len(text) and structural(text[n - 1]))
|
|
precision[_precision(s, blocks)] += 1
|
|
rel_map = related(concepts)
|
|
kinds = {d.rel: _kind(d) for d in concepts}
|
|
islands: dict[str, Any] = {"total": sum(not v for v in rel_map.values())}
|
|
for k in sorted(set(kinds.values())):
|
|
islands[k] = sum(not rel_map[r] for r, kk in kinds.items() if kk == k)
|
|
pages = {d.rel for d in concepts}
|
|
unused = None
|
|
if skeleton is not None:
|
|
unused = sum(e.get("kind") == "skeleton" and f"{e['id']}.md" not in pages for e in skeleton["entries"])
|
|
tests13 = None
|
|
if skeleton is not None:
|
|
new_ids = {e["id"] for e in skeleton["entries"] if e.get("kind") == "new" and e.get("tests")}
|
|
tests13 = sum(f"{i}.md" in pages for i in new_ids)
|
|
v = versions(bundle, root)
|
|
dups = duplicates(concepts)
|
|
return {
|
|
"schema": STRUCTURE_SCHEMA,
|
|
"pages": len(concepts),
|
|
"page_kinds": dict(Counter(kinds.values())),
|
|
"concepts_per_source": {n: per_source[n] for n in order if per_source[n]},
|
|
"skeleton_unused": unused,
|
|
"duplicates": {"pairs": len(dups), "by_rule": dict(Counter(r for _, _, r in dups)),
|
|
"list": [list(p) for p in dups]},
|
|
"merge_candidates": sum(k == "merge-candidate" for k in kinds.values()),
|
|
"islands": islands,
|
|
"documents_per_page": {"mean": round(sum(docs_per_page) / len(docs_per_page), 2) if docs_per_page else 0,
|
|
"max": max(docs_per_page, default=0)},
|
|
"version_numbers_in_text": {"flagged": len(v.flagged), "claims": v.total},
|
|
"concept_tests": {"test2": test2, "test4": test4, "tests1and3_self_declared": tests13},
|
|
"quotes": {"entries": quotes["entries"], "structural": quotes["structural"], "code": quotes["code"]},
|
|
"precision": {k: precision[k] for k in ("line", "block", "section", "document")},
|
|
}
|
|
|
|
|
|
def _uncited_cause(sentence: str, block_text: str) -> str:
|
|
s = sentence.strip()
|
|
if FENCE.match(block_text): # the auditor's sentence text masks code spans, so a fence is read on the block
|
|
return "fence line"
|
|
if s.startswith("|"):
|
|
return "table row"
|
|
if LINK.search(block_text) and not LINK_ONLY.match(mask_code(block_text)):
|
|
return "link lead-in"
|
|
return "prose"
|
|
|
|
|
|
def audit_record(bundle: Path, root: Path, variant: str, out: Path) -> dict[str, Any]:
|
|
record_all: dict[str, Any] = (json.loads(out.read_text(encoding="utf-8")) if out.is_file()
|
|
else {"schema": RESULT_SCHEMA, "variants": {}})
|
|
if variant in record_all["variants"]:
|
|
raise RunError(f"{out} already records variant {variant}")
|
|
b = load_bundle(bundle, root)
|
|
found = audit(bundle, root)
|
|
uncited: Counter[str] = Counter()
|
|
for d in b.concepts:
|
|
for block in prose_blocks(d):
|
|
if LINK_ONLY.match(mask_code(block.text)):
|
|
continue
|
|
for s in sentences(block.text):
|
|
if not FOOTNOTE_REF.search(s):
|
|
uncited[_uncited_cause(s, block.text)] += 1
|
|
yaml_form = 0
|
|
for d in b.concepts:
|
|
for entry in _entries(d).values():
|
|
q = entry.get("x-quote")
|
|
yaml_form += q is not None and not isinstance(q, str)
|
|
yaml_form += sum(f.check == "timestamp-form" for f in found)
|
|
listed = claims_mod.extract(bundle, root)
|
|
ok, total = claims_mod.criterion_1(listed)["total"]
|
|
rec = {"exit": 1 if found else 0, "findings": dict(sorted(Counter(f.check for f in found).items())),
|
|
"uncited": dict(sorted(uncited.items())), "yaml_form": yaml_form, "criterion_1": [ok, total]}
|
|
record_all["variants"][variant] = rec
|
|
out.parent.mkdir(parents=True, exist_ok=True)
|
|
out.write_text(json.dumps(record_all, ensure_ascii=False, indent=1) + "\n", encoding="utf-8")
|
|
return rec
|
|
|
|
|
|
def _write_new(path: Path, data: Any) -> None:
|
|
if path.exists():
|
|
raise RunError(f"{path} exists; refusing to overwrite")
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
path.write_text(json.dumps(data, ensure_ascii=False, indent=1) + "\n", encoding="utf-8")
|
|
|
|
|
|
def build_parser() -> argparse.ArgumentParser:
|
|
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
sub = p.add_subparsers(dest="command", required=True)
|
|
a = sub.add_parser("audit")
|
|
a.add_argument("--bundle", required=True)
|
|
a.add_argument("--root", required=True)
|
|
a.add_argument("--variant", required=True)
|
|
a.add_argument("--out", required=True)
|
|
c = sub.add_parser("coverage")
|
|
s = sub.add_parser("structure")
|
|
for x in (c, s):
|
|
x.add_argument("--bundle", required=True)
|
|
x.add_argument("--root", required=True)
|
|
x.add_argument("--blocks", required=True)
|
|
x.add_argument("--out", required=True)
|
|
c.add_argument("--built", type=int, required=True)
|
|
c.add_argument("--pack-size", type=int)
|
|
s.add_argument("--skeleton")
|
|
return p
|
|
|
|
|
|
def main(argv: list[str] | None = None) -> int:
|
|
args = build_parser().parse_args(argv)
|
|
try:
|
|
bundle, root = Path(args.bundle), Path(args.root)
|
|
if not bundle.is_dir():
|
|
raise RunError(f"the bundle {bundle} is not a directory")
|
|
if args.command == "audit":
|
|
rec = audit_record(bundle, root, args.variant, Path(args.out))
|
|
print(f"{args.variant}: audit exit {rec['exit']}, criterion 1 {rec['criterion_1'][0]} of "
|
|
f"{rec['criterion_1'][1]}, findings {rec['findings']}")
|
|
return 0
|
|
blocks_doc = json.loads(Path(args.blocks).read_text(encoding="utf-8"))
|
|
order = [str(x) for x in blocks_doc["order"]]
|
|
if args.command == "coverage":
|
|
cov = coverage(bundle, root, blocks_doc, order[:args.built], args.pack_size)
|
|
_write_new(Path(args.out), {"schema": "okf-wiki/p1-coverage/1", "covered": cov.covered, "total": cov.total,
|
|
"built": args.built,
|
|
"per_position": {str(k): list(v) for k, v in cov.per_position.items()}})
|
|
print(f"coverage: {cov.covered} of {cov.total} blocks")
|
|
return 0
|
|
skeleton = json.loads(Path(args.skeleton).read_text(encoding="utf-8")) if args.skeleton else None
|
|
data = structure(bundle, root, blocks_doc, order, skeleton)
|
|
_write_new(Path(args.out), data)
|
|
print(f"structure: {data['pages']} pages, {data['duplicates']['pairs']} duplicate pairs, "
|
|
f"{data['islands']['total']} islands")
|
|
return 0
|
|
except (RunError, OSError, json.JSONDecodeError, KeyError) as exc:
|
|
print(f"measure cannot run: {exc}", file=sys.stderr)
|
|
return 2
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|