"""The trial's measures without a reader (TRIAL, P1): audit record, coverage, structure. It calls no model. audit --bundle B --root R --variant V --out FILE Runs the auditor in process and records, per variant, the exit status, findings per check, uncited sentences by cause (fence line, table row, link lead-in, prose), "yaml form" findings (a quote that YAML loaded as another type, a timestamp in the wrong form) and criterion 1 (N of N, claims.py's count). coverage --bundle B --root R --blocks F --built N [--pack-size K] --out FILE Blocks of the first N sources in the order with at least one body-cited source entry whose locator line lies inside the block (spans are not used), X of N blocks; per pack position with --pack-size. structure --bundle B --root R --blocks F [--skeleton I] --out FILE Pages, concepts per source (the earliest source in the order among a page's body-cited sources), skeleton entries without a page, duplicates (same name, alias or title Jaccard >= 0.6), merge candidates, islands, sources per page, version numbers in running text, concept tests 2 and 4, the share of quotes on a structural line and of code-supplied quotes (version sources left out), and the precision of each body-cited entry (line, block, section, document). --root is the folder that contains kilde/. Before reporting, structure and coverage check that at least one quote stands at its locator under --root, because a wrong root makes every quote unreadable. Exit 0 = written, 2 = could not run (an existing output, or a variant already in the audit record, is never overwritten). """ from __future__ import annotations import argparse import json import sys from collections import Counter from dataclasses import dataclass, field from pathlib import Path from typing import Any from okfwiki import claims as claims_mod from okfwiki.audit import ( FOOTNOTE_REF, LINK, LINK_ONLY, Document, audit, body_links, is_bundle_link, load_bundle, mask_code, prose_blocks, resolve, sentences, ) from okfwiki.view import is_version_heading, related, versions from p1.assemble import structural from p1.kpack import MERGE_JACCARD, norm from p1.split import FENCE, is_meta STRUCTURE_SCHEMA = "okf-wiki/p1-structure/1" RESULT_SCHEMA = "okf-wiki/p1-result/1" class RunError(Exception): """The measure cannot run as asked; nothing was written.""" @dataclass class Coverage: covered: int total: int per_position: dict[int, tuple[int, int, int]] = field(default_factory=dict) def _entries(doc: Document) -> dict[str, dict[str, Any]]: srcs = (doc.frontmatter or {}).get("sources") return {str(s["id"]): s for s in srcs if isinstance(s, dict) and "id" in s} if isinstance(srcs, list) else {} def cited(doc: Document) -> list[dict[str, Any]]: """The source entries a body footnote uses, in label order of first use.""" entries = _entries(doc) labels: list[str] = [] for block in prose_blocks(doc): for label in FOOTNOTE_REF.findall(mask_code(block.text)): if label not in labels: labels.append(label) return [entries[x] for x in labels if x in entries] def _file(s: dict[str, Any]) -> str: return Path(str(s.get("resource", ""))).name def _line(s: dict[str, Any]) -> int | None: loc = s.get("x-locator") return loc["line"] if isinstance(loc, dict) and type(loc.get("line")) is int else None class _Lines: def __init__(self, root: Path) -> None: self.root = root self.cache: dict[str, list[str] | None] = {} def get(self, name: str) -> list[str] | None: if name not in self.cache: p = self.root / "kilde" / name self.cache[name] = p.read_text(encoding="utf-8").split("\n") if p.is_file() else None return self.cache[name] def _check_root(concepts: list[Document], root: Path) -> None: lines = _Lines(root) seen = 0 for d in concepts: for s in _entries(d).values(): n, quote, text = _line(s), s.get("x-quote"), lines.get(_file(s)) seen += 1 if n and isinstance(quote, str) and text and n <= len(text) and quote in text[n - 1]: return if seen: raise RunError(f"no quote stands at its locator under {root}; is it the folder that contains kilde/?") def _block_of(blocks: list[dict[str, Any]], file: str, line: int) -> dict[str, Any] | None: return next((b for b in blocks if b["file"] == file and b["start"] <= line <= b["end"]), None) def coverage(bundle: Path, root: Path, blocks_doc: dict[str, Any], built: list[str], pack_size: int | None = None) -> Coverage: concepts = load_bundle(bundle, root).concepts _check_root(concepts, root) blocks = [b for b in blocks_doc["blocks"] if b["file"] in built] hit: set[tuple[str, int]] = set() for d in concepts: for s in cited(d): n = _line(s) b = _block_of(blocks, _file(s), n) if n else None if b is not None: hit.add((b["file"], b["start"])) result = Coverage(len(hit), len(blocks)) if pack_size: names = [str(x) for x in blocks_doc["order"] if x in built] for pos in range(1, pack_size + 1): files = {n for i, n in enumerate(names) if i % pack_size == pos - 1} if files: mine = [b for b in blocks if b["file"] in files] result.per_position[pos] = (sum((b["file"], b["start"]) in hit for b in mine), len(mine), len(files)) return result def _jaccard(a: str, b: str) -> float: x, y = set(a.split()), set(b.split()) return len(x & y) / len(x | y) if x | y else 0.0 def duplicates(concepts: list[Document]) -> list[tuple[str, str, str]]: info = [] for d in concepts: fm = d.frontmatter or {} aliases = fm.get("x-aliases") info.append((d.rel, norm(str(fm.get("title", ""))), {norm(str(a)) for a in aliases} if isinstance(aliases, list) else set())) pairs: list[tuple[str, str, str]] = [] for i, (a, na, aa) in enumerate(info): for b, nb, ab in info[i + 1:]: if na and na == nb: pairs.append((a, b, "norm")) elif na in ab or nb in aa: pairs.append((a, b, "alias")) elif _jaccard(na, nb) >= MERGE_JACCARD: pairs.append((a, b, "jaccard")) return pairs def _kind(d: Document) -> str: fm = d.frontmatter or {} if fm.get("x-merge-candidate"): return "merge-candidate" return {"Document map": "map", "Register entry": "register"}.get(str(fm.get("type", "")), "concept") def _precision(s: dict[str, Any], blocks: list[dict[str, Any]]) -> str: span = s.get("x-span") n = _line(s) or 0 a, z = (span[0], span[1]) if isinstance(span, list) and len(span) == 2 else (n, n) if a == z: return "line" inside = [b for b in blocks if b["file"] == _file(s) and b["end"] >= a and b["start"] <= z] if len(inside) == 1 and inside[0]["start"] <= a and z <= inside[0]["end"]: return "block" if inside and len({b["section"] for b in inside}) == 1: return "section" return "document" def structure(bundle: Path, root: Path, blocks_doc: dict[str, Any], order: list[str], skeleton: dict[str, Any] | None = None) -> dict[str, Any]: concepts = load_bundle(bundle, root).concepts _check_root(concepts, root) lines = _Lines(root) blocks = list(blocks_doc["blocks"]) rank = {n: i for i, n in enumerate(order)} per_source: Counter[str] = Counter() incoming: Counter[str] = Counter() rels = {d.rel for d in concepts} for d in concepts: for _line_no, target in body_links(d): other = resolve(d.rel, target) if is_bundle_link(target) else None if other in rels and other != d.rel: incoming[str(other)] += 1 quotes: Counter[str] = Counter() precision: Counter[str] = Counter() docs_per_page: list[int] = [] test2 = test4 = 0 for d in concepts: cs = cited(d) files = [_file(s) for s in cs if _file(s) in rank] if files: per_source[min(files, key=lambda f: rank[f])] += 1 docs_per_page.append(len(set(_file(s) for s in cs))) title = str((d.frontmatter or {}).get("title", "")) test2 += not is_meta(title) test4 += incoming[d.rel] >= 2 or len(set(files)) >= 2 for s in cs: if is_version_heading(s): continue quotes["entries"] += 1 quotes["code"] += s.get("x-quote-by") == "code" text, n = lines.get(_file(s)), _line(s) quotes["structural"] += bool(text and n and n <= len(text) and structural(text[n - 1])) precision[_precision(s, blocks)] += 1 rel_map = related(concepts) kinds = {d.rel: _kind(d) for d in concepts} islands: dict[str, Any] = {"total": sum(not v for v in rel_map.values())} for k in sorted(set(kinds.values())): islands[k] = sum(not rel_map[r] for r, kk in kinds.items() if kk == k) pages = {d.rel for d in concepts} unused = None if skeleton is not None: unused = sum(e.get("kind") == "skeleton" and f"{e['id']}.md" not in pages for e in skeleton["entries"]) tests13 = None if skeleton is not None: new_ids = {e["id"] for e in skeleton["entries"] if e.get("kind") == "new" and e.get("tests")} tests13 = sum(f"{i}.md" in pages for i in new_ids) v = versions(bundle, root) dups = duplicates(concepts) return { "schema": STRUCTURE_SCHEMA, "pages": len(concepts), "page_kinds": dict(Counter(kinds.values())), "concepts_per_source": {n: per_source[n] for n in order if per_source[n]}, "skeleton_unused": unused, "duplicates": {"pairs": len(dups), "by_rule": dict(Counter(r for _, _, r in dups)), "list": [list(p) for p in dups]}, "merge_candidates": sum(k == "merge-candidate" for k in kinds.values()), "islands": islands, "documents_per_page": {"mean": round(sum(docs_per_page) / len(docs_per_page), 2) if docs_per_page else 0, "max": max(docs_per_page, default=0)}, "version_numbers_in_text": {"flagged": len(v.flagged), "claims": v.total}, "concept_tests": {"test2": test2, "test4": test4, "tests1and3_self_declared": tests13}, "quotes": {"entries": quotes["entries"], "structural": quotes["structural"], "code": quotes["code"]}, "precision": {k: precision[k] for k in ("line", "block", "section", "document")}, } def _uncited_cause(sentence: str, block_text: str) -> str: s = sentence.strip() if FENCE.match(block_text): # the auditor's sentence text masks code spans, so a fence is read on the block return "fence line" if s.startswith("|"): return "table row" if LINK.search(block_text) and not LINK_ONLY.match(mask_code(block_text)): return "link lead-in" return "prose" def audit_record(bundle: Path, root: Path, variant: str, out: Path) -> dict[str, Any]: record_all: dict[str, Any] = (json.loads(out.read_text(encoding="utf-8")) if out.is_file() else {"schema": RESULT_SCHEMA, "variants": {}}) if variant in record_all["variants"]: raise RunError(f"{out} already records variant {variant}") b = load_bundle(bundle, root) found = audit(bundle, root) uncited: Counter[str] = Counter() for d in b.concepts: for block in prose_blocks(d): if LINK_ONLY.match(mask_code(block.text)): continue for s in sentences(block.text): if not FOOTNOTE_REF.search(s): uncited[_uncited_cause(s, block.text)] += 1 yaml_form = 0 for d in b.concepts: for entry in _entries(d).values(): q = entry.get("x-quote") yaml_form += q is not None and not isinstance(q, str) yaml_form += sum(f.check == "timestamp-form" for f in found) listed = claims_mod.extract(bundle, root) ok, total = claims_mod.criterion_1(listed)["total"] rec = {"exit": 1 if found else 0, "findings": dict(sorted(Counter(f.check for f in found).items())), "uncited": dict(sorted(uncited.items())), "yaml_form": yaml_form, "criterion_1": [ok, total]} record_all["variants"][variant] = rec out.parent.mkdir(parents=True, exist_ok=True) out.write_text(json.dumps(record_all, ensure_ascii=False, indent=1) + "\n", encoding="utf-8") return rec def _write_new(path: Path, data: Any) -> None: if path.exists(): raise RunError(f"{path} exists; refusing to overwrite") path.parent.mkdir(parents=True, exist_ok=True) path.write_text(json.dumps(data, ensure_ascii=False, indent=1) + "\n", encoding="utf-8") def build_parser() -> argparse.ArgumentParser: p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) sub = p.add_subparsers(dest="command", required=True) a = sub.add_parser("audit") a.add_argument("--bundle", required=True) a.add_argument("--root", required=True) a.add_argument("--variant", required=True) a.add_argument("--out", required=True) c = sub.add_parser("coverage") s = sub.add_parser("structure") for x in (c, s): x.add_argument("--bundle", required=True) x.add_argument("--root", required=True) x.add_argument("--blocks", required=True) x.add_argument("--out", required=True) c.add_argument("--built", type=int, required=True) c.add_argument("--pack-size", type=int) s.add_argument("--skeleton") return p def main(argv: list[str] | None = None) -> int: args = build_parser().parse_args(argv) try: bundle, root = Path(args.bundle), Path(args.root) if not bundle.is_dir(): raise RunError(f"the bundle {bundle} is not a directory") if args.command == "audit": rec = audit_record(bundle, root, args.variant, Path(args.out)) print(f"{args.variant}: audit exit {rec['exit']}, criterion 1 {rec['criterion_1'][0]} of " f"{rec['criterion_1'][1]}, findings {rec['findings']}") return 0 blocks_doc = json.loads(Path(args.blocks).read_text(encoding="utf-8")) order = [str(x) for x in blocks_doc["order"]] if args.command == "coverage": cov = coverage(bundle, root, blocks_doc, order[:args.built], args.pack_size) _write_new(Path(args.out), {"schema": "okf-wiki/p1-coverage/1", "covered": cov.covered, "total": cov.total, "built": args.built, "per_position": {str(k): list(v) for k, v in cov.per_position.items()}}) print(f"coverage: {cov.covered} of {cov.total} blocks") return 0 skeleton = json.loads(Path(args.skeleton).read_text(encoding="utf-8")) if args.skeleton else None data = structure(bundle, root, blocks_doc, order, skeleton) _write_new(Path(args.out), data) print(f"structure: {data['pages']} pages, {data['duplicates']['pairs']} duplicate pairs, " f"{data['islands']['total']} islands") return 0 except (RunError, OSError, json.JSONDecodeError, KeyError) as exc: print(f"measure cannot run: {exc}", file=sys.stderr) return 2 if __name__ == "__main__": sys.exit(main())