"""Repair gate (order 34, contracts K18 and K19): the judge of the repair command, run before the command exists. No model call. The gate is fail-closed: a command that is missing, crashes, times out, refuses with another code or changes more or less than the case says, fails the case, and the gate is GREEN only when every case passes. The gate builds its own collection from the synthetic fixtures in gates/repair/ (a K1 source root about beekeeping, three docs pages and one release note, and the compiler's fragments for them) with the frozen tools: kblocks order and blocks, kpack skeleton, pack and accept, kassemble build --at a fixed time, klink accept and kassemble build --links. No generated page is in git. The base must come out clean (no auditor finding, criterion 1 N of N) or the gate cannot run. Every case then runs in a fresh copy of that base: 1. the command's steps run with the placeholders filled in; each step's exit code and refusal codes are checked; 2. a case that writes is built with kassemble build --links --replace at another fixed time, and measured by the gate's own code, apart from the command's report: exactly the concept pages the case names have other bytes (index.md and log.md left out); the claims (the frozen claims extract) that are gone or new, by id and excerpt_sha256, are exactly the ids the case derives from its hand-written sentences, so every other claim keeps its id and excerpt_sha256; the auditor's findings are exactly the case's (check and file), and criterion 1 is N of N; the links file holds exactly the base records without the case's removed records and with its added ones, each placed at its anchor; 3. the command's diff runs on the old and the new collection, and its ids.json and sider.json must equal what the gate measured (a case without a repairs log, the link commands, leaves this out); 4. a refused step changes no file in the work folder, the links file, the collection or the output folder (a tree hash before and after); task and linkset change none of the first three, and their output is checked as the case says. The denominator comes from the sources: the pages, sentences, links and findings each case expects are written by hand in gates/repair.manifest.json as page names and sentence text before and after; the gate turns a sentence into its claim id with the frozen klink.key_text, and a fragment into its footnote with the frozen kassemble.label. Before anything runs, every file under gates/repair/ is checked against the manifest's sha256: a changed, missing or extra file stops the gate (exit 2). Usage: python3 gates/repair_gate.py --manifest gates/repair.manifest.json [--timeout 120] -- ... The command is the program the case's arguments are appended to, e.g. `python3 -m okfwiki.krepair`; it runs from the repository root. Placeholders in a case's arguments: {work} {wiki} {wiki_old} {links} {sources} {blocks} {claims} {pairs} {out} {diff}, and the step's rendered inputs {answers} {list} {agent}. Exit: 0 GREEN, 1 RED, 2 the gate could not run. """ from __future__ import annotations import argparse import hashlib import io import json import shutil import subprocess import sys import tempfile from collections import Counter from contextlib import redirect_stderr, redirect_stdout from dataclasses import dataclass, field from pathlib import Path from typing import Any, Callable ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT)) from okfwiki import audit, claims, kassemble, kblocks, klink, kpack # noqa: E402 from okfwiki.claims import ClaimList # noqa: E402 SCHEMA = "okf-wiki/repair-gate/1" ANSWERS_SCHEMA = "okf-wiki/repair-answers/1" LIST_SCHEMA = "okf-wiki/repair-list/1" PAGES_SCHEMA = "okf-wiki/repair-pages/1" CASES_DIR = "repair" # beside the manifest RESERVED = ("index.md", "log.md") EMPTY_SHA = hashlib.sha256(b"").hexdigest() STALE_SHA = hashlib.sha256(b"stale").hexdigest() class GateError(Exception): """The gate cannot produce a measurement (bad manifest, drifted fixtures, a base that does not build clean).""" # --- files ----------------------------------------------------------------------------------------------------- def sha256_file(path: Path) -> str: return hashlib.sha256(path.read_bytes()).hexdigest() def hash_tree(root: Path) -> dict[str, str]: """sha256 of every regular file under root (or of root itself when it is a file), keyed by relative POSIX path.""" if root.is_file(): return {".": sha256_file(root)} out: dict[str, str] = {} if not root.exists(): return out for p in sorted(root.rglob("*")): if p.is_file() and "__pycache__" not in p.parts: out[p.relative_to(root).as_posix()] = sha256_file(p) return out def load_manifest(path: Path) -> dict[str, Any]: try: data = json.loads(path.read_text(encoding="utf-8")) except (OSError, json.JSONDecodeError) as exc: raise GateError(f"cannot read manifest {path}: {exc}") from exc if not isinstance(data, dict) or data.get("schema") != SCHEMA: raise GateError(f"manifest schema must be {SCHEMA}") for key in ("files", "base", "cases", "build_at", "refusal_codes"): if key not in data: raise GateError(f"manifest has no {key}") names = [c.get("name") for c in data["cases"]] if not names or len(names) != len(set(names)) or not all(isinstance(n, str) and n for n in names): raise GateError("case names are missing or not unique") codes = set(data["refusal_codes"]) for case in data["cases"]: steps = case.get("steps") if not isinstance(steps, list) or not steps: raise GateError(f"case {case['name']} has no steps") for step in steps: if step.get("exit") not in (0, 1) or not isinstance(step.get("argv"), list): raise GateError(f"case {case['name']}: a step needs argv and exit 0 or 1") unknown = set(step.get("refused", [])) - codes if unknown: raise GateError(f"case {case['name']}: refusal codes not in the manifest: {sorted(unknown)}") if (step["exit"] == 1) != bool(step.get("refused")): raise GateError(f"case {case['name']}: a step exits 1 exactly when it names refusal codes") return data def verify_fixtures(manifest: dict[str, Any], cases_root: Path) -> None: expected: dict[str, str] = manifest["files"] actual = hash_tree(cases_root) problems = [f"missing file {rel}" for rel in sorted(set(expected) - set(actual))] problems += [f"unexpected file {rel}" for rel in sorted(set(actual) - set(expected))] problems += [f"sha256 mismatch in {rel}" for rel in sorted(set(actual) & set(expected)) if actual[rel] != expected[rel]] if problems: raise GateError("fixtures do not match the manifest:\n " + "\n ".join(problems)) # --- the frozen tools, in process ------------------------------------------------------------------------------ def quiet(fn: Callable[[list[str]], int], argv: list[str]) -> tuple[int, str]: out = io.StringIO() with redirect_stdout(out), redirect_stderr(out): code = fn(argv) return code, out.getvalue() def must(fn: Callable[[list[str]], int], argv: list[str], what: str, ok: tuple[int, ...] = (0,)) -> str: code, out = quiet(fn, argv) if code not in ok: raise GateError(f"{what} exited {code}:\n{out}") return out def sentence_key(text: str, n: int = 0) -> str: """The sentence key klink gives the n-th sentence (from 0) with this text on a page.""" return hashlib.sha256(klink.key_text(text).encode("utf-8")).hexdigest()[:16] + f"-{n}" def claim_id(page: str, text: str, n: int = 0) -> str: return f"{page}#{sentence_key(text, n)}" def fragment_label(sources: Path, key: dict[str, Any]) -> str: """The footnote of a fragment by the frozen kassemble.label, its quote line found in its anchor span; or the literal footnote a case gives.""" if "footnote" in key: return str(key["footnote"]) lines = (sources / "kilde" / key["file"]).read_text(encoding="utf-8").split("\n") line = next((n for n in range(key["start"], key["end"] + 1) if key["quote"] in lines[n - 1]), None) if line is None: raise GateError(f"the quote of key {key} is not in its span") return kassemble.label({"file": key["file"], "start": key["start"], "end": key["end"], "quote_line": line, "quote": key["quote"], "provenance": key["provenance"]}) @dataclass class Base: root: Path sources: Path work: Path wiki: Path links: Path blocks: Path claims: Path pairs: Path found: ClaimList = field(default_factory=ClaimList) def build_base(manifest: dict[str, Any], cases_root: Path, tmp: Path) -> Base: spec = manifest["base"] root = tmp / "base" root.mkdir() b = Base(root, root / "sources", root / "work", root / "wiki", root / "links.json", root / "work" / "blocks.json", root / "claims.json", root / "pairs.json") shutil.copytree(cases_root / spec["sources"], b.sources) b.work.mkdir() order = b.work / "order.txt" must(kblocks.main, ["order", str(b.sources), "--out", str(order)], "kblocks order") must(kblocks.main, ["blocks", str(b.sources), "--order", str(order), "--out", str(b.blocks)], "kblocks blocks") must(kpack.main, ["skeleton", "--sources", str(b.sources), "--work", str(b.work)], "kpack skeleton") for n, rel in enumerate(spec["fragments"], 1): must(kpack.main, ["pack", str(n), "--blocks", str(b.blocks), "--sources", str(b.sources), "--work", str(b.work), "--size", str(spec["size"])], f"kpack pack {n}") dest = b.work / "K-fragments" / f"pack-{n}.json" dest.parent.mkdir(parents=True, exist_ok=True) shutil.copyfile(cases_root / rel, dest) out = must(kpack.main, ["accept", str(n), "--blocks", str(b.blocks), "--sources", str(b.sources), "--work", str(b.work)], f"kpack accept {n}") given = len(json.loads((cases_root / rel).read_text(encoding="utf-8"))["fragments"]) if f"accepted {given} of {given}" not in out: raise GateError(f"pack {n}: not every fixture fragment was accepted:\n{out}") build = ["build", "--work", str(b.work), "--sources", str(b.sources), "--out", str(b.wiki), "--at", spec["at"]] must(kassemble.main, build, "kassemble build") links_spec = json.loads((cases_root / spec["links"]).read_text(encoding="utf-8"))["links"] proposals = {"links": [{"page": x["page"], "sentence": sentence_key(x["sentence"], x.get("n", 0)), "anchor": x["anchor"], "target": x["target"]} for x in links_spec], "none": []} prop_path = root / "proposals.json" prop_path.write_text(json.dumps(proposals), encoding="utf-8") must(klink.main, ["accept", str(prop_path), "--bundle", str(b.wiki), "--links", str(b.links)], "klink accept") placed = json.loads(b.links.read_text(encoding="utf-8")) if placed["accepted"] != len(links_spec) or placed.get("rejected"): raise GateError(f"the base links were not all accepted: {placed['accepted']} of {len(links_spec)}, " f"rejected {placed.get('rejected')}") must(kassemble.main, build + ["--links", str(b.links), "--replace"], "kassemble build --links") prop_path.unlink() must(claims.main, ["list", str(b.wiki), str(b.sources), "--out", str(b.claims), "--pairs", str(b.pairs)], "claims list of the base") b.found = claims.extract(b.wiki, b.sources) findings = audit.audit(b.wiki, b.sources) if findings: raise GateError("the base has auditor findings: " + "; ".join(f"{f.check} {f.file}" for f in findings)) return b # --- rendering a case's inputs --------------------------------------------------------------------------------- def render_answers(answers: list[dict[str, Any]], sources: Path) -> dict[str, Any]: out = [] for a in answers: item: dict[str, Any] = {"concept_id": a["key"]["concept_id"], "footnote": fragment_label(sources, a["key"]), "action": a["action"]} if a["action"] == "replace": item["fragments"] = a["fragments"] out.append(item) return {"schema": ANSWERS_SCHEMA, "answers": out} def render_list(spec: dict[str, Any], b: Base) -> dict[str, Any]: """A repair list in K17's schema from a case's sentences and links, its ids, hashes and lines from the base.""" by_id = {c.id: c for c in b.found.claims} k1 = {str(e["path"]): str(e["kind"]) for e in json.loads((b.sources / "source-manifest.json").read_text(encoding="utf-8"))["sources"]} rows = [] for c in spec.get("claims", []): cid = claim_id(c["page"], c["text"], c.get("n", 0)) if cid not in by_id: raise GateError(f"list claim {cid} ({c['text']!r}) is not a claim of the base") found = by_id[cid] kinds = [k1.get(r.lstrip("/"), "docs") for r in found.resources] rows.append({"id": cid, "excerpt_sha256": found.excerpt_sha256, "page": found.page, "line": found.line, "provenance": found.provenance, "kind": "notes" if kinds and kinds[0] == "release-notes" else "docs", "sources": [{"file": f"kilde/{e.file}", "line": e.line} for e in found.excerpts], "origins": ["o33"], "reasons": c.get("reasons", ["not-supported"])}) records = json.loads(b.links.read_text(encoding="utf-8"))["links"] links = [] for x in spec.get("links", []): key = sentence_key(x["sentence"], x.get("n", 0)) rec = next((r for r in records if r["page"] == x["page"] and r["sentence"] == key and r["anchor"] == x["anchor"] and r["target"] == x["target"]), None) if rec is None: raise GateError(f"list link {x} is not a record of the base") links.append({**rec, "origin": "o33-kandidat", "known_positive": bool(x.get("known_positive", False))}) return {"schema": LIST_SCHEMA, "claims_sha256": STALE_SHA if spec.get("stale") else sha256_file(b.claims), "verdicts_sha256": EMPTY_SHA, "claims": rows, "pages": [], "pairs": [], "links": links} # --- running the command --------------------------------------------------------------------------------------- @dataclass class Ran: code: int | None codes: list[str] output: str error: str = "" bare: int = 0 # refusal lines that name no claim or record def run_command(template: list[str], argv: list[str], timeout: float) -> Ran: full = template + argv try: proc = subprocess.run(full, capture_output=True, text=True, timeout=timeout, check=False, cwd=ROOT) except FileNotFoundError: return Ran(None, [], "", f"command not found: {full[0]}") except PermissionError: return Ran(None, [], "", f"command not executable: {full[0]}") except subprocess.TimeoutExpired: return Ran(None, [], "", f"command timed out after {timeout:g}s") output = proc.stdout + proc.stderr codes, bare = [], 0 for line in output.splitlines(): if line.startswith("refused: "): parts = line[len("refused: "):].split(" ", 1) codes.append(parts[0]) bare += not (len(parts) == 2 and parts[1].strip()) return Ran(proc.returncode, codes, output, bare=bare) def fill(argv: list[Any], places: dict[str, str]) -> list[str]: """The step's arguments with the placeholders filled in; an argument {"sentence": text, "n": n} becomes that sentence's key.""" out = [] for a in argv: if isinstance(a, dict): out.append(sentence_key(str(a["sentence"]), int(a.get("n", 0)))) continue for k, v in places.items(): a = a.replace("{" + k + "}", v) out.append(str(a)) return out # --- measuring ------------------------------------------------------------------------------------------------- @dataclass class Measured: changed: list[str] gone: list[str] claims_gone: Counter[str] claims_came: Counter[str] findings: Counter[tuple[str, str]] criterion_1: tuple[int, int] unkeyed: int def concept_pages(wiki: Path) -> dict[str, bytes]: return {p.relative_to(wiki).as_posix(): p.read_bytes() for p in sorted(wiki.rglob("*.md")) if p.name not in RESERVED} def measure(old: Path, new: Path, sources: Path) -> Measured: a, z = concept_pages(old), concept_pages(new) changed = sorted(n for n in set(a) | set(z) if a.get(n) != z.get(n)) before = Counter((c.id, c.excerpt_sha256) for c in claims.extract(old, sources).claims) after_list = claims.extract(new, sources) after = Counter((c.id, c.excerpt_sha256) for c in after_list.claims) gone = Counter(i for i, _ in (before - after).elements()) came = Counter(i for i, _ in (after - before).elements()) findings = Counter((f.check, f.file) for f in audit.audit(new, sources)) total = claims.criterion_1(after_list)["total"] return Measured(changed, sorted(set(a) - set(z)), gone, came, findings, (total[0], total[1]), len(after_list.unkeyed)) def link_set(records: list[dict[str, Any]]) -> Counter[tuple[str, str, str, str]]: return Counter((str(r["page"]), str(r["sentence"]), str(r["anchor"]), str(r["target"])) for r in records) def link_tuple(x: dict[str, Any]) -> tuple[str, str, str, str]: return x["page"], sentence_key(x["sentence"], x.get("n", 0)), x["anchor"], x["target"] def misplaced_links(records: list[dict[str, Any]], wiki: Path) -> list[str]: """Records whose `at` does not hold their anchor in their sentence on the built page (link-reduced, as klink).""" pages = klink.read_pages(wiki) bad = [] for r in records: page = pages.get(str(r["page"])) s = next((s for s in klink.body_sentences(page.body) if s.key == r["sentence"]), None) if page else None at = int(r["at"]) if s is None or s.text[at:at + len(str(r["anchor"]))] != r["anchor"]: bad.append(f"{r['page']} {r['sentence']} {r['anchor']!r} at {r['at']}") return bad def excerpt_lines(wiki: Path, sources: Path, cid: str) -> list[list[Any]] | None: c = next((c for c in claims.extract(wiki, sources).claims if c.id == cid), None) return None if c is None else [[e.file, e.line] for e in c.excerpts] # --- one case -------------------------------------------------------------------------------------------------- @dataclass class Copy: root: Path places: dict[str, str] def path(self, name: str) -> Path: return Path(self.places[name]) def trees(self, *names: str) -> dict[str, dict[str, str]]: return {n: hash_tree(self.path(n)) for n in names} def fresh_copy(b: Base, tmp: Path, name: str) -> Copy: root = tmp / name root.mkdir() for d in ("sources", "work", "wiki"): shutil.copytree(b.root / d, root / d) shutil.copytree(b.wiki, root / "wiki_old") for f in ("links.json", "claims.json", "pairs.json"): shutil.copyfile(b.root / f, root / f) for d in ("in", "out", "diff"): (root / d).mkdir() places = {"work": root / "work", "wiki": root / "wiki", "wiki_old": root / "wiki_old", "links": root / "links.json", "sources": root / "sources", "blocks": root / "work" / "blocks.json", "claims": root / "claims.json", "pairs": root / "pairs.json", "out": root / "out", "diff": root / "diff", "in": root / "in"} return Copy(root, {k: str(v) for k, v in places.items()}) def render_step(step: dict[str, Any], i: int, c: Copy, b: Base, cases_root: Path) -> dict[str, str]: places = dict(c.places) if "answers" in step: p = c.path("in") / f"answers-{i}.json" p.write_text(json.dumps(render_answers(step["answers"], b.sources), indent=1), encoding="utf-8") places["answers"] = str(p) if "list" in step: p = c.path("in") / f"list-{i}.json" p.write_text(json.dumps(render_list(step["list"], b), indent=1), encoding="utf-8") places["list"] = str(p) if "agent" in step: p = c.path("in") / f"agent-{i}.md" shutil.copyfile(cases_root / step["agent"], p) places["agent"] = str(p) return places def check_task(case: dict[str, Any], c: Copy, b: Base) -> list[str]: problems: list[str] = [] out = c.path("out") tasks = sorted(out.rglob("*.parts.json")) if not tasks: return ["task wrote no .parts.json"] texts: dict[str, str] = {} answers: list[str] = [] for t in tasks: try: doc = json.loads(t.read_text(encoding="utf-8")) parts = [Path(p) if Path(p).is_absolute() else out / p for p in doc["parts"]] answer = doc["answer"] if not parts or not isinstance(answer, str) or not answer: raise ValueError("no parts or no answer path") except (OSError, ValueError, KeyError, TypeError) as exc: problems.append(f"{t.name}: not a part list with parts and one answer path ({exc})") continue answers.append(answer) body = [] for p in parts: if not p.is_file(): problems.append(f"{t.name}: part {p} does not exist") continue text = p.read_text(encoding="utf-8") if len(text) > kpack.PART_CHARS: problems.append(f"{t.name}: part {p.name} has {len(text)} characters, over {kpack.PART_CHARS}") body.append(text) texts[t.name] = "\n".join(body) if len(set(answers)) != len(answers): problems.append("two tasks share an answer path") for want in case["task"]: label = fragment_label(b.sources, want["key"]) holders = [n for n, text in texts.items() if label in text] if len(holders) != 1: problems.append(f"fragment {want['key']['concept_id']} {label} stands in {len(holders)} tasks, not 1") continue for s in want.get("must_contain", []): if s not in texts[holders[0]]: problems.append(f"the task with {label} lacks {s!r}") return problems def check_linkset(case: dict[str, Any], c: Copy) -> list[str]: problems: list[str] = [] out = c.path("out") groups = sorted(out.glob("group-*.sentences.json")) if not groups: return ["linkset wrote no group-.sentences.json"] found: dict[tuple[str, str], list[str]] = {} for g in groups: stem = g.name[: -len(".sentences.json")] cat = out / f"{stem}.catalogue" try: doc = json.loads(g.read_text(encoding="utf-8")) pages = doc["pages"] if doc.get("schema") == klink.SENTENCES_SCHEMA else None if not isinstance(pages, dict): raise ValueError(f"not {klink.SENTENCES_SCHEMA} with pages") ids = klink.read_ids(cat) except (OSError, ValueError, KeyError, TypeError) as exc: problems.append(f"{stem}: unreadable sentence file or catalogue ({exc})") continue for page, keys in pages.items(): for k in keys: found[(str(page), str(k))] = ids target = c.root / "klink-task" / stem shutil.rmtree(target, ignore_errors=True) code, said = quiet(klink.main, ["task", c.places["wiki"], "--sentences", str(g), "--catalogue", str(cat), "--out", str(target)]) if code != 0: problems.append(f"{stem}: klink task cannot run on it (exit {code}): {said.strip()[-200:]}") wanted = {(x["page"], sentence_key(x["sentence"], x.get("n", 0))): x["target"] for x in case["linkset"]} if set(found) != set(wanted): problems.append(f"sentences: {sorted(found)} written, {sorted(wanted)} wanted") for k, target in wanted.items(): if k in found and target.removesuffix(".md") not in found[k]: problems.append(f"the catalogue for {k} lacks its current target {target}") return problems def check_written(case: dict[str, Any], c: Copy, b: Base, template: list[str], at: str, timeout: float, repairs_log: bool) -> list[str]: problems: list[str] = [] code, said = quiet(kassemble.main, ["build", "--work", c.places["work"], "--sources", c.places["sources"], "--out", c.places["wiki"], "--links", c.places["links"], "--replace", "--at", at]) if code != 0: return [f"kassemble build after the command exited {code}: {said.strip()[-300:]}"] m = measure(c.path("wiki_old"), c.path("wiki"), c.path("sources")) if m.changed != sorted(case.get("pages_changed", [])): problems.append(f"pages changed: {m.changed}, the case says {sorted(case.get('pages_changed', []))}") if m.gone != sorted(case.get("pages_gone", [])): problems.append(f"pages gone: {m.gone}, the case says {sorted(case.get('pages_gone', []))}") want_gone = Counter(claim_id(x["page"], x["text"], x.get("n", 0)) for x in case.get("claims_before", [])) want_came = Counter(claim_id(x["page"], x["text"], x.get("n", 0)) for x in case.get("claims_after", [])) if m.claims_gone != want_gone: problems.append(f"claims gone or changed: {sorted(m.claims_gone.elements())}, the case says " f"{sorted(want_gone.elements())}") if m.claims_came != want_came: problems.append(f"claims new or changed: {sorted(m.claims_came.elements())}, the case says " f"{sorted(want_came.elements())}") for x in case.get("claims_after", []): if "excerpts" in x: got = excerpt_lines(c.path("wiki"), c.path("sources"), claim_id(x["page"], x["text"], x.get("n", 0))) if got != x["excerpts"]: problems.append(f"excerpts of {x['text']!r}: {got}, the case says {x['excerpts']}") want_findings = Counter((f["check"], f["file"]) for f in case.get("findings", [])) if m.findings != want_findings: problems.append(f"auditor findings: {sorted(m.findings.elements())}, the case says " f"{sorted(want_findings.elements())}") if not (m.criterion_1[1] and m.criterion_1[0] == m.criterion_1[1] and m.unkeyed == 0): problems.append(f"criterion 1: {m.criterion_1[0]} of {m.criterion_1[1]}, unkeyed {m.unkeyed}") records = json.loads(c.path("links").read_text(encoding="utf-8"))["links"] base = link_set(json.loads(b.links.read_text(encoding="utf-8"))["links"]) want = base - Counter(link_tuple(x) for x in case.get("links_removed", [])) want.update(Counter(link_tuple(x) for x in case.get("links_added", []))) if link_set(records) != want: problems.append(f"links file: {sorted(link_set(records).elements())}, the case says {sorted(want.elements())}") problems += [f"link record not at its anchor: {x}" for x in misplaced_links(records, c.path("wiki"))] if repairs_log: problems += check_diff(c, template, m, timeout) return problems def check_diff(c: Copy, template: list[str], m: Measured, timeout: float) -> list[str]: out = c.path("diff") argv = ["diff", "--old", c.places["wiki_old"], "--new", c.places["wiki"], "--sources", c.places["sources"], "--repairs", str(c.path("work") / "K-work" / "repairs.json"), "--out", str(out)] ran = run_command(template, argv, timeout) if ran.code != 0: return [f"diff exited {ran.code} {ran.error}: {ran.output.strip()[-300:]}"] problems = [] try: ids = json.loads((out / "ids.json").read_text(encoding="utf-8"))["ids"] pages = json.loads((out / "sider.json").read_text(encoding="utf-8"))["pages"] except (OSError, ValueError, KeyError, TypeError) as exc: return [f"diff output unreadable: {exc}"] if sorted(ids) != sorted(m.claims_came.elements()): problems.append(f"diff ids.json: {sorted(ids)}, the gate measured {sorted(m.claims_came.elements())}") if sorted(pages) != m.changed: problems.append(f"diff sider.json: {sorted(pages)}, the gate measured {m.changed}") return problems def run_case(case: dict[str, Any], b: Base, template: list[str], cases_root: Path, tmp: Path, at: str, timeout: float) -> list[str]: c = fresh_copy(b, tmp, case["name"]) fixed = c.trees("sources", "wiki_old") problems: list[str] = [] last: dict[str, Any] = {} for i, step in enumerate(case["steps"], 1): places = render_step(step, i, c, b, cases_root) before = c.trees("work", "wiki", "links", "out") ran = run_command(template, fill(step["argv"], places), timeout) if ran.code != step["exit"]: tail = ran.error or ran.output.strip()[-300:] problems.append(f"step {i} ({step['argv'][0]}): exit {ran.code}, the case says {step['exit']}: {tail}") return problems if sorted(set(ran.codes)) != sorted(set(step.get("refused", []))): problems.append(f"step {i}: refusal codes {sorted(set(ran.codes))}, the case says " f"{sorted(set(step.get('refused', [])))}") if ran.bare: problems.append(f"step {i}: {ran.bare} refusal line(s) name no claim or record") after = c.trees("work", "wiki", "links", "out") if step["exit"] == 1: moved = [n for n in before if before[n] != after[n]] if moved: problems.append(f"step {i} refused but changed {', '.join(moved)}") elif step["argv"][0] in ("task", "linkset"): moved = [n for n in ("work", "wiki", "links") if before[n] != after[n]] if moved: problems.append(f"step {i} ({step['argv'][0]}) changed {', '.join(moved)}") last = step if last["exit"] == 0: if last["argv"][0] == "task": problems += check_task(case, c, b) elif last["argv"][0] == "linkset": problems += check_linkset(case, c) else: problems += check_written(case, c, b, template, at, timeout, last["argv"][0] == "fix") if c.trees("sources", "wiki_old") != fixed: problems.append("the sources or the old collection changed") return problems # --- the gate -------------------------------------------------------------------------------------------------- def report(cases: list[dict[str, Any]], results: dict[str, list[str]], command: list[str]) -> bool: print(f"repair gate: {len(cases)} cases; command: {' '.join(command)}") for case in cases: problems = results[case["name"]] print(f" {case['name']:<44} {'PASS' if not problems else 'FAIL'}") for p in problems: print(f" {p}") passed = sum(not results[c["name"]] for c in cases) print(f"passed {passed} of {len(cases)}") green = passed == len(cases) print(f"verdict: {'GREEN' if green else 'RED'}") return green def run(manifest_path: Path, command: list[str], timeout: float) -> int: try: manifest = load_manifest(manifest_path) cases_root = manifest_path.parent / CASES_DIR verify_fixtures(manifest, cases_root) with tempfile.TemporaryDirectory(prefix="repair-gate-") as tmp_name: tmp = Path(tmp_name) base = build_base(manifest, cases_root, tmp) results = {case["name"]: run_case(case, base, command, cases_root, tmp, manifest["build_at"], timeout) for case in manifest["cases"]} except GateError as exc: print(f"GATE CANNOT RUN: {exc}") return 2 return 0 if report(manifest["cases"], results, command) else 1 def main(argv: list[str] | None = None) -> int: args_in = sys.argv[1:] if argv is None else argv if "--" in args_in: split = args_in.index("--") own, command = args_in[:split], args_in[split + 1:] else: own, command = args_in, [] parser = argparse.ArgumentParser(description="Run the repair gate (fail-closed, no model calls).") parser.add_argument("--manifest", type=Path, required=True) parser.add_argument("--timeout", type=float, default=120.0, help="seconds per command run") args = parser.parse_args(own) if not command: print("usage error: give the command after --, e.g. -- python3 -m okfwiki.krepair") return 2 return run(args.manifest, command, args.timeout) if __name__ == "__main__": sys.exit(main())