okf-wiki/gates/repair_gate.py
Kjell Tore Guttormsen 0fd86a23a3
okf-wiki 1.1.1
First public release, published as one snapshot. What changed between
versions is in CHANGELOG.md.

Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
2026-11-14 21:03:20 +01:00

665 lines
32 KiB
Python

"""Repair gate (order 34, contracts K18 and K19): the judge of the repair command, run before the command exists.
No model call. The gate is fail-closed: a command that is missing, crashes, times out, refuses with another code or
changes more or less than the case says, fails the case, and the gate is GREEN only when every case passes.
The gate builds its own collection from the synthetic fixtures in gates/repair/ (a K1 source root about beekeeping,
three docs pages and one release note, and the compiler's fragments for them) with the frozen tools: kblocks order and
blocks, kpack skeleton, pack and accept, kassemble build --at a fixed time, klink accept and kassemble build --links.
No generated page is in git. The base must come out clean (no auditor finding, criterion 1 N of N) or the gate cannot
run. Every case then runs in a fresh copy of that base:
1. the command's steps run with the placeholders filled in; each step's exit code and refusal codes are checked;
2. a case that writes is built with kassemble build --links --replace at another fixed time, and measured by the gate's
own code, apart from the command's report: exactly the concept pages the case names have other bytes (index.md and
log.md left out); the claims (the frozen claims extract) that are gone or new, by id and excerpt_sha256, are exactly
the ids the case derives from its hand-written sentences, so every other claim keeps its id and excerpt_sha256;
the auditor's findings are exactly the case's (check and file), and criterion 1 is N of N; the links file holds
exactly the base records without the case's removed records and with its added ones, each placed at its anchor;
3. the command's diff runs on the old and the new collection, and its ids.json and sider.json must equal what the gate
measured (a case without a repairs log, the link commands, leaves this out);
4. a refused step changes no file in the work folder, the links file, the collection or the output folder (a tree
hash before and after); task and linkset change none of the first three, and their output is checked as the case
says.
The denominator comes from the sources: the pages, sentences, links and findings each case expects are written by hand
in gates/repair.manifest.json as page names and sentence text before and after; the gate turns a sentence into its
claim id with the frozen klink.key_text, and a fragment into its footnote with the frozen kassemble.label. Before
anything runs, every file under gates/repair/ is checked against the manifest's sha256: a changed, missing or extra
file stops the gate (exit 2).
Usage:
python3 gates/repair_gate.py --manifest gates/repair.manifest.json [--timeout 120] -- <command> ...
The command is the program the case's arguments are appended to, e.g. `python3 -m okfwiki.krepair`; it runs from the
repository root. Placeholders in a case's arguments: {work} {wiki} {wiki_old} {links} {sources} {blocks} {claims}
{pairs} {out} {diff}, and the step's rendered inputs {answers} {list} {agent}.
Exit: 0 GREEN, 1 RED, 2 the gate could not run.
"""
from __future__ import annotations
import argparse
import hashlib
import io
import json
import shutil
import subprocess
import sys
import tempfile
from collections import Counter
from contextlib import redirect_stderr, redirect_stdout
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any, Callable
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT))
from okfwiki import audit, claims, kassemble, kblocks, klink, kpack # noqa: E402
from okfwiki.claims import ClaimList # noqa: E402
SCHEMA = "okf-wiki/repair-gate/1"
ANSWERS_SCHEMA = "okf-wiki/repair-answers/1"
LIST_SCHEMA = "okf-wiki/repair-list/1"
PAGES_SCHEMA = "okf-wiki/repair-pages/1"
CASES_DIR = "repair" # beside the manifest
RESERVED = ("index.md", "log.md")
EMPTY_SHA = hashlib.sha256(b"").hexdigest()
STALE_SHA = hashlib.sha256(b"stale").hexdigest()
class GateError(Exception):
"""The gate cannot produce a measurement (bad manifest, drifted fixtures, a base that does not build clean)."""
# --- files -----------------------------------------------------------------------------------------------------
def sha256_file(path: Path) -> str:
return hashlib.sha256(path.read_bytes()).hexdigest()
def hash_tree(root: Path) -> dict[str, str]:
"""sha256 of every regular file under root (or of root itself when it is a file), keyed by relative POSIX path."""
if root.is_file():
return {".": sha256_file(root)}
out: dict[str, str] = {}
if not root.exists():
return out
for p in sorted(root.rglob("*")):
if p.is_file() and "__pycache__" not in p.parts:
out[p.relative_to(root).as_posix()] = sha256_file(p)
return out
def load_manifest(path: Path) -> dict[str, Any]:
try:
data = json.loads(path.read_text(encoding="utf-8"))
except (OSError, json.JSONDecodeError) as exc:
raise GateError(f"cannot read manifest {path}: {exc}") from exc
if not isinstance(data, dict) or data.get("schema") != SCHEMA:
raise GateError(f"manifest schema must be {SCHEMA}")
for key in ("files", "base", "cases", "build_at", "refusal_codes"):
if key not in data:
raise GateError(f"manifest has no {key}")
names = [c.get("name") for c in data["cases"]]
if not names or len(names) != len(set(names)) or not all(isinstance(n, str) and n for n in names):
raise GateError("case names are missing or not unique")
codes = set(data["refusal_codes"])
for case in data["cases"]:
steps = case.get("steps")
if not isinstance(steps, list) or not steps:
raise GateError(f"case {case['name']} has no steps")
for step in steps:
if step.get("exit") not in (0, 1) or not isinstance(step.get("argv"), list):
raise GateError(f"case {case['name']}: a step needs argv and exit 0 or 1")
unknown = set(step.get("refused", [])) - codes
if unknown:
raise GateError(f"case {case['name']}: refusal codes not in the manifest: {sorted(unknown)}")
if (step["exit"] == 1) != bool(step.get("refused")):
raise GateError(f"case {case['name']}: a step exits 1 exactly when it names refusal codes")
return data
def verify_fixtures(manifest: dict[str, Any], cases_root: Path) -> None:
expected: dict[str, str] = manifest["files"]
actual = hash_tree(cases_root)
problems = [f"missing file {rel}" for rel in sorted(set(expected) - set(actual))]
problems += [f"unexpected file {rel}" for rel in sorted(set(actual) - set(expected))]
problems += [f"sha256 mismatch in {rel}" for rel in sorted(set(actual) & set(expected))
if actual[rel] != expected[rel]]
if problems:
raise GateError("fixtures do not match the manifest:\n " + "\n ".join(problems))
# --- the frozen tools, in process ------------------------------------------------------------------------------
def quiet(fn: Callable[[list[str]], int], argv: list[str]) -> tuple[int, str]:
out = io.StringIO()
with redirect_stdout(out), redirect_stderr(out):
code = fn(argv)
return code, out.getvalue()
def must(fn: Callable[[list[str]], int], argv: list[str], what: str, ok: tuple[int, ...] = (0,)) -> str:
code, out = quiet(fn, argv)
if code not in ok:
raise GateError(f"{what} exited {code}:\n{out}")
return out
def sentence_key(text: str, n: int = 0) -> str:
"""The sentence key klink gives the n-th sentence (from 0) with this text on a page."""
return hashlib.sha256(klink.key_text(text).encode("utf-8")).hexdigest()[:16] + f"-{n}"
def claim_id(page: str, text: str, n: int = 0) -> str:
return f"{page}#{sentence_key(text, n)}"
def fragment_label(sources: Path, key: dict[str, Any]) -> str:
"""The footnote of a fragment by the frozen kassemble.label, its quote line found in its anchor span; or the
literal footnote a case gives."""
if "footnote" in key:
return str(key["footnote"])
lines = (sources / "kilde" / key["file"]).read_text(encoding="utf-8").split("\n")
line = next((n for n in range(key["start"], key["end"] + 1) if key["quote"] in lines[n - 1]), None)
if line is None:
raise GateError(f"the quote of key {key} is not in its span")
return kassemble.label({"file": key["file"], "start": key["start"], "end": key["end"], "quote_line": line,
"quote": key["quote"], "provenance": key["provenance"]})
@dataclass
class Base:
root: Path
sources: Path
work: Path
wiki: Path
links: Path
blocks: Path
claims: Path
pairs: Path
found: ClaimList = field(default_factory=ClaimList)
def build_base(manifest: dict[str, Any], cases_root: Path, tmp: Path) -> Base:
spec = manifest["base"]
root = tmp / "base"
root.mkdir()
b = Base(root, root / "sources", root / "work", root / "wiki", root / "links.json", root / "work" / "blocks.json",
root / "claims.json", root / "pairs.json")
shutil.copytree(cases_root / spec["sources"], b.sources)
b.work.mkdir()
order = b.work / "order.txt"
must(kblocks.main, ["order", str(b.sources), "--out", str(order)], "kblocks order")
must(kblocks.main, ["blocks", str(b.sources), "--order", str(order), "--out", str(b.blocks)], "kblocks blocks")
must(kpack.main, ["skeleton", "--sources", str(b.sources), "--work", str(b.work)], "kpack skeleton")
for n, rel in enumerate(spec["fragments"], 1):
must(kpack.main, ["pack", str(n), "--blocks", str(b.blocks), "--sources", str(b.sources), "--work", str(b.work),
"--size", str(spec["size"])], f"kpack pack {n}")
dest = b.work / "K-fragments" / f"pack-{n}.json"
dest.parent.mkdir(parents=True, exist_ok=True)
shutil.copyfile(cases_root / rel, dest)
out = must(kpack.main, ["accept", str(n), "--blocks", str(b.blocks), "--sources", str(b.sources), "--work",
str(b.work)], f"kpack accept {n}")
given = len(json.loads((cases_root / rel).read_text(encoding="utf-8"))["fragments"])
if f"accepted {given} of {given}" not in out:
raise GateError(f"pack {n}: not every fixture fragment was accepted:\n{out}")
build = ["build", "--work", str(b.work), "--sources", str(b.sources), "--out", str(b.wiki), "--at", spec["at"]]
must(kassemble.main, build, "kassemble build")
links_spec = json.loads((cases_root / spec["links"]).read_text(encoding="utf-8"))["links"]
proposals = {"links": [{"page": x["page"], "sentence": sentence_key(x["sentence"], x.get("n", 0)),
"anchor": x["anchor"], "target": x["target"]} for x in links_spec], "none": []}
prop_path = root / "proposals.json"
prop_path.write_text(json.dumps(proposals), encoding="utf-8")
must(klink.main, ["accept", str(prop_path), "--bundle", str(b.wiki), "--links", str(b.links)], "klink accept")
placed = json.loads(b.links.read_text(encoding="utf-8"))
if placed["accepted"] != len(links_spec) or placed.get("rejected"):
raise GateError(f"the base links were not all accepted: {placed['accepted']} of {len(links_spec)}, "
f"rejected {placed.get('rejected')}")
must(kassemble.main, build + ["--links", str(b.links), "--replace"], "kassemble build --links")
prop_path.unlink()
must(claims.main, ["list", str(b.wiki), str(b.sources), "--out", str(b.claims), "--pairs", str(b.pairs)],
"claims list of the base")
b.found = claims.extract(b.wiki, b.sources)
findings = audit.audit(b.wiki, b.sources)
if findings:
raise GateError("the base has auditor findings: " + "; ".join(f"{f.check} {f.file}" for f in findings))
return b
# --- rendering a case's inputs ---------------------------------------------------------------------------------
def render_answers(answers: list[dict[str, Any]], sources: Path) -> dict[str, Any]:
out = []
for a in answers:
item: dict[str, Any] = {"concept_id": a["key"]["concept_id"], "footnote": fragment_label(sources, a["key"]),
"action": a["action"]}
if a["action"] == "replace":
item["fragments"] = a["fragments"]
out.append(item)
return {"schema": ANSWERS_SCHEMA, "answers": out}
def render_list(spec: dict[str, Any], b: Base) -> dict[str, Any]:
"""A repair list in K17's schema from a case's sentences and links, its ids, hashes and lines from the base."""
by_id = {c.id: c for c in b.found.claims}
k1 = {str(e["path"]): str(e["kind"])
for e in json.loads((b.sources / "source-manifest.json").read_text(encoding="utf-8"))["sources"]}
rows = []
for c in spec.get("claims", []):
cid = claim_id(c["page"], c["text"], c.get("n", 0))
if cid not in by_id:
raise GateError(f"list claim {cid} ({c['text']!r}) is not a claim of the base")
found = by_id[cid]
kinds = [k1.get(r.lstrip("/"), "docs") for r in found.resources]
rows.append({"id": cid, "excerpt_sha256": found.excerpt_sha256, "page": found.page, "line": found.line,
"provenance": found.provenance,
"kind": "notes" if kinds and kinds[0] == "release-notes" else "docs",
"sources": [{"file": f"kilde/{e.file}", "line": e.line} for e in found.excerpts],
"origins": ["o33"], "reasons": c.get("reasons", ["not-supported"])})
records = json.loads(b.links.read_text(encoding="utf-8"))["links"]
links = []
for x in spec.get("links", []):
key = sentence_key(x["sentence"], x.get("n", 0))
rec = next((r for r in records if r["page"] == x["page"] and r["sentence"] == key and r["anchor"] == x["anchor"]
and r["target"] == x["target"]), None)
if rec is None:
raise GateError(f"list link {x} is not a record of the base")
links.append({**rec, "origin": "o33-kandidat", "known_positive": bool(x.get("known_positive", False))})
return {"schema": LIST_SCHEMA, "claims_sha256": STALE_SHA if spec.get("stale") else sha256_file(b.claims),
"verdicts_sha256": EMPTY_SHA, "claims": rows, "pages": [], "pairs": [], "links": links}
# --- running the command ---------------------------------------------------------------------------------------
@dataclass
class Ran:
code: int | None
codes: list[str]
output: str
error: str = ""
bare: int = 0 # refusal lines that name no claim or record
def run_command(template: list[str], argv: list[str], timeout: float) -> Ran:
full = template + argv
try:
proc = subprocess.run(full, capture_output=True, text=True, timeout=timeout, check=False, cwd=ROOT)
except FileNotFoundError:
return Ran(None, [], "", f"command not found: {full[0]}")
except PermissionError:
return Ran(None, [], "", f"command not executable: {full[0]}")
except subprocess.TimeoutExpired:
return Ran(None, [], "", f"command timed out after {timeout:g}s")
output = proc.stdout + proc.stderr
codes, bare = [], 0
for line in output.splitlines():
if line.startswith("refused: "):
parts = line[len("refused: "):].split(" ", 1)
codes.append(parts[0])
bare += not (len(parts) == 2 and parts[1].strip())
return Ran(proc.returncode, codes, output, bare=bare)
def fill(argv: list[Any], places: dict[str, str]) -> list[str]:
"""The step's arguments with the placeholders filled in; an argument {"sentence": text, "n": n} becomes that
sentence's key."""
out = []
for a in argv:
if isinstance(a, dict):
out.append(sentence_key(str(a["sentence"]), int(a.get("n", 0))))
continue
for k, v in places.items():
a = a.replace("{" + k + "}", v)
out.append(str(a))
return out
# --- measuring -------------------------------------------------------------------------------------------------
@dataclass
class Measured:
changed: list[str]
gone: list[str]
claims_gone: Counter[str]
claims_came: Counter[str]
findings: Counter[tuple[str, str]]
criterion_1: tuple[int, int]
unkeyed: int
def concept_pages(wiki: Path) -> dict[str, bytes]:
return {p.relative_to(wiki).as_posix(): p.read_bytes() for p in sorted(wiki.rglob("*.md"))
if p.name not in RESERVED}
def measure(old: Path, new: Path, sources: Path) -> Measured:
a, z = concept_pages(old), concept_pages(new)
changed = sorted(n for n in set(a) | set(z) if a.get(n) != z.get(n))
before = Counter((c.id, c.excerpt_sha256) for c in claims.extract(old, sources).claims)
after_list = claims.extract(new, sources)
after = Counter((c.id, c.excerpt_sha256) for c in after_list.claims)
gone = Counter(i for i, _ in (before - after).elements())
came = Counter(i for i, _ in (after - before).elements())
findings = Counter((f.check, f.file) for f in audit.audit(new, sources))
total = claims.criterion_1(after_list)["total"]
return Measured(changed, sorted(set(a) - set(z)), gone, came, findings, (total[0], total[1]),
len(after_list.unkeyed))
def link_set(records: list[dict[str, Any]]) -> Counter[tuple[str, str, str, str]]:
return Counter((str(r["page"]), str(r["sentence"]), str(r["anchor"]), str(r["target"])) for r in records)
def link_tuple(x: dict[str, Any]) -> tuple[str, str, str, str]:
return x["page"], sentence_key(x["sentence"], x.get("n", 0)), x["anchor"], x["target"]
def misplaced_links(records: list[dict[str, Any]], wiki: Path) -> list[str]:
"""Records whose `at` does not hold their anchor in their sentence on the built page (link-reduced, as klink)."""
pages = klink.read_pages(wiki)
bad = []
for r in records:
page = pages.get(str(r["page"]))
s = next((s for s in klink.body_sentences(page.body) if s.key == r["sentence"]), None) if page else None
at = int(r["at"])
if s is None or s.text[at:at + len(str(r["anchor"]))] != r["anchor"]:
bad.append(f"{r['page']} {r['sentence']} {r['anchor']!r} at {r['at']}")
return bad
def excerpt_lines(wiki: Path, sources: Path, cid: str) -> list[list[Any]] | None:
c = next((c for c in claims.extract(wiki, sources).claims if c.id == cid), None)
return None if c is None else [[e.file, e.line] for e in c.excerpts]
# --- one case --------------------------------------------------------------------------------------------------
@dataclass
class Copy:
root: Path
places: dict[str, str]
def path(self, name: str) -> Path:
return Path(self.places[name])
def trees(self, *names: str) -> dict[str, dict[str, str]]:
return {n: hash_tree(self.path(n)) for n in names}
def fresh_copy(b: Base, tmp: Path, name: str) -> Copy:
root = tmp / name
root.mkdir()
for d in ("sources", "work", "wiki"):
shutil.copytree(b.root / d, root / d)
shutil.copytree(b.wiki, root / "wiki_old")
for f in ("links.json", "claims.json", "pairs.json"):
shutil.copyfile(b.root / f, root / f)
for d in ("in", "out", "diff"):
(root / d).mkdir()
places = {"work": root / "work", "wiki": root / "wiki", "wiki_old": root / "wiki_old", "links": root / "links.json",
"sources": root / "sources", "blocks": root / "work" / "blocks.json", "claims": root / "claims.json",
"pairs": root / "pairs.json", "out": root / "out", "diff": root / "diff", "in": root / "in"}
return Copy(root, {k: str(v) for k, v in places.items()})
def render_step(step: dict[str, Any], i: int, c: Copy, b: Base, cases_root: Path) -> dict[str, str]:
places = dict(c.places)
if "answers" in step:
p = c.path("in") / f"answers-{i}.json"
p.write_text(json.dumps(render_answers(step["answers"], b.sources), indent=1), encoding="utf-8")
places["answers"] = str(p)
if "list" in step:
p = c.path("in") / f"list-{i}.json"
p.write_text(json.dumps(render_list(step["list"], b), indent=1), encoding="utf-8")
places["list"] = str(p)
if "agent" in step:
p = c.path("in") / f"agent-{i}.md"
shutil.copyfile(cases_root / step["agent"], p)
places["agent"] = str(p)
return places
def check_task(case: dict[str, Any], c: Copy, b: Base) -> list[str]:
problems: list[str] = []
out = c.path("out")
tasks = sorted(out.rglob("*.parts.json"))
if not tasks:
return ["task wrote no <task>.parts.json"]
texts: dict[str, str] = {}
answers: list[str] = []
for t in tasks:
try:
doc = json.loads(t.read_text(encoding="utf-8"))
parts = [Path(p) if Path(p).is_absolute() else out / p for p in doc["parts"]]
answer = doc["answer"]
if not parts or not isinstance(answer, str) or not answer:
raise ValueError("no parts or no answer path")
except (OSError, ValueError, KeyError, TypeError) as exc:
problems.append(f"{t.name}: not a part list with parts and one answer path ({exc})")
continue
answers.append(answer)
body = []
for p in parts:
if not p.is_file():
problems.append(f"{t.name}: part {p} does not exist")
continue
text = p.read_text(encoding="utf-8")
if len(text) > kpack.PART_CHARS:
problems.append(f"{t.name}: part {p.name} has {len(text)} characters, over {kpack.PART_CHARS}")
body.append(text)
texts[t.name] = "\n".join(body)
if len(set(answers)) != len(answers):
problems.append("two tasks share an answer path")
for want in case["task"]:
label = fragment_label(b.sources, want["key"])
holders = [n for n, text in texts.items() if label in text]
if len(holders) != 1:
problems.append(f"fragment {want['key']['concept_id']} {label} stands in {len(holders)} tasks, not 1")
continue
for s in want.get("must_contain", []):
if s not in texts[holders[0]]:
problems.append(f"the task with {label} lacks {s!r}")
return problems
def check_linkset(case: dict[str, Any], c: Copy) -> list[str]:
problems: list[str] = []
out = c.path("out")
groups = sorted(out.glob("group-*.sentences.json"))
if not groups:
return ["linkset wrote no group-<n>.sentences.json"]
found: dict[tuple[str, str], list[str]] = {}
for g in groups:
stem = g.name[: -len(".sentences.json")]
cat = out / f"{stem}.catalogue"
try:
doc = json.loads(g.read_text(encoding="utf-8"))
pages = doc["pages"] if doc.get("schema") == klink.SENTENCES_SCHEMA else None
if not isinstance(pages, dict):
raise ValueError(f"not {klink.SENTENCES_SCHEMA} with pages")
ids = klink.read_ids(cat)
except (OSError, ValueError, KeyError, TypeError) as exc:
problems.append(f"{stem}: unreadable sentence file or catalogue ({exc})")
continue
for page, keys in pages.items():
for k in keys:
found[(str(page), str(k))] = ids
target = c.root / "klink-task" / stem
shutil.rmtree(target, ignore_errors=True)
code, said = quiet(klink.main, ["task", c.places["wiki"], "--sentences", str(g), "--catalogue", str(cat),
"--out", str(target)])
if code != 0:
problems.append(f"{stem}: klink task cannot run on it (exit {code}): {said.strip()[-200:]}")
wanted = {(x["page"], sentence_key(x["sentence"], x.get("n", 0))): x["target"] for x in case["linkset"]}
if set(found) != set(wanted):
problems.append(f"sentences: {sorted(found)} written, {sorted(wanted)} wanted")
for k, target in wanted.items():
if k in found and target.removesuffix(".md") not in found[k]:
problems.append(f"the catalogue for {k} lacks its current target {target}")
return problems
def check_written(case: dict[str, Any], c: Copy, b: Base, template: list[str], at: str, timeout: float,
repairs_log: bool) -> list[str]:
problems: list[str] = []
code, said = quiet(kassemble.main, ["build", "--work", c.places["work"], "--sources", c.places["sources"], "--out",
c.places["wiki"], "--links", c.places["links"], "--replace", "--at", at])
if code != 0:
return [f"kassemble build after the command exited {code}: {said.strip()[-300:]}"]
m = measure(c.path("wiki_old"), c.path("wiki"), c.path("sources"))
if m.changed != sorted(case.get("pages_changed", [])):
problems.append(f"pages changed: {m.changed}, the case says {sorted(case.get('pages_changed', []))}")
if m.gone != sorted(case.get("pages_gone", [])):
problems.append(f"pages gone: {m.gone}, the case says {sorted(case.get('pages_gone', []))}")
want_gone = Counter(claim_id(x["page"], x["text"], x.get("n", 0)) for x in case.get("claims_before", []))
want_came = Counter(claim_id(x["page"], x["text"], x.get("n", 0)) for x in case.get("claims_after", []))
if m.claims_gone != want_gone:
problems.append(f"claims gone or changed: {sorted(m.claims_gone.elements())}, the case says "
f"{sorted(want_gone.elements())}")
if m.claims_came != want_came:
problems.append(f"claims new or changed: {sorted(m.claims_came.elements())}, the case says "
f"{sorted(want_came.elements())}")
for x in case.get("claims_after", []):
if "excerpts" in x:
got = excerpt_lines(c.path("wiki"), c.path("sources"), claim_id(x["page"], x["text"], x.get("n", 0)))
if got != x["excerpts"]:
problems.append(f"excerpts of {x['text']!r}: {got}, the case says {x['excerpts']}")
want_findings = Counter((f["check"], f["file"]) for f in case.get("findings", []))
if m.findings != want_findings:
problems.append(f"auditor findings: {sorted(m.findings.elements())}, the case says "
f"{sorted(want_findings.elements())}")
if not (m.criterion_1[1] and m.criterion_1[0] == m.criterion_1[1] and m.unkeyed == 0):
problems.append(f"criterion 1: {m.criterion_1[0]} of {m.criterion_1[1]}, unkeyed {m.unkeyed}")
records = json.loads(c.path("links").read_text(encoding="utf-8"))["links"]
base = link_set(json.loads(b.links.read_text(encoding="utf-8"))["links"])
want = base - Counter(link_tuple(x) for x in case.get("links_removed", []))
want.update(Counter(link_tuple(x) for x in case.get("links_added", [])))
if link_set(records) != want:
problems.append(f"links file: {sorted(link_set(records).elements())}, the case says {sorted(want.elements())}")
problems += [f"link record not at its anchor: {x}" for x in misplaced_links(records, c.path("wiki"))]
if repairs_log:
problems += check_diff(c, template, m, timeout)
return problems
def check_diff(c: Copy, template: list[str], m: Measured, timeout: float) -> list[str]:
out = c.path("diff")
argv = ["diff", "--old", c.places["wiki_old"], "--new", c.places["wiki"], "--sources", c.places["sources"],
"--repairs", str(c.path("work") / "K-work" / "repairs.json"), "--out", str(out)]
ran = run_command(template, argv, timeout)
if ran.code != 0:
return [f"diff exited {ran.code} {ran.error}: {ran.output.strip()[-300:]}"]
problems = []
try:
ids = json.loads((out / "ids.json").read_text(encoding="utf-8"))["ids"]
pages = json.loads((out / "sider.json").read_text(encoding="utf-8"))["pages"]
except (OSError, ValueError, KeyError, TypeError) as exc:
return [f"diff output unreadable: {exc}"]
if sorted(ids) != sorted(m.claims_came.elements()):
problems.append(f"diff ids.json: {sorted(ids)}, the gate measured {sorted(m.claims_came.elements())}")
if sorted(pages) != m.changed:
problems.append(f"diff sider.json: {sorted(pages)}, the gate measured {m.changed}")
return problems
def run_case(case: dict[str, Any], b: Base, template: list[str], cases_root: Path, tmp: Path, at: str,
timeout: float) -> list[str]:
c = fresh_copy(b, tmp, case["name"])
fixed = c.trees("sources", "wiki_old")
problems: list[str] = []
last: dict[str, Any] = {}
for i, step in enumerate(case["steps"], 1):
places = render_step(step, i, c, b, cases_root)
before = c.trees("work", "wiki", "links", "out")
ran = run_command(template, fill(step["argv"], places), timeout)
if ran.code != step["exit"]:
tail = ran.error or ran.output.strip()[-300:]
problems.append(f"step {i} ({step['argv'][0]}): exit {ran.code}, the case says {step['exit']}: {tail}")
return problems
if sorted(set(ran.codes)) != sorted(set(step.get("refused", []))):
problems.append(f"step {i}: refusal codes {sorted(set(ran.codes))}, the case says "
f"{sorted(set(step.get('refused', [])))}")
if ran.bare:
problems.append(f"step {i}: {ran.bare} refusal line(s) name no claim or record")
after = c.trees("work", "wiki", "links", "out")
if step["exit"] == 1:
moved = [n for n in before if before[n] != after[n]]
if moved:
problems.append(f"step {i} refused but changed {', '.join(moved)}")
elif step["argv"][0] in ("task", "linkset"):
moved = [n for n in ("work", "wiki", "links") if before[n] != after[n]]
if moved:
problems.append(f"step {i} ({step['argv'][0]}) changed {', '.join(moved)}")
last = step
if last["exit"] == 0:
if last["argv"][0] == "task":
problems += check_task(case, c, b)
elif last["argv"][0] == "linkset":
problems += check_linkset(case, c)
else:
problems += check_written(case, c, b, template, at, timeout, last["argv"][0] == "fix")
if c.trees("sources", "wiki_old") != fixed:
problems.append("the sources or the old collection changed")
return problems
# --- the gate --------------------------------------------------------------------------------------------------
def report(cases: list[dict[str, Any]], results: dict[str, list[str]], command: list[str]) -> bool:
print(f"repair gate: {len(cases)} cases; command: {' '.join(command)}")
for case in cases:
problems = results[case["name"]]
print(f" {case['name']:<44} {'PASS' if not problems else 'FAIL'}")
for p in problems:
print(f" {p}")
passed = sum(not results[c["name"]] for c in cases)
print(f"passed {passed} of {len(cases)}")
green = passed == len(cases)
print(f"verdict: {'GREEN' if green else 'RED'}")
return green
def run(manifest_path: Path, command: list[str], timeout: float) -> int:
try:
manifest = load_manifest(manifest_path)
cases_root = manifest_path.parent / CASES_DIR
verify_fixtures(manifest, cases_root)
with tempfile.TemporaryDirectory(prefix="repair-gate-") as tmp_name:
tmp = Path(tmp_name)
base = build_base(manifest, cases_root, tmp)
results = {case["name"]: run_case(case, base, command, cases_root, tmp, manifest["build_at"], timeout)
for case in manifest["cases"]}
except GateError as exc:
print(f"GATE CANNOT RUN: {exc}")
return 2
return 0 if report(manifest["cases"], results, command) else 1
def main(argv: list[str] | None = None) -> int:
args_in = sys.argv[1:] if argv is None else argv
if "--" in args_in:
split = args_in.index("--")
own, command = args_in[:split], args_in[split + 1:]
else:
own, command = args_in, []
parser = argparse.ArgumentParser(description="Run the repair gate (fail-closed, no model calls).")
parser.add_argument("--manifest", type=Path, required=True)
parser.add_argument("--timeout", type=float, default=120.0, help="seconds per command run")
args = parser.parse_args(own)
if not command:
print("usage error: give the command after --, e.g. -- python3 -m okfwiki.krepair")
return 2
return run(args.manifest, command, args.timeout)
if __name__ == "__main__":
sys.exit(main())