First public release, published as one snapshot. What changed between versions is in CHANGELOG.md. Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
665 lines
32 KiB
Python
665 lines
32 KiB
Python
"""Repair gate (order 34, contracts K18 and K19): the judge of the repair command, run before the command exists.
|
|
|
|
No model call. The gate is fail-closed: a command that is missing, crashes, times out, refuses with another code or
|
|
changes more or less than the case says, fails the case, and the gate is GREEN only when every case passes.
|
|
|
|
The gate builds its own collection from the synthetic fixtures in gates/repair/ (a K1 source root about beekeeping,
|
|
three docs pages and one release note, and the compiler's fragments for them) with the frozen tools: kblocks order and
|
|
blocks, kpack skeleton, pack and accept, kassemble build --at a fixed time, klink accept and kassemble build --links.
|
|
No generated page is in git. The base must come out clean (no auditor finding, criterion 1 N of N) or the gate cannot
|
|
run. Every case then runs in a fresh copy of that base:
|
|
|
|
1. the command's steps run with the placeholders filled in; each step's exit code and refusal codes are checked;
|
|
2. a case that writes is built with kassemble build --links --replace at another fixed time, and measured by the gate's
|
|
own code, apart from the command's report: exactly the concept pages the case names have other bytes (index.md and
|
|
log.md left out); the claims (the frozen claims extract) that are gone or new, by id and excerpt_sha256, are exactly
|
|
the ids the case derives from its hand-written sentences, so every other claim keeps its id and excerpt_sha256;
|
|
the auditor's findings are exactly the case's (check and file), and criterion 1 is N of N; the links file holds
|
|
exactly the base records without the case's removed records and with its added ones, each placed at its anchor;
|
|
3. the command's diff runs on the old and the new collection, and its ids.json and sider.json must equal what the gate
|
|
measured (a case without a repairs log, the link commands, leaves this out);
|
|
4. a refused step changes no file in the work folder, the links file, the collection or the output folder (a tree
|
|
hash before and after); task and linkset change none of the first three, and their output is checked as the case
|
|
says.
|
|
|
|
The denominator comes from the sources: the pages, sentences, links and findings each case expects are written by hand
|
|
in gates/repair.manifest.json as page names and sentence text before and after; the gate turns a sentence into its
|
|
claim id with the frozen klink.key_text, and a fragment into its footnote with the frozen kassemble.label. Before
|
|
anything runs, every file under gates/repair/ is checked against the manifest's sha256: a changed, missing or extra
|
|
file stops the gate (exit 2).
|
|
|
|
Usage:
|
|
python3 gates/repair_gate.py --manifest gates/repair.manifest.json [--timeout 120] -- <command> ...
|
|
The command is the program the case's arguments are appended to, e.g. `python3 -m okfwiki.krepair`; it runs from the
|
|
repository root. Placeholders in a case's arguments: {work} {wiki} {wiki_old} {links} {sources} {blocks} {claims}
|
|
{pairs} {out} {diff}, and the step's rendered inputs {answers} {list} {agent}.
|
|
Exit: 0 GREEN, 1 RED, 2 the gate could not run.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import hashlib
|
|
import io
|
|
import json
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
from collections import Counter
|
|
from contextlib import redirect_stderr, redirect_stdout
|
|
from dataclasses import dataclass, field
|
|
from pathlib import Path
|
|
from typing import Any, Callable
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT))
|
|
|
|
from okfwiki import audit, claims, kassemble, kblocks, klink, kpack # noqa: E402
|
|
from okfwiki.claims import ClaimList # noqa: E402
|
|
|
|
SCHEMA = "okf-wiki/repair-gate/1"
|
|
ANSWERS_SCHEMA = "okf-wiki/repair-answers/1"
|
|
LIST_SCHEMA = "okf-wiki/repair-list/1"
|
|
PAGES_SCHEMA = "okf-wiki/repair-pages/1"
|
|
CASES_DIR = "repair" # beside the manifest
|
|
RESERVED = ("index.md", "log.md")
|
|
EMPTY_SHA = hashlib.sha256(b"").hexdigest()
|
|
STALE_SHA = hashlib.sha256(b"stale").hexdigest()
|
|
|
|
|
|
class GateError(Exception):
|
|
"""The gate cannot produce a measurement (bad manifest, drifted fixtures, a base that does not build clean)."""
|
|
|
|
|
|
# --- files -----------------------------------------------------------------------------------------------------
|
|
|
|
|
|
def sha256_file(path: Path) -> str:
|
|
return hashlib.sha256(path.read_bytes()).hexdigest()
|
|
|
|
|
|
def hash_tree(root: Path) -> dict[str, str]:
|
|
"""sha256 of every regular file under root (or of root itself when it is a file), keyed by relative POSIX path."""
|
|
if root.is_file():
|
|
return {".": sha256_file(root)}
|
|
out: dict[str, str] = {}
|
|
if not root.exists():
|
|
return out
|
|
for p in sorted(root.rglob("*")):
|
|
if p.is_file() and "__pycache__" not in p.parts:
|
|
out[p.relative_to(root).as_posix()] = sha256_file(p)
|
|
return out
|
|
|
|
|
|
def load_manifest(path: Path) -> dict[str, Any]:
|
|
try:
|
|
data = json.loads(path.read_text(encoding="utf-8"))
|
|
except (OSError, json.JSONDecodeError) as exc:
|
|
raise GateError(f"cannot read manifest {path}: {exc}") from exc
|
|
if not isinstance(data, dict) or data.get("schema") != SCHEMA:
|
|
raise GateError(f"manifest schema must be {SCHEMA}")
|
|
for key in ("files", "base", "cases", "build_at", "refusal_codes"):
|
|
if key not in data:
|
|
raise GateError(f"manifest has no {key}")
|
|
names = [c.get("name") for c in data["cases"]]
|
|
if not names or len(names) != len(set(names)) or not all(isinstance(n, str) and n for n in names):
|
|
raise GateError("case names are missing or not unique")
|
|
codes = set(data["refusal_codes"])
|
|
for case in data["cases"]:
|
|
steps = case.get("steps")
|
|
if not isinstance(steps, list) or not steps:
|
|
raise GateError(f"case {case['name']} has no steps")
|
|
for step in steps:
|
|
if step.get("exit") not in (0, 1) or not isinstance(step.get("argv"), list):
|
|
raise GateError(f"case {case['name']}: a step needs argv and exit 0 or 1")
|
|
unknown = set(step.get("refused", [])) - codes
|
|
if unknown:
|
|
raise GateError(f"case {case['name']}: refusal codes not in the manifest: {sorted(unknown)}")
|
|
if (step["exit"] == 1) != bool(step.get("refused")):
|
|
raise GateError(f"case {case['name']}: a step exits 1 exactly when it names refusal codes")
|
|
return data
|
|
|
|
|
|
def verify_fixtures(manifest: dict[str, Any], cases_root: Path) -> None:
|
|
expected: dict[str, str] = manifest["files"]
|
|
actual = hash_tree(cases_root)
|
|
problems = [f"missing file {rel}" for rel in sorted(set(expected) - set(actual))]
|
|
problems += [f"unexpected file {rel}" for rel in sorted(set(actual) - set(expected))]
|
|
problems += [f"sha256 mismatch in {rel}" for rel in sorted(set(actual) & set(expected))
|
|
if actual[rel] != expected[rel]]
|
|
if problems:
|
|
raise GateError("fixtures do not match the manifest:\n " + "\n ".join(problems))
|
|
|
|
|
|
# --- the frozen tools, in process ------------------------------------------------------------------------------
|
|
|
|
|
|
def quiet(fn: Callable[[list[str]], int], argv: list[str]) -> tuple[int, str]:
|
|
out = io.StringIO()
|
|
with redirect_stdout(out), redirect_stderr(out):
|
|
code = fn(argv)
|
|
return code, out.getvalue()
|
|
|
|
|
|
def must(fn: Callable[[list[str]], int], argv: list[str], what: str, ok: tuple[int, ...] = (0,)) -> str:
|
|
code, out = quiet(fn, argv)
|
|
if code not in ok:
|
|
raise GateError(f"{what} exited {code}:\n{out}")
|
|
return out
|
|
|
|
|
|
def sentence_key(text: str, n: int = 0) -> str:
|
|
"""The sentence key klink gives the n-th sentence (from 0) with this text on a page."""
|
|
return hashlib.sha256(klink.key_text(text).encode("utf-8")).hexdigest()[:16] + f"-{n}"
|
|
|
|
|
|
def claim_id(page: str, text: str, n: int = 0) -> str:
|
|
return f"{page}#{sentence_key(text, n)}"
|
|
|
|
|
|
def fragment_label(sources: Path, key: dict[str, Any]) -> str:
|
|
"""The footnote of a fragment by the frozen kassemble.label, its quote line found in its anchor span; or the
|
|
literal footnote a case gives."""
|
|
if "footnote" in key:
|
|
return str(key["footnote"])
|
|
lines = (sources / "kilde" / key["file"]).read_text(encoding="utf-8").split("\n")
|
|
line = next((n for n in range(key["start"], key["end"] + 1) if key["quote"] in lines[n - 1]), None)
|
|
if line is None:
|
|
raise GateError(f"the quote of key {key} is not in its span")
|
|
return kassemble.label({"file": key["file"], "start": key["start"], "end": key["end"], "quote_line": line,
|
|
"quote": key["quote"], "provenance": key["provenance"]})
|
|
|
|
|
|
@dataclass
|
|
class Base:
|
|
root: Path
|
|
sources: Path
|
|
work: Path
|
|
wiki: Path
|
|
links: Path
|
|
blocks: Path
|
|
claims: Path
|
|
pairs: Path
|
|
found: ClaimList = field(default_factory=ClaimList)
|
|
|
|
|
|
def build_base(manifest: dict[str, Any], cases_root: Path, tmp: Path) -> Base:
|
|
spec = manifest["base"]
|
|
root = tmp / "base"
|
|
root.mkdir()
|
|
b = Base(root, root / "sources", root / "work", root / "wiki", root / "links.json", root / "work" / "blocks.json",
|
|
root / "claims.json", root / "pairs.json")
|
|
shutil.copytree(cases_root / spec["sources"], b.sources)
|
|
b.work.mkdir()
|
|
order = b.work / "order.txt"
|
|
must(kblocks.main, ["order", str(b.sources), "--out", str(order)], "kblocks order")
|
|
must(kblocks.main, ["blocks", str(b.sources), "--order", str(order), "--out", str(b.blocks)], "kblocks blocks")
|
|
must(kpack.main, ["skeleton", "--sources", str(b.sources), "--work", str(b.work)], "kpack skeleton")
|
|
for n, rel in enumerate(spec["fragments"], 1):
|
|
must(kpack.main, ["pack", str(n), "--blocks", str(b.blocks), "--sources", str(b.sources), "--work", str(b.work),
|
|
"--size", str(spec["size"])], f"kpack pack {n}")
|
|
dest = b.work / "K-fragments" / f"pack-{n}.json"
|
|
dest.parent.mkdir(parents=True, exist_ok=True)
|
|
shutil.copyfile(cases_root / rel, dest)
|
|
out = must(kpack.main, ["accept", str(n), "--blocks", str(b.blocks), "--sources", str(b.sources), "--work",
|
|
str(b.work)], f"kpack accept {n}")
|
|
given = len(json.loads((cases_root / rel).read_text(encoding="utf-8"))["fragments"])
|
|
if f"accepted {given} of {given}" not in out:
|
|
raise GateError(f"pack {n}: not every fixture fragment was accepted:\n{out}")
|
|
build = ["build", "--work", str(b.work), "--sources", str(b.sources), "--out", str(b.wiki), "--at", spec["at"]]
|
|
must(kassemble.main, build, "kassemble build")
|
|
links_spec = json.loads((cases_root / spec["links"]).read_text(encoding="utf-8"))["links"]
|
|
proposals = {"links": [{"page": x["page"], "sentence": sentence_key(x["sentence"], x.get("n", 0)),
|
|
"anchor": x["anchor"], "target": x["target"]} for x in links_spec], "none": []}
|
|
prop_path = root / "proposals.json"
|
|
prop_path.write_text(json.dumps(proposals), encoding="utf-8")
|
|
must(klink.main, ["accept", str(prop_path), "--bundle", str(b.wiki), "--links", str(b.links)], "klink accept")
|
|
placed = json.loads(b.links.read_text(encoding="utf-8"))
|
|
if placed["accepted"] != len(links_spec) or placed.get("rejected"):
|
|
raise GateError(f"the base links were not all accepted: {placed['accepted']} of {len(links_spec)}, "
|
|
f"rejected {placed.get('rejected')}")
|
|
must(kassemble.main, build + ["--links", str(b.links), "--replace"], "kassemble build --links")
|
|
prop_path.unlink()
|
|
must(claims.main, ["list", str(b.wiki), str(b.sources), "--out", str(b.claims), "--pairs", str(b.pairs)],
|
|
"claims list of the base")
|
|
b.found = claims.extract(b.wiki, b.sources)
|
|
findings = audit.audit(b.wiki, b.sources)
|
|
if findings:
|
|
raise GateError("the base has auditor findings: " + "; ".join(f"{f.check} {f.file}" for f in findings))
|
|
return b
|
|
|
|
|
|
# --- rendering a case's inputs ---------------------------------------------------------------------------------
|
|
|
|
|
|
def render_answers(answers: list[dict[str, Any]], sources: Path) -> dict[str, Any]:
|
|
out = []
|
|
for a in answers:
|
|
item: dict[str, Any] = {"concept_id": a["key"]["concept_id"], "footnote": fragment_label(sources, a["key"]),
|
|
"action": a["action"]}
|
|
if a["action"] == "replace":
|
|
item["fragments"] = a["fragments"]
|
|
out.append(item)
|
|
return {"schema": ANSWERS_SCHEMA, "answers": out}
|
|
|
|
|
|
def render_list(spec: dict[str, Any], b: Base) -> dict[str, Any]:
|
|
"""A repair list in K17's schema from a case's sentences and links, its ids, hashes and lines from the base."""
|
|
by_id = {c.id: c for c in b.found.claims}
|
|
k1 = {str(e["path"]): str(e["kind"])
|
|
for e in json.loads((b.sources / "source-manifest.json").read_text(encoding="utf-8"))["sources"]}
|
|
rows = []
|
|
for c in spec.get("claims", []):
|
|
cid = claim_id(c["page"], c["text"], c.get("n", 0))
|
|
if cid not in by_id:
|
|
raise GateError(f"list claim {cid} ({c['text']!r}) is not a claim of the base")
|
|
found = by_id[cid]
|
|
kinds = [k1.get(r.lstrip("/"), "docs") for r in found.resources]
|
|
rows.append({"id": cid, "excerpt_sha256": found.excerpt_sha256, "page": found.page, "line": found.line,
|
|
"provenance": found.provenance,
|
|
"kind": "notes" if kinds and kinds[0] == "release-notes" else "docs",
|
|
"sources": [{"file": f"kilde/{e.file}", "line": e.line} for e in found.excerpts],
|
|
"origins": ["o33"], "reasons": c.get("reasons", ["not-supported"])})
|
|
records = json.loads(b.links.read_text(encoding="utf-8"))["links"]
|
|
links = []
|
|
for x in spec.get("links", []):
|
|
key = sentence_key(x["sentence"], x.get("n", 0))
|
|
rec = next((r for r in records if r["page"] == x["page"] and r["sentence"] == key and r["anchor"] == x["anchor"]
|
|
and r["target"] == x["target"]), None)
|
|
if rec is None:
|
|
raise GateError(f"list link {x} is not a record of the base")
|
|
links.append({**rec, "origin": "o33-kandidat", "known_positive": bool(x.get("known_positive", False))})
|
|
return {"schema": LIST_SCHEMA, "claims_sha256": STALE_SHA if spec.get("stale") else sha256_file(b.claims),
|
|
"verdicts_sha256": EMPTY_SHA, "claims": rows, "pages": [], "pairs": [], "links": links}
|
|
|
|
|
|
# --- running the command ---------------------------------------------------------------------------------------
|
|
|
|
|
|
@dataclass
|
|
class Ran:
|
|
code: int | None
|
|
codes: list[str]
|
|
output: str
|
|
error: str = ""
|
|
bare: int = 0 # refusal lines that name no claim or record
|
|
|
|
|
|
def run_command(template: list[str], argv: list[str], timeout: float) -> Ran:
|
|
full = template + argv
|
|
try:
|
|
proc = subprocess.run(full, capture_output=True, text=True, timeout=timeout, check=False, cwd=ROOT)
|
|
except FileNotFoundError:
|
|
return Ran(None, [], "", f"command not found: {full[0]}")
|
|
except PermissionError:
|
|
return Ran(None, [], "", f"command not executable: {full[0]}")
|
|
except subprocess.TimeoutExpired:
|
|
return Ran(None, [], "", f"command timed out after {timeout:g}s")
|
|
output = proc.stdout + proc.stderr
|
|
codes, bare = [], 0
|
|
for line in output.splitlines():
|
|
if line.startswith("refused: "):
|
|
parts = line[len("refused: "):].split(" ", 1)
|
|
codes.append(parts[0])
|
|
bare += not (len(parts) == 2 and parts[1].strip())
|
|
return Ran(proc.returncode, codes, output, bare=bare)
|
|
|
|
|
|
def fill(argv: list[Any], places: dict[str, str]) -> list[str]:
|
|
"""The step's arguments with the placeholders filled in; an argument {"sentence": text, "n": n} becomes that
|
|
sentence's key."""
|
|
out = []
|
|
for a in argv:
|
|
if isinstance(a, dict):
|
|
out.append(sentence_key(str(a["sentence"]), int(a.get("n", 0))))
|
|
continue
|
|
for k, v in places.items():
|
|
a = a.replace("{" + k + "}", v)
|
|
out.append(str(a))
|
|
return out
|
|
|
|
|
|
# --- measuring -------------------------------------------------------------------------------------------------
|
|
|
|
|
|
@dataclass
|
|
class Measured:
|
|
changed: list[str]
|
|
gone: list[str]
|
|
claims_gone: Counter[str]
|
|
claims_came: Counter[str]
|
|
findings: Counter[tuple[str, str]]
|
|
criterion_1: tuple[int, int]
|
|
unkeyed: int
|
|
|
|
|
|
def concept_pages(wiki: Path) -> dict[str, bytes]:
|
|
return {p.relative_to(wiki).as_posix(): p.read_bytes() for p in sorted(wiki.rglob("*.md"))
|
|
if p.name not in RESERVED}
|
|
|
|
|
|
def measure(old: Path, new: Path, sources: Path) -> Measured:
|
|
a, z = concept_pages(old), concept_pages(new)
|
|
changed = sorted(n for n in set(a) | set(z) if a.get(n) != z.get(n))
|
|
before = Counter((c.id, c.excerpt_sha256) for c in claims.extract(old, sources).claims)
|
|
after_list = claims.extract(new, sources)
|
|
after = Counter((c.id, c.excerpt_sha256) for c in after_list.claims)
|
|
gone = Counter(i for i, _ in (before - after).elements())
|
|
came = Counter(i for i, _ in (after - before).elements())
|
|
findings = Counter((f.check, f.file) for f in audit.audit(new, sources))
|
|
total = claims.criterion_1(after_list)["total"]
|
|
return Measured(changed, sorted(set(a) - set(z)), gone, came, findings, (total[0], total[1]),
|
|
len(after_list.unkeyed))
|
|
|
|
|
|
def link_set(records: list[dict[str, Any]]) -> Counter[tuple[str, str, str, str]]:
|
|
return Counter((str(r["page"]), str(r["sentence"]), str(r["anchor"]), str(r["target"])) for r in records)
|
|
|
|
|
|
def link_tuple(x: dict[str, Any]) -> tuple[str, str, str, str]:
|
|
return x["page"], sentence_key(x["sentence"], x.get("n", 0)), x["anchor"], x["target"]
|
|
|
|
|
|
def misplaced_links(records: list[dict[str, Any]], wiki: Path) -> list[str]:
|
|
"""Records whose `at` does not hold their anchor in their sentence on the built page (link-reduced, as klink)."""
|
|
pages = klink.read_pages(wiki)
|
|
bad = []
|
|
for r in records:
|
|
page = pages.get(str(r["page"]))
|
|
s = next((s for s in klink.body_sentences(page.body) if s.key == r["sentence"]), None) if page else None
|
|
at = int(r["at"])
|
|
if s is None or s.text[at:at + len(str(r["anchor"]))] != r["anchor"]:
|
|
bad.append(f"{r['page']} {r['sentence']} {r['anchor']!r} at {r['at']}")
|
|
return bad
|
|
|
|
|
|
def excerpt_lines(wiki: Path, sources: Path, cid: str) -> list[list[Any]] | None:
|
|
c = next((c for c in claims.extract(wiki, sources).claims if c.id == cid), None)
|
|
return None if c is None else [[e.file, e.line] for e in c.excerpts]
|
|
|
|
|
|
# --- one case --------------------------------------------------------------------------------------------------
|
|
|
|
|
|
@dataclass
|
|
class Copy:
|
|
root: Path
|
|
places: dict[str, str]
|
|
|
|
def path(self, name: str) -> Path:
|
|
return Path(self.places[name])
|
|
|
|
def trees(self, *names: str) -> dict[str, dict[str, str]]:
|
|
return {n: hash_tree(self.path(n)) for n in names}
|
|
|
|
|
|
def fresh_copy(b: Base, tmp: Path, name: str) -> Copy:
|
|
root = tmp / name
|
|
root.mkdir()
|
|
for d in ("sources", "work", "wiki"):
|
|
shutil.copytree(b.root / d, root / d)
|
|
shutil.copytree(b.wiki, root / "wiki_old")
|
|
for f in ("links.json", "claims.json", "pairs.json"):
|
|
shutil.copyfile(b.root / f, root / f)
|
|
for d in ("in", "out", "diff"):
|
|
(root / d).mkdir()
|
|
places = {"work": root / "work", "wiki": root / "wiki", "wiki_old": root / "wiki_old", "links": root / "links.json",
|
|
"sources": root / "sources", "blocks": root / "work" / "blocks.json", "claims": root / "claims.json",
|
|
"pairs": root / "pairs.json", "out": root / "out", "diff": root / "diff", "in": root / "in"}
|
|
return Copy(root, {k: str(v) for k, v in places.items()})
|
|
|
|
|
|
def render_step(step: dict[str, Any], i: int, c: Copy, b: Base, cases_root: Path) -> dict[str, str]:
|
|
places = dict(c.places)
|
|
if "answers" in step:
|
|
p = c.path("in") / f"answers-{i}.json"
|
|
p.write_text(json.dumps(render_answers(step["answers"], b.sources), indent=1), encoding="utf-8")
|
|
places["answers"] = str(p)
|
|
if "list" in step:
|
|
p = c.path("in") / f"list-{i}.json"
|
|
p.write_text(json.dumps(render_list(step["list"], b), indent=1), encoding="utf-8")
|
|
places["list"] = str(p)
|
|
if "agent" in step:
|
|
p = c.path("in") / f"agent-{i}.md"
|
|
shutil.copyfile(cases_root / step["agent"], p)
|
|
places["agent"] = str(p)
|
|
return places
|
|
|
|
|
|
def check_task(case: dict[str, Any], c: Copy, b: Base) -> list[str]:
|
|
problems: list[str] = []
|
|
out = c.path("out")
|
|
tasks = sorted(out.rglob("*.parts.json"))
|
|
if not tasks:
|
|
return ["task wrote no <task>.parts.json"]
|
|
texts: dict[str, str] = {}
|
|
answers: list[str] = []
|
|
for t in tasks:
|
|
try:
|
|
doc = json.loads(t.read_text(encoding="utf-8"))
|
|
parts = [Path(p) if Path(p).is_absolute() else out / p for p in doc["parts"]]
|
|
answer = doc["answer"]
|
|
if not parts or not isinstance(answer, str) or not answer:
|
|
raise ValueError("no parts or no answer path")
|
|
except (OSError, ValueError, KeyError, TypeError) as exc:
|
|
problems.append(f"{t.name}: not a part list with parts and one answer path ({exc})")
|
|
continue
|
|
answers.append(answer)
|
|
body = []
|
|
for p in parts:
|
|
if not p.is_file():
|
|
problems.append(f"{t.name}: part {p} does not exist")
|
|
continue
|
|
text = p.read_text(encoding="utf-8")
|
|
if len(text) > kpack.PART_CHARS:
|
|
problems.append(f"{t.name}: part {p.name} has {len(text)} characters, over {kpack.PART_CHARS}")
|
|
body.append(text)
|
|
texts[t.name] = "\n".join(body)
|
|
if len(set(answers)) != len(answers):
|
|
problems.append("two tasks share an answer path")
|
|
for want in case["task"]:
|
|
label = fragment_label(b.sources, want["key"])
|
|
holders = [n for n, text in texts.items() if label in text]
|
|
if len(holders) != 1:
|
|
problems.append(f"fragment {want['key']['concept_id']} {label} stands in {len(holders)} tasks, not 1")
|
|
continue
|
|
for s in want.get("must_contain", []):
|
|
if s not in texts[holders[0]]:
|
|
problems.append(f"the task with {label} lacks {s!r}")
|
|
return problems
|
|
|
|
|
|
def check_linkset(case: dict[str, Any], c: Copy) -> list[str]:
|
|
problems: list[str] = []
|
|
out = c.path("out")
|
|
groups = sorted(out.glob("group-*.sentences.json"))
|
|
if not groups:
|
|
return ["linkset wrote no group-<n>.sentences.json"]
|
|
found: dict[tuple[str, str], list[str]] = {}
|
|
for g in groups:
|
|
stem = g.name[: -len(".sentences.json")]
|
|
cat = out / f"{stem}.catalogue"
|
|
try:
|
|
doc = json.loads(g.read_text(encoding="utf-8"))
|
|
pages = doc["pages"] if doc.get("schema") == klink.SENTENCES_SCHEMA else None
|
|
if not isinstance(pages, dict):
|
|
raise ValueError(f"not {klink.SENTENCES_SCHEMA} with pages")
|
|
ids = klink.read_ids(cat)
|
|
except (OSError, ValueError, KeyError, TypeError) as exc:
|
|
problems.append(f"{stem}: unreadable sentence file or catalogue ({exc})")
|
|
continue
|
|
for page, keys in pages.items():
|
|
for k in keys:
|
|
found[(str(page), str(k))] = ids
|
|
target = c.root / "klink-task" / stem
|
|
shutil.rmtree(target, ignore_errors=True)
|
|
code, said = quiet(klink.main, ["task", c.places["wiki"], "--sentences", str(g), "--catalogue", str(cat),
|
|
"--out", str(target)])
|
|
if code != 0:
|
|
problems.append(f"{stem}: klink task cannot run on it (exit {code}): {said.strip()[-200:]}")
|
|
wanted = {(x["page"], sentence_key(x["sentence"], x.get("n", 0))): x["target"] for x in case["linkset"]}
|
|
if set(found) != set(wanted):
|
|
problems.append(f"sentences: {sorted(found)} written, {sorted(wanted)} wanted")
|
|
for k, target in wanted.items():
|
|
if k in found and target.removesuffix(".md") not in found[k]:
|
|
problems.append(f"the catalogue for {k} lacks its current target {target}")
|
|
return problems
|
|
|
|
|
|
def check_written(case: dict[str, Any], c: Copy, b: Base, template: list[str], at: str, timeout: float,
|
|
repairs_log: bool) -> list[str]:
|
|
problems: list[str] = []
|
|
code, said = quiet(kassemble.main, ["build", "--work", c.places["work"], "--sources", c.places["sources"], "--out",
|
|
c.places["wiki"], "--links", c.places["links"], "--replace", "--at", at])
|
|
if code != 0:
|
|
return [f"kassemble build after the command exited {code}: {said.strip()[-300:]}"]
|
|
m = measure(c.path("wiki_old"), c.path("wiki"), c.path("sources"))
|
|
if m.changed != sorted(case.get("pages_changed", [])):
|
|
problems.append(f"pages changed: {m.changed}, the case says {sorted(case.get('pages_changed', []))}")
|
|
if m.gone != sorted(case.get("pages_gone", [])):
|
|
problems.append(f"pages gone: {m.gone}, the case says {sorted(case.get('pages_gone', []))}")
|
|
want_gone = Counter(claim_id(x["page"], x["text"], x.get("n", 0)) for x in case.get("claims_before", []))
|
|
want_came = Counter(claim_id(x["page"], x["text"], x.get("n", 0)) for x in case.get("claims_after", []))
|
|
if m.claims_gone != want_gone:
|
|
problems.append(f"claims gone or changed: {sorted(m.claims_gone.elements())}, the case says "
|
|
f"{sorted(want_gone.elements())}")
|
|
if m.claims_came != want_came:
|
|
problems.append(f"claims new or changed: {sorted(m.claims_came.elements())}, the case says "
|
|
f"{sorted(want_came.elements())}")
|
|
for x in case.get("claims_after", []):
|
|
if "excerpts" in x:
|
|
got = excerpt_lines(c.path("wiki"), c.path("sources"), claim_id(x["page"], x["text"], x.get("n", 0)))
|
|
if got != x["excerpts"]:
|
|
problems.append(f"excerpts of {x['text']!r}: {got}, the case says {x['excerpts']}")
|
|
want_findings = Counter((f["check"], f["file"]) for f in case.get("findings", []))
|
|
if m.findings != want_findings:
|
|
problems.append(f"auditor findings: {sorted(m.findings.elements())}, the case says "
|
|
f"{sorted(want_findings.elements())}")
|
|
if not (m.criterion_1[1] and m.criterion_1[0] == m.criterion_1[1] and m.unkeyed == 0):
|
|
problems.append(f"criterion 1: {m.criterion_1[0]} of {m.criterion_1[1]}, unkeyed {m.unkeyed}")
|
|
records = json.loads(c.path("links").read_text(encoding="utf-8"))["links"]
|
|
base = link_set(json.loads(b.links.read_text(encoding="utf-8"))["links"])
|
|
want = base - Counter(link_tuple(x) for x in case.get("links_removed", []))
|
|
want.update(Counter(link_tuple(x) for x in case.get("links_added", [])))
|
|
if link_set(records) != want:
|
|
problems.append(f"links file: {sorted(link_set(records).elements())}, the case says {sorted(want.elements())}")
|
|
problems += [f"link record not at its anchor: {x}" for x in misplaced_links(records, c.path("wiki"))]
|
|
if repairs_log:
|
|
problems += check_diff(c, template, m, timeout)
|
|
return problems
|
|
|
|
|
|
def check_diff(c: Copy, template: list[str], m: Measured, timeout: float) -> list[str]:
|
|
out = c.path("diff")
|
|
argv = ["diff", "--old", c.places["wiki_old"], "--new", c.places["wiki"], "--sources", c.places["sources"],
|
|
"--repairs", str(c.path("work") / "K-work" / "repairs.json"), "--out", str(out)]
|
|
ran = run_command(template, argv, timeout)
|
|
if ran.code != 0:
|
|
return [f"diff exited {ran.code} {ran.error}: {ran.output.strip()[-300:]}"]
|
|
problems = []
|
|
try:
|
|
ids = json.loads((out / "ids.json").read_text(encoding="utf-8"))["ids"]
|
|
pages = json.loads((out / "sider.json").read_text(encoding="utf-8"))["pages"]
|
|
except (OSError, ValueError, KeyError, TypeError) as exc:
|
|
return [f"diff output unreadable: {exc}"]
|
|
if sorted(ids) != sorted(m.claims_came.elements()):
|
|
problems.append(f"diff ids.json: {sorted(ids)}, the gate measured {sorted(m.claims_came.elements())}")
|
|
if sorted(pages) != m.changed:
|
|
problems.append(f"diff sider.json: {sorted(pages)}, the gate measured {m.changed}")
|
|
return problems
|
|
|
|
|
|
def run_case(case: dict[str, Any], b: Base, template: list[str], cases_root: Path, tmp: Path, at: str,
|
|
timeout: float) -> list[str]:
|
|
c = fresh_copy(b, tmp, case["name"])
|
|
fixed = c.trees("sources", "wiki_old")
|
|
problems: list[str] = []
|
|
last: dict[str, Any] = {}
|
|
for i, step in enumerate(case["steps"], 1):
|
|
places = render_step(step, i, c, b, cases_root)
|
|
before = c.trees("work", "wiki", "links", "out")
|
|
ran = run_command(template, fill(step["argv"], places), timeout)
|
|
if ran.code != step["exit"]:
|
|
tail = ran.error or ran.output.strip()[-300:]
|
|
problems.append(f"step {i} ({step['argv'][0]}): exit {ran.code}, the case says {step['exit']}: {tail}")
|
|
return problems
|
|
if sorted(set(ran.codes)) != sorted(set(step.get("refused", []))):
|
|
problems.append(f"step {i}: refusal codes {sorted(set(ran.codes))}, the case says "
|
|
f"{sorted(set(step.get('refused', [])))}")
|
|
if ran.bare:
|
|
problems.append(f"step {i}: {ran.bare} refusal line(s) name no claim or record")
|
|
after = c.trees("work", "wiki", "links", "out")
|
|
if step["exit"] == 1:
|
|
moved = [n for n in before if before[n] != after[n]]
|
|
if moved:
|
|
problems.append(f"step {i} refused but changed {', '.join(moved)}")
|
|
elif step["argv"][0] in ("task", "linkset"):
|
|
moved = [n for n in ("work", "wiki", "links") if before[n] != after[n]]
|
|
if moved:
|
|
problems.append(f"step {i} ({step['argv'][0]}) changed {', '.join(moved)}")
|
|
last = step
|
|
if last["exit"] == 0:
|
|
if last["argv"][0] == "task":
|
|
problems += check_task(case, c, b)
|
|
elif last["argv"][0] == "linkset":
|
|
problems += check_linkset(case, c)
|
|
else:
|
|
problems += check_written(case, c, b, template, at, timeout, last["argv"][0] == "fix")
|
|
if c.trees("sources", "wiki_old") != fixed:
|
|
problems.append("the sources or the old collection changed")
|
|
return problems
|
|
|
|
|
|
# --- the gate --------------------------------------------------------------------------------------------------
|
|
|
|
|
|
def report(cases: list[dict[str, Any]], results: dict[str, list[str]], command: list[str]) -> bool:
|
|
print(f"repair gate: {len(cases)} cases; command: {' '.join(command)}")
|
|
for case in cases:
|
|
problems = results[case["name"]]
|
|
print(f" {case['name']:<44} {'PASS' if not problems else 'FAIL'}")
|
|
for p in problems:
|
|
print(f" {p}")
|
|
passed = sum(not results[c["name"]] for c in cases)
|
|
print(f"passed {passed} of {len(cases)}")
|
|
green = passed == len(cases)
|
|
print(f"verdict: {'GREEN' if green else 'RED'}")
|
|
return green
|
|
|
|
|
|
def run(manifest_path: Path, command: list[str], timeout: float) -> int:
|
|
try:
|
|
manifest = load_manifest(manifest_path)
|
|
cases_root = manifest_path.parent / CASES_DIR
|
|
verify_fixtures(manifest, cases_root)
|
|
with tempfile.TemporaryDirectory(prefix="repair-gate-") as tmp_name:
|
|
tmp = Path(tmp_name)
|
|
base = build_base(manifest, cases_root, tmp)
|
|
results = {case["name"]: run_case(case, base, command, cases_root, tmp, manifest["build_at"], timeout)
|
|
for case in manifest["cases"]}
|
|
except GateError as exc:
|
|
print(f"GATE CANNOT RUN: {exc}")
|
|
return 2
|
|
return 0 if report(manifest["cases"], results, command) else 1
|
|
|
|
|
|
def main(argv: list[str] | None = None) -> int:
|
|
args_in = sys.argv[1:] if argv is None else argv
|
|
if "--" in args_in:
|
|
split = args_in.index("--")
|
|
own, command = args_in[:split], args_in[split + 1:]
|
|
else:
|
|
own, command = args_in, []
|
|
parser = argparse.ArgumentParser(description="Run the repair gate (fail-closed, no model calls).")
|
|
parser.add_argument("--manifest", type=Path, required=True)
|
|
parser.add_argument("--timeout", type=float, default=120.0, help="seconds per command run")
|
|
args = parser.parse_args(own)
|
|
if not command:
|
|
print("usage error: give the command after --, e.g. -- python3 -m okfwiki.krepair")
|
|
return 2
|
|
return run(args.manifest, command, args.timeout)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|