okf-wiki/gates/tests/test_repair_gate.py
Kjell Tore Guttormsen 0fd86a23a3
okf-wiki 1.1.1
First public release, published as one snapshot. What changed between
versions is in CHANGELOG.md.

Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
2026-11-14 21:03:20 +01:00

332 lines
18 KiB
Python

"""Tests for gates/repair_gate.py, the fail-closed judge of the repair command (order 34, contracts K18 and K19).
Order 34 A3: the three stubs in gates/repair/stubs/ (one that does nothing, one that refuses everything, one that gives
every page a new generated.at) each turn the gate RED; a manifest whose fixtures have changed stops the gate (exit 2);
and a missing command, or a missing module, is RED. Beside them, the parts of the gate a command cannot be run against
yet (the repair command does not exist when this is written) are tried with handmade outputs: the base the gate builds,
the keys it computes, the checks of task and linkset, and a refusal that must name its code and write nothing. Every
fixture is synthetic text about beekeeping (CLAUDE.md rule 3).
Run: python3 -m unittest discover -s gates/tests -p 'test_repair_gate.py'
"""
from __future__ import annotations
import io
import json
import shutil
import sys
import tempfile
import unittest
from contextlib import redirect_stdout
from pathlib import Path
from typing import Any
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
import repair_gate as gate # noqa: E402
from okfwiki import audit, claims, kassemble, klink, kpack # noqa: E402
GATES = Path(__file__).resolve().parents[1]
MANIFEST = GATES / "repair.manifest.json"
STUBS = GATES / "repair" / "stubs"
PY = sys.executable
# The kinds of case order 34 A1 asks for, by the names the manifest gives them.
REQUIRED = (
"replace-one-with-one", "replace-one-with-two", "two-sentence-fragment", "merged-fragment", "release-note-fragment",
"context-kept", "context-other", "remove-one", "remove-last-on-page", "remove-last-on-main-page",
"remove-last-from-source-on-main-page", "shared-footnote", "equal-sentences-first-removed", "repair-makes-equal",
"link-anchor-stays", "link-anchor-gone", "unlink", "relink", "linkset-two-candidates", "two-repairs-two-pages",
"refuse-quote-not-verbatim", "refuse-quote-other-source", "refuse-unknown-key", "refuse-stale-bundle",
"refuse-duplicate-answer", "refuse-stale-list", "refuse-relink-missing-target", "refuse-unlink-wrong-at",
"refuse-unlink-wrong-anchor", "identity",
)
def run_gate(*command: str, manifest: Path = MANIFEST) -> tuple[int, str]:
out = io.StringIO()
with redirect_stdout(out):
code = gate.main(["--manifest", str(manifest), "--timeout", "60", "--", *command])
return code, out.getvalue()
def case(name: str) -> dict[str, Any]:
cases: list[dict[str, Any]] = gate.load_manifest(MANIFEST)["cases"]
return next(c for c in cases if c["name"] == name)
class StubsTest(unittest.TestCase):
"""A3: each stub is RED, and no case passes with it."""
def assert_red(self, stub: str) -> str:
code, out = run_gate(PY, str(STUBS / stub))
self.assertEqual(code, 1, out)
self.assertIn("verdict: RED", out)
n = len(gate.load_manifest(MANIFEST)["cases"])
self.assertIn(f"passed 0 of {n}", out)
return out
def test_a_command_that_does_nothing_is_red(self) -> None:
out = self.assert_red("noop.py")
self.assertIn("pages changed: [], the case says ['honey-harvest.md']", out)
def test_a_command_that_refuses_everything_is_red(self) -> None:
out = self.assert_red("refuse_all.py")
self.assertIn("refusal codes ['x'], the case says ['unknown-key']", out)
self.assertIn("1 refusal line(s) name no claim or record", out)
def test_a_command_that_touches_every_page_is_red(self) -> None:
out = self.assert_red("touch_generated.py")
lines = out.splitlines()
at = next(i for i, line in enumerate(lines) if line.startswith(" identity "))
identity = "\n".join(line for line in lines[at + 1:at + 4] if line.startswith(" "))
self.assertIn("pages changed: ['brood.md', 'frames.md', 'hive.md'", identity)
class CannotRunTest(unittest.TestCase):
def setUp(self) -> None:
self._tmp = tempfile.TemporaryDirectory()
self.tmp = Path(self._tmp.name)
shutil.copyfile(MANIFEST, self.tmp / MANIFEST.name)
shutil.copytree(GATES / "repair", self.tmp / "repair", ignore=shutil.ignore_patterns("__pycache__"))
self.manifest = self.tmp / MANIFEST.name
def tearDown(self) -> None:
self._tmp.cleanup()
def test_a_changed_fixture_stops_the_gate(self) -> None:
p = self.tmp / "repair" / "kilder" / "kilde" / "docs-en-hive.md"
p.chmod(0o644)
p.write_text(p.read_text(encoding="utf-8").replace("warm and calm", "cold and windy"), encoding="utf-8")
code, out = run_gate(PY, str(STUBS / "noop.py"), manifest=self.manifest)
self.assertEqual(code, 2, out)
self.assertIn("GATE CANNOT RUN", out)
self.assertIn("sha256 mismatch in kilder/kilde/docs-en-hive.md", out)
def test_an_extra_or_a_missing_fixture_stops_the_gate(self) -> None:
(self.tmp / "repair" / "extra.json").write_text("{}", encoding="utf-8")
(self.tmp / "repair" / "agent.md").unlink()
code, out = run_gate(PY, str(STUBS / "noop.py"), manifest=self.manifest)
self.assertEqual(code, 2, out)
self.assertIn("unexpected file extra.json", out)
self.assertIn("missing file agent.md", out)
def test_a_changed_case_is_no_other_manifest_than_the_one_reported(self) -> None:
doc = json.loads(self.manifest.read_text(encoding="utf-8"))
doc["cases"][0]["steps"][0]["exit"] = 2
self.manifest.write_text(json.dumps(doc), encoding="utf-8")
code, out = run_gate(PY, str(STUBS / "noop.py"), manifest=self.manifest)
self.assertEqual(code, 2, out)
self.assertIn("a step needs argv and exit 0 or 1", out)
def test_a_refusal_code_the_manifest_does_not_define_stops_the_gate(self) -> None:
doc = json.loads(self.manifest.read_text(encoding="utf-8"))
del doc["refusal_codes"]["unknown-key"]
self.manifest.write_text(json.dumps(doc), encoding="utf-8")
code, out = run_gate(PY, str(STUBS / "noop.py"), manifest=self.manifest)
self.assertEqual(code, 2, out)
self.assertIn("refusal codes not in the manifest", out)
def test_no_command_is_a_usage_error(self) -> None:
code, out = run_gate(manifest=self.manifest)
self.assertEqual(code, 2)
self.assertIn("usage error", out)
def test_a_missing_command_is_red(self) -> None:
code, out = run_gate(str(self.tmp / "no-such-command"))
self.assertEqual(code, 1, out)
self.assertIn("command not found", out)
self.assertIn("verdict: RED", out)
def test_a_missing_module_is_red(self) -> None:
code, out = run_gate(PY, "-m", "okfwiki.no_such_module_here")
self.assertEqual(code, 1, out)
self.assertIn("No module named okfwiki.no_such_module_here", out)
self.assertIn("passed 0 of", out)
class BaseTest(unittest.TestCase):
"""The collection the gate builds, and the checks a command cannot be run against yet."""
_tmp: tempfile.TemporaryDirectory[str]
base: gate.Base
manifest: dict[str, Any]
@classmethod
def setUpClass(cls) -> None:
cls._tmp = tempfile.TemporaryDirectory()
cls.manifest = gate.load_manifest(MANIFEST)
gate.verify_fixtures(cls.manifest, GATES / "repair")
cls.base = gate.build_base(cls.manifest, GATES / "repair", Path(cls._tmp.name))
@classmethod
def tearDownClass(cls) -> None:
cls._tmp.cleanup()
def setUp(self) -> None:
self._case_tmp = tempfile.TemporaryDirectory()
self.tmp = Path(self._case_tmp.name)
def tearDown(self) -> None:
self._case_tmp.cleanup()
def test_the_base_is_clean_and_has_the_pages_the_cases_are_written_for(self) -> None:
pages = sorted(gate.concept_pages(self.base.wiki))
self.assertEqual(pages, ["brood.md", "frames.md", "hive.md", "honey-harvest.md", "queen-cells.md", "smoker.md",
"swarming.md", "water.md"])
self.assertEqual(audit.audit(self.base.wiki, self.base.sources), [])
found = claims.extract(self.base.wiki, self.base.sources)
self.assertEqual(claims.criterion_1(found)["total"], (20, 20))
self.assertEqual(len(json.loads(self.base.links.read_text(encoding="utf-8"))["links"]), 5)
def test_every_case_names_pages_and_sentences_of_the_base_or_of_its_own_answers(self) -> None:
pages = set(gate.concept_pages(self.base.wiki))
ids = {c.id for c in self.base.found.claims}
for c in self.manifest["cases"]:
for p in c.get("pages_changed", []) + c.get("pages_gone", []):
self.assertIn(p, pages, c["name"])
for x in c.get("claims_before", []):
self.assertIn(gate.claim_id(x["page"], x["text"], x.get("n", 0)), ids, c["name"])
for x in c.get("links_removed", []):
self.assertIn(gate.link_tuple(x), gate.link_set(json.loads(self.base.links.read_text())["links"]))
def test_the_cases_cover_what_order_34_a1_names(self) -> None:
names = {c["name"] for c in self.manifest["cases"]}
self.assertEqual(set(REQUIRED) - names, set())
def test_a_sentence_key_is_klinks_key_on_the_built_page(self) -> None:
page = (self.base.wiki / "water.md").read_text(encoding="utf-8")
keys = [s.key for s in klink.page_sentences(page)]
self.assertEqual(keys, [gate.sentence_key("Bees need a water source near the hive.", 0),
gate.sentence_key("Bees need a water source near the hive.", 1)])
def test_a_fragment_key_is_the_footnote_on_its_page_and_two_concepts_share_one(self) -> None:
key = {"concept_id": "frames", "file": "docs-en-hive.md", "start": 8, "end": 10,
"quote": "holds wax comb where bees store honey and raise brood", "provenance": "documented"}
label = gate.fragment_label(self.base.sources, key)
for page in ("frames.md", "brood.md"):
self.assertIn(f"[^{label}]", (self.base.wiki / page).read_text(encoding="utf-8"))
records = json.loads((self.base.work / "K-work" / "accepted.json").read_text(encoding="utf-8"))["fragments"]
self.assertEqual(sorted(r["concept_id"] for r in records if kassemble.label(r) == label), ["brood", "frames"])
def test_a_rebuild_without_change_measures_nothing(self) -> None:
c = gate.fresh_copy(self.base, self.tmp, "same")
code, out = gate.quiet(kassemble.main, ["build", "--work", c.places["work"], "--sources", c.places["sources"],
"--out", c.places["wiki"], "--links", c.places["links"], "--replace",
"--at", self.manifest["build_at"]])
self.assertEqual(code, 0, out)
m = gate.measure(c.path("wiki_old"), c.path("wiki"), c.path("sources"))
self.assertEqual((m.changed, sum(m.claims_gone.values()), sum(m.claims_came.values())), ([], 0, 0))
def test_a_list_is_rendered_in_k17s_schema_with_the_hash_of_the_claim_list(self) -> None:
doc = gate.render_list({"claims": [{"page": "frames.md", "text": "Frames can be lifted out one by one."}],
"links": [{"page": "hive.md", "sentence": "Light the smoker before you lift the lid.",
"anchor": "smoker", "target": "smoker.md"}]}, self.base)
self.assertEqual(doc["schema"], "okf-wiki/repair-list/1")
self.assertEqual(doc["claims_sha256"], gate.sha256_file(self.base.claims))
self.assertEqual(set(doc), {"schema", "claims_sha256", "verdicts_sha256", "claims", "pages", "pairs", "links"})
row = doc["claims"][0]
self.assertEqual((row["page"], row["provenance"], row["kind"], row["sources"]),
("frames.md", "documented", "docs", [{"file": "kilde/docs-en-hive.md", "line": 8}]))
self.assertEqual((doc["links"][0]["at"], doc["links"][0]["origin"]), (10, "o33-kandidat"))
stale = gate.render_list({"claims": [], "stale": True}, self.base)
self.assertNotEqual(stale["claims_sha256"], doc["claims_sha256"])
def test_answers_are_keyed_by_concept_and_footnote(self) -> None:
doc = gate.render_answers(case("refuse-unknown-key")["steps"][0]["answers"]
+ case("remove-one")["steps"][0]["answers"], self.base.sources)
self.assertEqual(doc["schema"], "okf-wiki/repair-answers/1")
self.assertEqual(doc["answers"][0], {"concept_id": "hive", "footnote": "s0000000000", "action": "remove"})
self.assertEqual(doc["answers"][1]["concept_id"], "brood")
self.assertIn(f"[^{doc['answers'][1]['footnote']}]", (self.base.wiki / "brood.md").read_text(encoding="utf-8"))
def fake(self, body: str) -> list[str]:
script = self.tmp / f"fake-{len(list(self.tmp.iterdir()))}.py"
script.write_text(body, encoding="utf-8")
return [PY, str(script)]
def run_one(self, name: str, command: list[str]) -> list[str]:
where = Path(tempfile.mkdtemp(dir=self.tmp))
return gate.run_case(case(name), self.base, command, GATES / "repair", where, self.manifest["build_at"], 60)
def test_a_refusal_passes_only_with_its_code_and_nothing_written(self) -> None:
right = self.fake("import sys\nprint('refused: unknown-key hive s0000000000')\nsys.exit(1)\n")
self.assertEqual(self.run_one("refuse-unknown-key", right), [])
wrong_code = self.fake("import sys\nprint('refused: stale-bundle hive s0000000000')\nsys.exit(1)\n")
problems = self.run_one("refuse-unknown-key", wrong_code)
self.assertTrue(any("refusal codes ['stale-bundle']" in p for p in problems))
no_record = self.fake("import sys\nprint('refused: unknown-key')\nsys.exit(1)\n")
self.assertEqual(self.run_one("refuse-unknown-key", no_record),
["step 1: 1 refusal line(s) name no claim or record"])
writes = self.fake("import sys, pathlib\nargs = sys.argv[1:]\n"
"w = pathlib.Path(args[args.index('--work') + 1])\n"
"(w / 'K-work' / 'repairs.json').write_text('{}')\n"
"print('refused: unknown-key hive s0000000000')\nsys.exit(1)\n")
self.assertTrue(any("refused but changed work" in p for p in self.run_one("refuse-unknown-key", writes)))
crash = self.fake("raise RuntimeError('a crash exits 1 too')\n")
self.assertTrue(any("refusal codes []" in p for p in self.run_one("refuse-unknown-key", crash)))
def test_a_link_command_is_judged_by_the_links_file_and_the_built_page(self) -> None:
unlink = self.fake(
"import json, sys\nargs = sys.argv[1:]\np = args[args.index('--links') + 1]\nd = json.load(open(p))\n"
"d['links'] = [r for r in d['links'] if r['anchor'] != 'smoker']\nd['accepted'] = len(d['links'])\n"
"json.dump(d, open(p, 'w'))\n")
self.assertEqual(self.run_one("unlink", unlink), [])
self.assertTrue(self.run_one("relink", unlink))
def task_output(self, out: Path, parts: dict[str, list[str]], answers: dict[str, str]) -> None:
for name, texts in parts.items():
paths = []
for i, t in enumerate(texts, 1):
p = out / f"{name}-part-{i}.md"
p.write_text(t, encoding="utf-8")
paths.append(str(p))
(out / f"{name}.parts.json").write_text(json.dumps({"parts": paths, "answer": answers[name]}),
encoding="utf-8")
def test_task_output_is_judged_by_where_each_fragment_stands_and_what_its_task_holds(self) -> None:
want = case("task")
c = gate.fresh_copy(self.base, self.tmp, "task")
labels = [gate.fragment_label(self.base.sources, t["key"]) for t in want["task"]]
full = ["\n".join([lab] + t["must_contain"]) for lab, t in zip(labels, want["task"])]
self.task_output(c.path("out"), {"t1": [full[0], full[1]], "t2": [full[2]]}, {"t1": "a1.json", "t2": "a2.json"})
self.assertEqual(gate.check_task(want, c, self.base), [])
short = gate.fresh_copy(self.base, self.tmp, "task-short")
cut = full[1].replace("A queen cell looks like a peanut shell.", "")
self.task_output(short.path("out"), {"t1": [full[0], cut], "t2": [full[2]]}, {"t1": "a1.json", "t2": "a2.json"})
self.assertEqual(gate.check_task(want, short, self.base),
[f"the task with {labels[1]} lacks 'A queen cell looks like a peanut shell.'"])
twice = gate.fresh_copy(self.base, self.tmp, "task-twice")
self.task_output(twice.path("out"), {"t1": [full[0], full[1]], "t2": [full[2], labels[0]]},
{"t1": "a.json", "t2": "a.json"})
problems = gate.check_task(want, twice, self.base)
self.assertIn("two tasks share an answer path", problems)
self.assertTrue(any("stands in 2 tasks" in p for p in problems))
big = gate.fresh_copy(self.base, self.tmp, "task-big")
self.task_output(big.path("out"), {"t1": ["\n".join(full) + "x" * kpack.PART_CHARS]}, {"t1": "a.json"})
self.assertTrue(any("over 60000" in p for p in gate.check_task(want, big, self.base)))
def test_linkset_output_is_judged_by_klink_task_and_the_current_targets(self) -> None:
want = case("linkset-two-candidates")
c = gate.fresh_copy(self.base, self.tmp, "linkset")
out = c.path("out")
keys = [(x["page"], gate.sentence_key(x["sentence"], x.get("n", 0))) for x in want["linkset"]]
def group(n: int, page: str, key: str, ids: list[str]) -> None:
(out / f"group-{n}.sentences.json").write_text(json.dumps(
{"schema": klink.SENTENCES_SCHEMA, "pages": {page: [key]}}), encoding="utf-8")
(out / f"group-{n}.catalogue").write_text("\n".join(ids) + "\n", encoding="utf-8")
group(1, *keys[0], ["smoker", "hive"])
group(2, *keys[1], ["hive", "honey-harvest"])
self.assertEqual(gate.check_linkset(want, c), [])
(out / "group-2.catalogue").write_text("honey-harvest\n", encoding="utf-8")
self.assertEqual(gate.check_linkset(want, c),
[f"the catalogue for {keys[1]} lacks its current target hive.md"])
(out / "group-2.sentences.json").unlink()
self.assertTrue(gate.check_linkset(want, c)[0].startswith("sentences:"))
if __name__ == "__main__":
unittest.main()