"""Tests for gates/calibration.py: the reader-calibration set checker, the seeded build with its call layout (K3), the stubs and the verdict gate. Every fixture is synthetic text written here (widgets and garden beds). No corpus text is used (CLAUDE.md rule 3). """ from __future__ import annotations import contextlib import hashlib import io import json import re import sys import tempfile import unittest from collections import Counter from pathlib import Path from typing import Any sys.path.insert(0, str(Path(__file__).resolve().parents[1])) import calibration # noqa: E402 REPO_ROOT = Path(__file__).resolve().parents[2] SOURCES = { "alpha.md": "# Alpha\n\nThe widget timeout is 45 seconds by default.\nRaising it above 90 seconds disables retries.\n", "beta.md": "# Beta\n\nBeta adds the --frobnicate flag for batch runs.\nThe flag is ignored on interactive runs.\n", "gamma.md": "# Gamma\n\nGamma stores its cache under the .gamma-cache directory.\nThe cache is cleared on every upgrade.\n", "north.md": "# North beds\n\n" + "".join(f"Bed N{i} gets {i + 2} litres of water every morning.\n" for i in range(1, 31)), "south.md": "# South beds\n\n" + "".join(f"Bed S{i} is weeded every {i + 3} days in summer.\n" for i in range(1, 31)), } ALPHA = "The widget timeout is 45 seconds by default." ALPHA2 = "Raising it above 90 seconds disables retries." BETA = "Beta adds the --frobnicate flag for batch runs." BETA2 = "The flag is ignored on interactive runs." GAMMA = "Gamma stores its cache under the .gamma-cache directory." GAMMA2 = "The cache is cleared on every upgrade." UNSUPPORTED, SUPPORTED, FILLER = "unsupported", "supported", "filler-supported" CALL_ID = re.compile(r"\bk\d{3,}\b") CLAIM_ID = re.compile(r"\bc\d{3,}\b") def ex(name: str, text: str) -> dict[str, Any]: return {"file": name, "text": text} def draft_pairs() -> list[dict[str, Any]]: """Six unsupported pairs (one per kind) and six supported pairs (two per kind): a valid small draft.""" u = [ ("overgeneralisation", "Every Alpha setting defaults to 45 seconds.", [ex("alpha.md", ALPHA)]), ("wrong-number", "The widget timeout is 30 seconds by default.", [ex("alpha.md", ALPHA)]), ("swapped-cause", "Retries being disabled is what raises the timeout above 90 seconds.", [ex("alpha.md", ALPHA2)]), ("not-mentioned", "Beta writes a summary file after each batch run.", [ex("beta.md", BETA)]), ("reversed-polarity", "The --frobnicate flag also applies to interactive runs.", [ex("beta.md", BETA2)]), ( "conflated-sources", "Beta stores its cache under the .gamma-cache directory.", [ex("beta.md", BETA), ex("gamma.md", GAMMA)], ), ] s = [ ("paraphrase", "By default, the widget waits 45 seconds before timing out.", [ex("alpha.md", ALPHA)]), ("paraphrase", "Upgrading clears the Gamma cache.", [ex("gamma.md", GAMMA2)]), ("inference", "A widget left at its default timeout still retries.", [ex("alpha.md", ALPHA + "\n" + ALPHA2)]), ("inference", "Batch runs are the ones where --frobnicate has an effect.", [ex("beta.md", BETA + "\n" + BETA2)]), ( "synthesis", "An upgrade empties the directory Gamma uses for its cache.", [ex("gamma.md", GAMMA), ex("gamma.md", GAMMA2)], ), ( "synthesis", "Both a 45-second default timeout and a --frobnicate flag for batch runs exist.", [ex("alpha.md", ALPHA), ex("beta.md", BETA)], ), ] pairs: list[dict[str, Any]] = [] for n, (kind, claim, excerpts) in enumerate(u, 1): pairs.append({"id": f"u{n:02d}", "label": UNSUPPORTED, "kind": kind, "claim": claim, "excerpts": excerpts, "why": "planted"}) for n, (kind, claim, excerpts) in enumerate(s, 1): pairs.append({"id": f"s{n:02d}", "label": SUPPORTED, "kind": kind, "claim": claim, "excerpts": excerpts, "why": "follows"}) return pairs def north(i: int) -> str: return f"Bed N{i} gets {i + 2} litres of water every morning." def south(i: int) -> str: return f"Bed S{i} is weeded every {i + 3} days in summer." def garden_draft(n_unsupported: int = 20, n_supported: int = 6, extra_fillers: int = 4) -> list[dict[str, Any]]: """More unsupported than supported pairs, each unsupported with its supported twin (filler-supported), plus a few fillers without a twin. Every kind of both counted labels is present.""" pairs: list[dict[str, Any]] = [] for i in range(1, n_unsupported + 1): kind = calibration.UNSUPPORTED_KINDS[(i - 1) % len(calibration.UNSUPPORTED_KINDS)] excerpts = [ex("north.md", north(i))] claim = { "overgeneralisation": f"Every north bed, N{i} included, gets {i + 2} litres of water each morning.", "wrong-number": f"Bed N{i} gets {i + 50} litres of water each morning.", "swapped-cause": f"Watering bed N{i} each morning is what makes it need {i + 2} litres.", "not-mentioned": f"Bed N{i} gets {i + 2} litres of water each morning and is mulched in spring.", "reversed-polarity": f"Bed N{i} never gets {i + 2} litres of water in the morning.", "conflated-sources": f"Bed N{i} is weeded every {i + 3} days in summer.", }[kind] twin, twin_kind = f"Each morning, bed N{i} receives {i + 2} litres of water.", "paraphrase" if kind == "conflated-sources": excerpts = [ex("north.md", north(i)), ex("south.md", south(i))] twin = f"Bed N{i} gets {i + 2} litres each morning, and bed S{i} is weeded every {i + 3} days in summer." twin_kind = "synthesis" pairs.append({"id": f"u{i:02d}", "label": UNSUPPORTED, "kind": kind, "claim": claim, "excerpts": excerpts, "why": "planted"}) pairs.append({"id": f"t{i:02d}", "label": FILLER, "kind": twin_kind, "twin_of": f"u{i:02d}", "claim": twin, "excerpts": [dict(e) for e in excerpts], "why": "the supported reading"}) for n in range(1, n_supported + 1): j = 20 + n kind = calibration.SUPPORTED_KINDS[(n - 1) % len(calibration.SUPPORTED_KINDS)] claim, excerpts = { "paraphrase": (f"In summer, bed S{j} has its weeds pulled every {j + 3} days.", [ex("south.md", south(j))]), "inference": (f"Bed S{j} is weeded more often than once a season in summer.", [ex("south.md", south(j))]), "synthesis": ( f"Bed N{j} is watered each morning and bed S{j} is weeded every {j + 3} days in summer.", [ex("north.md", north(j)), ex("south.md", south(j))], ), }[kind] pairs.append({"id": f"s{n:02d}", "label": SUPPORTED, "kind": kind, "claim": claim, "excerpts": excerpts, "why": "follows"}) for n in range(1, extra_fillers + 1): j = 26 + n pairs.append({"id": f"f{n:02d}", "label": FILLER, "kind": "paraphrase", "claim": f"Each morning, bed N{j} receives {j + 2} litres of water.", "excerpts": [ex("north.md", north(j))], "why": "follows"}) return pairs def sha256_file(path: Path) -> str: return hashlib.sha256(path.read_bytes()).hexdigest() class Workspace: """A temporary sources directory and helpers to write drafts, verdicts and build outputs.""" def __init__(self) -> None: self._tmp = tempfile.TemporaryDirectory() self.root = Path(self._tmp.name) self.sources = self.root / "sources" self.sources.mkdir() for name, text in SOURCES.items(): (self.sources / name).write_text(text, encoding="utf-8") def close(self) -> None: self._tmp.cleanup() def write_json(self, name: str, data: Any) -> Path: path = self.root / name path.write_text(json.dumps(data), encoding="utf-8") return path def write_draft(self, pairs: list[dict[str, Any]], name: str = "draft.json") -> Path: return self.write_json(name, {"schema": calibration.DRAFT_SCHEMA, "pairs": pairs}) def paths(self) -> dict[str, Path]: return { "set": self.root / "out" / "pairs.json", "key": self.root / "out" / "key.json", "calls": self.root / "out" / "calls.json", "manifest": self.root / "calibration.manifest.json", } def build( self, pairs: list[dict[str, Any]] | None = None, seed: int = 7, lone: int = 10, dense: int = 5 ) -> tuple[int, str]: draft = self.write_draft(garden_draft() if pairs is None else pairs) p = self.paths() return run( "build", "--draft", str(draft), "--sources", str(self.sources), "--seed", str(seed), "--lone", str(lone), "--dense", str(dense), "--set-out", str(p["set"]), "--key-out", str(p["key"]), "--calls-out", str(p["calls"]), "--manifest", str(p["manifest"]), ) def frozen_args(self) -> list[str]: p = self.paths() return ["--set", str(p["set"]), "--key", str(p["key"]), "--calls", str(p["calls"]), "--manifest", str(p["manifest"]), "--sources", str(self.sources)] def gate(self, verdicts: Path | None) -> tuple[int, str]: return run("gate", *self.frozen_args(), "--verdicts", str(verdicts or self.root / "no-such-verdicts.json")) def check(self) -> tuple[int, str]: return run("check", *self.frozen_args()) def load(self, which: str) -> Any: return json.loads(self.paths()[which].read_text(encoding="utf-8")) def labels(self) -> dict[str, str]: return {k["id"]: k["label"] for k in self.load("key")["labels"]} def positions(self) -> dict[str, int]: return {i: n for c in self.load("calls")["calls"] for n, i in enumerate(c["ids"], 1)} def rewrite_calls(self, layout: dict[str, Any]) -> None: """Write a changed layout and make the manifest's sha256 follow it, so only the layout rule under test fails.""" p = self.paths() p["calls"].write_text(json.dumps(layout, indent=1, sort_keys=True) + "\n", encoding="utf-8") manifest = self.load("manifest") manifest["calls"]["sha256"] = sha256_file(p["calls"]) p["manifest"].write_text(json.dumps(manifest, indent=1) + "\n", encoding="utf-8") def run(*argv: str) -> tuple[int, str]: out, err = io.StringIO(), io.StringIO() with contextlib.redirect_stdout(out), contextlib.redirect_stderr(err): try: code = calibration.main(list(argv)) except SystemExit as exc: # argparse refuses an unknown option code = exc.code if isinstance(exc.code, int) else 2 return code, out.getvalue() + err.getvalue() class Base(unittest.TestCase): def setUp(self) -> None: self.ws = Workspace() def tearDown(self) -> None: self.ws.close() class CheckDraftTest(Base): def check(self, pairs: list[dict[str, Any]]) -> calibration.CheckReport: return calibration.check_pairs(pairs, self.ws.sources) def test_valid_draft_passes_and_counts_kinds(self) -> None: rep = self.check(draft_pairs()) self.assertTrue(rep.passed, rep.lines()) self.assertEqual(rep.unsupported, 6) self.assertEqual(rep.supported, 6) self.assertEqual(rep.excerpts_verbatim, rep.excerpts) self.assertEqual(rep.by_kind["wrong-number"], 1) self.assertEqual(rep.by_kind["synthesis"], 2) def test_excerpt_not_verbatim_fails(self) -> None: pairs = draft_pairs() pairs[0]["excerpts"] = [ex("alpha.md", "The widget timeout is 46 seconds by default.")] rep = self.check(pairs) self.assertFalse(rep.passed) self.assertEqual(rep.excerpts_verbatim, rep.excerpts - 1) def test_missing_source_file_fails(self) -> None: pairs = draft_pairs() pairs[0]["excerpts"] = [ex("nowhere.md", ALPHA)] self.assertFalse(self.check(pairs).passed) def test_path_outside_sources_fails(self) -> None: pairs = draft_pairs() pairs[0]["excerpts"] = [ex("../alpha.md", ALPHA)] self.assertFalse(self.check(pairs).passed) def test_unbalanced_labels_pass_the_draft_check(self) -> None: # The counts are the manifest's business now (order 3, point 1), not a balance rule. rep = self.check(garden_draft()) self.assertTrue(rep.passed, rep.lines()) self.assertEqual((rep.unsupported, rep.supported, rep.filler), (20, 6, 24)) def test_every_unsupported_kind_must_be_present(self) -> None: pairs = draft_pairs() pairs[1]["kind"] = "overgeneralisation" pairs[1]["claim"] = "Every widget setting is 45 seconds long." rep = self.check(pairs) self.assertFalse(rep.passed) self.assertTrue(any("wrong-number" in e for e in rep.errors), rep.errors) def test_every_supported_kind_must_be_present(self) -> None: pairs = draft_pairs() for p in pairs: if p["kind"] == "inference": p["kind"] = "paraphrase" rep = self.check(pairs) self.assertFalse(rep.passed) self.assertTrue(any("inference" in e for e in rep.errors), rep.errors) def test_unknown_kind_or_label_fails(self) -> None: pairs = draft_pairs() pairs[0]["kind"] = "vibes" self.assertFalse(self.check(pairs).passed) pairs = draft_pairs() pairs[0]["label"] = "maybe" self.assertFalse(self.check(pairs).passed) def test_kind_must_match_label(self) -> None: pairs = draft_pairs() pairs[0]["kind"] = "paraphrase" self.assertFalse(self.check(pairs).passed) def test_filler_takes_a_supported_kind(self) -> None: pairs = garden_draft() twin = next(p for p in pairs if p["label"] == FILLER) twin["kind"] = "wrong-number" self.assertFalse(self.check(pairs).passed) def test_filler_may_not_be_a_verbatim_copy(self) -> None: pairs = garden_draft() twin = next(p for p in pairs if p["label"] == FILLER and p["kind"] == "paraphrase") twin["claim"] = twin["excerpts"][0]["text"] rep = self.check(pairs) self.assertFalse(rep.passed) self.assertTrue(any(twin["id"] in e and "verbatim" in e for e in rep.errors), rep.errors) def test_twin_of_must_name_an_unsupported_pair(self) -> None: for target in ("s01", "nowhere"): pairs = garden_draft() next(p for p in pairs if p["label"] == FILLER)["twin_of"] = target rep = self.check(pairs) self.assertFalse(rep.passed, target) self.assertTrue(any("twin_of" in e for e in rep.errors), rep.errors) def test_wrong_number_claim_needs_a_number_absent_from_the_excerpts(self) -> None: pairs = draft_pairs() pairs[1]["claim"] = "The widget timeout is 45 seconds, which is short." rep = self.check(pairs) self.assertFalse(rep.passed) self.assertTrue(any("u02" in e and "number" in e for e in rep.errors), rep.errors) def test_conflated_sources_needs_two_files(self) -> None: pairs = draft_pairs() pairs[5]["excerpts"] = [ex("beta.md", BETA), ex("beta.md", BETA2)] self.assertFalse(self.check(pairs).passed) def test_synthesis_needs_two_excerpts(self) -> None: pairs = draft_pairs() pairs[10]["excerpts"] = [ex("gamma.md", GAMMA)] self.assertFalse(self.check(pairs).passed) def test_supported_claim_may_not_be_a_verbatim_copy(self) -> None: pairs = draft_pairs() pairs[6]["claim"] = "the widget timeout is 45 seconds by default" rep = self.check(pairs) self.assertFalse(rep.passed) self.assertTrue(any("s01" in e and "verbatim" in e for e in rep.errors), rep.errors) def test_duplicate_ids_fail(self) -> None: pairs = draft_pairs() pairs[1]["id"] = "u01" self.assertFalse(self.check(pairs).passed) def test_too_many_excerpts_fail(self) -> None: pairs = draft_pairs() pairs[0]["excerpts"] = [ex("alpha.md", ALPHA), ex("beta.md", BETA), ex("gamma.md", GAMMA)] self.assertFalse(self.check(pairs).passed) def test_empty_draft_fails(self) -> None: self.assertFalse(self.check([]).passed) def test_check_command_on_draft(self) -> None: draft = self.ws.write_draft(draft_pairs()) code, out = run("check", "--draft", str(draft), "--sources", str(self.ws.sources)) self.assertEqual(code, 0, out) self.assertIn("excerpts verbatim: 15 of 15", out) self.assertIn("PASS", out) def test_check_command_missing_sources_is_a_fail(self) -> None: draft = self.ws.write_draft(draft_pairs()) code, out = run("check", "--draft", str(draft), "--sources", str(self.ws.root / "absent")) self.assertEqual(code, 1) self.assertIn("FAIL", out) class BuildTest(Base): def built(self, **kwargs: Any) -> None: code, out = self.ws.build(**kwargs) self.assertEqual(code, 0, out) def test_build_writes_set_key_layout_and_manifest(self) -> None: self.built() pset, key, layout, manifest = (self.ws.load(w) for w in ("set", "key", "calls", "manifest")) p = self.ws.paths() self.assertEqual(pset["schema"], calibration.SET_SCHEMA) ids = [q["id"] for q in pset["pairs"]] self.assertEqual(ids, [f"c{n:03d}" for n in range(1, 121)]) for q in pset["pairs"]: self.assertEqual(set(q), {"id", "claim", "excerpts"}) for e in q["excerpts"]: self.assertEqual(set(e), {"file", "line", "text"}) self.assertNotIn("unsupported", json.dumps(pset)) self.assertNotIn("filler", json.dumps(pset)) self.assertEqual(sorted(k["id"] for k in key["labels"]), ids) self.assertEqual(layout["schema"], "okf-wiki/reader-calls/1") self.assertEqual(layout["call_size_max"], 10) self.assertEqual(layout["set_sha256"], sha256_file(p["set"])) self.assertEqual(len(layout["calls"]), 12) self.assertEqual([c["call"] for c in layout["calls"]], [f"k{n:03d}" for n in range(1, 13)]) self.assertEqual(manifest["seed"], 7) self.assertEqual(manifest["set"]["sha256"], sha256_file(p["set"])) self.assertEqual(manifest["key"]["sha256"], sha256_file(p["key"])) self.assertEqual(manifest["calls"]["sha256"], sha256_file(p["calls"])) self.assertEqual(manifest["calls"]["calls"], 12) counts = manifest["counts"] self.assertEqual((counts["unsupported"], counts["supported"], counts["filler-supported"]), (20, 6, 94)) self.assertEqual(counts["filler_distinct"], 24) self.assertEqual(sum(v["unsupported"] for v in counts["by_position"].values()), 20) self.assertEqual(manifest["thresholds"], {"caught_min": 20, "accepted_min": 6}) self.assertEqual(manifest["sources"]["files"], len(SOURCES)) self.assertEqual(manifest["history"], []) self.assertNotIn("claim", json.dumps(manifest)) def test_every_claim_stands_in_exactly_one_call_of_at_most_ten(self) -> None: self.built() placed = [i for c in self.ws.load("calls")["calls"] for i in c["ids"]] self.assertEqual(sorted(placed), sorted(self.ws.labels())) self.assertEqual(len(placed), len(set(placed))) self.assertTrue(all(len(c["ids"]) <= 10 for c in self.ws.load("calls")["calls"])) def test_lone_unsupported_stand_at_every_position(self) -> None: self.built() labels, pos = self.ws.labels(), self.ws.positions() alone = [] for c in self.ws.load("calls")["calls"]: u = [i for i in c["ids"] if labels[i] == UNSUPPORTED] if len(u) == 1: alone.append(pos[u[0]]) self.assertGreaterEqual(len(alone), 10) self.assertEqual(set(alone), set(range(1, 11))) self.assertEqual(self.ws.load("manifest")["calls"]["unsupported_alone"], len(alone)) def test_each_counted_supported_pair_stands_once(self) -> None: self.built() drafts = Counter(k["draft_id"] for k in self.ws.load("key")["labels"] if k["label"] == SUPPORTED) self.assertEqual(drafts, Counter({f"s{n:02d}": 1 for n in range(1, 7)})) def test_a_twin_never_shares_a_call_with_its_unsupported_pair_and_no_filler_stands_twice_in_a_call(self) -> None: self.built() draft_of = {k["id"]: k["draft_id"] for k in self.ws.load("key")["labels"]} for c in self.ws.load("calls")["calls"]: members = [draft_of[i] for i in c["ids"]] self.assertEqual(len(members), len(set(members)), c["call"]) for m in members: if m.startswith("t"): self.assertNotIn("u" + m[1:], members, c["call"]) def test_same_seed_gives_the_same_bytes_and_another_seed_another_layout(self) -> None: self.built(seed=7) first = {w: self.ws.paths()[w].read_bytes() for w in ("set", "key", "calls")} other = Workspace() try: self.assertEqual(other.build(seed=7)[0], 0) self.assertEqual(first, {w: other.paths()[w].read_bytes() for w in ("set", "key", "calls")}) finally: other.close() other = Workspace() try: self.assertEqual(other.build(seed=8)[0], 0) self.assertNotEqual(first["calls"], other.paths()["calls"].read_bytes()) self.assertNotEqual(first["set"], other.paths()["set"].read_bytes()) finally: other.close() def test_order_does_not_follow_the_labels(self) -> None: self.built() labels = self.ws.labels() order = [labels[q["id"]] for q in self.ws.load("set")["pairs"]] self.assertNotEqual(order, sorted(order)) self.assertNotEqual(order, sorted(order, reverse=True)) def test_build_moves_the_old_manifest_into_history(self) -> None: old = {"schema": "okf-wiki/calibration-manifest/1", "seed": 1, "history": [{"schema": "older"}]} self.ws.paths()["manifest"].write_text(json.dumps(old), encoding="utf-8") self.built() history = self.ws.load("manifest")["history"] self.assertEqual(history, [{"schema": "older"}, {"schema": "okf-wiki/calibration-manifest/1", "seed": 1}]) def test_build_refuses_a_failing_draft(self) -> None: pairs = garden_draft() pairs[0]["excerpts"] = [ex("north.md", "not in the file at all, not anywhere")] code, out = self.ws.build(pairs) self.assertEqual(code, 1) self.assertFalse(self.ws.paths()["set"].exists()) def test_build_refuses_when_a_position_would_have_no_unsupported_claim(self) -> None: code, out = self.ws.build(garden_draft(n_unsupported=9), lone=9) self.assertEqual(code, 1, out) self.assertIn("position 10", out) self.assertFalse(any(self.ws.paths()[w].exists() for w in ("set", "key", "calls", "manifest"))) def test_build_refuses_slots_it_cannot_fill(self) -> None: pairs = [p for p in garden_draft() if p["label"] != FILLER] code, out = self.ws.build(pairs) self.assertEqual(code, 1, out) self.assertIn("filler", out) self.assertFalse(self.ws.paths()["calls"].exists()) def test_build_refuses_impossible_parameters(self) -> None: for lone, dense in ((21, 5), (10, 11), (10, 0), (-1, 5)): code, out = self.ws.build(lone=lone, dense=dense) self.assertEqual(code, 1, (lone, dense, out)) self.assertFalse(self.ws.paths()["set"].exists()) def test_build_never_overwrites_a_frozen_set(self) -> None: self.built() code, out = self.ws.build() self.assertEqual(code, 1) self.assertIn("frozen", out) self.ws.paths()["set"].unlink() self.ws.paths()["key"].unlink() code, out = self.ws.build() self.assertEqual(code, 1) self.assertIn("frozen", out) class GateBase(Base): """A built set, its key, and helpers to write verdicts and stubs for the gate.""" def setUp(self) -> None: super().setUp() code, out = self.ws.build() self.assertEqual(code, 0, out) self.key = self.ws.labels() def verdicts(self, mapping: dict[str, str], reader: str = "test", models: Any = ("claude-test-1",), name: str = "verdicts.json") -> Path: doc: dict[str, Any] = {"reader": reader, "verdicts": [{"id": i, "verdict": v} for i, v in mapping.items()]} if models is not None: doc["models"] = list(models) if isinstance(models, tuple) else models return self.ws.write_json(name, doc) def perfect(self) -> dict[str, str]: return {i: (UNSUPPORTED if lab == UNSUPPORTED else SUPPORTED) for i, lab in self.key.items()} def stub(self, *args: str) -> Path: path = self.ws.root / "stub.json" code, out = run("stub", "--set", str(self.ws.paths()["set"]), "--out", str(path), *args) self.assertEqual(code, 0, out) return path def per_position(self) -> dict[int, int]: pos = self.ws.positions() return dict(Counter(pos[i] for i, lab in self.key.items() if lab == UNSUPPORTED)) class GateTest(GateBase): def test_no_reader_scores_zero_and_is_red(self) -> None: code, out = self.ws.gate(None) self.assertEqual(code, 1) self.assertIn("unsupported caught 0 of 20 · supported accepted 0 of 6", out) self.assertIn("fail-closed", out) self.assertIn("RED", out) def test_the_key_as_judge_is_green(self) -> None: code, out = self.ws.gate(self.stub("--key", str(self.ws.paths()["key"]))) self.assertEqual(code, 0, out) self.assertIn("unsupported caught 20 of 20 · supported accepted 6 of 6", out) self.assertIn("filler accepted 94 of 94", out) self.assertIn("GREEN", out) def test_reject_all_stub_is_red(self) -> None: code, out = self.ws.gate(self.stub("--verdict", UNSUPPORTED)) self.assertEqual(code, 1) self.assertIn("unsupported caught 20 of 20 · supported accepted 0 of 6", out) self.assertIn("RED", out) def test_accept_all_stub_is_red(self) -> None: code, out = self.ws.gate(self.stub("--verdict", SUPPORTED)) self.assertEqual(code, 1) self.assertIn("unsupported caught 0 of 20 · supported accepted 6 of 6", out) self.assertIn("RED", out) def test_a_stub_mild_at_one_position_is_red_and_the_position_is_named(self) -> None: stub = self.stub("--key", str(self.ws.paths()["key"]), "--calls", str(self.ws.paths()["calls"]), "--accept-at", "7") code, out = self.ws.gate(stub) self.assertEqual(code, 1, out) n7 = self.per_position()[7] self.assertIn(f"7 0 of {n7}", out) self.assertIn(f"unsupported caught {20 - n7} of 20", out) not_met = [line for line in out.splitlines() if line.startswith("not met:")] self.assertEqual(len(not_met), 1, out) self.assertIn("position 7", not_met[0]) self.assertNotIn("position 6", not_met[0]) self.assertIn("RED", out) def test_caught_per_position_is_written_and_no_line_per_call_or_pair(self) -> None: code, out = self.ws.gate(self.verdicts(self.perfect())) self.assertEqual(code, 0, out) counts = self.per_position() line = next(line for line in out.splitlines() if line.startswith("caught by position: ")) self.assertEqual(line, "caught by position: " + ", ".join(f"{p} {counts[p]} of {counts[p]}" for p in range(1, 11))) self.assertIsNone(CALL_ID.search(out), out) self.assertIsNone(CLAIM_ID.search(out), out) def test_the_gate_writes_no_per_pair_results(self) -> None: code, out = run("gate", *self.ws.frozen_args(), "--verdicts", str(self.verdicts(self.perfect())), "--out", str(self.ws.root / "results.json")) self.assertNotEqual(code, 0) self.assertFalse((self.ws.root / "results.json").exists()) def test_reports_caught_per_kind(self) -> None: code, out = self.ws.gate(self.verdicts(self.perfect())) self.assertIn("caught by kind: overgeneralisation 4 of 4", out) self.assertIn("conflated-sources 3 of 3", out) def test_filler_never_counts_in_the_thresholds(self) -> None: v = self.perfect() for i, lab in self.key.items(): if lab == FILLER: v[i] = UNSUPPORTED code, out = self.ws.gate(self.verdicts(v)) self.assertEqual(code, 0, out) self.assertIn("filler accepted 0 of 94", out) self.assertIn("GREEN", out) def test_a_missing_filler_verdict_is_red(self) -> None: v = self.perfect() del v[next(i for i, lab in self.key.items() if lab == FILLER)] code, out = self.ws.gate(self.verdicts(v)) self.assertEqual(code, 1, out) self.assertIn("missing verdicts: 1", out) def test_one_missed_unsupported_is_red(self) -> None: v = self.perfect() v[next(i for i, lab in self.key.items() if lab == UNSUPPORTED)] = SUPPORTED code, out = self.ws.gate(self.verdicts(v)) self.assertEqual(code, 1) self.assertIn("unsupported caught 19 of 20", out) def test_missing_verdict_counts_as_wrong(self) -> None: v = self.perfect() gone_u = next(i for i, lab in self.key.items() if lab == UNSUPPORTED) gone_s = next(i for i, lab in self.key.items() if lab == SUPPORTED) del v[gone_u], v[gone_s] code, out = self.ws.gate(self.verdicts(v)) self.assertEqual(code, 1) self.assertIn("missing verdicts: 2", out) self.assertIn("unsupported caught 19 of 20 · supported accepted 5 of 6", out) def test_invalid_or_duplicate_verdict_counts_as_missing(self) -> None: v = self.perfect() first = next(iter(v)) path = self.ws.write_json( "verdicts.json", { "reader": "test", "verdicts": [{"id": i, "verdict": val} for i, val in v.items()] + [{"id": first, "verdict": SUPPORTED}, {"id": "c999", "verdict": SUPPORTED}], }, ) code, out = self.ws.gate(path) self.assertEqual(code, 1) self.assertIn("missing verdicts: 1", out) for bad in ("probably", FILLER): code, out = self.ws.gate(self.verdicts({**v, first: bad})) self.assertEqual(code, 1, bad) self.assertIn("missing verdicts: 1", out) def test_accepted_threshold_is_read_from_the_manifest(self) -> None: manifest = self.ws.load("manifest") manifest["thresholds"]["accepted_min"] = 5 self.ws.paths()["manifest"].write_text(json.dumps(manifest), encoding="utf-8") v = self.perfect() v[next(i for i, lab in self.key.items() if lab == SUPPORTED)] = UNSUPPORTED code, out = self.ws.gate(self.verdicts(v)) self.assertEqual(code, 0, out) self.assertIn("supported accepted 5 of 6", out) def test_gate_refuses_a_changed_set_key_or_layout(self) -> None: for which in ("set", "key", "calls"): path = self.ws.paths()[which] before = path.read_text(encoding="utf-8") path.write_text(before + " ", encoding="utf-8") code, out = self.ws.gate(self.verdicts(self.perfect())) self.assertEqual(code, 1, which) self.assertIn("manifest", out) path.write_text(before, encoding="utf-8") def test_gate_refuses_other_sources(self) -> None: (self.ws.sources / "alpha.md").write_text("changed\n", encoding="utf-8") code, out = self.ws.gate(self.verdicts(self.perfect())) self.assertEqual(code, 1) self.assertIn("tree_sha256", out) class GateModelTest(GateBase): """PLAN § Rammer 23: the gate is bound to the model the reader ran on. It names the model it scored, is GREEN only for verdicts of exactly one model, and records a GREEN run in the file --passed names; collect reads that file.""" def passed(self) -> Path: return self.ws.root / "passed.json" def gate_passed(self, verdicts: Path) -> tuple[int, str]: return run("gate", *self.ws.frozen_args(), "--verdicts", str(verdicts), "--passed", str(self.passed())) def test_the_gate_names_the_model_it_scored(self) -> None: code, out = self.ws.gate(self.verdicts(self.perfect(), models=["claude-test-2"])) self.assertEqual(code, 0, out) self.assertIn("model: claude-test-2\n", out) self.assertIn("GREEN", out) def test_verdicts_that_name_no_model_are_red(self) -> None: for models in (None, [], "claude-test-2", [3]): with self.subTest(models=models): code, out = self.ws.gate(self.verdicts(self.perfect(), models=models)) self.assertEqual(code, 1, out) self.assertIn("model: (none)\n", out) self.assertIn("the verdicts name no model", out) self.assertNotIn("GREEN", out) self.assertIn("RED", out) def test_verdicts_of_two_models_are_red(self) -> None: code, out = self.ws.gate(self.verdicts(self.perfect(), models=["claude-test-1", "claude-test-2"])) self.assertEqual(code, 1, out) self.assertIn("model: claude-test-1, claude-test-2\n", out) self.assertIn("the verdicts name 2 models", out) self.assertIn("RED", out) def test_a_stub_names_the_model_stub_and_its_meaning_is_kept(self) -> None: stub = self.stub("--key", str(self.ws.paths()["key"])) self.assertEqual(json.loads(stub.read_text(encoding="utf-8"))["models"], ["stub"]) code, out = self.ws.gate(stub) self.assertEqual(code, 0, out) self.assertIn("model: stub\n", out) code, out = self.ws.gate(self.stub("--verdict", SUPPORTED)) self.assertEqual(code, 1, out) self.assertIn("model: stub\n", out) def test_a_green_run_is_recorded_with_its_hashes(self) -> None: verdicts = self.verdicts(self.perfect(), reader="r7", models=["claude-test-2"]) code, out = self.gate_passed(verdicts) self.assertEqual(code, 0, out) p = self.ws.paths() record = json.loads(self.passed().read_text(encoding="utf-8")) self.assertEqual(record, {"schema": "okf-wiki/calibration-passed/1", "passed": [{ "model": "claude-test-2", "set_sha256": sha256_file(p["set"]), "key_sha256": sha256_file(p["key"]), "verdicts_sha256": sha256_file(verdicts), "reader": "r7"}]}) self.assertIn(str(self.passed()), out) def test_the_same_green_run_is_recorded_once_and_the_file_has_no_time(self) -> None: verdicts = self.verdicts(self.perfect(), models=["claude-test-2"]) self.assertEqual(self.gate_passed(verdicts)[0], 0) first = self.passed().read_bytes() self.assertEqual(self.gate_passed(verdicts)[0], 0) self.assertEqual(self.passed().read_bytes(), first) self.assertEqual(len(json.loads(first)["passed"]), 1) def test_another_green_run_is_added_after_the_first(self) -> None: self.assertEqual(self.gate_passed(self.verdicts(self.perfect(), models=["claude-test-2"]))[0], 0) self.assertEqual(self.gate_passed(self.verdicts(self.perfect(), models=["claude-test-3"], name="v3.json"))[0], 0) record = json.loads(self.passed().read_text(encoding="utf-8")) self.assertEqual([e["model"] for e in record["passed"]], ["claude-test-2", "claude-test-3"]) def test_a_red_run_never_writes_the_record(self) -> None: v = self.perfect() v[next(i for i, lab in self.key.items() if lab == UNSUPPORTED)] = SUPPORTED code, out = self.gate_passed(self.verdicts(v)) self.assertEqual(code, 1, out) self.assertFalse(self.passed().exists()) code, out = self.gate_passed(self.verdicts(self.perfect(), models=None)) self.assertEqual(code, 1, out) self.assertFalse(self.passed().exists()) self.assertEqual(self.gate_passed(self.verdicts(self.perfect()))[0], 0) before = self.passed().read_bytes() self.assertEqual(self.gate_passed(self.verdicts(v, models=["claude-test-9"]))[0], 1) self.assertEqual(self.passed().read_bytes(), before) def test_a_record_that_cannot_be_read_is_a_fail_and_is_left_as_it_was(self) -> None: for bad in ("{", json.dumps({"schema": "okf-wiki/other/1", "passed": []}), json.dumps({"schema": "okf-wiki/calibration-passed/1", "passed": {}})): with self.subTest(bad=bad): self.passed().write_text(bad, encoding="utf-8") code, out = self.gate_passed(self.verdicts(self.perfect())) self.assertEqual(code, 1, out) self.assertIn("FAIL", out) self.assertEqual(self.passed().read_text(encoding="utf-8"), bad) class LayoutRuleTest(Base): """A layout that breaks K3 point 1 is refused by the gate and by check, even when the manifest follows it.""" def setUp(self) -> None: super().setUp() code, out = self.ws.build() self.assertEqual(code, 0, out) self.layout = self.ws.load("calls") self.key = self.ws.labels() self.green = self.ws.write_json( "verdicts.json", {"reader": "key", "models": ["claude-test-1"], "verdicts": [{"id": i, "verdict": UNSUPPORTED if lab == UNSUPPORTED else SUPPORTED} for i, lab in self.key.items()]}, ) def refused(self, needle: str) -> None: self.ws.rewrite_calls(self.layout) code, out = self.ws.gate(self.green) self.assertEqual(code, 1, out) self.assertIn(needle, out) self.assertNotIn("GREEN", out) code, out = self.ws.check() self.assertEqual(code, 1, out) self.assertIn(needle, out) def test_the_unchanged_layout_passes(self) -> None: self.ws.rewrite_calls(self.layout) self.assertEqual(self.ws.gate(self.green)[0], 0) self.assertEqual(self.ws.check()[0], 0) def test_an_id_given_twice_is_refused(self) -> None: self.layout["calls"].append({"call": "k999", "ids": [self.layout["calls"][1]["ids"][0]]}) self.refused("more than one place") def test_an_id_missing_from_every_call_is_refused(self) -> None: self.layout["calls"][-1]["ids"].pop() self.refused("in no call") def test_an_id_not_in_the_set_is_refused(self) -> None: self.layout["calls"][-1]["ids"][-1] = "c999" self.refused("not in the set") def test_a_wrong_set_sha256_is_refused(self) -> None: self.layout["set_sha256"] = "0" * 64 self.refused("set_sha256") def test_a_call_over_ten_is_refused(self) -> None: moved = self.layout["calls"][1]["ids"].pop() self.layout["calls"][0]["ids"].append(moved) self.refused("more than 10") def test_a_call_size_max_over_ten_is_refused(self) -> None: self.layout["call_size_max"] = 11 self.refused("call_size_max") def test_a_position_without_unsupported_is_refused(self) -> None: for c in self.layout["calls"]: ids = c["ids"] if self.key[ids[2]] == UNSUPPORTED: swap = next(n for n, i in enumerate(ids) if self.key[i] != UNSUPPORTED and n != 2) ids[2], ids[swap] = ids[swap], ids[2] self.refused("position 3 has no unsupported claim") class CheckBuiltSetTest(Base): def setUp(self) -> None: super().setUp() code, out = self.ws.build() self.assertEqual(code, 0, out) def excerpts(self) -> int: return sum(len(q["excerpts"]) for q in self.ws.load("set")["pairs"]) def test_check_passes_on_the_built_set_and_states_the_layout(self) -> None: code, out = self.ws.check() self.assertEqual(code, 0, out) n = self.excerpts() self.assertIn(f"excerpt lines correct: {n} of {n}", out) self.assertIn("pairs: 120 (20 unsupported, 6 supported, 94 filler-supported)", out) self.assertIn("unsupported per call: 1 in 10 calls, 5 in 2 calls", out) self.assertIn("PASS", out) def test_more_unsupported_than_supported_passes_when_the_manifest_says_so(self) -> None: self.assertEqual(self.ws.check()[0], 0) manifest = self.ws.load("manifest") manifest["counts"]["unsupported"] = 6 self.ws.paths()["manifest"].write_text(json.dumps(manifest), encoding="utf-8") code, out = self.ws.check() self.assertEqual(code, 1, out) self.assertIn("manifest", out) def test_check_catches_a_wrong_line(self) -> None: data = self.ws.load("set") data["pairs"][0]["excerpts"][0]["line"] += 1 self.ws.paths()["set"].write_text(json.dumps(data), encoding="utf-8") code, out = self.ws.check() self.assertEqual(code, 1) n = self.excerpts() self.assertIn(f"excerpt lines correct: {n - 1} of {n}", out) def test_check_of_a_built_set_needs_the_layout_and_the_manifest(self) -> None: p = self.ws.paths() code, out = run("check", "--set", str(p["set"]), "--key", str(p["key"]), "--sources", str(self.ws.sources)) self.assertEqual(code, 1) self.assertIn("FAIL", out) class StubTest(Base): def setUp(self) -> None: super().setUp() code, out = self.ws.build() self.assertEqual(code, 0, out) def stub(self, *args: str) -> tuple[int, str, dict[str, str]]: path = self.ws.root / "stub.json" code, out = run("stub", "--set", str(self.ws.paths()["set"]), "--out", str(path), *args) verdicts = {} if path.exists(): verdicts = {v["id"]: v["verdict"] for v in json.loads(path.read_text(encoding="utf-8"))["verdicts"]} path.unlink() return code, out, verdicts def test_the_same_verdict_for_every_claim(self) -> None: code, out, v = self.stub("--verdict", UNSUPPORTED) self.assertEqual(code, 0, out) self.assertEqual(set(v.values()), {UNSUPPORTED}) self.assertEqual(len(v), 120) def test_the_key_gives_supported_for_every_filler(self) -> None: code, out, v = self.stub("--key", str(self.ws.paths()["key"])) self.assertEqual(code, 0, out) labels = self.ws.labels() self.assertEqual(v, {i: UNSUPPORTED if lab == UNSUPPORTED else SUPPORTED for i, lab in labels.items()}) def test_accept_at_changes_only_that_position(self) -> None: code, out, v = self.stub("--verdict", UNSUPPORTED, "--calls", str(self.ws.paths()["calls"]), "--accept-at", "4") self.assertEqual(code, 0, out) pos = self.ws.positions() self.assertEqual({i for i, val in v.items() if val == SUPPORTED}, {i for i, n in pos.items() if n == 4}) def test_accept_at_needs_the_layout_and_a_position_from_one_to_ten(self) -> None: self.assertEqual(self.stub("--verdict", UNSUPPORTED, "--accept-at", "4")[0], 1) for bad in ("0", "11"): code, out, v = self.stub("--verdict", UNSUPPORTED, "--calls", str(self.ws.paths()["calls"]), "--accept-at", bad) self.assertEqual(code, 1, bad) def test_verdict_and_key_exclude_each_other(self) -> None: self.assertNotEqual(self.stub("--verdict", UNSUPPORTED, "--key", str(self.ws.paths()["key"]))[0], 0) self.assertNotEqual(self.stub()[0], 0) class NoModelCallTest(unittest.TestCase): def test_tool_imports_no_network_or_model_client(self) -> None: text = (REPO_ROOT / "gates" / "calibration.py").read_text(encoding="utf-8") for banned in ("anthropic", "openai", "urllib", "http.client", "requests", "subprocess", "socket"): self.assertNotIn(f"import {banned}", text) self.assertNotIn(f"from {banned}", text) if __name__ == "__main__": unittest.main()