First public release, published as one snapshot. What changed between versions is in CHANGELOG.md. Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
938 lines
43 KiB
Python
938 lines
43 KiB
Python
"""Tests for gates/calibration.py: the reader-calibration set checker, the seeded build with its call layout (K3), the
|
|
stubs and the verdict gate.
|
|
|
|
Every fixture is synthetic text written here (widgets and garden beds). No corpus text is used (CLAUDE.md rule 3).
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import contextlib
|
|
import hashlib
|
|
import io
|
|
import json
|
|
import re
|
|
import sys
|
|
import tempfile
|
|
import unittest
|
|
from collections import Counter
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
|
|
|
import calibration # noqa: E402
|
|
|
|
REPO_ROOT = Path(__file__).resolve().parents[2]
|
|
|
|
SOURCES = {
|
|
"alpha.md": "# Alpha\n\nThe widget timeout is 45 seconds by default.\nRaising it above 90 seconds disables retries.\n",
|
|
"beta.md": "# Beta\n\nBeta adds the --frobnicate flag for batch runs.\nThe flag is ignored on interactive runs.\n",
|
|
"gamma.md": "# Gamma\n\nGamma stores its cache under the .gamma-cache directory.\nThe cache is cleared on every upgrade.\n",
|
|
"north.md": "# North beds\n\n" + "".join(f"Bed N{i} gets {i + 2} litres of water every morning.\n" for i in range(1, 31)),
|
|
"south.md": "# South beds\n\n" + "".join(f"Bed S{i} is weeded every {i + 3} days in summer.\n" for i in range(1, 31)),
|
|
}
|
|
|
|
ALPHA = "The widget timeout is 45 seconds by default."
|
|
ALPHA2 = "Raising it above 90 seconds disables retries."
|
|
BETA = "Beta adds the --frobnicate flag for batch runs."
|
|
BETA2 = "The flag is ignored on interactive runs."
|
|
GAMMA = "Gamma stores its cache under the .gamma-cache directory."
|
|
GAMMA2 = "The cache is cleared on every upgrade."
|
|
|
|
UNSUPPORTED, SUPPORTED, FILLER = "unsupported", "supported", "filler-supported"
|
|
CALL_ID = re.compile(r"\bk\d{3,}\b")
|
|
CLAIM_ID = re.compile(r"\bc\d{3,}\b")
|
|
|
|
|
|
def ex(name: str, text: str) -> dict[str, Any]:
|
|
return {"file": name, "text": text}
|
|
|
|
|
|
def draft_pairs() -> list[dict[str, Any]]:
|
|
"""Six unsupported pairs (one per kind) and six supported pairs (two per kind): a valid small draft."""
|
|
u = [
|
|
("overgeneralisation", "Every Alpha setting defaults to 45 seconds.", [ex("alpha.md", ALPHA)]),
|
|
("wrong-number", "The widget timeout is 30 seconds by default.", [ex("alpha.md", ALPHA)]),
|
|
("swapped-cause", "Retries being disabled is what raises the timeout above 90 seconds.", [ex("alpha.md", ALPHA2)]),
|
|
("not-mentioned", "Beta writes a summary file after each batch run.", [ex("beta.md", BETA)]),
|
|
("reversed-polarity", "The --frobnicate flag also applies to interactive runs.", [ex("beta.md", BETA2)]),
|
|
(
|
|
"conflated-sources",
|
|
"Beta stores its cache under the .gamma-cache directory.",
|
|
[ex("beta.md", BETA), ex("gamma.md", GAMMA)],
|
|
),
|
|
]
|
|
s = [
|
|
("paraphrase", "By default, the widget waits 45 seconds before timing out.", [ex("alpha.md", ALPHA)]),
|
|
("paraphrase", "Upgrading clears the Gamma cache.", [ex("gamma.md", GAMMA2)]),
|
|
("inference", "A widget left at its default timeout still retries.", [ex("alpha.md", ALPHA + "\n" + ALPHA2)]),
|
|
("inference", "Batch runs are the ones where --frobnicate has an effect.", [ex("beta.md", BETA + "\n" + BETA2)]),
|
|
(
|
|
"synthesis",
|
|
"An upgrade empties the directory Gamma uses for its cache.",
|
|
[ex("gamma.md", GAMMA), ex("gamma.md", GAMMA2)],
|
|
),
|
|
(
|
|
"synthesis",
|
|
"Both a 45-second default timeout and a --frobnicate flag for batch runs exist.",
|
|
[ex("alpha.md", ALPHA), ex("beta.md", BETA)],
|
|
),
|
|
]
|
|
pairs: list[dict[str, Any]] = []
|
|
for n, (kind, claim, excerpts) in enumerate(u, 1):
|
|
pairs.append({"id": f"u{n:02d}", "label": UNSUPPORTED, "kind": kind, "claim": claim, "excerpts": excerpts, "why": "planted"})
|
|
for n, (kind, claim, excerpts) in enumerate(s, 1):
|
|
pairs.append({"id": f"s{n:02d}", "label": SUPPORTED, "kind": kind, "claim": claim, "excerpts": excerpts, "why": "follows"})
|
|
return pairs
|
|
|
|
|
|
def north(i: int) -> str:
|
|
return f"Bed N{i} gets {i + 2} litres of water every morning."
|
|
|
|
|
|
def south(i: int) -> str:
|
|
return f"Bed S{i} is weeded every {i + 3} days in summer."
|
|
|
|
|
|
def garden_draft(n_unsupported: int = 20, n_supported: int = 6, extra_fillers: int = 4) -> list[dict[str, Any]]:
|
|
"""More unsupported than supported pairs, each unsupported with its supported twin (filler-supported), plus a few
|
|
fillers without a twin. Every kind of both counted labels is present."""
|
|
pairs: list[dict[str, Any]] = []
|
|
for i in range(1, n_unsupported + 1):
|
|
kind = calibration.UNSUPPORTED_KINDS[(i - 1) % len(calibration.UNSUPPORTED_KINDS)]
|
|
excerpts = [ex("north.md", north(i))]
|
|
claim = {
|
|
"overgeneralisation": f"Every north bed, N{i} included, gets {i + 2} litres of water each morning.",
|
|
"wrong-number": f"Bed N{i} gets {i + 50} litres of water each morning.",
|
|
"swapped-cause": f"Watering bed N{i} each morning is what makes it need {i + 2} litres.",
|
|
"not-mentioned": f"Bed N{i} gets {i + 2} litres of water each morning and is mulched in spring.",
|
|
"reversed-polarity": f"Bed N{i} never gets {i + 2} litres of water in the morning.",
|
|
"conflated-sources": f"Bed N{i} is weeded every {i + 3} days in summer.",
|
|
}[kind]
|
|
twin, twin_kind = f"Each morning, bed N{i} receives {i + 2} litres of water.", "paraphrase"
|
|
if kind == "conflated-sources":
|
|
excerpts = [ex("north.md", north(i)), ex("south.md", south(i))]
|
|
twin = f"Bed N{i} gets {i + 2} litres each morning, and bed S{i} is weeded every {i + 3} days in summer."
|
|
twin_kind = "synthesis"
|
|
pairs.append({"id": f"u{i:02d}", "label": UNSUPPORTED, "kind": kind, "claim": claim, "excerpts": excerpts, "why": "planted"})
|
|
pairs.append({"id": f"t{i:02d}", "label": FILLER, "kind": twin_kind, "twin_of": f"u{i:02d}", "claim": twin,
|
|
"excerpts": [dict(e) for e in excerpts], "why": "the supported reading"})
|
|
for n in range(1, n_supported + 1):
|
|
j = 20 + n
|
|
kind = calibration.SUPPORTED_KINDS[(n - 1) % len(calibration.SUPPORTED_KINDS)]
|
|
claim, excerpts = {
|
|
"paraphrase": (f"In summer, bed S{j} has its weeds pulled every {j + 3} days.", [ex("south.md", south(j))]),
|
|
"inference": (f"Bed S{j} is weeded more often than once a season in summer.", [ex("south.md", south(j))]),
|
|
"synthesis": (
|
|
f"Bed N{j} is watered each morning and bed S{j} is weeded every {j + 3} days in summer.",
|
|
[ex("north.md", north(j)), ex("south.md", south(j))],
|
|
),
|
|
}[kind]
|
|
pairs.append({"id": f"s{n:02d}", "label": SUPPORTED, "kind": kind, "claim": claim, "excerpts": excerpts, "why": "follows"})
|
|
for n in range(1, extra_fillers + 1):
|
|
j = 26 + n
|
|
pairs.append({"id": f"f{n:02d}", "label": FILLER, "kind": "paraphrase",
|
|
"claim": f"Each morning, bed N{j} receives {j + 2} litres of water.",
|
|
"excerpts": [ex("north.md", north(j))], "why": "follows"})
|
|
return pairs
|
|
|
|
|
|
def sha256_file(path: Path) -> str:
|
|
return hashlib.sha256(path.read_bytes()).hexdigest()
|
|
|
|
|
|
class Workspace:
|
|
"""A temporary sources directory and helpers to write drafts, verdicts and build outputs."""
|
|
|
|
def __init__(self) -> None:
|
|
self._tmp = tempfile.TemporaryDirectory()
|
|
self.root = Path(self._tmp.name)
|
|
self.sources = self.root / "sources"
|
|
self.sources.mkdir()
|
|
for name, text in SOURCES.items():
|
|
(self.sources / name).write_text(text, encoding="utf-8")
|
|
|
|
def close(self) -> None:
|
|
self._tmp.cleanup()
|
|
|
|
def write_json(self, name: str, data: Any) -> Path:
|
|
path = self.root / name
|
|
path.write_text(json.dumps(data), encoding="utf-8")
|
|
return path
|
|
|
|
def write_draft(self, pairs: list[dict[str, Any]], name: str = "draft.json") -> Path:
|
|
return self.write_json(name, {"schema": calibration.DRAFT_SCHEMA, "pairs": pairs})
|
|
|
|
def paths(self) -> dict[str, Path]:
|
|
return {
|
|
"set": self.root / "out" / "pairs.json",
|
|
"key": self.root / "out" / "key.json",
|
|
"calls": self.root / "out" / "calls.json",
|
|
"manifest": self.root / "calibration.manifest.json",
|
|
}
|
|
|
|
def build(
|
|
self, pairs: list[dict[str, Any]] | None = None, seed: int = 7, lone: int = 10, dense: int = 5
|
|
) -> tuple[int, str]:
|
|
draft = self.write_draft(garden_draft() if pairs is None else pairs)
|
|
p = self.paths()
|
|
return run(
|
|
"build", "--draft", str(draft), "--sources", str(self.sources), "--seed", str(seed),
|
|
"--lone", str(lone), "--dense", str(dense),
|
|
"--set-out", str(p["set"]), "--key-out", str(p["key"]), "--calls-out", str(p["calls"]),
|
|
"--manifest", str(p["manifest"]),
|
|
)
|
|
|
|
def frozen_args(self) -> list[str]:
|
|
p = self.paths()
|
|
return ["--set", str(p["set"]), "--key", str(p["key"]), "--calls", str(p["calls"]),
|
|
"--manifest", str(p["manifest"]), "--sources", str(self.sources)]
|
|
|
|
def gate(self, verdicts: Path | None) -> tuple[int, str]:
|
|
return run("gate", *self.frozen_args(), "--verdicts", str(verdicts or self.root / "no-such-verdicts.json"))
|
|
|
|
def check(self) -> tuple[int, str]:
|
|
return run("check", *self.frozen_args())
|
|
|
|
def load(self, which: str) -> Any:
|
|
return json.loads(self.paths()[which].read_text(encoding="utf-8"))
|
|
|
|
def labels(self) -> dict[str, str]:
|
|
return {k["id"]: k["label"] for k in self.load("key")["labels"]}
|
|
|
|
def positions(self) -> dict[str, int]:
|
|
return {i: n for c in self.load("calls")["calls"] for n, i in enumerate(c["ids"], 1)}
|
|
|
|
def rewrite_calls(self, layout: dict[str, Any]) -> None:
|
|
"""Write a changed layout and make the manifest's sha256 follow it, so only the layout rule under test fails."""
|
|
p = self.paths()
|
|
p["calls"].write_text(json.dumps(layout, indent=1, sort_keys=True) + "\n", encoding="utf-8")
|
|
manifest = self.load("manifest")
|
|
manifest["calls"]["sha256"] = sha256_file(p["calls"])
|
|
p["manifest"].write_text(json.dumps(manifest, indent=1) + "\n", encoding="utf-8")
|
|
|
|
|
|
def run(*argv: str) -> tuple[int, str]:
|
|
out, err = io.StringIO(), io.StringIO()
|
|
with contextlib.redirect_stdout(out), contextlib.redirect_stderr(err):
|
|
try:
|
|
code = calibration.main(list(argv))
|
|
except SystemExit as exc: # argparse refuses an unknown option
|
|
code = exc.code if isinstance(exc.code, int) else 2
|
|
return code, out.getvalue() + err.getvalue()
|
|
|
|
|
|
class Base(unittest.TestCase):
|
|
def setUp(self) -> None:
|
|
self.ws = Workspace()
|
|
|
|
def tearDown(self) -> None:
|
|
self.ws.close()
|
|
|
|
|
|
class CheckDraftTest(Base):
|
|
def check(self, pairs: list[dict[str, Any]]) -> calibration.CheckReport:
|
|
return calibration.check_pairs(pairs, self.ws.sources)
|
|
|
|
def test_valid_draft_passes_and_counts_kinds(self) -> None:
|
|
rep = self.check(draft_pairs())
|
|
self.assertTrue(rep.passed, rep.lines())
|
|
self.assertEqual(rep.unsupported, 6)
|
|
self.assertEqual(rep.supported, 6)
|
|
self.assertEqual(rep.excerpts_verbatim, rep.excerpts)
|
|
self.assertEqual(rep.by_kind["wrong-number"], 1)
|
|
self.assertEqual(rep.by_kind["synthesis"], 2)
|
|
|
|
def test_excerpt_not_verbatim_fails(self) -> None:
|
|
pairs = draft_pairs()
|
|
pairs[0]["excerpts"] = [ex("alpha.md", "The widget timeout is 46 seconds by default.")]
|
|
rep = self.check(pairs)
|
|
self.assertFalse(rep.passed)
|
|
self.assertEqual(rep.excerpts_verbatim, rep.excerpts - 1)
|
|
|
|
def test_missing_source_file_fails(self) -> None:
|
|
pairs = draft_pairs()
|
|
pairs[0]["excerpts"] = [ex("nowhere.md", ALPHA)]
|
|
self.assertFalse(self.check(pairs).passed)
|
|
|
|
def test_path_outside_sources_fails(self) -> None:
|
|
pairs = draft_pairs()
|
|
pairs[0]["excerpts"] = [ex("../alpha.md", ALPHA)]
|
|
self.assertFalse(self.check(pairs).passed)
|
|
|
|
def test_unbalanced_labels_pass_the_draft_check(self) -> None:
|
|
# The counts are the manifest's business now (order 3, point 1), not a balance rule.
|
|
rep = self.check(garden_draft())
|
|
self.assertTrue(rep.passed, rep.lines())
|
|
self.assertEqual((rep.unsupported, rep.supported, rep.filler), (20, 6, 24))
|
|
|
|
def test_every_unsupported_kind_must_be_present(self) -> None:
|
|
pairs = draft_pairs()
|
|
pairs[1]["kind"] = "overgeneralisation"
|
|
pairs[1]["claim"] = "Every widget setting is 45 seconds long."
|
|
rep = self.check(pairs)
|
|
self.assertFalse(rep.passed)
|
|
self.assertTrue(any("wrong-number" in e for e in rep.errors), rep.errors)
|
|
|
|
def test_every_supported_kind_must_be_present(self) -> None:
|
|
pairs = draft_pairs()
|
|
for p in pairs:
|
|
if p["kind"] == "inference":
|
|
p["kind"] = "paraphrase"
|
|
rep = self.check(pairs)
|
|
self.assertFalse(rep.passed)
|
|
self.assertTrue(any("inference" in e for e in rep.errors), rep.errors)
|
|
|
|
def test_unknown_kind_or_label_fails(self) -> None:
|
|
pairs = draft_pairs()
|
|
pairs[0]["kind"] = "vibes"
|
|
self.assertFalse(self.check(pairs).passed)
|
|
pairs = draft_pairs()
|
|
pairs[0]["label"] = "maybe"
|
|
self.assertFalse(self.check(pairs).passed)
|
|
|
|
def test_kind_must_match_label(self) -> None:
|
|
pairs = draft_pairs()
|
|
pairs[0]["kind"] = "paraphrase"
|
|
self.assertFalse(self.check(pairs).passed)
|
|
|
|
def test_filler_takes_a_supported_kind(self) -> None:
|
|
pairs = garden_draft()
|
|
twin = next(p for p in pairs if p["label"] == FILLER)
|
|
twin["kind"] = "wrong-number"
|
|
self.assertFalse(self.check(pairs).passed)
|
|
|
|
def test_filler_may_not_be_a_verbatim_copy(self) -> None:
|
|
pairs = garden_draft()
|
|
twin = next(p for p in pairs if p["label"] == FILLER and p["kind"] == "paraphrase")
|
|
twin["claim"] = twin["excerpts"][0]["text"]
|
|
rep = self.check(pairs)
|
|
self.assertFalse(rep.passed)
|
|
self.assertTrue(any(twin["id"] in e and "verbatim" in e for e in rep.errors), rep.errors)
|
|
|
|
def test_twin_of_must_name_an_unsupported_pair(self) -> None:
|
|
for target in ("s01", "nowhere"):
|
|
pairs = garden_draft()
|
|
next(p for p in pairs if p["label"] == FILLER)["twin_of"] = target
|
|
rep = self.check(pairs)
|
|
self.assertFalse(rep.passed, target)
|
|
self.assertTrue(any("twin_of" in e for e in rep.errors), rep.errors)
|
|
|
|
def test_wrong_number_claim_needs_a_number_absent_from_the_excerpts(self) -> None:
|
|
pairs = draft_pairs()
|
|
pairs[1]["claim"] = "The widget timeout is 45 seconds, which is short."
|
|
rep = self.check(pairs)
|
|
self.assertFalse(rep.passed)
|
|
self.assertTrue(any("u02" in e and "number" in e for e in rep.errors), rep.errors)
|
|
|
|
def test_conflated_sources_needs_two_files(self) -> None:
|
|
pairs = draft_pairs()
|
|
pairs[5]["excerpts"] = [ex("beta.md", BETA), ex("beta.md", BETA2)]
|
|
self.assertFalse(self.check(pairs).passed)
|
|
|
|
def test_synthesis_needs_two_excerpts(self) -> None:
|
|
pairs = draft_pairs()
|
|
pairs[10]["excerpts"] = [ex("gamma.md", GAMMA)]
|
|
self.assertFalse(self.check(pairs).passed)
|
|
|
|
def test_supported_claim_may_not_be_a_verbatim_copy(self) -> None:
|
|
pairs = draft_pairs()
|
|
pairs[6]["claim"] = "the widget timeout is 45 seconds by default"
|
|
rep = self.check(pairs)
|
|
self.assertFalse(rep.passed)
|
|
self.assertTrue(any("s01" in e and "verbatim" in e for e in rep.errors), rep.errors)
|
|
|
|
def test_duplicate_ids_fail(self) -> None:
|
|
pairs = draft_pairs()
|
|
pairs[1]["id"] = "u01"
|
|
self.assertFalse(self.check(pairs).passed)
|
|
|
|
def test_too_many_excerpts_fail(self) -> None:
|
|
pairs = draft_pairs()
|
|
pairs[0]["excerpts"] = [ex("alpha.md", ALPHA), ex("beta.md", BETA), ex("gamma.md", GAMMA)]
|
|
self.assertFalse(self.check(pairs).passed)
|
|
|
|
def test_empty_draft_fails(self) -> None:
|
|
self.assertFalse(self.check([]).passed)
|
|
|
|
def test_check_command_on_draft(self) -> None:
|
|
draft = self.ws.write_draft(draft_pairs())
|
|
code, out = run("check", "--draft", str(draft), "--sources", str(self.ws.sources))
|
|
self.assertEqual(code, 0, out)
|
|
self.assertIn("excerpts verbatim: 15 of 15", out)
|
|
self.assertIn("PASS", out)
|
|
|
|
def test_check_command_missing_sources_is_a_fail(self) -> None:
|
|
draft = self.ws.write_draft(draft_pairs())
|
|
code, out = run("check", "--draft", str(draft), "--sources", str(self.ws.root / "absent"))
|
|
self.assertEqual(code, 1)
|
|
self.assertIn("FAIL", out)
|
|
|
|
|
|
class BuildTest(Base):
|
|
def built(self, **kwargs: Any) -> None:
|
|
code, out = self.ws.build(**kwargs)
|
|
self.assertEqual(code, 0, out)
|
|
|
|
def test_build_writes_set_key_layout_and_manifest(self) -> None:
|
|
self.built()
|
|
pset, key, layout, manifest = (self.ws.load(w) for w in ("set", "key", "calls", "manifest"))
|
|
p = self.ws.paths()
|
|
self.assertEqual(pset["schema"], calibration.SET_SCHEMA)
|
|
ids = [q["id"] for q in pset["pairs"]]
|
|
self.assertEqual(ids, [f"c{n:03d}" for n in range(1, 121)])
|
|
for q in pset["pairs"]:
|
|
self.assertEqual(set(q), {"id", "claim", "excerpts"})
|
|
for e in q["excerpts"]:
|
|
self.assertEqual(set(e), {"file", "line", "text"})
|
|
self.assertNotIn("unsupported", json.dumps(pset))
|
|
self.assertNotIn("filler", json.dumps(pset))
|
|
self.assertEqual(sorted(k["id"] for k in key["labels"]), ids)
|
|
self.assertEqual(layout["schema"], "okf-wiki/reader-calls/1")
|
|
self.assertEqual(layout["call_size_max"], 10)
|
|
self.assertEqual(layout["set_sha256"], sha256_file(p["set"]))
|
|
self.assertEqual(len(layout["calls"]), 12)
|
|
self.assertEqual([c["call"] for c in layout["calls"]], [f"k{n:03d}" for n in range(1, 13)])
|
|
self.assertEqual(manifest["seed"], 7)
|
|
self.assertEqual(manifest["set"]["sha256"], sha256_file(p["set"]))
|
|
self.assertEqual(manifest["key"]["sha256"], sha256_file(p["key"]))
|
|
self.assertEqual(manifest["calls"]["sha256"], sha256_file(p["calls"]))
|
|
self.assertEqual(manifest["calls"]["calls"], 12)
|
|
counts = manifest["counts"]
|
|
self.assertEqual((counts["unsupported"], counts["supported"], counts["filler-supported"]), (20, 6, 94))
|
|
self.assertEqual(counts["filler_distinct"], 24)
|
|
self.assertEqual(sum(v["unsupported"] for v in counts["by_position"].values()), 20)
|
|
self.assertEqual(manifest["thresholds"], {"caught_min": 20, "accepted_min": 6})
|
|
self.assertEqual(manifest["sources"]["files"], len(SOURCES))
|
|
self.assertEqual(manifest["history"], [])
|
|
self.assertNotIn("claim", json.dumps(manifest))
|
|
|
|
def test_every_claim_stands_in_exactly_one_call_of_at_most_ten(self) -> None:
|
|
self.built()
|
|
placed = [i for c in self.ws.load("calls")["calls"] for i in c["ids"]]
|
|
self.assertEqual(sorted(placed), sorted(self.ws.labels()))
|
|
self.assertEqual(len(placed), len(set(placed)))
|
|
self.assertTrue(all(len(c["ids"]) <= 10 for c in self.ws.load("calls")["calls"]))
|
|
|
|
def test_lone_unsupported_stand_at_every_position(self) -> None:
|
|
self.built()
|
|
labels, pos = self.ws.labels(), self.ws.positions()
|
|
alone = []
|
|
for c in self.ws.load("calls")["calls"]:
|
|
u = [i for i in c["ids"] if labels[i] == UNSUPPORTED]
|
|
if len(u) == 1:
|
|
alone.append(pos[u[0]])
|
|
self.assertGreaterEqual(len(alone), 10)
|
|
self.assertEqual(set(alone), set(range(1, 11)))
|
|
self.assertEqual(self.ws.load("manifest")["calls"]["unsupported_alone"], len(alone))
|
|
|
|
def test_each_counted_supported_pair_stands_once(self) -> None:
|
|
self.built()
|
|
drafts = Counter(k["draft_id"] for k in self.ws.load("key")["labels"] if k["label"] == SUPPORTED)
|
|
self.assertEqual(drafts, Counter({f"s{n:02d}": 1 for n in range(1, 7)}))
|
|
|
|
def test_a_twin_never_shares_a_call_with_its_unsupported_pair_and_no_filler_stands_twice_in_a_call(self) -> None:
|
|
self.built()
|
|
draft_of = {k["id"]: k["draft_id"] for k in self.ws.load("key")["labels"]}
|
|
for c in self.ws.load("calls")["calls"]:
|
|
members = [draft_of[i] for i in c["ids"]]
|
|
self.assertEqual(len(members), len(set(members)), c["call"])
|
|
for m in members:
|
|
if m.startswith("t"):
|
|
self.assertNotIn("u" + m[1:], members, c["call"])
|
|
|
|
def test_same_seed_gives_the_same_bytes_and_another_seed_another_layout(self) -> None:
|
|
self.built(seed=7)
|
|
first = {w: self.ws.paths()[w].read_bytes() for w in ("set", "key", "calls")}
|
|
other = Workspace()
|
|
try:
|
|
self.assertEqual(other.build(seed=7)[0], 0)
|
|
self.assertEqual(first, {w: other.paths()[w].read_bytes() for w in ("set", "key", "calls")})
|
|
finally:
|
|
other.close()
|
|
other = Workspace()
|
|
try:
|
|
self.assertEqual(other.build(seed=8)[0], 0)
|
|
self.assertNotEqual(first["calls"], other.paths()["calls"].read_bytes())
|
|
self.assertNotEqual(first["set"], other.paths()["set"].read_bytes())
|
|
finally:
|
|
other.close()
|
|
|
|
def test_order_does_not_follow_the_labels(self) -> None:
|
|
self.built()
|
|
labels = self.ws.labels()
|
|
order = [labels[q["id"]] for q in self.ws.load("set")["pairs"]]
|
|
self.assertNotEqual(order, sorted(order))
|
|
self.assertNotEqual(order, sorted(order, reverse=True))
|
|
|
|
def test_build_moves_the_old_manifest_into_history(self) -> None:
|
|
old = {"schema": "okf-wiki/calibration-manifest/1", "seed": 1, "history": [{"schema": "older"}]}
|
|
self.ws.paths()["manifest"].write_text(json.dumps(old), encoding="utf-8")
|
|
self.built()
|
|
history = self.ws.load("manifest")["history"]
|
|
self.assertEqual(history, [{"schema": "older"}, {"schema": "okf-wiki/calibration-manifest/1", "seed": 1}])
|
|
|
|
def test_build_refuses_a_failing_draft(self) -> None:
|
|
pairs = garden_draft()
|
|
pairs[0]["excerpts"] = [ex("north.md", "not in the file at all, not anywhere")]
|
|
code, out = self.ws.build(pairs)
|
|
self.assertEqual(code, 1)
|
|
self.assertFalse(self.ws.paths()["set"].exists())
|
|
|
|
def test_build_refuses_when_a_position_would_have_no_unsupported_claim(self) -> None:
|
|
code, out = self.ws.build(garden_draft(n_unsupported=9), lone=9)
|
|
self.assertEqual(code, 1, out)
|
|
self.assertIn("position 10", out)
|
|
self.assertFalse(any(self.ws.paths()[w].exists() for w in ("set", "key", "calls", "manifest")))
|
|
|
|
def test_build_refuses_slots_it_cannot_fill(self) -> None:
|
|
pairs = [p for p in garden_draft() if p["label"] != FILLER]
|
|
code, out = self.ws.build(pairs)
|
|
self.assertEqual(code, 1, out)
|
|
self.assertIn("filler", out)
|
|
self.assertFalse(self.ws.paths()["calls"].exists())
|
|
|
|
def test_build_refuses_impossible_parameters(self) -> None:
|
|
for lone, dense in ((21, 5), (10, 11), (10, 0), (-1, 5)):
|
|
code, out = self.ws.build(lone=lone, dense=dense)
|
|
self.assertEqual(code, 1, (lone, dense, out))
|
|
self.assertFalse(self.ws.paths()["set"].exists())
|
|
|
|
def test_build_never_overwrites_a_frozen_set(self) -> None:
|
|
self.built()
|
|
code, out = self.ws.build()
|
|
self.assertEqual(code, 1)
|
|
self.assertIn("frozen", out)
|
|
self.ws.paths()["set"].unlink()
|
|
self.ws.paths()["key"].unlink()
|
|
code, out = self.ws.build()
|
|
self.assertEqual(code, 1)
|
|
self.assertIn("frozen", out)
|
|
|
|
|
|
class GateBase(Base):
|
|
"""A built set, its key, and helpers to write verdicts and stubs for the gate."""
|
|
|
|
def setUp(self) -> None:
|
|
super().setUp()
|
|
code, out = self.ws.build()
|
|
self.assertEqual(code, 0, out)
|
|
self.key = self.ws.labels()
|
|
|
|
def verdicts(self, mapping: dict[str, str], reader: str = "test", models: Any = ("claude-test-1",),
|
|
name: str = "verdicts.json") -> Path:
|
|
doc: dict[str, Any] = {"reader": reader, "verdicts": [{"id": i, "verdict": v} for i, v in mapping.items()]}
|
|
if models is not None:
|
|
doc["models"] = list(models) if isinstance(models, tuple) else models
|
|
return self.ws.write_json(name, doc)
|
|
|
|
def perfect(self) -> dict[str, str]:
|
|
return {i: (UNSUPPORTED if lab == UNSUPPORTED else SUPPORTED) for i, lab in self.key.items()}
|
|
|
|
def stub(self, *args: str) -> Path:
|
|
path = self.ws.root / "stub.json"
|
|
code, out = run("stub", "--set", str(self.ws.paths()["set"]), "--out", str(path), *args)
|
|
self.assertEqual(code, 0, out)
|
|
return path
|
|
|
|
def per_position(self) -> dict[int, int]:
|
|
pos = self.ws.positions()
|
|
return dict(Counter(pos[i] for i, lab in self.key.items() if lab == UNSUPPORTED))
|
|
|
|
|
|
class GateTest(GateBase):
|
|
def test_no_reader_scores_zero_and_is_red(self) -> None:
|
|
code, out = self.ws.gate(None)
|
|
self.assertEqual(code, 1)
|
|
self.assertIn("unsupported caught 0 of 20 · supported accepted 0 of 6", out)
|
|
self.assertIn("fail-closed", out)
|
|
self.assertIn("RED", out)
|
|
|
|
def test_the_key_as_judge_is_green(self) -> None:
|
|
code, out = self.ws.gate(self.stub("--key", str(self.ws.paths()["key"])))
|
|
self.assertEqual(code, 0, out)
|
|
self.assertIn("unsupported caught 20 of 20 · supported accepted 6 of 6", out)
|
|
self.assertIn("filler accepted 94 of 94", out)
|
|
self.assertIn("GREEN", out)
|
|
|
|
def test_reject_all_stub_is_red(self) -> None:
|
|
code, out = self.ws.gate(self.stub("--verdict", UNSUPPORTED))
|
|
self.assertEqual(code, 1)
|
|
self.assertIn("unsupported caught 20 of 20 · supported accepted 0 of 6", out)
|
|
self.assertIn("RED", out)
|
|
|
|
def test_accept_all_stub_is_red(self) -> None:
|
|
code, out = self.ws.gate(self.stub("--verdict", SUPPORTED))
|
|
self.assertEqual(code, 1)
|
|
self.assertIn("unsupported caught 0 of 20 · supported accepted 6 of 6", out)
|
|
self.assertIn("RED", out)
|
|
|
|
def test_a_stub_mild_at_one_position_is_red_and_the_position_is_named(self) -> None:
|
|
stub = self.stub("--key", str(self.ws.paths()["key"]), "--calls", str(self.ws.paths()["calls"]), "--accept-at", "7")
|
|
code, out = self.ws.gate(stub)
|
|
self.assertEqual(code, 1, out)
|
|
n7 = self.per_position()[7]
|
|
self.assertIn(f"7 0 of {n7}", out)
|
|
self.assertIn(f"unsupported caught {20 - n7} of 20", out)
|
|
not_met = [line for line in out.splitlines() if line.startswith("not met:")]
|
|
self.assertEqual(len(not_met), 1, out)
|
|
self.assertIn("position 7", not_met[0])
|
|
self.assertNotIn("position 6", not_met[0])
|
|
self.assertIn("RED", out)
|
|
|
|
def test_caught_per_position_is_written_and_no_line_per_call_or_pair(self) -> None:
|
|
code, out = self.ws.gate(self.verdicts(self.perfect()))
|
|
self.assertEqual(code, 0, out)
|
|
counts = self.per_position()
|
|
line = next(line for line in out.splitlines() if line.startswith("caught by position: "))
|
|
self.assertEqual(line, "caught by position: " + ", ".join(f"{p} {counts[p]} of {counts[p]}" for p in range(1, 11)))
|
|
self.assertIsNone(CALL_ID.search(out), out)
|
|
self.assertIsNone(CLAIM_ID.search(out), out)
|
|
|
|
def test_the_gate_writes_no_per_pair_results(self) -> None:
|
|
code, out = run("gate", *self.ws.frozen_args(), "--verdicts", str(self.verdicts(self.perfect())),
|
|
"--out", str(self.ws.root / "results.json"))
|
|
self.assertNotEqual(code, 0)
|
|
self.assertFalse((self.ws.root / "results.json").exists())
|
|
|
|
def test_reports_caught_per_kind(self) -> None:
|
|
code, out = self.ws.gate(self.verdicts(self.perfect()))
|
|
self.assertIn("caught by kind: overgeneralisation 4 of 4", out)
|
|
self.assertIn("conflated-sources 3 of 3", out)
|
|
|
|
def test_filler_never_counts_in_the_thresholds(self) -> None:
|
|
v = self.perfect()
|
|
for i, lab in self.key.items():
|
|
if lab == FILLER:
|
|
v[i] = UNSUPPORTED
|
|
code, out = self.ws.gate(self.verdicts(v))
|
|
self.assertEqual(code, 0, out)
|
|
self.assertIn("filler accepted 0 of 94", out)
|
|
self.assertIn("GREEN", out)
|
|
|
|
def test_a_missing_filler_verdict_is_red(self) -> None:
|
|
v = self.perfect()
|
|
del v[next(i for i, lab in self.key.items() if lab == FILLER)]
|
|
code, out = self.ws.gate(self.verdicts(v))
|
|
self.assertEqual(code, 1, out)
|
|
self.assertIn("missing verdicts: 1", out)
|
|
|
|
def test_one_missed_unsupported_is_red(self) -> None:
|
|
v = self.perfect()
|
|
v[next(i for i, lab in self.key.items() if lab == UNSUPPORTED)] = SUPPORTED
|
|
code, out = self.ws.gate(self.verdicts(v))
|
|
self.assertEqual(code, 1)
|
|
self.assertIn("unsupported caught 19 of 20", out)
|
|
|
|
def test_missing_verdict_counts_as_wrong(self) -> None:
|
|
v = self.perfect()
|
|
gone_u = next(i for i, lab in self.key.items() if lab == UNSUPPORTED)
|
|
gone_s = next(i for i, lab in self.key.items() if lab == SUPPORTED)
|
|
del v[gone_u], v[gone_s]
|
|
code, out = self.ws.gate(self.verdicts(v))
|
|
self.assertEqual(code, 1)
|
|
self.assertIn("missing verdicts: 2", out)
|
|
self.assertIn("unsupported caught 19 of 20 · supported accepted 5 of 6", out)
|
|
|
|
def test_invalid_or_duplicate_verdict_counts_as_missing(self) -> None:
|
|
v = self.perfect()
|
|
first = next(iter(v))
|
|
path = self.ws.write_json(
|
|
"verdicts.json",
|
|
{
|
|
"reader": "test",
|
|
"verdicts": [{"id": i, "verdict": val} for i, val in v.items()]
|
|
+ [{"id": first, "verdict": SUPPORTED}, {"id": "c999", "verdict": SUPPORTED}],
|
|
},
|
|
)
|
|
code, out = self.ws.gate(path)
|
|
self.assertEqual(code, 1)
|
|
self.assertIn("missing verdicts: 1", out)
|
|
for bad in ("probably", FILLER):
|
|
code, out = self.ws.gate(self.verdicts({**v, first: bad}))
|
|
self.assertEqual(code, 1, bad)
|
|
self.assertIn("missing verdicts: 1", out)
|
|
|
|
def test_accepted_threshold_is_read_from_the_manifest(self) -> None:
|
|
manifest = self.ws.load("manifest")
|
|
manifest["thresholds"]["accepted_min"] = 5
|
|
self.ws.paths()["manifest"].write_text(json.dumps(manifest), encoding="utf-8")
|
|
v = self.perfect()
|
|
v[next(i for i, lab in self.key.items() if lab == SUPPORTED)] = UNSUPPORTED
|
|
code, out = self.ws.gate(self.verdicts(v))
|
|
self.assertEqual(code, 0, out)
|
|
self.assertIn("supported accepted 5 of 6", out)
|
|
|
|
def test_gate_refuses_a_changed_set_key_or_layout(self) -> None:
|
|
for which in ("set", "key", "calls"):
|
|
path = self.ws.paths()[which]
|
|
before = path.read_text(encoding="utf-8")
|
|
path.write_text(before + " ", encoding="utf-8")
|
|
code, out = self.ws.gate(self.verdicts(self.perfect()))
|
|
self.assertEqual(code, 1, which)
|
|
self.assertIn("manifest", out)
|
|
path.write_text(before, encoding="utf-8")
|
|
|
|
def test_gate_refuses_other_sources(self) -> None:
|
|
(self.ws.sources / "alpha.md").write_text("changed\n", encoding="utf-8")
|
|
code, out = self.ws.gate(self.verdicts(self.perfect()))
|
|
self.assertEqual(code, 1)
|
|
self.assertIn("tree_sha256", out)
|
|
|
|
|
|
class GateModelTest(GateBase):
|
|
"""PLAN § Rammer 23: the gate is bound to the model the reader ran on. It names the model it scored, is GREEN only
|
|
for verdicts of exactly one model, and records a GREEN run in the file --passed names; collect reads that file."""
|
|
|
|
def passed(self) -> Path:
|
|
return self.ws.root / "passed.json"
|
|
|
|
def gate_passed(self, verdicts: Path) -> tuple[int, str]:
|
|
return run("gate", *self.ws.frozen_args(), "--verdicts", str(verdicts), "--passed", str(self.passed()))
|
|
|
|
def test_the_gate_names_the_model_it_scored(self) -> None:
|
|
code, out = self.ws.gate(self.verdicts(self.perfect(), models=["claude-test-2"]))
|
|
self.assertEqual(code, 0, out)
|
|
self.assertIn("model: claude-test-2\n", out)
|
|
self.assertIn("GREEN", out)
|
|
|
|
def test_verdicts_that_name_no_model_are_red(self) -> None:
|
|
for models in (None, [], "claude-test-2", [3]):
|
|
with self.subTest(models=models):
|
|
code, out = self.ws.gate(self.verdicts(self.perfect(), models=models))
|
|
self.assertEqual(code, 1, out)
|
|
self.assertIn("model: (none)\n", out)
|
|
self.assertIn("the verdicts name no model", out)
|
|
self.assertNotIn("GREEN", out)
|
|
self.assertIn("RED", out)
|
|
|
|
def test_verdicts_of_two_models_are_red(self) -> None:
|
|
code, out = self.ws.gate(self.verdicts(self.perfect(), models=["claude-test-1", "claude-test-2"]))
|
|
self.assertEqual(code, 1, out)
|
|
self.assertIn("model: claude-test-1, claude-test-2\n", out)
|
|
self.assertIn("the verdicts name 2 models", out)
|
|
self.assertIn("RED", out)
|
|
|
|
def test_a_stub_names_the_model_stub_and_its_meaning_is_kept(self) -> None:
|
|
stub = self.stub("--key", str(self.ws.paths()["key"]))
|
|
self.assertEqual(json.loads(stub.read_text(encoding="utf-8"))["models"], ["stub"])
|
|
code, out = self.ws.gate(stub)
|
|
self.assertEqual(code, 0, out)
|
|
self.assertIn("model: stub\n", out)
|
|
code, out = self.ws.gate(self.stub("--verdict", SUPPORTED))
|
|
self.assertEqual(code, 1, out)
|
|
self.assertIn("model: stub\n", out)
|
|
|
|
def test_a_green_run_is_recorded_with_its_hashes(self) -> None:
|
|
verdicts = self.verdicts(self.perfect(), reader="r7", models=["claude-test-2"])
|
|
code, out = self.gate_passed(verdicts)
|
|
self.assertEqual(code, 0, out)
|
|
p = self.ws.paths()
|
|
record = json.loads(self.passed().read_text(encoding="utf-8"))
|
|
self.assertEqual(record, {"schema": "okf-wiki/calibration-passed/1", "passed": [{
|
|
"model": "claude-test-2", "set_sha256": sha256_file(p["set"]), "key_sha256": sha256_file(p["key"]),
|
|
"verdicts_sha256": sha256_file(verdicts), "reader": "r7"}]})
|
|
self.assertIn(str(self.passed()), out)
|
|
|
|
def test_the_same_green_run_is_recorded_once_and_the_file_has_no_time(self) -> None:
|
|
verdicts = self.verdicts(self.perfect(), models=["claude-test-2"])
|
|
self.assertEqual(self.gate_passed(verdicts)[0], 0)
|
|
first = self.passed().read_bytes()
|
|
self.assertEqual(self.gate_passed(verdicts)[0], 0)
|
|
self.assertEqual(self.passed().read_bytes(), first)
|
|
self.assertEqual(len(json.loads(first)["passed"]), 1)
|
|
|
|
def test_another_green_run_is_added_after_the_first(self) -> None:
|
|
self.assertEqual(self.gate_passed(self.verdicts(self.perfect(), models=["claude-test-2"]))[0], 0)
|
|
self.assertEqual(self.gate_passed(self.verdicts(self.perfect(), models=["claude-test-3"],
|
|
name="v3.json"))[0], 0)
|
|
record = json.loads(self.passed().read_text(encoding="utf-8"))
|
|
self.assertEqual([e["model"] for e in record["passed"]], ["claude-test-2", "claude-test-3"])
|
|
|
|
def test_a_red_run_never_writes_the_record(self) -> None:
|
|
v = self.perfect()
|
|
v[next(i for i, lab in self.key.items() if lab == UNSUPPORTED)] = SUPPORTED
|
|
code, out = self.gate_passed(self.verdicts(v))
|
|
self.assertEqual(code, 1, out)
|
|
self.assertFalse(self.passed().exists())
|
|
code, out = self.gate_passed(self.verdicts(self.perfect(), models=None))
|
|
self.assertEqual(code, 1, out)
|
|
self.assertFalse(self.passed().exists())
|
|
self.assertEqual(self.gate_passed(self.verdicts(self.perfect()))[0], 0)
|
|
before = self.passed().read_bytes()
|
|
self.assertEqual(self.gate_passed(self.verdicts(v, models=["claude-test-9"]))[0], 1)
|
|
self.assertEqual(self.passed().read_bytes(), before)
|
|
|
|
def test_a_record_that_cannot_be_read_is_a_fail_and_is_left_as_it_was(self) -> None:
|
|
for bad in ("{", json.dumps({"schema": "okf-wiki/other/1", "passed": []}),
|
|
json.dumps({"schema": "okf-wiki/calibration-passed/1", "passed": {}})):
|
|
with self.subTest(bad=bad):
|
|
self.passed().write_text(bad, encoding="utf-8")
|
|
code, out = self.gate_passed(self.verdicts(self.perfect()))
|
|
self.assertEqual(code, 1, out)
|
|
self.assertIn("FAIL", out)
|
|
self.assertEqual(self.passed().read_text(encoding="utf-8"), bad)
|
|
|
|
|
|
class LayoutRuleTest(Base):
|
|
"""A layout that breaks K3 point 1 is refused by the gate and by check, even when the manifest follows it."""
|
|
|
|
def setUp(self) -> None:
|
|
super().setUp()
|
|
code, out = self.ws.build()
|
|
self.assertEqual(code, 0, out)
|
|
self.layout = self.ws.load("calls")
|
|
self.key = self.ws.labels()
|
|
self.green = self.ws.write_json(
|
|
"verdicts.json",
|
|
{"reader": "key", "models": ["claude-test-1"],
|
|
"verdicts": [{"id": i, "verdict": UNSUPPORTED if lab == UNSUPPORTED else SUPPORTED}
|
|
for i, lab in self.key.items()]},
|
|
)
|
|
|
|
def refused(self, needle: str) -> None:
|
|
self.ws.rewrite_calls(self.layout)
|
|
code, out = self.ws.gate(self.green)
|
|
self.assertEqual(code, 1, out)
|
|
self.assertIn(needle, out)
|
|
self.assertNotIn("GREEN", out)
|
|
code, out = self.ws.check()
|
|
self.assertEqual(code, 1, out)
|
|
self.assertIn(needle, out)
|
|
|
|
def test_the_unchanged_layout_passes(self) -> None:
|
|
self.ws.rewrite_calls(self.layout)
|
|
self.assertEqual(self.ws.gate(self.green)[0], 0)
|
|
self.assertEqual(self.ws.check()[0], 0)
|
|
|
|
def test_an_id_given_twice_is_refused(self) -> None:
|
|
self.layout["calls"].append({"call": "k999", "ids": [self.layout["calls"][1]["ids"][0]]})
|
|
self.refused("more than one place")
|
|
|
|
def test_an_id_missing_from_every_call_is_refused(self) -> None:
|
|
self.layout["calls"][-1]["ids"].pop()
|
|
self.refused("in no call")
|
|
|
|
def test_an_id_not_in_the_set_is_refused(self) -> None:
|
|
self.layout["calls"][-1]["ids"][-1] = "c999"
|
|
self.refused("not in the set")
|
|
|
|
def test_a_wrong_set_sha256_is_refused(self) -> None:
|
|
self.layout["set_sha256"] = "0" * 64
|
|
self.refused("set_sha256")
|
|
|
|
def test_a_call_over_ten_is_refused(self) -> None:
|
|
moved = self.layout["calls"][1]["ids"].pop()
|
|
self.layout["calls"][0]["ids"].append(moved)
|
|
self.refused("more than 10")
|
|
|
|
def test_a_call_size_max_over_ten_is_refused(self) -> None:
|
|
self.layout["call_size_max"] = 11
|
|
self.refused("call_size_max")
|
|
|
|
def test_a_position_without_unsupported_is_refused(self) -> None:
|
|
for c in self.layout["calls"]:
|
|
ids = c["ids"]
|
|
if self.key[ids[2]] == UNSUPPORTED:
|
|
swap = next(n for n, i in enumerate(ids) if self.key[i] != UNSUPPORTED and n != 2)
|
|
ids[2], ids[swap] = ids[swap], ids[2]
|
|
self.refused("position 3 has no unsupported claim")
|
|
|
|
|
|
class CheckBuiltSetTest(Base):
|
|
def setUp(self) -> None:
|
|
super().setUp()
|
|
code, out = self.ws.build()
|
|
self.assertEqual(code, 0, out)
|
|
|
|
def excerpts(self) -> int:
|
|
return sum(len(q["excerpts"]) for q in self.ws.load("set")["pairs"])
|
|
|
|
def test_check_passes_on_the_built_set_and_states_the_layout(self) -> None:
|
|
code, out = self.ws.check()
|
|
self.assertEqual(code, 0, out)
|
|
n = self.excerpts()
|
|
self.assertIn(f"excerpt lines correct: {n} of {n}", out)
|
|
self.assertIn("pairs: 120 (20 unsupported, 6 supported, 94 filler-supported)", out)
|
|
self.assertIn("unsupported per call: 1 in 10 calls, 5 in 2 calls", out)
|
|
self.assertIn("PASS", out)
|
|
|
|
def test_more_unsupported_than_supported_passes_when_the_manifest_says_so(self) -> None:
|
|
self.assertEqual(self.ws.check()[0], 0)
|
|
manifest = self.ws.load("manifest")
|
|
manifest["counts"]["unsupported"] = 6
|
|
self.ws.paths()["manifest"].write_text(json.dumps(manifest), encoding="utf-8")
|
|
code, out = self.ws.check()
|
|
self.assertEqual(code, 1, out)
|
|
self.assertIn("manifest", out)
|
|
|
|
def test_check_catches_a_wrong_line(self) -> None:
|
|
data = self.ws.load("set")
|
|
data["pairs"][0]["excerpts"][0]["line"] += 1
|
|
self.ws.paths()["set"].write_text(json.dumps(data), encoding="utf-8")
|
|
code, out = self.ws.check()
|
|
self.assertEqual(code, 1)
|
|
n = self.excerpts()
|
|
self.assertIn(f"excerpt lines correct: {n - 1} of {n}", out)
|
|
|
|
def test_check_of_a_built_set_needs_the_layout_and_the_manifest(self) -> None:
|
|
p = self.ws.paths()
|
|
code, out = run("check", "--set", str(p["set"]), "--key", str(p["key"]), "--sources", str(self.ws.sources))
|
|
self.assertEqual(code, 1)
|
|
self.assertIn("FAIL", out)
|
|
|
|
|
|
class StubTest(Base):
|
|
def setUp(self) -> None:
|
|
super().setUp()
|
|
code, out = self.ws.build()
|
|
self.assertEqual(code, 0, out)
|
|
|
|
def stub(self, *args: str) -> tuple[int, str, dict[str, str]]:
|
|
path = self.ws.root / "stub.json"
|
|
code, out = run("stub", "--set", str(self.ws.paths()["set"]), "--out", str(path), *args)
|
|
verdicts = {}
|
|
if path.exists():
|
|
verdicts = {v["id"]: v["verdict"] for v in json.loads(path.read_text(encoding="utf-8"))["verdicts"]}
|
|
path.unlink()
|
|
return code, out, verdicts
|
|
|
|
def test_the_same_verdict_for_every_claim(self) -> None:
|
|
code, out, v = self.stub("--verdict", UNSUPPORTED)
|
|
self.assertEqual(code, 0, out)
|
|
self.assertEqual(set(v.values()), {UNSUPPORTED})
|
|
self.assertEqual(len(v), 120)
|
|
|
|
def test_the_key_gives_supported_for_every_filler(self) -> None:
|
|
code, out, v = self.stub("--key", str(self.ws.paths()["key"]))
|
|
self.assertEqual(code, 0, out)
|
|
labels = self.ws.labels()
|
|
self.assertEqual(v, {i: UNSUPPORTED if lab == UNSUPPORTED else SUPPORTED for i, lab in labels.items()})
|
|
|
|
def test_accept_at_changes_only_that_position(self) -> None:
|
|
code, out, v = self.stub("--verdict", UNSUPPORTED, "--calls", str(self.ws.paths()["calls"]), "--accept-at", "4")
|
|
self.assertEqual(code, 0, out)
|
|
pos = self.ws.positions()
|
|
self.assertEqual({i for i, val in v.items() if val == SUPPORTED}, {i for i, n in pos.items() if n == 4})
|
|
|
|
def test_accept_at_needs_the_layout_and_a_position_from_one_to_ten(self) -> None:
|
|
self.assertEqual(self.stub("--verdict", UNSUPPORTED, "--accept-at", "4")[0], 1)
|
|
for bad in ("0", "11"):
|
|
code, out, v = self.stub("--verdict", UNSUPPORTED, "--calls", str(self.ws.paths()["calls"]), "--accept-at", bad)
|
|
self.assertEqual(code, 1, bad)
|
|
|
|
def test_verdict_and_key_exclude_each_other(self) -> None:
|
|
self.assertNotEqual(self.stub("--verdict", UNSUPPORTED, "--key", str(self.ws.paths()["key"]))[0], 0)
|
|
self.assertNotEqual(self.stub()[0], 0)
|
|
|
|
|
|
class NoModelCallTest(unittest.TestCase):
|
|
def test_tool_imports_no_network_or_model_client(self) -> None:
|
|
text = (REPO_ROOT / "gates" / "calibration.py").read_text(encoding="utf-8")
|
|
for banned in ("anthropic", "openai", "urllib", "http.client", "requests", "subprocess", "socket"):
|
|
self.assertNotIn(f"import {banned}", text)
|
|
self.assertNotIn(f"from {banned}", text)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|