First public release, published as one snapshot. What changed between versions is in CHANGELOG.md. Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
790 lines
38 KiB
Python
790 lines
38 KiB
Python
"""Tests for gates/answerset.py: the question set for criterion 4 on collection 1 (contract K5 point 1).
|
|
|
|
check, inside, split, gate and stub on synthetic K1 roots about bread and gardens. No corpus text is used
|
|
(CLAUDE.md rule 3).
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import contextlib
|
|
import copy
|
|
import hashlib
|
|
import io
|
|
import json
|
|
import random
|
|
import sys
|
|
import tempfile
|
|
import unittest
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
|
|
|
import answerset # noqa: E402
|
|
|
|
REPO_ROOT = Path(__file__).resolve().parents[2]
|
|
|
|
RISING = "kilde/docs-bread-rising.md"
|
|
OVEN = "kilde/docs-bread-oven.md"
|
|
SOIL = "kilde/docs-garden-soil.md"
|
|
WATER = "kilde/docs-garden-water.md"
|
|
FROST = "kilde/docs-garden-frost.md"
|
|
RELEASE = "kilde/release-v1.2.0.md"
|
|
|
|
TEXTS: dict[str, tuple[str, bool]] = {
|
|
RISING: ("# Rising\n\nThe dough rises for 90 minutes at room temperature.\n"
|
|
"Use the `--slow-proof` flag for a cold rise overnight.\n", True),
|
|
OVEN: ("# Oven\n\nBake the loaf at 230 degrees with the lid on.\nThe crust does not form without steam.\n", True),
|
|
SOIL: ("# Soil\n\nGarden beds need compost every spring.\nSet `SOIL_MODE=loam` for heavy clay.\n", True),
|
|
WATER: ("# Water\n\nWater the tomatoes twice a week in July.\nThe drip line is called hose-9.\n", True),
|
|
FROST: ("# Frost\n\nCover the seedlings when frost is forecast below 2 degrees.\n", True),
|
|
RELEASE: ("# 1.2.0\n\n- Added the `--slow-proof` flag for cold rises.\n- The oven timer no longer resets on restart.\n",
|
|
False),
|
|
}
|
|
PART_PATHS = (RISING, OVEN, SOIL, WATER)
|
|
|
|
SPAN = {
|
|
RISING: "The dough rises for 90 minutes at room temperature.",
|
|
OVEN: "Bake the loaf at 230 degrees with the lid on.",
|
|
SOIL: "Set `SOIL_MODE=loam` for heavy clay.",
|
|
WATER: "The drip line is called hose-9.",
|
|
FROST: "Cover the seedlings when frost is forecast below 2 degrees.",
|
|
RELEASE: "Added the `--slow-proof` flag for cold rises.",
|
|
}
|
|
KEY = {RISING: "90 minutes", OVEN: "230 degrees", SOIL: "SOIL_MODE=loam", WATER: "hose-9", FROST: "seedlings",
|
|
RELEASE: "--slow-proof"}
|
|
|
|
|
|
def dump(data: Any) -> bytes:
|
|
return (json.dumps(data, indent=1, sort_keys=True, ensure_ascii=False) + "\n").encode("utf-8")
|
|
|
|
|
|
def sha(data: bytes) -> str:
|
|
return hashlib.sha256(data).hexdigest()
|
|
|
|
|
|
def write_root(root: Path, paths: tuple[str, ...] | None = None, frozen_at: str = "2026-10-08T12:00:00Z",
|
|
subset_of: dict[str, str] | None = None) -> Path:
|
|
"""A K1 root: kilde/ files and source-manifest.json, written as contract K1 says."""
|
|
sources = []
|
|
for path in sorted(paths if paths is not None else TEXTS):
|
|
text, main = TEXTS[path]
|
|
data = text.encode("utf-8")
|
|
target = root / path
|
|
target.parent.mkdir(parents=True, exist_ok=True)
|
|
target.write_bytes(data)
|
|
kind = "release-notes" if path.startswith("kilde/release-") else "docs"
|
|
entry: dict[str, Any] = {"bytes": len(data), "kind": kind, "lines": len(text.splitlines()),
|
|
"main_topic": main, "path": path, "sha256": sha(data),
|
|
"url": "https://example.invalid/" + path}
|
|
if kind == "release-notes":
|
|
entry["version"] = "1.2.0"
|
|
sources.append(entry)
|
|
manifest = {"collection": "bread-and-garden", "frozen_at": frozen_at, "schema": "okf-wiki/source-manifest/1",
|
|
"sources": sources, "subset_of": subset_of}
|
|
root.mkdir(parents=True, exist_ok=True)
|
|
(root / "source-manifest.json").write_bytes(dump(manifest))
|
|
return root
|
|
|
|
|
|
def root_ref(root: Path) -> dict[str, str]:
|
|
data = (root / "source-manifest.json").read_bytes()
|
|
return {"frozen_at": json.loads(data)["frozen_at"], "manifest_sha256": sha(data)}
|
|
|
|
|
|
def question(qid: str, paths: tuple[str, ...], group: str) -> dict[str, Any]:
|
|
return {
|
|
"id": qid,
|
|
"question": "How do the synthetic notes about bread and gardens fit together?",
|
|
"group": group,
|
|
"evidence": [{"path": p, "span": SPAN[p]} for p in paths],
|
|
"answer_keys": [{"path": p, "text": KEY[p]} for p in paths],
|
|
}
|
|
|
|
|
|
def q23(qid: str = "q01") -> dict[str, Any]:
|
|
return question(qid, (RISING, OVEN), "docs-2-3")
|
|
|
|
|
|
def q4(qid: str = "q02") -> dict[str, Any]:
|
|
return question(qid, (RISING, OVEN, SOIL, WATER), "docs-4")
|
|
|
|
|
|
def qhist(qid: str = "q03") -> dict[str, Any]:
|
|
return question(qid, (RISING, RELEASE), "history")
|
|
|
|
|
|
def make_set(root: Path, questions: list[dict[str, Any]]) -> dict[str, Any]:
|
|
return {"schema": "okf-wiki/answer-set/1", "sources": root_ref(root), "questions": questions}
|
|
|
|
|
|
def run(argv: list[str]) -> tuple[int, str]:
|
|
out, err = io.StringIO(), io.StringIO()
|
|
with contextlib.redirect_stdout(out), contextlib.redirect_stderr(err):
|
|
code = answerset.main(argv)
|
|
return code, out.getvalue() + err.getvalue()
|
|
|
|
|
|
class Base(unittest.TestCase):
|
|
def setUp(self) -> None:
|
|
self._tmp = tempfile.TemporaryDirectory()
|
|
self.tmp = Path(self._tmp.name)
|
|
self.root = write_root(self.tmp / "copy")
|
|
|
|
def tearDown(self) -> None:
|
|
self._tmp.cleanup()
|
|
|
|
def write_set(self, questions: list[dict[str, Any]], name: str = "set.json", root: Path | None = None) -> Path:
|
|
path = self.tmp / name
|
|
path.write_bytes(dump(make_set(root or self.root, questions)))
|
|
return path
|
|
|
|
def check(self, questions: list[dict[str, Any]], root: Path | None = None) -> tuple[int, str]:
|
|
return run(["check", "--set", str(self.write_set(questions, root=root)), "--sources", str(root or self.root)])
|
|
|
|
|
|
class CheckTest(Base):
|
|
def test_a_good_set_passes_with_the_groups_counted(self) -> None:
|
|
code, out = self.check([q23(), q4(), qhist()])
|
|
self.assertEqual(code, 0, out)
|
|
self.assertIn("questions: 3", out)
|
|
self.assertIn("groups: docs-2-3 1, docs-4 1, history 1", out)
|
|
self.assertIn("spans verbatim: 8 of 8", out)
|
|
self.assertIn("questions with >= 2 distinct paths: 3 of 3", out)
|
|
self.assertIn("questions whose group matches K1 and the path count: 3 of 3", out)
|
|
self.assertIn("questions with a key in every path's spans: 3 of 3", out)
|
|
self.assertIn("answer keys that are wording: 0 of 8", out)
|
|
self.assertTrue(out.rstrip().endswith("PASS"), out)
|
|
|
|
def test_an_empty_set_fails(self) -> None:
|
|
code, out = self.check([])
|
|
self.assertEqual(code, 1)
|
|
self.assertIn("FAIL", out)
|
|
|
|
def test_a_span_that_does_not_stand_fails(self) -> None:
|
|
q = q23()
|
|
q["evidence"][0]["span"] = "The dough rises for 80 minutes at room temperature."
|
|
code, out = self.check([q, q4()])
|
|
self.assertEqual(code, 1)
|
|
self.assertIn("spans verbatim: 5 of 6", out)
|
|
self.assertIn("FAIL", out)
|
|
|
|
def test_a_span_must_stand_byte_for_byte(self) -> None:
|
|
q = q23()
|
|
q["evidence"][0]["span"] = "The dough rises for 90 minutes at room temperature."
|
|
code, out = self.check([q])
|
|
self.assertEqual(code, 1)
|
|
self.assertIn("spans verbatim: 1 of 2", out)
|
|
|
|
def test_a_question_with_one_path_fails(self) -> None:
|
|
q = question("q09", (RISING,), "docs-2-3")
|
|
code, out = self.check([q23(), q])
|
|
self.assertEqual(code, 1)
|
|
self.assertIn("questions with >= 2 distinct paths: 1 of 2", out)
|
|
|
|
def test_two_spans_in_one_file_are_one_path(self) -> None:
|
|
q = q23()
|
|
q["evidence"] = [{"path": RISING, "span": SPAN[RISING]},
|
|
{"path": RISING, "span": "Use the `--slow-proof` flag for a cold rise overnight."}]
|
|
q["answer_keys"] = [{"path": RISING, "text": "90 minutes"}]
|
|
code, out = self.check([q])
|
|
self.assertEqual(code, 1)
|
|
self.assertIn("questions with >= 2 distinct paths: 0 of 1", out)
|
|
|
|
def test_history_is_set_by_a_source_without_a_main_topic(self) -> None:
|
|
q = qhist()
|
|
q["group"] = "docs-2-3"
|
|
code, out = self.check([q])
|
|
self.assertEqual(code, 1)
|
|
self.assertIn("questions whose group matches K1 and the path count: 0 of 1", out)
|
|
|
|
def test_four_paths_are_docs_4(self) -> None:
|
|
q = q4()
|
|
q["group"] = "docs-2-3"
|
|
code, out = self.check([q])
|
|
self.assertEqual(code, 1)
|
|
self.assertIn("questions whose group matches K1 and the path count: 0 of 1", out)
|
|
|
|
def test_three_paths_are_docs_2_3(self) -> None:
|
|
q = question("q05", (RISING, OVEN, SOIL), "docs-4")
|
|
self.assertEqual(self.check([q])[0], 1)
|
|
q["group"] = "docs-2-3"
|
|
self.assertEqual(self.check([q])[0], 0)
|
|
|
|
def test_an_unknown_group_fails(self) -> None:
|
|
q = q23()
|
|
q["group"] = "lookup"
|
|
self.assertEqual(self.check([q])[0], 1)
|
|
|
|
def test_a_wording_key_fails(self) -> None:
|
|
q = q23()
|
|
q["answer_keys"][0]["text"] = "rises for 90 minutes"
|
|
code, out = self.check([q])
|
|
self.assertEqual(code, 1)
|
|
self.assertIn("answer keys that are wording: 1 of 2", out)
|
|
|
|
def test_a_negated_key_must_stand_negated(self) -> None:
|
|
q = q23()
|
|
q["answer_keys"][0]["negated"] = True
|
|
code, out = self.check([q])
|
|
self.assertEqual(code, 1)
|
|
self.assertIn("negated keys negated in their span: 0 of 1", out)
|
|
|
|
def test_a_negated_key_that_stands_negated_passes(self) -> None:
|
|
q = q23()
|
|
q["evidence"][1]["span"] = "The crust does not form without steam."
|
|
q["answer_keys"][1] = {"path": OVEN, "text": "form", "negated": True}
|
|
code, out = self.check([q])
|
|
self.assertEqual(code, 0, out)
|
|
self.assertIn("negated keys negated in their span: 1 of 1", out)
|
|
|
|
def test_a_path_without_a_key_in_its_span_fails(self) -> None:
|
|
q = q23()
|
|
q["answer_keys"] = [{"path": RISING, "text": "90 minutes"}]
|
|
code, out = self.check([q])
|
|
self.assertEqual(code, 1)
|
|
self.assertIn("questions with a key in every path's spans: 0 of 1", out)
|
|
|
|
def test_a_key_that_stands_only_in_another_path_fails(self) -> None:
|
|
q = q23()
|
|
q["answer_keys"][1] = {"path": OVEN, "text": "90 minutes"}
|
|
code, out = self.check([q])
|
|
self.assertEqual(code, 1)
|
|
self.assertIn("questions with a key in every path's spans: 0 of 1", out)
|
|
|
|
def test_a_path_that_k1_does_not_list_fails(self) -> None:
|
|
q = q23()
|
|
q["evidence"][1]["path"] = "kilde/docs-bread-crumb.md"
|
|
q["answer_keys"][1]["path"] = "kilde/docs-bread-crumb.md"
|
|
code, out = self.check([q])
|
|
self.assertEqual(code, 1)
|
|
self.assertIn("error:", out)
|
|
|
|
def test_a_file_that_differs_from_k1_fails(self) -> None:
|
|
(self.root / OVEN).write_text(TEXTS[OVEN][0] + "Extra line.\n", encoding="utf-8")
|
|
code, out = self.check([q23()])
|
|
self.assertEqual(code, 1)
|
|
self.assertIn("sha256", out)
|
|
|
|
def test_the_set_must_name_the_root_it_was_made_from(self) -> None:
|
|
other = write_root(self.tmp / "other", frozen_at="2026-10-09T12:00:00Z")
|
|
path = self.write_set([q23()], root=other)
|
|
code, out = run(["check", "--set", str(path), "--sources", str(self.root)])
|
|
self.assertEqual(code, 1)
|
|
self.assertIn("manifest_sha256", out)
|
|
|
|
def test_a_root_without_k1_cannot_run(self) -> None:
|
|
path = self.write_set([q23()])
|
|
(self.root / "source-manifest.json").unlink()
|
|
code, out = run(["check", "--set", str(path), "--sources", str(self.root)])
|
|
self.assertEqual(code, 1)
|
|
self.assertIn("FAIL", out)
|
|
|
|
def test_a_wrong_schema_cannot_run(self) -> None:
|
|
data = make_set(self.root, [q23()])
|
|
data["schema"] = "okf-wiki/multidoc-set/1"
|
|
path = self.tmp / "wrong.json"
|
|
path.write_bytes(dump(data))
|
|
self.assertEqual(run(["check", "--set", str(path), "--sources", str(self.root)])[0], 1)
|
|
|
|
def test_duplicate_ids_fail(self) -> None:
|
|
self.assertEqual(self.check([q23("q01"), q4("q01")])[0], 1)
|
|
|
|
|
|
class InsideTest(Base):
|
|
def setUp(self) -> None:
|
|
super().setUp()
|
|
ref = root_ref(self.root)
|
|
self.part = write_root(self.tmp / "part", PART_PATHS, frozen_at="2026-10-08T19:00:00Z", subset_of=ref)
|
|
|
|
def test_counts_the_questions_with_every_span_in_the_part_root(self) -> None:
|
|
path = self.write_set([q23(), q4(), qhist()])
|
|
code, out = run(["inside", "--set", str(path), "--part-root", str(self.part)])
|
|
self.assertEqual(code, 0, out)
|
|
self.assertIn("questions with every span in the part root: 2 of 3", out)
|
|
|
|
def test_require_all_fails_unless_every_question_is_inside(self) -> None:
|
|
path = self.write_set([q23(), qhist()])
|
|
code, out = run(["inside", "--set", str(path), "--part-root", str(self.part), "--require-all"])
|
|
self.assertEqual(code, 1)
|
|
self.assertIn("1 of 2", out)
|
|
inside = self.write_set([q23(), q4()], name="inside.json", root=self.part)
|
|
code, out = run(["inside", "--set", str(inside), "--part-root", str(self.part), "--require-all"])
|
|
self.assertEqual(code, 0, out)
|
|
self.assertIn("2 of 2", out)
|
|
|
|
def test_a_span_that_is_not_in_the_part_file_is_outside(self) -> None:
|
|
(self.part / OVEN).write_text("# Oven\n\nSomething else entirely.\n", encoding="utf-8")
|
|
path = self.write_set([q23()])
|
|
code, out = run(["inside", "--set", str(path), "--part-root", str(self.part)])
|
|
self.assertIn("0 of 1", out)
|
|
|
|
|
|
class SplitAndGateBase(Base):
|
|
"""Eight questions (4 docs-2-3, 2 docs-4, 2 history), a part of two, split with seed 7, 4 held out."""
|
|
|
|
def setUp(self) -> None:
|
|
super().setUp()
|
|
self.part_root = write_root(self.tmp / "part", PART_PATHS, frozen_at="2026-10-08T19:00:00Z",
|
|
subset_of=root_ref(self.root))
|
|
questions = ([q23(f"q{i:02d}") for i in (1, 2, 3, 4)] + [q4(f"q{i:02d}") for i in (5, 6)]
|
|
+ [qhist(f"q{i:02d}") for i in (7, 8)])
|
|
for n, q in enumerate(questions):
|
|
q["question"] = f"Synthetic question number {n} about bread and gardens?"
|
|
self.all = self.write_set(questions, name="all.json")
|
|
self.part = self.write_set([q23("p01"), q4("p02")], name="part.json", root=self.part_root)
|
|
self.out = self.tmp / "out"
|
|
|
|
def split(self, held: Path, seed: int = 7, out: Path | None = None) -> tuple[int, str]:
|
|
out = out or self.out
|
|
return run(["split", "--set", str(self.all), "--sources", str(self.root), "--seed", str(seed),
|
|
"--held-out-count", "4", "--set-out", str(out / "set.json"), "--set-label", "set.json",
|
|
"--held-out", str(held), "--part", str(self.part), "--part-sources", str(self.part_root),
|
|
"--part-label", "part.json", "--manifest", str(out / "manifest.json")])
|
|
|
|
|
|
class SplitTest(SplitAndGateBase):
|
|
def test_split_is_stratified_by_group_and_renumbers_both_parts(self) -> None:
|
|
held = self.tmp / "outside" / "held-out.json"
|
|
code, out = self.split(held)
|
|
self.assertEqual(code, 0, out)
|
|
dev = json.loads((self.out / "set.json").read_text(encoding="utf-8"))
|
|
kept = json.loads(held.read_text(encoding="utf-8"))
|
|
self.assertEqual([q["id"] for q in dev["questions"]], ["d01", "d02", "d03", "d04"])
|
|
self.assertEqual([q["id"] for q in kept["questions"]], ["h01", "h02", "h03", "h04"])
|
|
groups = sorted(q["group"] for q in kept["questions"])
|
|
self.assertEqual(groups, ["docs-2-3", "docs-2-3", "docs-4", "history"])
|
|
self.assertEqual(dev["sources"], kept["sources"])
|
|
self.assertEqual(dev["schema"], "okf-wiki/answer-set/1")
|
|
|
|
def test_the_manifest_holds_seed_counts_and_sha256_only(self) -> None:
|
|
held = self.tmp / "outside" / "held-out.json"
|
|
self.assertEqual(self.split(held)[0], 0)
|
|
manifest = json.loads((self.out / "manifest.json").read_text(encoding="utf-8"))
|
|
self.assertEqual(manifest["schema"], "okf-wiki/answer-set-manifest/1")
|
|
self.assertEqual(manifest["seed"], 7)
|
|
self.assertEqual(manifest["set"]["sha256"], sha((self.out / "set.json").read_bytes()))
|
|
self.assertEqual(manifest["set"]["file"], "set.json")
|
|
self.assertEqual(manifest["held_out"]["sha256"], sha(held.read_bytes()))
|
|
self.assertEqual(manifest["part"]["sha256"], sha(self.part.read_bytes()))
|
|
self.assertEqual(manifest["part"]["file"], "part.json")
|
|
self.assertEqual(manifest["groups"]["held_out"], {"docs-2-3": 2, "docs-4": 1, "history": 1})
|
|
self.assertEqual(manifest["groups"]["set"], {"docs-2-3": 2, "docs-4": 1, "history": 1})
|
|
self.assertEqual(manifest["groups"]["part"], {"docs-2-3": 1, "docs-4": 1, "history": 0})
|
|
self.assertEqual(manifest["sources"]["copy"], root_ref(self.root))
|
|
self.assertEqual(manifest["sources"]["part"], root_ref(self.part_root))
|
|
text = (self.out / "manifest.json").read_text(encoding="utf-8")
|
|
self.assertNotIn("bread", text)
|
|
self.assertNotIn(SPAN[RISING], text)
|
|
|
|
def test_two_runs_with_one_seed_are_byte_identical(self) -> None:
|
|
first, second = self.tmp / "a" / "held.json", self.tmp / "b" / "held.json"
|
|
self.assertEqual(self.split(first, out=self.tmp / "o1")[0], 0)
|
|
self.assertEqual(self.split(second, out=self.tmp / "o2")[0], 0)
|
|
self.assertEqual(first.read_bytes(), second.read_bytes())
|
|
self.assertEqual((self.tmp / "o1" / "set.json").read_bytes(), (self.tmp / "o2" / "set.json").read_bytes())
|
|
|
|
def test_split_refuses_to_overwrite_the_held_out_file(self) -> None:
|
|
held = self.tmp / "outside" / "held-out.json"
|
|
self.assertEqual(self.split(held)[0], 0)
|
|
before = held.read_bytes()
|
|
code, out = self.split(held, seed=8, out=self.tmp / "again")
|
|
self.assertEqual(code, 1)
|
|
self.assertIn("exists", out)
|
|
self.assertEqual(held.read_bytes(), before)
|
|
|
|
def test_split_refuses_a_held_out_path_in_the_repository(self) -> None:
|
|
held = REPO_ROOT / "gates-local" / "answerset" / "test-held-out-never-written.json"
|
|
code, out = self.split(held)
|
|
self.assertEqual(code, 1)
|
|
self.assertIn("inside the repository", out)
|
|
self.assertFalse(held.exists())
|
|
|
|
def test_split_refuses_a_set_that_fails_the_check(self) -> None:
|
|
data = json.loads(self.all.read_text(encoding="utf-8"))
|
|
data["questions"][0]["evidence"][0]["span"] = "No such sentence stands in any synthetic file."
|
|
self.all.write_bytes(dump(data))
|
|
held = self.tmp / "outside" / "held-out.json"
|
|
self.assertEqual(self.split(held)[0], 1)
|
|
self.assertFalse(held.exists())
|
|
|
|
def test_split_refuses_a_part_with_a_span_outside_its_root(self) -> None:
|
|
self.part.write_bytes(dump(make_set(self.part_root, [q23("p01"), qhist("p02")])))
|
|
held = self.tmp / "outside" / "held-out.json"
|
|
self.assertEqual(self.split(held)[0], 1)
|
|
self.assertFalse(held.exists())
|
|
|
|
|
|
def fasit(q: dict[str, Any]) -> dict[str, Any]:
|
|
text = " ".join(("not " if k.get("negated") else "") + k["text"] + "." for k in q["answer_keys"])
|
|
return {"id": q["id"], "answer": text, "citations": [{"path": e["path"], "quote": e["span"]} for e in q["evidence"]]}
|
|
|
|
|
|
class GateBase(SplitAndGateBase):
|
|
def setUp(self) -> None:
|
|
super().setUp()
|
|
self.held = self.tmp / "outside" / "held-out.json"
|
|
code, out = self.split(self.held)
|
|
self.assertEqual(code, 0, out)
|
|
self.set = self.out / "set.json"
|
|
self.manifest = self.out / "manifest.json"
|
|
self.questions: list[dict[str, Any]] = json.loads(self.set.read_text(encoding="utf-8"))["questions"]
|
|
|
|
def answers(self, answers: list[dict[str, Any]], arm: str = "bundle", name: str = "answers.json") -> Path:
|
|
path = self.tmp / name
|
|
path.write_bytes(dump({"schema": "okf-wiki/answers/1", "arm": arm, "answers": answers}))
|
|
return path
|
|
|
|
def gate(self, answers: Path, arm: str = "bundle", set_path: Path | None = None, root: Path | None = None,
|
|
out: Path | None = None) -> tuple[int, str]:
|
|
argv = ["gate", "--set", str(set_path or self.set), "--manifest", str(self.manifest),
|
|
"--sources", str(root or self.root), "--arm", arm, "--answers", str(answers)]
|
|
return run(argv + (["--out", str(out)] if out else []))
|
|
|
|
|
|
class GateTest(GateBase):
|
|
def test_a_missing_answers_file_scores_0_red_exit_1(self) -> None:
|
|
code, out = self.gate(self.tmp / "none.json")
|
|
self.assertEqual(code, 1)
|
|
self.assertIn("correct answer and source: 0 of 4", out)
|
|
self.assertIn("fail-closed", out)
|
|
self.assertIn("RED", out)
|
|
|
|
def test_the_fasit_scores_n_of_n_green(self) -> None:
|
|
code, out = self.gate(self.answers([fasit(q) for q in self.questions]))
|
|
self.assertEqual(code, 0, out)
|
|
self.assertIn("correct answer and source: 4 of 4", out)
|
|
self.assertIn("GREEN", out)
|
|
|
|
def test_right_answers_without_citations_score_0(self) -> None:
|
|
answers = [dict(fasit(q), citations=[]) for q in self.questions]
|
|
code, out = self.gate(self.answers(answers))
|
|
self.assertEqual(code, 0, out)
|
|
self.assertIn("correct answer: 4 of 4", out)
|
|
self.assertIn("correct answer and source: 0 of 4", out)
|
|
self.assertIn("RED", out)
|
|
|
|
def test_a_citation_of_only_some_needed_paths_is_wrong(self) -> None:
|
|
answers = [fasit(q) for q in self.questions]
|
|
answers[0]["citations"] = answers[0]["citations"][:1]
|
|
code, out = self.gate(self.answers(answers))
|
|
self.assertIn("correct answer and source: 3 of 4", out)
|
|
|
|
def test_a_quote_counts_with_whitespace_collapsed_and_a_leading_slash(self) -> None:
|
|
answers = [fasit(q) for q in self.questions]
|
|
for c in answers[0]["citations"]:
|
|
c["path"] = "/" + c["path"]
|
|
c["quote"] = " " + c["quote"].replace(" ", "\n ") + " "
|
|
self.assertIn("correct answer and source: 4 of 4", self.gate(self.answers(answers))[1])
|
|
|
|
def test_a_quote_that_does_not_stand_is_wrong(self) -> None:
|
|
answers = [fasit(q) for q in self.questions]
|
|
answers[0]["citations"][0]["quote"] = "The dough rises for 80 minutes."
|
|
self.assertIn("correct answer and source: 3 of 4", self.gate(self.answers(answers))[1])
|
|
|
|
def test_a_missing_answer_counts_as_wrong(self) -> None:
|
|
code, out = self.gate(self.answers([fasit(q) for q in self.questions[1:]]))
|
|
self.assertEqual(code, 1)
|
|
self.assertIn("missing answers: 1", out)
|
|
self.assertIn("correct answer and source: 3 of 4", out)
|
|
|
|
def test_answers_for_another_arm_cannot_run(self) -> None:
|
|
path = self.answers([fasit(q) for q in self.questions], arm="filsok")
|
|
self.assertEqual(self.gate(path, arm="bundle")[0], 1)
|
|
|
|
def test_an_unknown_arm_cannot_run(self) -> None:
|
|
path = self.answers([fasit(q) for q in self.questions], arm="server")
|
|
self.assertEqual(self.gate(path, arm="server")[0], 1)
|
|
|
|
def test_a_set_whose_sha256_is_not_in_the_manifest_cannot_run(self) -> None:
|
|
changed = self.tmp / "changed.json"
|
|
changed.write_bytes(self.set.read_bytes() + b"\n")
|
|
code, out = self.gate(self.answers([fasit(q) for q in self.questions]), set_path=changed)
|
|
self.assertEqual(code, 1)
|
|
self.assertIn("sha256", out)
|
|
|
|
def test_a_root_whose_manifest_differs_cannot_run(self) -> None:
|
|
other = write_root(self.tmp / "other", frozen_at="2026-10-09T12:00:00Z")
|
|
code, out = self.gate(self.answers([fasit(q) for q in self.questions]), root=other)
|
|
self.assertEqual(code, 1)
|
|
self.assertIn("sha256", out)
|
|
|
|
def test_the_part_is_scored_against_the_part_root_only(self) -> None:
|
|
part_q = json.loads(self.part.read_text(encoding="utf-8"))["questions"]
|
|
answers = self.answers([fasit(q) for q in part_q])
|
|
code, out = self.gate(answers, set_path=self.part, root=self.part_root)
|
|
self.assertEqual(code, 0, out)
|
|
self.assertIn("set: part, 2 questions", out)
|
|
self.assertEqual(self.gate(answers, set_path=self.part, root=self.root)[0], 1)
|
|
|
|
def test_a_negated_key_needs_the_negation_in_the_answer(self) -> None:
|
|
data = json.loads(self.all.read_text(encoding="utf-8"))
|
|
for q in data["questions"]:
|
|
if q["group"] == "docs-2-3":
|
|
q["evidence"][1]["span"] = "The crust does not form without steam."
|
|
q["answer_keys"][1] = {"path": OVEN, "text": "form", "negated": True}
|
|
self.all.write_bytes(dump(data))
|
|
held = self.tmp / "outside2" / "held.json"
|
|
out_dir = self.tmp / "neg"
|
|
self.assertEqual(self.split(held, out=out_dir)[0], 0)
|
|
self.manifest = out_dir / "manifest.json"
|
|
questions = json.loads((out_dir / "set.json").read_text(encoding="utf-8"))["questions"]
|
|
answers = [fasit(x) for x in questions]
|
|
good = self.gate(self.answers(answers), set_path=out_dir / "set.json")[1]
|
|
self.assertIn("correct answer and source: 4 of 4", good)
|
|
for a, x in zip(answers, questions):
|
|
a["answer"] = " ".join(k["text"] + "." for k in x["answer_keys"])
|
|
plain = self.gate(self.answers(answers), set_path=out_dir / "set.json")[1]
|
|
self.assertEqual(sum(1 for x in questions if x["group"] == "docs-2-3"), 2)
|
|
self.assertIn("correct answer and source: 2 of 4", plain)
|
|
|
|
def test_out_writes_the_k5_results_sorted_and_byte_identical(self) -> None:
|
|
answers = [fasit(q) for q in self.questions]
|
|
answers[1]["citations"] = answers[1]["citations"][:1]
|
|
answers[2]["answer"] = "Nothing useful."
|
|
first, second = self.tmp / "r1.json", self.tmp / "r2.json"
|
|
self.gate(self.answers(list(reversed(answers))), out=first)
|
|
self.gate(self.answers(answers), out=second)
|
|
self.assertEqual(first.read_bytes(), second.read_bytes())
|
|
doc = json.loads(first.read_text(encoding="utf-8"))
|
|
self.assertEqual(doc["schema"], "okf-wiki/answer-results/1")
|
|
self.assertEqual(doc["arm"], "bundle")
|
|
self.assertEqual(doc["set_sha256"], sha(self.set.read_bytes()))
|
|
ids = [r["id"] for r in doc["results"]]
|
|
self.assertEqual(ids, sorted(ids))
|
|
by_id = {r["id"]: r for r in doc["results"]}
|
|
q1, q2 = self.questions[1], self.questions[2]
|
|
r1, r2 = by_id[q1["id"]], by_id[q2["id"]]
|
|
self.assertEqual(set(r1), {"id", "group", "correct", "keys_held", "keys", "paths_cited", "paths_needed"})
|
|
self.assertEqual((r1["correct"], r1["paths_cited"]), (False, 1))
|
|
self.assertEqual(r1["paths_needed"], len({e["path"] for e in q1["evidence"]}))
|
|
self.assertEqual((r2["correct"], r2["keys_held"], r2["keys"]), (False, 0, len(q2["answer_keys"])))
|
|
self.assertEqual(r1["group"], q1["group"])
|
|
self.assertEqual(first.read_bytes(), dump(doc))
|
|
|
|
|
|
class StubTest(GateBase):
|
|
def test_stub_fasit_scores_n_of_n_and_stub_without_citations_0(self) -> None:
|
|
fas, bare = self.tmp / "fasit.json", self.tmp / "bare.json"
|
|
self.assertEqual(run(["stub", "--set", str(self.set), "--arm", "bundle", "--kind", "fasit",
|
|
"--out", str(fas)])[0], 0)
|
|
self.assertEqual(run(["stub", "--set", str(self.set), "--arm", "bundle", "--kind", "no-citations",
|
|
"--out", str(bare)])[0], 0)
|
|
self.assertIn("correct answer and source: 4 of 4", self.gate(fas)[1])
|
|
code, out = self.gate(bare)
|
|
self.assertEqual(code, 0)
|
|
self.assertIn("correct answer: 4 of 4", out)
|
|
self.assertIn("correct answer and source: 0 of 4", out)
|
|
|
|
def test_stub_answers_are_k5_answers(self) -> None:
|
|
fas = self.tmp / "fasit.json"
|
|
run(["stub", "--set", str(self.set), "--arm", "filsok", "--kind", "fasit", "--out", str(fas)])
|
|
doc = json.loads(fas.read_text(encoding="utf-8"))
|
|
self.assertEqual((doc["schema"], doc["arm"]), ("okf-wiki/answers/1", "filsok"))
|
|
self.assertEqual(set(doc["answers"][0]["citations"][0]), {"path", "quote"})
|
|
|
|
|
|
HARD_PATHS = (RISING, OVEN, SOIL, WATER, FROST)
|
|
|
|
|
|
def q3(qid: str) -> dict[str, Any]:
|
|
return question(qid, (RISING, OVEN, FROST), "docs-2-3")
|
|
|
|
|
|
def hard_questions(three: int, four: int, two: int = 0) -> list[dict[str, Any]]:
|
|
"""`three` questions with 3 paths, `four` with 4 and `two` with 2, each with its own wording, ids k01.."""
|
|
questions = [q3("") for _ in range(three)] + [q4("") for _ in range(four)] + [q23("") for _ in range(two)]
|
|
for n, q in enumerate(questions, 1):
|
|
q["id"] = f"k{n:02d}"
|
|
q["question"] = f"Synthetic hard question number {n} about bread and gardens?"
|
|
return questions
|
|
|
|
|
|
class HardSetBase(Base):
|
|
"""The harder set (order 21): every question needs at least 3 documents and at least 20 of 30 need 4 or more.
|
|
Its root is a K1 subset root of the copy, as the first part is."""
|
|
|
|
def setUp(self) -> None:
|
|
super().setUp()
|
|
self.hard_root = write_root(self.tmp / "hard-root", HARD_PATHS, frozen_at="2026-10-08T19:00:00Z",
|
|
subset_of=root_ref(self.root))
|
|
self.out = self.tmp / "frozen"
|
|
|
|
def hard_set(self, questions: list[dict[str, Any]], name: str = "drafts.json") -> Path:
|
|
return self.write_set(questions, name=name, root=self.hard_root)
|
|
|
|
def check_hard(self, path: Path, min_paths: int = 3, min_many: int = 20) -> tuple[int, str]:
|
|
return run(["check", "--set", str(path), "--sources", str(self.hard_root), "--min-paths", str(min_paths),
|
|
"--min-many", str(min_many)])
|
|
|
|
def freeze(self, path: Path, seed: int = 11, out: Path | None = None, count: int = 30,
|
|
min_many: int = 20) -> tuple[int, str]:
|
|
out = out or self.out
|
|
return run(["freeze", "--set", str(path), "--sources", str(self.hard_root), "--seed", str(seed),
|
|
"--count", str(count), "--min-paths", "3", "--min-many", str(min_many), "--prefix", "x",
|
|
"--set-out", str(out / "hard.json"), "--set-label", "hard.json",
|
|
"--manifest", str(out / "hard.manifest.json")])
|
|
|
|
|
|
class HardCheckTest(HardSetBase):
|
|
def test_ten_with_three_and_twenty_with_four_pass_the_thresholds(self) -> None:
|
|
code, out = self.check_hard(self.hard_set(hard_questions(10, 20)))
|
|
self.assertEqual(code, 0, out)
|
|
self.assertIn("questions with >= 3 distinct paths: 30 of 30", out)
|
|
self.assertIn("questions with >= 4 distinct paths: 20 of 30 (required >= 20)", out)
|
|
self.assertIn("groups: docs-2-3 10, docs-4 20, history 0", out)
|
|
self.assertIn("PASS", out)
|
|
|
|
def test_a_question_with_two_paths_fails_min_paths_3(self) -> None:
|
|
path = self.hard_set(hard_questions(9, 20, two=1))
|
|
code, out = self.check_hard(path)
|
|
self.assertEqual(code, 1)
|
|
self.assertIn("questions with >= 3 distinct paths: 29 of 30", out)
|
|
self.assertIn("FAIL", out)
|
|
code, out = run(["check", "--set", str(path), "--sources", str(self.hard_root)])
|
|
self.assertEqual(code, 0, out)
|
|
self.assertIn("questions with >= 2 distinct paths: 30 of 30", out)
|
|
|
|
def test_nineteen_of_thirty_with_four_fail_min_many_20(self) -> None:
|
|
code, out = self.check_hard(self.hard_set(hard_questions(11, 19)))
|
|
self.assertEqual(code, 1)
|
|
self.assertIn("questions with >= 4 distinct paths: 19 of 30 (required >= 20)", out)
|
|
self.assertIn("FAIL", out)
|
|
|
|
def test_the_default_check_prints_no_many_line(self) -> None:
|
|
code, out = run(["check", "--set", str(self.hard_set(hard_questions(10, 20))), "--sources",
|
|
str(self.hard_root)])
|
|
self.assertEqual(code, 0, out)
|
|
self.assertNotIn("required >=", out)
|
|
|
|
|
|
class FreezeTest(HardSetBase):
|
|
def test_freeze_writes_the_set_and_a_manifest_with_seed_counts_sha256_and_roots_only(self) -> None:
|
|
code, out = self.freeze(self.hard_set(hard_questions(10, 20)))
|
|
self.assertEqual(code, 0, out)
|
|
frozen = (self.out / "hard.json").read_bytes()
|
|
manifest = json.loads((self.out / "hard.manifest.json").read_text(encoding="utf-8"))
|
|
self.assertEqual(manifest["schema"], "okf-wiki/answer-set-manifest/1")
|
|
self.assertEqual(manifest["seed"], 11)
|
|
self.assertEqual(manifest["rules"], {"min_paths": 3, "many_paths": 4, "min_many": 20})
|
|
self.assertEqual(manifest["part"], {"file": "hard.json", "sha256": sha(frozen), "questions": 30})
|
|
self.assertEqual(manifest["groups"], {"part": {"docs-2-3": 10, "docs-4": 20, "history": 0}})
|
|
self.assertEqual(manifest["sources"], {"part": root_ref(self.hard_root), "copy": root_ref(self.root)})
|
|
self.assertNotIn("set", manifest)
|
|
self.assertNotIn("held_out", manifest)
|
|
text = (self.out / "hard.manifest.json").read_text(encoding="utf-8")
|
|
self.assertNotIn("bread", text)
|
|
self.assertNotIn(SPAN[RISING], text)
|
|
data = json.loads(frozen)
|
|
self.assertEqual(data["sources"], root_ref(self.hard_root))
|
|
self.assertEqual(data["schema"], "okf-wiki/answer-set/1")
|
|
|
|
def test_freeze_renumbers_in_a_seeded_order_and_is_byte_identical(self) -> None:
|
|
drafts = self.hard_set(hard_questions(10, 20))
|
|
self.assertEqual(self.freeze(drafts, out=self.tmp / "o1")[0], 0)
|
|
self.assertEqual(self.freeze(drafts, out=self.tmp / "o2")[0], 0)
|
|
first = (self.tmp / "o1" / "hard.json").read_bytes()
|
|
self.assertEqual(first, (self.tmp / "o2" / "hard.json").read_bytes())
|
|
questions = json.loads(first)["questions"]
|
|
self.assertEqual([q["id"] for q in questions], [f"x{n:02d}" for n in range(1, 31)])
|
|
order = sorted(f"k{n:02d}" for n in range(1, 31))
|
|
random.Random(11).shuffle(order)
|
|
wording = {f"k{n:02d}": f"Synthetic hard question number {n} about bread and gardens?" for n in range(1, 31)}
|
|
self.assertEqual([q["question"] for q in questions], [wording[i] for i in order])
|
|
self.assertEqual(self.freeze(drafts, seed=12, out=self.tmp / "o3")[0], 0)
|
|
other = json.loads((self.tmp / "o3" / "hard.json").read_text(encoding="utf-8"))["questions"]
|
|
self.assertNotEqual([q["question"] for q in other], [q["question"] for q in questions])
|
|
self.assertEqual(sorted(q["question"] for q in other), sorted(q["question"] for q in questions))
|
|
|
|
def test_freeze_refuses_a_set_that_fails_the_thresholds(self) -> None:
|
|
code, out = self.freeze(self.hard_set(hard_questions(11, 19)))
|
|
self.assertEqual(code, 1)
|
|
self.assertIn("19 of 30", out)
|
|
self.assertFalse((self.out / "hard.json").exists())
|
|
self.assertFalse((self.out / "hard.manifest.json").exists())
|
|
|
|
def test_freeze_refuses_a_set_with_another_count(self) -> None:
|
|
code, out = self.freeze(self.hard_set(hard_questions(9, 20)))
|
|
self.assertEqual(code, 1)
|
|
self.assertIn("29", out)
|
|
self.assertFalse((self.out / "hard.json").exists())
|
|
|
|
def test_freeze_refuses_a_set_that_fails_the_check(self) -> None:
|
|
questions = hard_questions(10, 20)
|
|
questions[0]["evidence"][0]["span"] = "No such sentence stands in any synthetic file."
|
|
self.assertEqual(self.freeze(self.hard_set(questions))[0], 1)
|
|
self.assertFalse((self.out / "hard.manifest.json").exists())
|
|
|
|
def test_freeze_never_overwrites_a_frozen_set_or_manifest(self) -> None:
|
|
drafts = self.hard_set(hard_questions(10, 20))
|
|
self.assertEqual(self.freeze(drafts)[0], 0)
|
|
before = ((self.out / "hard.json").read_bytes(), (self.out / "hard.manifest.json").read_bytes())
|
|
code, out = self.freeze(drafts, seed=12)
|
|
self.assertEqual(code, 1)
|
|
self.assertIn("exists", out)
|
|
self.assertEqual(before, ((self.out / "hard.json").read_bytes(), (self.out / "hard.manifest.json").read_bytes()))
|
|
(self.out / "hard.manifest.json").unlink()
|
|
code, out = self.freeze(drafts, seed=12)
|
|
self.assertEqual(code, 1)
|
|
self.assertIn("exists", out)
|
|
self.assertEqual((self.out / "hard.json").read_bytes(), before[0])
|
|
|
|
|
|
class HardGateTest(HardSetBase):
|
|
def setUp(self) -> None:
|
|
super().setUp()
|
|
code, out = self.freeze(self.hard_set(hard_questions(10, 20)))
|
|
self.assertEqual(code, 0, out)
|
|
self.set = self.out / "hard.json"
|
|
self.manifest = self.out / "hard.manifest.json"
|
|
|
|
def gate(self, answers: Path, root: Path | None = None) -> tuple[int, str]:
|
|
return run(["gate", "--set", str(self.set), "--manifest", str(self.manifest), "--sources",
|
|
str(root or self.hard_root), "--arm", "filsok", "--answers", str(answers)])
|
|
|
|
def stub(self, kind: str) -> Path:
|
|
path = self.tmp / f"{kind}.json"
|
|
self.assertEqual(run(["stub", "--set", str(self.set), "--arm", "filsok", "--kind", kind,
|
|
"--out", str(path)])[0], 0)
|
|
return path
|
|
|
|
def test_the_fasit_is_green_and_no_citations_red_on_the_frozen_set(self) -> None:
|
|
code, out = self.gate(self.stub("fasit"))
|
|
self.assertEqual(code, 0, out)
|
|
self.assertIn("set: part, 30 questions", out)
|
|
self.assertIn("correct answer and source: 30 of 30", out)
|
|
self.assertIn("GREEN", out)
|
|
code, out = self.gate(self.stub("no-citations"))
|
|
self.assertEqual(code, 0, out)
|
|
self.assertIn("correct answer: 30 of 30", out)
|
|
self.assertIn("correct answer and source: 0 of 30", out)
|
|
self.assertIn("RED", out)
|
|
|
|
def test_no_answers_score_0_of_30_red_exit_1(self) -> None:
|
|
code, out = self.gate(self.tmp / "none.json")
|
|
self.assertEqual(code, 1)
|
|
self.assertIn("correct answer and source: 0 of 30", out)
|
|
self.assertIn("RED", out)
|
|
|
|
def test_the_frozen_set_is_judged_against_its_own_root_only(self) -> None:
|
|
code, out = self.gate(self.stub("fasit"), root=self.root)
|
|
self.assertEqual(code, 1)
|
|
self.assertIn("sha256", out)
|
|
|
|
|
|
class DumpTest(unittest.TestCase):
|
|
def test_dump_is_the_k5_form(self) -> None:
|
|
data = {"b": [1, {"z": "æ", "a": None}], "a": True}
|
|
self.assertEqual(answerset.dump(data), dump(data))
|
|
|
|
def test_group_of_follows_k1_and_the_path_count(self) -> None:
|
|
main = {RISING: True, OVEN: True, SOIL: True, WATER: True, RELEASE: False}
|
|
self.assertEqual(answerset.group_of([RISING, OVEN], main), "docs-2-3")
|
|
self.assertEqual(answerset.group_of([RISING, OVEN, SOIL], main), "docs-2-3")
|
|
self.assertEqual(answerset.group_of([RISING, OVEN, SOIL, WATER], main), "docs-4")
|
|
self.assertEqual(answerset.group_of([RISING, RELEASE], main), "history")
|
|
self.assertEqual(answerset.group_of([RISING, OVEN, SOIL, RELEASE], main), "history")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|