okf-wiki/gates/tests/test_answerset.py
Kjell Tore Guttormsen 0fd86a23a3
okf-wiki 1.1.1
First public release, published as one snapshot. What changed between
versions is in CHANGELOG.md.

Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
2026-11-14 21:03:20 +01:00

790 lines
38 KiB
Python

"""Tests for gates/answerset.py: the question set for criterion 4 on collection 1 (contract K5 point 1).
check, inside, split, gate and stub on synthetic K1 roots about bread and gardens. No corpus text is used
(CLAUDE.md rule 3).
"""
from __future__ import annotations
import contextlib
import copy
import hashlib
import io
import json
import random
import sys
import tempfile
import unittest
from pathlib import Path
from typing import Any
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
import answerset # noqa: E402
REPO_ROOT = Path(__file__).resolve().parents[2]
RISING = "kilde/docs-bread-rising.md"
OVEN = "kilde/docs-bread-oven.md"
SOIL = "kilde/docs-garden-soil.md"
WATER = "kilde/docs-garden-water.md"
FROST = "kilde/docs-garden-frost.md"
RELEASE = "kilde/release-v1.2.0.md"
TEXTS: dict[str, tuple[str, bool]] = {
RISING: ("# Rising\n\nThe dough rises for 90 minutes at room temperature.\n"
"Use the `--slow-proof` flag for a cold rise overnight.\n", True),
OVEN: ("# Oven\n\nBake the loaf at 230 degrees with the lid on.\nThe crust does not form without steam.\n", True),
SOIL: ("# Soil\n\nGarden beds need compost every spring.\nSet `SOIL_MODE=loam` for heavy clay.\n", True),
WATER: ("# Water\n\nWater the tomatoes twice a week in July.\nThe drip line is called hose-9.\n", True),
FROST: ("# Frost\n\nCover the seedlings when frost is forecast below 2 degrees.\n", True),
RELEASE: ("# 1.2.0\n\n- Added the `--slow-proof` flag for cold rises.\n- The oven timer no longer resets on restart.\n",
False),
}
PART_PATHS = (RISING, OVEN, SOIL, WATER)
SPAN = {
RISING: "The dough rises for 90 minutes at room temperature.",
OVEN: "Bake the loaf at 230 degrees with the lid on.",
SOIL: "Set `SOIL_MODE=loam` for heavy clay.",
WATER: "The drip line is called hose-9.",
FROST: "Cover the seedlings when frost is forecast below 2 degrees.",
RELEASE: "Added the `--slow-proof` flag for cold rises.",
}
KEY = {RISING: "90 minutes", OVEN: "230 degrees", SOIL: "SOIL_MODE=loam", WATER: "hose-9", FROST: "seedlings",
RELEASE: "--slow-proof"}
def dump(data: Any) -> bytes:
return (json.dumps(data, indent=1, sort_keys=True, ensure_ascii=False) + "\n").encode("utf-8")
def sha(data: bytes) -> str:
return hashlib.sha256(data).hexdigest()
def write_root(root: Path, paths: tuple[str, ...] | None = None, frozen_at: str = "2026-10-08T12:00:00Z",
subset_of: dict[str, str] | None = None) -> Path:
"""A K1 root: kilde/ files and source-manifest.json, written as contract K1 says."""
sources = []
for path in sorted(paths if paths is not None else TEXTS):
text, main = TEXTS[path]
data = text.encode("utf-8")
target = root / path
target.parent.mkdir(parents=True, exist_ok=True)
target.write_bytes(data)
kind = "release-notes" if path.startswith("kilde/release-") else "docs"
entry: dict[str, Any] = {"bytes": len(data), "kind": kind, "lines": len(text.splitlines()),
"main_topic": main, "path": path, "sha256": sha(data),
"url": "https://example.invalid/" + path}
if kind == "release-notes":
entry["version"] = "1.2.0"
sources.append(entry)
manifest = {"collection": "bread-and-garden", "frozen_at": frozen_at, "schema": "okf-wiki/source-manifest/1",
"sources": sources, "subset_of": subset_of}
root.mkdir(parents=True, exist_ok=True)
(root / "source-manifest.json").write_bytes(dump(manifest))
return root
def root_ref(root: Path) -> dict[str, str]:
data = (root / "source-manifest.json").read_bytes()
return {"frozen_at": json.loads(data)["frozen_at"], "manifest_sha256": sha(data)}
def question(qid: str, paths: tuple[str, ...], group: str) -> dict[str, Any]:
return {
"id": qid,
"question": "How do the synthetic notes about bread and gardens fit together?",
"group": group,
"evidence": [{"path": p, "span": SPAN[p]} for p in paths],
"answer_keys": [{"path": p, "text": KEY[p]} for p in paths],
}
def q23(qid: str = "q01") -> dict[str, Any]:
return question(qid, (RISING, OVEN), "docs-2-3")
def q4(qid: str = "q02") -> dict[str, Any]:
return question(qid, (RISING, OVEN, SOIL, WATER), "docs-4")
def qhist(qid: str = "q03") -> dict[str, Any]:
return question(qid, (RISING, RELEASE), "history")
def make_set(root: Path, questions: list[dict[str, Any]]) -> dict[str, Any]:
return {"schema": "okf-wiki/answer-set/1", "sources": root_ref(root), "questions": questions}
def run(argv: list[str]) -> tuple[int, str]:
out, err = io.StringIO(), io.StringIO()
with contextlib.redirect_stdout(out), contextlib.redirect_stderr(err):
code = answerset.main(argv)
return code, out.getvalue() + err.getvalue()
class Base(unittest.TestCase):
def setUp(self) -> None:
self._tmp = tempfile.TemporaryDirectory()
self.tmp = Path(self._tmp.name)
self.root = write_root(self.tmp / "copy")
def tearDown(self) -> None:
self._tmp.cleanup()
def write_set(self, questions: list[dict[str, Any]], name: str = "set.json", root: Path | None = None) -> Path:
path = self.tmp / name
path.write_bytes(dump(make_set(root or self.root, questions)))
return path
def check(self, questions: list[dict[str, Any]], root: Path | None = None) -> tuple[int, str]:
return run(["check", "--set", str(self.write_set(questions, root=root)), "--sources", str(root or self.root)])
class CheckTest(Base):
def test_a_good_set_passes_with_the_groups_counted(self) -> None:
code, out = self.check([q23(), q4(), qhist()])
self.assertEqual(code, 0, out)
self.assertIn("questions: 3", out)
self.assertIn("groups: docs-2-3 1, docs-4 1, history 1", out)
self.assertIn("spans verbatim: 8 of 8", out)
self.assertIn("questions with >= 2 distinct paths: 3 of 3", out)
self.assertIn("questions whose group matches K1 and the path count: 3 of 3", out)
self.assertIn("questions with a key in every path's spans: 3 of 3", out)
self.assertIn("answer keys that are wording: 0 of 8", out)
self.assertTrue(out.rstrip().endswith("PASS"), out)
def test_an_empty_set_fails(self) -> None:
code, out = self.check([])
self.assertEqual(code, 1)
self.assertIn("FAIL", out)
def test_a_span_that_does_not_stand_fails(self) -> None:
q = q23()
q["evidence"][0]["span"] = "The dough rises for 80 minutes at room temperature."
code, out = self.check([q, q4()])
self.assertEqual(code, 1)
self.assertIn("spans verbatim: 5 of 6", out)
self.assertIn("FAIL", out)
def test_a_span_must_stand_byte_for_byte(self) -> None:
q = q23()
q["evidence"][0]["span"] = "The dough rises for 90 minutes at room temperature."
code, out = self.check([q])
self.assertEqual(code, 1)
self.assertIn("spans verbatim: 1 of 2", out)
def test_a_question_with_one_path_fails(self) -> None:
q = question("q09", (RISING,), "docs-2-3")
code, out = self.check([q23(), q])
self.assertEqual(code, 1)
self.assertIn("questions with >= 2 distinct paths: 1 of 2", out)
def test_two_spans_in_one_file_are_one_path(self) -> None:
q = q23()
q["evidence"] = [{"path": RISING, "span": SPAN[RISING]},
{"path": RISING, "span": "Use the `--slow-proof` flag for a cold rise overnight."}]
q["answer_keys"] = [{"path": RISING, "text": "90 minutes"}]
code, out = self.check([q])
self.assertEqual(code, 1)
self.assertIn("questions with >= 2 distinct paths: 0 of 1", out)
def test_history_is_set_by_a_source_without_a_main_topic(self) -> None:
q = qhist()
q["group"] = "docs-2-3"
code, out = self.check([q])
self.assertEqual(code, 1)
self.assertIn("questions whose group matches K1 and the path count: 0 of 1", out)
def test_four_paths_are_docs_4(self) -> None:
q = q4()
q["group"] = "docs-2-3"
code, out = self.check([q])
self.assertEqual(code, 1)
self.assertIn("questions whose group matches K1 and the path count: 0 of 1", out)
def test_three_paths_are_docs_2_3(self) -> None:
q = question("q05", (RISING, OVEN, SOIL), "docs-4")
self.assertEqual(self.check([q])[0], 1)
q["group"] = "docs-2-3"
self.assertEqual(self.check([q])[0], 0)
def test_an_unknown_group_fails(self) -> None:
q = q23()
q["group"] = "lookup"
self.assertEqual(self.check([q])[0], 1)
def test_a_wording_key_fails(self) -> None:
q = q23()
q["answer_keys"][0]["text"] = "rises for 90 minutes"
code, out = self.check([q])
self.assertEqual(code, 1)
self.assertIn("answer keys that are wording: 1 of 2", out)
def test_a_negated_key_must_stand_negated(self) -> None:
q = q23()
q["answer_keys"][0]["negated"] = True
code, out = self.check([q])
self.assertEqual(code, 1)
self.assertIn("negated keys negated in their span: 0 of 1", out)
def test_a_negated_key_that_stands_negated_passes(self) -> None:
q = q23()
q["evidence"][1]["span"] = "The crust does not form without steam."
q["answer_keys"][1] = {"path": OVEN, "text": "form", "negated": True}
code, out = self.check([q])
self.assertEqual(code, 0, out)
self.assertIn("negated keys negated in their span: 1 of 1", out)
def test_a_path_without_a_key_in_its_span_fails(self) -> None:
q = q23()
q["answer_keys"] = [{"path": RISING, "text": "90 minutes"}]
code, out = self.check([q])
self.assertEqual(code, 1)
self.assertIn("questions with a key in every path's spans: 0 of 1", out)
def test_a_key_that_stands_only_in_another_path_fails(self) -> None:
q = q23()
q["answer_keys"][1] = {"path": OVEN, "text": "90 minutes"}
code, out = self.check([q])
self.assertEqual(code, 1)
self.assertIn("questions with a key in every path's spans: 0 of 1", out)
def test_a_path_that_k1_does_not_list_fails(self) -> None:
q = q23()
q["evidence"][1]["path"] = "kilde/docs-bread-crumb.md"
q["answer_keys"][1]["path"] = "kilde/docs-bread-crumb.md"
code, out = self.check([q])
self.assertEqual(code, 1)
self.assertIn("error:", out)
def test_a_file_that_differs_from_k1_fails(self) -> None:
(self.root / OVEN).write_text(TEXTS[OVEN][0] + "Extra line.\n", encoding="utf-8")
code, out = self.check([q23()])
self.assertEqual(code, 1)
self.assertIn("sha256", out)
def test_the_set_must_name_the_root_it_was_made_from(self) -> None:
other = write_root(self.tmp / "other", frozen_at="2026-10-09T12:00:00Z")
path = self.write_set([q23()], root=other)
code, out = run(["check", "--set", str(path), "--sources", str(self.root)])
self.assertEqual(code, 1)
self.assertIn("manifest_sha256", out)
def test_a_root_without_k1_cannot_run(self) -> None:
path = self.write_set([q23()])
(self.root / "source-manifest.json").unlink()
code, out = run(["check", "--set", str(path), "--sources", str(self.root)])
self.assertEqual(code, 1)
self.assertIn("FAIL", out)
def test_a_wrong_schema_cannot_run(self) -> None:
data = make_set(self.root, [q23()])
data["schema"] = "okf-wiki/multidoc-set/1"
path = self.tmp / "wrong.json"
path.write_bytes(dump(data))
self.assertEqual(run(["check", "--set", str(path), "--sources", str(self.root)])[0], 1)
def test_duplicate_ids_fail(self) -> None:
self.assertEqual(self.check([q23("q01"), q4("q01")])[0], 1)
class InsideTest(Base):
def setUp(self) -> None:
super().setUp()
ref = root_ref(self.root)
self.part = write_root(self.tmp / "part", PART_PATHS, frozen_at="2026-10-08T19:00:00Z", subset_of=ref)
def test_counts_the_questions_with_every_span_in_the_part_root(self) -> None:
path = self.write_set([q23(), q4(), qhist()])
code, out = run(["inside", "--set", str(path), "--part-root", str(self.part)])
self.assertEqual(code, 0, out)
self.assertIn("questions with every span in the part root: 2 of 3", out)
def test_require_all_fails_unless_every_question_is_inside(self) -> None:
path = self.write_set([q23(), qhist()])
code, out = run(["inside", "--set", str(path), "--part-root", str(self.part), "--require-all"])
self.assertEqual(code, 1)
self.assertIn("1 of 2", out)
inside = self.write_set([q23(), q4()], name="inside.json", root=self.part)
code, out = run(["inside", "--set", str(inside), "--part-root", str(self.part), "--require-all"])
self.assertEqual(code, 0, out)
self.assertIn("2 of 2", out)
def test_a_span_that_is_not_in_the_part_file_is_outside(self) -> None:
(self.part / OVEN).write_text("# Oven\n\nSomething else entirely.\n", encoding="utf-8")
path = self.write_set([q23()])
code, out = run(["inside", "--set", str(path), "--part-root", str(self.part)])
self.assertIn("0 of 1", out)
class SplitAndGateBase(Base):
"""Eight questions (4 docs-2-3, 2 docs-4, 2 history), a part of two, split with seed 7, 4 held out."""
def setUp(self) -> None:
super().setUp()
self.part_root = write_root(self.tmp / "part", PART_PATHS, frozen_at="2026-10-08T19:00:00Z",
subset_of=root_ref(self.root))
questions = ([q23(f"q{i:02d}") for i in (1, 2, 3, 4)] + [q4(f"q{i:02d}") for i in (5, 6)]
+ [qhist(f"q{i:02d}") for i in (7, 8)])
for n, q in enumerate(questions):
q["question"] = f"Synthetic question number {n} about bread and gardens?"
self.all = self.write_set(questions, name="all.json")
self.part = self.write_set([q23("p01"), q4("p02")], name="part.json", root=self.part_root)
self.out = self.tmp / "out"
def split(self, held: Path, seed: int = 7, out: Path | None = None) -> tuple[int, str]:
out = out or self.out
return run(["split", "--set", str(self.all), "--sources", str(self.root), "--seed", str(seed),
"--held-out-count", "4", "--set-out", str(out / "set.json"), "--set-label", "set.json",
"--held-out", str(held), "--part", str(self.part), "--part-sources", str(self.part_root),
"--part-label", "part.json", "--manifest", str(out / "manifest.json")])
class SplitTest(SplitAndGateBase):
def test_split_is_stratified_by_group_and_renumbers_both_parts(self) -> None:
held = self.tmp / "outside" / "held-out.json"
code, out = self.split(held)
self.assertEqual(code, 0, out)
dev = json.loads((self.out / "set.json").read_text(encoding="utf-8"))
kept = json.loads(held.read_text(encoding="utf-8"))
self.assertEqual([q["id"] for q in dev["questions"]], ["d01", "d02", "d03", "d04"])
self.assertEqual([q["id"] for q in kept["questions"]], ["h01", "h02", "h03", "h04"])
groups = sorted(q["group"] for q in kept["questions"])
self.assertEqual(groups, ["docs-2-3", "docs-2-3", "docs-4", "history"])
self.assertEqual(dev["sources"], kept["sources"])
self.assertEqual(dev["schema"], "okf-wiki/answer-set/1")
def test_the_manifest_holds_seed_counts_and_sha256_only(self) -> None:
held = self.tmp / "outside" / "held-out.json"
self.assertEqual(self.split(held)[0], 0)
manifest = json.loads((self.out / "manifest.json").read_text(encoding="utf-8"))
self.assertEqual(manifest["schema"], "okf-wiki/answer-set-manifest/1")
self.assertEqual(manifest["seed"], 7)
self.assertEqual(manifest["set"]["sha256"], sha((self.out / "set.json").read_bytes()))
self.assertEqual(manifest["set"]["file"], "set.json")
self.assertEqual(manifest["held_out"]["sha256"], sha(held.read_bytes()))
self.assertEqual(manifest["part"]["sha256"], sha(self.part.read_bytes()))
self.assertEqual(manifest["part"]["file"], "part.json")
self.assertEqual(manifest["groups"]["held_out"], {"docs-2-3": 2, "docs-4": 1, "history": 1})
self.assertEqual(manifest["groups"]["set"], {"docs-2-3": 2, "docs-4": 1, "history": 1})
self.assertEqual(manifest["groups"]["part"], {"docs-2-3": 1, "docs-4": 1, "history": 0})
self.assertEqual(manifest["sources"]["copy"], root_ref(self.root))
self.assertEqual(manifest["sources"]["part"], root_ref(self.part_root))
text = (self.out / "manifest.json").read_text(encoding="utf-8")
self.assertNotIn("bread", text)
self.assertNotIn(SPAN[RISING], text)
def test_two_runs_with_one_seed_are_byte_identical(self) -> None:
first, second = self.tmp / "a" / "held.json", self.tmp / "b" / "held.json"
self.assertEqual(self.split(first, out=self.tmp / "o1")[0], 0)
self.assertEqual(self.split(second, out=self.tmp / "o2")[0], 0)
self.assertEqual(first.read_bytes(), second.read_bytes())
self.assertEqual((self.tmp / "o1" / "set.json").read_bytes(), (self.tmp / "o2" / "set.json").read_bytes())
def test_split_refuses_to_overwrite_the_held_out_file(self) -> None:
held = self.tmp / "outside" / "held-out.json"
self.assertEqual(self.split(held)[0], 0)
before = held.read_bytes()
code, out = self.split(held, seed=8, out=self.tmp / "again")
self.assertEqual(code, 1)
self.assertIn("exists", out)
self.assertEqual(held.read_bytes(), before)
def test_split_refuses_a_held_out_path_in_the_repository(self) -> None:
held = REPO_ROOT / "gates-local" / "answerset" / "test-held-out-never-written.json"
code, out = self.split(held)
self.assertEqual(code, 1)
self.assertIn("inside the repository", out)
self.assertFalse(held.exists())
def test_split_refuses_a_set_that_fails_the_check(self) -> None:
data = json.loads(self.all.read_text(encoding="utf-8"))
data["questions"][0]["evidence"][0]["span"] = "No such sentence stands in any synthetic file."
self.all.write_bytes(dump(data))
held = self.tmp / "outside" / "held-out.json"
self.assertEqual(self.split(held)[0], 1)
self.assertFalse(held.exists())
def test_split_refuses_a_part_with_a_span_outside_its_root(self) -> None:
self.part.write_bytes(dump(make_set(self.part_root, [q23("p01"), qhist("p02")])))
held = self.tmp / "outside" / "held-out.json"
self.assertEqual(self.split(held)[0], 1)
self.assertFalse(held.exists())
def fasit(q: dict[str, Any]) -> dict[str, Any]:
text = " ".join(("not " if k.get("negated") else "") + k["text"] + "." for k in q["answer_keys"])
return {"id": q["id"], "answer": text, "citations": [{"path": e["path"], "quote": e["span"]} for e in q["evidence"]]}
class GateBase(SplitAndGateBase):
def setUp(self) -> None:
super().setUp()
self.held = self.tmp / "outside" / "held-out.json"
code, out = self.split(self.held)
self.assertEqual(code, 0, out)
self.set = self.out / "set.json"
self.manifest = self.out / "manifest.json"
self.questions: list[dict[str, Any]] = json.loads(self.set.read_text(encoding="utf-8"))["questions"]
def answers(self, answers: list[dict[str, Any]], arm: str = "bundle", name: str = "answers.json") -> Path:
path = self.tmp / name
path.write_bytes(dump({"schema": "okf-wiki/answers/1", "arm": arm, "answers": answers}))
return path
def gate(self, answers: Path, arm: str = "bundle", set_path: Path | None = None, root: Path | None = None,
out: Path | None = None) -> tuple[int, str]:
argv = ["gate", "--set", str(set_path or self.set), "--manifest", str(self.manifest),
"--sources", str(root or self.root), "--arm", arm, "--answers", str(answers)]
return run(argv + (["--out", str(out)] if out else []))
class GateTest(GateBase):
def test_a_missing_answers_file_scores_0_red_exit_1(self) -> None:
code, out = self.gate(self.tmp / "none.json")
self.assertEqual(code, 1)
self.assertIn("correct answer and source: 0 of 4", out)
self.assertIn("fail-closed", out)
self.assertIn("RED", out)
def test_the_fasit_scores_n_of_n_green(self) -> None:
code, out = self.gate(self.answers([fasit(q) for q in self.questions]))
self.assertEqual(code, 0, out)
self.assertIn("correct answer and source: 4 of 4", out)
self.assertIn("GREEN", out)
def test_right_answers_without_citations_score_0(self) -> None:
answers = [dict(fasit(q), citations=[]) for q in self.questions]
code, out = self.gate(self.answers(answers))
self.assertEqual(code, 0, out)
self.assertIn("correct answer: 4 of 4", out)
self.assertIn("correct answer and source: 0 of 4", out)
self.assertIn("RED", out)
def test_a_citation_of_only_some_needed_paths_is_wrong(self) -> None:
answers = [fasit(q) for q in self.questions]
answers[0]["citations"] = answers[0]["citations"][:1]
code, out = self.gate(self.answers(answers))
self.assertIn("correct answer and source: 3 of 4", out)
def test_a_quote_counts_with_whitespace_collapsed_and_a_leading_slash(self) -> None:
answers = [fasit(q) for q in self.questions]
for c in answers[0]["citations"]:
c["path"] = "/" + c["path"]
c["quote"] = " " + c["quote"].replace(" ", "\n ") + " "
self.assertIn("correct answer and source: 4 of 4", self.gate(self.answers(answers))[1])
def test_a_quote_that_does_not_stand_is_wrong(self) -> None:
answers = [fasit(q) for q in self.questions]
answers[0]["citations"][0]["quote"] = "The dough rises for 80 minutes."
self.assertIn("correct answer and source: 3 of 4", self.gate(self.answers(answers))[1])
def test_a_missing_answer_counts_as_wrong(self) -> None:
code, out = self.gate(self.answers([fasit(q) for q in self.questions[1:]]))
self.assertEqual(code, 1)
self.assertIn("missing answers: 1", out)
self.assertIn("correct answer and source: 3 of 4", out)
def test_answers_for_another_arm_cannot_run(self) -> None:
path = self.answers([fasit(q) for q in self.questions], arm="filsok")
self.assertEqual(self.gate(path, arm="bundle")[0], 1)
def test_an_unknown_arm_cannot_run(self) -> None:
path = self.answers([fasit(q) for q in self.questions], arm="server")
self.assertEqual(self.gate(path, arm="server")[0], 1)
def test_a_set_whose_sha256_is_not_in_the_manifest_cannot_run(self) -> None:
changed = self.tmp / "changed.json"
changed.write_bytes(self.set.read_bytes() + b"\n")
code, out = self.gate(self.answers([fasit(q) for q in self.questions]), set_path=changed)
self.assertEqual(code, 1)
self.assertIn("sha256", out)
def test_a_root_whose_manifest_differs_cannot_run(self) -> None:
other = write_root(self.tmp / "other", frozen_at="2026-10-09T12:00:00Z")
code, out = self.gate(self.answers([fasit(q) for q in self.questions]), root=other)
self.assertEqual(code, 1)
self.assertIn("sha256", out)
def test_the_part_is_scored_against_the_part_root_only(self) -> None:
part_q = json.loads(self.part.read_text(encoding="utf-8"))["questions"]
answers = self.answers([fasit(q) for q in part_q])
code, out = self.gate(answers, set_path=self.part, root=self.part_root)
self.assertEqual(code, 0, out)
self.assertIn("set: part, 2 questions", out)
self.assertEqual(self.gate(answers, set_path=self.part, root=self.root)[0], 1)
def test_a_negated_key_needs_the_negation_in_the_answer(self) -> None:
data = json.loads(self.all.read_text(encoding="utf-8"))
for q in data["questions"]:
if q["group"] == "docs-2-3":
q["evidence"][1]["span"] = "The crust does not form without steam."
q["answer_keys"][1] = {"path": OVEN, "text": "form", "negated": True}
self.all.write_bytes(dump(data))
held = self.tmp / "outside2" / "held.json"
out_dir = self.tmp / "neg"
self.assertEqual(self.split(held, out=out_dir)[0], 0)
self.manifest = out_dir / "manifest.json"
questions = json.loads((out_dir / "set.json").read_text(encoding="utf-8"))["questions"]
answers = [fasit(x) for x in questions]
good = self.gate(self.answers(answers), set_path=out_dir / "set.json")[1]
self.assertIn("correct answer and source: 4 of 4", good)
for a, x in zip(answers, questions):
a["answer"] = " ".join(k["text"] + "." for k in x["answer_keys"])
plain = self.gate(self.answers(answers), set_path=out_dir / "set.json")[1]
self.assertEqual(sum(1 for x in questions if x["group"] == "docs-2-3"), 2)
self.assertIn("correct answer and source: 2 of 4", plain)
def test_out_writes_the_k5_results_sorted_and_byte_identical(self) -> None:
answers = [fasit(q) for q in self.questions]
answers[1]["citations"] = answers[1]["citations"][:1]
answers[2]["answer"] = "Nothing useful."
first, second = self.tmp / "r1.json", self.tmp / "r2.json"
self.gate(self.answers(list(reversed(answers))), out=first)
self.gate(self.answers(answers), out=second)
self.assertEqual(first.read_bytes(), second.read_bytes())
doc = json.loads(first.read_text(encoding="utf-8"))
self.assertEqual(doc["schema"], "okf-wiki/answer-results/1")
self.assertEqual(doc["arm"], "bundle")
self.assertEqual(doc["set_sha256"], sha(self.set.read_bytes()))
ids = [r["id"] for r in doc["results"]]
self.assertEqual(ids, sorted(ids))
by_id = {r["id"]: r for r in doc["results"]}
q1, q2 = self.questions[1], self.questions[2]
r1, r2 = by_id[q1["id"]], by_id[q2["id"]]
self.assertEqual(set(r1), {"id", "group", "correct", "keys_held", "keys", "paths_cited", "paths_needed"})
self.assertEqual((r1["correct"], r1["paths_cited"]), (False, 1))
self.assertEqual(r1["paths_needed"], len({e["path"] for e in q1["evidence"]}))
self.assertEqual((r2["correct"], r2["keys_held"], r2["keys"]), (False, 0, len(q2["answer_keys"])))
self.assertEqual(r1["group"], q1["group"])
self.assertEqual(first.read_bytes(), dump(doc))
class StubTest(GateBase):
def test_stub_fasit_scores_n_of_n_and_stub_without_citations_0(self) -> None:
fas, bare = self.tmp / "fasit.json", self.tmp / "bare.json"
self.assertEqual(run(["stub", "--set", str(self.set), "--arm", "bundle", "--kind", "fasit",
"--out", str(fas)])[0], 0)
self.assertEqual(run(["stub", "--set", str(self.set), "--arm", "bundle", "--kind", "no-citations",
"--out", str(bare)])[0], 0)
self.assertIn("correct answer and source: 4 of 4", self.gate(fas)[1])
code, out = self.gate(bare)
self.assertEqual(code, 0)
self.assertIn("correct answer: 4 of 4", out)
self.assertIn("correct answer and source: 0 of 4", out)
def test_stub_answers_are_k5_answers(self) -> None:
fas = self.tmp / "fasit.json"
run(["stub", "--set", str(self.set), "--arm", "filsok", "--kind", "fasit", "--out", str(fas)])
doc = json.loads(fas.read_text(encoding="utf-8"))
self.assertEqual((doc["schema"], doc["arm"]), ("okf-wiki/answers/1", "filsok"))
self.assertEqual(set(doc["answers"][0]["citations"][0]), {"path", "quote"})
HARD_PATHS = (RISING, OVEN, SOIL, WATER, FROST)
def q3(qid: str) -> dict[str, Any]:
return question(qid, (RISING, OVEN, FROST), "docs-2-3")
def hard_questions(three: int, four: int, two: int = 0) -> list[dict[str, Any]]:
"""`three` questions with 3 paths, `four` with 4 and `two` with 2, each with its own wording, ids k01.."""
questions = [q3("") for _ in range(three)] + [q4("") for _ in range(four)] + [q23("") for _ in range(two)]
for n, q in enumerate(questions, 1):
q["id"] = f"k{n:02d}"
q["question"] = f"Synthetic hard question number {n} about bread and gardens?"
return questions
class HardSetBase(Base):
"""The harder set (order 21): every question needs at least 3 documents and at least 20 of 30 need 4 or more.
Its root is a K1 subset root of the copy, as the first part is."""
def setUp(self) -> None:
super().setUp()
self.hard_root = write_root(self.tmp / "hard-root", HARD_PATHS, frozen_at="2026-10-08T19:00:00Z",
subset_of=root_ref(self.root))
self.out = self.tmp / "frozen"
def hard_set(self, questions: list[dict[str, Any]], name: str = "drafts.json") -> Path:
return self.write_set(questions, name=name, root=self.hard_root)
def check_hard(self, path: Path, min_paths: int = 3, min_many: int = 20) -> tuple[int, str]:
return run(["check", "--set", str(path), "--sources", str(self.hard_root), "--min-paths", str(min_paths),
"--min-many", str(min_many)])
def freeze(self, path: Path, seed: int = 11, out: Path | None = None, count: int = 30,
min_many: int = 20) -> tuple[int, str]:
out = out or self.out
return run(["freeze", "--set", str(path), "--sources", str(self.hard_root), "--seed", str(seed),
"--count", str(count), "--min-paths", "3", "--min-many", str(min_many), "--prefix", "x",
"--set-out", str(out / "hard.json"), "--set-label", "hard.json",
"--manifest", str(out / "hard.manifest.json")])
class HardCheckTest(HardSetBase):
def test_ten_with_three_and_twenty_with_four_pass_the_thresholds(self) -> None:
code, out = self.check_hard(self.hard_set(hard_questions(10, 20)))
self.assertEqual(code, 0, out)
self.assertIn("questions with >= 3 distinct paths: 30 of 30", out)
self.assertIn("questions with >= 4 distinct paths: 20 of 30 (required >= 20)", out)
self.assertIn("groups: docs-2-3 10, docs-4 20, history 0", out)
self.assertIn("PASS", out)
def test_a_question_with_two_paths_fails_min_paths_3(self) -> None:
path = self.hard_set(hard_questions(9, 20, two=1))
code, out = self.check_hard(path)
self.assertEqual(code, 1)
self.assertIn("questions with >= 3 distinct paths: 29 of 30", out)
self.assertIn("FAIL", out)
code, out = run(["check", "--set", str(path), "--sources", str(self.hard_root)])
self.assertEqual(code, 0, out)
self.assertIn("questions with >= 2 distinct paths: 30 of 30", out)
def test_nineteen_of_thirty_with_four_fail_min_many_20(self) -> None:
code, out = self.check_hard(self.hard_set(hard_questions(11, 19)))
self.assertEqual(code, 1)
self.assertIn("questions with >= 4 distinct paths: 19 of 30 (required >= 20)", out)
self.assertIn("FAIL", out)
def test_the_default_check_prints_no_many_line(self) -> None:
code, out = run(["check", "--set", str(self.hard_set(hard_questions(10, 20))), "--sources",
str(self.hard_root)])
self.assertEqual(code, 0, out)
self.assertNotIn("required >=", out)
class FreezeTest(HardSetBase):
def test_freeze_writes_the_set_and_a_manifest_with_seed_counts_sha256_and_roots_only(self) -> None:
code, out = self.freeze(self.hard_set(hard_questions(10, 20)))
self.assertEqual(code, 0, out)
frozen = (self.out / "hard.json").read_bytes()
manifest = json.loads((self.out / "hard.manifest.json").read_text(encoding="utf-8"))
self.assertEqual(manifest["schema"], "okf-wiki/answer-set-manifest/1")
self.assertEqual(manifest["seed"], 11)
self.assertEqual(manifest["rules"], {"min_paths": 3, "many_paths": 4, "min_many": 20})
self.assertEqual(manifest["part"], {"file": "hard.json", "sha256": sha(frozen), "questions": 30})
self.assertEqual(manifest["groups"], {"part": {"docs-2-3": 10, "docs-4": 20, "history": 0}})
self.assertEqual(manifest["sources"], {"part": root_ref(self.hard_root), "copy": root_ref(self.root)})
self.assertNotIn("set", manifest)
self.assertNotIn("held_out", manifest)
text = (self.out / "hard.manifest.json").read_text(encoding="utf-8")
self.assertNotIn("bread", text)
self.assertNotIn(SPAN[RISING], text)
data = json.loads(frozen)
self.assertEqual(data["sources"], root_ref(self.hard_root))
self.assertEqual(data["schema"], "okf-wiki/answer-set/1")
def test_freeze_renumbers_in_a_seeded_order_and_is_byte_identical(self) -> None:
drafts = self.hard_set(hard_questions(10, 20))
self.assertEqual(self.freeze(drafts, out=self.tmp / "o1")[0], 0)
self.assertEqual(self.freeze(drafts, out=self.tmp / "o2")[0], 0)
first = (self.tmp / "o1" / "hard.json").read_bytes()
self.assertEqual(first, (self.tmp / "o2" / "hard.json").read_bytes())
questions = json.loads(first)["questions"]
self.assertEqual([q["id"] for q in questions], [f"x{n:02d}" for n in range(1, 31)])
order = sorted(f"k{n:02d}" for n in range(1, 31))
random.Random(11).shuffle(order)
wording = {f"k{n:02d}": f"Synthetic hard question number {n} about bread and gardens?" for n in range(1, 31)}
self.assertEqual([q["question"] for q in questions], [wording[i] for i in order])
self.assertEqual(self.freeze(drafts, seed=12, out=self.tmp / "o3")[0], 0)
other = json.loads((self.tmp / "o3" / "hard.json").read_text(encoding="utf-8"))["questions"]
self.assertNotEqual([q["question"] for q in other], [q["question"] for q in questions])
self.assertEqual(sorted(q["question"] for q in other), sorted(q["question"] for q in questions))
def test_freeze_refuses_a_set_that_fails_the_thresholds(self) -> None:
code, out = self.freeze(self.hard_set(hard_questions(11, 19)))
self.assertEqual(code, 1)
self.assertIn("19 of 30", out)
self.assertFalse((self.out / "hard.json").exists())
self.assertFalse((self.out / "hard.manifest.json").exists())
def test_freeze_refuses_a_set_with_another_count(self) -> None:
code, out = self.freeze(self.hard_set(hard_questions(9, 20)))
self.assertEqual(code, 1)
self.assertIn("29", out)
self.assertFalse((self.out / "hard.json").exists())
def test_freeze_refuses_a_set_that_fails_the_check(self) -> None:
questions = hard_questions(10, 20)
questions[0]["evidence"][0]["span"] = "No such sentence stands in any synthetic file."
self.assertEqual(self.freeze(self.hard_set(questions))[0], 1)
self.assertFalse((self.out / "hard.manifest.json").exists())
def test_freeze_never_overwrites_a_frozen_set_or_manifest(self) -> None:
drafts = self.hard_set(hard_questions(10, 20))
self.assertEqual(self.freeze(drafts)[0], 0)
before = ((self.out / "hard.json").read_bytes(), (self.out / "hard.manifest.json").read_bytes())
code, out = self.freeze(drafts, seed=12)
self.assertEqual(code, 1)
self.assertIn("exists", out)
self.assertEqual(before, ((self.out / "hard.json").read_bytes(), (self.out / "hard.manifest.json").read_bytes()))
(self.out / "hard.manifest.json").unlink()
code, out = self.freeze(drafts, seed=12)
self.assertEqual(code, 1)
self.assertIn("exists", out)
self.assertEqual((self.out / "hard.json").read_bytes(), before[0])
class HardGateTest(HardSetBase):
def setUp(self) -> None:
super().setUp()
code, out = self.freeze(self.hard_set(hard_questions(10, 20)))
self.assertEqual(code, 0, out)
self.set = self.out / "hard.json"
self.manifest = self.out / "hard.manifest.json"
def gate(self, answers: Path, root: Path | None = None) -> tuple[int, str]:
return run(["gate", "--set", str(self.set), "--manifest", str(self.manifest), "--sources",
str(root or self.hard_root), "--arm", "filsok", "--answers", str(answers)])
def stub(self, kind: str) -> Path:
path = self.tmp / f"{kind}.json"
self.assertEqual(run(["stub", "--set", str(self.set), "--arm", "filsok", "--kind", kind,
"--out", str(path)])[0], 0)
return path
def test_the_fasit_is_green_and_no_citations_red_on_the_frozen_set(self) -> None:
code, out = self.gate(self.stub("fasit"))
self.assertEqual(code, 0, out)
self.assertIn("set: part, 30 questions", out)
self.assertIn("correct answer and source: 30 of 30", out)
self.assertIn("GREEN", out)
code, out = self.gate(self.stub("no-citations"))
self.assertEqual(code, 0, out)
self.assertIn("correct answer: 30 of 30", out)
self.assertIn("correct answer and source: 0 of 30", out)
self.assertIn("RED", out)
def test_no_answers_score_0_of_30_red_exit_1(self) -> None:
code, out = self.gate(self.tmp / "none.json")
self.assertEqual(code, 1)
self.assertIn("correct answer and source: 0 of 30", out)
self.assertIn("RED", out)
def test_the_frozen_set_is_judged_against_its_own_root_only(self) -> None:
code, out = self.gate(self.stub("fasit"), root=self.root)
self.assertEqual(code, 1)
self.assertIn("sha256", out)
class DumpTest(unittest.TestCase):
def test_dump_is_the_k5_form(self) -> None:
data = {"b": [1, {"z": "æ", "a": None}], "a": True}
self.assertEqual(answerset.dump(data), dump(data))
def test_group_of_follows_k1_and_the_path_count(self) -> None:
main = {RISING: True, OVEN: True, SOIL: True, WATER: True, RELEASE: False}
self.assertEqual(answerset.group_of([RISING, OVEN], main), "docs-2-3")
self.assertEqual(answerset.group_of([RISING, OVEN, SOIL], main), "docs-2-3")
self.assertEqual(answerset.group_of([RISING, OVEN, SOIL, WATER], main), "docs-4")
self.assertEqual(answerset.group_of([RISING, RELEASE], main), "history")
self.assertEqual(answerset.group_of([RISING, OVEN, SOIL, RELEASE], main), "history")
if __name__ == "__main__":
unittest.main()