"""Tests for okfwiki/claims.py: the claim list behind criterion 1 and 2 (PLAN.md ยง Ferdig-kriteriet). Every body sentence of every concept page is a claim; headings and pure link lists are the listed exceptions. Criterion 1 counts the claims whose every footnote resolves to a verbatim quote at its locator. Criterion 2 counts the claims a reader judged supported, over the same denominator. The fixtures are synthetic text about bread (CLAUDE.md rule 3). Run: python3 -m unittest discover -s tests """ from __future__ import annotations import contextlib import hashlib import io import json import sys import tempfile import unittest from pathlib import Path from typing import Any ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT)) from okfwiki import claims, klink, reader # noqa: E402 from okfwiki.reader import load_pairs # noqa: E402 SOURCE = """# Bread notes Sourdough needs a starter that is fed daily. The dough rests for twelve hours before baking. A hot oven gives a dark crust. """ OVEN = """# Oven notes Preheat the oven for forty minutes. """ PAGE = """--- type: Concept title: Sourdough description: A bread raised by a starter. sources: - id: b-2 resource: /kilde/bread.md title: Bread notes x-locator: { line: 2 } x-quote: 'Sourdough needs a starter that is fed daily.' - id: b-3 resource: /kilde/bread.md title: Bread notes x-locator: { line: 3 } x-quote: 'The dough rests for twelve hours' - id: o-2 resource: /kilde/oven.md title: Oven notes x-locator: { line: 2 } x-quote: 'Preheat the oven for forty minutes.' generated: { by: claude-code/claude-opus-5-5, at: 2026-09-27T09:00:00Z } --- # Sourdough **Short:** Sourdough needs a starter (see `starter.cfg`) that is fed every day.[^b-2] The dough rests for twelve hours.[^b-3] ## Baking - Rest the dough for twelve hours, then preheat the oven for forty minutes.[^b-3][^o-2] - Feed the starter daily.[^b-2][^b-2] ## Related - [Crust](crust.md) [^b-2]: Bread notes, line 2 [^b-3]: Bread notes, line 3 [^o-2]: Oven notes, line 2 """ CRUST = """--- type: Concept title: Crust sources: - id: b-4 resource: /kilde/bread.md title: Bread notes x-locator: { line: 4 } x-quote: 'A hot oven gives a dark crust.' generated: { by: claude-code/claude-opus-5-5, at: 2026-09-27T09:00:00Z } --- # Crust A hot oven gives a dark crust.[^b-4] [^b-4]: Bread notes, line 4 """ INDEX = """# Bread * [Sourdough](sourdough.md) - A bread raised by a starter. * [Crust](crust.md) - The outer layer. """ class Fixture(unittest.TestCase): def setUp(self) -> None: self._tmp = tempfile.TemporaryDirectory(prefix="claims-test-") self.root = Path(self._tmp.name) self.sources = self.root / "sources" (self.sources / "kilde").mkdir(parents=True) (self.sources / "kilde" / "bread.md").write_text(SOURCE, encoding="utf-8") (self.sources / "kilde" / "oven.md").write_text(OVEN, encoding="utf-8") self.bundle = self.root / "bundle" self.bundle.mkdir() def tearDown(self) -> None: self._tmp.cleanup() def write(self, page: str = PAGE, crust: str = CRUST) -> claims.ClaimList: for name, text in (("sourdough.md", page), ("crust.md", crust), ("index.md", INDEX)): (self.bundle / name).write_text(text, encoding="utf-8") return claims.extract(self.bundle, self.sources) class ExtractTest(Fixture): def test_every_body_sentence_is_a_claim_in_page_order(self) -> None: found = self.write() self.assertEqual( [("crust.md", "A hot oven gives a dark crust."), ("sourdough.md", "**Short:** Sourdough needs a starter (see `starter.cfg`) that is fed every day."), ("sourdough.md", "The dough rests for twelve hours."), ("sourdough.md", "Rest the dough for twelve hours, then preheat the oven for forty minutes."), ("sourdough.md", "Feed the starter daily.")], [(c.page, c.claim) for c in found.claims]) keys = {rel: [x.key for x in klink.page_sentences((self.bundle / rel).read_text(encoding="utf-8"))] for rel in ("crust.md", "sourdough.md")} self.assertEqual([f"crust.md#{keys['crust.md'][0]}"] + [f"sourdough.md#{k}" for k in keys["sourdough.md"][:4]], [c.id for c in found.claims]) def test_claims_carry_their_file_line(self) -> None: found = self.write() text = (self.bundle / "sourdough.md").read_text(encoding="utf-8").splitlines() first = found.claims[1] self.assertIn("**Short:**", text[first.line - 1]) def test_headings_and_link_lists_are_listed_exceptions_not_claims(self) -> None: found = self.write() kinds = sorted((e.page, e.kind, e.text) for e in found.exceptions) self.assertIn(("sourdough.md", "heading", "Sourdough"), kinds) self.assertIn(("sourdough.md", "heading", "Related"), kinds) self.assertIn(("sourdough.md", "link-list", "[Crust](crust.md)"), kinds) self.assertIn(("crust.md", "heading", "Crust"), kinds) self.assertEqual(5, len(found.exceptions)) def test_excerpts_are_the_cited_quotes_by_file_name_and_line_without_duplicates(self) -> None: found = self.write() both = found.claims[3] self.assertEqual([("bread.md", 3, "The dough rests for twelve hours"), ("oven.md", 2, "Preheat the oven for forty minutes.")], [(e.file, e.line, e.text) for e in both.excerpts]) twice = found.claims[4] self.assertEqual(1, len(twice.excerpts)) self.assertEqual(["b-2"], twice.footnotes) def test_all_quotes_at_their_locator_is_criterion_1_true(self) -> None: found = self.write() self.assertEqual([True] * 5, [c.quote_ok for c in found.claims]) def test_a_sentence_without_footnote_is_a_claim_that_fails(self) -> None: found = self.write(crust=CRUST.replace("dark crust.[^b-4]", "dark crust.[^b-4] Salt helps.")) salt = [c for c in found.claims if c.claim == "Salt helps."] self.assertEqual(1, len(salt)) self.assertFalse(salt[0].quote_ok) self.assertEqual([], salt[0].excerpts) def test_one_quote_off_its_locator_fails_every_claim_that_cites_it(self) -> None: found = self.write(page=PAGE.replace("x-locator: { line: 3 }", "x-locator: { line: 4 }")) self.assertEqual([True, True, False, False, True], [c.quote_ok for c in found.claims]) def test_a_footnote_with_no_source_entry_fails(self) -> None: found = self.write(crust=CRUST.replace("crust.[^b-4]\n", "crust.[^b-9]\n")) self.assertFalse(found.claims[0].quote_ok) def test_reserved_files_hold_no_claims(self) -> None: found = self.write() self.assertNotIn("index.md", {c.page for c in found.claims}) class CriterionOneTest(Fixture): def test_totals_per_page_and_per_cited_file(self) -> None: found = self.write(page=PAGE.replace("x-locator: { line: 3 }", "x-locator: { line: 4 }")) report = claims.criterion_1(found) self.assertEqual((3, 5), report["total"]) self.assertEqual({"crust.md": (1, 1), "sourdough.md": (2, 4)}, report["per_page"]) self.assertEqual({"bread.md": (3, 5), "oven.md": (0, 1)}, report["per_file"]) class PairsTest(Fixture): def test_pairs_file_is_a_reader_set_of_the_claims_with_excerpts(self) -> None: found = self.write(crust=CRUST.replace("dark crust.[^b-4]", "dark crust.[^b-4] Salt helps.")) out = self.root / "pairs.json" claims.write_pairs(found, out) pairs = load_pairs(out) self.assertEqual(5, len(pairs)) self.assertNotIn("Salt helps.", [p.claim for p in pairs]) raw = json.loads(out.read_text(encoding="utf-8")) self.assertEqual({"id", "claim", "excerpts"}, set(raw["pairs"][0])) class CriterionTwoTest(Fixture): def test_supported_over_the_same_denominator_and_missing_counts_as_unsupported(self) -> None: found = self.write(crust=CRUST.replace("dark crust.[^b-4]", "dark crust.[^b-4] Salt helps.")) ids = [c.id for c in found.claims] verdicts = {"reader": "r", "verdicts": [{"id": ids[0], "verdict": "supported"}, {"id": ids[2], "verdict": "supported"}, {"id": ids[3], "verdict": "unsupported"}, {"id": ids[4], "verdict": "maybe"}]} report = claims.criterion_2(found, verdicts) self.assertEqual((2, 6), report["total"]) self.assertEqual({"crust.md": (1, 2), "sourdough.md": (1, 4)}, report["per_page"]) self.assertEqual(sorted(set(ids) - {ids[0], ids[2]}), report["not_supported"]) class CommandTest(Fixture): def test_list_writes_claims_and_pairs_and_exits_0_when_all_quotes_hold(self) -> None: self.write() out, pairs = self.root / "claims.json", self.root / "pairs.json" code = claims.main(["list", str(self.bundle), str(self.sources), "--out", str(out), "--pairs", str(pairs)]) self.assertEqual(0, code) data = json.loads(out.read_text(encoding="utf-8")) self.assertEqual(5, len(data["claims"])) self.assertEqual(5, len(data["exceptions"])) self.assertEqual(5, len(load_pairs(pairs))) def test_list_exits_1_when_a_claim_fails(self) -> None: self.write(crust=CRUST.replace("dark crust.[^b-4]", "dark crust.[^b-4] Salt helps.")) code = claims.main(["list", str(self.bundle), str(self.sources), "--out", str(self.root / "c.json")]) self.assertEqual(1, code) def test_score_reads_what_list_wrote_and_exits_0_only_at_n_of_n(self) -> None: self.write() out = self.root / "claims.json" claims.main(["list", str(self.bundle), str(self.sources), "--out", str(out)]) ids = [c["id"] for c in json.loads(out.read_text(encoding="utf-8"))["claims"]] v = self.root / "v.json" v.write_text(json.dumps({"reader": "r", "verdicts": [{"id": i, "verdict": "supported"} for i in ids]})) self.assertEqual(0, claims.main(["score", "--claims", str(out), "--verdicts", str(v)])) v.write_text(json.dumps({"reader": "r", "verdicts": [{"id": i, "verdict": "supported"} for i in ids[1:]]})) self.assertEqual(1, claims.main(["score", "--claims", str(out), "--verdicts", str(v)])) def test_missing_directories_exit_2(self) -> None: self.assertEqual(2, claims.main(["list", str(self.root / "nope"), str(self.sources), "--out", str(self.root / "c")])) def claim_row(cid: str, page: str, provenance: str = "documented", files: tuple[str, ...] = ("bread.md",), text: str = "A claim.") -> dict[str, object]: excerpts = [{"file": f, "line": 1, "text": f"Quote from {f}."} for f in files] return {"id": cid, "page": page, "line": 1, "claim": text, "footnotes": ["a"], "excerpts": excerpts, "quote_ok": True, "excerpt_sha256": claims.excerpt_digest([claims.Excerpt(**e) for e in excerpts]), # type: ignore[arg-type] "provenance": provenance, "resources": [f"/kilde/{f}" for f in files]} def claims_file(path: Path, rows: list[dict[str, object]]) -> Path: path.write_text(json.dumps({"schema": "okf-wiki/claims/1", "claims": rows, "exceptions": [], "unkeyed": []}), encoding="utf-8") return path class StableIdTest(Fixture): """K6 point 1: the id is #, the key being klink's sentence key for the sentence at the same place.""" def test_an_unchanged_sentence_keeps_its_id_when_the_page_is_assembled_again(self) -> None: before = {c.claim: c.id for c in self.write().claims} page = PAGE.replace("**Short:** Sourdough", "Bakers love it.[^b-2] **Short:** Sourdough").replace( "The dough rests for twelve\nhours.[^b-3]", "The [dough](crust.md) rests for twelve\nhours.[^b-3]") after = {c.claim: c.id for c in self.write(page=page).claims} self.assertEqual(before["The dough rests for twelve hours."], after["The [dough](crust.md) rests for twelve hours."]) self.assertEqual(before["Feed the starter daily."], after["Feed the starter daily."]) self.assertTrue(all(i.startswith(("crust.md#", "sourdough.md#")) for i in after.values())) def test_the_same_sentence_twice_on_a_page_is_counted(self) -> None: twice = "dark crust.[^b-4] A hot oven gives a dark crust.[^b-4]\n" found = self.write(crust=CRUST.replace("dark crust.[^b-4]\n", twice)) ids = [c.id for c in found.claims if c.page == "crust.md"] self.assertEqual(2, len(ids)) self.assertEqual(ids[0][:-2] + "-0", ids[0]) self.assertEqual(ids[0][:-2] + "-1", ids[1]) def test_a_new_excerpt_keeps_the_id_and_changes_the_excerpt_digest(self) -> None: a = self.write().claims[0] shorter = CRUST.replace("x-quote: 'A hot oven gives a dark crust.'", "x-quote: 'A hot oven gives'") b = self.write(crust=shorter).claims[0] self.assertEqual(a.id, b.id) self.assertNotEqual(a.excerpt_sha256, b.excerpt_sha256) self.assertEqual(64, len(a.excerpt_sha256)) def test_a_claim_klink_does_not_see_is_listed_never_given_an_id(self) -> None: found = self.write(crust=CRUST.replace("[^b-4]\n\n[^b-4]:", "[^b-4]\n\n```\nMix the flour.\n```\n\n[^b-4]:")) self.assertEqual(["``` Mix the flour. ```"], [u.text for u in found.unkeyed]) self.assertFalse([c for c in found.claims if "Mix the flour." in c.claim]) self.assertEqual((5, 6), claims.criterion_1(found)["total"]) code = claims.main(["list", str(self.bundle), str(self.sources), "--out", str(self.root / "c.json")]) self.assertEqual(1, code) self.assertEqual(1, len(json.loads((self.root / "c.json").read_text(encoding="utf-8"))["unkeyed"])) class ProvenanceTest(Fixture): """K6 point 2: the strongest mark among the cited entries, merged > inferred > documented; missing is strongest.""" def marked(self, b2: str | None, o2: str | None) -> list[str]: page = PAGE for label, mark in (("b-2", b2), ("o-2", o2)): if mark is not None: page = page.replace(f" - id: {label}\n", f" - id: {label}\n x-provenance: {mark}\n") page = page.replace(" - id: b-3\n", " - id: b-3\n x-provenance: documented\n") return [c.provenance for c in self.write(page=page).claims if c.page == "sourdough.md"] def test_each_claim_carries_the_strongest_mark_it_cites(self) -> None: self.assertEqual(["documented"] * 4, self.marked("documented", "documented")) self.assertEqual(["inferred", "documented", "inferred", "inferred"], self.marked("inferred", "inferred")) self.assertEqual(["documented", "documented", "merged", "documented"], self.marked("documented", "merged")) def test_a_missing_or_unknown_mark_is_missing(self) -> None: self.assertEqual(["missing", "documented", "documented", "missing"], self.marked(None, "documented")) self.assertEqual(["documented", "documented", "missing", "documented"], self.marked("documented", "guessed")) class CallsTest(Fixture): """K6 point 3: claims of one page stay together in calls of at most ten, in page order.""" def rows(self) -> list[dict[str, object]]: return [claim_row(f"a.md#{i:02d}", "a.md") for i in range(12)] + [claim_row("b.md#00", "b.md")] def pairs(self, rows: list[dict[str, object]]) -> Path: out = self.root / "pairs.json" out.write_text(json.dumps({"schema": "okf-wiki/claim-pairs/1", "pairs": [ {"id": r["id"], "claim": r["claim"], "excerpts": r["excerpts"]} for r in rows]}), encoding="utf-8") return out def test_a_page_over_ten_is_split_and_pages_never_share_a_call(self) -> None: rows = self.rows() c, p, out = claims_file(self.root / "c.json", rows), self.pairs(rows), self.root / "calls.json" self.assertEqual(0, claims.main(["calls", "--claims", str(c), "--pairs", str(p), "--out", str(out)])) layout = json.loads(out.read_text(encoding="utf-8")) self.assertEqual([10, 2, 1], [len(x["ids"]) for x in layout["calls"]]) self.assertEqual([r["id"] for r in rows], [i for x in layout["calls"] for i in x["ids"]]) self.assertEqual(3, len(reader.load_calls(out, p))) def test_ids_restrict_the_layout_and_claims_without_a_pair_are_left_out(self) -> None: rows = self.rows() c, p, out = claims_file(self.root / "c.json", rows), self.pairs(rows[:11]), self.root / "calls.json" ids = self.root / "ids.json" ids.write_text(json.dumps({"ids": ["a.md#03", "a.md#11", "b.md#00"]}), encoding="utf-8") self.assertEqual(0, claims.main(["calls", "--claims", str(c), "--pairs", str(p), "--out", str(out), "--ids", str(ids)])) layout = json.loads(out.read_text(encoding="utf-8")) self.assertEqual([["a.md#03"]], [x["ids"] for x in layout["calls"]]) def calls(self, rows: list[dict[str, object]], *extra: str) -> tuple[int, Path, Path, str]: c, p, out = claims_file(self.root / "c.json", rows), self.pairs(rows), self.root / "calls.json" err = io.StringIO() with contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(err): code = claims.main(["calls", "--claims", str(c), "--pairs", str(p), "--out", str(out), *extra]) return code, p, out, err.getvalue() def test_without_size_the_layout_is_ten_per_call_as_before(self) -> None: code, _, out, _ = self.calls(self.rows()) default = out.read_bytes() self.assertEqual(0, code) layout = json.loads(default) self.assertEqual(10, layout["call_size_max"]) self.assertEqual(["k001", "k002", "k003"], [x["call"] for x in layout["calls"]]) self.assertEqual(0, self.calls(self.rows(), "--size", "10")[0]) self.assertEqual(default, out.read_bytes()) def test_size_groups_each_page_in_chunks_of_at_most_n(self) -> None: code, p, out, _ = self.calls(self.rows(), "--size", "3") self.assertEqual(0, code) layout = json.loads(out.read_text(encoding="utf-8")) self.assertEqual(3, layout["call_size_max"]) self.assertEqual([3, 3, 3, 3, 1], [len(x["ids"]) for x in layout["calls"]]) self.assertEqual([r["id"] for r in self.rows()], [i for x in layout["calls"] for i in x["ids"]]) self.assertEqual(5, len(reader.load_calls(out, p))) def test_size_one_puts_every_claim_alone_in_a_call(self) -> None: code, p, out, _ = self.calls(self.rows(), "--size", "1") self.assertEqual(0, code) layout = json.loads(out.read_text(encoding="utf-8")) self.assertEqual(1, layout["call_size_max"]) self.assertEqual([[r["id"]] for r in self.rows()], [x["ids"] for x in layout["calls"]]) self.assertEqual(13, len(reader.load_calls(out, p))) def test_size_outside_one_to_ten_is_refused_and_nothing_is_written(self) -> None: for size in ("0", "11", "-1"): with self.subTest(size): code, _, out, err = self.calls(self.rows(), "--size", size) self.assertEqual(2, code) self.assertIn("cannot run", err) self.assertIn("--size", err) self.assertFalse(out.exists()) def test_over_999_calls_get_four_digit_names_the_reader_accepts(self) -> None: rows = [claim_row(f"a.md#{i:04d}", "a.md") for i in range(1001)] code, p, out, _ = self.calls(rows, "--size", "1") self.assertEqual(0, code) names = [c.name for c in reader.load_calls(out, p)] self.assertEqual(["k001", "k999", "k1000", "k1001"], [names[0], *names[998:]]) self.assertEqual(1001, len(set(names))) class SampleTest(Fixture): """K6 point 4: judged claims are skipped; the part, inferred, merged and missing are read; at least --min others are drawn with a fixed seed.""" def setUp(self) -> None: super().setUp() part = self.root / "del1" (part / "kilde").mkdir(parents=True) (part / "source-manifest.json").write_text(json.dumps({ "schema": "okf-wiki/source-manifest/1", "sources": [{"path": "kilde/part.md", "main_topic": True}]}), encoding="utf-8") self.part = part rows = [claim_row(f"p.md#{i:03d}", "p.md") for i in range(400)] rows += [claim_row("q.md#part", "q.md", files=("part.md",)), claim_row("q.md#inf", "q.md", "inferred"), claim_row("q.md#mrg", "q.md", "merged"), claim_row("q.md#mis", "q.md", "missing"), claim_row("q.md#old", "q.md", "merged"), claim_row("q.md#new", "q.md", "merged")] self.claims = claims_file(self.root / "c.json", rows) changed = claim_row("q.md#new", "q.md", "merged", text="Old text.") changed["excerpt_sha256"] = "0" * 64 self.judged = claims_file(self.root / "judged.json", [claim_row("q.md#old", "q.md", "merged"), changed, claim_row("p.md#000", "p.md")]) def sample(self, seed: int, out: str, minimum: int = 300) -> dict[str, Any]: code = claims.main(["sample", "--claims", str(self.claims), "--judged", str(self.judged), "--part-root", str(self.part), "--seed", str(seed), "--min", str(minimum), "--out", str(self.root / out)]) self.assertEqual(0, code) data: dict[str, Any] = json.loads((self.root / out).read_text(encoding="utf-8")) return data def test_must_read_and_a_seeded_draw_of_the_others(self) -> None: data = self.sample(7, "s.json") must = set(data["must_read"]) self.assertEqual({"q.md#part", "q.md#inf", "q.md#mrg", "q.md#mis", "q.md#new"}, must) drawn = data["sample"] self.assertEqual(300, len(drawn)) self.assertNotIn("p.md#000", drawn) self.assertNotIn("q.md#old", data["ids"]) self.assertEqual(305, len(data["ids"])) self.assertEqual(7, data["seed"]) def test_the_same_seed_gives_the_same_file_and_another_seed_another_draw(self) -> None: self.sample(7, "a.json"), self.sample(7, "b.json"), self.sample(8, "c.json") a, b, c = ((self.root / n).read_bytes() for n in ("a.json", "b.json", "c.json")) self.assertEqual(a, b) self.assertNotEqual(json.loads(a)["sample"], json.loads(c)["sample"]) def test_fewer_others_than_min_are_all_read(self) -> None: data = self.sample(7, "s.json", minimum=1000) self.assertEqual(399, len(data["sample"])) def test_the_sample_file_records_what_it_was_drawn_from(self) -> None: data = self.sample(7, "s.json") self.assertEqual(hashlib.sha256(self.claims.read_bytes()).hexdigest(), data["claims_sha256"]) self.assertEqual(hashlib.sha256(self.judged.read_bytes()).hexdigest(), data["judged_sha256"]) self.assertEqual(hashlib.sha256("\n".join(data["ids"]).encode("utf-8")).hexdigest(), data["ids_sha256"]) self.assertEqual(claims.error_bound(0, 300), data["error_bound_at_zero"]) def test_a_min_under_300_is_refused(self) -> None: # K6 point 4: at least 300 others, so that 0 not supported bounds the error under 1 %. err = io.StringIO() with contextlib.redirect_stderr(err): code = claims.main(["sample", "--claims", str(self.claims), "--judged", str(self.judged), "--part-root", str(self.part), "--seed", "7", "--min", "299", "--out", str(self.root / "s.json")]) self.assertEqual(2, code) self.assertIn("300", err.getvalue()) self.assertFalse((self.root / "s.json").exists()) def of_all(self, seed: int, out: str, minimum: int = 300) -> dict[str, Any]: code = claims.main(["sample", "--claims", str(self.claims), "--of-all", "--seed", str(seed), "--min", str(minimum), "--out", str(self.root / out)]) self.assertEqual(0, code) data: dict[str, Any] = json.loads((self.root / out).read_text(encoding="utf-8")) return data def test_of_all_draws_among_every_claim_and_adds_nothing(self) -> None: # The operator 13.11.2026: the reading after a changed build is 300 random claims of all, whatever their mark. data = self.of_all(7, "s.json") self.assertEqual([], data["must_read"]) self.assertEqual(300, len(data["sample"])) self.assertEqual(data["sample"], data["ids"]) self.assertTrue(data["of_all"]) self.assertEqual({"claims": 406, "judged": 0, "must_read": 0, "others": 406, "sample": 300}, data["counts"]) order = [c["id"] for c in json.loads(self.claims.read_text(encoding="utf-8"))["claims"]] self.assertEqual([i for i in order if i in set(data["sample"])], data["sample"]) self.assertEqual(hashlib.sha256(self.claims.read_bytes()).hexdigest(), data["claims_sha256"]) self.assertIsNone(data["judged_sha256"]) self.assertIsNone(data["part_manifest_sha256"]) def test_of_all_can_draw_a_claim_of_any_mark_and_one_judged_before(self) -> None: drawn: set[str] = set() for seed in range(12): drawn |= set(self.of_all(seed, f"s{seed}.json")["sample"]) for wanted in ("q.md#inf", "q.md#mrg", "q.md#mis", "q.md#part", "q.md#old", "p.md#000"): self.assertIn(wanted, drawn) def test_of_all_with_the_same_seed_gives_the_same_file(self) -> None: self.of_all(7, "a.json"), self.of_all(7, "b.json"), self.of_all(8, "c.json") a, b, c = ((self.root / n).read_bytes() for n in ("a.json", "b.json", "c.json")) self.assertEqual(a, b) self.assertNotEqual(json.loads(a)["sample"], json.loads(c)["sample"]) def test_of_all_takes_no_judged_list_or_part(self) -> None: for extra in (["--judged", str(self.judged)], ["--part-root", str(self.part)]): err = io.StringIO() with contextlib.redirect_stderr(err): code = claims.main(["sample", "--claims", str(self.claims), "--of-all", *extra, "--seed", "7", "--min", "300", "--out", str(self.root / "s.json")]) self.assertEqual(2, code) self.assertIn("--of-all", err.getvalue()) self.assertFalse((self.root / "s.json").exists()) def test_without_of_all_the_judged_list_and_the_part_are_required(self) -> None: err = io.StringIO() with contextlib.redirect_stderr(err): code = claims.main(["sample", "--claims", str(self.claims), "--seed", "7", "--min", "300", "--out", str(self.root / "s.json")]) self.assertEqual(2, code) self.assertIn("--judged", err.getvalue()) self.assertFalse((self.root / "s.json").exists()) class ErrorBoundTest(unittest.TestCase): """K6 point 4: the upper Clopper-Pearson bound at 95 % (one-sided) for k errors in n.""" def test_zero_errors_in_299_is_under_one_percent_and_in_298_is_not(self) -> None: self.assertLess(claims.error_bound(0, 299), 0.01) self.assertGreaterEqual(claims.error_bound(0, 298), 0.01) def test_known_values(self) -> None: self.assertAlmostEqual(claims.error_bound(1, 10), 0.3942, places=4) self.assertAlmostEqual(claims.error_bound(0, 10), 1 - 0.05 ** 0.1, places=6) self.assertEqual(claims.error_bound(10, 10), 1.0) class ScoreTest(Fixture): """K6 point 5: criterion 2 over the same claims, a claim without a verdict counts as wrong, and the claims not supported are counted per provenance mark.""" def test_not_supported_per_mark_and_the_ids_restrict_the_denominator(self) -> None: rows = [claim_row("a.md#1", "a.md"), claim_row("a.md#2", "a.md", "inferred"), claim_row("a.md#3", "a.md", "merged"), claim_row("a.md#4", "a.md")] c = claims_file(self.root / "c.json", rows) v = self.root / "v.json" v.write_text(json.dumps({"reader": "r", "verdicts": [{"id": "a.md#1", "verdict": "supported"}, {"id": "a.md#2", "verdict": "unsupported"}, {"id": "a.md#4", "verdict": "supported"}]})) found = claims.load_claims(c) report = claims.criterion_2(found, json.loads(v.read_text())) self.assertEqual((2, 4), report["total"]) self.assertEqual({"documented": (0, 2), "inferred": (1, 1), "merged": (1, 1)}, report["not_supported_per_mark"]) ids = self.root / "ids.json" ids.write_text(json.dumps({"ids": ["a.md#1", "a.md#4"], "sample": ["a.md#1", "a.md#4"]})) out = io.StringIO() with contextlib.redirect_stdout(out): code = claims.main(["score", "--claims", str(c), "--verdicts", str(v), "--ids", str(ids)]) self.assertEqual(0, code) self.assertIn("criterion 2, supported: 2 of 2", out.getvalue()) self.assertIn("0 not supported in a sample of 2", out.getvalue()) def test_the_bound_counts_the_sampled_claims_not_supported(self) -> None: c = claims_file(self.root / "c.json", [claim_row("a.md#1", "a.md"), claim_row("a.md#2", "a.md"), claim_row("a.md#3", "a.md", "merged")]) v = self.root / "v.json" v.write_text(json.dumps({"reader": "r", "verdicts": [{"id": "a.md#1", "verdict": "unsupported"}, {"id": "a.md#2", "verdict": "supported"}, {"id": "a.md#3", "verdict": "unsupported"}]})) ids = self.root / "ids.json" ids.write_text(json.dumps({"ids": ["a.md#1", "a.md#2", "a.md#3"], "sample": ["a.md#1", "a.md#2"]})) out = io.StringIO() with contextlib.redirect_stdout(out): code = claims.main(["score", "--claims", str(c), "--verdicts", str(v), "--ids", str(ids)]) self.assertEqual(1, code) self.assertIn(f"1 not supported in a sample of 2: upper bound {claims.error_bound(1, 2):.2%}", out.getvalue()) def test_list_that_cannot_write_its_output_exits_2(self) -> None: bundle, sources = self.root / "b", self.root / "s" bundle.mkdir() sources.mkdir() err = io.StringIO() with contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(err): code = claims.main(["list", str(bundle), str(sources), "--out", str(self.root / "no" / "c.json")]) self.assertEqual(2, code) self.assertIn("cannot run", err.getvalue()) class K6CommandsTest(unittest.TestCase): def test_every_k6_command_answers_help(self) -> None: for module, command in ((claims, "list"), (claims, "calls"), (claims, "sample"), (claims, "score"), (reader, "render"), (reader, "collect")): with self.subTest(command), contextlib.redirect_stdout(io.StringIO()): with self.assertRaises(SystemExit) as stop: module.main([command, "--help"]) self.assertEqual(0, stop.exception.code) class NoModelTest(unittest.TestCase): def test_tool_imports_no_network_or_model_client(self) -> None: text = (ROOT / "okfwiki" / "claims.py").read_text(encoding="utf-8") for banned in ("anthropic", "openai", "urllib", "http.client", "requests", "subprocess", "socket"): self.assertNotIn(banned, text) if __name__ == "__main__": unittest.main()