"""Tests for tools/claim_features.py: what each claim says beyond its quote, counted by code (wave 4 order 20, K9). The fixture is a source of baking notes, two wiki pages and three claims, written by hand; every expected number below is worked out by hand from it. The text is synthetic, about bread (CLAUDE.md rule 3). Run: python3 -m unittest discover -s tests -p 'test_claim_features.py' """ from __future__ import annotations import contextlib import importlib.util import io import json import sys import tempfile import unittest from pathlib import Path from typing import Any from unittest import mock from okfwiki import kblocks ROOT = Path(__file__).resolve().parents[1] # Loaded from its file, as tests/test_part_metrics.py loads its tool; mypy checks the tool when it is named. _SPEC = importlib.util.spec_from_file_location("claim_features", ROOT / "tools" / "claim_features.py") assert _SPEC is not None and _SPEC.loader is not None cf: Any = importlib.util.module_from_spec(_SPEC) sys.modules["claim_features"] = cf _SPEC.loader.exec_module(cf) SOURCE = "\n".join([ "# Baking bread", # 1 "", # 2 "Bread needs flour, water and salt.", # 3 "", # 4 "## Sourdough starter", # 5 "", # 6 "```sh", # 7 "# feed the starter daily", # 8 a comment in code, not a heading "```", # 9 "", # 10 "Feed the starter with rye flour every morning.", # 11 "", # 12 "### Warm kitchens", # 13 "", # 14 "A warm kitchen makes the dough rise faster.", # 15 "", ]) QUOTE_1 = "Feed the starter with rye flour every morning." QUOTE_2 = "A warm kitchen makes the dough rise faster." QUOTE_3 = "Bread needs flour, water and salt." CLAIM_1 = "The sourdough starter is fed with rye flour every morning." CLAIM_2 = "A warm kitchen always makes every dough rise twice as fast." CLAIM_3 = "Bread needs flour, water and salt." def page(title: str) -> str: return f"---\ntype: Concept\ntitle: {title}\n---\n\nBody.\n" def claim(cid: str, page_name: str, text: str, line: int, quote: str, provenance: str = "documented") -> dict[str, Any]: return {"id": cid, "page": page_name, "line": 6, "claim": text, "footnotes": ["s1"], "excerpts": [{"file": "docs-bread.md", "line": line, "text": quote}], "quote_ok": True, "excerpt_sha256": "0" * 64, "provenance": provenance, "resources": ["/kilde/docs-bread.md"]} class Fixture(unittest.TestCase): def setUp(self) -> None: self._tmp = tempfile.TemporaryDirectory() root = Path(self._tmp.name) self.sources = root / "src" (self.sources / "kilde").mkdir(parents=True) (self.sources / "kilde" / "docs-bread.md").write_text(SOURCE, encoding="utf-8") self.wiki = root / "wiki" self.wiki.mkdir() (self.wiki / "starter.md").write_text(page("Sourdough starter care"), encoding="utf-8") (self.wiki / "kitchen.md").write_text(page("Kitchen warmth"), encoding="utf-8") self.claims = root / "claims.json" self.claims.write_text(json.dumps({"schema": "okf-wiki/claims/1", "claims": [ claim("c1", "starter.md", CLAIM_1, 11, QUOTE_1), claim("c2", "kitchen.md", CLAIM_2, 15, QUOTE_2, "merged"), claim("c3", "kitchen.md", CLAIM_3, 3, QUOTE_3)], "exceptions": [], "unkeyed": []}), encoding="utf-8") self.verdicts = root / "verdicts.json" self.verdicts.write_text(json.dumps({"reader": "okf-wiki-reader", "verdicts": [ {"id": "c1", "verdict": "supported"}, {"id": "c2", "verdict": "unsupported"}, {"id": "c3", "verdict": "supported"}]}), encoding="utf-8") self.out = root / "out" def tearDown(self) -> None: self._tmp.cleanup() def run_tool(self, name: str, *extra: str) -> tuple[int, str, dict[str, Any]]: out = self.out / name buf = io.StringIO() with contextlib.redirect_stdout(buf), contextlib.redirect_stderr(io.StringIO()): code = cf.main(["--claims", str(self.claims), "--sources", str(self.sources), "--wiki", str(self.wiki), "--out", str(out), *extra]) doc = json.loads(out.read_text(encoding="utf-8")) if out.is_file() else {} return code, buf.getvalue(), doc class FeaturesTest(Fixture): def by_id(self, doc: dict[str, Any]) -> dict[str, dict[str, Any]]: return {c["id"]: c for c in doc["claims"]} def test_the_words_outside_the_quote_by_hand(self) -> None: code, _, doc = self.run_tool("f.json", "--verdicts", str(self.verdicts)) self.assertEqual(0, code) c1, c2, c3 = (self.by_id(doc)[x] for x in ("c1", "c2", "c3")) # claim 1: sourdough starter fed rye flour every morning (7 content words); the quote has feed, starter, rye, # flour, every, morning: "sourd" and "fed" are not among the quote's first five letters self.assertEqual((7, ["sourdough", "fed"], 2), (c1["content_words"], c1["outside"], c1["outside_count"])) self.assertEqual(["sourdough"], c1["outside_title"]) # "Sourdough starter care" self.assertEqual(["sourdough"], c1["outside_headings"]) self.assertEqual((1, len(QUOTE_1), len(CLAIM_1)), (c1["quotes"], c1["quote_chars"], c1["claim_chars"])) # claim 2: "fast" is not "faste" (faster) on five letters; always, every and twice are not in the quote self.assertEqual((9, ["always", "every", "twice", "fast"]), (c2["content_words"], c2["outside"])) self.assertEqual(([], []), (c2["outside_title"], c2["outside_headings"])) self.assertEqual((0, []), (c3["outside_count"], c3["outside"])) self.assertEqual(("supported", "unsupported", "merged"), (c1["verdict"], c2["verdict"], c2["provenance"])) self.assertEqual(["docs-bread.md"], c1["files"]) def test_the_heading_path_is_the_one_kblocks_gives(self) -> None: _, _, doc = self.run_tool("f.json") found = self.by_id(doc) self.assertEqual(["Baking bread", "Sourdough starter"], found["c1"]["headings"]) # line 8 is code self.assertEqual(["Baking bread", "Sourdough starter", "Warm kitchens"], found["c2"]["headings"]) self.assertEqual(["Baking bread"], found["c3"]["headings"]) # the tool asks kblocks.headings_of, so a change there (order 22 owns it) reaches the features with mock.patch.object(kblocks, "headings_of", return_value=[(2, 1, "Rye loaves")]): _, _, again = self.run_tool("g.json") self.assertEqual(["Rye loaves"], self.by_id(again)["c1"]["headings"]) def test_without_verdicts_every_verdict_is_null(self) -> None: code, _, doc = self.run_tool("f.json") self.assertEqual(0, code) self.assertEqual([None, None, None], [c["verdict"] for c in doc["claims"]]) def test_two_runs_are_byte_identical_and_an_output_is_never_overwritten(self) -> None: self.run_tool("a.json", "--verdicts", str(self.verdicts)) self.run_tool("b.json", "--verdicts", str(self.verdicts)) a, b = (self.out / "a.json").read_bytes(), (self.out / "b.json").read_bytes() self.assertEqual(a, b) self.assertEqual(json.dumps(json.loads(a), indent=1, sort_keys=True, ensure_ascii=False) + "\n", a.decode()) self.assertEqual(2, self.run_tool("a.json")[0]) self.assertEqual(a, (self.out / "a.json").read_bytes()) def test_a_missing_source_or_page_cannot_run(self) -> None: (self.wiki / "kitchen.md").unlink() self.assertEqual(2, self.run_tool("f.json")[0]) self.assertFalse((self.out / "f.json").exists()) class SummaryTest(Fixture): def test_the_share_rejected_per_number_of_words_outside(self) -> None: code, text, _ = self.run_tool("f.json", "--verdicts", str(self.verdicts), "--summary") self.assertEqual(0, code) self.assertIn("words outside the quote 0: 1 claims, 0 unsupported (0.0 %)", text) self.assertIn("words outside the quote 2: 1 claims, 0 unsupported (0.0 %)", text) self.assertIn("words outside the quote 3-4: 1 claims, 1 unsupported (100.0 %)", text) self.assertIn("words outside the quote 5+: 0 claims", text) self.assertIn("provenance merged: 1 claims, 1 unsupported (100.0 %)", text) self.assertIn("source docs-bread.md: 3 claims, 1 unsupported (33.3 %)", text) # medians per verdict: words outside 1 (supported: 2 and 0) and 4; with a title word 1 of 2 and 0 of 1 self.assertIn("median words outside: supported 1, unsupported 4", text) self.assertIn("with a word from the page title outside the quote: supported 1 of 2 (50.0 %), " "unsupported 0 of 1 (0.0 %)", text) def test_a_claim_without_a_verdict_is_left_out_of_the_share_and_counted_apart(self) -> None: """Review finding 49 (order 39): a partial reading must not lower the rejected share.""" rows = [{"verdict": v, "outside_count": 0, "provenance": "documented", "files": ["a.md"], "quote_chars": 1, "claim_chars": 1, "content_words": 1, "outside_title": [], "outside_headings": []} for v in ("unsupported", "supported", None, None)] text = cf.summary(rows) self.assertIn("words outside the quote 0: 4 claims, 1 unsupported (50.0 % of 2 judged), 2 not judged", text) self.assertIn("source a.md: 4 claims, 1 unsupported (50.0 % of 2 judged), 2 not judged", text) self.assertIn("words outside the quote 1: 0 claims, 0 unsupported (-)", text) def test_without_verdicts_the_summary_counts_only(self) -> None: code, text, _ = self.run_tool("f.json", "--summary") self.assertEqual(0, code) self.assertIn("words outside the quote 0: 1 claims, no verdicts", text) class NoModelTest(unittest.TestCase): def test_the_tool_imports_no_network_or_model_client(self) -> None: source = (ROOT / "tools" / "claim_features.py").read_text(encoding="utf-8") for banned in ("anthropic", "openai", "urllib", "http.client", "requests", "subprocess", "socket"): self.assertNotIn(f"import {banned}", source) if __name__ == "__main__": unittest.main()