First public release, published as one snapshot. What changed between versions is in CHANGELOG.md. Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
195 lines
10 KiB
Python
195 lines
10 KiB
Python
"""Tests for tools/claim_features.py: what each claim says beyond its quote, counted by code (wave 4 order 20, K9).
|
|
|
|
The fixture is a source of baking notes, two wiki pages and three claims, written by hand; every expected number below
|
|
is worked out by hand from it. The text is synthetic, about bread (CLAUDE.md rule 3).
|
|
Run: python3 -m unittest discover -s tests -p 'test_claim_features.py'
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import contextlib
|
|
import importlib.util
|
|
import io
|
|
import json
|
|
import sys
|
|
import tempfile
|
|
import unittest
|
|
from pathlib import Path
|
|
from typing import Any
|
|
from unittest import mock
|
|
|
|
from okfwiki import kblocks
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
|
|
# Loaded from its file, as tests/test_part_metrics.py loads its tool; mypy checks the tool when it is named.
|
|
_SPEC = importlib.util.spec_from_file_location("claim_features", ROOT / "tools" / "claim_features.py")
|
|
assert _SPEC is not None and _SPEC.loader is not None
|
|
cf: Any = importlib.util.module_from_spec(_SPEC)
|
|
sys.modules["claim_features"] = cf
|
|
_SPEC.loader.exec_module(cf)
|
|
|
|
SOURCE = "\n".join([
|
|
"# Baking bread", # 1
|
|
"", # 2
|
|
"Bread needs flour, water and salt.", # 3
|
|
"", # 4
|
|
"## Sourdough starter", # 5
|
|
"", # 6
|
|
"```sh", # 7
|
|
"# feed the starter daily", # 8 a comment in code, not a heading
|
|
"```", # 9
|
|
"", # 10
|
|
"Feed the starter with rye flour every morning.", # 11
|
|
"", # 12
|
|
"### Warm kitchens", # 13
|
|
"", # 14
|
|
"A warm kitchen makes the dough rise faster.", # 15
|
|
"",
|
|
])
|
|
|
|
QUOTE_1 = "Feed the starter with rye flour every morning."
|
|
QUOTE_2 = "A warm kitchen makes the dough rise faster."
|
|
QUOTE_3 = "Bread needs flour, water and salt."
|
|
CLAIM_1 = "The sourdough starter is fed with rye flour every morning."
|
|
CLAIM_2 = "A warm kitchen always makes every dough rise twice as fast."
|
|
CLAIM_3 = "Bread needs flour, water and salt."
|
|
|
|
|
|
def page(title: str) -> str:
|
|
return f"---\ntype: Concept\ntitle: {title}\n---\n\nBody.\n"
|
|
|
|
|
|
def claim(cid: str, page_name: str, text: str, line: int, quote: str, provenance: str = "documented") -> dict[str, Any]:
|
|
return {"id": cid, "page": page_name, "line": 6, "claim": text, "footnotes": ["s1"],
|
|
"excerpts": [{"file": "docs-bread.md", "line": line, "text": quote}], "quote_ok": True,
|
|
"excerpt_sha256": "0" * 64, "provenance": provenance, "resources": ["/kilde/docs-bread.md"]}
|
|
|
|
|
|
class Fixture(unittest.TestCase):
|
|
def setUp(self) -> None:
|
|
self._tmp = tempfile.TemporaryDirectory()
|
|
root = Path(self._tmp.name)
|
|
self.sources = root / "src"
|
|
(self.sources / "kilde").mkdir(parents=True)
|
|
(self.sources / "kilde" / "docs-bread.md").write_text(SOURCE, encoding="utf-8")
|
|
self.wiki = root / "wiki"
|
|
self.wiki.mkdir()
|
|
(self.wiki / "starter.md").write_text(page("Sourdough starter care"), encoding="utf-8")
|
|
(self.wiki / "kitchen.md").write_text(page("Kitchen warmth"), encoding="utf-8")
|
|
self.claims = root / "claims.json"
|
|
self.claims.write_text(json.dumps({"schema": "okf-wiki/claims/1", "claims": [
|
|
claim("c1", "starter.md", CLAIM_1, 11, QUOTE_1),
|
|
claim("c2", "kitchen.md", CLAIM_2, 15, QUOTE_2, "merged"),
|
|
claim("c3", "kitchen.md", CLAIM_3, 3, QUOTE_3)], "exceptions": [], "unkeyed": []}), encoding="utf-8")
|
|
self.verdicts = root / "verdicts.json"
|
|
self.verdicts.write_text(json.dumps({"reader": "okf-wiki-reader", "verdicts": [
|
|
{"id": "c1", "verdict": "supported"}, {"id": "c2", "verdict": "unsupported"},
|
|
{"id": "c3", "verdict": "supported"}]}), encoding="utf-8")
|
|
self.out = root / "out"
|
|
|
|
def tearDown(self) -> None:
|
|
self._tmp.cleanup()
|
|
|
|
def run_tool(self, name: str, *extra: str) -> tuple[int, str, dict[str, Any]]:
|
|
out = self.out / name
|
|
buf = io.StringIO()
|
|
with contextlib.redirect_stdout(buf), contextlib.redirect_stderr(io.StringIO()):
|
|
code = cf.main(["--claims", str(self.claims), "--sources", str(self.sources), "--wiki", str(self.wiki),
|
|
"--out", str(out), *extra])
|
|
doc = json.loads(out.read_text(encoding="utf-8")) if out.is_file() else {}
|
|
return code, buf.getvalue(), doc
|
|
|
|
|
|
class FeaturesTest(Fixture):
|
|
def by_id(self, doc: dict[str, Any]) -> dict[str, dict[str, Any]]:
|
|
return {c["id"]: c for c in doc["claims"]}
|
|
|
|
def test_the_words_outside_the_quote_by_hand(self) -> None:
|
|
code, _, doc = self.run_tool("f.json", "--verdicts", str(self.verdicts))
|
|
self.assertEqual(0, code)
|
|
c1, c2, c3 = (self.by_id(doc)[x] for x in ("c1", "c2", "c3"))
|
|
# claim 1: sourdough starter fed rye flour every morning (7 content words); the quote has feed, starter, rye,
|
|
# flour, every, morning: "sourd" and "fed" are not among the quote's first five letters
|
|
self.assertEqual((7, ["sourdough", "fed"], 2), (c1["content_words"], c1["outside"], c1["outside_count"]))
|
|
self.assertEqual(["sourdough"], c1["outside_title"]) # "Sourdough starter care"
|
|
self.assertEqual(["sourdough"], c1["outside_headings"])
|
|
self.assertEqual((1, len(QUOTE_1), len(CLAIM_1)), (c1["quotes"], c1["quote_chars"], c1["claim_chars"]))
|
|
# claim 2: "fast" is not "faste" (faster) on five letters; always, every and twice are not in the quote
|
|
self.assertEqual((9, ["always", "every", "twice", "fast"]), (c2["content_words"], c2["outside"]))
|
|
self.assertEqual(([], []), (c2["outside_title"], c2["outside_headings"]))
|
|
self.assertEqual((0, []), (c3["outside_count"], c3["outside"]))
|
|
self.assertEqual(("supported", "unsupported", "merged"), (c1["verdict"], c2["verdict"], c2["provenance"]))
|
|
self.assertEqual(["docs-bread.md"], c1["files"])
|
|
|
|
def test_the_heading_path_is_the_one_kblocks_gives(self) -> None:
|
|
_, _, doc = self.run_tool("f.json")
|
|
found = self.by_id(doc)
|
|
self.assertEqual(["Baking bread", "Sourdough starter"], found["c1"]["headings"]) # line 8 is code
|
|
self.assertEqual(["Baking bread", "Sourdough starter", "Warm kitchens"], found["c2"]["headings"])
|
|
self.assertEqual(["Baking bread"], found["c3"]["headings"])
|
|
# the tool asks kblocks.headings_of, so a change there (order 22 owns it) reaches the features
|
|
with mock.patch.object(kblocks, "headings_of", return_value=[(2, 1, "Rye loaves")]):
|
|
_, _, again = self.run_tool("g.json")
|
|
self.assertEqual(["Rye loaves"], self.by_id(again)["c1"]["headings"])
|
|
|
|
def test_without_verdicts_every_verdict_is_null(self) -> None:
|
|
code, _, doc = self.run_tool("f.json")
|
|
self.assertEqual(0, code)
|
|
self.assertEqual([None, None, None], [c["verdict"] for c in doc["claims"]])
|
|
|
|
def test_two_runs_are_byte_identical_and_an_output_is_never_overwritten(self) -> None:
|
|
self.run_tool("a.json", "--verdicts", str(self.verdicts))
|
|
self.run_tool("b.json", "--verdicts", str(self.verdicts))
|
|
a, b = (self.out / "a.json").read_bytes(), (self.out / "b.json").read_bytes()
|
|
self.assertEqual(a, b)
|
|
self.assertEqual(json.dumps(json.loads(a), indent=1, sort_keys=True, ensure_ascii=False) + "\n", a.decode())
|
|
self.assertEqual(2, self.run_tool("a.json")[0])
|
|
self.assertEqual(a, (self.out / "a.json").read_bytes())
|
|
|
|
def test_a_missing_source_or_page_cannot_run(self) -> None:
|
|
(self.wiki / "kitchen.md").unlink()
|
|
self.assertEqual(2, self.run_tool("f.json")[0])
|
|
self.assertFalse((self.out / "f.json").exists())
|
|
|
|
|
|
class SummaryTest(Fixture):
|
|
def test_the_share_rejected_per_number_of_words_outside(self) -> None:
|
|
code, text, _ = self.run_tool("f.json", "--verdicts", str(self.verdicts), "--summary")
|
|
self.assertEqual(0, code)
|
|
self.assertIn("words outside the quote 0: 1 claims, 0 unsupported (0.0 %)", text)
|
|
self.assertIn("words outside the quote 2: 1 claims, 0 unsupported (0.0 %)", text)
|
|
self.assertIn("words outside the quote 3-4: 1 claims, 1 unsupported (100.0 %)", text)
|
|
self.assertIn("words outside the quote 5+: 0 claims", text)
|
|
self.assertIn("provenance merged: 1 claims, 1 unsupported (100.0 %)", text)
|
|
self.assertIn("source docs-bread.md: 3 claims, 1 unsupported (33.3 %)", text)
|
|
# medians per verdict: words outside 1 (supported: 2 and 0) and 4; with a title word 1 of 2 and 0 of 1
|
|
self.assertIn("median words outside: supported 1, unsupported 4", text)
|
|
self.assertIn("with a word from the page title outside the quote: supported 1 of 2 (50.0 %), "
|
|
"unsupported 0 of 1 (0.0 %)", text)
|
|
|
|
def test_a_claim_without_a_verdict_is_left_out_of_the_share_and_counted_apart(self) -> None:
|
|
"""Review finding 49 (order 39): a partial reading must not lower the rejected share."""
|
|
rows = [{"verdict": v, "outside_count": 0, "provenance": "documented", "files": ["a.md"], "quote_chars": 1,
|
|
"claim_chars": 1, "content_words": 1, "outside_title": [], "outside_headings": []}
|
|
for v in ("unsupported", "supported", None, None)]
|
|
text = cf.summary(rows)
|
|
self.assertIn("words outside the quote 0: 4 claims, 1 unsupported (50.0 % of 2 judged), 2 not judged", text)
|
|
self.assertIn("source a.md: 4 claims, 1 unsupported (50.0 % of 2 judged), 2 not judged", text)
|
|
self.assertIn("words outside the quote 1: 0 claims, 0 unsupported (-)", text)
|
|
|
|
def test_without_verdicts_the_summary_counts_only(self) -> None:
|
|
code, text, _ = self.run_tool("f.json", "--summary")
|
|
self.assertEqual(0, code)
|
|
self.assertIn("words outside the quote 0: 1 claims, no verdicts", text)
|
|
|
|
|
|
class NoModelTest(unittest.TestCase):
|
|
def test_the_tool_imports_no_network_or_model_client(self) -> None:
|
|
source = (ROOT / "tools" / "claim_features.py").read_text(encoding="utf-8")
|
|
for banned in ("anthropic", "openai", "urllib", "http.client", "requests", "subprocess", "socket"):
|
|
self.assertNotIn(f"import {banned}", source)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|