First public release, published as one snapshot. What changed between versions is in CHANGELOG.md. Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
614 lines
31 KiB
Python
614 lines
31 KiB
Python
"""Tests for okfwiki/claims.py: the claim list behind criterion 1 and 2 (PLAN.md § Ferdig-kriteriet).
|
|
|
|
Every body sentence of every concept page is a claim; headings and pure link lists are the listed exceptions.
|
|
Criterion 1 counts the claims whose every footnote resolves to a verbatim quote at its locator. Criterion 2 counts
|
|
the claims a reader judged supported, over the same denominator. The fixtures are synthetic text about bread
|
|
(CLAUDE.md rule 3). Run: python3 -m unittest discover -s tests
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import contextlib
|
|
import hashlib
|
|
import io
|
|
import json
|
|
import sys
|
|
import tempfile
|
|
import unittest
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT))
|
|
|
|
from okfwiki import claims, klink, reader # noqa: E402
|
|
from okfwiki.reader import load_pairs # noqa: E402
|
|
|
|
SOURCE = """# Bread notes
|
|
Sourdough needs a starter that is fed daily.
|
|
The dough rests for twelve hours before baking.
|
|
A hot oven gives a dark crust.
|
|
"""
|
|
|
|
OVEN = """# Oven notes
|
|
Preheat the oven for forty minutes.
|
|
"""
|
|
|
|
PAGE = """---
|
|
type: Concept
|
|
title: Sourdough
|
|
description: A bread raised by a starter.
|
|
sources:
|
|
- id: b-2
|
|
resource: /kilde/bread.md
|
|
title: Bread notes
|
|
x-locator: { line: 2 }
|
|
x-quote: 'Sourdough needs a starter that is fed daily.'
|
|
- id: b-3
|
|
resource: /kilde/bread.md
|
|
title: Bread notes
|
|
x-locator: { line: 3 }
|
|
x-quote: 'The dough rests for twelve hours'
|
|
- id: o-2
|
|
resource: /kilde/oven.md
|
|
title: Oven notes
|
|
x-locator: { line: 2 }
|
|
x-quote: 'Preheat the oven for forty minutes.'
|
|
generated: { by: claude-code/claude-opus-5-5, at: 2026-09-27T09:00:00Z }
|
|
---
|
|
|
|
# Sourdough
|
|
|
|
**Short:** Sourdough needs a starter (see `starter.cfg`) that is fed every day.[^b-2] The dough rests for twelve
|
|
hours.[^b-3]
|
|
|
|
## Baking
|
|
- Rest the dough for twelve hours, then preheat the oven for forty minutes.[^b-3][^o-2]
|
|
- Feed the starter daily.[^b-2][^b-2]
|
|
|
|
## Related
|
|
- [Crust](crust.md)
|
|
|
|
[^b-2]: Bread notes, line 2
|
|
[^b-3]: Bread notes, line 3
|
|
[^o-2]: Oven notes, line 2
|
|
"""
|
|
|
|
CRUST = """---
|
|
type: Concept
|
|
title: Crust
|
|
sources:
|
|
- id: b-4
|
|
resource: /kilde/bread.md
|
|
title: Bread notes
|
|
x-locator: { line: 4 }
|
|
x-quote: 'A hot oven gives a dark crust.'
|
|
generated: { by: claude-code/claude-opus-5-5, at: 2026-09-27T09:00:00Z }
|
|
---
|
|
|
|
# Crust
|
|
|
|
A hot oven gives a dark crust.[^b-4]
|
|
|
|
[^b-4]: Bread notes, line 4
|
|
"""
|
|
|
|
INDEX = """# Bread
|
|
|
|
* [Sourdough](sourdough.md) - A bread raised by a starter.
|
|
* [Crust](crust.md) - The outer layer.
|
|
"""
|
|
|
|
|
|
class Fixture(unittest.TestCase):
|
|
def setUp(self) -> None:
|
|
self._tmp = tempfile.TemporaryDirectory(prefix="claims-test-")
|
|
self.root = Path(self._tmp.name)
|
|
self.sources = self.root / "sources"
|
|
(self.sources / "kilde").mkdir(parents=True)
|
|
(self.sources / "kilde" / "bread.md").write_text(SOURCE, encoding="utf-8")
|
|
(self.sources / "kilde" / "oven.md").write_text(OVEN, encoding="utf-8")
|
|
self.bundle = self.root / "bundle"
|
|
self.bundle.mkdir()
|
|
|
|
def tearDown(self) -> None:
|
|
self._tmp.cleanup()
|
|
|
|
def write(self, page: str = PAGE, crust: str = CRUST) -> claims.ClaimList:
|
|
for name, text in (("sourdough.md", page), ("crust.md", crust), ("index.md", INDEX)):
|
|
(self.bundle / name).write_text(text, encoding="utf-8")
|
|
return claims.extract(self.bundle, self.sources)
|
|
|
|
|
|
class ExtractTest(Fixture):
|
|
def test_every_body_sentence_is_a_claim_in_page_order(self) -> None:
|
|
found = self.write()
|
|
self.assertEqual(
|
|
[("crust.md", "A hot oven gives a dark crust."),
|
|
("sourdough.md", "**Short:** Sourdough needs a starter (see `starter.cfg`) that is fed every day."),
|
|
("sourdough.md", "The dough rests for twelve hours."),
|
|
("sourdough.md", "Rest the dough for twelve hours, then preheat the oven for forty minutes."),
|
|
("sourdough.md", "Feed the starter daily.")],
|
|
[(c.page, c.claim) for c in found.claims])
|
|
keys = {rel: [x.key for x in klink.page_sentences((self.bundle / rel).read_text(encoding="utf-8"))]
|
|
for rel in ("crust.md", "sourdough.md")}
|
|
self.assertEqual([f"crust.md#{keys['crust.md'][0]}"] + [f"sourdough.md#{k}" for k in keys["sourdough.md"][:4]],
|
|
[c.id for c in found.claims])
|
|
|
|
def test_claims_carry_their_file_line(self) -> None:
|
|
found = self.write()
|
|
text = (self.bundle / "sourdough.md").read_text(encoding="utf-8").splitlines()
|
|
first = found.claims[1]
|
|
self.assertIn("**Short:**", text[first.line - 1])
|
|
|
|
def test_headings_and_link_lists_are_listed_exceptions_not_claims(self) -> None:
|
|
found = self.write()
|
|
kinds = sorted((e.page, e.kind, e.text) for e in found.exceptions)
|
|
self.assertIn(("sourdough.md", "heading", "Sourdough"), kinds)
|
|
self.assertIn(("sourdough.md", "heading", "Related"), kinds)
|
|
self.assertIn(("sourdough.md", "link-list", "[Crust](crust.md)"), kinds)
|
|
self.assertIn(("crust.md", "heading", "Crust"), kinds)
|
|
self.assertEqual(5, len(found.exceptions))
|
|
|
|
def test_excerpts_are_the_cited_quotes_by_file_name_and_line_without_duplicates(self) -> None:
|
|
found = self.write()
|
|
both = found.claims[3]
|
|
self.assertEqual([("bread.md", 3, "The dough rests for twelve hours"),
|
|
("oven.md", 2, "Preheat the oven for forty minutes.")],
|
|
[(e.file, e.line, e.text) for e in both.excerpts])
|
|
twice = found.claims[4]
|
|
self.assertEqual(1, len(twice.excerpts))
|
|
self.assertEqual(["b-2"], twice.footnotes)
|
|
|
|
def test_all_quotes_at_their_locator_is_criterion_1_true(self) -> None:
|
|
found = self.write()
|
|
self.assertEqual([True] * 5, [c.quote_ok for c in found.claims])
|
|
|
|
def test_a_sentence_without_footnote_is_a_claim_that_fails(self) -> None:
|
|
found = self.write(crust=CRUST.replace("dark crust.[^b-4]", "dark crust.[^b-4] Salt helps."))
|
|
salt = [c for c in found.claims if c.claim == "Salt helps."]
|
|
self.assertEqual(1, len(salt))
|
|
self.assertFalse(salt[0].quote_ok)
|
|
self.assertEqual([], salt[0].excerpts)
|
|
|
|
def test_one_quote_off_its_locator_fails_every_claim_that_cites_it(self) -> None:
|
|
found = self.write(page=PAGE.replace("x-locator: { line: 3 }", "x-locator: { line: 4 }"))
|
|
self.assertEqual([True, True, False, False, True], [c.quote_ok for c in found.claims])
|
|
|
|
def test_a_footnote_with_no_source_entry_fails(self) -> None:
|
|
found = self.write(crust=CRUST.replace("crust.[^b-4]\n", "crust.[^b-9]\n"))
|
|
self.assertFalse(found.claims[0].quote_ok)
|
|
|
|
def test_reserved_files_hold_no_claims(self) -> None:
|
|
found = self.write()
|
|
self.assertNotIn("index.md", {c.page for c in found.claims})
|
|
|
|
|
|
class CriterionOneTest(Fixture):
|
|
def test_totals_per_page_and_per_cited_file(self) -> None:
|
|
found = self.write(page=PAGE.replace("x-locator: { line: 3 }", "x-locator: { line: 4 }"))
|
|
report = claims.criterion_1(found)
|
|
self.assertEqual((3, 5), report["total"])
|
|
self.assertEqual({"crust.md": (1, 1), "sourdough.md": (2, 4)}, report["per_page"])
|
|
self.assertEqual({"bread.md": (3, 5), "oven.md": (0, 1)}, report["per_file"])
|
|
|
|
|
|
class PairsTest(Fixture):
|
|
def test_pairs_file_is_a_reader_set_of_the_claims_with_excerpts(self) -> None:
|
|
found = self.write(crust=CRUST.replace("dark crust.[^b-4]", "dark crust.[^b-4] Salt helps."))
|
|
out = self.root / "pairs.json"
|
|
claims.write_pairs(found, out)
|
|
pairs = load_pairs(out)
|
|
self.assertEqual(5, len(pairs))
|
|
self.assertNotIn("Salt helps.", [p.claim for p in pairs])
|
|
raw = json.loads(out.read_text(encoding="utf-8"))
|
|
self.assertEqual({"id", "claim", "excerpts"}, set(raw["pairs"][0]))
|
|
|
|
|
|
class CriterionTwoTest(Fixture):
|
|
def test_supported_over_the_same_denominator_and_missing_counts_as_unsupported(self) -> None:
|
|
found = self.write(crust=CRUST.replace("dark crust.[^b-4]", "dark crust.[^b-4] Salt helps."))
|
|
ids = [c.id for c in found.claims]
|
|
verdicts = {"reader": "r", "verdicts": [{"id": ids[0], "verdict": "supported"},
|
|
{"id": ids[2], "verdict": "supported"},
|
|
{"id": ids[3], "verdict": "unsupported"},
|
|
{"id": ids[4], "verdict": "maybe"}]}
|
|
report = claims.criterion_2(found, verdicts)
|
|
self.assertEqual((2, 6), report["total"])
|
|
self.assertEqual({"crust.md": (1, 2), "sourdough.md": (1, 4)}, report["per_page"])
|
|
self.assertEqual(sorted(set(ids) - {ids[0], ids[2]}), report["not_supported"])
|
|
|
|
|
|
class CommandTest(Fixture):
|
|
def test_list_writes_claims_and_pairs_and_exits_0_when_all_quotes_hold(self) -> None:
|
|
self.write()
|
|
out, pairs = self.root / "claims.json", self.root / "pairs.json"
|
|
code = claims.main(["list", str(self.bundle), str(self.sources), "--out", str(out), "--pairs", str(pairs)])
|
|
self.assertEqual(0, code)
|
|
data = json.loads(out.read_text(encoding="utf-8"))
|
|
self.assertEqual(5, len(data["claims"]))
|
|
self.assertEqual(5, len(data["exceptions"]))
|
|
self.assertEqual(5, len(load_pairs(pairs)))
|
|
|
|
def test_list_exits_1_when_a_claim_fails(self) -> None:
|
|
self.write(crust=CRUST.replace("dark crust.[^b-4]", "dark crust.[^b-4] Salt helps."))
|
|
code = claims.main(["list", str(self.bundle), str(self.sources), "--out", str(self.root / "c.json")])
|
|
self.assertEqual(1, code)
|
|
|
|
def test_score_reads_what_list_wrote_and_exits_0_only_at_n_of_n(self) -> None:
|
|
self.write()
|
|
out = self.root / "claims.json"
|
|
claims.main(["list", str(self.bundle), str(self.sources), "--out", str(out)])
|
|
ids = [c["id"] for c in json.loads(out.read_text(encoding="utf-8"))["claims"]]
|
|
v = self.root / "v.json"
|
|
v.write_text(json.dumps({"reader": "r", "verdicts": [{"id": i, "verdict": "supported"} for i in ids]}))
|
|
self.assertEqual(0, claims.main(["score", "--claims", str(out), "--verdicts", str(v)]))
|
|
v.write_text(json.dumps({"reader": "r", "verdicts": [{"id": i, "verdict": "supported"} for i in ids[1:]]}))
|
|
self.assertEqual(1, claims.main(["score", "--claims", str(out), "--verdicts", str(v)]))
|
|
|
|
def test_missing_directories_exit_2(self) -> None:
|
|
self.assertEqual(2, claims.main(["list", str(self.root / "nope"), str(self.sources), "--out", str(self.root / "c")]))
|
|
|
|
|
|
def claim_row(cid: str, page: str, provenance: str = "documented", files: tuple[str, ...] = ("bread.md",),
|
|
text: str = "A claim.") -> dict[str, object]:
|
|
excerpts = [{"file": f, "line": 1, "text": f"Quote from {f}."} for f in files]
|
|
return {"id": cid, "page": page, "line": 1, "claim": text, "footnotes": ["a"], "excerpts": excerpts,
|
|
"quote_ok": True,
|
|
"excerpt_sha256": claims.excerpt_digest([claims.Excerpt(**e) for e in excerpts]), # type: ignore[arg-type]
|
|
"provenance": provenance, "resources": [f"/kilde/{f}" for f in files]}
|
|
|
|
|
|
def claims_file(path: Path, rows: list[dict[str, object]]) -> Path:
|
|
path.write_text(json.dumps({"schema": "okf-wiki/claims/1", "claims": rows, "exceptions": [], "unkeyed": []}),
|
|
encoding="utf-8")
|
|
return path
|
|
|
|
|
|
class StableIdTest(Fixture):
|
|
"""K6 point 1: the id is <page>#<key>, the key being klink's sentence key for the sentence at the same place."""
|
|
|
|
def test_an_unchanged_sentence_keeps_its_id_when_the_page_is_assembled_again(self) -> None:
|
|
before = {c.claim: c.id for c in self.write().claims}
|
|
page = PAGE.replace("**Short:** Sourdough", "Bakers love it.[^b-2] **Short:** Sourdough").replace(
|
|
"The dough rests for twelve\nhours.[^b-3]", "The [dough](crust.md) rests for twelve\nhours.[^b-3]")
|
|
after = {c.claim: c.id for c in self.write(page=page).claims}
|
|
self.assertEqual(before["The dough rests for twelve hours."],
|
|
after["The [dough](crust.md) rests for twelve hours."])
|
|
self.assertEqual(before["Feed the starter daily."], after["Feed the starter daily."])
|
|
self.assertTrue(all(i.startswith(("crust.md#", "sourdough.md#")) for i in after.values()))
|
|
|
|
def test_the_same_sentence_twice_on_a_page_is_counted(self) -> None:
|
|
twice = "dark crust.[^b-4] A hot oven gives a dark crust.[^b-4]\n"
|
|
found = self.write(crust=CRUST.replace("dark crust.[^b-4]\n", twice))
|
|
ids = [c.id for c in found.claims if c.page == "crust.md"]
|
|
self.assertEqual(2, len(ids))
|
|
self.assertEqual(ids[0][:-2] + "-0", ids[0])
|
|
self.assertEqual(ids[0][:-2] + "-1", ids[1])
|
|
|
|
def test_a_new_excerpt_keeps_the_id_and_changes_the_excerpt_digest(self) -> None:
|
|
a = self.write().claims[0]
|
|
shorter = CRUST.replace("x-quote: 'A hot oven gives a dark crust.'", "x-quote: 'A hot oven gives'")
|
|
b = self.write(crust=shorter).claims[0]
|
|
self.assertEqual(a.id, b.id)
|
|
self.assertNotEqual(a.excerpt_sha256, b.excerpt_sha256)
|
|
self.assertEqual(64, len(a.excerpt_sha256))
|
|
|
|
def test_a_claim_klink_does_not_see_is_listed_never_given_an_id(self) -> None:
|
|
found = self.write(crust=CRUST.replace("[^b-4]\n\n[^b-4]:", "[^b-4]\n\n```\nMix the flour.\n```\n\n[^b-4]:"))
|
|
self.assertEqual(["``` Mix the flour. ```"], [u.text for u in found.unkeyed])
|
|
self.assertFalse([c for c in found.claims if "Mix the flour." in c.claim])
|
|
self.assertEqual((5, 6), claims.criterion_1(found)["total"])
|
|
code = claims.main(["list", str(self.bundle), str(self.sources), "--out", str(self.root / "c.json")])
|
|
self.assertEqual(1, code)
|
|
self.assertEqual(1, len(json.loads((self.root / "c.json").read_text(encoding="utf-8"))["unkeyed"]))
|
|
|
|
|
|
class ProvenanceTest(Fixture):
|
|
"""K6 point 2: the strongest mark among the cited entries, merged > inferred > documented; missing is strongest."""
|
|
|
|
def marked(self, b2: str | None, o2: str | None) -> list[str]:
|
|
page = PAGE
|
|
for label, mark in (("b-2", b2), ("o-2", o2)):
|
|
if mark is not None:
|
|
page = page.replace(f" - id: {label}\n", f" - id: {label}\n x-provenance: {mark}\n")
|
|
page = page.replace(" - id: b-3\n", " - id: b-3\n x-provenance: documented\n")
|
|
return [c.provenance for c in self.write(page=page).claims if c.page == "sourdough.md"]
|
|
|
|
def test_each_claim_carries_the_strongest_mark_it_cites(self) -> None:
|
|
self.assertEqual(["documented"] * 4, self.marked("documented", "documented"))
|
|
self.assertEqual(["inferred", "documented", "inferred", "inferred"], self.marked("inferred", "inferred"))
|
|
self.assertEqual(["documented", "documented", "merged", "documented"], self.marked("documented", "merged"))
|
|
|
|
def test_a_missing_or_unknown_mark_is_missing(self) -> None:
|
|
self.assertEqual(["missing", "documented", "documented", "missing"], self.marked(None, "documented"))
|
|
self.assertEqual(["documented", "documented", "missing", "documented"], self.marked("documented", "guessed"))
|
|
|
|
|
|
class CallsTest(Fixture):
|
|
"""K6 point 3: claims of one page stay together in calls of at most ten, in page order."""
|
|
|
|
def rows(self) -> list[dict[str, object]]:
|
|
return [claim_row(f"a.md#{i:02d}", "a.md") for i in range(12)] + [claim_row("b.md#00", "b.md")]
|
|
|
|
def pairs(self, rows: list[dict[str, object]]) -> Path:
|
|
out = self.root / "pairs.json"
|
|
out.write_text(json.dumps({"schema": "okf-wiki/claim-pairs/1", "pairs": [
|
|
{"id": r["id"], "claim": r["claim"], "excerpts": r["excerpts"]} for r in rows]}), encoding="utf-8")
|
|
return out
|
|
|
|
def test_a_page_over_ten_is_split_and_pages_never_share_a_call(self) -> None:
|
|
rows = self.rows()
|
|
c, p, out = claims_file(self.root / "c.json", rows), self.pairs(rows), self.root / "calls.json"
|
|
self.assertEqual(0, claims.main(["calls", "--claims", str(c), "--pairs", str(p), "--out", str(out)]))
|
|
layout = json.loads(out.read_text(encoding="utf-8"))
|
|
self.assertEqual([10, 2, 1], [len(x["ids"]) for x in layout["calls"]])
|
|
self.assertEqual([r["id"] for r in rows], [i for x in layout["calls"] for i in x["ids"]])
|
|
self.assertEqual(3, len(reader.load_calls(out, p)))
|
|
|
|
def test_ids_restrict_the_layout_and_claims_without_a_pair_are_left_out(self) -> None:
|
|
rows = self.rows()
|
|
c, p, out = claims_file(self.root / "c.json", rows), self.pairs(rows[:11]), self.root / "calls.json"
|
|
ids = self.root / "ids.json"
|
|
ids.write_text(json.dumps({"ids": ["a.md#03", "a.md#11", "b.md#00"]}), encoding="utf-8")
|
|
self.assertEqual(0, claims.main(["calls", "--claims", str(c), "--pairs", str(p), "--out", str(out),
|
|
"--ids", str(ids)]))
|
|
layout = json.loads(out.read_text(encoding="utf-8"))
|
|
self.assertEqual([["a.md#03"]], [x["ids"] for x in layout["calls"]])
|
|
|
|
def calls(self, rows: list[dict[str, object]], *extra: str) -> tuple[int, Path, Path, str]:
|
|
c, p, out = claims_file(self.root / "c.json", rows), self.pairs(rows), self.root / "calls.json"
|
|
err = io.StringIO()
|
|
with contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(err):
|
|
code = claims.main(["calls", "--claims", str(c), "--pairs", str(p), "--out", str(out), *extra])
|
|
return code, p, out, err.getvalue()
|
|
|
|
def test_without_size_the_layout_is_ten_per_call_as_before(self) -> None:
|
|
code, _, out, _ = self.calls(self.rows())
|
|
default = out.read_bytes()
|
|
self.assertEqual(0, code)
|
|
layout = json.loads(default)
|
|
self.assertEqual(10, layout["call_size_max"])
|
|
self.assertEqual(["k001", "k002", "k003"], [x["call"] for x in layout["calls"]])
|
|
self.assertEqual(0, self.calls(self.rows(), "--size", "10")[0])
|
|
self.assertEqual(default, out.read_bytes())
|
|
|
|
def test_size_groups_each_page_in_chunks_of_at_most_n(self) -> None:
|
|
code, p, out, _ = self.calls(self.rows(), "--size", "3")
|
|
self.assertEqual(0, code)
|
|
layout = json.loads(out.read_text(encoding="utf-8"))
|
|
self.assertEqual(3, layout["call_size_max"])
|
|
self.assertEqual([3, 3, 3, 3, 1], [len(x["ids"]) for x in layout["calls"]])
|
|
self.assertEqual([r["id"] for r in self.rows()], [i for x in layout["calls"] for i in x["ids"]])
|
|
self.assertEqual(5, len(reader.load_calls(out, p)))
|
|
|
|
def test_size_one_puts_every_claim_alone_in_a_call(self) -> None:
|
|
code, p, out, _ = self.calls(self.rows(), "--size", "1")
|
|
self.assertEqual(0, code)
|
|
layout = json.loads(out.read_text(encoding="utf-8"))
|
|
self.assertEqual(1, layout["call_size_max"])
|
|
self.assertEqual([[r["id"]] for r in self.rows()], [x["ids"] for x in layout["calls"]])
|
|
self.assertEqual(13, len(reader.load_calls(out, p)))
|
|
|
|
def test_size_outside_one_to_ten_is_refused_and_nothing_is_written(self) -> None:
|
|
for size in ("0", "11", "-1"):
|
|
with self.subTest(size):
|
|
code, _, out, err = self.calls(self.rows(), "--size", size)
|
|
self.assertEqual(2, code)
|
|
self.assertIn("cannot run", err)
|
|
self.assertIn("--size", err)
|
|
self.assertFalse(out.exists())
|
|
|
|
def test_over_999_calls_get_four_digit_names_the_reader_accepts(self) -> None:
|
|
rows = [claim_row(f"a.md#{i:04d}", "a.md") for i in range(1001)]
|
|
code, p, out, _ = self.calls(rows, "--size", "1")
|
|
self.assertEqual(0, code)
|
|
names = [c.name for c in reader.load_calls(out, p)]
|
|
self.assertEqual(["k001", "k999", "k1000", "k1001"], [names[0], *names[998:]])
|
|
self.assertEqual(1001, len(set(names)))
|
|
|
|
|
|
class SampleTest(Fixture):
|
|
"""K6 point 4: judged claims are skipped; the part, inferred, merged and missing are read; at least --min others
|
|
are drawn with a fixed seed."""
|
|
|
|
def setUp(self) -> None:
|
|
super().setUp()
|
|
part = self.root / "del1"
|
|
(part / "kilde").mkdir(parents=True)
|
|
(part / "source-manifest.json").write_text(json.dumps({
|
|
"schema": "okf-wiki/source-manifest/1", "sources": [{"path": "kilde/part.md", "main_topic": True}]}),
|
|
encoding="utf-8")
|
|
self.part = part
|
|
rows = [claim_row(f"p.md#{i:03d}", "p.md") for i in range(400)]
|
|
rows += [claim_row("q.md#part", "q.md", files=("part.md",)), claim_row("q.md#inf", "q.md", "inferred"),
|
|
claim_row("q.md#mrg", "q.md", "merged"), claim_row("q.md#mis", "q.md", "missing"),
|
|
claim_row("q.md#old", "q.md", "merged"), claim_row("q.md#new", "q.md", "merged")]
|
|
self.claims = claims_file(self.root / "c.json", rows)
|
|
changed = claim_row("q.md#new", "q.md", "merged", text="Old text.")
|
|
changed["excerpt_sha256"] = "0" * 64
|
|
self.judged = claims_file(self.root / "judged.json", [claim_row("q.md#old", "q.md", "merged"), changed,
|
|
claim_row("p.md#000", "p.md")])
|
|
|
|
def sample(self, seed: int, out: str, minimum: int = 300) -> dict[str, Any]:
|
|
code = claims.main(["sample", "--claims", str(self.claims), "--judged", str(self.judged), "--part-root",
|
|
str(self.part), "--seed", str(seed), "--min", str(minimum), "--out", str(self.root / out)])
|
|
self.assertEqual(0, code)
|
|
data: dict[str, Any] = json.loads((self.root / out).read_text(encoding="utf-8"))
|
|
return data
|
|
|
|
def test_must_read_and_a_seeded_draw_of_the_others(self) -> None:
|
|
data = self.sample(7, "s.json")
|
|
must = set(data["must_read"])
|
|
self.assertEqual({"q.md#part", "q.md#inf", "q.md#mrg", "q.md#mis", "q.md#new"}, must)
|
|
drawn = data["sample"]
|
|
self.assertEqual(300, len(drawn))
|
|
self.assertNotIn("p.md#000", drawn)
|
|
self.assertNotIn("q.md#old", data["ids"])
|
|
self.assertEqual(305, len(data["ids"]))
|
|
self.assertEqual(7, data["seed"])
|
|
|
|
def test_the_same_seed_gives_the_same_file_and_another_seed_another_draw(self) -> None:
|
|
self.sample(7, "a.json"), self.sample(7, "b.json"), self.sample(8, "c.json")
|
|
a, b, c = ((self.root / n).read_bytes() for n in ("a.json", "b.json", "c.json"))
|
|
self.assertEqual(a, b)
|
|
self.assertNotEqual(json.loads(a)["sample"], json.loads(c)["sample"])
|
|
|
|
def test_fewer_others_than_min_are_all_read(self) -> None:
|
|
data = self.sample(7, "s.json", minimum=1000)
|
|
self.assertEqual(399, len(data["sample"]))
|
|
|
|
def test_the_sample_file_records_what_it_was_drawn_from(self) -> None:
|
|
data = self.sample(7, "s.json")
|
|
self.assertEqual(hashlib.sha256(self.claims.read_bytes()).hexdigest(), data["claims_sha256"])
|
|
self.assertEqual(hashlib.sha256(self.judged.read_bytes()).hexdigest(), data["judged_sha256"])
|
|
self.assertEqual(hashlib.sha256("\n".join(data["ids"]).encode("utf-8")).hexdigest(), data["ids_sha256"])
|
|
self.assertEqual(claims.error_bound(0, 300), data["error_bound_at_zero"])
|
|
|
|
def test_a_min_under_300_is_refused(self) -> None:
|
|
# K6 point 4: at least 300 others, so that 0 not supported bounds the error under 1 %.
|
|
err = io.StringIO()
|
|
with contextlib.redirect_stderr(err):
|
|
code = claims.main(["sample", "--claims", str(self.claims), "--judged", str(self.judged), "--part-root",
|
|
str(self.part), "--seed", "7", "--min", "299", "--out", str(self.root / "s.json")])
|
|
self.assertEqual(2, code)
|
|
self.assertIn("300", err.getvalue())
|
|
self.assertFalse((self.root / "s.json").exists())
|
|
|
|
def of_all(self, seed: int, out: str, minimum: int = 300) -> dict[str, Any]:
|
|
code = claims.main(["sample", "--claims", str(self.claims), "--of-all", "--seed", str(seed), "--min",
|
|
str(minimum), "--out", str(self.root / out)])
|
|
self.assertEqual(0, code)
|
|
data: dict[str, Any] = json.loads((self.root / out).read_text(encoding="utf-8"))
|
|
return data
|
|
|
|
def test_of_all_draws_among_every_claim_and_adds_nothing(self) -> None:
|
|
# The operator 13.11.2026: the reading after a changed build is 300 random claims of all, whatever their mark.
|
|
data = self.of_all(7, "s.json")
|
|
self.assertEqual([], data["must_read"])
|
|
self.assertEqual(300, len(data["sample"]))
|
|
self.assertEqual(data["sample"], data["ids"])
|
|
self.assertTrue(data["of_all"])
|
|
self.assertEqual({"claims": 406, "judged": 0, "must_read": 0, "others": 406, "sample": 300}, data["counts"])
|
|
order = [c["id"] for c in json.loads(self.claims.read_text(encoding="utf-8"))["claims"]]
|
|
self.assertEqual([i for i in order if i in set(data["sample"])], data["sample"])
|
|
self.assertEqual(hashlib.sha256(self.claims.read_bytes()).hexdigest(), data["claims_sha256"])
|
|
self.assertIsNone(data["judged_sha256"])
|
|
self.assertIsNone(data["part_manifest_sha256"])
|
|
|
|
def test_of_all_can_draw_a_claim_of_any_mark_and_one_judged_before(self) -> None:
|
|
drawn: set[str] = set()
|
|
for seed in range(12):
|
|
drawn |= set(self.of_all(seed, f"s{seed}.json")["sample"])
|
|
for wanted in ("q.md#inf", "q.md#mrg", "q.md#mis", "q.md#part", "q.md#old", "p.md#000"):
|
|
self.assertIn(wanted, drawn)
|
|
|
|
def test_of_all_with_the_same_seed_gives_the_same_file(self) -> None:
|
|
self.of_all(7, "a.json"), self.of_all(7, "b.json"), self.of_all(8, "c.json")
|
|
a, b, c = ((self.root / n).read_bytes() for n in ("a.json", "b.json", "c.json"))
|
|
self.assertEqual(a, b)
|
|
self.assertNotEqual(json.loads(a)["sample"], json.loads(c)["sample"])
|
|
|
|
def test_of_all_takes_no_judged_list_or_part(self) -> None:
|
|
for extra in (["--judged", str(self.judged)], ["--part-root", str(self.part)]):
|
|
err = io.StringIO()
|
|
with contextlib.redirect_stderr(err):
|
|
code = claims.main(["sample", "--claims", str(self.claims), "--of-all", *extra, "--seed", "7", "--min",
|
|
"300", "--out", str(self.root / "s.json")])
|
|
self.assertEqual(2, code)
|
|
self.assertIn("--of-all", err.getvalue())
|
|
self.assertFalse((self.root / "s.json").exists())
|
|
|
|
def test_without_of_all_the_judged_list_and_the_part_are_required(self) -> None:
|
|
err = io.StringIO()
|
|
with contextlib.redirect_stderr(err):
|
|
code = claims.main(["sample", "--claims", str(self.claims), "--seed", "7", "--min", "300", "--out",
|
|
str(self.root / "s.json")])
|
|
self.assertEqual(2, code)
|
|
self.assertIn("--judged", err.getvalue())
|
|
self.assertFalse((self.root / "s.json").exists())
|
|
|
|
|
|
class ErrorBoundTest(unittest.TestCase):
|
|
"""K6 point 4: the upper Clopper-Pearson bound at 95 % (one-sided) for k errors in n."""
|
|
|
|
def test_zero_errors_in_299_is_under_one_percent_and_in_298_is_not(self) -> None:
|
|
self.assertLess(claims.error_bound(0, 299), 0.01)
|
|
self.assertGreaterEqual(claims.error_bound(0, 298), 0.01)
|
|
|
|
def test_known_values(self) -> None:
|
|
self.assertAlmostEqual(claims.error_bound(1, 10), 0.3942, places=4)
|
|
self.assertAlmostEqual(claims.error_bound(0, 10), 1 - 0.05 ** 0.1, places=6)
|
|
self.assertEqual(claims.error_bound(10, 10), 1.0)
|
|
|
|
|
|
class ScoreTest(Fixture):
|
|
"""K6 point 5: criterion 2 over the same claims, a claim without a verdict counts as wrong, and the claims not
|
|
supported are counted per provenance mark."""
|
|
|
|
def test_not_supported_per_mark_and_the_ids_restrict_the_denominator(self) -> None:
|
|
rows = [claim_row("a.md#1", "a.md"), claim_row("a.md#2", "a.md", "inferred"),
|
|
claim_row("a.md#3", "a.md", "merged"), claim_row("a.md#4", "a.md")]
|
|
c = claims_file(self.root / "c.json", rows)
|
|
v = self.root / "v.json"
|
|
v.write_text(json.dumps({"reader": "r", "verdicts": [{"id": "a.md#1", "verdict": "supported"},
|
|
{"id": "a.md#2", "verdict": "unsupported"},
|
|
{"id": "a.md#4", "verdict": "supported"}]}))
|
|
found = claims.load_claims(c)
|
|
report = claims.criterion_2(found, json.loads(v.read_text()))
|
|
self.assertEqual((2, 4), report["total"])
|
|
self.assertEqual({"documented": (0, 2), "inferred": (1, 1), "merged": (1, 1)}, report["not_supported_per_mark"])
|
|
ids = self.root / "ids.json"
|
|
ids.write_text(json.dumps({"ids": ["a.md#1", "a.md#4"], "sample": ["a.md#1", "a.md#4"]}))
|
|
out = io.StringIO()
|
|
with contextlib.redirect_stdout(out):
|
|
code = claims.main(["score", "--claims", str(c), "--verdicts", str(v), "--ids", str(ids)])
|
|
self.assertEqual(0, code)
|
|
self.assertIn("criterion 2, supported: 2 of 2", out.getvalue())
|
|
self.assertIn("0 not supported in a sample of 2", out.getvalue())
|
|
|
|
def test_the_bound_counts_the_sampled_claims_not_supported(self) -> None:
|
|
c = claims_file(self.root / "c.json", [claim_row("a.md#1", "a.md"), claim_row("a.md#2", "a.md"),
|
|
claim_row("a.md#3", "a.md", "merged")])
|
|
v = self.root / "v.json"
|
|
v.write_text(json.dumps({"reader": "r", "verdicts": [{"id": "a.md#1", "verdict": "unsupported"},
|
|
{"id": "a.md#2", "verdict": "supported"},
|
|
{"id": "a.md#3", "verdict": "unsupported"}]}))
|
|
ids = self.root / "ids.json"
|
|
ids.write_text(json.dumps({"ids": ["a.md#1", "a.md#2", "a.md#3"], "sample": ["a.md#1", "a.md#2"]}))
|
|
out = io.StringIO()
|
|
with contextlib.redirect_stdout(out):
|
|
code = claims.main(["score", "--claims", str(c), "--verdicts", str(v), "--ids", str(ids)])
|
|
self.assertEqual(1, code)
|
|
self.assertIn(f"1 not supported in a sample of 2: upper bound {claims.error_bound(1, 2):.2%}", out.getvalue())
|
|
|
|
def test_list_that_cannot_write_its_output_exits_2(self) -> None:
|
|
bundle, sources = self.root / "b", self.root / "s"
|
|
bundle.mkdir()
|
|
sources.mkdir()
|
|
err = io.StringIO()
|
|
with contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(err):
|
|
code = claims.main(["list", str(bundle), str(sources), "--out", str(self.root / "no" / "c.json")])
|
|
self.assertEqual(2, code)
|
|
self.assertIn("cannot run", err.getvalue())
|
|
|
|
|
|
class K6CommandsTest(unittest.TestCase):
|
|
def test_every_k6_command_answers_help(self) -> None:
|
|
for module, command in ((claims, "list"), (claims, "calls"), (claims, "sample"), (claims, "score"),
|
|
(reader, "render"), (reader, "collect")):
|
|
with self.subTest(command), contextlib.redirect_stdout(io.StringIO()):
|
|
with self.assertRaises(SystemExit) as stop:
|
|
module.main([command, "--help"])
|
|
self.assertEqual(0, stop.exception.code)
|
|
|
|
|
|
class NoModelTest(unittest.TestCase):
|
|
def test_tool_imports_no_network_or_model_client(self) -> None:
|
|
text = (ROOT / "okfwiki" / "claims.py").read_text(encoding="utf-8")
|
|
for banned in ("anthropic", "openai", "urllib", "http.client", "requests", "subprocess", "socket"):
|
|
self.assertNotIn(banned, text)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|