"""`okf check` holds the two forms v1.1 order C added to the payload (C6). - `subquestions_unindexed`: a payload asking several sub-questions carries `questions`, and every excerpt then names the sub-questions it answered as indices into that list. An index a reader cannot look up names nothing. - `passage_malformed`: an excerpt delivered as a passage of a larger concept carries `passage: {start, end, of}`, and a place that is not a place -- backwards, past the end, not whole numbers -- sends a reader to the wrong characters of the concept it fetches. Each rule is held against a payload the pre-pass really produced (0 findings) and against that payload broken one way at a time. """ from __future__ import annotations import copy import sys from pathlib import Path from typing import Any import pytest from llm_ingestion_okf import consume, contract_check from llm_ingestion_okf import skill as okf_skill TOOLS = Path(__file__).resolve().parent.parent / "tools" if str(TOOLS) not in sys.path: sys.path.insert(0, str(TOOLS)) import okf_retrieval_gate as retrieval # noqa: E402 @pytest.fixture(scope="module") def bundle(tmp_path_factory: pytest.TempPathFactory) -> Path: spec = retrieval.BundleSpec( "contract-new-form", ( retrieval.DocumentSpec( "cabin", "cabin.md", ( retrieval.ConceptSpec( slug="stove", title="Stove", body="The stove is lit with birch and kept burning all night. ", repeat=200, ), retrieval.ConceptSpec( slug="well", title="Well", body="The well is drained in autumn." ), ), ), ), ) return retrieval.build_bundle(tmp_path_factory.mktemp("contract") / "bundle", spec) def _codes(payload: dict[str, Any]) -> list[str]: report = contract_check.check(okf_skill.render_generic(), payload) return [finding.code for finding in report.findings] def _multi(bundle: Path) -> dict[str, Any]: return consume.build_multi_payload( bundle, questions=["How is the stove lit?", "When is the well drained?"] ) def _passage(bundle: Path) -> dict[str, Any]: payload = consume.build_payload(bundle, question="How is the stove lit with birch?") assert any("passage" in excerpt for excerpt in payload["excerpts"]), "the premise" return payload def test_the_checker_has_nineteen_rules() -> None: assert len(contract_check.RULES) == 19 assert contract_check.rule_subquestions_indexed in contract_check.RULES assert contract_check.rule_passage_placed in contract_check.RULES def test_real_payloads_of_both_forms_are_conformant(bundle: Path) -> None: assert _codes(_multi(bundle)) == [] assert _codes(_passage(bundle)) == [] @pytest.mark.parametrize( "break_it", [ lambda p: p["excerpts"][0].__setitem__("subquestions", [2]), lambda p: p["excerpts"][0].__setitem__("subquestions", []), lambda p: p["excerpts"][0].__setitem__("subquestions", [0, 0]), lambda p: p["excerpts"][0].__setitem__("subquestions", ["0"]), lambda p: p["excerpts"][0].pop("subquestions"), lambda p: p.pop("questions"), ], ids=["out-of-range", "empty", "repeated", "not-a-number", "missing", "no-questions"], ) def test_a_subquestion_index_a_reader_cannot_look_up_is_refused( bundle: Path, break_it: Any ) -> None: payload = copy.deepcopy(_multi(bundle)) break_it(payload) codes = _codes(payload) assert codes and set(codes) == {"subquestions_unindexed"} def _passage_excerpt(payload: dict[str, Any]) -> dict[str, Any]: return next(excerpt for excerpt in payload["excerpts"] if "passage" in excerpt) @pytest.mark.parametrize( "passage", [ {"start": 10, "end": 5, "of": 100}, {"start": 0, "end": 101, "of": 100}, {"start": -1, "end": 5, "of": 100}, {"start": 0, "end": 5}, {"start": "0", "end": 5, "of": 100}, "0-5", ], ids=["backwards", "past-the-end", "negative", "no-of", "not-a-number", "not-a-mapping"], ) def test_a_passage_that_is_not_a_place_is_refused(bundle: Path, passage: object) -> None: payload = copy.deepcopy(_passage(bundle)) _passage_excerpt(payload)["passage"] = passage assert _codes(payload) == ["passage_malformed"]