feat(check): the contract and the checker state the payload's v1.1 forms

C6. `docs/consumption-contract.md` now says what the pre-pass emits:

- SS 2.5 point 2: sub-questions are written in the bundle's terms and given to
  the pre-pass in ONE run where it takes several;
- SS 8's example is `okf-consumption/2` as shipped: `withheld` is the
  `{total, by_rule, nearest, complete}` mapping, `coverage` carries
  `absent_terms` and `weak`, an excerpt carries `text`, `text_sha256`,
  `passage` and `own_title`; point 1 closes the identity on `withheld.total`;
- point 7 keeps the falsified verdicts on record and states the one reading
  carried since v1.1, `weak`, with its rule: a word held in no form, or
  nothing delivered -- whether a word exists at all, which does not move with
  question style or corpus size the way the two shares did;
- point 8 the passage (`{start, end, of}`, `text_sha256` over the passage,
  `sha256` still the file's), point 9 several sub-questions in one run (one
  cut, `questions`, `subquestions`, per-sub-question coverage), point 10 a
  title inherited from the heading above, with `own_title`;
- SS 10 names this library's default ranking (BM25) and `--ranking fusion`.

`okf check` holds the two new forms a reader acts on: `passage_malformed`
(not whole numbers with 0 <= start < end <= of) and `subquestions_unindexed`
(not distinct indices into `questions`, or indices in a payload listing none).
17 -> 19 rules; the two tests that pin the published count move with it.
Each rule is held against a real payload (0 findings) and against that
payload broken six ways.

Editing the contract moved the SS 7.4 known-positive, measured once after the
edit: 19 837 -> 23 672 encoded, 19 358 -> 23 092 raw, delta 479 -> 580. The
example payload and `skills/okf-consume` are regenerated by the published
recipe and `okf check` reports 19 rules, 0 findings on them.

Suite on a clean tree after `git add`: 2443 passed, 2 skipped, 4 xfailed.
ruff, ruff format, mypy --strict clean.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Kjell Tore Guttormsen 2026-09-21 08:47:46 +02:00
commit 977040f575
11 changed files with 308 additions and 28 deletions

View file

@ -156,13 +156,14 @@ def test_a_payload_declaring_no_identity_is_left_to_its_own_rule(tmp_path: Path)
def test_the_rule_count_is_the_denominator_the_report_quotes() -> None:
"""The nevner moved 15 -> 16 with this rule and 16 -> 17 with
`parent_unfollowable` (K3-21), and every consumer quoting the old number is
"""The nevner moved 15 -> 16 with this rule, 16 -> 17 with
`parent_unfollowable` (K3-21) and 17 -> 19 with `subquestions_unindexed`
and `passage_malformed` (v1.1 C6), and every consumer quoting the old number is
quoting a number that has changed."""
text, payload = instantiate(BUNDLE_A)
report = contract_check.check(text, payload)
assert report.rules_evaluated == len(contract_check.RULES) == 17
assert "17 rules" in report.render()
assert report.rules_evaluated == len(contract_check.RULES) == 19
assert "19 rules" in report.render()
def test_the_generator_writes_an_identity_the_checker_can_read() -> None:

View file

@ -0,0 +1,127 @@
"""`okf check` holds the two forms v1.1 order C added to the payload (C6).
- `subquestions_unindexed`: a payload asking several sub-questions carries
`questions`, and every excerpt then names the sub-questions it answered as
indices into that list. An index a reader cannot look up names nothing.
- `passage_malformed`: an excerpt delivered as a passage of a larger concept
carries `passage: {start, end, of}`, and a place that is not a place --
backwards, past the end, not whole numbers -- sends a reader to the wrong
characters of the concept it fetches.
Each rule is held against a payload the pre-pass really produced (0 findings)
and against that payload broken one way at a time.
"""
from __future__ import annotations
import copy
import sys
from pathlib import Path
from typing import Any
import pytest
from llm_ingestion_okf import consume, contract_check
from llm_ingestion_okf import skill as okf_skill
TOOLS = Path(__file__).resolve().parent.parent / "tools"
if str(TOOLS) not in sys.path:
sys.path.insert(0, str(TOOLS))
import okf_retrieval_gate as retrieval # noqa: E402
@pytest.fixture(scope="module")
def bundle(tmp_path_factory: pytest.TempPathFactory) -> Path:
spec = retrieval.BundleSpec(
"contract-new-form",
(
retrieval.DocumentSpec(
"cabin",
"cabin.md",
(
retrieval.ConceptSpec(
slug="stove",
title="Stove",
body="The stove is lit with birch and kept burning all night. ",
repeat=200,
),
retrieval.ConceptSpec(
slug="well", title="Well", body="The well is drained in autumn."
),
),
),
),
)
return retrieval.build_bundle(tmp_path_factory.mktemp("contract") / "bundle", spec)
def _codes(payload: dict[str, Any]) -> list[str]:
report = contract_check.check(okf_skill.render_generic(), payload)
return [finding.code for finding in report.findings]
def _multi(bundle: Path) -> dict[str, Any]:
return consume.build_multi_payload(
bundle, questions=["How is the stove lit?", "When is the well drained?"]
)
def _passage(bundle: Path) -> dict[str, Any]:
payload = consume.build_payload(bundle, question="How is the stove lit with birch?")
assert any("passage" in excerpt for excerpt in payload["excerpts"]), "the premise"
return payload
def test_the_checker_has_nineteen_rules() -> None:
assert len(contract_check.RULES) == 19
assert contract_check.rule_subquestions_indexed in contract_check.RULES
assert contract_check.rule_passage_placed in contract_check.RULES
def test_real_payloads_of_both_forms_are_conformant(bundle: Path) -> None:
assert _codes(_multi(bundle)) == []
assert _codes(_passage(bundle)) == []
@pytest.mark.parametrize(
"break_it",
[
lambda p: p["excerpts"][0].__setitem__("subquestions", [2]),
lambda p: p["excerpts"][0].__setitem__("subquestions", []),
lambda p: p["excerpts"][0].__setitem__("subquestions", [0, 0]),
lambda p: p["excerpts"][0].__setitem__("subquestions", ["0"]),
lambda p: p["excerpts"][0].pop("subquestions"),
lambda p: p.pop("questions"),
],
ids=["out-of-range", "empty", "repeated", "not-a-number", "missing", "no-questions"],
)
def test_a_subquestion_index_a_reader_cannot_look_up_is_refused(
bundle: Path, break_it: Any
) -> None:
payload = copy.deepcopy(_multi(bundle))
break_it(payload)
codes = _codes(payload)
assert codes and set(codes) == {"subquestions_unindexed"}
def _passage_excerpt(payload: dict[str, Any]) -> dict[str, Any]:
return next(excerpt for excerpt in payload["excerpts"] if "passage" in excerpt)
@pytest.mark.parametrize(
"passage",
[
{"start": 10, "end": 5, "of": 100},
{"start": 0, "end": 101, "of": 100},
{"start": -1, "end": 5, "of": 100},
{"start": 0, "end": 5},
{"start": "0", "end": 5, "of": 100},
"0-5",
],
ids=["backwards", "past-the-end", "negative", "no-of", "not-a-number", "not-a-mapping"],
)
def test_a_passage_that_is_not_a_place_is_refused(bundle: Path, passage: object) -> None:
payload = copy.deepcopy(_passage(bundle))
_passage_excerpt(payload)["passage"] = passage
assert _codes(payload) == ["passage_malformed"]

View file

@ -186,13 +186,13 @@ def _pair(tmp_path: Path) -> tuple[str, dict[str, Any]]:
return text, dict(payload)
def test_the_checker_has_seventeen_rules_and_accepts_a_followable_parent(tmp_path: Path) -> None:
def test_the_checker_has_nineteen_rules_and_accepts_a_followable_parent(tmp_path: Path) -> None:
text, payload = _pair(tmp_path)
assert any("parent" in excerpt for excerpt in payload["excerpts"])
report = contract_check.check(text, payload)
assert report.findings == ()
assert report.rules_evaluated == len(contract_check.RULES) == 17
assert "17 rules" in report.render()
assert report.rules_evaluated == len(contract_check.RULES) == 19
assert "19 rules" in report.render()
@pytest.mark.parametrize(