feat(check): the contract and the checker state the payload's v1.1 forms
C6. `docs/consumption-contract.md` now says what the pre-pass emits:
- SS 2.5 point 2: sub-questions are written in the bundle's terms and given to
the pre-pass in ONE run where it takes several;
- SS 8's example is `okf-consumption/2` as shipped: `withheld` is the
`{total, by_rule, nearest, complete}` mapping, `coverage` carries
`absent_terms` and `weak`, an excerpt carries `text`, `text_sha256`,
`passage` and `own_title`; point 1 closes the identity on `withheld.total`;
- point 7 keeps the falsified verdicts on record and states the one reading
carried since v1.1, `weak`, with its rule: a word held in no form, or
nothing delivered -- whether a word exists at all, which does not move with
question style or corpus size the way the two shares did;
- point 8 the passage (`{start, end, of}`, `text_sha256` over the passage,
`sha256` still the file's), point 9 several sub-questions in one run (one
cut, `questions`, `subquestions`, per-sub-question coverage), point 10 a
title inherited from the heading above, with `own_title`;
- SS 10 names this library's default ranking (BM25) and `--ranking fusion`.
`okf check` holds the two new forms a reader acts on: `passage_malformed`
(not whole numbers with 0 <= start < end <= of) and `subquestions_unindexed`
(not distinct indices into `questions`, or indices in a payload listing none).
17 -> 19 rules; the two tests that pin the published count move with it.
Each rule is held against a real payload (0 findings) and against that
payload broken six ways.
Editing the contract moved the SS 7.4 known-positive, measured once after the
edit: 19 837 -> 23 672 encoded, 19 358 -> 23 092 raw, delta 479 -> 580. The
example payload and `skills/okf-consume` are regenerated by the published
recipe and `okf check` reports 19 rules, 0 findings on them.
Suite on a clean tree after `git add`: 2443 passed, 2 skipped, 4 xfailed.
ruff, ruff format, mypy --strict clean.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
parent
da6faf8776
commit
977040f575
11 changed files with 308 additions and 28 deletions
|
|
@ -156,13 +156,14 @@ def test_a_payload_declaring_no_identity_is_left_to_its_own_rule(tmp_path: Path)
|
|||
|
||||
|
||||
def test_the_rule_count_is_the_denominator_the_report_quotes() -> None:
|
||||
"""The nevner moved 15 -> 16 with this rule and 16 -> 17 with
|
||||
`parent_unfollowable` (K3-21), and every consumer quoting the old number is
|
||||
"""The nevner moved 15 -> 16 with this rule, 16 -> 17 with
|
||||
`parent_unfollowable` (K3-21) and 17 -> 19 with `subquestions_unindexed`
|
||||
and `passage_malformed` (v1.1 C6), and every consumer quoting the old number is
|
||||
quoting a number that has changed."""
|
||||
text, payload = instantiate(BUNDLE_A)
|
||||
report = contract_check.check(text, payload)
|
||||
assert report.rules_evaluated == len(contract_check.RULES) == 17
|
||||
assert "17 rules" in report.render()
|
||||
assert report.rules_evaluated == len(contract_check.RULES) == 19
|
||||
assert "19 rules" in report.render()
|
||||
|
||||
|
||||
def test_the_generator_writes_an_identity_the_checker_can_read() -> None:
|
||||
|
|
|
|||
127
tests/test_contract_subquestions_passage.py
Normal file
127
tests/test_contract_subquestions_passage.py
Normal file
|
|
@ -0,0 +1,127 @@
|
|||
"""`okf check` holds the two forms v1.1 order C added to the payload (C6).
|
||||
|
||||
- `subquestions_unindexed`: a payload asking several sub-questions carries
|
||||
`questions`, and every excerpt then names the sub-questions it answered as
|
||||
indices into that list. An index a reader cannot look up names nothing.
|
||||
- `passage_malformed`: an excerpt delivered as a passage of a larger concept
|
||||
carries `passage: {start, end, of}`, and a place that is not a place --
|
||||
backwards, past the end, not whole numbers -- sends a reader to the wrong
|
||||
characters of the concept it fetches.
|
||||
|
||||
Each rule is held against a payload the pre-pass really produced (0 findings)
|
||||
and against that payload broken one way at a time.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
from llm_ingestion_okf import consume, contract_check
|
||||
from llm_ingestion_okf import skill as okf_skill
|
||||
|
||||
TOOLS = Path(__file__).resolve().parent.parent / "tools"
|
||||
if str(TOOLS) not in sys.path:
|
||||
sys.path.insert(0, str(TOOLS))
|
||||
|
||||
import okf_retrieval_gate as retrieval # noqa: E402
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def bundle(tmp_path_factory: pytest.TempPathFactory) -> Path:
|
||||
spec = retrieval.BundleSpec(
|
||||
"contract-new-form",
|
||||
(
|
||||
retrieval.DocumentSpec(
|
||||
"cabin",
|
||||
"cabin.md",
|
||||
(
|
||||
retrieval.ConceptSpec(
|
||||
slug="stove",
|
||||
title="Stove",
|
||||
body="The stove is lit with birch and kept burning all night. ",
|
||||
repeat=200,
|
||||
),
|
||||
retrieval.ConceptSpec(
|
||||
slug="well", title="Well", body="The well is drained in autumn."
|
||||
),
|
||||
),
|
||||
),
|
||||
),
|
||||
)
|
||||
return retrieval.build_bundle(tmp_path_factory.mktemp("contract") / "bundle", spec)
|
||||
|
||||
|
||||
def _codes(payload: dict[str, Any]) -> list[str]:
|
||||
report = contract_check.check(okf_skill.render_generic(), payload)
|
||||
return [finding.code for finding in report.findings]
|
||||
|
||||
|
||||
def _multi(bundle: Path) -> dict[str, Any]:
|
||||
return consume.build_multi_payload(
|
||||
bundle, questions=["How is the stove lit?", "When is the well drained?"]
|
||||
)
|
||||
|
||||
|
||||
def _passage(bundle: Path) -> dict[str, Any]:
|
||||
payload = consume.build_payload(bundle, question="How is the stove lit with birch?")
|
||||
assert any("passage" in excerpt for excerpt in payload["excerpts"]), "the premise"
|
||||
return payload
|
||||
|
||||
|
||||
def test_the_checker_has_nineteen_rules() -> None:
|
||||
assert len(contract_check.RULES) == 19
|
||||
assert contract_check.rule_subquestions_indexed in contract_check.RULES
|
||||
assert contract_check.rule_passage_placed in contract_check.RULES
|
||||
|
||||
|
||||
def test_real_payloads_of_both_forms_are_conformant(bundle: Path) -> None:
|
||||
assert _codes(_multi(bundle)) == []
|
||||
assert _codes(_passage(bundle)) == []
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"break_it",
|
||||
[
|
||||
lambda p: p["excerpts"][0].__setitem__("subquestions", [2]),
|
||||
lambda p: p["excerpts"][0].__setitem__("subquestions", []),
|
||||
lambda p: p["excerpts"][0].__setitem__("subquestions", [0, 0]),
|
||||
lambda p: p["excerpts"][0].__setitem__("subquestions", ["0"]),
|
||||
lambda p: p["excerpts"][0].pop("subquestions"),
|
||||
lambda p: p.pop("questions"),
|
||||
],
|
||||
ids=["out-of-range", "empty", "repeated", "not-a-number", "missing", "no-questions"],
|
||||
)
|
||||
def test_a_subquestion_index_a_reader_cannot_look_up_is_refused(
|
||||
bundle: Path, break_it: Any
|
||||
) -> None:
|
||||
payload = copy.deepcopy(_multi(bundle))
|
||||
break_it(payload)
|
||||
codes = _codes(payload)
|
||||
assert codes and set(codes) == {"subquestions_unindexed"}
|
||||
|
||||
|
||||
def _passage_excerpt(payload: dict[str, Any]) -> dict[str, Any]:
|
||||
return next(excerpt for excerpt in payload["excerpts"] if "passage" in excerpt)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"passage",
|
||||
[
|
||||
{"start": 10, "end": 5, "of": 100},
|
||||
{"start": 0, "end": 101, "of": 100},
|
||||
{"start": -1, "end": 5, "of": 100},
|
||||
{"start": 0, "end": 5},
|
||||
{"start": "0", "end": 5, "of": 100},
|
||||
"0-5",
|
||||
],
|
||||
ids=["backwards", "past-the-end", "negative", "no-of", "not-a-number", "not-a-mapping"],
|
||||
)
|
||||
def test_a_passage_that_is_not_a_place_is_refused(bundle: Path, passage: object) -> None:
|
||||
payload = copy.deepcopy(_passage(bundle))
|
||||
_passage_excerpt(payload)["passage"] = passage
|
||||
assert _codes(payload) == ["passage_malformed"]
|
||||
|
|
@ -186,13 +186,13 @@ def _pair(tmp_path: Path) -> tuple[str, dict[str, Any]]:
|
|||
return text, dict(payload)
|
||||
|
||||
|
||||
def test_the_checker_has_seventeen_rules_and_accepts_a_followable_parent(tmp_path: Path) -> None:
|
||||
def test_the_checker_has_nineteen_rules_and_accepts_a_followable_parent(tmp_path: Path) -> None:
|
||||
text, payload = _pair(tmp_path)
|
||||
assert any("parent" in excerpt for excerpt in payload["excerpts"])
|
||||
report = contract_check.check(text, payload)
|
||||
assert report.findings == ()
|
||||
assert report.rules_evaluated == len(contract_check.RULES) == 17
|
||||
assert "17 rules" in report.render()
|
||||
assert report.rules_evaluated == len(contract_check.RULES) == 19
|
||||
assert "19 rules" in report.render()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue