Kjell Tore Guttormsen
977040f575
C6. `docs/consumption-contract.md` now says what the pre-pass emits:
- SS 2.5 point 2: sub-questions are written in the bundle's terms and given to
the pre-pass in ONE run where it takes several;
- SS 8's example is `okf-consumption/2` as shipped: `withheld` is the
`{total, by_rule, nearest, complete}` mapping, `coverage` carries
`absent_terms` and `weak`, an excerpt carries `text`, `text_sha256`,
`passage` and `own_title`; point 1 closes the identity on `withheld.total`;
- point 7 keeps the falsified verdicts on record and states the one reading
carried since v1.1, `weak`, with its rule: a word held in no form, or
nothing delivered -- whether a word exists at all, which does not move with
question style or corpus size the way the two shares did;
- point 8 the passage (`{start, end, of}`, `text_sha256` over the passage,
`sha256` still the file's), point 9 several sub-questions in one run (one
cut, `questions`, `subquestions`, per-sub-question coverage), point 10 a
title inherited from the heading above, with `own_title`;
- SS 10 names this library's default ranking (BM25) and `--ranking fusion`.
`okf check` holds the two new forms a reader acts on: `passage_malformed`
(not whole numbers with 0 <= start < end <= of) and `subquestions_unindexed`
(not distinct indices into `questions`, or indices in a payload listing none).
17 -> 19 rules; the two tests that pin the published count move with it.
Each rule is held against a real payload (0 findings) and against that
payload broken six ways.
Editing the contract moved the SS 7.4 known-positive, measured once after the
edit: 19 837 -> 23 672 encoded, 19 358 -> 23 092 raw, delta 479 -> 580. The
example payload and `skills/okf-consume` are regenerated by the published
recipe and `okf check` reports 19 rules, 0 findings on them.
Suite on a clean tree after `git add`: 2443 passed, 2 skipped, 4 xfailed.
ruff, ruff format, mypy --strict clean.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
127 lines
4.4 KiB
Python
127 lines
4.4 KiB
Python
"""`okf check` holds the two forms v1.1 order C added to the payload (C6).
|
|
|
|
- `subquestions_unindexed`: a payload asking several sub-questions carries
|
|
`questions`, and every excerpt then names the sub-questions it answered as
|
|
indices into that list. An index a reader cannot look up names nothing.
|
|
- `passage_malformed`: an excerpt delivered as a passage of a larger concept
|
|
carries `passage: {start, end, of}`, and a place that is not a place --
|
|
backwards, past the end, not whole numbers -- sends a reader to the wrong
|
|
characters of the concept it fetches.
|
|
|
|
Each rule is held against a payload the pre-pass really produced (0 findings)
|
|
and against that payload broken one way at a time.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import copy
|
|
import sys
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
import pytest
|
|
|
|
from llm_ingestion_okf import consume, contract_check
|
|
from llm_ingestion_okf import skill as okf_skill
|
|
|
|
TOOLS = Path(__file__).resolve().parent.parent / "tools"
|
|
if str(TOOLS) not in sys.path:
|
|
sys.path.insert(0, str(TOOLS))
|
|
|
|
import okf_retrieval_gate as retrieval # noqa: E402
|
|
|
|
|
|
@pytest.fixture(scope="module")
|
|
def bundle(tmp_path_factory: pytest.TempPathFactory) -> Path:
|
|
spec = retrieval.BundleSpec(
|
|
"contract-new-form",
|
|
(
|
|
retrieval.DocumentSpec(
|
|
"cabin",
|
|
"cabin.md",
|
|
(
|
|
retrieval.ConceptSpec(
|
|
slug="stove",
|
|
title="Stove",
|
|
body="The stove is lit with birch and kept burning all night. ",
|
|
repeat=200,
|
|
),
|
|
retrieval.ConceptSpec(
|
|
slug="well", title="Well", body="The well is drained in autumn."
|
|
),
|
|
),
|
|
),
|
|
),
|
|
)
|
|
return retrieval.build_bundle(tmp_path_factory.mktemp("contract") / "bundle", spec)
|
|
|
|
|
|
def _codes(payload: dict[str, Any]) -> list[str]:
|
|
report = contract_check.check(okf_skill.render_generic(), payload)
|
|
return [finding.code for finding in report.findings]
|
|
|
|
|
|
def _multi(bundle: Path) -> dict[str, Any]:
|
|
return consume.build_multi_payload(
|
|
bundle, questions=["How is the stove lit?", "When is the well drained?"]
|
|
)
|
|
|
|
|
|
def _passage(bundle: Path) -> dict[str, Any]:
|
|
payload = consume.build_payload(bundle, question="How is the stove lit with birch?")
|
|
assert any("passage" in excerpt for excerpt in payload["excerpts"]), "the premise"
|
|
return payload
|
|
|
|
|
|
def test_the_checker_has_nineteen_rules() -> None:
|
|
assert len(contract_check.RULES) == 19
|
|
assert contract_check.rule_subquestions_indexed in contract_check.RULES
|
|
assert contract_check.rule_passage_placed in contract_check.RULES
|
|
|
|
|
|
def test_real_payloads_of_both_forms_are_conformant(bundle: Path) -> None:
|
|
assert _codes(_multi(bundle)) == []
|
|
assert _codes(_passage(bundle)) == []
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"break_it",
|
|
[
|
|
lambda p: p["excerpts"][0].__setitem__("subquestions", [2]),
|
|
lambda p: p["excerpts"][0].__setitem__("subquestions", []),
|
|
lambda p: p["excerpts"][0].__setitem__("subquestions", [0, 0]),
|
|
lambda p: p["excerpts"][0].__setitem__("subquestions", ["0"]),
|
|
lambda p: p["excerpts"][0].pop("subquestions"),
|
|
lambda p: p.pop("questions"),
|
|
],
|
|
ids=["out-of-range", "empty", "repeated", "not-a-number", "missing", "no-questions"],
|
|
)
|
|
def test_a_subquestion_index_a_reader_cannot_look_up_is_refused(
|
|
bundle: Path, break_it: Any
|
|
) -> None:
|
|
payload = copy.deepcopy(_multi(bundle))
|
|
break_it(payload)
|
|
codes = _codes(payload)
|
|
assert codes and set(codes) == {"subquestions_unindexed"}
|
|
|
|
|
|
def _passage_excerpt(payload: dict[str, Any]) -> dict[str, Any]:
|
|
return next(excerpt for excerpt in payload["excerpts"] if "passage" in excerpt)
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"passage",
|
|
[
|
|
{"start": 10, "end": 5, "of": 100},
|
|
{"start": 0, "end": 101, "of": 100},
|
|
{"start": -1, "end": 5, "of": 100},
|
|
{"start": 0, "end": 5},
|
|
{"start": "0", "end": 5, "of": 100},
|
|
"0-5",
|
|
],
|
|
ids=["backwards", "past-the-end", "negative", "no-of", "not-a-number", "not-a-mapping"],
|
|
)
|
|
def test_a_passage_that_is_not_a_place_is_refused(bundle: Path, passage: object) -> None:
|
|
payload = copy.deepcopy(_passage(bundle))
|
|
_passage_excerpt(payload)["passage"] = passage
|
|
assert _codes(payload) == ["passage_malformed"]
|