feat(tools): okf_contract_check reads the contract's checkable half

Fourteen rules, each with its own code: one "invalid" verdict over fourteen
different defects is a diagnostic no caller can act on. The report quotes its
own denominators -- rules run, excerpts and withheld entries examined -- because
a checker that exempted itself from section 5 would be stating the rule it
breaks.

Three exit codes, not two. "The check did not run" and "the check failed" are
different outcomes, and an unread file reported as a failed check is the fourth
face of the verification law.

The shipped template plus its example payload is the known-positive arm, so a
checker that refuses everything cannot be green on the thirteen negative ones.
A last test asserts every literal the checker enforces appears in the contract
document: two copies of a closed set drift, and the copy nobody reads is the
one that goes wrong.

996 -> 1023 tests.
This commit is contained in:
Kjell Tore Guttormsen 2026-09-02 16:09:27 +02:00
commit bc0b4130f1
2 changed files with 716 additions and 0 deletions

View file

@ -0,0 +1,243 @@
"""The consumption contract, checked rather than described.
`docs/consumption-contract.md` states what a consumption skill and its pre-pass
payload MUST carry. Prose has no test, and this repository has already paid for
that once: a published promise without a test goes false without anyone
noticing. So the contract's mechanically checkable half is checked here.
Two disciplines the checker itself is held to:
- **A known-positive, shipped.** The template and its example payload must
PASS. Without that arm a checker that refuses everything is green on every
negative case -- the harness-lies-red failure, one level down.
- **One mutation per arm, each with its own code.** A single "invalid" verdict
over fourteen different defects is a diagnostic no caller can act on, so each
mutation asserts the code it produces, not merely that something failed.
The last test is the anti-drift gate: every literal the checker enforces must
appear in the contract document. Two copies of a closed set drift, and the
copy nobody reads is the one that goes wrong.
"""
from __future__ import annotations
import json
import subprocess
import sys
from pathlib import Path
from typing import Any
import pytest
PROJECT_ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(PROJECT_ROOT / "tools"))
import okf_contract_check # noqa: E402
TEMPLATE = PROJECT_ROOT / "skills" / "okf-consume-template" / "SKILL.md"
EXAMPLE = TEMPLATE.parent / "references" / "example-payload.json"
CONTRACT = PROJECT_ROOT / "docs" / "consumption-contract.md"
def load_example() -> dict[str, Any]:
payload: dict[str, Any] = json.loads(EXAMPLE.read_text(encoding="utf-8"))
return payload
def codes(skill_text: str, payload: Any) -> list[str]:
return [finding.code for finding in okf_contract_check.check(skill_text, payload).findings]
# --- The known-positive, which every negative arm depends on ----------------
def test_shipped_template_and_example_payload_are_conformant() -> None:
"""The CONTROL. A checker that cannot pass anything proves nothing below."""
report = okf_contract_check.check(TEMPLATE.read_text(encoding="utf-8"), load_example())
assert report.findings == ()
def test_the_report_carries_its_denominators() -> None:
""" "Conformant" without a denominator is unmeasured, per the contract's own
section 5. The report says how many rules ran over how many units."""
report = okf_contract_check.check(TEMPLATE.read_text(encoding="utf-8"), load_example())
assert report.rules_evaluated == len(okf_contract_check.RULES)
assert report.rules_evaluated > 0
assert report.excerpts_examined == 3
assert report.withheld_examined == 2
rendered = report.render()
assert str(report.rules_evaluated) in rendered
assert "3" in rendered and "2" in rendered
# --- Payload mutations, one per arm -----------------------------------------
def test_broken_denominator_identity_is_named() -> None:
payload = load_example()
payload["denominators"]["considered"] = 6
assert "denominator_identity" in codes(TEMPLATE.read_text(encoding="utf-8"), payload)
def test_delivered_count_disagreeing_with_the_list_is_named() -> None:
payload = load_example()
payload["excerpts"] = payload["excerpts"][:2]
assert "denominator_list_mismatch" in codes(TEMPLATE.read_text(encoding="utf-8"), payload)
def test_a_missing_adjudication_key_is_not_absence() -> None:
"""The contract's sharpest rule: `unknown` is written, never omitted."""
payload = load_example()
del payload["excerpts"][0]["adjudication"]
assert "state_missing" in codes(TEMPLATE.read_text(encoding="utf-8"), payload)
def test_an_adjudication_value_outside_the_closed_set_is_named() -> None:
payload = load_example()
payload["excerpts"][0]["adjudication"] = "absent"
assert "state_not_in_set" in codes(TEMPLATE.read_text(encoding="utf-8"), payload)
def test_a_trust_tier_outside_the_closed_set_is_named() -> None:
payload = load_example()
payload["excerpts"][1]["trust_tier"] = "trusted"
assert "state_not_in_set" in codes(TEMPLATE.read_text(encoding="utf-8"), payload)
@pytest.mark.parametrize("key", ["bundle_id", "concept_id", "sha256"])
def test_an_excerpt_missing_its_source_marking_is_named(key: str) -> None:
payload = load_example()
del payload["excerpts"][0][key]
assert "source_marking_incomplete" in codes(TEMPLATE.read_text(encoding="utf-8"), payload)
def test_a_bundle_without_a_ref_is_named() -> None:
payload = load_example()
payload["bundle"]["ref"] = ""
assert "ref_missing" in codes(TEMPLATE.read_text(encoding="utf-8"), payload)
def test_a_withheld_concept_without_a_rule_is_an_undeclared_cut() -> None:
payload = load_example()
del payload["withheld"][0]["rule"]
assert "cut_undeclared" in codes(TEMPLATE.read_text(encoding="utf-8"), payload)
def test_spending_over_the_limit_is_the_gate_firing() -> None:
payload = load_example()
payload["budget"]["spent"] = payload["budget"]["limit"] + 1
assert "budget_exceeded" in codes(TEMPLATE.read_text(encoding="utf-8"), payload)
def test_an_instrument_that_missed_its_known_positive_is_named() -> None:
payload = load_example()
payload["budget"]["known_positive"]["measured"] = 10405
assert "instrument_unvalidated" in codes(TEMPLATE.read_text(encoding="utf-8"), payload)
def test_a_budget_without_an_instrument_is_named() -> None:
payload = load_example()
payload["budget"]["instrument"] = ""
assert "budget_undeclared" in codes(TEMPLATE.read_text(encoding="utf-8"), payload)
def test_a_payload_naming_no_contract_revision_is_named() -> None:
payload = load_example()
del payload["contract"]
assert "contract_unversioned" in codes(TEMPLATE.read_text(encoding="utf-8"), payload)
def test_a_payload_that_is_not_a_mapping_is_refused_not_crashed() -> None:
assert "payload_invalid" in codes(TEMPLATE.read_text(encoding="utf-8"), ["not", "a", "mapping"])
# --- Skill mutations --------------------------------------------------------
def test_a_missing_required_section_is_named() -> None:
text = TEMPLATE.read_text(encoding="utf-8").replace("## Denominators", "## Counts")
assert "skill_section_missing" in codes(text, load_example())
def test_a_missing_required_marking_is_named() -> None:
text = TEMPLATE.read_text(encoding="utf-8").replace("[sourced-not-sufficient]", "[thin]")
assert "skill_marking_missing" in codes(text, load_example())
def test_a_translated_marking_literal_does_not_count() -> None:
"""One literal string, no variants, no translations."""
text = TEMPLATE.read_text(encoding="utf-8").replace(
"[unverifiable-from-bundle]", "[ikke-verifiserbar-fra-bundle]"
)
assert "skill_marking_missing" in codes(text, load_example())
def test_a_skill_omitting_an_adjudication_value_is_named() -> None:
text = TEMPLATE.read_text(encoding="utf-8").replace("`unknown`", "`missing`")
assert "skill_state_missing" in codes(text, load_example())
# --- The command line -------------------------------------------------------
def test_the_command_exits_zero_on_the_shipped_pair() -> None:
exit_code = okf_contract_check.main(["--skill", str(TEMPLATE), "--payload", str(EXAMPLE)])
assert exit_code == 0
def test_the_command_exits_one_on_a_non_conformant_payload(tmp_path: Path) -> None:
payload = load_example()
payload["denominators"]["considered"] = 6
broken = tmp_path / "payload.json"
broken.write_text(json.dumps(payload), encoding="utf-8")
assert okf_contract_check.main(["--skill", str(TEMPLATE), "--payload", str(broken)]) == 1
def test_the_command_separates_could_not_run_from_non_conformant(tmp_path: Path) -> None:
"""Exit 2, never 1: "the check did not run" and "the check failed" are
different outcomes, and collapsing them is the fourth face of the
verification law."""
missing = tmp_path / "absent.json"
assert okf_contract_check.main(["--skill", str(TEMPLATE), "--payload", str(missing)]) == 2
unreadable = tmp_path / "bad.json"
unreadable.write_text("{not json", encoding="utf-8")
assert okf_contract_check.main(["--skill", str(TEMPLATE), "--payload", str(unreadable)]) == 2
def test_the_module_runs_as_a_script() -> None:
result = subprocess.run(
[
sys.executable,
str(PROJECT_ROOT / "tools" / "okf_contract_check.py"),
"--skill",
str(TEMPLATE),
"--payload",
str(EXAMPLE),
],
capture_output=True,
text=True,
check=False,
)
assert result.returncode == 0, result.stderr
assert "conformant" in result.stdout
# --- The anti-drift gate ----------------------------------------------------
def test_every_literal_the_checker_enforces_is_in_the_contract() -> None:
"""The checker and the document state one closed set each. If they can
disagree, the copy nobody reads is the one that goes wrong."""
contract = CONTRACT.read_text(encoding="utf-8")
for literal in (
*okf_contract_check.REQUIRED_MARKINGS,
*okf_contract_check.ADJUDICATION_STATES,
*okf_contract_check.TRUST_TIERS,
):
assert literal in contract, literal
for heading in okf_contract_check.REQUIRED_SECTIONS:
assert heading.lower() in contract.lower(), heading
def test_the_contract_document_is_reachable_from_the_template() -> None:
assert "docs/consumption-contract.md" in TEMPLATE.read_text(encoding="utf-8")

473
tools/okf_contract_check.py Normal file
View file

@ -0,0 +1,473 @@
"""Check a consumption skill and one pre-pass payload against the contract.
`docs/consumption-contract.md` is normative prose. Prose has no test, so this
command reads the mechanically checkable half of it: the payload shape (SS 3, 5,
6, 7, 8) and the fixed vocabulary a conformant `SKILL.md` must carry (SS 4, 6).
**What it cannot see, said here rather than implied.** The division of labour
(SS 2) and the prohibitions (SS 9) are properties of a RUN -- whether an agent
went looking for context the pre-pass withheld, whether a retrieval tool was
pointed at the verdict layer. No static check reaches them. Conformance here is
the floor, never the proof, and a green run means the payload carries what a
claim would have to rest on -- not that the claim was made honestly.
**Every rule has its own code.** One "invalid" verdict over fourteen different
defects is a diagnostic no caller can act on. The codes are the contract's
paragraphs made addressable.
**The report carries denominators**, because the contract requires them of its
consumers and a checker exempting itself would be stating the rule it breaks:
how many rules ran, over how many excerpts and withheld entries.
Exit codes are three, not two: 0 conformant, 1 non-conformant, 2 the check did
not run. Collapsing 2 into 1 would report an unread file as a failed check.
It lives outside `src/`, so it never enters a wheel and no consumer's install
surface changes because it exists.
"""
from __future__ import annotations
import argparse
import json
from collections.abc import Callable, Mapping, Sequence
from dataclasses import dataclass
from pathlib import Path
from typing import Any
#: SS 4.1. Five literals, spelled exactly. `[unverifiable-from-bundle]` is one
#: literal string -- no variants, no translations -- so this is a membership
#: test on the literal, never on a translated concept name.
REQUIRED_MARKINGS = (
"extracted",
"derived",
"[unverifiable-from-bundle]",
"[unread]",
"[sourced-not-sufficient]",
)
#: SS 6.1. A closed set of three, and the third is a real state: `unknown` says
#: the concept carries no `adjudication` key, which is what an older bundle
#: looks like. Collapsing it into `proposed` or into absence reports "not
#: judged" where the truth is "we cannot tell", and only one of those is a fact
#: about the concept.
ADJUDICATION_STATES = ("proposed", "adjudicated", "unknown")
#: SS 6.2, from SPEC SS 5.3: derived from `verified`, lowest to highest.
TRUST_TIERS = ("unverified", "machine-confirmed", "human-reviewed")
#: The level-2 headings a conformant skill carries. Checked by name because the
#: alternative -- searching the prose for the obligation -- passes on any
#: document that happens to use the words.
REQUIRED_SECTIONS = (
"Pre-pass",
"Division of labour",
"Markings",
"States",
"Budget",
"Denominators",
"Prohibitions",
)
@dataclass(frozen=True)
class Finding:
"""One contract paragraph, unmet, named by its code."""
code: str
message: str
@dataclass(frozen=True)
class Report:
"""Findings plus the denominators they were measured over."""
findings: tuple[Finding, ...]
rules_evaluated: int
excerpts_examined: int
withheld_examined: int
def render(self) -> str:
denominator = (
f"{self.rules_evaluated} rules over {self.excerpts_examined} excerpts "
f"and {self.withheld_examined} withheld entries"
)
if not self.findings:
return f"conformant: {denominator}, 0 findings"
lines = [f"NOT conformant: {denominator}, {len(self.findings)} findings"]
lines += [f" {finding.code}: {finding.message}" for finding in self.findings]
return "\n".join(lines)
@dataclass(frozen=True)
class Context:
"""What every rule reads. `payload` is empty when the payload is not a
mapping at all, so each rule stays a total function over its input."""
skill: str
payload: Mapping[str, Any]
payload_is_mapping: bool
def _mapping(value: object) -> Mapping[str, Any]:
return value if isinstance(value, Mapping) else {}
def _sequence(value: object) -> Sequence[Any]:
if isinstance(value, Sequence) and not isinstance(value, (str, bytes)):
return value
return ()
def _text(value: object) -> str:
return value if isinstance(value, str) else ""
def _whole(value: object) -> int | None:
# `bool` is an `int` in Python and would make `True` a budget. Excluded by
# name rather than trusted not to arrive.
return value if isinstance(value, int) and not isinstance(value, bool) else None
# --- Payload rules -----------------------------------------------------------
def rule_payload_shape(ctx: Context) -> list[Finding]:
if ctx.payload_is_mapping:
return []
return [Finding("payload_invalid", "the payload is not a JSON object (SS 8)")]
def rule_contract_version(ctx: Context) -> list[Finding]:
if not ctx.payload_is_mapping or _text(ctx.payload.get("contract")):
return []
return [
Finding(
"contract_unversioned",
"the payload names no `contract` revision, so a reader cannot tell "
"which revision it is holding (SS 8.2)",
)
]
def rule_bundle_ref(ctx: Context) -> list[Finding]:
if not ctx.payload_is_mapping:
return []
bundle = _mapping(ctx.payload.get("bundle"))
findings = []
if not _text(bundle.get("ref")):
findings.append(
Finding(
"ref_missing",
"the payload names no bundle `ref`; a version is the producer's "
"assertion, a ref is a fact about bytes (SS 3.3)",
)
)
if not _text(bundle.get("bundle_id")):
findings.append(Finding("ref_missing", "the payload names no `bundle.bundle_id` (SS 3.1)"))
return findings
def rule_excerpt_source_marking(ctx: Context) -> list[Finding]:
if not ctx.payload_is_mapping:
return []
findings = []
for position, raw in enumerate(_sequence(ctx.payload.get("excerpts"))):
excerpt = _mapping(raw)
for key in ("bundle_id", "concept_id", "sha256"):
if not _text(excerpt.get(key)):
findings.append(
Finding(
"source_marking_incomplete",
f"excerpt {position} carries no {key!r}; identity across "
"bundles is the (bundle_id, concept_id) tuple with the "
"digest of the bytes it was taken from (SS 3.1, SS 3.2)",
)
)
return findings
def rule_excerpt_states(ctx: Context) -> list[Finding]:
if not ctx.payload_is_mapping:
return []
findings = []
closed: tuple[tuple[str, tuple[str, ...]], ...] = (
("adjudication", ADJUDICATION_STATES),
("trust_tier", TRUST_TIERS),
)
for position, raw in enumerate(_sequence(ctx.payload.get("excerpts"))):
excerpt = _mapping(raw)
for key, allowed in closed:
if key not in excerpt:
findings.append(
Finding(
"state_missing",
f"excerpt {position} carries no {key!r}; the third state is "
"written explicitly, never omitted -- omission collapses "
'"we cannot tell" into a fact about the concept (SS 6.1)',
)
)
continue
value = _text(excerpt.get(key))
if value not in allowed:
findings.append(
Finding(
"state_not_in_set",
f"excerpt {position} has {key}={value!r}, outside the closed "
f"set {allowed} (SS 6)",
)
)
return findings
def rule_denominator_identity(ctx: Context) -> list[Finding]:
if not ctx.payload_is_mapping:
return []
counts = _mapping(ctx.payload.get("denominators"))
values = {key: _whole(counts.get(key)) for key in ("considered", "withheld", "delivered")}
missing = sorted(key for key, value in values.items() if value is None)
if missing:
return [
Finding(
"denominator_identity",
f"the payload reports no whole-number {', '.join(missing)}; a count "
"that is not reported is unmeasured, not zero (SS 5.1)",
)
]
considered = values["considered"]
withheld = values["withheld"]
delivered = values["delivered"]
assert considered is not None and withheld is not None and delivered is not None
if considered != withheld + delivered:
return [
Finding(
"denominator_identity",
f"considered ({considered}) != withheld ({withheld}) + delivered "
f"({delivered}); a count that does not close is not a denominator "
"(SS 5.2)",
)
]
return []
def rule_denominator_lists(ctx: Context) -> list[Finding]:
if not ctx.payload_is_mapping:
return []
counts = _mapping(ctx.payload.get("denominators"))
findings = []
for key, member in (("delivered", "excerpts"), ("withheld", "withheld")):
declared = _whole(counts.get(key))
if declared is None:
continue
actual = len(_sequence(ctx.payload.get(member)))
if declared != actual:
findings.append(
Finding(
"denominator_list_mismatch",
f"denominators.{key} is {declared} but {member} holds {actual}; "
"the count and the list are two statements of one fact (SS 8.1)",
)
)
return findings
def rule_withheld_rules(ctx: Context) -> list[Finding]:
if not ctx.payload_is_mapping:
return []
findings = []
for position, raw in enumerate(_sequence(ctx.payload.get("withheld"))):
entry = _mapping(raw)
for key in ("concept_id", "rule"):
if not _text(entry.get(key)):
findings.append(
Finding(
"cut_undeclared",
f"withheld entry {position} carries no {key!r}; a visible "
"drop is worth more than a silent one (SS 5.3)",
)
)
return findings
def rule_budget_declared(ctx: Context) -> list[Finding]:
if not ctx.payload_is_mapping:
return []
budget = _mapping(ctx.payload.get("budget"))
findings = []
for key in ("unit", "instrument"):
if not _text(budget.get(key)):
findings.append(
Finding(
"budget_undeclared",
f"the budget names no {key!r}; a number without its instrument "
"and unit is not a measurement (SS 7.1)",
)
)
limit = _whole(budget.get("limit"))
if limit is None or limit <= 0:
findings.append(
Finding(
"budget_undeclared",
'"bounded" without a bound is a denominator failure in prose (SS 7.1)',
)
)
if _whole(budget.get("spent")) is None:
findings.append(Finding("budget_undeclared", "the budget reports no `spent` (SS 7.2)"))
return findings
def rule_budget_gate(ctx: Context) -> list[Finding]:
if not ctx.payload_is_mapping:
return []
budget = _mapping(ctx.payload.get("budget"))
limit = _whole(budget.get("limit"))
spent = _whole(budget.get("spent"))
if limit is None or spent is None or spent <= limit:
return []
return [
Finding(
"budget_exceeded",
f"spent ({spent}) exceeds limit ({limit}); the cut strategy is wrong "
"for this bundle, which is a finding requiring a decision and never "
"a retry with a narrower question (SS 7.3)",
)
]
def rule_instrument_validated(ctx: Context) -> list[Finding]:
if not ctx.payload_is_mapping:
return []
known = _mapping(_mapping(ctx.payload.get("budget")).get("known_positive"))
expected = _whole(known.get("expected"))
measured = _whole(known.get("measured"))
if expected is None or measured is None or expected <= 0 or not _text(known.get("case")):
return [
Finding(
"instrument_unvalidated",
"the budget carries no usable `known_positive` (case, expected, "
"measured); an instrument that has not reproduced a known figure "
"has not been shown to count (SS 7.4)",
)
]
if expected != measured:
return [
Finding(
"instrument_unvalidated",
f"the known-positive expected {expected} and the instrument "
f"measured {measured} (SS 7.4)",
)
]
return []
# --- Skill rules -------------------------------------------------------------
def rule_skill_sections(ctx: Context) -> list[Finding]:
return [
Finding(
"skill_section_missing",
f"the skill carries no `## {section}` section (SS 8 of this checker's "
"reading; the headings are fixed so they can be checked by name)",
)
for section in REQUIRED_SECTIONS
if f"## {section}" not in ctx.skill
]
def rule_skill_markings(ctx: Context) -> list[Finding]:
return [
Finding(
"skill_marking_missing",
f"the skill does not carry the required marking {marking!r} verbatim "
"(SS 4.1); one literal string, no variants, no translations",
)
for marking in REQUIRED_MARKINGS
if marking not in ctx.skill
]
def rule_skill_states(ctx: Context) -> list[Finding]:
return [
Finding(
"skill_state_missing",
f"the skill does not name the state {state!r}, so its consumer cannot "
"be held to the closed set (SS 6)",
)
for state in (*ADJUDICATION_STATES, *TRUST_TIERS)
if f"`{state}`" not in ctx.skill
]
#: Every rule, in report order. `len(RULES)` is the denominator the report
#: quotes: "how many rules ran" is the number that makes "0 findings" mean
#: something.
RULES: tuple[Callable[[Context], list[Finding]], ...] = (
rule_payload_shape,
rule_contract_version,
rule_bundle_ref,
rule_excerpt_source_marking,
rule_excerpt_states,
rule_denominator_identity,
rule_denominator_lists,
rule_withheld_rules,
rule_budget_declared,
rule_budget_gate,
rule_instrument_validated,
rule_skill_sections,
rule_skill_markings,
rule_skill_states,
)
def check(skill_text: str, payload: object) -> Report:
"""Run every rule. No rule short-circuits another: a caller fixing one
defect should not discover a second only on the next run."""
is_mapping = isinstance(payload, Mapping)
ctx = Context(
skill=skill_text,
payload=payload if isinstance(payload, Mapping) else {},
payload_is_mapping=is_mapping,
)
findings: list[Finding] = []
for rule in RULES:
findings.extend(rule(ctx))
return Report(
findings=tuple(findings),
rules_evaluated=len(RULES),
excerpts_examined=len(_sequence(ctx.payload.get("excerpts"))),
withheld_examined=len(_sequence(ctx.payload.get("withheld"))),
)
def parse_args(argv: list[str] | None) -> argparse.Namespace:
parser = argparse.ArgumentParser(
description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter
)
parser.add_argument("--skill", type=Path, required=True, help="the SKILL.md to check")
parser.add_argument("--payload", type=Path, required=True, help="one pre-pass payload (JSON)")
return parser.parse_args(argv)
def main(argv: list[str] | None = None) -> int:
args = parse_args(argv)
try:
skill_text = args.skill.read_text(encoding="utf-8")
except OSError as exc:
print(f"could not read the skill: {exc}")
return 2
try:
payload = json.loads(args.payload.read_text(encoding="utf-8"))
except OSError as exc:
print(f"could not read the payload: {exc}")
return 2
except json.JSONDecodeError as exc:
print(f"the payload is not readable JSON: {exc}")
return 2
report = check(skill_text, payload)
print(report.render())
return 1 if report.findings else 0
if __name__ == "__main__":
raise SystemExit(main())