feat(check): a skill and a payload naming different bundles is a finding

`okf check` had fifteen rules and none asked whether the skill and the payload
were talking about the same bundle. Reproduced on this HEAD before any code
moved: three pairs reported `conformant: 15 rules over 8 excerpts and 438
withheld entries, 0 findings` -- a skill generated from one corpus against
another corpus's payload, the unfilled template against that payload, and a
payload sharing the skill's `bundle_id` at a foreign `ref`. All three now exit 1
with one `bundle_mismatch` finding over 16 rules.

BOTH halves are compared and the `ref` half is load-bearing: three distinct
builds on this machine carry one `bundle_id`, so an id comparison would pass a
stale skill. SS 3.3: "a version is the producer's assertion; a ref is a fact
about bytes". An identity the rule cannot read is a finding, never a silent
pass -- that is what refuses the unfilled template.

No new field: the identity was already in the generated skill's prose, now
factored into `skill.identity_line` and read back by
`contract_check.skill_identity`. Generated skill bytes unchanged, measured on
both tracked bundles on one interpreter.

The rule's first real find is this repository's own hand-made
`skills/okf-consume/SKILL.md`, which predates `okf skill` and declares no
identity a reader can act on: 1 of 1. Nine tests that asserted the old, false
conformance now pair a skill with its own bundle's payload.

Measured, nothing else moved: `~/okf-test/dokumenter` `diff -r` empty old
source vs new on one interpreter (52 files, 26 concepts), `okf project` still
byte-equal to `okf build`, K2 pin unmodified and green (453 concepts, ranks
1,1,1,1,1,5), known-negative `{}` unchanged at 9 findings.

Report: docs/2026-09-10-k3-runde15-bundle-mismatch.md

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Kjell Tore Guttormsen 2026-09-10 23:42:06 +02:00
commit 7cca9e079e
9 changed files with 468 additions and 56 deletions

View file

@ -32,6 +32,7 @@ import sys
import unicodedata
from collections.abc import Mapping
from pathlib import Path
from typing import Any
import pytest
@ -41,10 +42,28 @@ sys.path.insert(0, str(PROJECT_ROOT / "tools"))
import okf_consume # noqa: E402
import okf_consume_measure # noqa: E402
import okf_contract_check # noqa: E402
import okf_skill # noqa: E402
from llm_ingestion_okf.materialize import parse_frontmatter # noqa: E402
TEMPLATE = PROJECT_ROOT / "skills" / "okf-consume-template" / "SKILL.md"
def _skill_declaring(payload: dict[str, Any]) -> str:
"""A skill declaring the bundle THIS payload declares.
The template cannot stand in for one any more: `<CORPUS>` and `<REF>` are
placeholders, and since 2026-09-10 an identity `okf check` cannot read is
a `bundle_mismatch` finding. The sentence comes from the generator rather
than being copied beside it."""
bundle = payload["bundle"]
return TEMPLATE.read_text(encoding="utf-8").replace(
okf_skill.TEMPLATE_HEADER,
okf_skill.identity_line(bundle["bundle_id"], bundle["ref"]) + ".",
1,
)
GOLDEN = PROJECT_ROOT / "examples" / "ingest-golden-segmented-okf-v0-2" / "expected-bundle"
@ -730,8 +749,9 @@ def _payload(
return okf_consume.build_payload(root, question=question, **kwargs) # type: ignore[arg-type]
def test_the_payload_passes_the_checker_against_the_template_with_zero_findings() -> None:
report = okf_contract_check.check(TEMPLATE.read_text(encoding="utf-8"), _payload())
def test_the_payload_passes_the_checker_against_a_skill_for_its_own_bundle() -> None:
payload = _payload()
report = okf_contract_check.check(_skill_declaring(payload), payload)
assert report.findings == ()
@ -855,7 +875,7 @@ def test_a_question_with_no_answer_returns_a_measured_empty_set_not_a_guess() ->
def test_the_empty_payload_still_passes_the_checker() -> None:
payload = _payload(question="Hva er reglene for sveising av titan i vakuum?")
assert okf_contract_check.check(TEMPLATE.read_text(encoding="utf-8"), payload).findings == ()
assert okf_contract_check.check(_skill_declaring(payload), payload).findings == ()
# --- Corpus-conditional arms --------------------------------------------------
@ -1072,12 +1092,16 @@ def test_the_payload_written_by_the_cli_passes_the_checker(tmp_path: Path) -> No
assert (
_run(str(FIXTURE), "--question", "Hvordan skal prisene fylles ut?", "--out", str(out))
).returncode == 0
skill_file = tmp_path / "SKILL.md"
skill_file.write_text(
_skill_declaring(json.loads(out.read_text(encoding="utf-8"))), encoding="utf-8"
)
checked = subprocess.run(
[
sys.executable,
str(PROJECT_ROOT / "tools" / "okf_contract_check.py"),
"--skill",
str(TEMPLATE),
str(skill_file),
"--payload",
str(out),
],
@ -1134,9 +1158,14 @@ def test_every_rule_the_pre_pass_can_emit_is_named_in_the_skill() -> None:
assert rule in text, rule
def test_the_skill_and_a_real_payload_pass_the_checker_together() -> None:
payload = _payload()
assert okf_contract_check.check(SKILL.read_text(encoding="utf-8"), payload).findings == ()
def test_the_skill_and_a_real_payload_pass_the_checker_together(tmp_path: Path) -> None:
# A GENERATED skill, against a payload from the bundle it was generated for.
# The shipped `skills/okf-consume/SKILL.md` cannot serve here: it predates
# `okf skill` and declares no bundle identity a reader can act on, which is
# a `bundle_mismatch` finding and is recorded as one rather than worked
# around.
text, payload = okf_skill.render(GOLDEN, out=tmp_path / "skill")
assert okf_contract_check.check(text, payload).findings == ()
def test_the_shipped_example_payload_is_current_and_regenerates_byte_for_byte() -> None:
@ -1793,9 +1822,8 @@ def test_a_payload_carrying_a_reservation_still_passes_the_checker(tmp_path: Pat
# SS 8 permits additional members; a declaration the checker refuses would
# buy honesty at the price of conformance.
root = _eviction_bundle(tmp_path / "bundle")
report = okf_contract_check.check(
TEMPLATE.read_text(encoding="utf-8"), _eviction_payload(root, reserve_top_rank=True)
)
payload = _eviction_payload(root, reserve_top_rank=True)
report = okf_contract_check.check(_skill_declaring(payload), payload)
assert report.findings == ()
@ -2492,9 +2520,10 @@ def test_the_checker_refuses_an_excerpt_that_cannot_be_named() -> None:
# `title` is the defect po measured, and a checker that passes it certifies
# a payload a model cannot cite from.
payload = json.loads((SKILL.parent / "references" / "example-payload.json").read_text("utf-8"))
assert okf_contract_check.check(SKILL.read_text(encoding="utf-8"), payload).findings == ()
skill_text = _skill_declaring(payload)
assert okf_contract_check.check(skill_text, payload).findings == ()
del payload["excerpts"][0]["title"]
codes = [f.code for f in okf_contract_check.check(SKILL.read_text("utf-8"), payload).findings]
codes = [f.code for f in okf_contract_check.check(skill_text, payload).findings]
assert codes == ["excerpt_unnamed"]