feat(check): a skill and a payload naming different bundles is a finding
`okf check` had fifteen rules and none asked whether the skill and the payload
were talking about the same bundle. Reproduced on this HEAD before any code
moved: three pairs reported `conformant: 15 rules over 8 excerpts and 438
withheld entries, 0 findings` -- a skill generated from one corpus against
another corpus's payload, the unfilled template against that payload, and a
payload sharing the skill's `bundle_id` at a foreign `ref`. All three now exit 1
with one `bundle_mismatch` finding over 16 rules.
BOTH halves are compared and the `ref` half is load-bearing: three distinct
builds on this machine carry one `bundle_id`, so an id comparison would pass a
stale skill. SS 3.3: "a version is the producer's assertion; a ref is a fact
about bytes". An identity the rule cannot read is a finding, never a silent
pass -- that is what refuses the unfilled template.
No new field: the identity was already in the generated skill's prose, now
factored into `skill.identity_line` and read back by
`contract_check.skill_identity`. Generated skill bytes unchanged, measured on
both tracked bundles on one interpreter.
The rule's first real find is this repository's own hand-made
`skills/okf-consume/SKILL.md`, which predates `okf skill` and declares no
identity a reader can act on: 1 of 1. Nine tests that asserted the old, false
conformance now pair a skill with its own bundle's payload.
Measured, nothing else moved: `~/okf-test/dokumenter` `diff -r` empty old
source vs new on one interpreter (52 files, 26 concepts), `okf project` still
byte-equal to `okf build`, K2 pin unmodified and green (453 concepts, ranks
1,1,1,1,1,5), known-negative `{}` unchanged at 9 findings.
Report: docs/2026-09-10-k3-runde15-bundle-mismatch.md
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
parent
b5df3355c5
commit
7cca9e079e
9 changed files with 468 additions and 56 deletions
|
|
@ -32,6 +32,7 @@ import sys
|
|||
import unicodedata
|
||||
from collections.abc import Mapping
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
|
|
@ -41,10 +42,28 @@ sys.path.insert(0, str(PROJECT_ROOT / "tools"))
|
|||
import okf_consume # noqa: E402
|
||||
import okf_consume_measure # noqa: E402
|
||||
import okf_contract_check # noqa: E402
|
||||
import okf_skill # noqa: E402
|
||||
|
||||
from llm_ingestion_okf.materialize import parse_frontmatter # noqa: E402
|
||||
|
||||
TEMPLATE = PROJECT_ROOT / "skills" / "okf-consume-template" / "SKILL.md"
|
||||
|
||||
|
||||
def _skill_declaring(payload: dict[str, Any]) -> str:
|
||||
"""A skill declaring the bundle THIS payload declares.
|
||||
|
||||
The template cannot stand in for one any more: `<CORPUS>` and `<REF>` are
|
||||
placeholders, and since 2026-09-10 an identity `okf check` cannot read is
|
||||
a `bundle_mismatch` finding. The sentence comes from the generator rather
|
||||
than being copied beside it."""
|
||||
bundle = payload["bundle"]
|
||||
return TEMPLATE.read_text(encoding="utf-8").replace(
|
||||
okf_skill.TEMPLATE_HEADER,
|
||||
okf_skill.identity_line(bundle["bundle_id"], bundle["ref"]) + ".",
|
||||
1,
|
||||
)
|
||||
|
||||
|
||||
GOLDEN = PROJECT_ROOT / "examples" / "ingest-golden-segmented-okf-v0-2" / "expected-bundle"
|
||||
|
||||
|
||||
|
|
@ -730,8 +749,9 @@ def _payload(
|
|||
return okf_consume.build_payload(root, question=question, **kwargs) # type: ignore[arg-type]
|
||||
|
||||
|
||||
def test_the_payload_passes_the_checker_against_the_template_with_zero_findings() -> None:
|
||||
report = okf_contract_check.check(TEMPLATE.read_text(encoding="utf-8"), _payload())
|
||||
def test_the_payload_passes_the_checker_against_a_skill_for_its_own_bundle() -> None:
|
||||
payload = _payload()
|
||||
report = okf_contract_check.check(_skill_declaring(payload), payload)
|
||||
assert report.findings == ()
|
||||
|
||||
|
||||
|
|
@ -855,7 +875,7 @@ def test_a_question_with_no_answer_returns_a_measured_empty_set_not_a_guess() ->
|
|||
|
||||
def test_the_empty_payload_still_passes_the_checker() -> None:
|
||||
payload = _payload(question="Hva er reglene for sveising av titan i vakuum?")
|
||||
assert okf_contract_check.check(TEMPLATE.read_text(encoding="utf-8"), payload).findings == ()
|
||||
assert okf_contract_check.check(_skill_declaring(payload), payload).findings == ()
|
||||
|
||||
|
||||
# --- Corpus-conditional arms --------------------------------------------------
|
||||
|
|
@ -1072,12 +1092,16 @@ def test_the_payload_written_by_the_cli_passes_the_checker(tmp_path: Path) -> No
|
|||
assert (
|
||||
_run(str(FIXTURE), "--question", "Hvordan skal prisene fylles ut?", "--out", str(out))
|
||||
).returncode == 0
|
||||
skill_file = tmp_path / "SKILL.md"
|
||||
skill_file.write_text(
|
||||
_skill_declaring(json.loads(out.read_text(encoding="utf-8"))), encoding="utf-8"
|
||||
)
|
||||
checked = subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
str(PROJECT_ROOT / "tools" / "okf_contract_check.py"),
|
||||
"--skill",
|
||||
str(TEMPLATE),
|
||||
str(skill_file),
|
||||
"--payload",
|
||||
str(out),
|
||||
],
|
||||
|
|
@ -1134,9 +1158,14 @@ def test_every_rule_the_pre_pass_can_emit_is_named_in_the_skill() -> None:
|
|||
assert rule in text, rule
|
||||
|
||||
|
||||
def test_the_skill_and_a_real_payload_pass_the_checker_together() -> None:
|
||||
payload = _payload()
|
||||
assert okf_contract_check.check(SKILL.read_text(encoding="utf-8"), payload).findings == ()
|
||||
def test_the_skill_and_a_real_payload_pass_the_checker_together(tmp_path: Path) -> None:
|
||||
# A GENERATED skill, against a payload from the bundle it was generated for.
|
||||
# The shipped `skills/okf-consume/SKILL.md` cannot serve here: it predates
|
||||
# `okf skill` and declares no bundle identity a reader can act on, which is
|
||||
# a `bundle_mismatch` finding and is recorded as one rather than worked
|
||||
# around.
|
||||
text, payload = okf_skill.render(GOLDEN, out=tmp_path / "skill")
|
||||
assert okf_contract_check.check(text, payload).findings == ()
|
||||
|
||||
|
||||
def test_the_shipped_example_payload_is_current_and_regenerates_byte_for_byte() -> None:
|
||||
|
|
@ -1793,9 +1822,8 @@ def test_a_payload_carrying_a_reservation_still_passes_the_checker(tmp_path: Pat
|
|||
# SS 8 permits additional members; a declaration the checker refuses would
|
||||
# buy honesty at the price of conformance.
|
||||
root = _eviction_bundle(tmp_path / "bundle")
|
||||
report = okf_contract_check.check(
|
||||
TEMPLATE.read_text(encoding="utf-8"), _eviction_payload(root, reserve_top_rank=True)
|
||||
)
|
||||
payload = _eviction_payload(root, reserve_top_rank=True)
|
||||
report = okf_contract_check.check(_skill_declaring(payload), payload)
|
||||
assert report.findings == ()
|
||||
|
||||
|
||||
|
|
@ -2492,9 +2520,10 @@ def test_the_checker_refuses_an_excerpt_that_cannot_be_named() -> None:
|
|||
# `title` is the defect po measured, and a checker that passes it certifies
|
||||
# a payload a model cannot cite from.
|
||||
payload = json.loads((SKILL.parent / "references" / "example-payload.json").read_text("utf-8"))
|
||||
assert okf_contract_check.check(SKILL.read_text(encoding="utf-8"), payload).findings == ()
|
||||
skill_text = _skill_declaring(payload)
|
||||
assert okf_contract_check.check(skill_text, payload).findings == ()
|
||||
del payload["excerpts"][0]["title"]
|
||||
codes = [f.code for f in okf_contract_check.check(SKILL.read_text("utf-8"), payload).findings]
|
||||
codes = [f.code for f in okf_contract_check.check(skill_text, payload).findings]
|
||||
assert codes == ["excerpt_unnamed"]
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue