feat(check): a skill and a payload naming different bundles is a finding
`okf check` had fifteen rules and none asked whether the skill and the payload
were talking about the same bundle. Reproduced on this HEAD before any code
moved: three pairs reported `conformant: 15 rules over 8 excerpts and 438
withheld entries, 0 findings` -- a skill generated from one corpus against
another corpus's payload, the unfilled template against that payload, and a
payload sharing the skill's `bundle_id` at a foreign `ref`. All three now exit 1
with one `bundle_mismatch` finding over 16 rules.
BOTH halves are compared and the `ref` half is load-bearing: three distinct
builds on this machine carry one `bundle_id`, so an id comparison would pass a
stale skill. SS 3.3: "a version is the producer's assertion; a ref is a fact
about bytes". An identity the rule cannot read is a finding, never a silent
pass -- that is what refuses the unfilled template.
No new field: the identity was already in the generated skill's prose, now
factored into `skill.identity_line` and read back by
`contract_check.skill_identity`. Generated skill bytes unchanged, measured on
both tracked bundles on one interpreter.
The rule's first real find is this repository's own hand-made
`skills/okf-consume/SKILL.md`, which predates `okf skill` and declares no
identity a reader can act on: 1 of 1. Nine tests that asserted the old, false
conformance now pair a skill with its own bundle's payload.
Measured, nothing else moved: `~/okf-test/dokumenter` `diff -r` empty old
source vs new on one interpreter (52 files, 26 concepts), `okf project` still
byte-equal to `okf build`, K2 pin unmodified and green (453 concepts, ranks
1,1,1,1,1,5), known-negative `{}` unchanged at 9 findings.
Report: docs/2026-09-10-k3-runde15-bundle-mismatch.md
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
parent
b5df3355c5
commit
7cca9e079e
9 changed files with 468 additions and 56 deletions
|
|
@ -33,6 +33,7 @@ PROJECT_ROOT = Path(__file__).resolve().parents[1]
|
|||
sys.path.insert(0, str(PROJECT_ROOT / "tools"))
|
||||
|
||||
import okf_contract_check # noqa: E402
|
||||
import okf_skill # noqa: E402
|
||||
|
||||
TEMPLATE = PROJECT_ROOT / "skills" / "okf-consume-template" / "SKILL.md"
|
||||
EXAMPLE = TEMPLATE.parent / "references" / "example-payload.json"
|
||||
|
|
@ -44,6 +45,32 @@ def load_example() -> dict[str, Any]:
|
|||
return payload
|
||||
|
||||
|
||||
def declaring(skill_text: str, payload: dict[str, Any]) -> str:
|
||||
"""The template with the one hole `bundle_mismatch` reads filled in.
|
||||
|
||||
The template alone can no longer be a conformant skill: `<CORPUS>` and
|
||||
`<REF>` are placeholders, and an identity the checker cannot read is a
|
||||
finding by design. The sentence is the generator's own, taken from it
|
||||
rather than copied, so the two cannot drift apart.
|
||||
"""
|
||||
bundle = payload["bundle"]
|
||||
return skill_text.replace(
|
||||
okf_skill.TEMPLATE_HEADER,
|
||||
okf_skill.identity_line(bundle["bundle_id"], bundle["ref"]) + ".",
|
||||
1,
|
||||
)
|
||||
|
||||
|
||||
def instantiated() -> str:
|
||||
return declaring(TEMPLATE.read_text(encoding="utf-8"), load_example())
|
||||
|
||||
|
||||
def instantiated_file(tmp_path: Path) -> Path:
|
||||
target = tmp_path / "SKILL.md"
|
||||
target.write_text(instantiated(), encoding="utf-8")
|
||||
return target
|
||||
|
||||
|
||||
def codes(skill_text: str, payload: Any) -> list[str]:
|
||||
return [finding.code for finding in okf_contract_check.check(skill_text, payload).findings]
|
||||
|
||||
|
|
@ -51,16 +78,20 @@ def codes(skill_text: str, payload: Any) -> list[str]:
|
|||
# --- The known-positive, which every negative arm depends on ----------------
|
||||
|
||||
|
||||
def test_shipped_template_and_example_payload_are_conformant() -> None:
|
||||
"""The CONTROL. A checker that cannot pass anything proves nothing below."""
|
||||
report = okf_contract_check.check(TEMPLATE.read_text(encoding="utf-8"), load_example())
|
||||
def test_an_instantiated_skill_and_its_own_example_payload_are_conformant() -> None:
|
||||
"""The CONTROL. A checker that cannot pass anything proves nothing below.
|
||||
|
||||
It is the INSTANTIATED skill, not the template: since 2026-09-10 a skill
|
||||
declaring no readable bundle identity is `bundle_mismatch`, measured on
|
||||
the template itself in `test_bundle_identity.py`."""
|
||||
report = okf_contract_check.check(instantiated(), load_example())
|
||||
assert report.findings == ()
|
||||
|
||||
|
||||
def test_the_report_carries_its_denominators() -> None:
|
||||
""" "Conformant" without a denominator is unmeasured, per the contract's own
|
||||
section 5. The report says how many rules ran over how many units."""
|
||||
report = okf_contract_check.check(TEMPLATE.read_text(encoding="utf-8"), load_example())
|
||||
report = okf_contract_check.check(instantiated(), load_example())
|
||||
assert report.rules_evaluated == len(okf_contract_check.RULES)
|
||||
assert report.rules_evaluated > 0
|
||||
assert report.excerpts_examined == 3
|
||||
|
|
@ -76,112 +107,112 @@ def test_the_report_carries_its_denominators() -> None:
|
|||
def test_broken_denominator_identity_is_named() -> None:
|
||||
payload = load_example()
|
||||
payload["denominators"]["considered"] = 6
|
||||
assert "denominator_identity" in codes(TEMPLATE.read_text(encoding="utf-8"), payload)
|
||||
assert "denominator_identity" in codes(instantiated(), payload)
|
||||
|
||||
|
||||
def test_delivered_count_disagreeing_with_the_list_is_named() -> None:
|
||||
payload = load_example()
|
||||
payload["excerpts"] = payload["excerpts"][:2]
|
||||
assert "denominator_list_mismatch" in codes(TEMPLATE.read_text(encoding="utf-8"), payload)
|
||||
assert "denominator_list_mismatch" in codes(instantiated(), payload)
|
||||
|
||||
|
||||
def test_a_missing_adjudication_key_is_not_absence() -> None:
|
||||
"""The contract's sharpest rule: `unknown` is written, never omitted."""
|
||||
payload = load_example()
|
||||
del payload["excerpts"][0]["adjudication"]
|
||||
assert "state_missing" in codes(TEMPLATE.read_text(encoding="utf-8"), payload)
|
||||
assert "state_missing" in codes(instantiated(), payload)
|
||||
|
||||
|
||||
def test_an_adjudication_value_outside_the_closed_set_is_named() -> None:
|
||||
payload = load_example()
|
||||
payload["excerpts"][0]["adjudication"] = "absent"
|
||||
assert "state_not_in_set" in codes(TEMPLATE.read_text(encoding="utf-8"), payload)
|
||||
assert "state_not_in_set" in codes(instantiated(), payload)
|
||||
|
||||
|
||||
def test_a_trust_tier_outside_the_closed_set_is_named() -> None:
|
||||
payload = load_example()
|
||||
payload["excerpts"][1]["trust_tier"] = "trusted"
|
||||
assert "state_not_in_set" in codes(TEMPLATE.read_text(encoding="utf-8"), payload)
|
||||
assert "state_not_in_set" in codes(instantiated(), payload)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("key", ["bundle_id", "concept_id", "sha256"])
|
||||
def test_an_excerpt_missing_its_source_marking_is_named(key: str) -> None:
|
||||
payload = load_example()
|
||||
del payload["excerpts"][0][key]
|
||||
assert "source_marking_incomplete" in codes(TEMPLATE.read_text(encoding="utf-8"), payload)
|
||||
assert "source_marking_incomplete" in codes(instantiated(), payload)
|
||||
|
||||
|
||||
def test_a_bundle_without_a_ref_is_named() -> None:
|
||||
payload = load_example()
|
||||
payload["bundle"]["ref"] = ""
|
||||
assert "ref_missing" in codes(TEMPLATE.read_text(encoding="utf-8"), payload)
|
||||
assert "ref_missing" in codes(instantiated(), payload)
|
||||
|
||||
|
||||
def test_a_withheld_concept_without_a_rule_is_an_undeclared_cut() -> None:
|
||||
payload = load_example()
|
||||
del payload["withheld"][0]["rule"]
|
||||
assert "cut_undeclared" in codes(TEMPLATE.read_text(encoding="utf-8"), payload)
|
||||
assert "cut_undeclared" in codes(instantiated(), payload)
|
||||
|
||||
|
||||
def test_spending_over_the_limit_is_the_gate_firing() -> None:
|
||||
payload = load_example()
|
||||
payload["budget"]["spent"] = payload["budget"]["limit"] + 1
|
||||
assert "budget_exceeded" in codes(TEMPLATE.read_text(encoding="utf-8"), payload)
|
||||
assert "budget_exceeded" in codes(instantiated(), payload)
|
||||
|
||||
|
||||
def test_an_instrument_that_missed_its_known_positive_is_named() -> None:
|
||||
payload = load_example()
|
||||
payload["budget"]["known_positive"]["measured"] = 10405
|
||||
assert "instrument_unvalidated" in codes(TEMPLATE.read_text(encoding="utf-8"), payload)
|
||||
assert "instrument_unvalidated" in codes(instantiated(), payload)
|
||||
|
||||
|
||||
def test_a_budget_without_an_instrument_is_named() -> None:
|
||||
payload = load_example()
|
||||
payload["budget"]["instrument"] = ""
|
||||
assert "budget_undeclared" in codes(TEMPLATE.read_text(encoding="utf-8"), payload)
|
||||
assert "budget_undeclared" in codes(instantiated(), payload)
|
||||
|
||||
|
||||
def test_a_payload_naming_no_contract_revision_is_named() -> None:
|
||||
payload = load_example()
|
||||
del payload["contract"]
|
||||
assert "contract_unversioned" in codes(TEMPLATE.read_text(encoding="utf-8"), payload)
|
||||
assert "contract_unversioned" in codes(instantiated(), payload)
|
||||
|
||||
|
||||
def test_a_payload_that_is_not_a_mapping_is_refused_not_crashed() -> None:
|
||||
assert "payload_invalid" in codes(TEMPLATE.read_text(encoding="utf-8"), ["not", "a", "mapping"])
|
||||
assert "payload_invalid" in codes(instantiated(), ["not", "a", "mapping"])
|
||||
|
||||
|
||||
# --- Skill mutations --------------------------------------------------------
|
||||
|
||||
|
||||
def test_a_missing_required_section_is_named() -> None:
|
||||
text = TEMPLATE.read_text(encoding="utf-8").replace("## Denominators", "## Counts")
|
||||
text = instantiated().replace("## Denominators", "## Counts")
|
||||
assert "skill_section_missing" in codes(text, load_example())
|
||||
|
||||
|
||||
def test_a_missing_required_marking_is_named() -> None:
|
||||
text = TEMPLATE.read_text(encoding="utf-8").replace("[sourced-not-sufficient]", "[thin]")
|
||||
text = instantiated().replace("[sourced-not-sufficient]", "[thin]")
|
||||
assert "skill_marking_missing" in codes(text, load_example())
|
||||
|
||||
|
||||
def test_a_translated_marking_literal_does_not_count() -> None:
|
||||
"""One literal string, no variants, no translations."""
|
||||
text = TEMPLATE.read_text(encoding="utf-8").replace(
|
||||
"[unverifiable-from-bundle]", "[ikke-verifiserbar-fra-bundle]"
|
||||
)
|
||||
text = instantiated().replace("[unverifiable-from-bundle]", "[ikke-verifiserbar-fra-bundle]")
|
||||
assert "skill_marking_missing" in codes(text, load_example())
|
||||
|
||||
|
||||
def test_a_skill_omitting_an_adjudication_value_is_named() -> None:
|
||||
text = TEMPLATE.read_text(encoding="utf-8").replace("`unknown`", "`missing`")
|
||||
text = instantiated().replace("`unknown`", "`missing`")
|
||||
assert "skill_state_missing" in codes(text, load_example())
|
||||
|
||||
|
||||
# --- The command line -------------------------------------------------------
|
||||
|
||||
|
||||
def test_the_command_exits_zero_on_the_shipped_pair() -> None:
|
||||
exit_code = okf_contract_check.main(["--skill", str(TEMPLATE), "--payload", str(EXAMPLE)])
|
||||
def test_the_command_exits_zero_on_an_instantiated_pair(tmp_path: Path) -> None:
|
||||
exit_code = okf_contract_check.main(
|
||||
["--skill", str(instantiated_file(tmp_path)), "--payload", str(EXAMPLE)]
|
||||
)
|
||||
assert exit_code == 0
|
||||
|
||||
|
||||
|
|
@ -190,7 +221,12 @@ def test_the_command_exits_one_on_a_non_conformant_payload(tmp_path: Path) -> No
|
|||
payload["denominators"]["considered"] = 6
|
||||
broken = tmp_path / "payload.json"
|
||||
broken.write_text(json.dumps(payload), encoding="utf-8")
|
||||
assert okf_contract_check.main(["--skill", str(TEMPLATE), "--payload", str(broken)]) == 1
|
||||
assert (
|
||||
okf_contract_check.main(
|
||||
["--skill", str(instantiated_file(tmp_path)), "--payload", str(broken)]
|
||||
)
|
||||
== 1
|
||||
)
|
||||
|
||||
|
||||
def test_the_command_separates_could_not_run_from_non_conformant(tmp_path: Path) -> None:
|
||||
|
|
@ -204,13 +240,13 @@ def test_the_command_separates_could_not_run_from_non_conformant(tmp_path: Path)
|
|||
assert okf_contract_check.main(["--skill", str(TEMPLATE), "--payload", str(unreadable)]) == 2
|
||||
|
||||
|
||||
def test_the_module_runs_as_a_script() -> None:
|
||||
def test_the_module_runs_as_a_script(tmp_path: Path) -> None:
|
||||
result = subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
str(PROJECT_ROOT / "tools" / "okf_contract_check.py"),
|
||||
"--skill",
|
||||
str(TEMPLATE),
|
||||
str(instantiated_file(tmp_path)),
|
||||
"--payload",
|
||||
str(EXAMPLE),
|
||||
],
|
||||
|
|
|
|||
|
|
@ -32,6 +32,7 @@ import sys
|
|||
import unicodedata
|
||||
from collections.abc import Mapping
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
|
|
@ -41,10 +42,28 @@ sys.path.insert(0, str(PROJECT_ROOT / "tools"))
|
|||
import okf_consume # noqa: E402
|
||||
import okf_consume_measure # noqa: E402
|
||||
import okf_contract_check # noqa: E402
|
||||
import okf_skill # noqa: E402
|
||||
|
||||
from llm_ingestion_okf.materialize import parse_frontmatter # noqa: E402
|
||||
|
||||
TEMPLATE = PROJECT_ROOT / "skills" / "okf-consume-template" / "SKILL.md"
|
||||
|
||||
|
||||
def _skill_declaring(payload: dict[str, Any]) -> str:
|
||||
"""A skill declaring the bundle THIS payload declares.
|
||||
|
||||
The template cannot stand in for one any more: `<CORPUS>` and `<REF>` are
|
||||
placeholders, and since 2026-09-10 an identity `okf check` cannot read is
|
||||
a `bundle_mismatch` finding. The sentence comes from the generator rather
|
||||
than being copied beside it."""
|
||||
bundle = payload["bundle"]
|
||||
return TEMPLATE.read_text(encoding="utf-8").replace(
|
||||
okf_skill.TEMPLATE_HEADER,
|
||||
okf_skill.identity_line(bundle["bundle_id"], bundle["ref"]) + ".",
|
||||
1,
|
||||
)
|
||||
|
||||
|
||||
GOLDEN = PROJECT_ROOT / "examples" / "ingest-golden-segmented-okf-v0-2" / "expected-bundle"
|
||||
|
||||
|
||||
|
|
@ -730,8 +749,9 @@ def _payload(
|
|||
return okf_consume.build_payload(root, question=question, **kwargs) # type: ignore[arg-type]
|
||||
|
||||
|
||||
def test_the_payload_passes_the_checker_against_the_template_with_zero_findings() -> None:
|
||||
report = okf_contract_check.check(TEMPLATE.read_text(encoding="utf-8"), _payload())
|
||||
def test_the_payload_passes_the_checker_against_a_skill_for_its_own_bundle() -> None:
|
||||
payload = _payload()
|
||||
report = okf_contract_check.check(_skill_declaring(payload), payload)
|
||||
assert report.findings == ()
|
||||
|
||||
|
||||
|
|
@ -855,7 +875,7 @@ def test_a_question_with_no_answer_returns_a_measured_empty_set_not_a_guess() ->
|
|||
|
||||
def test_the_empty_payload_still_passes_the_checker() -> None:
|
||||
payload = _payload(question="Hva er reglene for sveising av titan i vakuum?")
|
||||
assert okf_contract_check.check(TEMPLATE.read_text(encoding="utf-8"), payload).findings == ()
|
||||
assert okf_contract_check.check(_skill_declaring(payload), payload).findings == ()
|
||||
|
||||
|
||||
# --- Corpus-conditional arms --------------------------------------------------
|
||||
|
|
@ -1072,12 +1092,16 @@ def test_the_payload_written_by_the_cli_passes_the_checker(tmp_path: Path) -> No
|
|||
assert (
|
||||
_run(str(FIXTURE), "--question", "Hvordan skal prisene fylles ut?", "--out", str(out))
|
||||
).returncode == 0
|
||||
skill_file = tmp_path / "SKILL.md"
|
||||
skill_file.write_text(
|
||||
_skill_declaring(json.loads(out.read_text(encoding="utf-8"))), encoding="utf-8"
|
||||
)
|
||||
checked = subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
str(PROJECT_ROOT / "tools" / "okf_contract_check.py"),
|
||||
"--skill",
|
||||
str(TEMPLATE),
|
||||
str(skill_file),
|
||||
"--payload",
|
||||
str(out),
|
||||
],
|
||||
|
|
@ -1134,9 +1158,14 @@ def test_every_rule_the_pre_pass_can_emit_is_named_in_the_skill() -> None:
|
|||
assert rule in text, rule
|
||||
|
||||
|
||||
def test_the_skill_and_a_real_payload_pass_the_checker_together() -> None:
|
||||
payload = _payload()
|
||||
assert okf_contract_check.check(SKILL.read_text(encoding="utf-8"), payload).findings == ()
|
||||
def test_the_skill_and_a_real_payload_pass_the_checker_together(tmp_path: Path) -> None:
|
||||
# A GENERATED skill, against a payload from the bundle it was generated for.
|
||||
# The shipped `skills/okf-consume/SKILL.md` cannot serve here: it predates
|
||||
# `okf skill` and declares no bundle identity a reader can act on, which is
|
||||
# a `bundle_mismatch` finding and is recorded as one rather than worked
|
||||
# around.
|
||||
text, payload = okf_skill.render(GOLDEN, out=tmp_path / "skill")
|
||||
assert okf_contract_check.check(text, payload).findings == ()
|
||||
|
||||
|
||||
def test_the_shipped_example_payload_is_current_and_regenerates_byte_for_byte() -> None:
|
||||
|
|
@ -1793,9 +1822,8 @@ def test_a_payload_carrying_a_reservation_still_passes_the_checker(tmp_path: Pat
|
|||
# SS 8 permits additional members; a declaration the checker refuses would
|
||||
# buy honesty at the price of conformance.
|
||||
root = _eviction_bundle(tmp_path / "bundle")
|
||||
report = okf_contract_check.check(
|
||||
TEMPLATE.read_text(encoding="utf-8"), _eviction_payload(root, reserve_top_rank=True)
|
||||
)
|
||||
payload = _eviction_payload(root, reserve_top_rank=True)
|
||||
report = okf_contract_check.check(_skill_declaring(payload), payload)
|
||||
assert report.findings == ()
|
||||
|
||||
|
||||
|
|
@ -2492,9 +2520,10 @@ def test_the_checker_refuses_an_excerpt_that_cannot_be_named() -> None:
|
|||
# `title` is the defect po measured, and a checker that passes it certifies
|
||||
# a payload a model cannot cite from.
|
||||
payload = json.loads((SKILL.parent / "references" / "example-payload.json").read_text("utf-8"))
|
||||
assert okf_contract_check.check(SKILL.read_text(encoding="utf-8"), payload).findings == ()
|
||||
skill_text = _skill_declaring(payload)
|
||||
assert okf_contract_check.check(skill_text, payload).findings == ()
|
||||
del payload["excerpts"][0]["title"]
|
||||
codes = [f.code for f in okf_contract_check.check(SKILL.read_text("utf-8"), payload).findings]
|
||||
codes = [f.code for f in okf_contract_check.check(skill_text, payload).findings]
|
||||
assert codes == ["excerpt_unnamed"]
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -2,9 +2,15 @@
|
|||
|
||||
`tools/okf_skill.py` turns one OKF bundle into one instantiated `SKILL.md` that
|
||||
`tools/okf_contract_check.py` accepts. The discipline here is the one measurement
|
||||
that decided the form: **the checker cannot tell an instantiated skill from an
|
||||
unfilled template**, and passes a skill built for a different bundle against this
|
||||
that decided the form: the checker could not tell an instantiated skill from an
|
||||
unfilled template, and passed a skill built for a different bundle against this
|
||||
one's payload. So every gate the checker does not have is a test here.
|
||||
|
||||
**That measurement is closed on its identity half since 2026-09-10.** The
|
||||
`bundle_mismatch` rule refuses both forms, and `tests/test_bundle_identity.py`
|
||||
holds the arms. The gates below are the ones it still does not have: what the
|
||||
generated skill MEASURES -- the per-bundle denominators, the breaking point,
|
||||
the conditional-field list -- is not something any static pairing check reaches.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue