open/llm-ingestion-okf
1
0
Fork 0
Code Issues Releases 1 Activity

feat(check): a skill and a payload naming different bundles is a finding

Browse source 
`okf check` had fifteen rules and none asked whether the skill and the payload
were talking about the same bundle. Reproduced on this HEAD before any code
moved: three pairs reported `conformant: 15 rules over 8 excerpts and 438
withheld entries, 0 findings` -- a skill generated from one corpus against
another corpus's payload, the unfilled template against that payload, and a
payload sharing the skill's `bundle_id` at a foreign `ref`. All three now exit 1
with one `bundle_mismatch` finding over 16 rules.

BOTH halves are compared and the `ref` half is load-bearing: three distinct
builds on this machine carry one `bundle_id`, so an id comparison would pass a
stale skill. SS 3.3: "a version is the producer's assertion; a ref is a fact
about bytes". An identity the rule cannot read is a finding, never a silent
pass -- that is what refuses the unfilled template.

No new field: the identity was already in the generated skill's prose, now
factored into `skill.identity_line` and read back by
`contract_check.skill_identity`. Generated skill bytes unchanged, measured on
both tracked bundles on one interpreter.

The rule's first real find is this repository's own hand-made
`skills/okf-consume/SKILL.md`, which predates `okf skill` and declares no
identity a reader can act on: 1 of 1. Nine tests that asserted the old, false
conformance now pair a skill with its own bundle's payload.

Measured, nothing else moved: `~/okf-test/dokumenter` `diff -r` empty old
source vs new on one interpreter (52 files, 26 concepts), `okf project` still
byte-equal to `okf build`, K2 pin unmodified and green (453 concepts, ranks
1,1,1,1,1,5), known-negative `{}` unchanged at 9 findings.

Report: docs/2026-09-10-k3-runde15-bundle-mismatch.md

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Kjell Tore Guttormsen 2026-09-10 23:42:06 +02:00
commit 7cca9e079e
9 changed files with 468 additions and 56 deletions
Download patch file Download diff file Expand all files Collapse all files

92
tests/test_contract_check.py
View file

@ -33,6 +33,7 @@ PROJECT_ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(PROJECT_ROOT / "tools"))
import okf_contract_check # noqa: E402
import okf_skill # noqa: E402
TEMPLATE = PROJECT_ROOT / "skills" / "okf-consume-template" / "SKILL.md"
EXAMPLE = TEMPLATE.parent / "references" / "example-payload.json"
@ -44,6 +45,32 @@ def load_example() -> dict[str, Any]:
return payload
def declaring(skill_text: str, payload: dict[str, Any]) -> str:
"""The template with the one hole `bundle_mismatch` reads filled in.
The template alone can no longer be a conformant skill: `<CORPUS>` and
`<REF>` are placeholders, and an identity the checker cannot read is a
finding by design. The sentence is the generator's own, taken from it
rather than copied, so the two cannot drift apart.
"""
bundle = payload["bundle"]
return skill_text.replace(
okf_skill.TEMPLATE_HEADER,
okf_skill.identity_line(bundle["bundle_id"], bundle["ref"]) + ".",
1,
)
def instantiated() -> str:
return declaring(TEMPLATE.read_text(encoding="utf-8"), load_example())
def instantiated_file(tmp_path: Path) -> Path:
target = tmp_path / "SKILL.md"
target.write_text(instantiated(), encoding="utf-8")
return target
def codes(skill_text: str, payload: Any) -> list[str]:
return [finding.code for finding in okf_contract_check.check(skill_text, payload).findings]
@ -51,16 +78,20 @@ def codes(skill_text: str, payload: Any) -> list[str]:
# --- The known-positive, which every negative arm depends on ----------------
def test_shipped_template_and_example_payload_are_conformant() -> None:
"""The CONTROL. A checker that cannot pass anything proves nothing below."""
report = okf_contract_check.check(TEMPLATE.read_text(encoding="utf-8"), load_example())
def test_an_instantiated_skill_and_its_own_example_payload_are_conformant() -> None:
"""The CONTROL. A checker that cannot pass anything proves nothing below.
It is the INSTANTIATED skill, not the template: since 2026-09-10 a skill
declaring no readable bundle identity is `bundle_mismatch`, measured on
the template itself in `test_bundle_identity.py`."""
report = okf_contract_check.check(instantiated(), load_example())
assert report.findings == ()
def test_the_report_carries_its_denominators() -> None:
""" "Conformant" without a denominator is unmeasured, per the contract's own
section 5. The report says how many rules ran over how many units."""
report = okf_contract_check.check(TEMPLATE.read_text(encoding="utf-8"), load_example())
report = okf_contract_check.check(instantiated(), load_example())
assert report.rules_evaluated == len(okf_contract_check.RULES)
assert report.rules_evaluated > 0
assert report.excerpts_examined == 3
@ -76,112 +107,112 @@ def test_the_report_carries_its_denominators() -> None:
def test_broken_denominator_identity_is_named() -> None:
payload = load_example()
payload["denominators"]["considered"] = 6
assert "denominator_identity" in codes(TEMPLATE.read_text(encoding="utf-8"), payload)
assert "denominator_identity" in codes(instantiated(), payload)
def test_delivered_count_disagreeing_with_the_list_is_named() -> None:
payload = load_example()
payload["excerpts"] = payload["excerpts"][:2]
assert "denominator_list_mismatch" in codes(TEMPLATE.read_text(encoding="utf-8"), payload)
assert "denominator_list_mismatch" in codes(instantiated(), payload)
def test_a_missing_adjudication_key_is_not_absence() -> None:
"""The contract's sharpest rule: `unknown` is written, never omitted."""
payload = load_example()
del payload["excerpts"][0]["adjudication"]
assert "state_missing" in codes(TEMPLATE.read_text(encoding="utf-8"), payload)
assert "state_missing" in codes(instantiated(), payload)
def test_an_adjudication_value_outside_the_closed_set_is_named() -> None:
payload = load_example()
payload["excerpts"][0]["adjudication"] = "absent"
assert "state_not_in_set" in codes(TEMPLATE.read_text(encoding="utf-8"), payload)
assert "state_not_in_set" in codes(instantiated(), payload)
def test_a_trust_tier_outside_the_closed_set_is_named() -> None:
payload = load_example()
payload["excerpts"][1]["trust_tier"] = "trusted"
assert "state_not_in_set" in codes(TEMPLATE.read_text(encoding="utf-8"), payload)
assert "state_not_in_set" in codes(instantiated(), payload)
@pytest.mark.parametrize("key", ["bundle_id", "concept_id", "sha256"])
def test_an_excerpt_missing_its_source_marking_is_named(key: str) -> None:
payload = load_example()
del payload["excerpts"][0][key]
assert "source_marking_incomplete" in codes(TEMPLATE.read_text(encoding="utf-8"), payload)
assert "source_marking_incomplete" in codes(instantiated(), payload)
def test_a_bundle_without_a_ref_is_named() -> None:
payload = load_example()
payload["bundle"]["ref"] = ""
assert "ref_missing" in codes(TEMPLATE.read_text(encoding="utf-8"), payload)
assert "ref_missing" in codes(instantiated(), payload)
def test_a_withheld_concept_without_a_rule_is_an_undeclared_cut() -> None:
payload = load_example()
del payload["withheld"][0]["rule"]
assert "cut_undeclared" in codes(TEMPLATE.read_text(encoding="utf-8"), payload)
assert "cut_undeclared" in codes(instantiated(), payload)
def test_spending_over_the_limit_is_the_gate_firing() -> None:
payload = load_example()
payload["budget"]["spent"] = payload["budget"]["limit"] + 1
assert "budget_exceeded" in codes(TEMPLATE.read_text(encoding="utf-8"), payload)
assert "budget_exceeded" in codes(instantiated(), payload)
def test_an_instrument_that_missed_its_known_positive_is_named() -> None:
payload = load_example()
payload["budget"]["known_positive"]["measured"] = 10405
assert "instrument_unvalidated" in codes(TEMPLATE.read_text(encoding="utf-8"), payload)
assert "instrument_unvalidated" in codes(instantiated(), payload)
def test_a_budget_without_an_instrument_is_named() -> None:
payload = load_example()
payload["budget"]["instrument"] = ""
assert "budget_undeclared" in codes(TEMPLATE.read_text(encoding="utf-8"), payload)
assert "budget_undeclared" in codes(instantiated(), payload)
def test_a_payload_naming_no_contract_revision_is_named() -> None:
payload = load_example()
del payload["contract"]
assert "contract_unversioned" in codes(TEMPLATE.read_text(encoding="utf-8"), payload)
assert "contract_unversioned" in codes(instantiated(), payload)
def test_a_payload_that_is_not_a_mapping_is_refused_not_crashed() -> None:
assert "payload_invalid" in codes(TEMPLATE.read_text(encoding="utf-8"), ["not", "a", "mapping"])
assert "payload_invalid" in codes(instantiated(), ["not", "a", "mapping"])
# --- Skill mutations --------------------------------------------------------
def test_a_missing_required_section_is_named() -> None:
text = TEMPLATE.read_text(encoding="utf-8").replace("## Denominators", "## Counts")
text = instantiated().replace("## Denominators", "## Counts")
assert "skill_section_missing" in codes(text, load_example())
def test_a_missing_required_marking_is_named() -> None:
text = TEMPLATE.read_text(encoding="utf-8").replace("[sourced-not-sufficient]", "[thin]")
text = instantiated().replace("[sourced-not-sufficient]", "[thin]")
assert "skill_marking_missing" in codes(text, load_example())
def test_a_translated_marking_literal_does_not_count() -> None:
"""One literal string, no variants, no translations."""
text = TEMPLATE.read_text(encoding="utf-8").replace(
"[unverifiable-from-bundle]", "[ikke-verifiserbar-fra-bundle]"
)
text = instantiated().replace("[unverifiable-from-bundle]", "[ikke-verifiserbar-fra-bundle]")
assert "skill_marking_missing" in codes(text, load_example())
def test_a_skill_omitting_an_adjudication_value_is_named() -> None:
text = TEMPLATE.read_text(encoding="utf-8").replace("`unknown`", "`missing`")
text = instantiated().replace("`unknown`", "`missing`")
assert "skill_state_missing" in codes(text, load_example())
# --- The command line -------------------------------------------------------
def test_the_command_exits_zero_on_the_shipped_pair() -> None:
exit_code = okf_contract_check.main(["--skill", str(TEMPLATE), "--payload", str(EXAMPLE)])
def test_the_command_exits_zero_on_an_instantiated_pair(tmp_path: Path) -> None:
exit_code = okf_contract_check.main(
["--skill", str(instantiated_file(tmp_path)), "--payload", str(EXAMPLE)]
)
assert exit_code == 0
@ -190,7 +221,12 @@ def test_the_command_exits_one_on_a_non_conformant_payload(tmp_path: Path) -> No
payload["denominators"]["considered"] = 6
broken = tmp_path / "payload.json"
broken.write_text(json.dumps(payload), encoding="utf-8")
assert okf_contract_check.main(["--skill", str(TEMPLATE), "--payload", str(broken)]) == 1
assert (
okf_contract_check.main(
["--skill", str(instantiated_file(tmp_path)), "--payload", str(broken)]
)
== 1
)
def test_the_command_separates_could_not_run_from_non_conformant(tmp_path: Path) -> None:
@ -204,13 +240,13 @@ def test_the_command_separates_could_not_run_from_non_conformant(tmp_path: Path)
assert okf_contract_check.main(["--skill", str(TEMPLATE), "--payload", str(unreadable)]) == 2
def test_the_module_runs_as_a_script() -> None:
def test_the_module_runs_as_a_script(tmp_path: Path) -> None:
result = subprocess.run(
[
sys.executable,
str(PROJECT_ROOT / "tools" / "okf_contract_check.py"),
"--skill",
str(TEMPLATE),
str(instantiated_file(tmp_path)),
"--payload",
str(EXAMPLE),
],