llm-ingestion-okf/tests/test_okf_skill.py

"""The consumption-skill generator, checked rather than described.

`tools/okf_skill.py` turns one OKF bundle into one instantiated `SKILL.md` that
`tools/okf_contract_check.py` accepts. The discipline here is the one measurement
that decided the form: **the checker cannot tell an instantiated skill from an
unfilled template**, and passes a skill built for a different bundle against this
one's payload. So every gate the checker does not have is a test here.
"""

from __future__ import annotations

import json
import re
import subprocess
import sys
from pathlib import Path

import pytest

PROJECT_ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(PROJECT_ROOT / "tools"))

import okf_consume  # noqa: E402
import okf_contract_check  # noqa: E402
import okf_skill  # noqa: E402

TEMPLATE = PROJECT_ROOT / "skills" / "okf-consume-template" / "SKILL.md"
GOLDEN = PROJECT_ROOT / "examples" / "ingest-golden-segmented-okf-v0-2" / "expected-bundle"
PROVENANCE = PROJECT_ROOT / "tests" / "fixtures" / "consume-provenance"

#: Two bundles that differ in every way the generator reads: id, ref, concept
#: count, and which conditional fields the producer wrote. One bundle would let
#: a constant masquerade as a measurement.
BUNDLES = (GOLDEN, PROVENANCE)


def _generate(bundle: Path, out: Path, *, force: bool = False) -> Path:
    written = okf_skill.generate(bundle, out=out, force=force)
    assert written.is_file()
    return written


def _run(*argv: str) -> subprocess.CompletedProcess[str]:
    return subprocess.run(
        [sys.executable, str(PROJECT_ROOT / "tools" / "okf_skill.py"), *argv],
        capture_output=True,
        text=True,
    )


@pytest.mark.parametrize("bundle", BUNDLES, ids=lambda path: path.name)
def test_the_generated_skill_leaves_no_hole_the_template_had(bundle: Path, tmp_path: Path) -> None:
    # The template's own rule: a copy that leaves one placeholder unfilled is
    # not configured, it is unfinished. The known-positive runs first, because
    # a pattern that cannot find would make the zero below mean nothing.
    holes = re.compile(r"<[A-Z][A-Z_]+[A-Z](?::[^>]*)?>")
    assert holes.findall(TEMPLATE.read_text(encoding="utf-8")), "the pattern cannot find"
    written = _generate(bundle, tmp_path / bundle.name)
    assert holes.findall(written.read_text(encoding="utf-8")) == []


@pytest.mark.parametrize("bundle", BUNDLES, ids=lambda path: path.name)
def test_a_generated_skill_and_a_payload_from_its_own_bundle_pass_the_checker(
    bundle: Path, tmp_path: Path
) -> None:
    written = _generate(bundle, tmp_path / bundle.name)
    payload = okf_consume.build_payload(bundle, question="Hva krever dette?")
    report = okf_contract_check.check(written.read_text(encoding="utf-8"), payload)
    assert report.findings == ()


@pytest.mark.parametrize("bundle", BUNDLES, ids=lambda path: path.name)
def test_the_generated_skill_names_the_bundle_and_the_ref_it_was_made_from(
    bundle: Path, tmp_path: Path
) -> None:
    # The gate the checker does not have. Measured 2026-09-08: the checker
    # passes the K2 skill against a payload from a different bundle, so nothing
    # mechanical stops a skill from certifying a corpus it never read.
    written = _generate(bundle, tmp_path / bundle.name)
    text = written.read_text(encoding="utf-8")
    assert okf_consume.bundle_ref(bundle) in text
    assert okf_consume.root_bundle_id_of(bundle) in text


def test_a_skill_generated_for_one_bundle_does_not_name_the_other(tmp_path: Path) -> None:
    first = _generate(GOLDEN, tmp_path / "first").read_text(encoding="utf-8")
    assert okf_consume.bundle_ref(PROVENANCE) not in first
    assert okf_consume.root_bundle_id_of(PROVENANCE) not in first


@pytest.mark.parametrize("bundle", BUNDLES, ids=lambda path: path.name)
def test_the_generated_commands_name_this_repository_nowhere(bundle: Path, tmp_path: Path) -> None:
    """O5's red measurement, made a test: no path into a checkout, anywhere.

    **This assertion REPLACES its own opposite, and the replacement is the
    point.** The test here until 2026-09-08 required the emitted commands to be
    ABSOLUTE, on the reasoning that a skill copied into someone else's
    `.claude/skills/` cannot resolve a relative `tools/okf_consume.py`. Both
    halves of that were true and the conclusion was still wrong: an absolute
    path into THIS clone is not portable either, it is merely portable-looking.
    Measured before the move, a skill generated from a checkout carried four
    lines naming this checkout by absolute path, two of them the commands a
    reader is told to run -- so the skill could not be moved, shared, or run by
    anyone without that clone at that exact path.

    It also caught nothing by then: the regex it looped over matched zero lines
    once the commands stopped being `python3 <file>.py`, so it was green over
    an empty set. The assertion below has a denominator that cannot go to zero.
    """
    written = _generate(bundle, tmp_path / bundle.name)
    text = written.read_text(encoding="utf-8")

    # The BUNDLE root is the one absolute path that belongs here: it points at
    # the caller's data. These fixtures happen to live inside this repository,
    # so it is removed before the search -- otherwise the search would find the
    # repository root inside the one path allowed to carry it.
    assert str(bundle.resolve()) in text
    rest = text.replace(str(bundle.resolve()), "<BUNDLE>")

    # The known-positive for the search: the string it hunts for DOES occur in
    # the environment running it, so the zero below means the generator kept it
    # out rather than the search being unable to find it.
    assert str(PROJECT_ROOT) in str(Path(__file__).resolve())
    assert str(PROJECT_ROOT) not in rest
    # No file under `tools/` at all, and not only the two commands: the
    # attribution lines named the generator by path too, which is a file the
    # reader does not have either.
    assert "tools/" not in rest

    # And what it names instead: commands resolved by PATH after an install.
    assert "\nokf consume \\\n" in text
    assert "\nokf check \\\n" in text


@pytest.mark.parametrize("bundle", BUNDLES, ids=lambda path: path.name)
def test_the_generated_skill_reports_the_conditional_fields_with_their_denominators(
    bundle: Path, tmp_path: Path
) -> None:
    # SS 6.4: absence is a measurement, not a fact. A per-bundle skill that does
    # not say how many of its concepts carry `req_number` cannot tell a reader
    # what a missing one means.
    written = _generate(bundle, tmp_path / bundle.name)
    text = written.read_text(encoding="utf-8")
    total = len(okf_consume.enumerate_concepts(bundle))
    assert f"of {total}" in text
    for field in ("req_number", "sources", "adjudication"):
        assert f"`{field}`" in text, field


@pytest.mark.parametrize("bundle", BUNDLES, ids=lambda path: path.name)
def test_the_generator_is_deterministic_at_the_byte(bundle: Path, tmp_path: Path) -> None:
    # Same bundle, same destination, same bytes. The DESTINATION is part of the
    # input on purpose: the skill names the path it was written to, so the
    # command a reader is told to run is one that exists.
    out = tmp_path / "a"
    first = _generate(bundle, out).read_bytes()
    second = _generate(bundle, out, force=True).read_bytes()
    assert first == second
    elsewhere = _generate(bundle, tmp_path / "b").read_bytes()
    assert elsewhere != first


def test_the_generated_frontmatter_is_what_claude_code_reads(tmp_path: Path) -> None:
    text = _generate(PROVENANCE, tmp_path / "out").read_text(encoding="utf-8")
    assert text.startswith("---\n")
    header = text.split("---\n", 2)[1]
    name = re.search(r"^name: (.+)$", header, flags=re.MULTILINE)
    description = re.search(r"^description: (.+)$", header, flags=re.MULTILINE)
    assert name and description
    # Claude Code's own constraint on a skill directory name.
    assert re.fullmatch(r"[a-z0-9]+(-[a-z0-9]+)*", name.group(1)), name.group(1)
    assert name.group(1) == tmp_path.joinpath("out").name or name.group(1)
    assert len(description.group(1)) > 40


def test_the_generator_refuses_a_directory_that_is_not_a_bundle(tmp_path: Path) -> None:
    plain = tmp_path / "just-a-folder"
    plain.mkdir()
    (plain / "notes.md").write_text("no manifest here\n", encoding="utf-8")
    result = _run(str(plain), "--out", str(tmp_path / "out"))
    assert result.returncode != 0
    assert "index.md" in result.stdout + result.stderr
    assert not (tmp_path / "out").exists()


def test_the_generator_refuses_a_bundle_whose_index_declares_no_id(tmp_path: Path) -> None:
    root = tmp_path / "bundle"
    root.mkdir()
    (root / "index.md").write_text("- [Something](something.md)\n", encoding="utf-8")
    result = _run(str(root), "--out", str(tmp_path / "out"))
    assert result.returncode != 0
    assert "bundle_id" in result.stdout + result.stderr


def test_the_generator_refuses_to_overwrite_without_being_asked(tmp_path: Path) -> None:
    out = tmp_path / "out"
    _generate(GOLDEN, out)
    result = _run(str(GOLDEN), "--out", str(out))
    assert result.returncode != 0
    assert "--force" in result.stdout + result.stderr
    forced = _run(str(GOLDEN), "--out", str(out), "--force")
    assert forced.returncode == 0


def test_the_cli_writes_the_same_bytes_the_function_does(tmp_path: Path) -> None:
    out = tmp_path / "shared"
    written = _generate(GOLDEN, out).read_bytes()
    result = _run(str(GOLDEN), "--out", str(out), "--force")
    assert result.returncode == 0
    assert (out / "SKILL.md").read_bytes() == written


def test_the_generated_skill_names_the_path_it_was_written_to(tmp_path: Path) -> None:
    # The checker command in the skill has to be runnable by whoever reads it.
    out = tmp_path / "somewhere"
    written = _generate(GOLDEN, out)
    assert str(written.resolve()) in written.read_text(encoding="utf-8")


def test_every_block_the_generator_replaces_is_still_in_the_template() -> None:
    # The anti-drift gate. The generator rewrites named blocks of the template
    # by exact string; an edit to the template that moves one would otherwise
    # produce a skill silently missing that rewrite.
    text = TEMPLATE.read_text(encoding="utf-8")
    for block in okf_skill.REPLACED_BLOCKS:
        assert text.count(block) == 1, block[:60]


def test_the_generated_skill_still_carries_every_literal_the_contract_fixes(
    tmp_path: Path,
) -> None:
    text = _generate(GOLDEN, tmp_path / "out").read_text(encoding="utf-8")
    for marking in okf_contract_check.REQUIRED_MARKINGS:
        assert marking in text, marking
    for section in okf_contract_check.REQUIRED_SECTIONS:
        assert f"## {section}" in text, section


def test_the_generated_skill_names_every_rule_the_pre_pass_can_emit(tmp_path: Path) -> None:
    text = _generate(GOLDEN, tmp_path / "out").read_text(encoding="utf-8")
    for rule in okf_consume.WITHHOLDING_RULES:
        assert rule in text, rule


def test_the_generated_skill_carries_a_payload_its_own_bundle_produced(tmp_path: Path) -> None:
    # The reference payload ships beside the skill, as it does for the
    # hand-instantiated copy, and is regenerated from this bundle rather than
    # copied from another.
    written = _generate(PROVENANCE, tmp_path / "out")
    example = written.parent / "references" / "example-payload.json"
    payload = json.loads(example.read_text(encoding="utf-8"))
    assert payload["bundle"]["bundle_id"] == okf_consume.root_bundle_id_of(PROVENANCE)
    assert payload["bundle"]["ref"] == okf_consume.bundle_ref(PROVENANCE)
    assert okf_contract_check.check(written.read_text(encoding="utf-8"), payload).findings == ()