llm-ingestion-okf/tests/test_okf_skill.py

"""The consumption-skill generator, checked rather than described.

`tools/okf_skill.py` turns one OKF bundle into one instantiated `SKILL.md` that
`tools/okf_contract_check.py` accepts. The discipline here is the one measurement
that decided the form: the checker could not tell an instantiated skill from an
unfilled template, and passed a skill built for a different bundle against this
one's payload. So every gate the checker does not have is a test here.

**That measurement is closed on its identity half since 2026-09-10.** The
`bundle_mismatch` rule refuses both forms, and `tests/test_bundle_identity.py`
holds the arms. The gates below are the ones it still does not have: what the
generated skill MEASURES -- the per-bundle denominators, the breaking point,
the conditional-field list -- is not something any static pairing check reaches.
"""

from __future__ import annotations

import json
import re
import shutil
import subprocess
import sys
from pathlib import Path

import pytest

PROJECT_ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(PROJECT_ROOT / "tools"))

import okf_consume  # noqa: E402
import okf_contract_check  # noqa: E402
import okf_skill  # noqa: E402

TEMPLATE = PROJECT_ROOT / "skills" / "okf-consume-template" / "SKILL.md"
GOLDEN = PROJECT_ROOT / "examples" / "ingest-golden-segmented-okf-v0-2" / "expected-bundle"
PROVENANCE = PROJECT_ROOT / "tests" / "fixtures" / "consume-provenance"

#: Two bundles that differ in every way the generator reads: id, ref, concept
#: count, and which conditional fields the producer wrote. One bundle would let
#: a constant masquerade as a measurement.
BUNDLES = (GOLDEN, PROVENANCE)


def _generate(bundle: Path, out: Path, *, force: bool = False) -> Path:
    written = okf_skill.generate(bundle, out=out, force=force)
    assert written.is_file()
    return written


def _run(*argv: str) -> subprocess.CompletedProcess[str]:
    return subprocess.run(
        [sys.executable, str(PROJECT_ROOT / "tools" / "okf_skill.py"), *argv],
        capture_output=True,
        text=True,
    )


@pytest.mark.parametrize("bundle", BUNDLES, ids=lambda path: path.name)
def test_the_generated_skill_leaves_no_hole_the_template_had(bundle: Path, tmp_path: Path) -> None:
    # The template's own rule: a copy that leaves one placeholder unfilled is
    # not configured, it is unfinished. The known-positive runs first, because
    # a pattern that cannot find would make the zero below mean nothing.
    holes = re.compile(r"<[A-Z][A-Z_]+[A-Z](?::[^>]*)?>")
    assert holes.findall(TEMPLATE.read_text(encoding="utf-8")), "the pattern cannot find"
    written = _generate(bundle, tmp_path / bundle.name)
    assert holes.findall(written.read_text(encoding="utf-8")) == []


@pytest.mark.parametrize("bundle", BUNDLES, ids=lambda path: path.name)
def test_a_generated_skill_and_a_payload_from_its_own_bundle_pass_the_checker(
    bundle: Path, tmp_path: Path
) -> None:
    written = _generate(bundle, tmp_path / bundle.name)
    payload = okf_consume.build_payload(bundle, question="Hva krever dette?")
    report = okf_contract_check.check(written.read_text(encoding="utf-8"), payload)
    assert report.findings == ()


@pytest.mark.parametrize("bundle", BUNDLES, ids=lambda path: path.name)
def test_the_generated_skill_names_the_bundle_and_the_ref_it_was_made_from(
    bundle: Path, tmp_path: Path
) -> None:
    # The gate the checker does not have. Measured 2026-09-08: the checker
    # passes the K2 skill against a payload from a different bundle, so nothing
    # mechanical stops a skill from certifying a corpus it never read.
    written = _generate(bundle, tmp_path / bundle.name)
    text = written.read_text(encoding="utf-8")
    assert okf_consume.bundle_ref(bundle) in text
    assert okf_consume.root_bundle_id_of(bundle) in text


def test_a_skill_generated_for_one_bundle_does_not_name_the_other(tmp_path: Path) -> None:
    first = _generate(GOLDEN, tmp_path / "first").read_text(encoding="utf-8")
    assert okf_consume.bundle_ref(PROVENANCE) not in first
    assert okf_consume.root_bundle_id_of(PROVENANCE) not in first


@pytest.mark.parametrize("bundle", BUNDLES, ids=lambda path: path.name)
def test_the_generated_commands_name_this_repository_nowhere(bundle: Path, tmp_path: Path) -> None:
    """O5's red measurement, made a test: no path into a checkout, anywhere.

    **This assertion REPLACES its own opposite, and the replacement is the
    point.** The test here until 2026-09-08 required the emitted commands to be
    ABSOLUTE, on the reasoning that a skill copied into someone else's
    `.claude/skills/` cannot resolve a relative `tools/okf_consume.py`. Both
    halves of that were true and the conclusion was still wrong: an absolute
    path into THIS clone is not portable either, it is merely portable-looking.
    Measured before the move, a skill generated from a checkout carried four
    lines naming this checkout by absolute path, two of them the commands a
    reader is told to run -- so the skill could not be moved, shared, or run by
    anyone without that clone at that exact path.

    It also caught nothing by then: the regex it looped over matched zero lines
    once the commands stopped being `python3 <file>.py`, so it was green over
    an empty set. The assertion below has a denominator that cannot go to zero.
    """
    written = _generate(bundle, tmp_path / bundle.name)
    text = written.read_text(encoding="utf-8")

    # The BUNDLE root is the one absolute path that belongs here: it points at
    # the caller's data. These fixtures happen to live inside this repository,
    # so it is removed before the search -- otherwise the search would find the
    # repository root inside the one path allowed to carry it.
    assert str(bundle.resolve()) in text
    rest = text.replace(str(bundle.resolve()), "<BUNDLE>")

    # The known-positive for the search: the string it hunts for DOES occur in
    # the environment running it, so the zero below means the generator kept it
    # out rather than the search being unable to find it.
    assert str(PROJECT_ROOT) in str(Path(__file__).resolve())
    assert str(PROJECT_ROOT) not in rest
    # No file under `tools/` at all, and not only the two commands: the
    # attribution lines named the generator by path too, which is a file the
    # reader does not have either.
    assert "tools/" not in rest

    # And what it names instead: commands resolved by PATH after an install.
    assert "\nokf consume \\\n" in text
    assert "\nokf check \\\n" in text


@pytest.mark.parametrize("bundle", BUNDLES, ids=lambda path: path.name)
def test_the_generated_skill_reports_the_conditional_fields_with_their_denominators(
    bundle: Path, tmp_path: Path
) -> None:
    # SS 6.4: absence is a measurement, not a fact. A per-bundle skill that does
    # not say how many of its concepts carry `req_number` cannot tell a reader
    # what a missing one means.
    written = _generate(bundle, tmp_path / bundle.name)
    text = written.read_text(encoding="utf-8")
    total = len(okf_consume.enumerate_concepts(bundle))
    assert f"of {total}" in text
    for field in ("req_number", "sources", "adjudication"):
        assert f"`{field}`" in text, field


@pytest.mark.parametrize("bundle", BUNDLES, ids=lambda path: path.name)
def test_the_generator_is_deterministic_at_the_byte(bundle: Path, tmp_path: Path) -> None:
    # Same bundle, same destination, same bytes. The DESTINATION is part of the
    # input on purpose: the skill names the path it was written to, so the
    # command a reader is told to run is one that exists.
    out = tmp_path / "a"
    first = _generate(bundle, out).read_bytes()
    second = _generate(bundle, out, force=True).read_bytes()
    assert first == second
    elsewhere = _generate(bundle, tmp_path / "b").read_bytes()
    assert elsewhere != first


def test_the_generated_frontmatter_is_what_claude_code_reads(tmp_path: Path) -> None:
    text = _generate(PROVENANCE, tmp_path / "out").read_text(encoding="utf-8")
    assert text.startswith("---\n")
    header = text.split("---\n", 2)[1]
    name = re.search(r"^name: (.+)$", header, flags=re.MULTILINE)
    description = re.search(r"^description: (.+)$", header, flags=re.MULTILINE)
    assert name and description
    # Claude Code's own constraint on a skill directory name.
    assert re.fullmatch(r"[a-z0-9]+(-[a-z0-9]+)*", name.group(1)), name.group(1)
    assert name.group(1) == tmp_path.joinpath("out").name or name.group(1)
    assert len(description.group(1)) > 40


def test_the_generator_refuses_a_directory_that_is_not_a_bundle(tmp_path: Path) -> None:
    plain = tmp_path / "just-a-folder"
    plain.mkdir()
    (plain / "notes.md").write_text("no manifest here\n", encoding="utf-8")
    result = _run(str(plain), "--out", str(tmp_path / "out"))
    assert result.returncode != 0
    assert "index.md" in result.stdout + result.stderr
    assert not (tmp_path / "out").exists()


def test_the_generator_refuses_a_bundle_whose_index_declares_no_id(tmp_path: Path) -> None:
    root = tmp_path / "bundle"
    root.mkdir()
    (root / "index.md").write_text("- [Something](something.md)\n", encoding="utf-8")
    result = _run(str(root), "--out", str(tmp_path / "out"))
    assert result.returncode != 0
    assert "bundle_id" in result.stdout + result.stderr


def test_the_generator_refuses_to_overwrite_without_being_asked(tmp_path: Path) -> None:
    out = tmp_path / "out"
    _generate(GOLDEN, out)
    result = _run(str(GOLDEN), "--out", str(out))
    assert result.returncode != 0
    assert "--force" in result.stdout + result.stderr
    forced = _run(str(GOLDEN), "--out", str(out), "--force")
    assert forced.returncode == 0


def test_the_cli_writes_the_same_bytes_the_function_does(tmp_path: Path) -> None:
    out = tmp_path / "shared"
    written = _generate(GOLDEN, out).read_bytes()
    result = _run(str(GOLDEN), "--out", str(out), "--force")
    assert result.returncode == 0
    assert (out / "SKILL.md").read_bytes() == written


def test_the_generated_skill_names_the_path_it_was_written_to(tmp_path: Path) -> None:
    # The checker command in the skill has to be runnable by whoever reads it.
    out = tmp_path / "somewhere"
    written = _generate(GOLDEN, out)
    assert str(written.resolve()) in written.read_text(encoding="utf-8")


def test_every_block_the_generator_replaces_is_still_in_the_template() -> None:
    # The anti-drift gate. The generator rewrites named blocks of the template
    # by exact string; an edit to the template that moves one would otherwise
    # produce a skill silently missing that rewrite.
    text = TEMPLATE.read_text(encoding="utf-8")
    for block in okf_skill.REPLACED_BLOCKS:
        assert text.count(block) == 1, block[:60]


def test_the_generated_skill_still_carries_every_literal_the_contract_fixes(
    tmp_path: Path,
) -> None:
    text = _generate(GOLDEN, tmp_path / "out").read_text(encoding="utf-8")
    for marking in okf_contract_check.REQUIRED_MARKINGS:
        assert marking in text, marking
    for section in okf_contract_check.REQUIRED_SECTIONS:
        assert f"## {section}" in text, section


def test_the_generated_skill_names_every_rule_the_pre_pass_can_emit(tmp_path: Path) -> None:
    text = _generate(GOLDEN, tmp_path / "out").read_text(encoding="utf-8")
    for rule in okf_consume.WITHHOLDING_RULES:
        assert rule in text, rule


def test_the_generated_skill_carries_a_payload_its_own_bundle_produced(tmp_path: Path) -> None:
    # The reference payload ships beside the skill, as it does for the
    # hand-instantiated copy, and is regenerated from this bundle rather than
    # copied from another.
    written = _generate(PROVENANCE, tmp_path / "out")
    example = written.parent / "references" / "example-payload.json"
    payload = json.loads(example.read_text(encoding="utf-8"))
    assert payload["bundle"]["bundle_id"] == okf_consume.root_bundle_id_of(PROVENANCE)
    assert payload["bundle"]["ref"] == okf_consume.bundle_ref(PROVENANCE)
    assert okf_contract_check.check(written.read_text(encoding="utf-8"), payload).findings == ()


# --- The three modes, and the paths a reader can actually run (O6) -------------

#: A path is absolute here if it starts a line or follows whitespace. The
#: measurement O5 published used `grep -c "^/"`, which cannot match a path
#: indented by two spaces -- which is the form the generator writes. Every
#: assertion below runs this pattern against a known-positive first.
ABSOLUTE = re.compile(r"(?:^|[ \t])(/[A-Za-z])", re.MULTILINE)


def _project_layout(tmp_path: Path, bundle: Path) -> tuple[Path, Path, str]:
    """Generate into the layout `okf project` writes, and return the pieces."""
    root = tmp_path / "prosjekt"
    identity = okf_consume.root_bundle_id_of(bundle)
    inside = root / ".okf" / identity
    inside.parent.mkdir(parents=True)
    shutil.copytree(bundle, inside)
    out = root / ".claude" / "skills" / f"{identity}-consume"
    written = _generate(inside, out)
    return root, written, identity


def test_the_pattern_that_looks_for_absolute_paths_can_find_one(tmp_path: Path) -> None:
    """Face 4 first: a query is shown capable of finding before a zero is read.

    O5's own published figure ("4 absolute paths -> 0") was measured with
    `grep -c "^/"` against a file whose paths are indented by two spaces, so the
    query could not have found one either way. This is that control.
    """
    known_positive = "prose with no path\n  /Users/x/bundle\nokf consume /Users/x/other\n"
    assert len(ABSOLUTE.findall(known_positive)) == 2
    assert len(re.findall(r"^/", known_positive, re.MULTILINE)) == 0


@pytest.mark.parametrize("bundle", BUNDLES, ids=lambda path: path.name)
def test_a_project_skill_names_its_bundle_and_itself_relative_to_the_project_root(
    bundle: Path, tmp_path: Path
) -> None:
    """The commands have to be runnable where `claude` is started: the project root.

    `okf project`'s own closing line tells the reader to start `claude` in the
    project root, so the skill's commands are read from there. An absolute path
    makes the skill unmovable, unshareable, and wrong for anyone whose clone
    lives elsewhere.
    """
    root, written, identity = _project_layout(tmp_path, bundle)
    text = written.read_text(encoding="utf-8")

    assert f".okf/{identity}" in text
    assert f".claude/skills/{identity}-consume/SKILL.md" in text
    assert str(root) not in text
    # Every absolute path left is a scratch write target, not a path into the
    # machine the skill was generated on. `/tmp/payload.json` is where the
    # pre-pass puts its payload; naming it relative would litter the project.
    leftover = [line for line in text.splitlines() if ABSOLUTE.search(line) and "/tmp/" not in line]
    assert leftover == []


@pytest.mark.parametrize("bundle", BUNDLES, ids=lambda path: path.name)
def test_the_generated_skill_declares_the_hypothesis_mode(bundle: Path, tmp_path: Path) -> None:
    """A hypothesis is answered per premise, not as one verdict over the whole.

    The three verdicts are literals, like the five markings: a reader that
    invents a fourth ("partly confirmed") has left the contract.
    """
    _, written, _ = _project_layout(tmp_path, bundle)
    text = written.read_text(encoding="utf-8")
    assert "## Modes" in text
    for literal in ("`confirmed`", "`refuted`", "`undecidable-from-bundle`"):
        assert literal in text
    assert "per premise" in text.lower()
    assert "[sourced-not-sufficient]" in text


@pytest.mark.parametrize("bundle", BUNDLES, ids=lambda path: path.name)
def test_the_generated_skill_declares_the_document_task_mode(bundle: Path, tmp_path: Path) -> None:
    """A task whose answer is a document carries the cut INTO the document.

    Dropping an ungrounded paragraph silently is the denominator failure with
    a nicer surface: the reader cannot see what the bundle did not cover.
    """
    _, written, _ = _project_layout(tmp_path, bundle)
    text = written.read_text(encoding="utf-8")
    assert "## Modes" in text
    assert "Task" in text
    for token in ("considered", "withheld", "delivered"):
        assert token in text


def test_the_five_markings_are_untouched_by_the_modes(tmp_path: Path) -> None:
    """The modes add no sixth marking. § 4.3 makes an undeclared extension the defect."""
    _, written, _ = _project_layout(tmp_path, GOLDEN)
    text = written.read_text(encoding="utf-8")
    for literal in okf_contract_check.REQUIRED_MARKINGS:
        assert literal in text


def test_a_project_skill_still_passes_the_contract_checker(tmp_path: Path) -> None:
    """Relative paths and a new section must not cost conformance."""
    _, written, _ = _project_layout(tmp_path, GOLDEN)
    payload = okf_consume.build_payload(GOLDEN, question="hva er kravet til pris?")
    report = okf_contract_check.check(written.read_text(encoding="utf-8"), payload)
    assert report.findings == ()


@pytest.mark.parametrize("bundle", BUNDLES, ids=lambda path: path.name)
def test_the_breaking_point_is_a_measurement_or_it_is_withheld(
    bundle: Path, tmp_path: Path
) -> None:
    """`0 concepts` was a division that never happened, printed as a number.

    The figure is EXTRAPOLATED from what one `withheld` entry costs, so a
    generation run that withheld nothing has no slope to extrapolate from:
    `per_withheld` was `0.0`, the guard returned the literal `0`, and the
    document told its reader the bundle's bookkeeping fills a 120000-byte
    budget at zero concepts -- before the bundle holds anything at all.

    Driven from both sides so a generator that simply stopped stating the
    figure would fail: the bundle that withholds nothing must say it could not
    measure it, and a bundle that withholds something must still print a
    positive count.
    """
    written = _generate(bundle, tmp_path / "out")
    text = written.read_text(encoding="utf-8")
    payload = json.loads((tmp_path / "out" / "references" / "example-payload.json").read_text())
    assert payload["withheld"], "the known-positive arm withheld nothing to extrapolate from"
    assert "**0 concepts**" not in text
    stated = re.search(r"At roughly\s+\*\*(\d+) concepts\*\*", text)
    assert stated is not None, "a bundle that withheld something states no figure"
    assert int(stated.group(1)) > 0


def test_a_generation_that_withheld_nothing_says_so_instead_of_printing_zero(
    tmp_path: Path,
) -> None:
    """The arm the SHIPPED skill is on, and the one that was wrong.

    `okf skill --example-question "Hva sier veiledningen om krav?"` delivers
    all three concepts of the golden bundle, so `withheld` is empty and there
    is no per-entry cost. The question is part of what the shipped file is
    (`skills/okf-consume/references/README.md`), which is why the defect was
    in the repository rather than only reachable in theory.
    """
    written = okf_skill.generate(
        GOLDEN,
        out=tmp_path / "out",
        question="Hva sier veiledningen om krav?",
        force=True,
    )
    text = written.read_text(encoding="utf-8")
    payload = json.loads((tmp_path / "out" / "references" / "example-payload.json").read_text())
    assert payload["withheld"] == [], "the premise of this arm no longer holds"
    assert "**0 concepts**" not in text
    assert "breaking point could not be measured" in text
    assert "At roughly" not in text