llm-ingestion-okf/tests/test_working_method.py

"""The generic skill states a WORKING METHOD and an ANSWER FORM, and is default.

Measured by the operator on a large real bundle
documentation, 2026-09-20: the generated skill was an audit contract. All the
discipline sat on the accounting -- markings, denominators, budget lines -- and
none of it on understanding the question, searching again, or writing one
coherent answer. Two sentences actively forbade the second of those.

These tests hold the repair from both sides: the five steps must be there, and
the two forbidding sentences must not come back.
"""

from __future__ import annotations

import json
import subprocess
import sys
from pathlib import Path

PROJECT_ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(PROJECT_ROOT / "src"))

from llm_ingestion_okf import contract_check, project, skill  # noqa: E402

GOLDEN = PROJECT_ROOT / "examples" / "ingest-golden-segmented-okf-v0-2" / "expected-bundle"

#: The two sentences that made the skill an audit contract. Removed, and named
#: here so a template edit cannot quietly restore either.
FORBIDDEN = (
    "Do not go looking for context the pre-pass deliberately withheld",
    "not something to retry with a narrower question",
)

#: The five links of the working method, each by a phrase the section must
#: carry. Phrases and not headings, so a rename does not silently pass.
METHOD_MARKS = (
    "## Working method",
    "Understand the question first",
    "Several searches are normal",
    "Several bundles",
    "Put it together",
    "## Answer form",
)


def _generic() -> str:
    return skill.render_generic()


def test_the_generic_skill_carries_every_link_of_the_working_method() -> None:
    text = _generic()
    missing = [mark for mark in METHOD_MARKS if mark not in text]
    assert missing == [], f"the generic skill states no {missing}"


def test_neither_forbidding_sentence_survives_in_any_shipped_skill_text() -> None:
    template = (PROJECT_ROOT / "skills" / "okf-consume-template" / "SKILL.md").read_text(
        encoding="utf-8"
    )
    shipped = (PROJECT_ROOT / "skills" / "okf-consume" / "SKILL.md").read_text(encoding="utf-8")
    generic = _generic()
    # The control: the strings are findable at all, measured on a text that
    # carries them, so the three zeros below are a measurement.
    carrier = "before: " + FORBIDDEN[0] + " and " + FORBIDDEN[1]
    for sentence in FORBIDDEN:
        assert sentence in carrier
        for name, text in (("template", template), ("shipped", shipped), ("generic", generic)):
            assert sentence not in text, f"{name} still forbids it"


def test_the_working_method_says_searching_again_is_expected() -> None:
    text = _generic()
    assert "allowed and expected" in text
    assert "okf consume" in text


def test_the_answer_form_names_the_jargon_it_keeps_out_of_the_answer() -> None:
    """The reader gets prose, not the instrument's vocabulary."""
    text = _generic()
    form = text.split("## Answer form", 1)[1].split("\n## ", 1)[0]
    for token in ("below_k", "sha256", "denominator"):
        assert token in form, f"the answer form does not name {token} as jargon to keep out"
    assert "the questioner's language" in form


def test_the_audit_trail_is_a_choice_and_the_answer_is_not() -> None:
    text = _generic()
    form = text.split("## Answer form", 1)[1].split("\n## ", 1)[0]
    assert "only when the questioner asks" in form


def test_the_generic_skill_is_what_okf_skill_writes_by_default(tmp_path: Path) -> None:
    """The default moves: one skill that serves any bundle and never goes stale.

    A per-bundle skill has to be regenerated every time its bundle is rebuilt,
    and it refuses out loud (`bundle_mismatch`) when it was not -- so its cost
    is not silence, it is a stopped session. The generic one has no bundle's
    numbers to go stale.
    """
    out = tmp_path / "generic"
    written = skill.generate_any(out=out)
    assert written.read_text(encoding="utf-8") == _generic()

    from llm_ingestion_okf.skill import main as skill_main

    assert skill_main([str(GOLDEN), "--out", str(tmp_path / "cli")]) == 0
    assert (tmp_path / "cli" / "SKILL.md").read_text(encoding="utf-8") == _generic()


def test_the_per_bundle_form_is_still_reachable(tmp_path: Path) -> None:
    from llm_ingestion_okf.skill import main as skill_main

    assert skill_main([str(GOLDEN), "--out", str(tmp_path / "one"), "--for-bundle"]) == 0
    text = (tmp_path / "one" / "SKILL.md").read_text(encoding="utf-8")
    assert "golden-segmented" in text
    assert text != _generic()


def test_okf_project_writes_the_generic_skill(tmp_path: Path) -> None:
    folder = tmp_path / "Dokumenter"
    folder.mkdir()
    (folder / "krav.md").write_text(
        "## 4 Grunnforhold\n\nGrunnen er morene over berg.\n", encoding="utf-8", newline=""
    )
    _, written, _ = project.create(folder, out=tmp_path / "project")
    assert written.read_text(encoding="utf-8") == _generic()


def test_the_checker_accepts_the_new_template_and_still_refuses_a_thin_one(
    tmp_path: Path,
) -> None:
    """`okf check`'s section rule follows the template, never the other way."""
    example = PROJECT_ROOT / "skills" / "okf-consume" / "references" / "example-payload.json"
    payload = json.loads(example.read_text(encoding="utf-8"))
    out = tmp_path / "generic"
    written = skill.generate_any(out=out)
    report = contract_check.check(written.read_text(encoding="utf-8"), payload)
    assert [finding.code for finding in report.findings] == []

    thin = written.read_text(encoding="utf-8").replace("## Working method", "## Notes")
    assert "skill_section_missing" in {
        finding.code for finding in contract_check.check(thin, payload).findings
    }


def test_the_installed_command_writes_the_generic_skill(tmp_path: Path) -> None:
    result = subprocess.run(
        [sys.executable, "-m", "llm_ingestion_okf.cli", "skill", "--out", str(tmp_path / "s")],
        capture_output=True,
        text=True,
        check=False,
        cwd=PROJECT_ROOT,
    )
    assert result.returncode == 0, result.stderr
    assert (tmp_path / "s" / "SKILL.md").read_text(encoding="utf-8") == _generic()