llm-ingestion-okf/tests/test_sts_description.py

"""An STS section's `description` is its own first spec point (K3-19 c).

SPEC SS 4.1: `description` is RECOMMENDED -- "A single sentence summarizing the
concept. Used by `index.md` generators, search snippets, and previews." The
spec sets NO length limit, in SS 4.1, SS 8 or SS 11. The limit here is this
package's own and it is structural: the FIRST `<p>` of the FIRST direct-child
`<sec sec-type="spec">` of a titled `<sec>`, whole. Measured on the one STS
document this row has: 2 026 of its N titled sections carry such a point.

WHAT THESE TESTS PIN:

- the description is the source's own words, never derived from the title and
  never invented where the source has none -- a section without a spec point
  of its own gets NO `description` key;
- a second spec point, and a second paragraph inside the first, never join it;
- a spec point belongs to the section it is a DIRECT child of, so a container
  does not borrow its child's;
- the plan carries it, because a plan is the record a rebuild replays and the
  title already travels there the same way;
- the gate sees it: it is document text persisted OUTSIDE the screened body,
  so it is kept only on the gate's non-blocking floor, as the sanitized text;
- a value stated for the run replaces it (`--frontmatter description=...`).
"""

from __future__ import annotations

from pathlib import Path

import pytest

from llm_ingestion_okf import cli, extract, propose
from llm_ingestion_okf.errors import IngestError
from llm_ingestion_okf.inbox import GateDecision, process_inbox
from llm_ingestion_okf.materialize import parse_frontmatter
from llm_ingestion_okf.profiles import SEGMENTED_OKF_V0_2
from llm_ingestion_okf.segmentation import parse_segmentation_plan

FIXTURES = Path(__file__).parent / "fixtures"
IDENTITY = FIXTURES / "sts-identity.xml"

FIRST = "Omfatter utskifting av skadde enkeltdeler i kjokkenbenk."
SECOND = "Omfatter maling av kjokkenbenk."
EXPECTED = {
    "Kjokkenbenk": None,
    "Utskifting av enkeltdeler": FIRST,
    "Utskifting av handtak": None,
    "Maling av kjokkenbenk": SECOND,
}
STAMP = "2026-09-08T12:00:00Z"


def _by_title(bundle: Path) -> dict[str, dict[str, str]]:
    return {
        values["title"]: values
        for values in (
            parse_frontmatter(path)
            for path in sorted(bundle.rglob("*.md"))
            if path.name not in ("index.md", "log.md")
        )
    }


def _build(tmp_path: Path, name: str, data: bytes, *extra: str) -> Path:
    inbox = tmp_path / "docs"
    inbox.mkdir()
    (inbox / name).write_bytes(data)
    bundle = tmp_path / "bundle"
    args = ["build", str(inbox), "--bundle", str(bundle), "--bundle-id", "d"]
    assert cli.main([*args, "--okf-version", "0.2", *extra]) == 0
    return bundle


# --- the reader --------------------------------------------------------------


def test_each_titled_section_carries_its_own_first_spec_paragraph() -> None:
    marks = extract.xml_outline(IDENTITY.name, IDENTITY.read_bytes())

    assert [(mark.title, mark.description) for mark in marks] == [
        ("17 Kjokkenbenk", None),
        ("17.612 Utskifting av enkeltdeler", FIRST),
        ("17.6121 Utskifting av handtak", None),
        ("17.613 Maling av kjokkenbenk", SECOND),
    ]


def test_the_extracted_text_does_not_move() -> None:
    """The description is read BESIDE the text, never out of it."""
    text = extract.extract_text(IDENTITY.name, IDENTITY.read_bytes())

    assert f"a) {FIRST}" in text.split("\n")
    assert "b) Et andre punkt blir aldri beskrivelsen." in text.split("\n")


# --- through the plan and the door -------------------------------------------


def test_the_description_reaches_the_concept_only_where_the_source_has_one(
    tmp_path: Path,
) -> None:
    concepts = _by_title(_build(tmp_path, IDENTITY.name, IDENTITY.read_bytes()))

    assert {title: values.get("description") for title, values in concepts.items()} == EXPECTED


def test_a_spec_point_a_yaml_reader_could_not_read_verbatim_is_not_written(
    tmp_path: Path,
) -> None:
    """Verbatim or not at all, as for the `sources` title.

    MEASURED ON THE REFERENCE STANDARD: 217 of the 2 024 descriptions the first build wrote carry
    `": "`, and a YAML reader refuses `description: a: b` -- PyYAML's
    `safe_load` rejected exactly those 217 concepts' frontmatter, which the
    same document's frontmatter had parsed without before the key existed.
    Quoting would write a value the line-oriented readers here return WITH its
    quotes, and cleaning would write a sentence the source does not carry. A
    colon with no space after it is an ordinary character and stays.
    """
    data = (
        IDENTITY.read_bytes()
        .replace(b"Omfatter maling av kjokkenbenk.", b"Omfatter maling: kjokkenbenk og skap.")
        .replace(b"Omfatter utskifting av skadde", b"Omfatter 1:2 utskifting av skadde")
    )
    concepts = _by_title(_build(tmp_path, IDENTITY.name, data))

    assert "description" not in concepts["Maling av kjokkenbenk"]
    assert concepts["Utskifting av enkeltdeler"]["description"] == FIRST.replace(
        "Omfatter utskifting", "Omfatter 1:2 utskifting"
    )


def test_a_type_that_declares_no_spec_point_gets_no_description(tmp_path: Path) -> None:
    markdown = b"# Innledning\n\na) Omfatter noe.\n\n# Omfang\n\nMer tekst her.\n"
    concepts = _by_title(_build(tmp_path, "notat.md", markdown))

    assert concepts
    assert not any("description" in values for values in concepts.values())


def test_a_stated_description_replaces_the_derived_one(tmp_path: Path) -> None:
    bundle = _build(
        tmp_path,
        IDENTITY.name,
        IDENTITY.read_bytes(),
        "--frontmatter",
        "description=Standard beskrivelsestekst",
    )

    assert {values["description"] for values in _by_title(bundle).values()} == {
        "Standard beskrivelsestekst"
    }


def _plan() -> dict[str, object]:
    data = IDENTITY.read_bytes()
    text = extract.extract_text(IDENTITY.name, data)
    return propose.build_plan(
        IDENTITY, text, data, okf_type="reference", proposed_at=STAMP, path_prefix="p900"
    )


def test_the_plan_carries_the_description_as_the_record_a_rebuild_replays() -> None:
    entries = _plan()["entries"]
    assert isinstance(entries, list)

    assert {entry["title"]: entry.get("description") for entry in entries} == EXPECTED
    plan = parse_segmentation_plan(_plan())
    assert {entry.title: entry.description for entry in plan.entries} == EXPECTED


@pytest.mark.parametrize("bad", ["", "to\nlinjer", 5], ids=["empty", "two-lines", "not-a-string"])
def test_the_plan_loader_refuses_a_description_it_could_not_write(bad: object) -> None:
    payload = _plan()
    entries = payload["entries"]
    assert isinstance(entries, list)
    entries[1]["description"] = bad

    with pytest.raises(IngestError) as caught:
        parse_segmentation_plan(payload)
    assert caught.value.code == "segmentation_plan_invalid"


def _door(tmp_path: Path, gate: object) -> dict[str, dict[str, str]]:
    inbox = tmp_path / "docs"
    inbox.mkdir()
    (inbox / IDENTITY.name).write_bytes(IDENTITY.read_bytes())
    bundle = tmp_path / "bundle"
    result = process_inbox(
        inbox,
        bundle,
        STAMP,
        okf_type="reference",
        gate=gate,  # type: ignore[arg-type]
        profile=SEGMENTED_OKF_V0_2,
        root_frontmatter_values={"okf_version": "0.2", "bundle_id": "d"},
        segmentation=parse_segmentation_plan(_plan()),
    )
    assert result.failed == ()
    return _by_title(bundle)


def test_a_description_the_gate_refuses_is_dropped_and_the_concept_is_kept(
    tmp_path: Path,
) -> None:
    def gate(text: str) -> GateDecision:
        refused = text == FIRST
        return GateDecision(sanitized_text=text, disposition="block" if refused else "warn")

    concepts = _door(tmp_path, gate)

    assert "description" not in concepts["Utskifting av enkeltdeler"]
    assert concepts["Maling av kjokkenbenk"]["description"] == SECOND


def test_the_description_written_is_the_gates_sanitized_text(tmp_path: Path) -> None:
    def gate(text: str) -> GateDecision:
        return GateDecision(sanitized_text=text.replace("skadde", "[fjernet]"), disposition="warn")

    concepts = _door(tmp_path, gate)

    assert concepts["Utskifting av enkeltdeler"]["description"] == FIRST.replace(
        "skadde", "[fjernet]"
    )