llm-ingestion-okf/tests/test_outline_gate.py

"""The outline gate: recovery yields to declaration, and a large span comes back.

Arm D RECOVERS a heading structure from an integer numbering run. Where a
document DECLARES its own headings -- ATX, or dotted-numbered, the two
grammars `RULE_HEADING` covers -- the recovery is a second, weaker source for
the same thing, and measured on the twelve-position reference it is the whole
of the remaining `pdf` excess: in every one of the four missing positions the
surplus is `rule:outline` entries and every `rule:heading` entry is a unit the
reference wants.

`propose.py`'s fold already states this principle, one step later: clause 2
excludes `RULE_OUTLINE` from voting on the unit level, because "a document
whose structure was recovered rather than declared has no unit level to read".
G1 is the same sentence moved from VOTING to ADMISSION.

G2 is the exception the reference forced. On one position the operator kept a
single recovered heading -- and its span covers 0.316 of the document while
every recovered heading the reference rejects covers 0.094 or less. The
threshold is bounded by that empty region on one side and by a measured
collapse on the other, the same shape `OCR_CID_SHARE` was chosen with.

Measured in `docs/2026-09-09-k3-runde6-outline-gaten-og-prioren.md`.
"""

from __future__ import annotations

from pathlib import Path

from llm_ingestion_okf import cli
from llm_ingestion_okf.propose import (
    OUTLINE_SHARE,
    RULE_HEADING,
    RULE_OUTLINE,
    build_plan,
    declares_headings,
    find_candidates,
)

#: A document that DECLARES a heading and also sustains a numbering run. The
#: run's members are short; the declared heading is not.
MIXED = (
    "## Innledning\n"
    + "Brodtekst under den erklaerte overskriften.\n" * 6
    + "1. Forste punkt\nEn linje under.\n"
    + "2. Andre punkt\nEn linje under.\n"
    + "3. Tredje punkt\nEn linje under.\n"
    + "Avsluttende avsnitt som ikke er en overskrift.\n"
)

#: The same numbering run with no declared heading anywhere. The gate must not
#: touch this document at all.
RECOVERED_ONLY = (
    "Et forord uten overskrift.\n"
    "1. Forste punkt\nEn linje under.\n"
    "2. Andre punkt\nEn linje under.\n"
    "3. Tredje punkt\nEn linje under.\n"
)

#: A declared heading, and a recovered run whose LAST member carries most of
#: the document. G2 must bring exactly that one back.
LARGE_TAIL = (
    "## Innledning\nEn kort innledning.\n"
    "1. Kort\nEn linje.\n"
    "2. Kort\nEn linje.\n"
    "3. Vedlegg\n" + "Et langt vedlegg som utgjor storsteparten av dokumentet.\n" * 40
)

ARMS = dict(outline_run=3, table_grid=True, unit_fold=True)


def _rules(text: str, **kwargs: object) -> list[str]:
    return [c.rule for c in find_candidates(text, **{**ARMS, **kwargs})]  # type: ignore[arg-type]


def test_the_predicate_reads_the_delivered_heading_grammar() -> None:
    assert declares_headings(find_candidates(MIXED, **ARMS)) is True
    assert declares_headings(find_candidates(RECOVERED_ONLY, **ARMS)) is False


def test_the_gate_drops_recovery_where_the_document_declares() -> None:
    assert RULE_OUTLINE in _rules(MIXED)
    gated = _rules(MIXED, outline_gate=True)
    assert RULE_OUTLINE not in gated
    assert RULE_HEADING in gated


def test_a_document_that_declares_nothing_is_untouched() -> None:
    """The known-negative, as IDENTICAL objects and not merely an equal count."""
    before = find_candidates(RECOVERED_ONLY, **ARMS)
    after = find_candidates(RECOVERED_ONLY, **ARMS, outline_gate=True)
    assert before == after
    assert [c.rule for c in after] == [RULE_OUTLINE] * 3


def test_a_span_over_the_share_is_readmitted() -> None:
    kept = [
        c for c in find_candidates(LARGE_TAIL, **ARMS, outline_gate=True) if c.rule == RULE_OUTLINE
    ]
    assert [c.title for c in kept] == ["Vedlegg"]
    span = kept[0].end - kept[0].start
    assert span / len(LARGE_TAIL) >= OUTLINE_SHARE


def test_the_gate_closes_the_span_it_removed_rather_than_dropping_the_text() -> None:
    """Admission, not post-filtering: no character falls outside every segment.

    A gate that filtered ENTRIES after the fact would leave the recovered
    heading's text in no segment at all -- the silent loss this library
    refuses everywhere else. Applied at admission the preceding span simply
    reaches further.
    """
    gated = find_candidates(MIXED, **ARMS, outline_gate=True)
    covered = max(c.end for c in gated)
    assert covered == len(MIXED)
    heading = [c for c in gated if c.rule == RULE_HEADING][0]
    assert MIXED[heading.start : heading.end].count("Forste punkt") == 1


def test_the_share_is_a_declared_constant() -> None:
    assert 0.10 <= OUTLINE_SHARE <= 0.30


def test_the_gate_reaches_the_plan(tmp_path: Path) -> None:
    plan = build_plan(
        Path("mixed.txt"),
        MIXED,
        b"",
        okf_type="reference",
        proposed_at="1970-01-01T00:00:00Z",
        outline_gate=True,
        **ARMS,
    )
    assert all(RULE_OUTLINE not in entry["derived"] for entry in plan["entries"])


def test_the_proposer_default_leaves_the_gate_off() -> None:
    """`propose.py`'s own defaults do not move; the COMMAND's default does.

    The gate and `--drop-wrapped-outline` moved TOGETHER on 2026-09-09, because
    neither carries the reference alone: the gate takes `pdf` from 2 of 8 to
    5 of 8, and the pair takes it to 7 of 8. The acceptance was not the
    reference: hit@8 on a K2 bundle built with both holds 5 of 6 at ranks
    1,1,1,1,1,-, no row losing rank 1.
    """
    assert RULE_OUTLINE in _rules(MIXED)
    assert cli.DEFAULT_OUTLINE_GATE is True
    assert cli.DEFAULT_DROP_WRAPPED_OUTLINE is True


def test_both_opt_outs_reproduce_the_previous_default() -> None:
    """A default a caller cannot turn off is not a default."""
    import subprocess
    import sys

    done = subprocess.run(
        [sys.executable, "-m", "llm_ingestion_okf.cli", "build", "--help"],
        capture_output=True,
        text=True,
        check=False,
    )
    assert "--no-outline-gate" in done.stdout
    assert "--keep-wrapped-outline" in done.stdout