"""The outline gate: recovery yields to declaration, and a large span comes back. Arm D RECOVERS a heading structure from an integer numbering run. Where a document DECLARES its own headings -- ATX, or dotted-numbered, the two grammars `RULE_HEADING` covers -- the recovery is a second, weaker source for the same thing, and measured on the twelve-position reference it is the whole of the remaining `pdf` excess: in every one of the four missing positions the surplus is `rule:outline` entries and every `rule:heading` entry is a unit the reference wants. `propose.py`'s fold already states this principle, one step later: clause 2 excludes `RULE_OUTLINE` from voting on the unit level, because "a document whose structure was recovered rather than declared has no unit level to read". G1 is the same sentence moved from VOTING to ADMISSION. G2 is the exception the reference forced. On one position the operator kept a single recovered heading -- and its span covers 0.316 of the document while every recovered heading the reference rejects covers 0.094 or less. The threshold is bounded by that empty region on one side and by a measured collapse on the other, the same shape `OCR_CID_SHARE` was chosen with. Measured in `docs/2026-09-09-k3-runde6-outline-gaten-og-prioren.md`. """ from __future__ import annotations from pathlib import Path from llm_ingestion_okf import cli from llm_ingestion_okf.propose import ( OUTLINE_SHARE, RULE_HEADING, RULE_OUTLINE, build_plan, declares_headings, find_candidates, ) #: A document that DECLARES a heading and also sustains a numbering run. The #: run's members are short; the declared heading is not. MIXED = ( "## Innledning\n" + "Brodtekst under den erklaerte overskriften.\n" * 6 + "1. Forste punkt\nEn linje under.\n" + "2. Andre punkt\nEn linje under.\n" + "3. Tredje punkt\nEn linje under.\n" + "Avsluttende avsnitt som ikke er en overskrift.\n" ) #: The same numbering run with no declared heading anywhere. The gate must not #: touch this document at all. RECOVERED_ONLY = ( "Et forord uten overskrift.\n" "1. Forste punkt\nEn linje under.\n" "2. Andre punkt\nEn linje under.\n" "3. Tredje punkt\nEn linje under.\n" ) #: A declared heading, and a recovered run whose LAST member carries most of #: the document. G2 must bring exactly that one back. LARGE_TAIL = ( "## Innledning\nEn kort innledning.\n" "1. Kort\nEn linje.\n" "2. Kort\nEn linje.\n" "3. Vedlegg\n" + "Et langt vedlegg som utgjor storsteparten av dokumentet.\n" * 40 ) ARMS = dict(outline_run=3, table_grid=True, unit_fold=True) def _rules(text: str, **kwargs: object) -> list[str]: return [c.rule for c in find_candidates(text, **{**ARMS, **kwargs})] # type: ignore[arg-type] def test_the_predicate_reads_the_delivered_heading_grammar() -> None: assert declares_headings(find_candidates(MIXED, **ARMS)) is True assert declares_headings(find_candidates(RECOVERED_ONLY, **ARMS)) is False def test_the_gate_drops_recovery_where_the_document_declares() -> None: assert RULE_OUTLINE in _rules(MIXED) gated = _rules(MIXED, outline_gate=True) assert RULE_OUTLINE not in gated assert RULE_HEADING in gated def test_a_document_that_declares_nothing_is_untouched() -> None: """The known-negative, as IDENTICAL objects and not merely an equal count.""" before = find_candidates(RECOVERED_ONLY, **ARMS) after = find_candidates(RECOVERED_ONLY, **ARMS, outline_gate=True) assert before == after assert [c.rule for c in after] == [RULE_OUTLINE] * 3 def test_a_span_over_the_share_is_readmitted() -> None: kept = [ c for c in find_candidates(LARGE_TAIL, **ARMS, outline_gate=True) if c.rule == RULE_OUTLINE ] assert [c.title for c in kept] == ["Vedlegg"] span = kept[0].end - kept[0].start assert span / len(LARGE_TAIL) >= OUTLINE_SHARE def test_the_gate_closes_the_span_it_removed_rather_than_dropping_the_text() -> None: """Admission, not post-filtering: no character falls outside every segment. A gate that filtered ENTRIES after the fact would leave the recovered heading's text in no segment at all -- the silent loss this library refuses everywhere else. Applied at admission the preceding span simply reaches further. """ gated = find_candidates(MIXED, **ARMS, outline_gate=True) covered = max(c.end for c in gated) assert covered == len(MIXED) heading = [c for c in gated if c.rule == RULE_HEADING][0] assert MIXED[heading.start : heading.end].count("Forste punkt") == 1 def test_the_share_is_a_declared_constant() -> None: assert 0.10 <= OUTLINE_SHARE <= 0.30 def test_the_gate_reaches_the_plan(tmp_path: Path) -> None: plan = build_plan( Path("mixed.txt"), MIXED, b"", okf_type="reference", proposed_at="1970-01-01T00:00:00Z", outline_gate=True, **ARMS, ) assert all(RULE_OUTLINE not in entry["derived"] for entry in plan["entries"]) def test_the_proposer_default_leaves_the_gate_off() -> None: """`propose.py`'s own defaults do not move; the COMMAND's default does. The gate and `--drop-wrapped-outline` moved TOGETHER on 2026-09-09, because neither carries the reference alone: the gate takes `pdf` from 2 of 8 to 5 of 8, and the pair takes it to 7 of 8. The acceptance was not the reference: hit@8 on a K2 bundle built with both holds 5 of 6 at ranks 1,1,1,1,1,-, no row losing rank 1. """ assert RULE_OUTLINE in _rules(MIXED) assert cli.DEFAULT_OUTLINE_GATE is True assert cli.DEFAULT_DROP_WRAPPED_OUTLINE is True def test_both_opt_outs_reproduce_the_previous_default() -> None: """A default a caller cannot turn off is not a default.""" import subprocess import sys done = subprocess.run( [sys.executable, "-m", "llm_ingestion_okf.cli", "build", "--help"], capture_output=True, text=True, check=False, ) assert "--no-outline-gate" in done.stdout assert "--keep-wrapped-outline" in done.stdout