llm-ingestion-okf/tests/test_bold_title.py

"""A bold standalone line is a title where the document declares nothing else.

THE HOLE THIS CLOSES. Round 9 met `rtf` end to end for the first time and the
row came back at 0 of 0 declared headings, 0 concepts, 1368 of 1368 characters
in no segment, N = 1: the container has no heading style, so the author's title
is bold text, the proposer proposes nothing, and the document reaches Door B's
inbox as one flat concept. Content preserved, structure zero.

WHAT THE GRAMMAR READS, AND WHERE IT LIVES. Not `rtf` markup. The converter
already emits that bold title as `**Kravspesifikasjon for tunnelbelysning**` in
the SAME markdown the other office rows produce, so this rule is a markdown
rule and no `rtf`-only grammar exists -- the same shape of decision as the PDF
font reader emitting ATX rather than a PDF-only heading form. Bold INSIDE a
paragraph is `text **bold** text` and is not a whole line, which is the
discriminator the rule rests on.

THREE PARAMETERS WERE SWEPT AND ONE CARRIES. Over 47 readable documents (3
hand-built office fixtures, 5 in the operator's folder, 39 of the 43 corpus
files), with the false-positive denominator measured FIRST:

  * a maximum title length (unlimited / 40 / 60 / 80 / 120 characters) is
    FLAT -- identical counts at every value;
  * requiring the line to stand alone between blank lines is FLAT -- the
    converter already writes one paragraph per line;
  * refusing a line that ends in terminal punctuation carries: false-positive
    lines fall 9-12 to 1-2, and the four candidates it drops are
    `er inngatt mellom:`, `Sted og dato:` and their kind -- sentence fragments
    a contract's cover page sets in bold, never section names.

A parameter measured flat buys nothing, so neither of the two is in the rule.

THE GATE IS G1, ALREADY SHIPPED. One false positive survived the punctuation
clause: a corpus `docx` that DECLARES two headings of its own and sets one
later line (`EGENERKLAERING`) in bold. The repair is not a fourth parameter --
it is the principle `_gate_outline` and `fold_units` clause 2 already carry,
that recovery yields to declaration. A bold-title candidate is admitted only
where the document declares no heading of its own, which takes the
false-positive count to 0 of 31 declaring documents BY CONSTRUCTION.

WHAT WAS MEASURED AND NOT ADDED. `heading_reserve_applies` yields to an
admitted Arm D run as well as to a declaration, and that axis was measured
here before being left out: over all 4 documents that reach this rule, the
outline gate admits 0 runs and the reserve applies on every one. n = 4, which
is small; the axis is flat over it, so coupling this rule to `outline_run`
would add a knob no measurement asks for.

HONESTY LIMIT: the `rtf` row rests on hand-built documents, not corpus files.
The corpus contains ZERO `rtf`. Its evidence class stays `constructed`.
"""

from __future__ import annotations

import json
from pathlib import Path

from llm_ingestion_okf import cli
from llm_ingestion_okf.extract import extract_text
from llm_ingestion_okf.propose import RULE_BOLD_TITLE, RULE_HEADING, find_candidates

#: Variant A stays with the three-container set; round 10's three variants live
#: in their own directory, because `test_k2_office_fixtures.py` reads N off that
#: one by listing it and Door B walks a drop directory recursively.
FIXTURES = Path(__file__).parent / "fixtures" / "k2-office"
RTF_VARIANTS = Path(__file__).parent / "fixtures" / "k2-rtf-variants"
FASIT_PATH = Path(__file__).parent / "fixtures" / "k2-office-fasit.json"

DEFAULT = dict(
    outline_run=cli.DEFAULT_OUTLINE_RUN,
    table_grid=cli.DEFAULT_TABLE_GRID,
    unit_fold=cli.DEFAULT_UNIT_FOLD,
    keep_table_heading=cli.DEFAULT_KEEP_TABLE_HEADING,
    sheet_section_rows=cli.DEFAULT_SHEET_SECTION_ROWS,
    drop_wrapped_outline=cli.DEFAULT_DROP_WRAPPED_OUTLINE,
    outline_gate=cli.DEFAULT_OUTLINE_GATE,
    first_span_from_zero=cli.DEFAULT_FIRST_SPAN_FROM_ZERO,
    close_span_gaps=cli.DEFAULT_CLOSE_SPAN_GAPS,
    contents_name=cli.DEFAULT_CONTENTS_NAME,
)

#: The shape the converter hands back for the `rtf` fixture: a bold title, a
#: prose paragraph, then a grid table of label/value pairs.
BOLD_TITLED = """**Kravspesifikasjon for tunnelbelysning**

Dokumentet samler kravene til belysning i vegtunneler over 500 meter.

  ----------------------- ------------------------------
  Dokumentnummer:         SVV-2026-0417
  Tittel:                 Tunnelbelysning i hovedlopet
  ----------------------- ------------------------------

**Luminansmatrise**

Matrisen gjelder per trafikklasse og er bindende.
"""

#: The known-negative for the whole-line clause. `**fet**` here is a phrase
#: inside a running sentence, which is what a bold RUN looks like and what the
#: rule must never read as a boundary.
BOLD_INSIDE_A_PARAGRAPH = """Dokumentet samler kravene, og et **fremhevet** ord
staar midt i en setning som fortsetter forbi det.

Neste avsnitt naevner **ogsaa** noe fremhevet i loepende tekst.
"""

#: The false-positive denominator in miniature: a document that declares its
#: own heading AND sets a later line in bold. The bold line must not become a
#: boundary here -- recovery yields to declaration.
DECLARES_AND_ALSO_BOLDS = """# Konkurransegrunnlag

Innledende avsnitt som beskriver konkurransen og dens omfang.

**EGENERKLAERING**

Leverandoren bekrefter at opplysningene i skjemaet er riktige.
"""

#: The punctuation clause's own material: a contract cover page whose bold
#: lines are sentence fragments, not section names.
BOLD_SENTENCE_FRAGMENTS = """**er inngaatt mellom:**

Byggherren og leverandoren, representert ved sine kontaktpersoner.

**Sted og dato:**

Oslo, 1. mars 2026, med signatur fra begge parter i to eksemplarer.
"""


def _fixture(name: str) -> Path:
    """Variant A in the three-container set, the three variants beside it."""
    return FIXTURES / name if (FIXTURES / name).is_file() else RTF_VARIANTS / name


def _titles(text: str, **overrides: object) -> list[str]:
    keywords = {**DEFAULT, **overrides}
    return [c.title for c in find_candidates(text, **keywords) if c.rule == RULE_BOLD_TITLE]


def test_a_bold_standalone_line_becomes_a_title_candidate() -> None:
    """The `rtf` row's whole finding: the title the container never declared."""
    assert _titles(BOLD_TITLED, bold_title=True) == [
        "Kravspesifikasjon for tunnelbelysning",
        "Luminansmatrise",
    ]


def test_bold_inside_a_paragraph_is_not_a_title_candidate() -> None:
    """The known-negative for the whole-line clause, in the same file."""
    assert _titles(BOLD_INSIDE_A_PARAGRAPH, bold_title=True) == []


def test_a_declared_heading_closes_the_rule_for_the_whole_document() -> None:
    """G1: recovery yields to declaration, so the false positive cannot occur.

    The document's own `# Konkurransegrunnlag` survives; the bold line below it
    proposes nothing. This is the single false positive measured over the 47
    documents, reduced to its mechanism.
    """
    assert _titles(DECLARES_AND_ALSO_BOLDS, bold_title=True) == []
    declared = [
        c.title
        for c in find_candidates(DECLARES_AND_ALSO_BOLDS, **{**DEFAULT, "bold_title": True})
        if c.rule == RULE_HEADING
    ]
    assert declared == ["Konkurransegrunnlag"], "the gate must not touch the declared heading"


def test_a_bold_line_ending_in_terminal_punctuation_is_not_a_title() -> None:
    """The one swept parameter that carried, pinned with its own material."""
    assert _titles(BOLD_SENTENCE_FRAGMENTS, bold_title=True) == []


def test_the_rule_proposes_nothing_when_the_flag_is_off() -> None:
    """The opt-out reproduces the pre-rule bytes on the rule's own material."""
    assert _titles(BOLD_TITLED, bold_title=False) == []
    assert find_candidates(BOLD_TITLED, **{**DEFAULT, "bold_title": False}) == find_candidates(
        BOLD_TITLED, **DEFAULT
    )


def test_the_rtf_row_recovers_every_authored_title_over_four_documents() -> None:
    """The row this round exists to move, pinned against the hand-written fasit.

    Round 9: 0 of 0 declared headings, 0 concepts, 1368 of 1368 characters in
    no segment, N = 1. Here: 6 of 6 AUTHORED titles over N = 4, 0 false bold
    titles, 0 of 1994 characters in no segment.

    `authored_titles` is deliberately not a count of what the container
    declares. `rtf` has no heading style, so a "declared heading" count for
    this type is 0 by construction and says nothing about whether the document
    has sections. The fasit counts what a reader of the SOURCE would call a
    section name, hand-written before this measurement ran.

    The fourth document is a control that failed informatively: it DECLARES
    `\\s1` and `\\s2` in a stylesheet, and the vendored converter discards the
    style and emits the same bold line. "Read the declared style" is therefore
    not a route that exists for this container -- which is why the rule reads
    markdown and not `rtf`.
    """
    fasit = json.loads(FASIT_PATH.read_text(encoding="utf-8"))["rtf_variants"]
    recovered = authored = false_titles = unsegmented = total = 0
    for name, want in sorted(fasit["documents"].items()):
        path = _fixture(name)
        text = extract_text(path.name, path.read_bytes())
        found = find_candidates(text, **{**DEFAULT, "bold_title": True})
        titles = [c.title for c in found]
        authored += want["authored_titles"]
        recovered += sum(1 for t in want["titles"] if t in titles)
        false_titles += sum(
            1 for c in found if c.rule == RULE_BOLD_TITLE and c.title not in want["titles"]
        )
        total += len(text)
        unsegmented += len(text) - sum(c.end - c.start for c in found)
    assert (recovered, authored) == (6, 6), "every authored title, over four containers"
    assert false_titles == 0, "the bold RUN inside a sentence must not become a title"
    assert (unsegmented, total) == (0, 1994), "no character left outside a segment"


def test_the_row_is_zero_without_the_rule_which_is_round_nine_reproduced() -> None:
    """The known-negative for the row: turn the rule off and the finding returns."""
    fasit = json.loads(FASIT_PATH.read_text(encoding="utf-8"))["rtf_variants"]
    for name in sorted(fasit["documents"]):
        path = _fixture(name)
        text = extract_text(path.name, path.read_bytes())
        found = find_candidates(text, **{**DEFAULT, "bold_title": False})
        assert found == [], f"{name} proposes nothing without the rule"


def test_the_container_declaring_a_heading_style_still_reaches_us_as_bold() -> None:
    """The measurement that felled both alternatives the order named.

    Alternative (i) was "read the declared headings out of the converter's
    markdown". There are none: this document declares `\\s1`/`\\s2` and the
    converter emits `**...**`. Alternative (ii) was "convert `rtf` to `docx`
    and let the `docx` grammar work" -- measured separately, that yields 0 ATX
    headings on this same document, because the loss happens in the `rtf`
    READER before any writer sees the style.
    """
    path = _fixture("krav-rikt-tekstformat-stil.rtf")
    text = extract_text(path.name, path.read_bytes())
    assert "**Kravspesifikasjon for tunnelbelysning**" in text
    assert [line for line in text.splitlines() if line.startswith("#")] == []