llm-ingestion-okf/tests/test_pdf_font_and_ocr.py

"""Round 4's two PDF readers: font-aware headings, and OCR behind an extra.

Both are OFF by default and both are about the same hole. A PDF carries no
notion of a heading -- a heading in a PDF is a typographic fact -- so the text
stream `pdfplumber` hands over has thrown away the only evidence there was, and
the segment proposer downstream sees a wall of prose. The `docx` path never had
that problem, because the converter emits ATX headings the proposer already
reads.

So the font reader's output is ATX in the SAME markdown the `docx` path
produces. `_ATX` applies unchanged and no new segmentation rule exists; the
tests below pin exactly that, because the alternative -- a PDF-only heading
grammar in `propose.py` -- would be a second definition of "heading" that can
drift from the one the office path already uses.

The OCR half is the other end: a page whose text never arrived at all, either
as nothing or as `(cid:N)` placeholder codes. Its engine is an OPTIONAL
dependency group and the tests here never require it -- the refusal is
exercised through the import probe (the module set to `None` in `sys.modules`,
which is what CPython treats as a failed import), and the success path through
an injected fake engine. That splits the two claims on purpose: the plumbing is
tested here, the reading quality is measured in
`docs/2026-09-08-k3-runde4-pdf-skrift-og-ocr.md` against a real document.
"""

from __future__ import annotations

import sys
import types
import warnings
from pathlib import Path

import pytest

from llm_ingestion_okf import extract as extract_module
from llm_ingestion_okf.errors import ExtractionError
from llm_ingestion_okf.extract import extract_text, source_units
from llm_ingestion_okf.propose import _ATX, find_candidates

FIXTURES = Path(__file__).parent / "fixtures"
FONT_PDF = FIXTURES / "font-heading-krav.pdf"
FLAT_PDF = FIXTURES / "two-line-krav.pdf"
NO_TEXT_PDF = FIXTURES / "no-text-layer.pdf"


def _read(path: Path) -> bytes:
    return path.read_bytes()


def _extract(path: Path, **kwargs: object) -> str:
    with warnings.catch_warnings():
        warnings.simplefilter("ignore")
        return extract_text(path.name, _read(path), **kwargs)  # type: ignore[arg-type]


# --- A: font-aware headings --------------------------------------------------


def test_the_font_reader_is_off_by_default() -> None:
    """The default extraction is what it was: no heading markers anywhere.

    Stated as a test rather than trusted to the signature default, because
    every byte-pinned golden in this suite depends on it.
    """
    text = _extract(FONT_PDF)
    assert text == (
        "Generelle tekniske krav\n"
        "Utkilingen skal ha helning 1:15.\n"
        "Merking\n"
        "Kravet gjelder alle veiklasser."
    )
    assert "#" not in text


def test_font_headings_emit_atx_the_proposer_already_reads() -> None:
    """Size class becomes ATX level, largest first, body untouched."""
    text = _extract(FONT_PDF, pdf_headings=True)
    assert text == (
        "# Generelle tekniske krav\n"
        "Utkilingen skal ha helning 1:15.\n"
        "## Merking\n"
        "Kravet gjelder alle veiklasser."
    )
    matched = [line for line in text.split("\n") if _ATX.match(line)]
    assert matched == ["# Generelle tekniske krav", "## Merking"]


def test_the_proposer_segments_the_font_headings_with_no_new_rule() -> None:
    """The whole point of emitting ATX: `find_candidates` needs no argument.

    Arm B -- every arm flag off -- is what runs here, so a candidate appearing
    is the heading rule reading the extraction, not an arm reading a PDF.
    """
    flat = find_candidates(_extract(FONT_PDF))
    marked = find_candidates(_extract(FONT_PDF, pdf_headings=True))
    assert [candidate.title for candidate in flat] == []
    assert [candidate.title for candidate in marked] == ["Generelle tekniske krav", "Merking"]
    assert {candidate.rule for candidate in marked} == {"rule:heading"}


def test_a_pdf_without_font_variation_is_byte_identical_with_the_flag_on() -> None:
    """The known-negative, and the reason the rule is a CONJUNCTION.

    `two-line-krav.pdf` is one font at one size. Nothing is larger than the
    body median and nothing is bold, so the reader must return the same bytes
    it returns with the flag off -- a rule that fired here would mark the first
    line of every flat document as a chapter.
    """
    assert _extract(FLAT_PDF, pdf_headings=True) == _extract(FLAT_PDF)


def test_the_locator_indexes_the_text_the_flag_produced() -> None:
    """A unit table built against the other rendering points at the wrong place.

    `source_units` re-derives page offsets from the parse, so it has to be told
    which rendering it is describing. Without the argument the page start would
    be computed from an unmarked page and land mid-heading in a marked one.
    """
    data = _read(FONT_PDF)
    text = _extract(FONT_PDF, pdf_headings=True)
    units = source_units(FONT_PDF.name, data, text, pdf_headings=True)
    assert units is not None
    assert units.unit == "pages"
    assert units.starts == (0,)
    assert text[units.starts[0] :].startswith("# Generelle tekniske krav")


# --- B: OCR behind the optional group ---------------------------------------


def test_ocr_without_the_optional_group_refuses_with_a_code(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """A missing engine is a typed refusal, never a traceback.

    Reached through the import probe rather than through the absence of the
    package, so the test states the same thing on a machine where the group IS
    installed. `None` in `sys.modules` is what CPython treats as a failed
    import.
    """
    monkeypatch.setitem(sys.modules, "rapidocr", None)
    with pytest.raises(ExtractionError) as excinfo:
        _extract(NO_TEXT_PDF, ocr=True)
    assert excinfo.value.code == "extractor_ocr_group_missing"
    message = str(excinfo.value)
    assert "ocr" in message
    assert "rapidocr" in message


def _fake_engine(lines: tuple[str, ...]) -> types.ModuleType:
    """A `rapidocr` stand-in returning fixed lines, so the SEAM is testable.

    The engine's reading quality is not this suite's claim and cannot be: it
    would need the optional group, a real scan, and a judgement about what the
    page says. What IS this suite's claim is that the trigger fires on the
    right pages, that the lines land in the page's place, and that nothing else
    moves -- all of which a fixed reader states better than a real one.
    """

    class _Result:
        txts = lines

    module = types.ModuleType("rapidocr")
    module.RapidOCR = lambda *args, **kwargs: lambda image: _Result()  # type: ignore[attr-defined]
    return module


def test_ocr_recovers_a_page_that_produced_no_text(monkeypatch: pytest.MonkeyPatch) -> None:
    """Without the flag this document is `extractor_empty_pdf`; with it, text."""
    monkeypatch.setitem(sys.modules, "rapidocr", _fake_engine(("Krav til helning", "1:15")))
    extract_module._pdf_pages.cache_clear()
    with pytest.raises(ExtractionError) as excinfo:
        _extract(NO_TEXT_PDF)
    assert excinfo.value.code == "extractor_empty_pdf"
    assert _extract(NO_TEXT_PDF, ocr=True) == "Krav til helning\n1:15"


def test_ocr_leaves_a_page_that_already_has_text_alone(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """The trigger is a measurement about the page, not a mode for the file.

    A readable page must come back byte-identical with `--ocr` on, or the flag
    would silently replace a document's own text with a guess about a picture
    of it.
    """
    monkeypatch.setitem(sys.modules, "rapidocr", _fake_engine(("WRONG",)))
    extract_module._pdf_pages.cache_clear()
    assert _extract(FLAT_PDF, ocr=True) == _extract(FLAT_PDF)


def test_the_cid_trigger_is_a_share_of_the_page_and_has_a_measured_floor() -> None:
    """The threshold is a number with a measurement behind it, not a taste.

    Pinned here so that moving it is a red test and a decision, and checked
    against the two shapes it has to separate: a page of placeholder codes and
    a page of prose.
    """
    assert extract_module.OCR_CID_SHARE == 0.10
    assert extract_module.cid_share("(cid:3)(cid:4)(cid:5)") == 1.0
    assert extract_module.cid_share("Krav til helning på utkilingen") == 0.0
    assert extract_module.cid_share("") == 0.0
    assert extract_module.needs_ocr("") is True
    assert extract_module.needs_ocr("   \n ") is True
    assert extract_module.needs_ocr("Krav til helning") is False


# --- both flags, reached from `okf build` ------------------------------------


#: A second document that ALWAYS yields a plan, so the run below is a run.
#: `okf build --segments on` refuses a corpus where nothing was proposed -- a
#: build asked to replay plans and given none would write a flat bundle and
#: report success -- so an inbox holding only the unsegmentable PDF would exit
#: 2 for a reason that has nothing to do with either flag.
ANCHOR = "# 1 Anker\n\nDette avsnittet finnes for at kjoeringen skal ha en plan.\n"


def _inbox(root: Path) -> Path:
    inbox = root / "inbox"
    inbox.mkdir(parents=True, exist_ok=True)
    (inbox / "anker.md").write_text(ANCHOR, encoding="utf-8", newline="")
    (inbox / FONT_PDF.name).write_bytes(_read(FONT_PDF))
    return inbox


def _plan_titles(plans: Path) -> list[str]:
    import json

    return [
        entry["title"]
        for path in sorted(plans.glob("*.json"))
        for entry in json.loads(path.read_text(encoding="utf-8"))["entries"]
    ]


def test_pdf_headings_reaches_the_proposer_from_the_build_command(tmp_path: Path) -> None:
    """The red test for round 4's A half: the flag exists and changes the plan.

    Asserted on the plan's TITLES rather than a count, for the reason every
    round-3 reach test is: a reader that cut in the wrong places would satisfy
    a count assertion exactly as well.

    ONE title from the PDF, not two, and that is the shipped default doing its
    job rather than the reader failing: Arm F folds a deeper heading into its
    parent, and `## Merking` is deeper than `# Generelle tekniske krav`. The
    reader's own output is both -- `test_font_headings_emit_atx...` above pins
    that -- and this is what the default does with it.
    """
    from llm_ingestion_okf import cli

    plans = tmp_path / "plans"
    assert (
        cli.main(
            [
                "build",
                str(_inbox(tmp_path)),
                "--bundle",
                str(tmp_path / "bundle"),
                "--bundle-id",
                "font-fixture",
                "--okf-version",
                "0.2",
                "--plans-dir",
                str(plans),
                "--pdf-headings",
                "font",
            ]
        )
        == 0
    )
    assert _plan_titles(plans) == ["1 Anker", "Generelle tekniske krav"]


def test_the_build_default_leaves_the_font_pdf_unsegmented(tmp_path: Path) -> None:
    """The control the test above rests on: same inbox, no flag, no plan.

    Without it, a change that turned the reader on by default would leave the
    assertion above green while moving every PDF bundle a consumer has built.
    """
    from llm_ingestion_okf import cli

    plans = tmp_path / "plans-plain"
    assert (
        cli.main(
            [
                "build",
                str(_inbox(tmp_path)),
                "--bundle",
                str(tmp_path / "bundle-plain"),
                "--bundle-id",
                "font-fixture",
                "--okf-version",
                "0.2",
                "--plans-dir",
                str(plans),
            ]
        )
        == 0
    )
    assert _plan_titles(plans) == ["1 Anker"]
    assert cli.DEFAULT_PDF_HEADINGS is False
    assert cli.DEFAULT_OCR is False


def test_ocr_without_the_group_is_a_coded_rejection_not_a_crash(
    tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
    """A missing engine must not take the run down; it accounts for the file.

    `merged + coded rejections == N` is the conservation identity every corpus
    run reports, and an engine nobody installed is exactly the kind of failure
    that would otherwise leave a document unaccounted for.
    """
    from llm_ingestion_okf import cli

    monkeypatch.setitem(sys.modules, "rapidocr", None)
    extract_module._pdf_pages.cache_clear()
    report = cli.build(
        _inbox(tmp_path),
        tmp_path / "bundle-ocr",
        bundle_id="font-fixture",
        okf_version="0.2",
        ocr=True,
    )
    assert dict(report.codes) == {"extractor_ocr_group_missing": 1}
    assert report.merged + report.rejected == report.n == 2