llm-ingestion-okf/tests/test_pdf_outline.py

"""The PDF's own `/Outlines` tree as a segmentation source.

MEASURED, OUTSIDE THIS REPOSITORY, ON ONE DOCUMENT: a 701-page process code
whose publisher also ships a NISO-STS structure for it. `okf build` on the
default recovered 1 967 of its 2 761 titled sections, 0 of 28 at the top level,
and **794 of 794 misses had their heading text present in the extracted text**
-- the boundary was never opened, the line was read. The same file carries an
`/Outlines` tree of 2 763 nodes which, after `re.sub(r"\\s+","",s).lower()`,
matches 2 761 of 2 761 STS titles exactly. The index shipped inside the file
and the build discarded it unopened.

WHAT THESE TESTS PIN, and why each one exists rather than "it found more":

- the LEVEL a node declares, not its distance from the root. A two-level tree
  cannot tell those apart, so the fixture has three.
- the LINE a mark lands on. 2 706 of 2 761 bookmarks in that document share a
  destination page with another bookmark, so a bridge that resolved to the page
  and stopped would be wrong on almost every node while still looking like it
  worked. The fixture's last page carries four lines and its second bookmark
  points at the third.
- a PDF with no `/Outlines` behaving IDENTICALLY with the arm on -- `pdfminer`
  raises `PDFNoOutlines` there, and "this file has no index" is not an error.
- an unresolvable `/Dest` being dropped and COUNTED. That document has 0 of
  2 763; a PDF in the wild has them, so without this the branch would ship
  having never run.
"""

from __future__ import annotations

from pathlib import Path

import pytest

from llm_ingestion_okf import extract, propose

pdfplumber = pytest.importorskip("pdfplumber")

FIXTURES = Path(__file__).parent / "fixtures"
OUTLINED = FIXTURES / "outlined-krav.pdf"
BROKEN = FIXTURES / "outline-broken-dest.pdf"
COLLISION = FIXTURES / "outline-collision.pdf"
NO_OUTLINE = FIXTURES / "three-page-krav.pdf"


def test_marks_carry_the_declared_level_and_the_line_they_land_on() -> None:
    data = OUTLINED.read_bytes()
    text = extract.extract_text(OUTLINED.name, data)
    lines = text.split("\n")
    outline = extract.pdf_outline(OUTLINED.name, data)

    assert outline.unresolved == 0
    assert [(mark.line, mark.level, mark.title) for mark in outline.marks] == [
        (0, 1, "1 Grunnlag"),
        (3, 2, "1.1 Omfang"),
        (6, 2, "1.2 Krav"),
        (8, 3, "1.2.1 Materialer"),
    ]
    # The offsets, stated against the text rather than against the tree: a mark
    # that named the right title on the wrong line would pass the list above if
    # the list were derived from the same walk.
    for mark in outline.marks:
        assert lines[mark.line] == mark.title


def test_the_last_mark_is_on_the_third_line_of_its_page_not_the_first() -> None:
    """Page resolution alone would put it two lines early, and still look right."""
    data = OUTLINED.read_bytes()
    marks = extract.pdf_outline(OUTLINED.name, data).marks
    page_three = [mark for mark in marks if mark.title.startswith("1.2")]
    assert [mark.line for mark in page_three] == [6, 8]


def test_a_pdf_with_no_outlines_yields_no_marks_and_no_error() -> None:
    data = NO_OUTLINE.read_bytes()
    outline = extract.pdf_outline(NO_OUTLINE.name, data)
    assert outline.marks == ()
    assert outline.unresolved == 0


def test_a_non_pdf_is_not_asked_the_question() -> None:
    outline = extract.pdf_outline("notat.md", b"# Overskrift\n\nBrodtekst.\n")
    assert outline.marks == ()
    assert outline.unresolved == 0


def test_an_unresolvable_destination_is_dropped_and_counted() -> None:
    data = BROKEN.read_bytes()
    outline = extract.pdf_outline(BROKEN.name, data)
    assert [mark.title for mark in outline.marks] == ["1 Grunnlag"]
    assert outline.unresolved == 1


def test_two_bookmarks_on_one_line_are_counted_not_lost() -> None:
    """The node that shares a line is DROPPED on purpose -- and never silently.

    Measured on R761: 2 763 nodes entered the bridge, 2 762 marks came out and
    `unresolved` was 0, so one node left no trace anywhere. The line it shared
    was line 0, between the tree's own root node and `SVV - Forside`.

    Keeping BOTH was measured on that same document and felled: the two
    candidates then open at the same offset, and the first closes with an
    EMPTY span (0, 0) that the orphan check deletes without a word -- the same
    node lost, one step later, plus a front-matter title that is in no fasit.
    So the rule is first-in-tree-order wins, and the loser is COUNTED.
    """
    data = COLLISION.read_bytes()
    outline = extract.pdf_outline(COLLISION.name, data)

    assert [(mark.line, mark.level, mark.title) for mark in outline.marks] == [
        (0, 1, "R761 Prosesskoden"),
    ]
    assert outline.unresolved == 0
    assert outline.collided == 1
    # The identity is the point, not the counter: every node the tree declared
    # is a mark, an unresolved destination, or a collision, and nothing else.
    assert len(outline.marks) + outline.unresolved + outline.collided == 2


def test_a_bookmark_tree_with_no_collision_counts_none() -> None:
    """The known-negative on the same field: the counter must be able to be 0."""
    outline = extract.pdf_outline(OUTLINED.name, OUTLINED.read_bytes())
    assert outline.collided == 0
    assert len(outline.marks) + outline.unresolved + outline.collided == 4

    broken = extract.pdf_outline(BROKEN.name, BROKEN.read_bytes())
    assert broken.collided == 0
    assert len(broken.marks) + broken.unresolved + broken.collided == 2


def test_the_outline_replaces_the_text_heuristics_when_it_is_given() -> None:
    data = OUTLINED.read_bytes()
    text = extract.extract_text(OUTLINED.name, data)
    marks = extract.pdf_outline(OUTLINED.name, data).marks

    candidates = propose.find_candidates(text, outline_marks=marks, close_span_gaps=True)
    assert [(c.number, c.title, c.level, c.rule) for c in candidates] == [
        ("1", "Grunnlag", 1, propose.RULE_PDF_OUTLINE),
        ("1.1", "Omfang", 2, propose.RULE_PDF_OUTLINE),
        ("1.2", "Krav", 2, propose.RULE_PDF_OUTLINE),
        ("1.2.1", "Materialer", 3, propose.RULE_PDF_OUTLINE),
    ]
    # The spans partition the text: the arm cuts, it does not drop.
    assert candidates[0].start == 0
    assert candidates[-1].end == len(text)


def test_a_declared_section_with_no_prose_of_its_own_survives() -> None:
    """The orphan check asks the wrong question of a publisher's own tree.

    Measured on that 701-page document: 683 of its 2 762 bookmark marks are
    followed immediately by their first subsection, with no prose between. They
    are container sections, not mis-detected headings -- the check exists to
    catch a heuristic's false positive, and there is no heuristic here. Left
    in, the arm scores 2 079 of 2 761 instead of 2 762 of 2 763.
    """
    text = "1 Grunnlag\n1.1 Omfang\nOmfanget dekker alt.\n"
    marks = (
        extract.OutlineMark(line=0, level=1, title="1 Grunnlag"),
        extract.OutlineMark(line=1, level=2, title="1.1 Omfang"),
    )
    candidates = propose.find_candidates(text, outline_marks=marks)
    assert [c.title for c in candidates] == ["Grunnlag", "Omfang"]


def test_an_empty_mark_list_leaves_every_other_rule_untouched() -> None:
    """`()` is "this file has no index", never "propose nothing"."""
    data = OUTLINED.read_bytes()
    text = extract.extract_text(OUTLINED.name, data)
    assert propose.find_candidates(text, outline_marks=()) == propose.find_candidates(text)