feat(extract): implement pdf behind the [extract] extra with pdfplumber

Order G2a. Populates the optional `[extract]` extra for the first time with
one parser, `pdfplumber>=0.11.10,<0.12` (MIT), and wires `pdf` through it.
The default install is untouched: exactly one runtime dependency, stdlib
otherwise, enforced by test_packaging.py.

The gate for `pdf` becomes an import probe rather than a frozenset membership
test, exactly as extract.py's docstring had promised. The rejection does not
change: without the extra, `pdf` still raises `extractor_extra_missing` with
the same message. That behaviour is asserted UNCONDITIONALLY via a sys.modules
monkeypatch, so it holds on machines where the parser is installed too — a
skip would have preserved nothing there. Verified in a clean venv without the
extra: 589 passed, 7 skipped; with it, 596 passed.

`docx`/`xlsx` are unchanged and still fail fast — the extra names exactly what
it ships.

The parser choice was forced by measurement, not preference (b73dd9d,
docs/2026-08-21-g2-pdf-extraction-measurement.md): on a real requirement table
pdfplumber keeps 4 of 4 rows with label and value on one line, where pypdf,
pdfminer.six and pymupdf each keep 0 of 4. pymupdf is additionally out on
licence (AGPL-3.0), which an MIT package must not push onto a consumer.

Three facts from that measurement are now carried in code rather than in a
report:

- Extracted text is pinned to an exact transitive parser version
  (pdfplumber pins pdfminer.six==20260107; date-stamped, no stability
  contract). tests/test_extract.py freezes the expected text of a committed
  hand-written fixture so a parser upgrade breaks something visible instead of
  drifting silently. Reasoning at the declaration site and in
  tests/fixtures/README.md.
- Determinism within a version is now held by a test, not only measured once.
- Drawn content does not survive extraction. Every pdf extraction emits the
  new `ExtractionWarning`: figures have no text to recover, so a bundle built
  from drawn documents is incomplete by construction. Stated categorically
  rather than detected — deciding "is there a figure here" is the layout
  heuristic G2b declined.

Two new error codes, both mirroring existing patterns: `extractor_empty_pdf`
(a scanned/image-only PDF, refused rather than persisted as an empty concept)
and `extractor_pdf_error` (parser failure wrapped, never leaked).

Structured table recovery (G2b) is NOT implemented and is documented as out of
scope: two independent parsers return the same wrong shape, so the breakage is
document geometry, not a library choice. PDFs enter as prose.

Also corrects an install promise this change would otherwise have published:
the README no longer presents a bare `pip install 'llm-ingestion-okf[extract]'`
as working, because the package is not on an index.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01HtNhsdHnMGtMi7U2mvMU8z
This commit is contained in:
Kjell Tore Guttormsen 2026-08-21 20:22:39 +02:00
commit 658b7aafe0
14 changed files with 1046 additions and 37 deletions

View file

@ -9,9 +9,24 @@ call — it is pure, deterministic plumbing.
from __future__ import annotations
import importlib.util
import sys
from pathlib import Path
import pytest
from llm_ingestion_okf import ExtractionError, extract_text
from llm_ingestion_okf import ExtractionError, ExtractionWarning, extract_text
FIXTURES = Path(__file__).parent / "fixtures"
# The parser-dependent tests below need the optional extra, so they are skipped
# without it — an optional extra that forced its parser on every test run would
# not be optional. The behaviour that must survive EITHER WAY is the rejection,
# and that one is asserted unconditionally above via the import probe.
requires_extract = pytest.mark.skipif(
importlib.util.find_spec("pdfplumber") is None,
reason="the optional [extract] extra is not installed",
)
# --- core extractors: happy path (a fixture per type) ---
@ -74,11 +89,11 @@ def test_missing_extension_fails_fast() -> None:
assert excinfo.value.code == "extractor_unknown"
# --- fail-fast: [extract]-gated binary types without the extra ---
# --- fail-fast: [extract]-gated binary types the extra ships no parser for ---
@pytest.mark.parametrize("name", ["doc.pdf", "doc.docx", "sheet.xlsx"])
def test_optional_type_without_extra_fails_fast(name: str) -> None:
@pytest.mark.parametrize("name", ["doc.docx", "sheet.xlsx"])
def test_optional_type_without_a_parser_fails_fast(name: str) -> None:
with pytest.raises(ExtractionError) as excinfo:
extract_text(name, b"binary")
assert excinfo.value.code == "extractor_extra_missing"
@ -87,6 +102,80 @@ def test_optional_type_without_extra_fails_fast(name: str) -> None:
assert "extract" in str(excinfo.value)
def test_pdf_without_the_extra_installed_keeps_the_same_rejection(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""The gate became an import probe; the rejection did not change.
Setting the module to None in sys.modules is what CPython treats as a
failed import, so this exercises the uninstalled path REGARDLESS of
whether the extra is installed in the running environment a skip would
have preserved nothing on the machine where the parser is present.
"""
monkeypatch.setitem(sys.modules, "pdfplumber", None)
with pytest.raises(ExtractionError) as excinfo:
extract_text("doc.pdf", b"%PDF-1.4")
assert excinfo.value.code == "extractor_extra_missing"
assert "extract" in str(excinfo.value)
# --- pdf: the [extract] parser (pdfplumber) ---
# The expected text is frozen against a committed fixture ON PURPOSE. Extraction
# is deterministic within a parser version and NOT guaranteed across one
# (pdfplumber pins pdfminer.six==20260107 exactly; pdfminer.six ships
# date-stamped releases with no stability contract). This literal is what makes
# a parser upgrade break something visible instead of drifting silently.
KRAV_TEXT = "Krav til helning på utkilingen\n60 og 70 1:15"
@requires_extract
def test_pdf_extracts_text_with_label_and_value_on_one_line() -> None:
data = (FIXTURES / "two-line-krav.pdf").read_bytes()
with pytest.warns(ExtractionWarning):
assert extract_text("krav.pdf", data) == KRAV_TEXT
@requires_extract
def test_pdf_extraction_is_byte_stable_across_calls() -> None:
"""Determinism is a promise this library already makes; hold it here too."""
data = (FIXTURES / "two-line-krav.pdf").read_bytes()
with pytest.warns(ExtractionWarning):
first = extract_text("krav.pdf", data)
second = extract_text("krav.pdf", data)
assert first == second
@requires_extract
def test_pdf_extraction_warns_that_drawn_content_is_not_recovered() -> None:
"""Figures are vector drawings: only the caption survives leg 2.
Categorically true of text extraction, so it is stated as a warning on
every PDF rather than guessed at per document detecting "is there a
figure here" would be exactly the layout heuristic this order declined.
"""
data = (FIXTURES / "two-line-krav.pdf").read_bytes()
with pytest.warns(ExtractionWarning, match="figures"):
extract_text("krav.pdf", data)
@requires_extract
def test_pdf_with_no_text_layer_fails_fast() -> None:
"""A scanned PDF yields nothing; an empty concept would be a silent skip."""
data = (FIXTURES / "no-text-layer.pdf").read_bytes()
with pytest.raises(ExtractionError) as excinfo:
extract_text("scan.pdf", data)
assert excinfo.value.code == "extractor_empty_pdf"
@requires_extract
def test_corrupt_pdf_fails_fast_typed() -> None:
"""Never a leaked pdfminer exception — the "always typed" doctrine holds."""
with pytest.raises(ExtractionError) as excinfo:
extract_text("broken.pdf", b"not a pdf at all")
assert excinfo.value.code == "extractor_pdf_error"
# --- fail-fast: corrupt (non-UTF-8) bytes on a text type ---