feat(extract): implement pdf behind the [extract] extra with pdfplumber
Order G2a. Populates the optional `[extract]` extra for the first time with
one parser, `pdfplumber>=0.11.10,<0.12` (MIT), and wires `pdf` through it.
The default install is untouched: exactly one runtime dependency, stdlib
otherwise, enforced by test_packaging.py.
The gate for `pdf` becomes an import probe rather than a frozenset membership
test, exactly as extract.py's docstring had promised. The rejection does not
change: without the extra, `pdf` still raises `extractor_extra_missing` with
the same message. That behaviour is asserted UNCONDITIONALLY via a sys.modules
monkeypatch, so it holds on machines where the parser is installed too — a
skip would have preserved nothing there. Verified in a clean venv without the
extra: 589 passed, 7 skipped; with it, 596 passed.
`docx`/`xlsx` are unchanged and still fail fast — the extra names exactly what
it ships.
The parser choice was forced by measurement, not preference (b73dd9d,
docs/2026-08-21-g2-pdf-extraction-measurement.md): on a real requirement table
pdfplumber keeps 4 of 4 rows with label and value on one line, where pypdf,
pdfminer.six and pymupdf each keep 0 of 4. pymupdf is additionally out on
licence (AGPL-3.0), which an MIT package must not push onto a consumer.
Three facts from that measurement are now carried in code rather than in a
report:
- Extracted text is pinned to an exact transitive parser version
(pdfplumber pins pdfminer.six==20260107; date-stamped, no stability
contract). tests/test_extract.py freezes the expected text of a committed
hand-written fixture so a parser upgrade breaks something visible instead of
drifting silently. Reasoning at the declaration site and in
tests/fixtures/README.md.
- Determinism within a version is now held by a test, not only measured once.
- Drawn content does not survive extraction. Every pdf extraction emits the
new `ExtractionWarning`: figures have no text to recover, so a bundle built
from drawn documents is incomplete by construction. Stated categorically
rather than detected — deciding "is there a figure here" is the layout
heuristic G2b declined.
Two new error codes, both mirroring existing patterns: `extractor_empty_pdf`
(a scanned/image-only PDF, refused rather than persisted as an empty concept)
and `extractor_pdf_error` (parser failure wrapped, never leaked).
Structured table recovery (G2b) is NOT implemented and is documented as out of
scope: two independent parsers return the same wrong shape, so the breakage is
document geometry, not a library choice. PDFs enter as prose.
Also corrects an install promise this change would otherwise have published:
the README no longer presents a bare `pip install 'llm-ingestion-okf[extract]'`
as working, because the package is not on an index.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01HtNhsdHnMGtMi7U2mvMU8z
This commit is contained in:
parent
b73dd9d6a4
commit
658b7aafe0
14 changed files with 1046 additions and 37 deletions
|
|
@ -9,9 +9,24 @@ call — it is pure, deterministic plumbing.
|
|||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from llm_ingestion_okf import ExtractionError, extract_text
|
||||
from llm_ingestion_okf import ExtractionError, ExtractionWarning, extract_text
|
||||
|
||||
FIXTURES = Path(__file__).parent / "fixtures"
|
||||
|
||||
# The parser-dependent tests below need the optional extra, so they are skipped
|
||||
# without it — an optional extra that forced its parser on every test run would
|
||||
# not be optional. The behaviour that must survive EITHER WAY is the rejection,
|
||||
# and that one is asserted unconditionally above via the import probe.
|
||||
requires_extract = pytest.mark.skipif(
|
||||
importlib.util.find_spec("pdfplumber") is None,
|
||||
reason="the optional [extract] extra is not installed",
|
||||
)
|
||||
|
||||
# --- core extractors: happy path (a fixture per type) ---
|
||||
|
||||
|
|
@ -74,11 +89,11 @@ def test_missing_extension_fails_fast() -> None:
|
|||
assert excinfo.value.code == "extractor_unknown"
|
||||
|
||||
|
||||
# --- fail-fast: [extract]-gated binary types without the extra ---
|
||||
# --- fail-fast: [extract]-gated binary types the extra ships no parser for ---
|
||||
|
||||
|
||||
@pytest.mark.parametrize("name", ["doc.pdf", "doc.docx", "sheet.xlsx"])
|
||||
def test_optional_type_without_extra_fails_fast(name: str) -> None:
|
||||
@pytest.mark.parametrize("name", ["doc.docx", "sheet.xlsx"])
|
||||
def test_optional_type_without_a_parser_fails_fast(name: str) -> None:
|
||||
with pytest.raises(ExtractionError) as excinfo:
|
||||
extract_text(name, b"binary")
|
||||
assert excinfo.value.code == "extractor_extra_missing"
|
||||
|
|
@ -87,6 +102,80 @@ def test_optional_type_without_extra_fails_fast(name: str) -> None:
|
|||
assert "extract" in str(excinfo.value)
|
||||
|
||||
|
||||
def test_pdf_without_the_extra_installed_keeps_the_same_rejection(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""The gate became an import probe; the rejection did not change.
|
||||
|
||||
Setting the module to None in sys.modules is what CPython treats as a
|
||||
failed import, so this exercises the uninstalled path REGARDLESS of
|
||||
whether the extra is installed in the running environment — a skip would
|
||||
have preserved nothing on the machine where the parser is present.
|
||||
"""
|
||||
monkeypatch.setitem(sys.modules, "pdfplumber", None)
|
||||
with pytest.raises(ExtractionError) as excinfo:
|
||||
extract_text("doc.pdf", b"%PDF-1.4")
|
||||
assert excinfo.value.code == "extractor_extra_missing"
|
||||
assert "extract" in str(excinfo.value)
|
||||
|
||||
|
||||
# --- pdf: the [extract] parser (pdfplumber) ---
|
||||
|
||||
# The expected text is frozen against a committed fixture ON PURPOSE. Extraction
|
||||
# is deterministic within a parser version and NOT guaranteed across one
|
||||
# (pdfplumber pins pdfminer.six==20260107 exactly; pdfminer.six ships
|
||||
# date-stamped releases with no stability contract). This literal is what makes
|
||||
# a parser upgrade break something visible instead of drifting silently.
|
||||
KRAV_TEXT = "Krav til helning på utkilingen\n60 og 70 1:15"
|
||||
|
||||
|
||||
@requires_extract
|
||||
def test_pdf_extracts_text_with_label_and_value_on_one_line() -> None:
|
||||
data = (FIXTURES / "two-line-krav.pdf").read_bytes()
|
||||
with pytest.warns(ExtractionWarning):
|
||||
assert extract_text("krav.pdf", data) == KRAV_TEXT
|
||||
|
||||
|
||||
@requires_extract
|
||||
def test_pdf_extraction_is_byte_stable_across_calls() -> None:
|
||||
"""Determinism is a promise this library already makes; hold it here too."""
|
||||
data = (FIXTURES / "two-line-krav.pdf").read_bytes()
|
||||
with pytest.warns(ExtractionWarning):
|
||||
first = extract_text("krav.pdf", data)
|
||||
second = extract_text("krav.pdf", data)
|
||||
assert first == second
|
||||
|
||||
|
||||
@requires_extract
|
||||
def test_pdf_extraction_warns_that_drawn_content_is_not_recovered() -> None:
|
||||
"""Figures are vector drawings: only the caption survives leg 2.
|
||||
|
||||
Categorically true of text extraction, so it is stated as a warning on
|
||||
every PDF rather than guessed at per document — detecting "is there a
|
||||
figure here" would be exactly the layout heuristic this order declined.
|
||||
"""
|
||||
data = (FIXTURES / "two-line-krav.pdf").read_bytes()
|
||||
with pytest.warns(ExtractionWarning, match="figures"):
|
||||
extract_text("krav.pdf", data)
|
||||
|
||||
|
||||
@requires_extract
|
||||
def test_pdf_with_no_text_layer_fails_fast() -> None:
|
||||
"""A scanned PDF yields nothing; an empty concept would be a silent skip."""
|
||||
data = (FIXTURES / "no-text-layer.pdf").read_bytes()
|
||||
with pytest.raises(ExtractionError) as excinfo:
|
||||
extract_text("scan.pdf", data)
|
||||
assert excinfo.value.code == "extractor_empty_pdf"
|
||||
|
||||
|
||||
@requires_extract
|
||||
def test_corrupt_pdf_fails_fast_typed() -> None:
|
||||
"""Never a leaked pdfminer exception — the "always typed" doctrine holds."""
|
||||
with pytest.raises(ExtractionError) as excinfo:
|
||||
extract_text("broken.pdf", b"not a pdf at all")
|
||||
assert excinfo.value.code == "extractor_pdf_error"
|
||||
|
||||
|
||||
# --- fail-fast: corrupt (non-UTF-8) bytes on a text type ---
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue