llm-ingestion-okf/src/llm_ingestion_okf/extract.py

"""Door B extraction registry: dropped file bytes -> text, per file type.

All file-type -> text extraction lives here (the guard is text-only). The core
registry is stdlib-only and deterministic: `md`/`txt` pass through, `csv` renders
the Phase 1 markdown table, `json` is fenced verbatim, and `html`/`htm` are
reduced to text with `html.parser`. Binary types (`pdf`/`docx`/`xlsx`) are
`[extract]`-gated: `pdf` is extracted with `pdfplumber` when the extra is
installed and rejected with the same typed error when it is not (the gate is an
import probe, not a membership test), while `docx`/`xlsx` ship no parser yet and
always fail fast. Never a silent skip and never a bundled parser in core.

`extract_text` returns the extracted text *content*; final LF framing and the
concept frontmatter are the materializer's concern (Phase 2 step 2), not this
registry's. No guard call and no model call anywhere in this module.
"""

from __future__ import annotations

import csv
import io
import warnings
from collections.abc import Callable
from html.parser import HTMLParser
from pathlib import Path

from .errors import ExtractionError, ExtractionWarning
from .render import render_fenced_block, render_table

# Binary types gated behind the optional `[extract]` extra that it ships no
# parser for. `.pdf` is no longer here: its gate is the import probe inside
# `_extract_pdf`, which raises the same code and the same message when the
# extra is absent. These two still fail fast unconditionally.
_UNPARSED_OPTIONAL_EXTENSIONS = frozenset({".docx", ".xlsx"})

# Text extraction recovers text. A figure is a vector drawing with no text to
# recover — only its caption survives — so any bundle built from drawn
# documents is incomplete by construction. Said out loud on every PDF rather
# than detected per document: deciding "is there a figure here" is a layout
# heuristic this library does not own.
_PDF_LOSSY_WARNING = (
    "PDF extraction recovers text only: figures, diagrams and images are not "
    "represented in the output (their captions are). A bundle built from "
    "drawn documents is incomplete by construction."
)

# Tags whose text content is never document prose.
_SKIP_TAGS = frozenset({"script", "style"})


def decode_text(data: bytes) -> str:
    """Decode file bytes as UTF-8 (BOM-stripping), typed on failure.

    utf-8-sig so a byte-order mark never leaks into the first character
    (baseline parity with Door A's read_csv). A non-UTF-8 file is a corrupt
    input: fail fast with a typed error rather than leaking UnicodeDecodeError.
    """
    try:
        return data.decode("utf-8-sig")
    except UnicodeDecodeError as exc:
        raise ExtractionError(
            f"file bytes are not valid UTF-8: {exc}", code="extractor_decode_error"
        ) from exc


def _extract_passthrough(data: bytes) -> str:
    """`md`/`txt`: the decoded text verbatim."""
    return decode_text(data)


def _extract_csv(data: bytes) -> str:
    """`csv`: parse with the stdlib reader, render the Phase 1 markdown table."""
    reader = csv.reader(io.StringIO(decode_text(data)))
    header = next(reader, None)
    if header is None:
        raise ExtractionError("CSV has no header row", code="extractor_empty_csv")
    rows = list(reader)
    return render_table(header, rows)


def _extract_json(data: bytes) -> str:
    """`json`: the decoded text verbatim inside a fenced block (Phase 1 renderer)."""
    return render_fenced_block(decode_text(data))


class _HTMLTextExtractor(HTMLParser):
    """Collect document text, skipping `script`/`style`, tags as word boundaries.

    Tags contribute no text of their own but do separate words: a boundary
    space is emitted at every tag so adjacent block text (``</h1><p>``) does not
    fuse. Runs of whitespace collapse to single spaces in :meth:`text`.
    """

    def __init__(self) -> None:
        super().__init__(convert_charrefs=True)
        self._parts: list[str] = []
        self._skip_depth = 0

    def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
        self._parts.append(" ")
        if tag in _SKIP_TAGS:
            self._skip_depth += 1

    def handle_startendtag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
        self._parts.append(" ")

    def handle_endtag(self, tag: str) -> None:
        if tag in _SKIP_TAGS and self._skip_depth > 0:
            self._skip_depth -= 1
        self._parts.append(" ")

    def handle_data(self, data: str) -> None:
        if self._skip_depth == 0:
            self._parts.append(data)

    def text(self) -> str:
        return " ".join("".join(self._parts).split())


def _extract_html(data: bytes) -> str:
    """`html`/`htm`: text via `html.parser`, script/style stripped (spec B3)."""
    parser = _HTMLTextExtractor()
    parser.feed(decode_text(data))
    parser.close()
    return parser.text()


def _extra_missing(suffix: str) -> ExtractionError:
    """The one rejection for a `[extract]` type without the extra installed.

    One constructor, one wording: the import probe and the still-unparsed
    types must be indistinguishable to a consumer, because they are the same
    fact — the extra is not installed.
    """
    return ExtractionError(
        f"extracting {suffix!r} requires the optional 'extract' extra "
        f"(pip install 'llm-ingestion-okf[extract]'); it is not installed",
        code="extractor_extra_missing",
    )


def _extract_pdf(data: bytes) -> str:
    """`pdf`: page text via `pdfplumber`, in page order, pages separated by a
    blank line.

    The gate is this import, not a membership test: without the `[extract]`
    extra the very same typed rejection is raised as for the types that ship
    no parser at all. Text is returned VERBATIM — no Unicode normalization,
    matching `md`/`txt` passthrough; normalizing would edit source content,
    and NFC folding belongs to filenames and titles, not to document bodies.

    `pdfplumber` was chosen on ONE measured property (2026-08-21,
    docs/2026-08-21-g2-pdf-extraction-measurement.md): on a real requirement
    table it keeps label and value on the same line, where pypdf, pdfminer.six
    and pymupdf each emit all labels then all values. Re-pairing those is
    guesswork, and in a requirements document a wrong pairing looks right.
    """
    try:
        import pdfplumber
    except ImportError as exc:
        raise _extra_missing(".pdf") from exc

    try:
        with pdfplumber.open(io.BytesIO(data)) as pdf:
            pages = [(page.extract_text() or "").rstrip() for page in pdf.pages]
    except ExtractionError:
        raise
    except Exception as exc:  # noqa: BLE001 - third-party parser, wrapped never leaked
        raise ExtractionError(
            f"the PDF parser failed on this file: {exc}", code="extractor_pdf_error"
        ) from exc

    text = "\n\n".join(page for page in pages if page)
    if not text:
        raise ExtractionError(
            "the PDF yielded no text on any page; a scanned or image-only "
            "document needs OCR, which this registry does not do",
            code="extractor_empty_pdf",
        )
    # After the parse, not before: a run that produced no text has nothing to
    # be lossy about, and warning there would just add noise to a failure.
    warnings.warn(_PDF_LOSSY_WARNING, ExtractionWarning, stacklevel=3)
    return text


_CORE_EXTRACTORS: dict[str, Callable[[bytes], str]] = {
    ".md": _extract_passthrough,
    ".txt": _extract_passthrough,
    ".csv": _extract_csv,
    ".json": _extract_json,
    ".html": _extract_html,
    ".htm": _extract_html,
}

# Types the `[extract]` extra ships a parser for. Kept separate from the core
# registry so "adds no runtime dependency" stays readable at a glance.
_OPTIONAL_EXTRACTORS: dict[str, Callable[[bytes], str]] = {
    ".pdf": _extract_pdf,
}


def extract_text(filename: str, data: bytes) -> str:
    """Convert one dropped file's bytes to OKF concept text, dispatched by type.

    `filename` supplies the extension (case-insensitive); `data` is the raw
    bytes. A core stdlib type is extracted; a `[extract]`-gated binary type
    without the extra, and any unregistered extension, fail fast with a typed
    :class:`ExtractionError`. Extracting a `pdf` also emits an
    :class:`ExtractionWarning`: drawn content has no text to recover.
    """
    suffix = Path(filename).suffix.lower()
    extractor = _CORE_EXTRACTORS.get(suffix) or _OPTIONAL_EXTRACTORS.get(suffix)
    if extractor is not None:
        return extractor(data)
    if suffix in _UNPARSED_OPTIONAL_EXTENSIONS:
        raise _extra_missing(suffix)
    raise ExtractionError(
        f"no extractor is registered for file extension {suffix!r} ({filename!r})",
        code="extractor_unknown",
    )