llm-ingestion-okf/src/llm_ingestion_okf/extract.py

"""Door B extraction registry: dropped file bytes -> text, per file type.

All file-type -> text extraction lives here (the guard is text-only). The core
registry is stdlib-only and deterministic: `md`/`txt` pass through, `csv` renders
the Phase 1 markdown table, `json` is fenced verbatim, and `html`/`htm` are
reduced to text with `html.parser`. Binary types are `[extract]`-gated and each
now has a reader: `pdf` through `pdfplumber`, and the five office rows
(`docx`/`xlsx`/`pptx`/`odt`/`rtf`) through a table-driven converter seam over
the vendored binary. Every one of those gates is an IMPORT PROBE rather than a
membership test, so an absent extra is rejected with the same typed error
whatever the type. Never a silent skip and never a bundled parser in core.

Two of the five office rows are `measured` and three are `unmeasured` -- the
corpus this arm was built on contains zero `pptx`, `odt` or `rtf` files, so
those rows work by construction and have never met a document anyone wrote.
`_EVIDENCE` carries that per row and the suite asserts it, because an
unmeasured row must not read as a supported one.

`extract_text` returns the extracted text *content*; final LF framing and the
concept frontmatter are the materializer's concern (Phase 2 step 2), not this
registry's. No guard call and no model call anywhere in this module.
"""

from __future__ import annotations

import csv
import functools
import io
import tempfile
import warnings
from collections.abc import Callable, Sequence
from html.parser import HTMLParser
from pathlib import Path

from .errors import ExtractionError, ExtractionWarning
from .render import render_fenced_block, render_table

# Binary types gated behind the optional `[extract]` extra that it ships no
# parser for. EMPTY, and kept rather than deleted: the dispatch branch it feeds
# still raises `extractor_extra_missing`, and a later type that arrives before
# its reader belongs here rather than in a new mechanism. Every type the extra
# names now has a reader -- `.pdf` through the import probe in `_extract_pdf`,
# the five office rows through the converter seam below -- so the gate for all
# of them is an import probe, which is why the two tests for that code reach it
# that way.
_UNPARSED_OPTIONAL_EXTENSIONS: frozenset[str] = frozenset()

# The office rows: suffix -> the converter's reader name. THESE ROWS AND NO
# OTHERS. `.html` is excluded although the converter can read it: it already
# has a stdlib extractor here, so routing it through the converter would buy
# nothing and would add CVE-2025-51591 (SSRF via an iframe in HTML input),
# unpatched in every converter version. `.epub` is excluded on the "no gain"
# half of the same reason.
_PANDOC_FORMATS: dict[str, str] = {
    ".docx": "docx",
    ".xlsx": "xlsx",
    ".pptx": "pptx",
    ".odt": "odt",
    ".rtf": "rtf",
}

# What each row's behaviour actually rests on, asserted in the suite rather
# than written in a comment that rots. `measured` means real corpus files and a
# hand-counted fasit; `unmeasured` means the corpus contains ZERO files of that
# type, so the row works by construction and has never been checked against a
# document anyone wrote. An unmeasured row must not read as a supported one.
_EVIDENCE: dict[str, str] = {
    ".docx": "measured",
    ".xlsx": "measured",
    ".pptx": "unmeasured",
    ".odt": "unmeasured",
    ".rtf": "unmeasured",
}

# Load-bearing, all three, and none of them hygiene:
#
#   --eol=lf     the defaults produce DIFFERENT BYTES (maximum line length 75
#   --wrap=none  against 447), which a byte-pinned golden registers as a change
#                nobody made.
#   -t markdown  never `-t plain`: plain destroys the headings the segment
#                proposer reads. Measured -- a document yielding 15 entries
#                including two real headings yields 13 with none under `plain`,
#                so the writer choice silently sets the ceiling for the arm
#                downstream of it.
_PANDOC_WRITER = "markdown"
_PANDOC_ARGS = ("--eol=lf", "--wrap=none")

# Conversion recovers text, on the same terms as PDF extraction: a drawing has
# no text to recover. Said out loud on every conversion rather than detected
# per document, for the same reason.
_OFFICE_LOSSY_WARNING = (
    "office-file conversion recovers text only: figures, diagrams, images and "
    "drawn shapes are not represented in the output (their captions are). A "
    "bundle built from drawn documents is incomplete by construction."
)

# Text extraction recovers text. A figure is a vector drawing with no text to
# recover — only its caption survives — so any bundle built from drawn
# documents is incomplete by construction. Said out loud on every PDF rather
# than detected per document: deciding "is there a figure here" is a layout
# heuristic this library does not own.
_PDF_LOSSY_WARNING = (
    "PDF extraction recovers text only: figures, diagrams and images are not "
    "represented in the output (their captions are). A bundle built from "
    "drawn documents is incomplete by construction."
)

# Tags whose text content is never document prose.
_SKIP_TAGS = frozenset({"script", "style"})


def decode_text(data: bytes) -> str:
    """Decode file bytes as UTF-8 (BOM-stripping), typed on failure.

    utf-8-sig so a byte-order mark never leaks into the first character
    (baseline parity with Door A's read_csv). A non-UTF-8 file is a corrupt
    input: fail fast with a typed error rather than leaking UnicodeDecodeError.
    """
    try:
        return data.decode("utf-8-sig")
    except UnicodeDecodeError as exc:
        raise ExtractionError(
            f"file bytes are not valid UTF-8: {exc}", code="extractor_decode_error"
        ) from exc


def _extract_passthrough(data: bytes) -> str:
    """`md`/`txt`: the decoded text verbatim."""
    return decode_text(data)


def _extract_csv(data: bytes) -> str:
    """`csv`: parse with the stdlib reader, render the Phase 1 markdown table."""
    reader = csv.reader(io.StringIO(decode_text(data)))
    header = next(reader, None)
    if header is None:
        raise ExtractionError("CSV has no header row", code="extractor_empty_csv")
    rows = list(reader)
    return render_table(header, rows)


def _extract_json(data: bytes) -> str:
    """`json`: the decoded text verbatim inside a fenced block (Phase 1 renderer)."""
    return render_fenced_block(decode_text(data))


class _HTMLTextExtractor(HTMLParser):
    """Collect document text, skipping `script`/`style`, tags as word boundaries.

    Tags contribute no text of their own but do separate words: a boundary
    space is emitted at every tag so adjacent block text (``</h1><p>``) does not
    fuse. Runs of whitespace collapse to single spaces in :meth:`text`.
    """

    def __init__(self) -> None:
        super().__init__(convert_charrefs=True)
        self._parts: list[str] = []
        self._skip_depth = 0

    def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
        self._parts.append(" ")
        if tag in _SKIP_TAGS:
            self._skip_depth += 1

    def handle_startendtag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
        self._parts.append(" ")

    def handle_endtag(self, tag: str) -> None:
        if tag in _SKIP_TAGS and self._skip_depth > 0:
            self._skip_depth -= 1
        self._parts.append(" ")

    def handle_data(self, data: str) -> None:
        if self._skip_depth == 0:
            self._parts.append(data)

    def text(self) -> str:
        return " ".join("".join(self._parts).split())


def _extract_html(data: bytes) -> str:
    """`html`/`htm`: text via `html.parser`, script/style stripped (spec B3)."""
    parser = _HTMLTextExtractor()
    parser.feed(decode_text(data))
    parser.close()
    return parser.text()


def _extra_missing(suffix: str) -> ExtractionError:
    """The one rejection for a `[extract]` type without the extra installed.

    One constructor, one wording: the import probe and the still-unparsed
    types must be indistinguishable to a consumer, because they are the same
    fact — the extra is not installed.
    """
    return ExtractionError(
        f"extracting {suffix!r} requires the optional 'extract' extra "
        f"(pip install 'llm-ingestion-okf[extract]'); it is not installed",
        code="extractor_extra_missing",
    )


def _extract_pdf(data: bytes) -> str:
    """`pdf`: page text via `pdfplumber`, in page order, pages separated by a
    blank line.

    The gate is this import, not a membership test: without the `[extract]`
    extra the very same typed rejection is raised as for the types that ship
    no parser at all. Text is returned VERBATIM — no Unicode normalization,
    matching `md`/`txt` passthrough; normalizing would edit source content,
    and NFC folding belongs to filenames and titles, not to document bodies.

    `pdfplumber` was chosen on ONE measured property (2026-08-21,
    docs/2026-08-21-g2-pdf-extraction-measurement.md): on a real requirement
    table it keeps label and value on the same line, where pypdf, pdfminer.six
    and pymupdf each emit all labels then all values. Re-pairing those is
    guesswork, and in a requirements document a wrong pairing looks right.
    """
    try:
        import pdfplumber
    except ImportError as exc:
        raise _extra_missing(".pdf") from exc

    try:
        with pdfplumber.open(io.BytesIO(data)) as pdf:
            pages = [(page.extract_text() or "").rstrip() for page in pdf.pages]
    except ExtractionError:
        raise
    except Exception as exc:  # noqa: BLE001 - third-party parser, wrapped never leaked
        raise ExtractionError(
            f"the PDF parser failed on this file: {exc}", code="extractor_pdf_error"
        ) from exc

    text = "\n\n".join(page for page in pages if page)
    if not text:
        raise ExtractionError(
            "the PDF yielded no text on any page; a scanned or image-only "
            "document needs OCR, which this registry does not do",
            code="extractor_empty_pdf",
        )
    # After the parse, not before: a run that produced no text has nothing to
    # be lossy about, and warning there would just add noise to a failure.
    warnings.warn(_PDF_LOSSY_WARNING, ExtractionWarning, stacklevel=3)
    return text


def _convert_bytes(source: bytes, to: str, format: str, extra_args: Sequence[str]) -> str:
    """The one converter call, isolated so the seam above it is testable.

    Separated for a reason beyond tidiness: every test of the seam's behaviour
    would otherwise need the binary present and a real office document, which
    would make the seam's own logic untestable on a machine without the extra.
    The conversion itself is covered by the frozen-text fixtures instead.

    THE INPUT GOES THROUGH A FILE, NOT THROUGH THE TEXT ENTRY POINT. Every
    format here is a binary container, and the converter's text entry point
    takes an `encoding` because it treats its source as text -- which corrupts
    a zip. Measured: a hand-laid `.xlsx` that pandoc reads correctly from disk
    fails through the text path with `Failed to unpack XLSX archive: not enough
    bytes`. A `.docx` of the same shape happened to survive, which is what
    makes this worth writing down: the defect is SILENT for some inputs and
    fatal for others, so "it worked on the file I tried" is not evidence here.

    The temporary directory is removed on every path, including the failure
    one, and nothing outside it is written.
    """
    import pypandoc

    from ._pandoc import converter_path

    with tempfile.TemporaryDirectory() as staging:
        staged = Path(staging) / f"input.{format}"
        staged.write_bytes(source)
        with converter_path():
            return str(
                pypandoc.convert_file(str(staged), to, format=format, extra_args=list(extra_args))
            )


def _extract_office(suffix: str, data: bytes) -> str:
    """The five office rows, converted through the vendored binary.

    Shaped after `_extract_pdf`: the gate is an import probe rather than a
    membership test, third-party failures are wrapped rather than leaked, empty
    output is refused rather than persisted, and the lossiness is stated after
    the parse rather than before it.
    """
    try:
        import pypandoc  # noqa: F401
    except ImportError as exc:
        raise _extra_missing(suffix) from exc

    try:
        text = _convert_bytes(data, _PANDOC_WRITER, _PANDOC_FORMATS[suffix], _PANDOC_ARGS)
    except ExtractionError:
        raise
    except Exception as exc:  # noqa: BLE001 - third-party converter, wrapped never leaked
        raise ExtractionError(
            f"the converter failed on this {suffix} file: {exc}",
            code="extractor_convert_error",
        ) from exc

    text = text.strip()
    if not text:
        raise ExtractionError(
            f"the converter returned no text for this {suffix} file; refused "
            "rather than persisted as an empty concept",
            code="extractor_empty_conversion",
        )

    # After the parse, not before: a run that produced no text has nothing to
    # be lossy about, and warning there would just add noise to a failure.
    warnings.warn(_OFFICE_LOSSY_WARNING, ExtractionWarning, stacklevel=3)
    return text


_CORE_EXTRACTORS: dict[str, Callable[[bytes], str]] = {
    ".md": _extract_passthrough,
    ".txt": _extract_passthrough,
    ".csv": _extract_csv,
    ".json": _extract_json,
    ".html": _extract_html,
    ".htm": _extract_html,
}

# Types the `[extract]` extra ships a parser for. Kept separate from the core
# registry so "adds no runtime dependency" stays readable at a glance.
_OPTIONAL_EXTRACTORS: dict[str, Callable[[bytes], str]] = {
    ".pdf": _extract_pdf,
    **{suffix: functools.partial(_extract_office, suffix) for suffix in _PANDOC_FORMATS},
}


def extract_text(
    filename: str, data: bytes, *, renderer: Callable[[str], str] | None = None
) -> str:
    """Convert one dropped file's bytes to OKF concept text, dispatched by type.

    `filename` supplies the extension (case-insensitive); `data` is the raw
    bytes. A core stdlib type is extracted; a `[extract]`-gated binary type
    without the extra, and any unregistered extension, fail fast with a typed
    :class:`ExtractionError`. Extracting a `pdf` also emits an
    :class:`ExtractionWarning`: drawn content has no text to recover.

    `renderer`, when given, is applied to the EXTRACTED TEXT before it is
    returned -- after extraction, never instead of it, so a renderer never has
    to re-implement a reader and the two cannot drift. It is a plain callable
    rather than anything profile-shaped ON PURPOSE: this module is the
    extraction registry and must not import the contract layer, or the
    dependency would run backwards and the registry would stop standing on its
    own. Resolving a profile's NAMED renderer to a function is the caller's
    job, in the layer that already holds the profile.

    The default is identity, which is what keeps every existing byte-pinned
    golden byte-pinned.
    """
    suffix = Path(filename).suffix.lower()
    extractor = _CORE_EXTRACTORS.get(suffix) or _OPTIONAL_EXTRACTORS.get(suffix)
    if extractor is not None:
        text = extractor(data)
        return renderer(text) if renderer is not None else text
    if suffix in _UNPARSED_OPTIONAL_EXTENSIONS:
        raise _extra_missing(suffix)
    raise ExtractionError(
        f"no extractor is registered for file extension {suffix!r} ({filename!r})",
        code="extractor_unknown",
    )