"""Door B extraction registry: dropped file bytes -> text, per file type. All file-type -> text extraction lives here (the guard is text-only). The core registry is stdlib-only and deterministic: `md`/`txt` pass through, `csv` renders the Phase 1 markdown table, `json` is fenced verbatim, and `html`/`htm` are reduced to text with `html.parser`. Binary types (`pdf`/`docx`/`xlsx`) are `[extract]`-gated: `pdf` is extracted with `pdfplumber` when the extra is installed and rejected with the same typed error when it is not (the gate is an import probe, not a membership test), while `docx`/`xlsx` ship no parser yet and always fail fast. Never a silent skip and never a bundled parser in core. `extract_text` returns the extracted text *content*; final LF framing and the concept frontmatter are the materializer's concern (Phase 2 step 2), not this registry's. No guard call and no model call anywhere in this module. """ from __future__ import annotations import csv import io import warnings from collections.abc import Callable from html.parser import HTMLParser from pathlib import Path from .errors import ExtractionError, ExtractionWarning from .render import render_fenced_block, render_table # Binary types gated behind the optional `[extract]` extra that it ships no # parser for. `.pdf` is no longer here: its gate is the import probe inside # `_extract_pdf`, which raises the same code and the same message when the # extra is absent. These two still fail fast unconditionally. _UNPARSED_OPTIONAL_EXTENSIONS = frozenset({".docx", ".xlsx"}) # Text extraction recovers text. A figure is a vector drawing with no text to # recover — only its caption survives — so any bundle built from drawn # documents is incomplete by construction. Said out loud on every PDF rather # than detected per document: deciding "is there a figure here" is a layout # heuristic this library does not own. _PDF_LOSSY_WARNING = ( "PDF extraction recovers text only: figures, diagrams and images are not " "represented in the output (their captions are). A bundle built from " "drawn documents is incomplete by construction." ) # Tags whose text content is never document prose. _SKIP_TAGS = frozenset({"script", "style"}) def decode_text(data: bytes) -> str: """Decode file bytes as UTF-8 (BOM-stripping), typed on failure. utf-8-sig so a byte-order mark never leaks into the first character (baseline parity with Door A's read_csv). A non-UTF-8 file is a corrupt input: fail fast with a typed error rather than leaking UnicodeDecodeError. """ try: return data.decode("utf-8-sig") except UnicodeDecodeError as exc: raise ExtractionError( f"file bytes are not valid UTF-8: {exc}", code="extractor_decode_error" ) from exc def _extract_passthrough(data: bytes) -> str: """`md`/`txt`: the decoded text verbatim.""" return decode_text(data) def _extract_csv(data: bytes) -> str: """`csv`: parse with the stdlib reader, render the Phase 1 markdown table.""" reader = csv.reader(io.StringIO(decode_text(data))) header = next(reader, None) if header is None: raise ExtractionError("CSV has no header row", code="extractor_empty_csv") rows = list(reader) return render_table(header, rows) def _extract_json(data: bytes) -> str: """`json`: the decoded text verbatim inside a fenced block (Phase 1 renderer).""" return render_fenced_block(decode_text(data)) class _HTMLTextExtractor(HTMLParser): """Collect document text, skipping `script`/`style`, tags as word boundaries. Tags contribute no text of their own but do separate words: a boundary space is emitted at every tag so adjacent block text (``

``) does not fuse. Runs of whitespace collapse to single spaces in :meth:`text`. """ def __init__(self) -> None: super().__init__(convert_charrefs=True) self._parts: list[str] = [] self._skip_depth = 0 def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: self._parts.append(" ") if tag in _SKIP_TAGS: self._skip_depth += 1 def handle_startendtag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: self._parts.append(" ") def handle_endtag(self, tag: str) -> None: if tag in _SKIP_TAGS and self._skip_depth > 0: self._skip_depth -= 1 self._parts.append(" ") def handle_data(self, data: str) -> None: if self._skip_depth == 0: self._parts.append(data) def text(self) -> str: return " ".join("".join(self._parts).split()) def _extract_html(data: bytes) -> str: """`html`/`htm`: text via `html.parser`, script/style stripped (spec B3).""" parser = _HTMLTextExtractor() parser.feed(decode_text(data)) parser.close() return parser.text() def _extra_missing(suffix: str) -> ExtractionError: """The one rejection for a `[extract]` type without the extra installed. One constructor, one wording: the import probe and the still-unparsed types must be indistinguishable to a consumer, because they are the same fact — the extra is not installed. """ return ExtractionError( f"extracting {suffix!r} requires the optional 'extract' extra " f"(pip install 'llm-ingestion-okf[extract]'); it is not installed", code="extractor_extra_missing", ) def _extract_pdf(data: bytes) -> str: """`pdf`: page text via `pdfplumber`, in page order, pages separated by a blank line. The gate is this import, not a membership test: without the `[extract]` extra the very same typed rejection is raised as for the types that ship no parser at all. Text is returned VERBATIM — no Unicode normalization, matching `md`/`txt` passthrough; normalizing would edit source content, and NFC folding belongs to filenames and titles, not to document bodies. `pdfplumber` was chosen on ONE measured property (2026-08-21, docs/2026-08-21-g2-pdf-extraction-measurement.md): on a real requirement table it keeps label and value on the same line, where pypdf, pdfminer.six and pymupdf each emit all labels then all values. Re-pairing those is guesswork, and in a requirements document a wrong pairing looks right. """ try: import pdfplumber except ImportError as exc: raise _extra_missing(".pdf") from exc try: with pdfplumber.open(io.BytesIO(data)) as pdf: pages = [(page.extract_text() or "").rstrip() for page in pdf.pages] except ExtractionError: raise except Exception as exc: # noqa: BLE001 - third-party parser, wrapped never leaked raise ExtractionError( f"the PDF parser failed on this file: {exc}", code="extractor_pdf_error" ) from exc text = "\n\n".join(page for page in pages if page) if not text: raise ExtractionError( "the PDF yielded no text on any page; a scanned or image-only " "document needs OCR, which this registry does not do", code="extractor_empty_pdf", ) # After the parse, not before: a run that produced no text has nothing to # be lossy about, and warning there would just add noise to a failure. warnings.warn(_PDF_LOSSY_WARNING, ExtractionWarning, stacklevel=3) return text _CORE_EXTRACTORS: dict[str, Callable[[bytes], str]] = { ".md": _extract_passthrough, ".txt": _extract_passthrough, ".csv": _extract_csv, ".json": _extract_json, ".html": _extract_html, ".htm": _extract_html, } # Types the `[extract]` extra ships a parser for. Kept separate from the core # registry so "adds no runtime dependency" stays readable at a glance. _OPTIONAL_EXTRACTORS: dict[str, Callable[[bytes], str]] = { ".pdf": _extract_pdf, } def extract_text(filename: str, data: bytes) -> str: """Convert one dropped file's bytes to OKF concept text, dispatched by type. `filename` supplies the extension (case-insensitive); `data` is the raw bytes. A core stdlib type is extracted; a `[extract]`-gated binary type without the extra, and any unregistered extension, fail fast with a typed :class:`ExtractionError`. Extracting a `pdf` also emits an :class:`ExtractionWarning`: drawn content has no text to recover. """ suffix = Path(filename).suffix.lower() extractor = _CORE_EXTRACTORS.get(suffix) or _OPTIONAL_EXTRACTORS.get(suffix) if extractor is not None: return extractor(data) if suffix in _UNPARSED_OPTIONAL_EXTENSIONS: raise _extra_missing(suffix) raise ExtractionError( f"no extractor is registered for file extension {suffix!r} ({filename!r})", code="extractor_unknown", )