feat(extract): Door B extraction registry (Phase 2 step 1)

First, guard-independent step of Phase 2: a stdlib-only registry mapping a
dropped file's extension to its text extractor, with the fail-fast gates that
keep binary parsing out of core. `extract_text(filename, data)` dispatches
(case-insensitively) to:

- `md`/`txt` — utf-8-sig passthrough (BOM never leaks, baseline parity with
  Door A's read_csv);
- `csv` — the Phase 1 `render_table` (renderer reused, not duplicated);
- `json` — verbatim inside `render_fenced_block`;
- `html`/`htm` — text via `html.parser`, `script`/`style` stripped, tags as
  word boundaries (spec B3: adequate for v1, richer is out of scope).

`pdf`/`docx`/`xlsx` are `[extract]`-gated; until that extra ships a parser they
fail fast with a typed error naming the extra — never a silent skip, never a
bundled parser in core. New `ExtractionError(IngestError)` carries four stable
codes (`extractor_unknown`, `extractor_extra_missing`, `extractor_decode_error`,
`extractor_empty_csv`); a non-UTF-8 file is a typed corrupt-input failure, never
a leaked UnicodeDecodeError. `extract_text` returns text content only — LF
framing and concept frontmatter are the materializer's job (step 2).

No runtime dependency and no guard call yet (the guard pin and 0.4.0 land with
the persist gate in steps 4–5). TDD: test_extract.py + the four codes in the
test_error_codes.py registry precede the implementation; mypy --strict, ruff,
and the `sanitize|quarantine|lexicon` boundary grep-gate all clean; the Phase 1
golden suite still passes byte-for-byte.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01HBbjgS5A55RVavoyjJC4FX

src/llm_ingestion_okf/__init__.py

@@ -18,6 +18,7 @@ typed error hierarchy rooted in IngestError.
 """
 
 from .errors import (
+    ExtractionError,
     IngestError,
     ManifestError,
     MaterializationError,
@@ -25,6 +26,7 @@ from .errors import (
     RenderError,
     SourceError,
 )
+from .extract import extract_text
 from .manifest import (
     Extraction,
     FileSource,
@@ -39,6 +41,7 @@ __version__ = "0.3.2"
 
 __all__ = [
     "Extraction",
+    "ExtractionError",
     "FileSource",
     "HttpSource",
     "IngestError",
@@ -50,6 +53,7 @@ __all__ = [
     "RenderError",
     "SourceError",
     "SqlSource",
+    "extract_text",
     "load_manifest",
     "materialize_bundle",
 ]

src/llm_ingestion_okf/errors.py

@@ -72,6 +72,23 @@ class SourceError(IngestError):
     """
 
 
+class ExtractionError(IngestError):
+    """A dropped file could not be converted to text (Door B, Phase 2).
+
+    File-type -> text extraction is text-only plumbing; a corrupt file, an
+    unknown type, or a binary type without its optional parser is a typed
+    per-file failure — never a silent skip, never a leaked stdlib error, and
+    never a bundled parser in core.
+
+    Codes:
+    - `extractor_unknown` — no extractor is registered for the file extension
+    - `extractor_extra_missing` — a `[extract]`-gated binary type (pdf/docx/
+      xlsx) was given but the optional extra is not installed
+    - `extractor_decode_error` — a text-type file's bytes are not valid UTF-8
+    - `extractor_empty_csv` — a CSV has no header row
+    """
+
+
 class MaterializationError(IngestError):
     """Materialization refused or failed (ingest-spec §5).
 

src/llm_ingestion_okf/extract.py

@@ -0,0 +1,144 @@
+"""Door B extraction registry: dropped file bytes -> text, per file type.
+
+All file-type -> text extraction lives here (the guard is text-only). The core
+registry is stdlib-only and deterministic: `md`/`txt` pass through, `csv` renders
+the Phase 1 markdown table, `json` is fenced verbatim, and `html`/`htm` are
+reduced to text with `html.parser`. Binary types (`pdf`/`docx`/`xlsx`) are
+`[extract]`-gated and — until that optional extra ships a parser — fail fast with
+a typed error naming the extra, never a silent skip and never a bundled parser in
+core.
+
+`extract_text` returns the extracted text *content*; final LF framing and the
+concept frontmatter are the materializer's concern (Phase 2 step 2), not this
+registry's. No guard call and no model call anywhere in this module.
+"""
+
+from __future__ import annotations
+
+import csv
+import io
+from collections.abc import Callable
+from html.parser import HTMLParser
+from pathlib import Path
+
+from .errors import ExtractionError
+from .render import render_fenced_block, render_table
+
+# Binary types gated behind the optional `[extract]` extra. The extra ships no
+# parser yet, so these always fail fast for now; when a parser lands the gate
+# becomes an import probe, but the rejection code and message stay the same.
+_OPTIONAL_EXTENSIONS = frozenset({".pdf", ".docx", ".xlsx"})
+
+# Tags whose text content is never document prose.
+_SKIP_TAGS = frozenset({"script", "style"})
+
+
+def _decode(data: bytes) -> str:
+    """Decode file bytes as UTF-8 (BOM-stripping), typed on failure.
+
+    utf-8-sig so a byte-order mark never leaks into the first character
+    (baseline parity with Door A's read_csv). A non-UTF-8 file is a corrupt
+    input: fail fast with a typed error rather than leaking UnicodeDecodeError.
+    """
+    try:
+        return data.decode("utf-8-sig")
+    except UnicodeDecodeError as exc:
+        raise ExtractionError(
+            f"file bytes are not valid UTF-8: {exc}", code="extractor_decode_error"
+        ) from exc
+
+
+def _extract_passthrough(data: bytes) -> str:
+    """`md`/`txt`: the decoded text verbatim."""
+    return _decode(data)
+
+
+def _extract_csv(data: bytes) -> str:
+    """`csv`: parse with the stdlib reader, render the Phase 1 markdown table."""
+    reader = csv.reader(io.StringIO(_decode(data)))
+    header = next(reader, None)
+    if header is None:
+        raise ExtractionError("CSV has no header row", code="extractor_empty_csv")
+    rows = list(reader)
+    return render_table(header, rows)
+
+
+def _extract_json(data: bytes) -> str:
+    """`json`: the decoded text verbatim inside a fenced block (Phase 1 renderer)."""
+    return render_fenced_block(_decode(data))
+
+
+class _HTMLTextExtractor(HTMLParser):
+    """Collect document text, skipping `script`/`style`, tags as word boundaries.
+
+    Tags contribute no text of their own but do separate words: a boundary
+    space is emitted at every tag so adjacent block text (``</h1><p>``) does not
+    fuse. Runs of whitespace collapse to single spaces in :meth:`text`.
+    """
+
+    def __init__(self) -> None:
+        super().__init__(convert_charrefs=True)
+        self._parts: list[str] = []
+        self._skip_depth = 0
+
+    def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
+        self._parts.append(" ")
+        if tag in _SKIP_TAGS:
+            self._skip_depth += 1
+
+    def handle_startendtag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
+        self._parts.append(" ")
+
+    def handle_endtag(self, tag: str) -> None:
+        if tag in _SKIP_TAGS and self._skip_depth > 0:
+            self._skip_depth -= 1
+        self._parts.append(" ")
+
+    def handle_data(self, data: str) -> None:
+        if self._skip_depth == 0:
+            self._parts.append(data)
+
+    def text(self) -> str:
+        return " ".join("".join(self._parts).split())
+
+
+def _extract_html(data: bytes) -> str:
+    """`html`/`htm`: text via `html.parser`, script/style stripped (spec B3)."""
+    parser = _HTMLTextExtractor()
+    parser.feed(_decode(data))
+    parser.close()
+    return parser.text()
+
+
+_CORE_EXTRACTORS: dict[str, Callable[[bytes], str]] = {
+    ".md": _extract_passthrough,
+    ".txt": _extract_passthrough,
+    ".csv": _extract_csv,
+    ".json": _extract_json,
+    ".html": _extract_html,
+    ".htm": _extract_html,
+}
+
+
+def extract_text(filename: str, data: bytes) -> str:
+    """Convert one dropped file's bytes to OKF concept text, dispatched by type.
+
+    `filename` supplies the extension (case-insensitive); `data` is the raw
+    bytes. A core stdlib type is extracted; a `[extract]`-gated binary type
+    without the extra, and any unregistered extension, fail fast with a typed
+    :class:`ExtractionError`.
+    """
+    suffix = Path(filename).suffix.lower()
+    extractor = _CORE_EXTRACTORS.get(suffix)
+    if extractor is not None:
+        return extractor(data)
+    if suffix in _OPTIONAL_EXTENSIONS:
+        raise ExtractionError(
+            f"extracting {suffix!r} requires the optional 'extract' extra "
+            f"(pip install 'llm-ingestion-okf[extract]'); it is not installed",
+            code="extractor_extra_missing",
+        )
+    raise ExtractionError(
+        f"no extractor is registered for file extension {suffix!r} ({filename!r})",
+        code="extractor_unknown",
+    )

tests/test_error_codes.py

@@ -19,6 +19,7 @@ import pytest
 import llm_ingestion_okf.connectors as connectors
 from llm_ingestion_okf.connectors import read_csv, read_http, read_sql, safe_resolve
 from llm_ingestion_okf.errors import (
+    ExtractionError,
     IngestError,
     ManifestError,
     MaterializationError,
@@ -26,6 +27,7 @@ from llm_ingestion_okf.errors import (
     RenderError,
     SourceError,
 )
+from llm_ingestion_okf.extract import extract_text
 from llm_ingestion_okf.manifest import load_manifest, load_manifest_bytes
 from llm_ingestion_okf.materialize import materialize_bundle
 from llm_ingestion_okf.render import sql_value_to_text
@@ -291,6 +293,33 @@ def test_unsupported_cell_type(value: object) -> None:
     assert code_of(excinfo) == "unsupported_cell_type"
 
 
+# --- ExtractionError codes ---
+
+
+def test_extractor_unknown() -> None:
+    with pytest.raises(ExtractionError) as excinfo:
+        extract_text("archive.zip", b"")
+    assert code_of(excinfo) == "extractor_unknown"
+
+
+def test_extractor_extra_missing() -> None:
+    with pytest.raises(ExtractionError) as excinfo:
+        extract_text("doc.pdf", b"binary")
+    assert code_of(excinfo) == "extractor_extra_missing"
+
+
+def test_extractor_decode_error() -> None:
+    with pytest.raises(ExtractionError) as excinfo:
+        extract_text("note.txt", b"\xffbad")
+    assert code_of(excinfo) == "extractor_decode_error"
+
+
+def test_extractor_empty_csv() -> None:
+    with pytest.raises(ExtractionError) as excinfo:
+        extract_text("empty.csv", b"")
+    assert code_of(excinfo) == "extractor_empty_csv"
+
+
 # --- MaterializationError codes ---
 
 

tests/test_extract.py

@@ -0,0 +1,107 @@
+"""Door B extraction registry: file bytes -> text, per file type (Phase 2 step 1).
+
+Core stdlib extractors (md/txt/csv/json/html) plus the fail-fast gates: unknown
+extension, the [extract]-gated binary types when the extra is absent, corrupt
+(non-UTF-8) bytes, and an empty CSV. All file-type->text extraction lives HERE
+(the guard is text-only); this step adds zero runtime dependency and no guard
+call — it is pure, deterministic plumbing.
+"""
+
+from __future__ import annotations
+
+import pytest
+
+from llm_ingestion_okf import ExtractionError, extract_text
+
+# --- core extractors: happy path (a fixture per type) ---
+
+
+def test_md_passthrough() -> None:
+    assert extract_text("note.md", b"# Title\n\nBody\n") == "# Title\n\nBody\n"
+
+
+def test_txt_passthrough() -> None:
+    assert extract_text("note.txt", b"plain text") == "plain text"
+
+
+def test_utf8_sig_bom_is_stripped() -> None:
+    # utf-8-sig so a BOM never leaks into the first character (baseline parity
+    # with Door A's read_csv).
+    assert extract_text("note.txt", b"\xef\xbb\xbfhello") == "hello"
+
+
+def test_csv_renders_the_phase_1_markdown_table() -> None:
+    out = extract_text("data.csv", b"name,age\nAda,36\n")
+    assert out == "| name | age |\n| --- | --- |\n| Ada | 36 |\n"
+
+
+def test_json_is_verbatim_inside_a_fenced_block() -> None:
+    out = extract_text("cfg.json", b'{"k": 1}\n')
+    assert out == '```\n{"k": 1}\n```\n'
+
+
+def test_html_text_via_htmlparser() -> None:
+    # Block boundaries separate words; tags themselves contribute no text.
+    out = extract_text("page.html", b"<h1>Title</h1><p>Hello <b>world</b></p>")
+    assert out == "Title Hello world"
+
+
+def test_html_skips_script_and_style() -> None:
+    html = b"<style>.x{color:red}</style><p>Keep</p><script>evil()</script>"
+    assert extract_text("page.html", html) == "Keep"
+
+
+def test_htm_is_an_html_alias() -> None:
+    assert extract_text("page.htm", b"<p>hi</p>") == "hi"
+
+
+def test_extension_dispatch_is_case_insensitive() -> None:
+    assert extract_text("NOTE.MD", b"x") == "x"
+
+
+# --- fail-fast: unknown extension ---
+
+
+def test_unknown_extension_fails_fast() -> None:
+    with pytest.raises(ExtractionError) as excinfo:
+        extract_text("archive.zip", b"PK\x03\x04")
+    assert excinfo.value.code == "extractor_unknown"
+
+
+def test_missing_extension_fails_fast() -> None:
+    with pytest.raises(ExtractionError) as excinfo:
+        extract_text("README", b"x")
+    assert excinfo.value.code == "extractor_unknown"
+
+
+# --- fail-fast: [extract]-gated binary types without the extra ---
+
+
+@pytest.mark.parametrize("name", ["doc.pdf", "doc.docx", "sheet.xlsx"])
+def test_optional_type_without_extra_fails_fast(name: str) -> None:
+    with pytest.raises(ExtractionError) as excinfo:
+        extract_text(name, b"binary")
+    assert excinfo.value.code == "extractor_extra_missing"
+    # The error names the extra so the operator knows the remedy — never a
+    # silent skip, never a bundled parser in core.
+    assert "extract" in str(excinfo.value)
+
+
+# --- fail-fast: corrupt (non-UTF-8) bytes on a text type ---
+
+
+def test_invalid_utf8_fails_fast_typed() -> None:
+    # Never a leaked UnicodeDecodeError — the "always typed" doctrine holds for
+    # Door B too (cf. Door A wrapping path ValueError as SourceError).
+    with pytest.raises(ExtractionError) as excinfo:
+        extract_text("note.txt", b"\xffbad")
+    assert excinfo.value.code == "extractor_decode_error"
+
+
+# --- fail-fast: empty CSV (no header row) ---
+
+
+def test_empty_csv_fails_fast() -> None:
+    with pytest.raises(ExtractionError) as excinfo:
+        extract_text("empty.csv", b"")
+    assert excinfo.value.code == "extractor_empty_csv"