feat(extract): five office formats through a table-driven converter seam

`_PANDOC_FORMATS` names the rows and no others: docx, xlsx, pptx, odt, rtf.
`.html` stays on its stdlib extractor -- routing it through the converter would
buy nothing and would add CVE-2025-51591 (SSRF via an iframe in HTML input),
unpatched in every converter version. `.epub` is out on the "no gain" half of
that.

`_EVIDENCE` records what each row rests on, asserted in the suite rather than
written in a comment: docx and xlsx are `measured`, and pptx, odt and rtf are
`unmeasured` because the corpus contains ZERO files of those types. Three of
five rows therefore leave this step working by construction and never checked
against a document anyone wrote, and the assertion is what keeps that visible.

Three converter arguments, all measured and none of them hygiene:
`--eol=lf --wrap=none` because the defaults produce different bytes (max line
length 75 against 447), and `-t markdown` never `-t plain` because plain
destroys the headings the segment proposer reads -- 15 entries with two real
headings become 13 with none.

`_UNPARSED_OPTIONAL_EXTENSIONS` is now empty and kept rather than deleted: the
branch still raises, and a future type arriving before its reader belongs there
rather than in a new mechanism. This is what the first step was for -- both
tests for `extractor_extra_missing` were repointed at the import probe before
the set emptied under them.

The converter call is isolated behind `_convert_bytes` so the seam's own logic
is testable without the binary; the conversion itself is pinned by frozen-text
fixtures in the next step. Checked live against a hand-laid docx through the
real vendored binary: heading and body both survive.

Suite 895 -> 908.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Kjell Tore Guttormsen 2026-09-02 14:09:49 +02:00
commit cd7b792aaf
2 changed files with 293 additions and 10 deletions

View file

@ -133,6 +133,174 @@ def test_pdf_without_the_extra_installed_keeps_the_same_rejection(
assert "extract" in str(excinfo.value)
# --- office types: the table-driven converter seam ---
def test_every_office_row_names_its_reader() -> None:
"""The table is the contract; these are all the rows there are.
Written as an exact equality rather than a series of `in` checks: a row
added without a fixture and without an evidence class is the failure mode
this pins, and only equality catches an ADDITION.
"""
from llm_ingestion_okf.extract import _PANDOC_FORMATS
assert _PANDOC_FORMATS == {
".docx": "docx",
".xlsx": "xlsx",
".pptx": "pptx",
".odt": "odt",
".rtf": "rtf",
}
def test_html_and_epub_are_excluded_from_the_converter_on_purpose() -> None:
"""`.html` has a stdlib extractor; routing it through the converter would
buy nothing and would add CVE-2025-51591 (SSRF via an iframe in HTML
input), which is unpatched in every converter version. `.epub` is out for
the same "no gain" half of that reason.
"""
from llm_ingestion_okf.extract import _CORE_EXTRACTORS, _PANDOC_FORMATS
assert ".html" not in _PANDOC_FORMATS
assert ".epub" not in _PANDOC_FORMATS
assert ".html" in _CORE_EXTRACTORS
def test_evidence_class_is_asserted_not_commented() -> None:
"""Which rows were measured is a fact about this work, not a footnote.
`.pptx`, `.odt` and `.rtf` have denominator ZERO in the corpus this arm was
measured on. A comment saying so rots; an assertion that names them keeps
an unmeasured row from quietly presenting as a supported one.
"""
from llm_ingestion_okf.extract import _EVIDENCE, _PANDOC_FORMATS
assert set(_EVIDENCE) == set(_PANDOC_FORMATS), "every row needs an evidence class"
assert {s for s, e in _EVIDENCE.items() if e == "measured"} == {".docx", ".xlsx"}
assert {s for s, e in _EVIDENCE.items() if e == "unmeasured"} == {
".pptx",
".odt",
".rtf",
}
def test_the_unparsed_set_is_empty_now_that_every_row_has_a_reader() -> None:
"""The membership branch that used to raise `extractor_extra_missing` has
no members left. This is why both tests for that code were repointed at the
import probe before this step landed.
"""
from llm_ingestion_okf.extract import _UNPARSED_OPTIONAL_EXTENSIONS
assert _UNPARSED_OPTIONAL_EXTENSIONS == frozenset()
def test_the_converter_is_called_with_the_load_bearing_arguments(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""`--eol=lf --wrap=none` and the markdown writer, all three measured.
Not hygiene, and not style. The defaults produce DIFFERENT BYTES (maximum
line length 75 against 447), which a byte-pinned golden would register as a
change nobody made. And `-t plain` destroys the headings the segment
proposer reads: measured, a document yielding 15 entries including two real
headings yields 13 with none once the writer is `plain`.
"""
import llm_ingestion_okf.extract as extract_module
seen: dict[str, object] = {}
def fake_convert(source: object, to: str, format: str, extra_args: object) -> str:
seen.update(to=to, format=format, extra_args=list(extra_args)) # type: ignore[call-overload]
return "# Heading\n\nBody."
monkeypatch.setattr(extract_module, "_convert_bytes", fake_convert)
with pytest.warns(ExtractionWarning):
text = extract_text("note.docx", b"PK\x03\x04")
assert text == "# Heading\n\nBody."
assert seen["to"] == "markdown"
assert seen["format"] == "docx"
assert "--eol=lf" in seen["extra_args"] # type: ignore[operator]
assert "--wrap=none" in seen["extra_args"] # type: ignore[operator]
@pytest.mark.parametrize(
("name", "reader"),
[
("a.docx", "docx"),
("b.xlsx", "xlsx"),
("c.pptx", "pptx"),
("d.odt", "odt"),
("e.rtf", "rtf"),
],
)
def test_each_row_dispatches_to_its_reader(
name: str, reader: str, monkeypatch: pytest.MonkeyPatch
) -> None:
import llm_ingestion_okf.extract as extract_module
seen: dict[str, object] = {}
def fake_convert(source: object, to: str, format: str, extra_args: object) -> str:
seen["format"] = format
return "text"
monkeypatch.setattr(extract_module, "_convert_bytes", fake_convert)
with pytest.warns(ExtractionWarning):
extract_text(name, b"bytes")
assert seen["format"] == reader
def test_an_empty_conversion_is_refused_not_persisted(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""Same reason as `extractor_empty_pdf`: an empty concept is the silent
skip this registry exists to prevent."""
import llm_ingestion_okf.extract as extract_module
monkeypatch.setattr(
extract_module,
"_convert_bytes",
lambda source, to, format, extra_args: " \n\t ",
)
with pytest.raises(ExtractionError) as excinfo:
extract_text("empty.docx", b"bytes")
assert excinfo.value.code == "extractor_empty_conversion"
def test_a_converter_failure_is_wrapped_never_leaked(
monkeypatch: pytest.MonkeyPatch,
) -> None:
import llm_ingestion_okf.extract as extract_module
def boom(source: object, to: str, format: str, extra_args: object) -> str:
raise RuntimeError("pandoc said no")
monkeypatch.setattr(extract_module, "_convert_bytes", boom)
with pytest.raises(ExtractionError) as excinfo:
extract_text("bad.docx", b"bytes")
assert excinfo.value.code == "extractor_convert_error"
assert "pandoc said no" in str(excinfo.value)
def test_office_conversion_warns_that_it_is_lossy(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""Drawn content does not survive extraction here either, and the warning
is emitted after the parse -- a run that produced no text has nothing to be
lossy about."""
import llm_ingestion_okf.extract as extract_module
monkeypatch.setattr(
extract_module,
"_convert_bytes",
lambda source, to, format, extra_args: "body",
)
with pytest.warns(ExtractionWarning):
extract_text("note.docx", b"bytes")
# --- pdf: the [extract] parser (pdfplumber) ---
# The expected text is frozen against a committed fixture ON PURPOSE. Extraction