feat(extract): five office formats through a table-driven converter seam
`_PANDOC_FORMATS` names the rows and no others: docx, xlsx, pptx, odt, rtf. `.html` stays on its stdlib extractor -- routing it through the converter would buy nothing and would add CVE-2025-51591 (SSRF via an iframe in HTML input), unpatched in every converter version. `.epub` is out on the "no gain" half of that. `_EVIDENCE` records what each row rests on, asserted in the suite rather than written in a comment: docx and xlsx are `measured`, and pptx, odt and rtf are `unmeasured` because the corpus contains ZERO files of those types. Three of five rows therefore leave this step working by construction and never checked against a document anyone wrote, and the assertion is what keeps that visible. Three converter arguments, all measured and none of them hygiene: `--eol=lf --wrap=none` because the defaults produce different bytes (max line length 75 against 447), and `-t markdown` never `-t plain` because plain destroys the headings the segment proposer reads -- 15 entries with two real headings become 13 with none. `_UNPARSED_OPTIONAL_EXTENSIONS` is now empty and kept rather than deleted: the branch still raises, and a future type arriving before its reader belongs there rather than in a new mechanism. This is what the first step was for -- both tests for `extractor_extra_missing` were repointed at the import probe before the set emptied under them. The converter call is isolated behind `_convert_bytes` so the seam's own logic is testable without the binary; the conversion itself is pinned by frozen-text fixtures in the next step. Checked live against a hand-laid docx through the real vendored binary: heading and body both survive. Suite 895 -> 908. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
parent
b9372ad7e8
commit
cd7b792aaf
2 changed files with 293 additions and 10 deletions
|
|
@ -133,6 +133,174 @@ def test_pdf_without_the_extra_installed_keeps_the_same_rejection(
|
|||
assert "extract" in str(excinfo.value)
|
||||
|
||||
|
||||
# --- office types: the table-driven converter seam ---
|
||||
|
||||
|
||||
def test_every_office_row_names_its_reader() -> None:
|
||||
"""The table is the contract; these are all the rows there are.
|
||||
|
||||
Written as an exact equality rather than a series of `in` checks: a row
|
||||
added without a fixture and without an evidence class is the failure mode
|
||||
this pins, and only equality catches an ADDITION.
|
||||
"""
|
||||
from llm_ingestion_okf.extract import _PANDOC_FORMATS
|
||||
|
||||
assert _PANDOC_FORMATS == {
|
||||
".docx": "docx",
|
||||
".xlsx": "xlsx",
|
||||
".pptx": "pptx",
|
||||
".odt": "odt",
|
||||
".rtf": "rtf",
|
||||
}
|
||||
|
||||
|
||||
def test_html_and_epub_are_excluded_from_the_converter_on_purpose() -> None:
|
||||
"""`.html` has a stdlib extractor; routing it through the converter would
|
||||
buy nothing and would add CVE-2025-51591 (SSRF via an iframe in HTML
|
||||
input), which is unpatched in every converter version. `.epub` is out for
|
||||
the same "no gain" half of that reason.
|
||||
"""
|
||||
from llm_ingestion_okf.extract import _CORE_EXTRACTORS, _PANDOC_FORMATS
|
||||
|
||||
assert ".html" not in _PANDOC_FORMATS
|
||||
assert ".epub" not in _PANDOC_FORMATS
|
||||
assert ".html" in _CORE_EXTRACTORS
|
||||
|
||||
|
||||
def test_evidence_class_is_asserted_not_commented() -> None:
|
||||
"""Which rows were measured is a fact about this work, not a footnote.
|
||||
|
||||
`.pptx`, `.odt` and `.rtf` have denominator ZERO in the corpus this arm was
|
||||
measured on. A comment saying so rots; an assertion that names them keeps
|
||||
an unmeasured row from quietly presenting as a supported one.
|
||||
"""
|
||||
from llm_ingestion_okf.extract import _EVIDENCE, _PANDOC_FORMATS
|
||||
|
||||
assert set(_EVIDENCE) == set(_PANDOC_FORMATS), "every row needs an evidence class"
|
||||
assert {s for s, e in _EVIDENCE.items() if e == "measured"} == {".docx", ".xlsx"}
|
||||
assert {s for s, e in _EVIDENCE.items() if e == "unmeasured"} == {
|
||||
".pptx",
|
||||
".odt",
|
||||
".rtf",
|
||||
}
|
||||
|
||||
|
||||
def test_the_unparsed_set_is_empty_now_that_every_row_has_a_reader() -> None:
|
||||
"""The membership branch that used to raise `extractor_extra_missing` has
|
||||
no members left. This is why both tests for that code were repointed at the
|
||||
import probe before this step landed.
|
||||
"""
|
||||
from llm_ingestion_okf.extract import _UNPARSED_OPTIONAL_EXTENSIONS
|
||||
|
||||
assert _UNPARSED_OPTIONAL_EXTENSIONS == frozenset()
|
||||
|
||||
|
||||
def test_the_converter_is_called_with_the_load_bearing_arguments(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""`--eol=lf --wrap=none` and the markdown writer, all three measured.
|
||||
|
||||
Not hygiene, and not style. The defaults produce DIFFERENT BYTES (maximum
|
||||
line length 75 against 447), which a byte-pinned golden would register as a
|
||||
change nobody made. And `-t plain` destroys the headings the segment
|
||||
proposer reads: measured, a document yielding 15 entries including two real
|
||||
headings yields 13 with none once the writer is `plain`.
|
||||
"""
|
||||
import llm_ingestion_okf.extract as extract_module
|
||||
|
||||
seen: dict[str, object] = {}
|
||||
|
||||
def fake_convert(source: object, to: str, format: str, extra_args: object) -> str:
|
||||
seen.update(to=to, format=format, extra_args=list(extra_args)) # type: ignore[call-overload]
|
||||
return "# Heading\n\nBody."
|
||||
|
||||
monkeypatch.setattr(extract_module, "_convert_bytes", fake_convert)
|
||||
with pytest.warns(ExtractionWarning):
|
||||
text = extract_text("note.docx", b"PK\x03\x04")
|
||||
|
||||
assert text == "# Heading\n\nBody."
|
||||
assert seen["to"] == "markdown"
|
||||
assert seen["format"] == "docx"
|
||||
assert "--eol=lf" in seen["extra_args"] # type: ignore[operator]
|
||||
assert "--wrap=none" in seen["extra_args"] # type: ignore[operator]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("name", "reader"),
|
||||
[
|
||||
("a.docx", "docx"),
|
||||
("b.xlsx", "xlsx"),
|
||||
("c.pptx", "pptx"),
|
||||
("d.odt", "odt"),
|
||||
("e.rtf", "rtf"),
|
||||
],
|
||||
)
|
||||
def test_each_row_dispatches_to_its_reader(
|
||||
name: str, reader: str, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
import llm_ingestion_okf.extract as extract_module
|
||||
|
||||
seen: dict[str, object] = {}
|
||||
|
||||
def fake_convert(source: object, to: str, format: str, extra_args: object) -> str:
|
||||
seen["format"] = format
|
||||
return "text"
|
||||
|
||||
monkeypatch.setattr(extract_module, "_convert_bytes", fake_convert)
|
||||
with pytest.warns(ExtractionWarning):
|
||||
extract_text(name, b"bytes")
|
||||
assert seen["format"] == reader
|
||||
|
||||
|
||||
def test_an_empty_conversion_is_refused_not_persisted(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""Same reason as `extractor_empty_pdf`: an empty concept is the silent
|
||||
skip this registry exists to prevent."""
|
||||
import llm_ingestion_okf.extract as extract_module
|
||||
|
||||
monkeypatch.setattr(
|
||||
extract_module,
|
||||
"_convert_bytes",
|
||||
lambda source, to, format, extra_args: " \n\t ",
|
||||
)
|
||||
with pytest.raises(ExtractionError) as excinfo:
|
||||
extract_text("empty.docx", b"bytes")
|
||||
assert excinfo.value.code == "extractor_empty_conversion"
|
||||
|
||||
|
||||
def test_a_converter_failure_is_wrapped_never_leaked(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
import llm_ingestion_okf.extract as extract_module
|
||||
|
||||
def boom(source: object, to: str, format: str, extra_args: object) -> str:
|
||||
raise RuntimeError("pandoc said no")
|
||||
|
||||
monkeypatch.setattr(extract_module, "_convert_bytes", boom)
|
||||
with pytest.raises(ExtractionError) as excinfo:
|
||||
extract_text("bad.docx", b"bytes")
|
||||
assert excinfo.value.code == "extractor_convert_error"
|
||||
assert "pandoc said no" in str(excinfo.value)
|
||||
|
||||
|
||||
def test_office_conversion_warns_that_it_is_lossy(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""Drawn content does not survive extraction here either, and the warning
|
||||
is emitted after the parse -- a run that produced no text has nothing to be
|
||||
lossy about."""
|
||||
import llm_ingestion_okf.extract as extract_module
|
||||
|
||||
monkeypatch.setattr(
|
||||
extract_module,
|
||||
"_convert_bytes",
|
||||
lambda source, to, format, extra_args: "body",
|
||||
)
|
||||
with pytest.warns(ExtractionWarning):
|
||||
extract_text("note.docx", b"bytes")
|
||||
|
||||
|
||||
# --- pdf: the [extract] parser (pdfplumber) ---
|
||||
|
||||
# The expected text is frozen against a committed fixture ON PURPOSE. Extraction
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue