test(extract): hand-built office fixtures with frozen extracted text
Three hand-laid OOXML containers, every part written out by hand and zipped
with a fixed date_time so they are byte-reproducible. No converter output
anywhere in them: a .docx written by the converter and read by the converter
proves only that the converter agrees with itself, and would stay green through
any conversion defect that is symmetric -- which is most of them.
two-line-krav.docx heading + label/value on one line (the docx mirror of
the PDF fixture)
no-styles-krav.docx the SAME document without word/styles.xml
two-line-krav.xlsx sheet name as heading + label/value on one row
THE FIXTURES FOUND A REAL DEFECT IN THE SEAM THEY WERE MEANT TO PIN. The
converter call used pypandoc's TEXT entry point, which takes an `encoding`
because it treats its source as text -- and that corrupts a zip. The xlsx
fixture failed with `Failed to unpack XLSX archive: not enough bytes` while
reading correctly from disk with the same binary. The docx of the same shape
happened to survive, which is the part worth writing down: the defect is silent
for some inputs and fatal for others, so "it worked on the file I tried" was
never evidence. Input now goes through a temporary file.
Two measurements while building, both the same shape -- structurally valid
input, silently reduced output, exit code 0, no warning:
- Without word/styles.xml the docx extracts as flat prose with no heading. A
fixture lacking that part would pin the body and pin nothing about structure.
Committed as a negative control that RUNS rather than a sentence in a README.
- With inline strings rather than a shared string table, the xlsx extracts with
the sheet name intact and every cell value gone. The fixture uses a dimension
element and a shared string table instead.
The frozen literals are pinned to a NAMED converter version, asserted beside
them: a frozen literal without one says "these bytes" without saying what
produced them.
Suite 908 -> 913. Fixtures regenerate byte-identically.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
parent
cd7b792aaf
commit
66a44f173b
7 changed files with 274 additions and 6 deletions
|
|
@ -310,6 +310,87 @@ def test_office_conversion_warns_that_it_is_lossy(
|
|||
# a parser upgrade break something visible instead of drifting silently.
|
||||
KRAV_TEXT = "Krav til helning på utkilingen\n60 og 70 1:15"
|
||||
|
||||
# The office fixtures are frozen the same way, and against a NAMED converter
|
||||
# version -- a frozen literal means nothing without one, because the thing it
|
||||
# pins is "this converter, on this input, produces these bytes". `_pandoc.py`
|
||||
# refuses any other version, so the two pins hold each other up.
|
||||
requires_pandoc = pytest.mark.skipif(
|
||||
importlib.util.find_spec("pypandoc") is None,
|
||||
reason="the optional [extract] extra is not installed",
|
||||
)
|
||||
|
||||
# docx: a heading and one requirement row with label and value on the SAME
|
||||
# line, mirroring the property the PDF fixture pins.
|
||||
DOCX_TEXT = "# Krav til helning\n\n60 og 70 1:15"
|
||||
|
||||
# xlsx: the sheet name becomes a heading and the rows become a table. The
|
||||
# label/value pairing survives on one row, which is the property that matters.
|
||||
XLSX_TEXT = (
|
||||
"## Krav {#sheet-1}\n\n Krav til helning \n"
|
||||
" ------------------ ------\n 60 og 70 1:15"
|
||||
)
|
||||
|
||||
# The negative control, committed rather than described: the SAME document
|
||||
# without `word/styles.xml`. The body survives and the heading marker does not.
|
||||
DOCX_NO_STYLES_TEXT = "Krav til helning\n\n60 og 70 1:15"
|
||||
|
||||
|
||||
@requires_pandoc
|
||||
def test_docx_extracts_to_its_frozen_text() -> None:
|
||||
data = (FIXTURES / "two-line-krav.docx").read_bytes()
|
||||
with pytest.warns(ExtractionWarning):
|
||||
assert extract_text("krav.docx", data) == DOCX_TEXT
|
||||
|
||||
|
||||
@requires_pandoc
|
||||
def test_xlsx_extracts_to_its_frozen_text() -> None:
|
||||
data = (FIXTURES / "two-line-krav.xlsx").read_bytes()
|
||||
with pytest.warns(ExtractionWarning):
|
||||
assert extract_text("krav.xlsx", data) == XLSX_TEXT
|
||||
|
||||
|
||||
@requires_pandoc
|
||||
def test_a_docx_without_a_styles_part_loses_its_heading() -> None:
|
||||
"""The negative control for the fixture policy, run rather than asserted.
|
||||
|
||||
`word/styles.xml` is what makes the converter see a heading. Without it the
|
||||
same document extracts as flat prose -- so a fixture built WITHOUT that
|
||||
part would pin the body and pin nothing at all about structure, while
|
||||
looking exactly as convincing.
|
||||
|
||||
Structure is the half the segment proposer reads, which is why this is a
|
||||
committed fixture and not a sentence in a README.
|
||||
"""
|
||||
data = (FIXTURES / "no-styles-krav.docx").read_bytes()
|
||||
with pytest.warns(ExtractionWarning):
|
||||
text = extract_text("krav.docx", data)
|
||||
assert text == DOCX_NO_STYLES_TEXT
|
||||
assert not text.startswith("#"), "the heading marker must be absent"
|
||||
assert DOCX_TEXT.startswith("#"), "and present in the fixture that has styles"
|
||||
|
||||
|
||||
@requires_pandoc
|
||||
def test_the_frozen_office_text_is_pinned_to_a_named_converter_version() -> None:
|
||||
"""A frozen literal without a named version pins nothing.
|
||||
|
||||
If the converter version ever moves, these literals must be re-measured
|
||||
rather than trusted -- so the version is asserted right where they live.
|
||||
"""
|
||||
from llm_ingestion_okf._pandoc import PANDOC_VERSION, resolve_pandoc
|
||||
|
||||
assert PANDOC_VERSION == "3.9"
|
||||
assert resolve_pandoc().is_file()
|
||||
|
||||
|
||||
@requires_pandoc
|
||||
def test_office_extraction_is_byte_stable_across_calls() -> None:
|
||||
data = (FIXTURES / "two-line-krav.docx").read_bytes()
|
||||
with pytest.warns(ExtractionWarning):
|
||||
first = extract_text("krav.docx", data)
|
||||
with pytest.warns(ExtractionWarning):
|
||||
second = extract_text("krav.docx", data)
|
||||
assert first == second
|
||||
|
||||
|
||||
@requires_extract
|
||||
def test_pdf_extracts_text_with_label_and_value_on_one_line() -> None:
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue