Kjell Tore Guttormsen
324cd0b1e5
RED, 11 failing. Reproduced first, before any code: `okf build` on the folder a publisher's own viewer delivers gives **110 of 110 unreadable, 0 plans, exit 2**, with `no extractor is registered for file extension '.xml'`. The conservation identity `merged + coded rejections == N` is never written at all -- the run aborts earlier on `FAILED - no segmentation plans` -- so the hole was not even visible as a count. The one xml file IS the whole product: R761 Prosesskoden:2025, the document round 12 met as a 701-page PDF, in NISO-STS form. Measured on the file with `xml.etree.ElementTree`: 7 715 `<sec>`, 2 761 with a `<title>`, 4 954 with a `<label>` and no title, 10 `<table-wrap>`, 12 528 `<p>`, root `<standard>`, 0 `<!DOCTYPE` and 0 `<!ENTITY`. Its `<sec>`-nesting depths over the titled sections are 28/118/500/1141/868/97/9 -- row for row the fasit's own distribution. The ceiling is therefore structural rather than computed. FOUR HAND-WRITTEN FIXTURES, none through `make_fixtures.py` and none serialised by `ElementTree`: a library that writes and reads its own format proves only that it agrees with itself. A known-positive STS mini, generic non-STS xml, a `<!DOCTYPE` with an entity expansion, and a malformed file. Two assertions that already existed are extended rather than duplicated: the converter fence, because a file routed to the converter is read by a second parser that never sees this reader's DTD refusal, and the evidence table, because a row without a class is the failure that test exists for. pytest -q: 11 failed, 1554 passed, 1 skipped. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
815 lines
32 KiB
Python
815 lines
32 KiB
Python
"""Door B extraction registry: file bytes -> text, per file type (Phase 2 step 1).
|
|
|
|
Core stdlib extractors (md/txt/csv/json/html) plus the fail-fast gates: unknown
|
|
extension, the [extract]-gated binary types when the extra is absent, corrupt
|
|
(non-UTF-8) bytes, and an empty CSV. All file-type->text extraction lives HERE
|
|
(the guard is text-only); this step adds zero runtime dependency and no guard
|
|
call — it is pure, deterministic plumbing.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import importlib.util
|
|
import os
|
|
import re
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from llm_ingestion_okf import ExtractionError, ExtractionWarning, extract_text
|
|
|
|
FIXTURES = Path(__file__).parent / "fixtures"
|
|
|
|
# The parser-dependent tests below need the optional extra, so they are skipped
|
|
# without it — an optional extra that forced its parser on every test run would
|
|
# not be optional. The behaviour that must survive EITHER WAY is the rejection,
|
|
# and that one is asserted unconditionally above via the import probe.
|
|
requires_extract = pytest.mark.skipif(
|
|
importlib.util.find_spec("pdfplumber") is None,
|
|
reason="the optional [extract] extra is not installed",
|
|
)
|
|
|
|
# --- core extractors: happy path (a fixture per type) ---
|
|
|
|
|
|
def test_md_passthrough() -> None:
|
|
assert extract_text("note.md", b"# Title\n\nBody\n") == "# Title\n\nBody\n"
|
|
|
|
|
|
def test_txt_passthrough() -> None:
|
|
assert extract_text("note.txt", b"plain text") == "plain text"
|
|
|
|
|
|
def test_utf8_sig_bom_is_stripped() -> None:
|
|
# utf-8-sig so a BOM never leaks into the first character (baseline parity
|
|
# with Door A's read_csv).
|
|
assert extract_text("note.txt", b"\xef\xbb\xbfhello") == "hello"
|
|
|
|
|
|
def test_csv_renders_the_phase_1_markdown_table() -> None:
|
|
out = extract_text("data.csv", b"name,age\nAda,36\n")
|
|
assert out == "| name | age |\n| --- | --- |\n| Ada | 36 |\n"
|
|
|
|
|
|
def test_json_is_verbatim_inside_a_fenced_block() -> None:
|
|
out = extract_text("cfg.json", b'{"k": 1}\n')
|
|
assert out == '```\n{"k": 1}\n```\n'
|
|
|
|
|
|
def test_html_text_via_htmlparser() -> None:
|
|
"""CHANGED 2026-09-09 BECAUSE THE BEHAVIOUR CHANGED, not to go green.
|
|
|
|
This asserted the collapsed one-line form -- the very form that made 828
|
|
of 828 real HTML sections unsegmentable. Block tags now open lines and
|
|
headings carry their ATX level; inline tags are still word boundaries,
|
|
which is the half of the old assertion that still holds and is still here.
|
|
"""
|
|
out = extract_text("page.html", b"<h1>Title</h1><p>Hello <b>world</b></p>")
|
|
assert out == "# Title\nHello world"
|
|
|
|
|
|
def test_html_skips_script_and_style() -> None:
|
|
"""UNCHANGED across round 11, and re-read to confirm it: `_SKIP_TAGS` still
|
|
holds exactly `script` and `style`, and a single block still renders as a
|
|
single line, so the old expectation is still the right one."""
|
|
html = b"<style>.x{color:red}</style><p>Keep</p><script>evil()</script>"
|
|
assert extract_text("page.html", html) == "Keep"
|
|
|
|
|
|
def test_htm_is_an_html_alias() -> None:
|
|
"""UNCHANGED across round 11, same reason as the test above."""
|
|
assert extract_text("page.htm", b"<p>hi</p>") == "hi"
|
|
|
|
|
|
# --- html: block structure survives extraction (round 11) -------------------
|
|
#
|
|
# The defect this closes, measured OUTSIDE this repo on 828 real sections:
|
|
# `text()` used to be `" ".join("".join(parts).split())`, and `str.split()`
|
|
# with no argument splits on newlines too, so every HTML file extracted to
|
|
# UNCONDITIONALLY ONE LINE. Every boundary grammar in `propose` is
|
|
# line-anchored (`_ATX`, `_NUMBERED`, `_TABLE_ROW`, `_GRID_RULE`, `_OUTLINE`,
|
|
# each with `^`), so 828 of 828 documents got zero boundaries and the run
|
|
# exited 2. The same 828 sections as markdown gave 828 plans.
|
|
|
|
_ADDED_ATX = re.compile(r"(?m)^#{1,6} ")
|
|
|
|
_HTML_KNOWN_POSITIVE = b"""<!doctype html>
|
|
<html><head><title>Doc</title></head><body>
|
|
<h1>Top</h1>
|
|
<p>Intro <b>bold</b> word.</p>
|
|
<h2>First part</h2>
|
|
<ul><li>alpha</li><li>beta</li></ul>
|
|
<h2>Second part</h2>
|
|
<table><tr><td>cell one</td><td>cell two</td></tr><tr><td>r2c1</td><td>r2c2</td></tr></table>
|
|
</body></html>
|
|
"""
|
|
|
|
|
|
def _legacy_html_text(data: bytes) -> str:
|
|
"""The pre-round-11 extractor, kept HERE as the invariant's reference.
|
|
|
|
The text-preservation invariant is stated against what the old extractor
|
|
returned for the same bytes, so it needs that string. Reproducing the ten
|
|
lines in the test is the only way to keep both forms available from one
|
|
definition of "what the old code did" -- exporting a second renderer from
|
|
the module would put a function in the package that nothing ships uses.
|
|
"""
|
|
from html.parser import HTMLParser
|
|
|
|
from llm_ingestion_okf.extract import _SKIP_TAGS
|
|
|
|
class _Legacy(HTMLParser):
|
|
def __init__(self) -> None:
|
|
super().__init__(convert_charrefs=True)
|
|
self.parts: list[str] = []
|
|
self.skip = 0
|
|
|
|
def handle_starttag(self, tag: str, attrs: object) -> None: # type: ignore[override]
|
|
self.parts.append(" ")
|
|
if tag in _SKIP_TAGS:
|
|
self.skip += 1
|
|
|
|
def handle_startendtag(self, tag: str, attrs: object) -> None: # type: ignore[override]
|
|
self.parts.append(" ")
|
|
|
|
def handle_endtag(self, tag: str) -> None:
|
|
if tag in _SKIP_TAGS and self.skip > 0:
|
|
self.skip -= 1
|
|
self.parts.append(" ")
|
|
|
|
def handle_data(self, data: str) -> None:
|
|
if self.skip == 0:
|
|
self.parts.append(data)
|
|
|
|
parser = _Legacy()
|
|
parser.feed(data.decode("utf-8-sig"))
|
|
parser.close()
|
|
return " ".join("".join(parser.parts).split())
|
|
|
|
|
|
def test_html_headings_become_atx_lines_at_the_tag_s_own_level() -> None:
|
|
"""The LEVEL is the tag's, not a flat `#` for every heading.
|
|
|
|
A flat prefix would give `propose._ATX` three level-1 boundaries where the
|
|
document declares one section and two subsections, so the assertion is on
|
|
the line CONTENT, never on "there are now several lines".
|
|
"""
|
|
out = extract_text("page.html", _HTML_KNOWN_POSITIVE)
|
|
lines = out.split("\n")
|
|
assert "# Top" in lines
|
|
assert "## First part" in lines
|
|
assert "## Second part" in lines
|
|
# Blocks are their own lines; inline markup stays a word boundary.
|
|
assert "Intro bold word." in lines
|
|
assert "alpha" in lines
|
|
assert "beta" in lines
|
|
# A table row is one line, its cells separated by a space.
|
|
assert "cell one cell two" in lines
|
|
assert "r2c1 r2c2" in lines
|
|
|
|
|
|
def test_html_without_a_heading_is_one_concept_s_worth_of_text_never_zero() -> None:
|
|
"""KNOWN-NEGATIVE: no heading in, no fabricated heading out.
|
|
|
|
The `rtf` row's honest result, and the shape this must keep: a document
|
|
that declares nothing lands as ONE concept with its content preserved --
|
|
never zero concepts and never zero characters.
|
|
"""
|
|
out = extract_text("flat.html", b"<p>First para.</p><p>Second para.</p>")
|
|
assert not _ADDED_ATX.search(out)
|
|
assert out.split("\n") == ["First para.", "Second para."]
|
|
|
|
|
|
def test_html_line_structure_preserves_every_non_whitespace_character() -> None:
|
|
"""Text preservation is an EXACT invariant, not a percentage.
|
|
|
|
Strip the ATX markers this extractor added and the non-whitespace sequence
|
|
must be identical to what the old extractor returned for the same bytes.
|
|
"""
|
|
new = extract_text("page.html", _HTML_KNOWN_POSITIVE)
|
|
assert _ADDED_ATX.search(new), "known-positive: the query must be able to find a marker"
|
|
stripped = _ADDED_ATX.sub("", new)
|
|
assert "".join(stripped.split()) == "".join(_legacy_html_text(_HTML_KNOWN_POSITIVE).split())
|
|
|
|
|
|
@pytest.mark.skipif(
|
|
not os.environ.get("OKF_HTML_CORPUS"),
|
|
reason=(
|
|
"set OKF_HTML_CORPUS to a directory of .html files to run the "
|
|
"corpus-wide invariant. The path is an ENVIRONMENT VARIABLE and not a "
|
|
"constant here on purpose: a corpus path names a consumer's own "
|
|
"export, and this repository is public"
|
|
),
|
|
)
|
|
def test_the_invariant_holds_over_a_whole_corpus_not_one_document() -> None:
|
|
"""One document proves the invariant is reachable, not that it holds.
|
|
|
|
Run over the SK1 corpus (828 `.html` sections of a published handbook) this
|
|
was 828 of 828 exact and a character ratio of 1.000000 against SK1's own
|
|
>= 99.8 % bar, with 7600 ATX markers added and 31 141 lines produced where
|
|
the old extractor produced 828 -- one per file, which was the defect.
|
|
"""
|
|
root = Path(os.environ["OKF_HTML_CORPUS"])
|
|
files = sorted(root.rglob("*.html"))
|
|
assert files, f"no .html under {root}: the corpus pointer is wrong, not the code"
|
|
markers = 0
|
|
for path in files:
|
|
data = path.read_bytes()
|
|
new = extract_text(path.name, data)
|
|
markers += len(_ADDED_ATX.findall(new))
|
|
stripped = _ADDED_ATX.sub("", new)
|
|
assert "".join(stripped.split()) == "".join(_legacy_html_text(data).split()), (
|
|
f"text preservation broke on {path.name}"
|
|
)
|
|
assert markers > 0, "known-positive: a corpus of prose must yield some heading"
|
|
|
|
|
|
def test_html_br_is_a_line_break_and_inline_tags_are_not() -> None:
|
|
out = extract_text("page.html", b"<p>one<br>two <em>three</em> four</p>")
|
|
assert out.split("\n") == ["one", "two three four"]
|
|
|
|
|
|
def test_extension_dispatch_is_case_insensitive() -> None:
|
|
assert extract_text("NOTE.MD", b"x") == "x"
|
|
|
|
|
|
# --- fail-fast: unknown extension ---
|
|
|
|
|
|
def test_unknown_extension_fails_fast() -> None:
|
|
with pytest.raises(ExtractionError) as excinfo:
|
|
extract_text("archive.zip", b"PK\x03\x04")
|
|
assert excinfo.value.code == "extractor_unknown"
|
|
|
|
|
|
def test_missing_extension_fails_fast() -> None:
|
|
with pytest.raises(ExtractionError) as excinfo:
|
|
extract_text("README", b"x")
|
|
assert excinfo.value.code == "extractor_unknown"
|
|
|
|
|
|
# --- fail-fast: an [extract]-gated type without the extra installed ---
|
|
|
|
|
|
def test_optional_type_without_the_extra_fails_fast(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
"""`extractor_extra_missing` reached through the import probe, not a suffix set.
|
|
|
|
This test used to reach the code through a `.docx` or `.xlsx` filename,
|
|
which sat in `_UNPARSED_OPTIONAL_EXTENSIONS` because the extra shipped no
|
|
parser for those types. That set is on its way to empty: once those types gain a converter,
|
|
a membership test can no longer raise this code at all, and a test pinned
|
|
to it would go red for the right reason at the worst moment.
|
|
|
|
The import probe is the durable path — it is how the gate actually works
|
|
(`extract.py`'s `_extract_pdf`), and it stays reachable no matter how many
|
|
types gain parsers.
|
|
"""
|
|
monkeypatch.setitem(sys.modules, "pdfplumber", None)
|
|
with pytest.raises(ExtractionError) as excinfo:
|
|
extract_text("report.pdf", b"%PDF-1.4")
|
|
assert excinfo.value.code == "extractor_extra_missing"
|
|
# The error names the extra so the operator knows the remedy — never a
|
|
# silent skip, never a bundled parser in core.
|
|
assert "extract" in str(excinfo.value)
|
|
|
|
|
|
def test_pdf_without_the_extra_installed_keeps_the_same_rejection(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
"""The gate became an import probe; the rejection did not change.
|
|
|
|
Setting the module to None in sys.modules is what CPython treats as a
|
|
failed import, so this exercises the uninstalled path REGARDLESS of
|
|
whether the extra is installed in the running environment — a skip would
|
|
have preserved nothing on the machine where the parser is present.
|
|
"""
|
|
monkeypatch.setitem(sys.modules, "pdfplumber", None)
|
|
with pytest.raises(ExtractionError) as excinfo:
|
|
extract_text("doc.pdf", b"%PDF-1.4")
|
|
assert excinfo.value.code == "extractor_extra_missing"
|
|
assert "extract" in str(excinfo.value)
|
|
|
|
|
|
# --- office types: the table-driven converter seam ---
|
|
|
|
|
|
def test_every_office_row_names_its_reader() -> None:
|
|
"""The table is the contract; these are all the rows there are.
|
|
|
|
Written as an exact equality rather than a series of `in` checks: a row
|
|
added without a fixture and without an evidence class is the failure mode
|
|
this pins, and only equality catches an ADDITION.
|
|
"""
|
|
from llm_ingestion_okf.extract import _PANDOC_FORMATS
|
|
|
|
assert _PANDOC_FORMATS == {
|
|
".docx": "docx",
|
|
".xlsx": "xlsx",
|
|
".pptx": "pptx",
|
|
".odt": "odt",
|
|
".rtf": "rtf",
|
|
}
|
|
|
|
|
|
def test_html_epub_and_xml_are_excluded_from_the_converter_on_purpose() -> None:
|
|
"""`.html` has a stdlib extractor; routing it through the converter would
|
|
buy nothing and would add CVE-2025-51591 (SSRF via an iframe in HTML
|
|
input), which is unpatched in every converter version. `.epub` is out for
|
|
the same "no gain" half of that reason.
|
|
|
|
`.xml` joins them on that same half PLUS one of its own: the XML reader
|
|
here refuses a DTD outright, and a file routed to the converter would be
|
|
read by a different parser that never sees that refusal. The hardening
|
|
would be true of the code and false of the file.
|
|
"""
|
|
from llm_ingestion_okf.extract import _CORE_EXTRACTORS, _PANDOC_FORMATS
|
|
|
|
assert ".html" not in _PANDOC_FORMATS
|
|
assert ".epub" not in _PANDOC_FORMATS
|
|
assert ".xml" not in _PANDOC_FORMATS
|
|
assert ".html" in _CORE_EXTRACTORS
|
|
assert ".xml" in _CORE_EXTRACTORS
|
|
|
|
|
|
def test_evidence_class_is_asserted_not_commented() -> None:
|
|
"""Which rows were measured is a fact about this work, not a footnote.
|
|
|
|
`.pptx`, `.odt` and `.rtf` have denominator ZERO in the corpus this arm was
|
|
measured on. A comment saying so rots; an assertion that names them keeps
|
|
an unmeasured row from quietly presenting as a supported one.
|
|
|
|
The table is NOT the converter's rows alone: `.html` and `.pdf` are
|
|
core-supported and carry a class here too -- `.html` since 2026-09-09
|
|
(`measured`, 828 files), `.pdf` since 2026-09-10 (`measured`, eight corpus
|
|
documents with a hand-counted fasit plus a publisher's own 2 761-section
|
|
structure for a 701-page one). `.pdf` was the row with the most measurement
|
|
behind it and no row in the table at all, which is the one way a table like
|
|
this can mislead while every entry in it is true.
|
|
"""
|
|
from llm_ingestion_okf.extract import _EVIDENCE, _PANDOC_FORMATS
|
|
|
|
assert set(_EVIDENCE) == set(_PANDOC_FORMATS) | {
|
|
".html",
|
|
".pdf",
|
|
".xml",
|
|
}, "every row needs a class"
|
|
assert {s for s, e in _EVIDENCE.items() if e == "measured"} == {
|
|
".pdf",
|
|
".docx",
|
|
".xlsx",
|
|
".html",
|
|
".xml",
|
|
}
|
|
# Since 2026-09-09 the three office rows are `constructed`, not
|
|
# `unmeasured`: each has now been put through end to end on a hand-built
|
|
# document with a hand-written fasit, and none of them has a corpus file.
|
|
# The set is asserted EMPTY rather than dropped -- a class with no members
|
|
# is a fact about this package, and a future row can re-enter it.
|
|
assert {s for s, e in _EVIDENCE.items() if e == "constructed"} == {
|
|
".pptx",
|
|
".odt",
|
|
".rtf",
|
|
}
|
|
assert {s for s, e in _EVIDENCE.items() if e == "unmeasured"} == set()
|
|
|
|
|
|
def test_the_unparsed_set_is_empty_now_that_every_row_has_a_reader() -> None:
|
|
"""The membership branch that used to raise `extractor_extra_missing` has
|
|
no members left. This is why both tests for that code were repointed at the
|
|
import probe before this step landed.
|
|
"""
|
|
from llm_ingestion_okf.extract import _UNPARSED_OPTIONAL_EXTENSIONS
|
|
|
|
assert _UNPARSED_OPTIONAL_EXTENSIONS == frozenset()
|
|
|
|
|
|
def test_the_converter_is_called_with_the_load_bearing_arguments(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
"""`--eol=lf --wrap=none` and the markdown writer, all three measured.
|
|
|
|
Not hygiene, and not style. The defaults produce DIFFERENT BYTES (maximum
|
|
line length 75 against 447), which a byte-pinned golden would register as a
|
|
change nobody made. And `-t plain` destroys the headings the segment
|
|
proposer reads: measured, a document yielding 15 entries including two real
|
|
headings yields 13 with none once the writer is `plain`.
|
|
"""
|
|
import llm_ingestion_okf.extract as extract_module
|
|
|
|
seen: dict[str, object] = {}
|
|
|
|
def fake_convert(source: object, to: str, format: str, extra_args: object) -> str:
|
|
seen.update(to=to, format=format, extra_args=list(extra_args)) # type: ignore[call-overload]
|
|
return "# Heading\n\nBody."
|
|
|
|
monkeypatch.setattr(extract_module, "_convert_bytes", fake_convert)
|
|
with pytest.warns(ExtractionWarning):
|
|
text = extract_text("note.docx", b"PK\x03\x04")
|
|
|
|
assert text == "# Heading\n\nBody."
|
|
assert seen["to"] == "markdown"
|
|
assert seen["format"] == "docx"
|
|
assert "--eol=lf" in seen["extra_args"] # type: ignore[operator]
|
|
assert "--wrap=none" in seen["extra_args"] # type: ignore[operator]
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("name", "reader"),
|
|
[
|
|
("a.docx", "docx"),
|
|
("b.xlsx", "xlsx"),
|
|
("c.pptx", "pptx"),
|
|
("d.odt", "odt"),
|
|
("e.rtf", "rtf"),
|
|
],
|
|
)
|
|
def test_each_row_dispatches_to_its_reader(
|
|
name: str, reader: str, monkeypatch: pytest.MonkeyPatch
|
|
) -> None:
|
|
import llm_ingestion_okf.extract as extract_module
|
|
|
|
seen: dict[str, object] = {}
|
|
|
|
def fake_convert(source: object, to: str, format: str, extra_args: object) -> str:
|
|
seen["format"] = format
|
|
return "text"
|
|
|
|
monkeypatch.setattr(extract_module, "_convert_bytes", fake_convert)
|
|
with pytest.warns(ExtractionWarning):
|
|
extract_text(name, b"bytes")
|
|
assert seen["format"] == reader
|
|
|
|
|
|
def test_an_empty_conversion_is_refused_not_persisted(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
"""Same reason as `extractor_empty_pdf`: an empty concept is the silent
|
|
skip this registry exists to prevent."""
|
|
import llm_ingestion_okf.extract as extract_module
|
|
|
|
monkeypatch.setattr(
|
|
extract_module,
|
|
"_convert_bytes",
|
|
lambda source, to, format, extra_args: " \n\t ",
|
|
)
|
|
with pytest.raises(ExtractionError) as excinfo:
|
|
extract_text("empty.docx", b"bytes")
|
|
assert excinfo.value.code == "extractor_empty_conversion"
|
|
|
|
|
|
def test_a_converter_failure_is_wrapped_never_leaked(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
import llm_ingestion_okf.extract as extract_module
|
|
|
|
def boom(source: object, to: str, format: str, extra_args: object) -> str:
|
|
raise RuntimeError("pandoc said no")
|
|
|
|
monkeypatch.setattr(extract_module, "_convert_bytes", boom)
|
|
with pytest.raises(ExtractionError) as excinfo:
|
|
extract_text("bad.docx", b"bytes")
|
|
assert excinfo.value.code == "extractor_convert_error"
|
|
assert "pandoc said no" in str(excinfo.value)
|
|
|
|
|
|
def test_office_conversion_warns_that_it_is_lossy(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
"""Drawn content does not survive extraction here either, and the warning
|
|
is emitted after the parse -- a run that produced no text has nothing to be
|
|
lossy about."""
|
|
import llm_ingestion_okf.extract as extract_module
|
|
|
|
monkeypatch.setattr(
|
|
extract_module,
|
|
"_convert_bytes",
|
|
lambda source, to, format, extra_args: "body",
|
|
)
|
|
with pytest.warns(ExtractionWarning):
|
|
extract_text("note.docx", b"bytes")
|
|
|
|
|
|
# --- pdf: the [extract] parser (pdfplumber) ---
|
|
|
|
# The expected text is frozen against a committed fixture ON PURPOSE. Extraction
|
|
# is deterministic within a parser version and NOT guaranteed across one
|
|
# (pdfplumber pins pdfminer.six==20260107 exactly; pdfminer.six ships
|
|
# date-stamped releases with no stability contract). This literal is what makes
|
|
# a parser upgrade break something visible instead of drifting silently.
|
|
KRAV_TEXT = "Krav til helning på utkilingen\n60 og 70 1:15"
|
|
|
|
# The office fixtures are frozen the same way, and against a NAMED converter
|
|
# version -- a frozen literal means nothing without one, because the thing it
|
|
# pins is "this converter, on this input, produces these bytes". `_pandoc.py`
|
|
# refuses any other version, so the two pins hold each other up.
|
|
requires_pandoc = pytest.mark.skipif(
|
|
importlib.util.find_spec("pypandoc") is None,
|
|
reason="the optional [extract] extra is not installed",
|
|
)
|
|
|
|
# docx: a heading and one requirement row with label and value on the SAME
|
|
# line, mirroring the property the PDF fixture pins.
|
|
DOCX_TEXT = "# Krav til helning\n\n60 og 70 1:15"
|
|
|
|
# xlsx: the sheet name becomes a heading and the rows become a table. The
|
|
# label/value pairing survives on one row, which is the property that matters.
|
|
#
|
|
# THIS LITERAL MOVED ONCE, deliberately, and the move is the fix reported in
|
|
# `docs/2026-09-08-prisform-og-loggen-k2.md`: the spreadsheet row now writes
|
|
# pipe tables, so the cells arrive delimited instead of padded. Every character
|
|
# of content is the same; only the table form changed.
|
|
XLSX_TEXT = "## Krav {#sheet-1}\n\n| Krav til helning | |\n|----|----|\n| 60 og 70 | 1:15 |"
|
|
|
|
# The negative control, committed rather than described: the SAME document
|
|
# without `word/styles.xml`. The body survives and the heading marker does not.
|
|
DOCX_NO_STYLES_TEXT = "Krav til helning\n\n60 og 70 1:15"
|
|
|
|
|
|
@requires_pandoc
|
|
def test_docx_extracts_to_its_frozen_text() -> None:
|
|
data = (FIXTURES / "two-line-krav.docx").read_bytes()
|
|
with pytest.warns(ExtractionWarning):
|
|
assert extract_text("krav.docx", data) == DOCX_TEXT
|
|
|
|
|
|
@requires_pandoc
|
|
def test_xlsx_extracts_to_its_frozen_text() -> None:
|
|
data = (FIXTURES / "two-line-krav.xlsx").read_bytes()
|
|
with pytest.warns(ExtractionWarning):
|
|
assert extract_text("krav.xlsx", data) == XLSX_TEXT
|
|
|
|
|
|
@requires_pandoc
|
|
def test_a_docx_without_a_styles_part_loses_its_heading() -> None:
|
|
"""The negative control for the fixture policy, run rather than asserted.
|
|
|
|
`word/styles.xml` is what makes the converter see a heading. Without it the
|
|
same document extracts as flat prose -- so a fixture built WITHOUT that
|
|
part would pin the body and pin nothing at all about structure, while
|
|
looking exactly as convincing.
|
|
|
|
Structure is the half the segment proposer reads, which is why this is a
|
|
committed fixture and not a sentence in a README.
|
|
"""
|
|
data = (FIXTURES / "no-styles-krav.docx").read_bytes()
|
|
with pytest.warns(ExtractionWarning):
|
|
text = extract_text("krav.docx", data)
|
|
assert text == DOCX_NO_STYLES_TEXT
|
|
assert not text.startswith("#"), "the heading marker must be absent"
|
|
assert DOCX_TEXT.startswith("#"), "and present in the fixture that has styles"
|
|
|
|
|
|
@requires_pandoc
|
|
def test_the_frozen_office_text_is_pinned_to_a_named_converter_version() -> None:
|
|
"""A frozen literal without a named version pins nothing.
|
|
|
|
If the converter version ever moves, these literals must be re-measured
|
|
rather than trusted -- so the version is asserted right where they live.
|
|
"""
|
|
from llm_ingestion_okf._pandoc import PANDOC_VERSION, resolve_pandoc
|
|
|
|
assert PANDOC_VERSION == "3.9"
|
|
assert resolve_pandoc().is_file()
|
|
|
|
|
|
@requires_pandoc
|
|
def test_office_extraction_is_byte_stable_across_calls() -> None:
|
|
data = (FIXTURES / "two-line-krav.docx").read_bytes()
|
|
with pytest.warns(ExtractionWarning):
|
|
first = extract_text("krav.docx", data)
|
|
with pytest.warns(ExtractionWarning):
|
|
second = extract_text("krav.docx", data)
|
|
assert first == second
|
|
|
|
|
|
@requires_extract
|
|
def test_pdf_extracts_text_with_label_and_value_on_one_line() -> None:
|
|
data = (FIXTURES / "two-line-krav.pdf").read_bytes()
|
|
with pytest.warns(ExtractionWarning):
|
|
assert extract_text("krav.pdf", data) == KRAV_TEXT
|
|
|
|
|
|
@requires_extract
|
|
def test_pdf_extraction_is_byte_stable_across_calls() -> None:
|
|
"""Determinism is a promise this library already makes; hold it here too."""
|
|
data = (FIXTURES / "two-line-krav.pdf").read_bytes()
|
|
with pytest.warns(ExtractionWarning):
|
|
first = extract_text("krav.pdf", data)
|
|
second = extract_text("krav.pdf", data)
|
|
assert first == second
|
|
|
|
|
|
@requires_extract
|
|
def test_pdf_extraction_warns_that_drawn_content_is_not_recovered() -> None:
|
|
"""Figures are vector drawings: only the caption survives leg 2.
|
|
|
|
Categorically true of text extraction, so it is stated as a warning on
|
|
every PDF rather than guessed at per document — detecting "is there a
|
|
figure here" would be exactly the layout heuristic this order declined.
|
|
"""
|
|
data = (FIXTURES / "two-line-krav.pdf").read_bytes()
|
|
with pytest.warns(ExtractionWarning, match="figures"):
|
|
extract_text("krav.pdf", data)
|
|
|
|
|
|
@requires_extract
|
|
def test_pdf_with_no_text_layer_fails_fast() -> None:
|
|
"""A scanned PDF yields nothing; an empty concept would be a silent skip."""
|
|
data = (FIXTURES / "no-text-layer.pdf").read_bytes()
|
|
with pytest.raises(ExtractionError) as excinfo:
|
|
extract_text("scan.pdf", data)
|
|
assert excinfo.value.code == "extractor_empty_pdf"
|
|
|
|
|
|
@requires_extract
|
|
def test_corrupt_pdf_fails_fast_typed() -> None:
|
|
"""Never a leaked pdfminer exception — the "always typed" doctrine holds."""
|
|
with pytest.raises(ExtractionError) as excinfo:
|
|
extract_text("broken.pdf", b"not a pdf at all")
|
|
assert excinfo.value.code == "extractor_pdf_error"
|
|
|
|
|
|
# --- fail-fast: corrupt (non-UTF-8) bytes on a text type ---
|
|
|
|
|
|
def test_invalid_utf8_fails_fast_typed() -> None:
|
|
# Never a leaked UnicodeDecodeError — the "always typed" doctrine holds for
|
|
# Door B too (cf. Door A wrapping path ValueError as SourceError).
|
|
with pytest.raises(ExtractionError) as excinfo:
|
|
extract_text("note.txt", b"\xffbad")
|
|
assert excinfo.value.code == "extractor_decode_error"
|
|
|
|
|
|
# --- fail-fast: empty CSV (no header row) ---
|
|
|
|
|
|
def test_empty_csv_fails_fast() -> None:
|
|
with pytest.raises(ExtractionError) as excinfo:
|
|
extract_text("empty.csv", b"")
|
|
assert excinfo.value.code == "extractor_empty_csv"
|
|
|
|
|
|
# --- xlsx: the spreadsheet form a consumer has to read ---
|
|
#
|
|
# Measured on K2 and reported upstream by the first consumer to read the bundle
|
|
# with a live model (`docs/2026-09-08-prisform-og-loggen-k2.md`): the converter's
|
|
# DEFAULT markdown writer emits simple tables, which pad every cell out to the
|
|
# width of the widest cell in its column. One long prose cell therefore turns
|
|
# every other row in that column into a whitespace carpet -- 887 characters
|
|
# between a label and its amount on the real sheet -- while the header row names
|
|
# a single column because only the first source cell in row 1 is filled. The
|
|
# bytes reach the reader and the STRUCTURE does not.
|
|
#
|
|
# `prisark.xlsx` is that shape in miniature, hand-laid rather than recorded, and
|
|
# it carries its own negative control on a second sheet.
|
|
|
|
PRISARK = "prisark.xlsx"
|
|
|
|
|
|
def _table_rows(text: str) -> list[list[str]]:
|
|
"""Every pipe-table row in `text`, as its cells, in order."""
|
|
rows = []
|
|
for line in text.split("\n"):
|
|
stripped = line.strip()
|
|
if not (stripped.startswith("|") and stripped.endswith("|")):
|
|
continue
|
|
cells = [cell.strip() for cell in stripped[1:-1].split("|")]
|
|
if all(set(cell) <= set("-:") and cell for cell in cells):
|
|
continue # the header separator is punctuation, not a row
|
|
rows.append(cells)
|
|
return rows
|
|
|
|
|
|
@requires_pandoc
|
|
def test_a_spreadsheet_keeps_its_columns_one_row_per_line() -> None:
|
|
"""The label and the amount arrive as separate cells on one line.
|
|
|
|
This is the property the whole change exists for. Asserted as properties
|
|
rather than only as a frozen literal, because a literal pins bytes and says
|
|
nothing about which of them was the point.
|
|
"""
|
|
data = (FIXTURES / PRISARK).read_bytes()
|
|
with pytest.warns(ExtractionWarning):
|
|
text = extract_text(PRISARK, data)
|
|
|
|
rows = _table_rows(text)
|
|
assert [
|
|
"01",
|
|
"Rigging og drift av byggeplass, medregnet alt som ikke er "
|
|
"priset spesifikt nedenfor og alt som er innkalkulert i de angitte "
|
|
"prisene",
|
|
"5647500",
|
|
] in rows
|
|
assert ["02", "Andel", "12.5"] in rows
|
|
|
|
longest = max((len(run) for run in re.findall(r" {2,}", text)), default=0)
|
|
assert longest <= 8, f"a whitespace run of {longest} is a carpet, not a column"
|
|
|
|
|
|
@requires_pandoc
|
|
def test_an_integral_amount_loses_the_converters_decimal_and_a_real_one_keeps_it() -> None:
|
|
"""`5647500` is a number; `92.0` in the same sheet is TEXT.
|
|
|
|
The converter renders both as `<digits>.0`, so the output alone cannot tell
|
|
them apart. The shared string table can, and is what the rewrite consults --
|
|
which is why this test asserts both directions from ONE document.
|
|
"""
|
|
data = (FIXTURES / PRISARK).read_bytes()
|
|
with pytest.warns(ExtractionWarning):
|
|
text = extract_text(PRISARK, data)
|
|
|
|
assert "5647500.0" not in text
|
|
assert "250000.0" not in text
|
|
assert "12.5" in text, "a genuine decimal is a value, not a converter artefact"
|
|
assert ["03", "92.0", "250000"] in _table_rows(text), (
|
|
"a shared-string cell reading 92.0 is author text and survives verbatim"
|
|
)
|
|
assert "Kode 4 \\| 5.0" in text, (
|
|
"a `5.0` INSIDE a cell is not a cell: the delimiter test is what sees that"
|
|
)
|
|
|
|
|
|
@requires_pandoc
|
|
def test_a_single_column_sheet_gains_no_columns() -> None:
|
|
"""The negative control, in the same document as the case it controls.
|
|
|
|
Sheet 2 has ONE column in the source. There is nothing to recover, so the
|
|
fix must not invent a second cell anywhere on it. Its three values arrive
|
|
in order and alone.
|
|
|
|
It is NOT byte-identical before and after the change, and that is measured
|
|
rather than glossed: the writer emits a pipe table for every table it
|
|
writes, so a one-column table changes delimiter form too. What must not
|
|
change is the cell content and the column count.
|
|
"""
|
|
data = (FIXTURES / PRISARK).read_bytes()
|
|
with pytest.warns(ExtractionWarning):
|
|
text = extract_text(PRISARK, data)
|
|
|
|
single = text.split("## Enkeltkolonne")[1]
|
|
rows = _table_rows(single)
|
|
assert [cells for cells in rows if any(cells)] == [
|
|
["Notat"],
|
|
["Ingen kolonner her"],
|
|
["Sum ikke oppgitt"],
|
|
]
|
|
|
|
|
|
# The scoping control. The same writer change applied to the other four office
|
|
# rows was MEASURED to move them (the odt fixture 1366 -> 1105 characters), so
|
|
# this digest can fail; it is not a tautology. The change is deliberately
|
|
# spreadsheet-only: a spreadsheet IS a grid and has no prose fallback, while
|
|
# moving docx/pptx/odt/rtf would move a corpus denominator nothing has measured.
|
|
# A red here means the writer stopped being scoped -- read the diff and decide.
|
|
OFFICE_TEXT_DIGESTS = {
|
|
"k2-office/krav-tekstdokument.odt": (
|
|
"58c9776f0d7f2b2a3a9d2774e4ae243b265c31b5b6b96914ef4db419fa66e4e2"
|
|
),
|
|
"k2-office/krav-presentasjon.pptx": (
|
|
"752420a04d651a416938ff9f0b3c2de5849bd2ea1d52063dd88aaae65ab99b90"
|
|
),
|
|
"k2-office/krav-rikt-tekstformat.rtf": (
|
|
"79cbf756eb482bb603f82c171d11efe74b9ba62ab9ef679ecd6bb8c3b3740ffb"
|
|
),
|
|
}
|
|
|
|
|
|
@requires_pandoc
|
|
@pytest.mark.parametrize("relative", sorted(OFFICE_TEXT_DIGESTS))
|
|
def test_the_other_office_rows_are_untouched_by_the_spreadsheet_writer(relative: str) -> None:
|
|
path = FIXTURES / relative
|
|
with pytest.warns(ExtractionWarning):
|
|
text = extract_text(path.name, path.read_bytes())
|
|
assert hashlib.sha256(text.encode("utf-8")).hexdigest() == OFFICE_TEXT_DIGESTS[relative]
|
|
|
|
|
|
# The whole fixture, frozen against the same named converter version as the
|
|
# literals above. The property tests say WHAT matters; this one catches any
|
|
# other byte moving without anybody noticing.
|
|
PRISARK_TEXT = (
|
|
"## Prisark {#sheet-1}\n\n"
|
|
"| Prisskjema | | |\n"
|
|
"|----|----|----|\n"
|
|
"| Post | Beskrivelse | Sum |\n"
|
|
"| 01 | Rigging og drift av byggeplass, medregnet alt som ikke er priset "
|
|
"spesifikt nedenfor og alt som er innkalkulert i de angitte prisene | 5647500 |\n"
|
|
"| 02 | Andel | 12.5 |\n"
|
|
"| 03 | 92.0 | 250000 |\n"
|
|
"| 04 | Kode 4 \\| 5.0 | |\n\n"
|
|
"## Enkeltkolonne {#sheet-2}\n\n"
|
|
"| Notat |\n"
|
|
"|----|\n"
|
|
"| Ingen kolonner her |\n"
|
|
"| Sum ikke oppgitt |"
|
|
)
|
|
|
|
|
|
@requires_pandoc
|
|
def test_prisark_extracts_to_its_frozen_text() -> None:
|
|
data = (FIXTURES / PRISARK).read_bytes()
|
|
with pytest.warns(ExtractionWarning):
|
|
assert extract_text(PRISARK, data) == PRISARK_TEXT
|