Kjell Tore Guttormsen
33d3269380
Round 3 of the 0.10.1 review. Rounds 1-3 each bound a NUMBER and the bomb
moved to the next one: the declared size, then the first `FlateDecode`, then
every `FlateDecode`. It now lives in a link this package had documented as
safe. `ASCII85Decode` was called bounded "by its own input because it
shrinks"; `z` is ASCII85's shorthand for four zero bytes, so one input byte
becomes four, and `base64.a85decode` appends one 4-byte object per group to a
list, so the DECODER costs about a hundred bytes of memory per byte of input
(measured on CPython 3.14: 101.4x at 1 MiB, 96.1x at 4 MiB, 94.5x at 16 MiB).
Measured on the pinned `0c3c490` tree, its own interpreter, idle machine: a
33 475-byte PDF decoding an image through `/Filter [/FlateDecode
/ASCII85Decode]` cost 3 827 003 392 bytes of peak RSS -- 114 000x the file --
and the picture was CARRIED, with no rejection at all.
Seven tests red, three green:
* the two bombs above, in their own interpreters, at the shipped bound;
* the cost ratio as a re-measurable known-negative, so the constant cannot
rot the first time CPython changes `a85decode`;
* the input cap against the corpora (9 668 image objects over 77 PDFs, 16
behind an `ASCII85Decode` link, largest input 450 739 bytes);
* THE PROPERTY, both payload fills: every chain of length 1-3 over the ten
filters pdfminer decodes -- 1 110 of them -- is either delivered under the
bound or refused with a code in the published vocabulary, and never paid
for on the way, which `tracemalloc` measures because that is where
`a85decode`'s cost lives.
Green but previously uncovered, which is the MAJOR of the same checkpoint:
`check_payload` at the END of `_check_stream_cost` could be deleted with the
whole suite still passing, because the second one after `get_data()` gives
the same code one step later. The two differ in whether the payment was made,
so the test asserts `get_data` was never called. The known-positive beside
the property -- every bounded chain still carries a 64-byte image -- is green
too, and a rule that refuses everything would pass the property alone.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
1084 lines
45 KiB
Python
1084 lines
45 KiB
Python
"""Two findings of the independent 0.10.0 review, as red tests (0.10.1).
|
|
|
|
Both land with the SHIPPED defaults (`--assets` on, `--gate
|
|
guard-trusted-source`), and both are new in 0.10.0 -- before it, no reader
|
|
read an `<img>` attribute or opened an image stream at all.
|
|
|
|
1. **A remote image reference became a LIVE markdown image link** in the
|
|
persisted concept, carrying an address the document's author controls,
|
|
query string included. This package opens no socket, but a consumer that
|
|
renders the markdown or lets an agent fetch images does, which turns "a
|
|
bundle was opened" into a beacon. The guard's `user-upload` tier refuses
|
|
such a line and the build's default tier does not, so the same bytes are
|
|
persisted under the default and refused one tier up.
|
|
2. **Nothing bounded a PDF image's size.** A 9.6 KB file declaring an
|
|
8000x8000 grayscale image of compressed zeros made `encode_png` allocate
|
|
width*height bytes twice over; measured peak RSS 83 MB at 3000x3000 and
|
|
276 MB at 8000x8000, linear in the pixel count. One such document -- or
|
|
one legitimately enormous scan -- takes the whole batch build with it,
|
|
before any gate, because the guard never sees image bytes.
|
|
|
|
The limit is READ OFF the corpora rather than chosen: over the 4 828 image
|
|
objects of the 43-document reference corpus the largest is 4 515 x 4 128
|
|
(18.6 MP, a landscape drawing), and over R761's 109 delivered pictures the
|
|
largest is 2 072 x 656 (1.4 MP). `MAX_IMAGE_PIXELS` sits above both with
|
|
room to spare, and anything larger is a counted refusal rather than a
|
|
killed build.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
import zlib
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from llm_ingestion_okf import assets, cli
|
|
from llm_ingestion_okf.assets import IMAGE_POINTER, render_missing
|
|
from llm_ingestion_okf.errors import ExtractionError
|
|
from llm_ingestion_okf.extract import extract_document
|
|
|
|
FIXTURES = Path(__file__).parent / "fixtures" / "image-inbox"
|
|
REMOTE = "https://collect.example.net/p.gif?doc=drift&u=SESSION"
|
|
|
|
#: Any markdown image whose target is not this bundle's own `assets/`.
|
|
FOREIGN_IMAGE_LINK = re.compile(r"!\[[^\]\n]*\]\(\s*(?!/assets/)([^)\s]+)")
|
|
|
|
|
|
# --- finding 1: a remote reference is inert ---------------------------------
|
|
|
|
|
|
def test_a_remote_reference_is_not_a_markdown_image_link() -> None:
|
|
line = render_missing(REMOTE, reason="the source is off this machine", label="fig", href=REMOTE)
|
|
assert "](" not in line
|
|
assert REMOTE in line, "the address is still stated -- a reader must see what was there"
|
|
|
|
|
|
def test_the_pointer_block_of_a_carried_image_is_unchanged() -> None:
|
|
"""The known-positive beside it: a local image keeps its image block, or
|
|
the fix has merely disarmed the whole capability."""
|
|
image = assets.read_image((FIXTURES / "graphics" / "figur-84-1.png").read_bytes(), name="f.png")
|
|
assert IMAGE_POINTER.search(assets.render_block(image)) is not None
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"document",
|
|
[
|
|
'<html><body><p>A</p><img src="{ref}" alt="fig"></body></html>',
|
|
'<standard xmlns:xlink="http://www.w3.org/1999/xlink"><body><sec><title>T</title>'
|
|
'<p>A</p><graphic xlink:href="{ref}"/></sec></body></standard>',
|
|
],
|
|
ids=["html", "sts"],
|
|
)
|
|
@pytest.mark.parametrize("ref", [REMOTE, "//collect.example.net/p.gif", "HTTPS://EVIL/p.gif"])
|
|
def test_no_reader_writes_a_live_link_for_a_remote_reference(document: str, ref: str) -> None:
|
|
"""A property over the readers that resolve references, not one string."""
|
|
suffix = ".html" if document.startswith("<html") else ".xml"
|
|
# `&` is an entity opener in XML, so the reference is escaped for that
|
|
# reader and not for the HTML one.
|
|
escaped = ref if suffix == ".html" else ref.replace("&", "&")
|
|
extracted = extract_document(
|
|
f"doc{suffix}", document.format(ref=escaped).encode("utf-8"), assets=True
|
|
)
|
|
foreign = FOREIGN_IMAGE_LINK.findall(extracted.text)
|
|
assert foreign == [], f"live image link(s) {foreign} for {ref}"
|
|
assert ref.split("?")[0].lower() in extracted.text.lower()
|
|
|
|
|
|
def test_a_data_uri_image_is_carried_and_leaves_no_foreign_link() -> None:
|
|
png = (FIXTURES / "graphics" / "figur-84-1.png").read_bytes()
|
|
import base64
|
|
|
|
uri = "data:image/png;base64," + base64.b64encode(png).decode("ascii")
|
|
extracted = extract_document(
|
|
"doc.html", f'<html><body><img src="{uri}"></body></html>'.encode(), assets=True
|
|
)
|
|
assert len(extracted.images) == 1
|
|
assert FOREIGN_IMAGE_LINK.findall(extracted.text) == []
|
|
|
|
|
|
def test_a_built_bundle_carries_no_foreign_image_link(tmp_path: Path) -> None:
|
|
"""The shipped fixture inbox holds one remote `<img>`; the bundle must
|
|
point at `assets/` and nowhere else."""
|
|
pytest.importorskip("pdfplumber")
|
|
pytest.importorskip("pypandoc")
|
|
bundle = tmp_path / "bundle"
|
|
code = cli.main(
|
|
[
|
|
"build",
|
|
str(FIXTURES),
|
|
"--bundle",
|
|
str(bundle),
|
|
"--bundle-id",
|
|
"limits",
|
|
"--okf-version",
|
|
"0.2",
|
|
]
|
|
)
|
|
assert code == 0
|
|
foreign: list[str] = []
|
|
for path in bundle.rglob("*.md"):
|
|
foreign += FOREIGN_IMAGE_LINK.findall(path.read_text(encoding="utf-8"))
|
|
assert foreign == []
|
|
|
|
|
|
# --- finding 2: a declared size that is too large is refused, not decoded ---
|
|
|
|
|
|
def _zeros_stream(total: int) -> bytes:
|
|
"""`total` bytes of zeros, deflated WITHOUT ever holding them.
|
|
|
|
The generator has to stay cheaper than the bomb it builds, or the test
|
|
measures its own fixture instead of the code under test.
|
|
"""
|
|
compressor = zlib.compressobj(9)
|
|
chunk = b"\x00" * (1 << 20)
|
|
parts = [compressor.compress(chunk) for _ in range(total >> 20)]
|
|
parts.append(compressor.flush())
|
|
return b"".join(parts)
|
|
|
|
|
|
def _nested_zeros_stream(total: int, links: int = 2) -> bytes:
|
|
"""The same zeros, deflated `links` times over -- the shape a PDF declares
|
|
as `/Filter [/FlateDecode /FlateDecode]`.
|
|
|
|
Deflate output of a long run of zeros is itself highly repetitive, so the
|
|
second pass shrinks 407 685 bytes to 795: the whole 400 MB stands behind
|
|
under a kilobyte of stream, which is why measuring only the first link of
|
|
the chain is not a bound at all.
|
|
"""
|
|
stream = _zeros_stream(total)
|
|
for _ in range(links - 1):
|
|
stream = zlib.compress(stream, 9)
|
|
return stream
|
|
|
|
|
|
def _bomb(
|
|
dimension: int,
|
|
content: bytes | None = None,
|
|
payload: bytes | None = None,
|
|
filters: str = "/FlateDecode",
|
|
) -> bytes:
|
|
"""A tiny PDF declaring one `dimension` x `dimension` grayscale image of
|
|
compressed zeros -- the review's own repro, built here. `content` replaces
|
|
the page's content stream, for a page that draws an INLINE image instead.
|
|
`payload` replaces the image stream, for a document whose DECLARED size and
|
|
whose actual stream are two different numbers. `filters` replaces the
|
|
`/Filter` entry, for a document whose stream is decoded by a CHAIN rather
|
|
than by one filter."""
|
|
if payload is None:
|
|
payload = zlib.compress(b"\x00" * (dimension * dimension), 9)
|
|
if content is None:
|
|
content = b"BT /F1 12 Tf 20 100 Td (bomb) Tj ET\nq 100 0 0 100 20 20 cm /Im0 Do Q\n"
|
|
objects = [
|
|
b"<< /Type /Catalog /Pages 2 0 R >>",
|
|
b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>",
|
|
b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 200 200] /Resources << /XObject "
|
|
b"<< /Im0 5 0 R >> /Font << /F1 6 0 R >> >> /Contents 4 0 R >>",
|
|
b"<< /Length %d >>\nstream\n" % len(content) + content + b"\nendstream",
|
|
(
|
|
"<< /Type /XObject /Subtype /Image /Width %d /Height %d /ColorSpace /DeviceGray "
|
|
"/BitsPerComponent 8 /Filter %s /Length %d >>\nstream\n"
|
|
% (dimension, dimension, filters, len(payload))
|
|
).encode("ascii")
|
|
+ payload
|
|
+ b"\nendstream",
|
|
b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>",
|
|
]
|
|
out = bytearray(b"%PDF-1.4\n")
|
|
offsets = []
|
|
for number, body in enumerate(objects, start=1):
|
|
offsets.append(len(out))
|
|
out += b"%d 0 obj\n" % number + body + b"\nendobj\n"
|
|
start = len(out)
|
|
out += b"xref\n0 %d\n0000000000 65535 f \n" % (len(objects) + 1)
|
|
for offset in offsets:
|
|
out += b"%010d 00000 n \n" % offset
|
|
out += b"trailer\n<< /Size %d /Root 1 0 R >>\nstartxref\n%d\n%%%%EOF\n" % (
|
|
len(objects) + 1,
|
|
start,
|
|
)
|
|
return bytes(out)
|
|
|
|
|
|
def test_the_limit_is_above_every_image_measured_in_the_corpora() -> None:
|
|
"""4 515 x 4 128 = 18.6 MP is the largest of the 4 828 objects measured in
|
|
the reference corpus; R761's largest delivered picture is 1.4 MP."""
|
|
assert assets.MAX_IMAGE_PIXELS > 4515 * 4128
|
|
assert assets.MAX_IMAGE_BYTES >= assets.MAX_IMAGE_PIXELS
|
|
|
|
|
|
def test_a_pdf_image_over_the_limit_is_refused_by_code() -> None:
|
|
pytest.importorskip("pdfplumber")
|
|
dimension = 1 + int(assets.MAX_IMAGE_PIXELS**0.5)
|
|
extracted = extract_document("bomb.pdf", _bomb(dimension), assets=True)
|
|
assert extracted.images == ()
|
|
assert [rejection.code for rejection in extracted.rejected] == ["asset_too_large"]
|
|
assert str(dimension) in extracted.rejected[0].reason
|
|
|
|
|
|
def test_a_pdf_image_under_the_limit_is_still_carried() -> None:
|
|
"""The boundary from the other side, on the same generator."""
|
|
pytest.importorskip("pdfplumber")
|
|
extracted = extract_document("small.pdf", _bomb(64), assets=True)
|
|
assert [(image.width, image.height) for image in extracted.images] == [(64, 64)]
|
|
|
|
|
|
def test_the_refusal_happens_before_the_stream_is_decompressed() -> None:
|
|
"""The limit is read off the DECLARED size, and the order is observable.
|
|
|
|
The image stream here is CORRUPT (its bytes are not deflate data) while
|
|
its declared size is over the bound. Decoding first gives
|
|
`asset_pdf_unsupported` ("could not be decoded"); reading the declared
|
|
size first gives `asset_too_large`. A check after `get_data()` has already
|
|
paid for the bomb it was meant to stop.
|
|
"""
|
|
pytest.importorskip("pdfplumber")
|
|
dimension = 1 + int(assets.MAX_IMAGE_PIXELS**0.5)
|
|
document = _bomb(dimension)
|
|
payload = zlib.compress(b"\x00" * (dimension * dimension), 9)
|
|
corrupt = document.replace(payload, b"\xff" * len(payload))
|
|
assert corrupt != document
|
|
extracted = extract_document("corrupt.pdf", corrupt, assets=True)
|
|
assert [rejection.code for rejection in extracted.rejected] == ["asset_too_large"]
|
|
|
|
|
|
def test_a_data_uri_over_the_limit_is_refused_before_decoding(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
monkeypatch.setattr(assets, "MAX_IMAGE_BYTES", 32)
|
|
uri = "data:image/png;base64," + "A" * 4096
|
|
extracted = extract_document(
|
|
"doc.html", f'<html><body><img src="{uri}"></body></html>'.encode(), assets=True
|
|
)
|
|
assert [rejection.code for rejection in extracted.rejected] == ["asset_too_large"]
|
|
|
|
|
|
def test_encode_png_refuses_the_same_size_on_its_own() -> None:
|
|
"""Defence in depth: the encoder does not trust its caller to have checked."""
|
|
dimension = 1 + int(assets.MAX_IMAGE_PIXELS**0.5)
|
|
with pytest.raises(ExtractionError) as excinfo:
|
|
assets.encode_png(dimension, dimension, b"", channels=1, palette=None, alpha=None)
|
|
assert excinfo.value.code == "asset_too_large"
|
|
|
|
|
|
# --- the determinism defect PM added to this order --------------------------
|
|
|
|
|
|
def test_an_inline_pdf_image_gets_a_stable_name() -> None:
|
|
"""pdfminer names an inline image (`BI ... EI`) from `id()` of a Python
|
|
object, so the pointer line changed between two runs of one build -- two
|
|
concept files of the reference corpus differed. Measured 2026-09-17."""
|
|
pytest.importorskip("pdfplumber")
|
|
from llm_ingestion_okf import extract as extract_module
|
|
|
|
inline = (
|
|
b"BT /F1 12 Tf 20 100 Td (t) Tj ET\n"
|
|
b"q 10 0 0 10 20 20 cm BI /W 2 /H 2 /CS /G /BPC 8 /F /AHx ID 00112233> EI Q\n"
|
|
)
|
|
document = _bomb(4, content=inline)
|
|
first = extract_document("inline.pdf", document, assets=True)
|
|
extract_module._pdf_pages.cache_clear()
|
|
second = extract_document("inline.pdf", document, assets=True)
|
|
names = [image.name for image in first.images] + [r.name for r in first.rejected]
|
|
again = [image.name for image in second.images] + [r.name for r in second.rejected]
|
|
assert names == again != []
|
|
assert not any(part.isdigit() and len(part) > 6 for name in names for part in name.split("-"))
|
|
|
|
|
|
# --- BLOCKER-1 of the 18.09 review: the bound must bind what the run PAYS ----
|
|
#
|
|
# `check_size` reads `/Width` and `/Height` out of the image dictionary, which
|
|
# is a CLAIM by an untrusted document, and the claim and the cost are two
|
|
# independent numbers: `/Length` is the COMPRESSED length and nothing in the
|
|
# dictionary states what `get_data()` will return. Measured on `230d1cb` by an
|
|
# independent review: a 389 626-byte PDF declaring 1x1 and carrying 400 MB of
|
|
# deflated zeros was CARRIED, with no rejection, at 834 MB of peak RSS -- and
|
|
# 1,2 GB of zeros at 2 436 MB, linear, about 2 100x the file size. The four
|
|
# mutations that suite already kills do not separate declared from actual, so
|
|
# they were all green while this stood.
|
|
|
|
#: What a bounded run of the 400 MB bomb may cost, in bytes of peak RSS.
|
|
#: Measured 2026-09-18 on this machine, same commit, same fixture: 892 MB
|
|
#: without the bound and 54 MB with it, and the bounded figure barely moves
|
|
#: when the stream triples (62 MB at 1,2 GB) because what grows is the
|
|
#: COMPRESSED input, which was already in memory. The bar sits between the
|
|
#: two, far enough above the bounded run that the interpreter's own footprint
|
|
#: on another machine cannot reach it.
|
|
PEAK_RSS_BOUND = 256 * 1024 * 1024
|
|
|
|
#: The stream the bomb inflates to. Over `MAX_IMAGE_BYTES` (256 MiB), so it is
|
|
#: refused at the real bound rather than at a monkeypatched one.
|
|
BOMB_STREAM_BYTES = 400 * 1024 * 1024
|
|
|
|
_CHILD = """
|
|
import resource, sys
|
|
sys.path.insert(0, {tests!r})
|
|
from test_asset_limits import _bomb, _zeros_stream, _nested_zeros_stream
|
|
from llm_ingestion_okf.extract import extract_document
|
|
|
|
document = {builder}
|
|
extracted = extract_document("bomb.pdf", document, assets=True)
|
|
peak = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
|
|
print(
|
|
len(document),
|
|
len(extracted.images),
|
|
",".join(rejection.code for rejection in extracted.rejected) or "-",
|
|
peak if sys.platform == "darwin" else peak * 1024,
|
|
)
|
|
"""
|
|
|
|
|
|
def _run_bomb(total: int, *, links: int = 1) -> tuple[int, int, str, int]:
|
|
"""The bomb in its own interpreter, so peak RSS is ITS peak and not the
|
|
high-water mark of every test that ran before it. `links` is how many
|
|
`FlateDecode` filters the stream is decoded by."""
|
|
builder = (
|
|
"_bomb(1, payload=_zeros_stream(%d))" % total
|
|
if links == 1
|
|
else "_bomb(1, payload=_nested_zeros_stream(%d, %d), filters=%r)"
|
|
% (total, links, "[" + " ".join(["/FlateDecode"] * links) + "]")
|
|
)
|
|
completed = subprocess.run(
|
|
[sys.executable, "-c", _CHILD.format(tests=str(Path(__file__).parent), builder=builder)],
|
|
capture_output=True,
|
|
text=True,
|
|
check=True,
|
|
)
|
|
size, carried, codes, peak = completed.stdout.split()
|
|
return int(size), int(carried), codes, int(peak)
|
|
|
|
|
|
def test_a_declared_size_of_one_pixel_does_not_licence_an_unbounded_stream() -> None:
|
|
"""The review's repro, at the shipped bound: 1x1 declared, 400 MB paid."""
|
|
pytest.importorskip("pdfplumber")
|
|
size, carried, codes, peak = _run_bomb(BOMB_STREAM_BYTES)
|
|
assert size < 2 * 1024 * 1024, "the fixture must stay a small file, or it proves nothing"
|
|
assert carried == 0, "a 400 MB stream was carried as a 1x1 picture"
|
|
assert codes == "asset_too_large"
|
|
assert peak < PEAK_RSS_BOUND, f"peak RSS {peak} bytes for a {size}-byte file"
|
|
|
|
|
|
def test_the_refusal_reads_the_stream_and_not_only_the_declaration() -> None:
|
|
"""The mutation the shipped suite could not kill.
|
|
|
|
An honest 20000x20000 declaration is refused by `check_size` alone, so a
|
|
test built on one is green whether or not the actual stream is bounded.
|
|
This document declares a size WITHIN the bound, which is the only shape
|
|
that separates the two checks.
|
|
"""
|
|
pytest.importorskip("pdfplumber")
|
|
small = zlib.compress(b"\x00" * 64, 9)
|
|
within = extract_document("small.pdf", _bomb(8, payload=small), assets=True)
|
|
assert [rejection.code for rejection in within.rejected] == [], "the control must be carried"
|
|
assert len(within.images) == 1
|
|
|
|
|
|
def test_a_stream_over_the_bound_is_refused_with_a_patched_bound() -> None:
|
|
"""The same rule, cheap, so it runs on every machine and every suite."""
|
|
pytest.importorskip("pdfplumber")
|
|
monkey = pytest.MonkeyPatch()
|
|
try:
|
|
monkey.setattr(assets, "MAX_IMAGE_BYTES", 4096)
|
|
extracted = extract_document(
|
|
"bomb.pdf", _bomb(1, payload=zlib.compress(b"\x00" * 1_000_000, 9)), assets=True
|
|
)
|
|
finally:
|
|
monkey.undo()
|
|
assert extracted.images == ()
|
|
assert [rejection.code for rejection in extracted.rejected] == ["asset_too_large"]
|
|
|
|
|
|
# --- MAJOR of the 18.09 review: a non-positive declaration is not a size -----
|
|
|
|
|
|
def test_a_non_positive_declared_size_is_refused_before_the_stream_is_read() -> None:
|
|
"""`-1 * 40_000_000_000` is NEGATIVE, so `pixels > MAX_IMAGE_PIXELS` was
|
|
false and `check_size` returned silently; 400 MB was then decompressed and
|
|
the refusal came from `encode_png` with `asset_samples_invalid` -- a code
|
|
about the sample buffer for a defect in the declaration.
|
|
|
|
The stream here is CORRUPT, so the order is observable: reading first gives
|
|
`asset_pdf_unsupported`, reading the declaration first gives the new code.
|
|
"""
|
|
pytest.importorskip("pdfplumber")
|
|
document = _bomb(4, payload=b"\xff" * 512).replace(
|
|
b"/Width 4 /Height 4", b"/Width -1 /Height 40000000000"
|
|
)
|
|
assert b"/Width -1" in document
|
|
extracted = extract_document("negative.pdf", document, assets=True)
|
|
assert extracted.images == ()
|
|
assert [rejection.code for rejection in extracted.rejected] == ["asset_size_invalid"]
|
|
|
|
|
|
def test_check_size_refuses_every_non_positive_pair_and_keeps_unknown_unknown() -> None:
|
|
for width, height in ((-1, 40_000_000_000), (0, 10), (10, 0), (-2, -2)):
|
|
with pytest.raises(ExtractionError) as excinfo:
|
|
assets.check_size(width, height, name="n")
|
|
assert excinfo.value.code == "asset_size_invalid"
|
|
# A size the container never declared is UNKNOWN, not invalid: there is no
|
|
# number to bound and inventing one would refuse a legitimate picture.
|
|
assets.check_size(None, None, name="n")
|
|
assets.check_size(None, 10, name="n")
|
|
|
|
|
|
# --- MINOR-3: the bound holds for a file carried verbatim, too ---------------
|
|
|
|
|
|
def _png_header(width: int, height: int) -> bytes:
|
|
"""A PNG whose IHDR declares `width` x `height` and whose body is a stub.
|
|
`read_image` sniffs and reads the header; it never decodes."""
|
|
|
|
def chunk(kind: bytes, payload: bytes) -> bytes:
|
|
return (
|
|
len(payload).to_bytes(4, "big")
|
|
+ kind
|
|
+ payload
|
|
+ zlib.crc32(kind + payload).to_bytes(4, "big")
|
|
)
|
|
|
|
ihdr = width.to_bytes(4, "big") + height.to_bytes(4, "big") + bytes([8, 0, 0, 0, 0])
|
|
return b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", ihdr) + chunk(b"IEND", b"")
|
|
|
|
|
|
def test_an_image_file_over_the_bound_is_refused_although_it_is_never_decoded() -> None:
|
|
"""A 7 000 x 7 000 PNG is 49 MP in 47 705 bytes. This package does not
|
|
decode a carried file, so it pays nothing -- but writing it into a bundle
|
|
hands the consumer the same bomb with `7000x7000 px` printed beside it,
|
|
and the README's first sentence says such an image is refused."""
|
|
with pytest.raises(ExtractionError) as excinfo:
|
|
assets.read_image(_png_header(7000, 7000), name="big.png")
|
|
assert excinfo.value.code == "asset_too_large"
|
|
|
|
|
|
def test_an_image_file_under_the_bound_is_still_read() -> None:
|
|
image = assets.read_image(_png_header(4515, 4128), name="drawing.png")
|
|
assert (image.width, image.height) == (4515, 4128)
|
|
|
|
|
|
# --- MINOR-1 and MINOR-2 of the 18.09 review --------------------------------
|
|
|
|
|
|
def test_a_remote_address_is_never_written_as_a_bare_url() -> None:
|
|
"""`inert` was only half true: the address was written TWICE, once in a
|
|
code span and once bare, and a GFM/linkify renderer autolinks the bare
|
|
one into `<a href="...">`. Measured with `markdown_it('gfm-like')`."""
|
|
line = render_missing(REMOTE, reason="the source is off this machine", href=REMOTE)
|
|
assert "](" not in line
|
|
assert REMOTE in line
|
|
for position in range(len(line)):
|
|
if line.startswith(REMOTE, position):
|
|
assert line[position - 1] == "`" and line[position + len(REMOTE)] == "`", (
|
|
f"a bare occurrence of the address at {position}: {line!r}"
|
|
)
|
|
|
|
|
|
def test_the_caption_of_a_remote_reference_is_still_stated() -> None:
|
|
"""`label` became a dead parameter in 0.10.1, so the alt text or figure
|
|
caption of an image the bundle does not carry was DROPPED -- a regression
|
|
against 0.10.0 and against this module's own reason for writing the line:
|
|
a reader cannot weigh an absence they were never shown."""
|
|
line = render_missing(
|
|
"p.gif", reason="the source is off this machine", label="Figur 84-1 Tverrprofil", href=None
|
|
)
|
|
assert "Figur 84-1 Tverrprofil" in line
|
|
with_href = render_missing(
|
|
REMOTE, reason="the source is off this machine", label="Figur 84-1 Tverrprofil", href=REMOTE
|
|
)
|
|
assert "Figur 84-1 Tverrprofil" in with_href
|
|
|
|
|
|
# --- BLOCKER of the 18.09 PM checkpoint: the CHAIN, not only its first link --
|
|
#
|
|
# `_check_inflated` read `filters[0]`, measured that one link, and returned.
|
|
# A PDF may decode a stream through a chain (`/Filter [/FlateDecode
|
|
# /FlateDecode]`), and the PM's repro measured what that costs on `0f308c1`:
|
|
# 1 606 bytes of stream became 835 MB of peak RSS and 2 839 bytes became
|
|
# 2 439 MB -- linear, about 860 000x the file size, and 400x worse than the
|
|
# single-link bomb the 0.10.1 fix was written for. The image is refused at the
|
|
# end by `check_payload`, after the memory has been spent; one 3 KB document
|
|
# takes a batch build with it.
|
|
#
|
|
# The corpora say the chain is not a hypothetical shape. Over the 5 142 image
|
|
# objects of the 78 PDFs on this machine (measured 2026-09-18), 596 decode
|
|
# through `[/FlateDecode /DCTDecode]`, 580 through `[/FlateDecode
|
|
# /ASCII85Decode]` and 16 through `[/ASCII85Decode /FlateDecode]` -- so a rule
|
|
# that refuses every chain would cost 1 192 real pictures, and a rule that
|
|
# bounds only the first link leaves all 16 of the last group unbounded.
|
|
|
|
|
|
def test_a_chain_of_two_flate_filters_costs_no_more_than_one() -> None:
|
|
"""The PM's repro at the shipped bound, in its own interpreter."""
|
|
pytest.importorskip("pdfplumber")
|
|
size, carried, codes, peak = _run_bomb(BOMB_STREAM_BYTES, links=2)
|
|
assert size < 2 * 1024 * 1024, "the fixture must stay a small file, or it proves nothing"
|
|
assert carried == 0, "a 400 MB stream was carried as a 1x1 picture"
|
|
assert codes == "asset_too_large"
|
|
assert peak < PEAK_RSS_BOUND, f"peak RSS {peak} bytes for a {size}-byte file"
|
|
|
|
|
|
def test_a_chain_of_three_flate_filters_costs_no_more_than_one() -> None:
|
|
"""A chain this package has never seen -- the bound must not be a list of
|
|
the shapes somebody thought of."""
|
|
pytest.importorskip("pdfplumber")
|
|
size, carried, codes, peak = _run_bomb(BOMB_STREAM_BYTES, links=3)
|
|
assert carried == 0
|
|
assert codes == "asset_too_large"
|
|
assert peak < PEAK_RSS_BOUND, f"peak RSS {peak} bytes for a {size}-byte file"
|
|
|
|
|
|
def test_every_link_of_a_flate_chain_is_measured_with_a_patched_bound() -> None:
|
|
"""The same rule, cheap, so it runs on every machine and every suite."""
|
|
pytest.importorskip("pdfplumber")
|
|
payload = _nested_zeros_stream(1 << 20, 2)
|
|
monkey = pytest.MonkeyPatch()
|
|
try:
|
|
monkey.setattr(assets, "MAX_IMAGE_BYTES", 4096)
|
|
extracted = extract_document(
|
|
"nested.pdf",
|
|
_bomb(1, payload=payload, filters="[/FlateDecode /FlateDecode]"),
|
|
assets=True,
|
|
)
|
|
finally:
|
|
monkey.undo()
|
|
assert len(payload) < 4096, "the stream itself is under the bound; only its output is over"
|
|
assert extracted.images == ()
|
|
assert [rejection.code for rejection in extracted.rejected] == ["asset_too_large"]
|
|
# WHICH check fired is the whole point: the backstop refuses this document
|
|
# too, after the memory is spent, so a test that reads only the code is
|
|
# green on the defect. The two refusals say different things.
|
|
assert "decompresses to more than" in extracted.rejected[0].reason, (
|
|
"refused by the backstop after decoding, not by the bound before it: "
|
|
+ extracted.rejected[0].reason
|
|
)
|
|
|
|
|
|
def test_a_flate_link_behind_an_ascii85_link_is_measured_too() -> None:
|
|
"""`[/ASCII85Decode /FlateDecode]` is 16 real objects of the corpora, and
|
|
`filters[0]` is not `FlateDecode`, so the old rule returned without
|
|
measuring anything at all."""
|
|
pytest.importorskip("pdfplumber")
|
|
import base64
|
|
|
|
payload = base64.a85encode(_zeros_stream(1 << 20), adobe=True)
|
|
monkey = pytest.MonkeyPatch()
|
|
try:
|
|
monkey.setattr(assets, "MAX_IMAGE_BYTES", 4096)
|
|
extracted = extract_document(
|
|
"a85.pdf",
|
|
_bomb(1, payload=payload, filters="[/ASCII85Decode /FlateDecode]"),
|
|
assets=True,
|
|
)
|
|
finally:
|
|
monkey.undo()
|
|
assert extracted.images == ()
|
|
assert [rejection.code for rejection in extracted.rejected] == ["asset_too_large"]
|
|
assert "decompresses to more than" in extracted.rejected[0].reason, (
|
|
"refused by the backstop after decoding, not by the bound before it: "
|
|
+ extracted.rejected[0].reason
|
|
)
|
|
|
|
|
|
def test_a_chained_image_within_the_bound_is_still_carried() -> None:
|
|
"""The known-positive beside all of them: the two chains the corpora
|
|
actually hold must still deliver their picture."""
|
|
pytest.importorskip("pdfplumber")
|
|
import base64
|
|
|
|
samples = zlib.compress(b"\x00" * (64 * 64), 9)
|
|
for filters, payload in (
|
|
("[/ASCII85Decode /FlateDecode]", base64.a85encode(samples, adobe=True)),
|
|
("[/FlateDecode]", samples),
|
|
):
|
|
extracted = extract_document(
|
|
"ok.pdf", _bomb(64, payload=payload, filters=filters), assets=True
|
|
)
|
|
assert [r.code for r in extracted.rejected] == [], filters
|
|
assert [(i.width, i.height) for i in extracted.images] == [(64, 64)], filters
|
|
|
|
|
|
def test_a_filter_the_bound_cannot_measure_is_refused_before_it_is_decoded() -> None:
|
|
"""`LZWDecode` and `RunLengthDecode` expand by an amount pdfminer will only
|
|
reveal by producing the whole output, so there is no measuring them a chunk
|
|
at a time. They are refused with a code BEFORE the decode, which is the
|
|
order the stream here makes observable: its bytes are not valid input for
|
|
either filter, so decoding first gives `asset_pdf_unsupported`.
|
|
"""
|
|
pytest.importorskip("pdfplumber")
|
|
for filters in ("/LZWDecode", "/RunLengthDecode", "[/FlateDecode /LZWDecode]"):
|
|
extracted = extract_document(
|
|
"unknown.pdf", _bomb(4, payload=b"\xff" * 512, filters=filters), assets=True
|
|
)
|
|
assert extracted.images == ()
|
|
assert [r.code for r in extracted.rejected] == ["asset_pdf_unbounded"], filters
|
|
|
|
|
|
def test_the_filters_this_package_bounds_are_the_ones_the_corpora_hold() -> None:
|
|
"""The published claim, as a test: these names are what the close-out
|
|
message and the four documentation surfaces say are carried."""
|
|
from llm_ingestion_okf import extract as extract_module
|
|
|
|
carried = extract_module.bounded_pdf_filters()
|
|
assert carried == {
|
|
"FlateDecode",
|
|
"ASCII85Decode",
|
|
"ASCIIHexDecode",
|
|
"DCTDecode",
|
|
"JPXDecode",
|
|
"JBIG2Decode",
|
|
}
|
|
|
|
|
|
def test_an_encrypted_stream_is_deciphered_and_then_bounded() -> None:
|
|
"""`stream.decipher is not None` returned without measuring, so a document
|
|
that declares encryption was a way past the bound. Deciphering does not
|
|
change a stream's length, so this package does what pdfminer's own
|
|
`decode()` does -- decipher first, then measure the filters.
|
|
"""
|
|
pytest.importorskip("pdfplumber")
|
|
from pdfminer.pdftypes import PDFStream
|
|
from pdfminer.psparser import LIT
|
|
|
|
from llm_ingestion_okf import extract as extract_module
|
|
|
|
def rot(objid: int, genno: int, data: bytes, attrs: dict[str, object]) -> bytes:
|
|
return bytes(byte ^ 0x5A for byte in data)
|
|
|
|
raw = _zeros_stream(1 << 20)
|
|
stream = PDFStream(
|
|
{"Width": 1, "Height": 1, "BitsPerComponent": 8, "Filter": LIT("FlateDecode")},
|
|
rot(0, 0, raw, {}),
|
|
decipher=rot,
|
|
)
|
|
stream.set_objid(1, 0)
|
|
monkey = pytest.MonkeyPatch()
|
|
try:
|
|
monkey.setattr(assets, "MAX_IMAGE_BYTES", 4096)
|
|
with pytest.raises(ExtractionError) as excinfo:
|
|
extract_module._check_stream_cost(stream, "encrypted")
|
|
finally:
|
|
monkey.undo()
|
|
assert excinfo.value.code == "asset_too_large"
|
|
|
|
|
|
# --- MAJOR of the 18.09 PM checkpoint: the backstop is covered ---------------
|
|
|
|
|
|
def test_the_backstop_refuses_a_payload_the_stream_bound_never_saw() -> None:
|
|
"""`check_payload(len(data))` after `get_data()` is the counted refusal the
|
|
docstrings, the README, the CHANGELOG and CLAUDE.md all point at -- and
|
|
deleting exactly that line passed the whole suite (2 132 tests) on
|
|
`0f308c1`, because every other test reaches a bound that fires earlier.
|
|
|
|
The path that reaches it is a stream pdfminer has ALREADY decoded: `decode()`
|
|
sets `rawdata` to `None`, so there is no raw stream left to measure and the
|
|
memory is spent before this package is asked anything. The picture is then
|
|
dropped by count rather than by bound, which is the weaker guarantee the
|
|
documents describe -- and nothing held it.
|
|
"""
|
|
pytest.importorskip("pdfplumber")
|
|
from pdfminer.pdftypes import PDFStream
|
|
from pdfminer.psparser import LIT
|
|
|
|
from llm_ingestion_okf import extract as extract_module
|
|
|
|
stream = PDFStream(
|
|
{"Width": 8, "Height": 8, "BitsPerComponent": 8, "Filter": LIT("FlateDecode")},
|
|
zlib.compress(b"\x00" * 4096, 9),
|
|
)
|
|
stream.set_objid(1, 0)
|
|
assert len(stream.get_data()) == 4096
|
|
assert stream.get_rawdata() is None, (
|
|
"the stream must be decoded already, or this proves nothing"
|
|
)
|
|
monkey = pytest.MonkeyPatch()
|
|
try:
|
|
monkey.setattr(assets, "MAX_IMAGE_BYTES", 1024)
|
|
with pytest.raises(ExtractionError) as excinfo:
|
|
extract_module._pdf_image(stream, "decoded")
|
|
finally:
|
|
monkey.undo()
|
|
assert excinfo.value.code == "asset_too_large", (
|
|
"without the backstop this falls through to the encoder and is reported "
|
|
"as a defect of the sample buffer"
|
|
)
|
|
|
|
|
|
def test_a_decoded_stream_under_the_bound_is_still_read() -> None:
|
|
"""The known-positive for the test above: the backstop must not be a rule
|
|
that refuses every already-decoded stream."""
|
|
pytest.importorskip("pdfplumber")
|
|
from pdfminer.pdftypes import PDFStream
|
|
from pdfminer.psparser import LIT
|
|
|
|
from llm_ingestion_okf import extract as extract_module
|
|
|
|
stream = PDFStream(
|
|
{
|
|
"Width": 8,
|
|
"Height": 8,
|
|
"BitsPerComponent": 8,
|
|
"ColorSpace": LIT("DeviceGray"),
|
|
"Filter": LIT("FlateDecode"),
|
|
},
|
|
zlib.compress(b"\x00" * 64, 9),
|
|
)
|
|
stream.set_objid(1, 0)
|
|
stream.get_data()
|
|
image = extract_module._pdf_image(stream, "decoded")
|
|
assert (image.width, image.height) == (8, 8)
|
|
|
|
|
|
# --- BLOCKER of the 18.09 round-3 checkpoint: the COST of a link, not the ---
|
|
# --- size of its output -----------------------------------------------------
|
|
#
|
|
# Round 1 bound what the dictionary DECLARES; the bomb moved into the stream.
|
|
# Round 2 bound the first `FlateDecode`; it moved into the second one. Round 3
|
|
# bound every `FlateDecode` link; it moved into a link this package had called
|
|
# safe. `ASCII85Decode` was documented as bounded "by its own input because it
|
|
# shrinks", and that is false: `z` is ASCII85's shorthand for four zero bytes,
|
|
# so one input byte becomes four. Worse, the DECODER's working set is far
|
|
# larger than either number -- `base64.a85decode` appends one 4-byte object per
|
|
# group to a list -- so the cost is about a hundred bytes of memory per byte of
|
|
# INPUT. Measured on CPython 3.14: 101.4x at 1 MiB of `z`, 96.1x at 4 MiB,
|
|
# 94.5x at 16 MiB. Nothing in the output size says so.
|
|
#
|
|
# Measured 2026-09-18 on the pinned `0c3c490` tree, its own interpreter, idle
|
|
# machine: a 33 475-byte PDF whose image decodes through `/Filter
|
|
# [/FlateDecode /ASCII85Decode]` cost 3 827 003 392 bytes of peak RSS -- about
|
|
# 114 000x the file -- and the picture was CARRIED, with no rejection at all.
|
|
#
|
|
# So the rule under test is not "bound the output of a link" but "bound what
|
|
# decoding a link COSTS", and it has to hold for every chain rather than for
|
|
# the ones a corpus happens to hold.
|
|
|
|
#: The filters pdfminer will decode, by their canonical names, plus `/Crypt`.
|
|
#: The property test below runs EVERY chain of length 1-3 over these.
|
|
DECODED_FILTERS = (
|
|
"FlateDecode",
|
|
"ASCII85Decode",
|
|
"ASCIIHexDecode",
|
|
"DCTDecode",
|
|
"JPXDecode",
|
|
"JBIG2Decode",
|
|
"LZWDecode",
|
|
"RunLengthDecode",
|
|
"CCITTFaxDecode",
|
|
"Crypt",
|
|
)
|
|
|
|
#: What a rejection may be CALLED. A refusal outside this vocabulary is a
|
|
#: defect even when it stops the bomb: a consumer counts these.
|
|
ASSET_REJECTION_CODES = frozenset(
|
|
{
|
|
"asset_too_large",
|
|
"asset_size_invalid",
|
|
"asset_pdf_unbounded",
|
|
"asset_pdf_unsupported",
|
|
"asset_samples_invalid",
|
|
"asset_type_unknown",
|
|
}
|
|
)
|
|
|
|
|
|
def _encode_for(literal: str, data: bytes) -> bytes:
|
|
"""Encode `data` so that DECODING it with `literal` gives `data` back.
|
|
|
|
The filters with no encoder here are the ones this package refuses unread,
|
|
so a chain containing one never reaches a decode at all.
|
|
"""
|
|
import base64
|
|
import binascii
|
|
|
|
if literal in ("FlateDecode", "Fl"):
|
|
return zlib.compress(data, 9)
|
|
if literal in ("ASCII85Decode", "A85"):
|
|
return base64.a85encode(data, adobe=True)
|
|
if literal in ("ASCIIHexDecode", "AHx"):
|
|
return binascii.hexlify(data) + b">"
|
|
return data
|
|
|
|
|
|
def _chain_stream(chain: tuple[str, ...], payload: bytes) -> bytes:
|
|
"""The stream bytes a document must hold for `chain` to decode to
|
|
`payload`. The LAST filter is applied to the payload first.
|
|
|
|
A payload of zeros is the amplifying case for BOTH ends of the chain: it
|
|
deflates to almost nothing, and `a85encode` writes a run of `z`, which is
|
|
the shorthand this bound exists for.
|
|
"""
|
|
for literal in reversed(chain):
|
|
payload = _encode_for(literal, payload)
|
|
return payload
|
|
|
|
|
|
def _filters_entry(chain: tuple[str, ...]) -> str:
|
|
return "[" + " ".join("/" + name for name in chain) + "]"
|
|
|
|
|
|
_CHAIN_CHILD = """
|
|
import resource, sys
|
|
sys.path.insert(0, {tests!r})
|
|
from test_asset_limits import _bomb, _chain_stream, _filters_entry
|
|
from llm_ingestion_okf.extract import extract_document
|
|
|
|
chain = {chain!r}
|
|
document = _bomb(
|
|
1, payload=_chain_stream(chain, b"\\x00" * {payload}), filters=_filters_entry(chain)
|
|
)
|
|
extracted = extract_document("bomb.pdf", document, assets=True)
|
|
peak = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
|
|
print(
|
|
len(document),
|
|
len(extracted.images),
|
|
",".join(r.code for r in extracted.rejected) or "-",
|
|
peak if sys.platform == "darwin" else peak * 1024,
|
|
)
|
|
"""
|
|
|
|
|
|
def _run_chain(chain: tuple[str, ...], *, payload: int) -> tuple[int, int, str, int]:
|
|
"""One chain's bomb in its own interpreter, so peak RSS is ITS peak and not
|
|
the high-water mark of every test that ran before it."""
|
|
completed = subprocess.run(
|
|
[
|
|
sys.executable,
|
|
"-c",
|
|
_CHAIN_CHILD.format(tests=str(Path(__file__).parent), chain=chain, payload=payload),
|
|
],
|
|
capture_output=True,
|
|
text=True,
|
|
check=True,
|
|
)
|
|
size, carried, codes, peak = completed.stdout.split()
|
|
return int(size), int(carried), codes, int(peak)
|
|
|
|
|
|
def test_an_ascii85_link_behind_a_flate_link_is_bounded_too() -> None:
|
|
"""The round-3 BLOCKER at the shipped bound, in its own interpreter."""
|
|
pytest.importorskip("pdfplumber")
|
|
size, carried, codes, peak = _run_chain(
|
|
("FlateDecode", "ASCII85Decode"), payload=128 * 1024 * 1024
|
|
)
|
|
assert size < 2 * 1024 * 1024, "the fixture must stay a small file, or it proves nothing"
|
|
assert carried == 0, "a run of `z` was carried as a 1x1 picture"
|
|
assert codes in ASSET_REJECTION_CODES, codes
|
|
assert peak < PEAK_RSS_BOUND, f"peak RSS {peak} bytes for a {size}-byte file"
|
|
|
|
|
|
def test_an_ascii85_link_on_its_own_is_bounded() -> None:
|
|
"""The same amplification with no filter in front of it: the stream IS the
|
|
run of `z`, so the cost must not be a multiple of the file."""
|
|
pytest.importorskip("pdfplumber")
|
|
size, carried, codes, peak = _run_chain(("ASCII85Decode",), payload=32 * 1024 * 1024)
|
|
assert carried == 0
|
|
assert codes in ASSET_REJECTION_CODES, codes
|
|
assert peak < PEAK_RSS_BOUND, f"peak RSS {peak} bytes for a {size}-byte file"
|
|
|
|
|
|
_COST_CHILD = """
|
|
import base64, resource, sys
|
|
base = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
|
|
data = b"z" * {count}
|
|
out = base64.a85decode(data)
|
|
peak = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
|
|
scale = 1 if sys.platform == "darwin" else 1024
|
|
print((peak - base) * scale / len(data))
|
|
"""
|
|
|
|
|
|
@pytest.mark.parametrize("count", [1 << 20, 4 << 20])
|
|
def test_the_ascii85_cost_ratio_is_not_below_the_one_this_package_measured(count: int) -> None:
|
|
"""The bound rests on a number measured from the DECODER, so the number has
|
|
to stay re-measurable or it rots the first time CPython changes
|
|
`a85decode`. This is the known-negative beside the bound: if the ratio here
|
|
ever rises above the constant, every cap derived from it is too generous
|
|
and this says so before a corpus does.
|
|
"""
|
|
completed = subprocess.run(
|
|
[sys.executable, "-c", _COST_CHILD.format(count=count)],
|
|
capture_output=True,
|
|
text=True,
|
|
check=True,
|
|
)
|
|
measured = float(completed.stdout.strip())
|
|
assert measured > 1.0, "the measurement itself must see the cost, or it proves nothing"
|
|
assert assets.PDF_FILTER_COST_RATIO["ASCII85Decode"] >= measured, (
|
|
f"a85decode costs {measured:.1f} bytes per input byte at {count} bytes of input, "
|
|
f"above the {assets.PDF_FILTER_COST_RATIO['ASCII85Decode']} this package budgets"
|
|
)
|
|
|
|
|
|
def test_the_ascii85_input_cap_stands_above_every_such_stream_in_the_corpora() -> None:
|
|
"""The cap is READ OFF the corpora, the posture `MAX_IMAGE_PIXELS` has.
|
|
Measured 2026-09-18 over the 9 668 image objects of the 77 PDFs on this
|
|
machine: 16 decode through an `ASCII85Decode` link, and the largest input
|
|
to one of them is 450 739 bytes.
|
|
"""
|
|
largest_measured = 450_739
|
|
cap = assets.MAX_FILTER_DECODE_BYTES / assets.PDF_FILTER_COST_RATIO["ASCII85Decode"]
|
|
assert cap > 10 * largest_measured, (
|
|
f"the cap is {cap:.0f} bytes, not an order of magnitude above the "
|
|
f"{largest_measured}-byte stream the corpora hold"
|
|
)
|
|
|
|
|
|
def _all_chains() -> list[tuple[str, ...]]:
|
|
import itertools
|
|
|
|
return [
|
|
tuple(chain)
|
|
for length in (1, 2, 3)
|
|
for chain in itertools.product(DECODED_FILTERS, repeat=length)
|
|
]
|
|
|
|
|
|
ALL_CHAINS = _all_chains()
|
|
|
|
|
|
@pytest.mark.parametrize("fill", [b"\x00", b"\xab"], ids=["zeros", "repeat"])
|
|
def test_no_chain_of_up_to_three_filters_is_carried_over_the_bound(fill: bytes) -> None:
|
|
"""THE PROPERTY, over every chain of length 1-3 the filters pdfminer
|
|
decodes can form -- 1 110 of them -- rather than over the shapes a corpus
|
|
happens to hold.
|
|
|
|
Each chain is given an amplifying payload and a small budget, and the
|
|
requirement is one sentence: the picture is either delivered with its bytes
|
|
under the bound, or refused with a code in the published vocabulary; never
|
|
carried over the bound, and never paid for on the way. `tracemalloc`
|
|
measures the paying, because it counts Python's own allocations, which is
|
|
exactly where `a85decode`'s cost lives.
|
|
"""
|
|
pytest.importorskip("pdfplumber")
|
|
import tracemalloc
|
|
|
|
from pdfminer.pdftypes import PDFStream
|
|
from pdfminer.psparser import LIT
|
|
|
|
from llm_ingestion_okf import extract as extract_module
|
|
|
|
budget = 8 * 1024
|
|
payload = fill * (128 * 1024)
|
|
ceiling = 4 * 1024 * 1024
|
|
carried_over_bound: list[tuple[str, ...]] = []
|
|
wrong_code: list[tuple[tuple[str, ...], str]] = []
|
|
over_ceiling: list[tuple[tuple[str, ...], int]] = []
|
|
monkey = pytest.MonkeyPatch()
|
|
try:
|
|
monkey.setattr(assets, "MAX_IMAGE_BYTES", budget)
|
|
monkey.setattr(assets, "MAX_FILTER_DECODE_BYTES", 2 * budget)
|
|
for chain in ALL_CHAINS:
|
|
stream = PDFStream(
|
|
{
|
|
"Width": 1,
|
|
"Height": 1,
|
|
"BitsPerComponent": 8,
|
|
"Filter": [LIT(name) for name in chain],
|
|
},
|
|
_chain_stream(chain, payload),
|
|
)
|
|
stream.set_objid(1, 0)
|
|
tracemalloc.start()
|
|
try:
|
|
extract_module._check_stream_cost(stream, "bomb")
|
|
data = stream.get_data()
|
|
except ExtractionError as exc:
|
|
if exc.code not in ASSET_REJECTION_CODES:
|
|
wrong_code.append((chain, exc.code))
|
|
except Exception:
|
|
pass # the reader behind the bound reports it in its own words
|
|
else:
|
|
if len(data) > budget:
|
|
carried_over_bound.append(chain)
|
|
finally:
|
|
_, peak = tracemalloc.get_traced_memory()
|
|
tracemalloc.stop()
|
|
if peak > ceiling:
|
|
over_ceiling.append((chain, peak))
|
|
finally:
|
|
monkey.undo()
|
|
assert carried_over_bound == [], (
|
|
f"{len(carried_over_bound)} of {len(ALL_CHAINS)} chains carried over the bound: "
|
|
f"{carried_over_bound[:4]}"
|
|
)
|
|
assert wrong_code == [], f"{len(wrong_code)} of {len(ALL_CHAINS)}: {wrong_code[:4]}"
|
|
assert over_ceiling == [], (
|
|
f"{len(over_ceiling)} of {len(ALL_CHAINS)} chains paid over {ceiling} bytes: "
|
|
f"{over_ceiling[:4]}"
|
|
)
|
|
|
|
|
|
def test_every_bounded_chain_still_carries_a_small_image() -> None:
|
|
"""The known-positive beside the property: over the same chain space,
|
|
restricted to the filters this package bounds, an image UNDER the bound
|
|
must still be delivered. A rule that refuses everything passes the property
|
|
above and is worthless."""
|
|
pytest.importorskip("pdfplumber")
|
|
import itertools
|
|
|
|
from pdfminer.pdftypes import PDFStream
|
|
from pdfminer.psparser import LIT
|
|
|
|
from llm_ingestion_okf import extract as extract_module
|
|
|
|
bounded = tuple(sorted(extract_module.bounded_pdf_filters()))
|
|
chains = [
|
|
tuple(chain) for length in (1, 2, 3) for chain in itertools.product(bounded, repeat=length)
|
|
]
|
|
refused: list[tuple[tuple[str, ...], str]] = []
|
|
for chain in chains:
|
|
stream = PDFStream(
|
|
{
|
|
"Width": 8,
|
|
"Height": 8,
|
|
"BitsPerComponent": 8,
|
|
"Filter": [LIT(name) for name in chain],
|
|
},
|
|
_chain_stream(chain, b"\x00" * 64),
|
|
)
|
|
stream.set_objid(1, 0)
|
|
try:
|
|
extract_module._check_stream_cost(stream, "small")
|
|
except ExtractionError as exc:
|
|
refused.append((chain, exc.code))
|
|
assert refused == [], f"{len(refused)} of {len(chains)} bounded chains refused a 64-byte image"
|
|
|
|
|
|
def test_the_stream_bound_refuses_before_get_data_is_ever_called() -> None:
|
|
"""MAJOR of the 18.09 checkpoint: `check_payload` at the END of
|
|
`_check_stream_cost` is what refuses a stream no filter in the chain
|
|
expands -- an unfiltered one, or one behind `DCTDecode` -- and deleting
|
|
that line passed the whole suite, because the SECOND `check_payload`, after
|
|
`get_data()`, produces the same code and the same words one step later.
|
|
|
|
A test reading the code cannot tell the two apart. What separates them is
|
|
whether the payment was made, so this one asserts `get_data` was never
|
|
called.
|
|
"""
|
|
pytest.importorskip("pdfplumber")
|
|
from pdfminer.pdftypes import PDFStream
|
|
|
|
from llm_ingestion_okf import extract as extract_module
|
|
|
|
stream = PDFStream({"Width": 1, "Height": 1, "BitsPerComponent": 8}, b"\x00" * 4096)
|
|
stream.set_objid(1, 0)
|
|
calls: list[int] = []
|
|
original = stream.get_data
|
|
|
|
def spy() -> bytes:
|
|
calls.append(1)
|
|
return original()
|
|
|
|
stream.get_data = spy # type: ignore[method-assign]
|
|
monkey = pytest.MonkeyPatch()
|
|
try:
|
|
monkey.setattr(assets, "MAX_IMAGE_BYTES", 1024)
|
|
with pytest.raises(ExtractionError) as excinfo:
|
|
extract_module._check_stream_cost(stream, "unfiltered")
|
|
finally:
|
|
monkey.undo()
|
|
assert excinfo.value.code == "asset_too_large"
|
|
assert calls == [], "refused only after the stream was decoded, which is the backstop"
|