Kjell Tore Guttormsen
88cf67f12e
Three STS fixtures still carried the section titles and labels of one real reference document, and three identifiers were copies of its codes with a letter or a word swapped. They now describe an invented kitchen counter and cookbook series: the titles, labels and descriptions of sts-identity.xml, sts-inherit.xml and sts-empty-label.xml, the P350/P351 document codes, the 99-0001 delivery prefix and chapter 7 of the image and accounting corpora. Generated fixtures are regenerated and the witness inventory's per-document totals are identical before and after; only names and text move. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
401 lines
16 KiB
Python
401 lines
16 KiB
Python
"""Binary assets: the layer that lets a bundle carry an image at all.
|
|
|
|
Until 0.10.0 no reader in this package fetched, named, described or copied a
|
|
single image, and the only writer into a bundle was
|
|
`materialize.write_bytes(bundle_dir, name, content: str)` -- UTF-8, text, no
|
|
binary path anywhere. A document whose table is a raster picture therefore
|
|
reached a concept as an absence with no denominator: measured on a long
|
|
reference standard, its text is carried in full while 12 `Tabell N-N` and 9
|
|
`Figur N-N` captions stand over nothing, and a section saying "... er gitt i
|
|
tabell 7-2" stands above an empty space.
|
|
|
|
THIS MODULE IS THE ONE PLACE THAT DECIDES WHAT AN IMAGE IS. Every reader hands
|
|
it bytes and gets back either a carried image or a coded rejection, so a
|
|
format's own quirks never reach the bundle layout, and "N images carried of M
|
|
found" has one definition for `pdf`, `docx`, `html` and `xml` alike.
|
|
|
|
Three properties are pinned here because a bundle is downstream of all of them:
|
|
|
|
- **the type is SNIFFED, never claimed.** A `.jpg` that is really a PNG would
|
|
otherwise be written under a name whose extension lies, and a consumer that
|
|
dispatches on the extension reads it wrong with full confidence.
|
|
- **the name is CONTENT-ADDRESSED.** `<sha256[:12]>-<reduced original name>`
|
|
makes two drops of one image one file, and makes a rebuild of the same
|
|
corpus produce the same bundle -- the byte-determinism rule this package
|
|
already holds for text.
|
|
- **the pointer is one GRAMMAR.** Step 2 (`okf describe`) has to find every
|
|
pointer mechanically in order to write a transcription under it, so the
|
|
block is a regex this module owns rather than a shape each reader invents.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import re
|
|
import zlib
|
|
|
|
import pytest
|
|
|
|
from llm_ingestion_okf import assets
|
|
from llm_ingestion_okf.errors import ExtractionError
|
|
|
|
# --- hand-laid image bytes -------------------------------------------------
|
|
#
|
|
# Written out here rather than committed as files: every byte is visible in the
|
|
# test that depends on it, and the readers below are header readers, so a
|
|
# header is the whole input they have. The PNG is a REAL image (zlib from the
|
|
# stdlib); the JPEG is a structurally valid header sequence and not a decodable
|
|
# photograph, which is exactly what the JPEG path needs -- it copies the bytes
|
|
# through and reads nothing but the SOF marker.
|
|
|
|
|
|
def _png(width: int, height: int) -> bytes:
|
|
def chunk(kind: bytes, payload: bytes) -> bytes:
|
|
return (
|
|
len(payload).to_bytes(4, "big")
|
|
+ kind
|
|
+ payload
|
|
+ zlib.crc32(kind + payload).to_bytes(4, "big")
|
|
)
|
|
|
|
ihdr = width.to_bytes(4, "big") + height.to_bytes(4, "big") + bytes([8, 0, 0, 0, 0])
|
|
raw = b"".join(b"\x00" + bytes([0x40] * width) for _ in range(height))
|
|
return (
|
|
b"\x89PNG\r\n\x1a\n"
|
|
+ chunk(b"IHDR", ihdr)
|
|
+ chunk(b"IDAT", zlib.compress(raw, 9))
|
|
+ chunk(b"IEND", b"")
|
|
)
|
|
|
|
|
|
def _jpeg(width: int, height: int) -> bytes:
|
|
sof = bytes([8, height >> 8, height & 0xFF, width >> 8, width & 0xFF, 1, 1, 0x11, 0])
|
|
return (
|
|
b"\xff\xd8"
|
|
b"\xff\xe0\x00\x10JFIF\x00\x01\x01\x00\x00\x01\x00\x01\x00\x00"
|
|
+ b"\xff\xc0"
|
|
+ (len(sof) + 2).to_bytes(2, "big")
|
|
+ sof
|
|
+ b"\xff\xd9"
|
|
)
|
|
|
|
|
|
def _gif(width: int, height: int) -> bytes:
|
|
return (
|
|
b"GIF89a"
|
|
+ width.to_bytes(2, "little")
|
|
+ height.to_bytes(2, "little")
|
|
+ b"\x00\x00\x00"
|
|
+ b";"
|
|
)
|
|
|
|
|
|
def _bmp(width: int, height: int) -> bytes:
|
|
header = b"BM" + (54).to_bytes(4, "little") + b"\x00" * 4 + (54).to_bytes(4, "little")
|
|
dib = (
|
|
(40).to_bytes(4, "little")
|
|
+ width.to_bytes(4, "little", signed=True)
|
|
+ height.to_bytes(4, "little", signed=True)
|
|
+ b"\x01\x00\x18\x00"
|
|
+ b"\x00" * 24
|
|
)
|
|
return header + dib
|
|
|
|
|
|
# --- the type is sniffed, never claimed ------------------------------------
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("data", "media_type", "suffix"),
|
|
[
|
|
(_png(4, 3), "image/png", ".png"),
|
|
(_jpeg(360, 269), "image/jpeg", ".jpg"),
|
|
(_gif(7, 5), "image/gif", ".gif"),
|
|
(_bmp(9, 2), "image/bmp", ".bmp"),
|
|
],
|
|
)
|
|
def test_sniff_reads_the_bytes(data: bytes, media_type: str, suffix: str) -> None:
|
|
assert assets.sniff(data) == (media_type, suffix)
|
|
|
|
|
|
def test_sniff_refuses_what_is_not_an_image() -> None:
|
|
assert assets.sniff(b"%PDF-1.7\n") is None
|
|
assert assets.sniff(b"") is None
|
|
|
|
|
|
def test_the_claimed_extension_never_decides() -> None:
|
|
"""A PNG named `.jpg` is carried as a PNG, under a `.png` name."""
|
|
image = assets.read_image(_png(4, 3), name="tabell-7-2.jpg")
|
|
assert image.media_type == "image/png"
|
|
assert assets.asset_name(image).endswith(".png")
|
|
|
|
|
|
def test_unknown_bytes_are_a_coded_rejection_not_a_crash() -> None:
|
|
with pytest.raises(ExtractionError) as excinfo:
|
|
assets.read_image(b"not an image at all", name="x.png")
|
|
assert excinfo.value.code == "asset_type_unknown"
|
|
|
|
|
|
# --- dimensions, from the header of each format ----------------------------
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("data", "size"),
|
|
[
|
|
(_png(4, 3), (4, 3)),
|
|
(_jpeg(360, 269), (360, 269)),
|
|
(_gif(7, 5), (7, 5)),
|
|
(_bmp(9, 2), (9, 2)),
|
|
],
|
|
)
|
|
def test_dimensions_from_the_header(data: bytes, size: tuple[int, int]) -> None:
|
|
assert assets.dimensions(data) == size
|
|
|
|
|
|
def test_dimensions_absent_is_absent_never_zero() -> None:
|
|
"""A format whose size this module does not read says so, rather than 0x0.
|
|
|
|
`0x0 px` in a concept is a measurement nobody took, printed as a fact.
|
|
|
|
The vehicle used to be a TIFF. Since the viewable-asset round a TIFF never reaches
|
|
`read_image`'s return at all -- it is a format no model can be shown and
|
|
there is no lossless conversion for it, so it is refused as
|
|
`asset_not_viewable`. The property still has a reachable case, and this is
|
|
it: a JPEG whose frame header never arrives. `sniff` types it from the
|
|
first three bytes, `_jpeg_dimensions` walks to the end and finds no SOF,
|
|
and the pointer has to say so rather than print a size.
|
|
"""
|
|
headless = b"\xff\xd8\xff\xfe\x00\x04ab\xff\xd9"
|
|
assert assets.sniff(headless) == ("image/jpeg", ".jpg")
|
|
assert assets.dimensions(headless) is None
|
|
image = assets.read_image(headless, name="scan.jpg")
|
|
assert image.width is None and image.height is None
|
|
assert "dimensions unknown" in assets.render_block(image)
|
|
|
|
|
|
# --- the name is content-addressed -----------------------------------------
|
|
|
|
|
|
def test_asset_name_is_digest_plus_a_reduced_original() -> None:
|
|
data = _jpeg(360, 269)
|
|
image = assets.read_image(data, name="99-0001 - Tabeller - Oppskriftsboka (P350-P351).jpg")
|
|
name = assets.asset_name(image)
|
|
assert name.startswith(hashlib.sha256(data).hexdigest()[: assets.DIGEST_PREFIX] + "-")
|
|
assert name.endswith(".jpg")
|
|
# The link target sits inside `](...)`, which a space or a parenthesis
|
|
# terminates -- and `structure._LINK` reads exactly that grammar. A name
|
|
# carrying either would produce a pointer no reader can follow.
|
|
assert re.fullmatch(r"[a-z0-9][a-z0-9.-]*", name), name
|
|
|
|
|
|
def test_two_drops_of_one_image_are_one_asset() -> None:
|
|
data = _png(4, 3)
|
|
first = assets.read_image(data, name="figur.png")
|
|
second = assets.read_image(data, name="figur.png")
|
|
assert assets.asset_name(first) == assets.asset_name(second)
|
|
|
|
|
|
def test_one_name_over_two_contents_stays_two_assets() -> None:
|
|
a = assets.read_image(_png(4, 3), name="figur.png")
|
|
b = assets.read_image(_png(5, 3), name="figur.png")
|
|
assert assets.asset_name(a) != assets.asset_name(b)
|
|
|
|
|
|
def test_a_nameless_image_still_gets_a_name() -> None:
|
|
image = assets.read_image(_png(4, 3), name="")
|
|
assert re.fullmatch(r"[0-9a-f]{12}\.png", assets.asset_name(image))
|
|
|
|
|
|
# --- the pointer is one grammar --------------------------------------------
|
|
|
|
|
|
def test_block_names_the_original_the_size_and_the_label() -> None:
|
|
image = assets.read_image(
|
|
_jpeg(360, 269),
|
|
name="99-0001 - Tabeller - Oppskriftsboka (P350-P351).jpg",
|
|
label="Tabell 7-2",
|
|
)
|
|
block = assets.render_block(image)
|
|
first, second = block.split("\n")
|
|
assert first == f"})"
|
|
assert second.startswith("Image: 99-0001 - Tabeller - Oppskriftsboka (P350-P351).jpg")
|
|
assert "360x269 px" in second
|
|
assert second.endswith("Tabell 7-2")
|
|
|
|
|
|
def test_the_label_falls_back_to_the_original_name() -> None:
|
|
"""Two of the four readers have no caption element at all.
|
|
|
|
A PDF image object and an STS `<graphic>` carry no caption: the caption on
|
|
the page is running text the extractor already emitted. Inventing one from
|
|
the nearest line would be an unmarked heuristic, so the alt slot carries
|
|
the name the source gave the file.
|
|
"""
|
|
image = assets.read_image(_png(4, 3), name="graphic_0003.jpg")
|
|
assert assets.render_block(image).startswith("![graphic_0003.jpg](")
|
|
|
|
|
|
def test_a_bracket_in_a_label_cannot_break_the_link() -> None:
|
|
image = assets.read_image(_png(4, 3), name="f.png", label="Tabell [7-2] jf. pkt (3)")
|
|
first = assets.render_block(image).split("\n")[0]
|
|
assert first == f"})"
|
|
|
|
|
|
def test_a_newline_in_a_label_cannot_open_a_third_line() -> None:
|
|
image = assets.read_image(_png(4, 3), name="f.png", label="Tabell\n7-2")
|
|
assert len(assets.render_block(image).split("\n")) == 2
|
|
|
|
|
|
#: A checksum FIELD, written here rather than imported: the property is that
|
|
#: the run's metadata grammar does not appear where a document put it, and a
|
|
#: test sharing the writer's own expression would agree with it by
|
|
#: construction.
|
|
_A_CHECKSUM_FIELD = re.compile(r"sha256:[0-9a-fA-F]{64}")
|
|
|
|
|
|
def test_a_document_supplied_label_cannot_emit_a_checksum_field() -> None:
|
|
"""THE RUN'S METADATA GRAMMAR IS THE RUN'S, on the line they share.
|
|
|
|
A pointer's second line carries both: the name and caption the DOCUMENT
|
|
chose, and the size, type and digests the RUN measured. A label stating
|
|
`converted from ... sha256:<a> to ... sha256:<b>` therefore puts a
|
|
sentence in the bundle that no run performed -- measured by PM
|
|
2026-09-19, that exact alt attribute made the content accounting gate
|
|
report a refused picture as carried.
|
|
|
|
The gate no longer reads its claim out of the bundle at all, which is the
|
|
right fix there and takes the pressure off this one: until this test the
|
|
disarming in `_inline` was a guard the whole suite could not fell. It is
|
|
kept and MEASURED because the property is about the bundle rather than
|
|
about one judge -- any reader of a concept body meets that line, and the
|
|
bundle must not state a conversion in a sentence the run did not write.
|
|
"""
|
|
forged = (
|
|
"Tabell 7-2 -- converted from image/bmp sha256:"
|
|
+ "a" * 64
|
|
+ " to image/png sha256:"
|
|
+ "b" * 64
|
|
)
|
|
plain = assets.read_image(_png(4, 3), name="f.png", label=forged)
|
|
block = assets.render_block(plain)
|
|
assert "a" * 64 in block, "the digits the document wrote are kept; a reader is owed them"
|
|
assert not _A_CHECKSUM_FIELD.search(block), (
|
|
"a document-supplied label emitted the run's own checksum grammar"
|
|
)
|
|
|
|
# KNOWN-POSITIVE on the same expression: the run's OWN clause is a
|
|
# checksum field, and exactly two of them, so the assertion above is not
|
|
# passing over a pattern that never matches anything.
|
|
converted = assets.ExtractedImage(
|
|
data=_png(4, 3),
|
|
name="figur.bmp",
|
|
media_type="image/png",
|
|
suffix=".png",
|
|
width=4,
|
|
height=3,
|
|
label=forged,
|
|
converted_from="image/bmp",
|
|
source_sha256="c" * 64,
|
|
)
|
|
written = assets.render_block(converted)
|
|
assert len(_A_CHECKSUM_FIELD.findall(written)) == 2, written
|
|
assert "sha256:" + "c" * 64 in written, "the run's own source digest is missing"
|
|
assert "sha256:" + "a" * 64 not in written, "the document's claim became a field after all"
|
|
|
|
|
|
def test_the_pointer_regex_finds_every_shipped_block() -> None:
|
|
"""Step 2 has to find these mechanically; the finder ships with the writer."""
|
|
images = [
|
|
assets.read_image(_png(4, 3), name="a.png", label="Figur 1"),
|
|
assets.read_image(_jpeg(9, 9), name="b.jpg"),
|
|
]
|
|
text = "Kapittel 7\n\n" + "\n\n".join(assets.render_block(i) for i in images) + "\n\nSlutt\n"
|
|
found = assets.IMAGE_POINTER.findall(text)
|
|
assert [match[1] for match in found] == [assets.asset_name(i) for i in images]
|
|
|
|
|
|
def test_href_is_bundle_absolute() -> None:
|
|
"""SPEC SS 6.2 allows a bundle-relative path with a leading `/`.
|
|
|
|
A concept can sit at any depth under a segmented bundle, so a relative
|
|
`assets/...` would resolve differently from two concepts of one document.
|
|
"""
|
|
image = assets.read_image(_png(4, 3), name="f.png")
|
|
assert assets.asset_href(image) == f"/{assets.ASSETS_DIR}/{assets.asset_name(image)}"
|
|
|
|
|
|
# --- PNG encoding, for samples that arrive without a container -------------
|
|
#
|
|
# A PDF image is usually not a file: `FlateDecode` hands back raw samples with
|
|
# the colour model in the dictionary beside them. Encoding those is the only
|
|
# way to carry them at all, and it is stdlib (`zlib`), so the `pdf` row does
|
|
# not gain a dependency and the output is not bound to a renderer's version.
|
|
|
|
|
|
def test_png_from_gray_samples_round_trips_the_header() -> None:
|
|
encoded = assets.encode_png(3, 2, bytes([0, 64, 128, 192, 255, 32]), channels=1)
|
|
assert assets.sniff(encoded) == ("image/png", ".png")
|
|
assert assets.dimensions(encoded) == (3, 2)
|
|
|
|
|
|
def test_png_from_rgb_samples_round_trips_the_header() -> None:
|
|
encoded = assets.encode_png(2, 1, bytes(range(6)), channels=3)
|
|
assert assets.dimensions(encoded) == (2, 1)
|
|
|
|
|
|
def test_png_from_indexed_samples_carries_the_palette() -> None:
|
|
palette = bytes([255, 0, 0, 0, 255, 0])
|
|
encoded = assets.encode_png(2, 1, bytes([0, 1]), channels=1, palette=palette)
|
|
assert assets.dimensions(encoded) == (2, 1)
|
|
assert b"PLTE" in encoded
|
|
|
|
|
|
def test_png_refuses_a_sample_count_that_does_not_fit() -> None:
|
|
"""Refused rather than padded: a short buffer is a misread dictionary."""
|
|
with pytest.raises(ExtractionError) as excinfo:
|
|
assets.encode_png(4, 4, b"\x00\x01", channels=1)
|
|
assert excinfo.value.code == "asset_samples_invalid"
|
|
|
|
|
|
def test_png_encoding_is_byte_stable() -> None:
|
|
first = assets.encode_png(3, 2, bytes([0, 64, 128, 192, 255, 32]), channels=1)
|
|
second = assets.encode_png(3, 2, bytes([0, 64, 128, 192, 255, 32]), channels=1)
|
|
assert first == second
|
|
|
|
|
|
def test_the_restated_reduction_is_materializes_own() -> None:
|
|
"""`assets` cannot import `materialize`, so the rule is held equal instead.
|
|
|
|
`extract.py`'s registry must not import the contract layer (its own
|
|
docstring says so, and `materialize` reaches `manifest` and `profiles`), so
|
|
the id-grammar reduction is written out a second time. This is what stops
|
|
the second copy becoming a second grammar.
|
|
"""
|
|
from llm_ingestion_okf.materialize import reduce_to_id_grammar
|
|
|
|
for value in (
|
|
"99-0001 - Tabeller - Oppskriftsboka (P350-P351)",
|
|
"Figur 11.1 Hevetidskrav",
|
|
"grafikk_med_æøå",
|
|
"---",
|
|
"",
|
|
):
|
|
assert assets._reduce(value) == reduce_to_id_grammar(value), value
|
|
|
|
|
|
def test_one_image_reached_by_two_paths_is_one_asset() -> None:
|
|
"""The asset name reads the BASENAME, never the path the document used.
|
|
|
|
Measured on the fixture inbox before this rule existed: the HTML document
|
|
points at `graphics/figur-7-1.png` and the STS document at
|
|
`figur-7-1.png` (resolved through the `graphics/` sibling convention), so
|
|
one image was written twice, under two names, from one run -- with the
|
|
digest in both of them announcing that the bytes were identical. The path a
|
|
document happened to use is not a property of the picture.
|
|
"""
|
|
data = _png(6, 4)
|
|
through_directory = assets.read_image(data, name="graphics/figur-7-1.png")
|
|
bare = assets.read_image(data, name="figur-7-1.png")
|
|
assert assets.asset_name(through_directory) == assets.asset_name(bare)
|
|
# The full original is not lost -- it moves to the line a person reads.
|
|
assert "graphics/figur-7-1.png" in assets.render_block(through_directory)
|