llm-ingestion-okf/tests/test_assets.py

"""Binary assets: the layer that lets a bundle carry an image at all.

Until 0.10.0 no reader in this package fetched, named, described or copied a
single image, and the only writer into a bundle was
`materialize.write_bytes(bundle_dir, name, content: str)` -- UTF-8, text, no
binary path anywhere. A document whose table is a raster picture therefore
reached a concept as an absence with no denominator: measured on R761
Prosesskoden:2025, the process text is carried in full while 12 `Tabell N-N`
and 9 `Figur N-N` captions stand over nothing, and process 84 says "toleranse-
klasse ... er gitt i tabell 84-2" above an empty space.

THIS MODULE IS THE ONE PLACE THAT DECIDES WHAT AN IMAGE IS. Every reader hands
it bytes and gets back either a carried image or a coded rejection, so a
format's own quirks never reach the bundle layout, and "N images carried of M
found" has one definition for `pdf`, `docx`, `html` and `xml` alike.

Three properties are pinned here because a bundle is downstream of all of them:

- **the type is SNIFFED, never claimed.** A `.jpg` that is really a PNG would
  otherwise be written under a name whose extension lies, and a consumer that
  dispatches on the extension reads it wrong with full confidence.
- **the name is CONTENT-ADDRESSED.** `<sha256[:12]>-<reduced original name>`
  makes two drops of one image one file, and makes a rebuild of the same
  corpus produce the same bundle -- the byte-determinism rule this package
  already holds for text.
- **the pointer is one GRAMMAR.** Step 2 (`okf describe`) has to find every
  pointer mechanically in order to write a transcription under it, so the
  block is a regex this module owns rather than a shape each reader invents.
"""

from __future__ import annotations

import hashlib
import re
import zlib

import pytest

from llm_ingestion_okf import assets
from llm_ingestion_okf.errors import ExtractionError

# --- hand-laid image bytes -------------------------------------------------
#
# Written out here rather than committed as files: every byte is visible in the
# test that depends on it, and the readers below are header readers, so a
# header is the whole input they have. The PNG is a REAL image (zlib from the
# stdlib); the JPEG is a structurally valid header sequence and not a decodable
# photograph, which is exactly what the JPEG path needs -- it copies the bytes
# through and reads nothing but the SOF marker.


def _png(width: int, height: int) -> bytes:
    def chunk(kind: bytes, payload: bytes) -> bytes:
        return (
            len(payload).to_bytes(4, "big")
            + kind
            + payload
            + zlib.crc32(kind + payload).to_bytes(4, "big")
        )

    ihdr = width.to_bytes(4, "big") + height.to_bytes(4, "big") + bytes([8, 0, 0, 0, 0])
    raw = b"".join(b"\x00" + bytes([0x40] * width) for _ in range(height))
    return (
        b"\x89PNG\r\n\x1a\n"
        + chunk(b"IHDR", ihdr)
        + chunk(b"IDAT", zlib.compress(raw, 9))
        + chunk(b"IEND", b"")
    )


def _jpeg(width: int, height: int) -> bytes:
    sof = bytes([8, height >> 8, height & 0xFF, width >> 8, width & 0xFF, 1, 1, 0x11, 0])
    return (
        b"\xff\xd8"
        b"\xff\xe0\x00\x10JFIF\x00\x01\x01\x00\x00\x01\x00\x01\x00\x00"
        + b"\xff\xc0"
        + (len(sof) + 2).to_bytes(2, "big")
        + sof
        + b"\xff\xd9"
    )


def _gif(width: int, height: int) -> bytes:
    return (
        b"GIF89a"
        + width.to_bytes(2, "little")
        + height.to_bytes(2, "little")
        + b"\x00\x00\x00"
        + b";"
    )


def _bmp(width: int, height: int) -> bytes:
    header = b"BM" + (54).to_bytes(4, "little") + b"\x00" * 4 + (54).to_bytes(4, "little")
    dib = (
        (40).to_bytes(4, "little")
        + width.to_bytes(4, "little", signed=True)
        + height.to_bytes(4, "little", signed=True)
        + b"\x01\x00\x18\x00"
        + b"\x00" * 24
    )
    return header + dib


# --- the type is sniffed, never claimed ------------------------------------


@pytest.mark.parametrize(
    ("data", "media_type", "suffix"),
    [
        (_png(4, 3), "image/png", ".png"),
        (_jpeg(360, 269), "image/jpeg", ".jpg"),
        (_gif(7, 5), "image/gif", ".gif"),
        (_bmp(9, 2), "image/bmp", ".bmp"),
    ],
)
def test_sniff_reads_the_bytes(data: bytes, media_type: str, suffix: str) -> None:
    assert assets.sniff(data) == (media_type, suffix)


def test_sniff_refuses_what_is_not_an_image() -> None:
    assert assets.sniff(b"%PDF-1.7\n") is None
    assert assets.sniff(b"") is None


def test_the_claimed_extension_never_decides() -> None:
    """A PNG named `.jpg` is carried as a PNG, under a `.png` name."""
    image = assets.read_image(_png(4, 3), name="tabell-84-2.jpg")
    assert image.media_type == "image/png"
    assert assets.asset_name(image).endswith(".png")


def test_unknown_bytes_are_a_coded_rejection_not_a_crash() -> None:
    with pytest.raises(ExtractionError) as excinfo:
        assets.read_image(b"not an image at all", name="x.png")
    assert excinfo.value.code == "asset_type_unknown"


# --- dimensions, from the header of each format ----------------------------


@pytest.mark.parametrize(
    ("data", "size"),
    [
        (_png(4, 3), (4, 3)),
        (_jpeg(360, 269), (360, 269)),
        (_gif(7, 5), (7, 5)),
        (_bmp(9, 2), (9, 2)),
    ],
)
def test_dimensions_from_the_header(data: bytes, size: tuple[int, int]) -> None:
    assert assets.dimensions(data) == size


def test_dimensions_absent_is_absent_never_zero() -> None:
    """A format whose size this module does not read says so, rather than 0x0.

    `0x0 px` in a concept is a measurement nobody took, printed as a fact.

    The vehicle used to be a TIFF. Since the viewable-asset round a TIFF never reaches
    `read_image`'s return at all -- it is a format no model can be shown and
    there is no lossless conversion for it, so it is refused as
    `asset_not_viewable`. The property still has a reachable case, and this is
    it: a JPEG whose frame header never arrives. `sniff` types it from the
    first three bytes, `_jpeg_dimensions` walks to the end and finds no SOF,
    and the pointer has to say so rather than print a size.
    """
    headless = b"\xff\xd8\xff\xfe\x00\x04ab\xff\xd9"
    assert assets.sniff(headless) == ("image/jpeg", ".jpg")
    assert assets.dimensions(headless) is None
    image = assets.read_image(headless, name="scan.jpg")
    assert image.width is None and image.height is None
    assert "dimensions unknown" in assets.render_block(image)


# --- the name is content-addressed -----------------------------------------


def test_asset_name_is_digest_plus_a_reduced_original() -> None:
    data = _jpeg(360, 269)
    image = assets.read_image(data, name="25-0143 - Tabeller - Prosesskoden (R761-R762).jpg")
    name = assets.asset_name(image)
    assert name.startswith(hashlib.sha256(data).hexdigest()[: assets.DIGEST_PREFIX] + "-")
    assert name.endswith(".jpg")
    # The link target sits inside `](...)`, which a space or a parenthesis
    # terminates -- and `structure._LINK` reads exactly that grammar. A name
    # carrying either would produce a pointer no reader can follow.
    assert re.fullmatch(r"[a-z0-9][a-z0-9.-]*", name), name


def test_two_drops_of_one_image_are_one_asset() -> None:
    data = _png(4, 3)
    first = assets.read_image(data, name="figur.png")
    second = assets.read_image(data, name="figur.png")
    assert assets.asset_name(first) == assets.asset_name(second)


def test_one_name_over_two_contents_stays_two_assets() -> None:
    a = assets.read_image(_png(4, 3), name="figur.png")
    b = assets.read_image(_png(5, 3), name="figur.png")
    assert assets.asset_name(a) != assets.asset_name(b)


def test_a_nameless_image_still_gets_a_name() -> None:
    image = assets.read_image(_png(4, 3), name="")
    assert re.fullmatch(r"[0-9a-f]{12}\.png", assets.asset_name(image))


# --- the pointer is one grammar --------------------------------------------


def test_block_names_the_original_the_size_and_the_label() -> None:
    image = assets.read_image(
        _jpeg(360, 269),
        name="25-0143 - Tabeller - Prosesskoden (R761-R762).jpg",
        label="Tabell 84-2",
    )
    block = assets.render_block(image)
    first, second = block.split("\n")
    assert first == f"![Tabell 84-2]({assets.asset_href(image)})"
    assert second.startswith("Image: 25-0143 - Tabeller - Prosesskoden (R761-R762).jpg")
    assert "360x269 px" in second
    assert second.endswith("Tabell 84-2")


def test_the_label_falls_back_to_the_original_name() -> None:
    """Two of the four readers have no caption element at all.

    A PDF image object and an STS `<graphic>` carry no caption: the caption on
    the page is running text the extractor already emitted. Inventing one from
    the nearest line would be an unmarked heuristic, so the alt slot carries
    the name the source gave the file.
    """
    image = assets.read_image(_png(4, 3), name="graphic_0003.jpg")
    assert assets.render_block(image).startswith("![graphic_0003.jpg](")


def test_a_bracket_in_a_label_cannot_break_the_link() -> None:
    image = assets.read_image(_png(4, 3), name="f.png", label="Tabell [84-2] jf. pkt (3)")
    first = assets.render_block(image).split("\n")[0]
    assert first == f"![Tabell (84-2) jf. pkt (3)]({assets.asset_href(image)})"


def test_a_newline_in_a_label_cannot_open_a_third_line() -> None:
    image = assets.read_image(_png(4, 3), name="f.png", label="Tabell\n84-2")
    assert len(assets.render_block(image).split("\n")) == 2


#: A checksum FIELD, written here rather than imported: the property is that
#: the run's metadata grammar does not appear where a document put it, and a
#: test sharing the writer's own expression would agree with it by
#: construction.
_A_CHECKSUM_FIELD = re.compile(r"sha256:[0-9a-fA-F]{64}")


def test_a_document_supplied_label_cannot_emit_a_checksum_field() -> None:
    """THE RUN'S METADATA GRAMMAR IS THE RUN'S, on the line they share.

    A pointer's second line carries both: the name and caption the DOCUMENT
    chose, and the size, type and digests the RUN measured. A label stating
    `converted from ... sha256:<a> to ... sha256:<b>` therefore puts a
    sentence in the bundle that no run performed -- measured by PM
    2026-09-19, that exact alt attribute made the content accounting gate
    report a refused picture as carried.

    The gate no longer reads its claim out of the bundle at all, which is the
    right fix there and takes the pressure off this one: until this test the
    disarming in `_inline` was a guard the whole suite could not fell. It is
    kept and MEASURED because the property is about the bundle rather than
    about one judge -- any reader of a concept body meets that line, and the
    bundle must not state a conversion in a sentence the run did not write.
    """
    forged = (
        "Tabell 84-2 -- converted from image/bmp sha256:"
        + "a" * 64
        + " to image/png sha256:"
        + "b" * 64
    )
    plain = assets.read_image(_png(4, 3), name="f.png", label=forged)
    block = assets.render_block(plain)
    assert "a" * 64 in block, "the digits the document wrote are kept; a reader is owed them"
    assert not _A_CHECKSUM_FIELD.search(block), (
        "a document-supplied label emitted the run's own checksum grammar"
    )

    # KNOWN-POSITIVE on the same expression: the run's OWN clause is a
    # checksum field, and exactly two of them, so the assertion above is not
    # passing over a pattern that never matches anything.
    converted = assets.ExtractedImage(
        data=_png(4, 3),
        name="figur.bmp",
        media_type="image/png",
        suffix=".png",
        width=4,
        height=3,
        label=forged,
        converted_from="image/bmp",
        source_sha256="c" * 64,
    )
    written = assets.render_block(converted)
    assert len(_A_CHECKSUM_FIELD.findall(written)) == 2, written
    assert "sha256:" + "c" * 64 in written, "the run's own source digest is missing"
    assert "sha256:" + "a" * 64 not in written, "the document's claim became a field after all"


def test_the_pointer_regex_finds_every_shipped_block() -> None:
    """Step 2 has to find these mechanically; the finder ships with the writer."""
    images = [
        assets.read_image(_png(4, 3), name="a.png", label="Figur 1"),
        assets.read_image(_jpeg(9, 9), name="b.jpg"),
    ]
    text = "Prosess 84\n\n" + "\n\n".join(assets.render_block(i) for i in images) + "\n\nSlutt\n"
    found = assets.IMAGE_POINTER.findall(text)
    assert [match[1] for match in found] == [assets.asset_name(i) for i in images]


def test_href_is_bundle_absolute() -> None:
    """SPEC SS 6.2 allows a bundle-relative path with a leading `/`.

    A concept can sit at any depth under a segmented bundle, so a relative
    `assets/...` would resolve differently from two concepts of one document.
    """
    image = assets.read_image(_png(4, 3), name="f.png")
    assert assets.asset_href(image) == f"/{assets.ASSETS_DIR}/{assets.asset_name(image)}"


# --- PNG encoding, for samples that arrive without a container -------------
#
# A PDF image is usually not a file: `FlateDecode` hands back raw samples with
# the colour model in the dictionary beside them. Encoding those is the only
# way to carry them at all, and it is stdlib (`zlib`), so the `pdf` row does
# not gain a dependency and the output is not bound to a renderer's version.


def test_png_from_gray_samples_round_trips_the_header() -> None:
    encoded = assets.encode_png(3, 2, bytes([0, 64, 128, 192, 255, 32]), channels=1)
    assert assets.sniff(encoded) == ("image/png", ".png")
    assert assets.dimensions(encoded) == (3, 2)


def test_png_from_rgb_samples_round_trips_the_header() -> None:
    encoded = assets.encode_png(2, 1, bytes(range(6)), channels=3)
    assert assets.dimensions(encoded) == (2, 1)


def test_png_from_indexed_samples_carries_the_palette() -> None:
    palette = bytes([255, 0, 0, 0, 255, 0])
    encoded = assets.encode_png(2, 1, bytes([0, 1]), channels=1, palette=palette)
    assert assets.dimensions(encoded) == (2, 1)
    assert b"PLTE" in encoded


def test_png_refuses_a_sample_count_that_does_not_fit() -> None:
    """Refused rather than padded: a short buffer is a misread dictionary."""
    with pytest.raises(ExtractionError) as excinfo:
        assets.encode_png(4, 4, b"\x00\x01", channels=1)
    assert excinfo.value.code == "asset_samples_invalid"


def test_png_encoding_is_byte_stable() -> None:
    first = assets.encode_png(3, 2, bytes([0, 64, 128, 192, 255, 32]), channels=1)
    second = assets.encode_png(3, 2, bytes([0, 64, 128, 192, 255, 32]), channels=1)
    assert first == second


def test_the_restated_reduction_is_materializes_own() -> None:
    """`assets` cannot import `materialize`, so the rule is held equal instead.

    `extract.py`'s registry must not import the contract layer (its own
    docstring says so, and `materialize` reaches `manifest` and `profiles`), so
    the id-grammar reduction is written out a second time. This is what stops
    the second copy becoming a second grammar.
    """
    from llm_ingestion_okf.materialize import reduce_to_id_grammar

    for value in (
        "25-0143 - Tabeller - Prosesskoden (R761-R762)",
        "Figur 11.1  Toleransekrav",
        "grafikk_med_æøå",
        "---",
        "",
    ):
        assert assets._reduce(value) == reduce_to_id_grammar(value), value


def test_one_image_reached_by_two_paths_is_one_asset() -> None:
    """The asset name reads the BASENAME, never the path the document used.

    Measured on the fixture inbox before this rule existed: the HTML document
    points at `graphics/figur-84-1.png` and the STS document at
    `figur-84-1.png` (resolved through the `graphics/` sibling convention), so
    one image was written twice, under two names, from one run -- with the
    digest in both of them announcing that the bytes were identical. The path a
    document happened to use is not a property of the picture.
    """
    data = _png(6, 4)
    through_directory = assets.read_image(data, name="graphics/figur-84-1.png")
    bare = assets.read_image(data, name="figur-84-1.png")
    assert assets.asset_name(through_directory) == assets.asset_name(bare)
    # The full original is not lost -- it moves to the line a person reads.
    assert "graphics/figur-84-1.png" in assets.render_block(through_directory)