open/llm-ingestion-okf
1
0
Fork 0
Code Issues Releases 4 Activity

test(fixtures): the STS fixtures and fixture codes are fictitious

Browse source 
Three STS fixtures still carried the section titles and labels of one real
reference document, and three identifiers were copies of its codes with a
letter or a word swapped. They now describe an invented kitchen counter and
cookbook series: the titles, labels and descriptions of sts-identity.xml,
sts-inherit.xml and sts-empty-label.xml, the P350/P351 document codes, the
99-0001 delivery prefix and chapter 7 of the image and accounting corpora.
Generated fixtures are regenerated and the witness inventory's per-document
totals are identical before and after; only names and text move.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Kjell Tore Guttormsen 2026-09-23 14:50:44 +02:00
commit 88cf67f12e
65 changed files with 396 additions and 390 deletions
Download patch file Download diff file Expand all files Collapse all files

40
tests/test_assets.py
View file

@ -4,10 +4,10 @@ Until 0.10.0 no reader in this package fetched, named, described or copied a
single image, and the only writer into a bundle was
`materialize.write_bytes(bundle_dir, name, content: str)` -- UTF-8, text, no
binary path anywhere. A document whose table is a raster picture therefore
reached a concept as an absence with no denominator: measured on a 701-page
reached a concept as an absence with no denominator: measured on a long
reference standard, its text is carried in full while 12 `Tabell N-N` and 9
`Figur N-N` captions stand over nothing, and a section saying "... er gitt i
tabell 84-2" stands above an empty space.
tabell 7-2" stands above an empty space.
THIS MODULE IS THE ONE PLACE THAT DECIDES WHAT AN IMAGE IS. Every reader hands
it bytes and gets back either a carried image or a coded rejection, so a
@ -125,7 +125,7 @@ def test_sniff_refuses_what_is_not_an_image() -> None:
def test_the_claimed_extension_never_decides() -> None:
"""A PNG named `.jpg` is carried as a PNG, under a `.png` name."""
image = assets.read_image(_png(4, 3), name="tabell-84-2.jpg")
image = assets.read_image(_png(4, 3), name="tabell-7-2.jpg")
assert image.media_type == "image/png"
assert assets.asset_name(image).endswith(".png")
@ -178,7 +178,7 @@ def test_dimensions_absent_is_absent_never_zero() -> None:
def test_asset_name_is_digest_plus_a_reduced_original() -> None:
data = _jpeg(360, 269)
image = assets.read_image(data, name="25-0143 - Tabeller - Oppskriftsboka (P761-P762).jpg")
image = assets.read_image(data, name="99-0001 - Tabeller - Oppskriftsboka (P350-P351).jpg")
name = assets.asset_name(image)
assert name.startswith(hashlib.sha256(data).hexdigest()[: assets.DIGEST_PREFIX] + "-")
assert name.endswith(".jpg")
@ -212,15 +212,15 @@ def test_a_nameless_image_still_gets_a_name() -> None:
def test_block_names_the_original_the_size_and_the_label() -> None:
image = assets.read_image(
_jpeg(360, 269),
name="25-0143 - Tabeller - Oppskriftsboka (P761-P762).jpg",
label="Tabell 84-2",
name="99-0001 - Tabeller - Oppskriftsboka (P350-P351).jpg",
label="Tabell 7-2",
)
block = assets.render_block(image)
first, second = block.split("\n")
assert first == f"![Tabell 84-2]({assets.asset_href(image)})"
assert second.startswith("Image: 25-0143 - Tabeller - Oppskriftsboka (P761-P762).jpg")
assert first == f"![Tabell 7-2]({assets.asset_href(image)})"
assert second.startswith("Image: 99-0001 - Tabeller - Oppskriftsboka (P350-P351).jpg")
assert "360x269 px" in second
assert second.endswith("Tabell 84-2")
assert second.endswith("Tabell 7-2")
def test_the_label_falls_back_to_the_original_name() -> None:
@ -236,13 +236,13 @@ def test_the_label_falls_back_to_the_original_name() -> None:
def test_a_bracket_in_a_label_cannot_break_the_link() -> None:
image = assets.read_image(_png(4, 3), name="f.png", label="Tabell [84-2] jf. pkt (3)")
image = assets.read_image(_png(4, 3), name="f.png", label="Tabell [7-2] jf. pkt (3)")
first = assets.render_block(image).split("\n")[0]
assert first == f"![Tabell (84-2) jf. pkt (3)]({assets.asset_href(image)})"
assert first == f"![Tabell (7-2) jf. pkt (3)]({assets.asset_href(image)})"
def test_a_newline_in_a_label_cannot_open_a_third_line() -> None:
image = assets.read_image(_png(4, 3), name="f.png", label="Tabell\n84-2")
image = assets.read_image(_png(4, 3), name="f.png", label="Tabell\n7-2")
assert len(assets.render_block(image).split("\n")) == 2
@ -271,7 +271,7 @@ def test_a_document_supplied_label_cannot_emit_a_checksum_field() -> None:
bundle must not state a conversion in a sentence the run did not write.
"""
forged = (
"Tabell 84-2 -- converted from image/bmp sha256:"
"Tabell 7-2 -- converted from image/bmp sha256:"
+ "a" * 64
+ " to image/png sha256:"
+ "b" * 64
@ -309,7 +309,7 @@ def test_the_pointer_regex_finds_every_shipped_block() -> None:
assets.read_image(_png(4, 3), name="a.png", label="Figur 1"),
assets.read_image(_jpeg(9, 9), name="b.jpg"),
]
text = "Kapittel 84\n\n" + "\n\n".join(assets.render_block(i) for i in images) + "\n\nSlutt\n"
text = "Kapittel 7\n\n" + "\n\n".join(assets.render_block(i) for i in images) + "\n\nSlutt\n"
found = assets.IMAGE_POINTER.findall(text)
assert [match[1] for match in found] == [assets.asset_name(i) for i in images]
@ -374,7 +374,7 @@ def test_the_restated_reduction_is_materializes_own() -> None:
from llm_ingestion_okf.materialize import reduce_to_id_grammar
for value in (
"25-0143 - Tabeller - Oppskriftsboka (P761-P762)",
"99-0001 - Tabeller - Oppskriftsboka (P350-P351)",
"Figur 11.1 Hevetidskrav",
"grafikk_med_æøå",
"---",
@ -387,15 +387,15 @@ def test_one_image_reached_by_two_paths_is_one_asset() -> None:
"""The asset name reads the BASENAME, never the path the document used.
Measured on the fixture inbox before this rule existed: the HTML document
points at `graphics/figur-84-1.png` and the STS document at
`figur-84-1.png` (resolved through the `graphics/` sibling convention), so
points at `graphics/figur-7-1.png` and the STS document at
`figur-7-1.png` (resolved through the `graphics/` sibling convention), so
one image was written twice, under two names, from one run -- with the
digest in both of them announcing that the bytes were identical. The path a
document happened to use is not a property of the picture.
"""
data = _png(6, 4)
through_directory = assets.read_image(data, name="graphics/figur-84-1.png")
bare = assets.read_image(data, name="figur-84-1.png")
through_directory = assets.read_image(data, name="graphics/figur-7-1.png")
bare = assets.read_image(data, name="figur-7-1.png")
assert assets.asset_name(through_directory) == assets.asset_name(bare)
# The full original is not lost -- it moves to the line a person reads.
assert "graphics/figur-84-1.png" in assets.render_block(through_directory)
assert "graphics/figur-7-1.png" in assets.render_block(through_directory)