open/llm-ingestion-okf
1
0
Fork 0
Code Issues Releases 4 Activity
llm-ingestion-okf/tests/fixtures/make_image_fixtures.py
Kjell Tore Guttormsen 9d1f4b14ed test(fixtures): replace sector-specific example material with generic, fictitious examples — green 
Every fixture, test document, tool example and document now uses an invented
kitchen-and-baking handbook series, written in this repository. The package's
behaviour is unchanged; src/ changes are comments and help text only.

- Generated fixtures are regenerated from their generators. Their structural
  counts are identical before and after: elements, images, rows, cells,
  headings, bookmarks and the witness inventory's per-document totals. The
  image-inbox and accounting documents are renamed kapittel-84-*.
- tools/okf_accounting_gate.py: the two options that named one real corpus
  each are replaced by a generic, repeatable --corpus PATH with no default.
  Row 5 compares the PDF pair alone. Gate verdict unchanged: RED rows 2, 3, 6.
- tools/okf_witness.py: the STS JSON reader for one publisher's delivery is
  removed, along with its three twins and five tests. The mutation harness
  loses W09.
- docs/: 13 dated reports that documented runs on a retired reference corpus
  are removed, and 40 are neutralized. Dead links are removed, and no new
  dangling path is introduced.
- The synthetic MCP-gate corpus and the residual probe words are neutral.

Valgt: keep the `okf quality --fasit` bar value (the measured fraction, one corpus) and
rewrite only its provenance, because the verdict stays unchanged and the
number names nothing.

Term check with the local list: 0 of 411 tracked files, 0 file names, 0 of
27 binary fixtures. Suite after git add: 2457 passed, 1 skipped. The base
tree had 2460 passed and 2 skipped; five tests went with the JSON reader and
four were added by the term check. ruff, ruff format and mypy --strict src/
are clean.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-09-23 14:52:02 +02:00

327 lines
15 KiB
Python
Raw Blame History

"""Regenerate the image-bearing fixtures for the asset path (0.10.0).
Five documents, one per reader, each carrying a KNOWN number of images so a
gate can state "carried N of M" with M read out of the source rather than out
of this package. Written in a second file rather than appended to
`make_fixtures.py` for one reason: every fixture that file emits is byte-pinned
by a test, and the object numbering of the PDF builders is part of those bytes.
Adding an XObject to a shared builder would regenerate files whose whole value
is that they have not moved.
The same policy holds here as there: no generator library. The PNG is written
out with `zlib` from the stdlib, the JPEG as a header sequence (the readers
copy JPEG bytes through and read nothing but the frame marker, so a decodable
photograph would test nothing extra and could not be hand-audited), and the
containers are assembled part by part.
Run from the repository root: python3 tests/fixtures/make_image_fixtures.py
"""
from __future__ import annotations
import io
import struct
import zipfile
import zlib
from pathlib import Path
HERE = Path(__file__).parent
IMAGES = HERE / "image-inbox"
_XML = '<?xml version="1.0" encoding="UTF-8" standalone="yes"?>'
_ZIP_DATE = (2020, 1, 1, 0, 0, 0)
def png(width: int, height: int, value: int = 0x40) -> bytes:
"""A real, single-channel PNG of a flat grey."""
def chunk(kind: bytes, payload: bytes) -> bytes:
return (
len(payload).to_bytes(4, "big")
+ kind
+ payload
+ zlib.crc32(kind + payload).to_bytes(4, "big")
)
ihdr = struct.pack(">IIBBBBB", width, height, 8, 0, 0, 0, 0)
raw = b"".join(b"\x00" + bytes([value] * width) for _ in range(height))
return (
b"\x89PNG\r\n\x1a\n"
+ chunk(b"IHDR", ihdr)
+ chunk(b"IDAT", zlib.compress(raw, 9))
+ chunk(b"IEND", b"")
)
def jpeg_header(width: int, height: int) -> bytes:
"""A JPEG's marker sequence: SOI, JFIF, a baseline frame header, EOI.
Not a decodable photograph, and that is the point of it. The `pdf` reader
passes `DCTDecode` bytes through untouched and reads only the frame header
for the size, which is precisely what a real reference PDF's table images
need -- 29 of its 50 image objects are `DCTDecode`. A fixture that also carried entropy
data would exercise no additional line of this package and could not be read
byte by byte by a person.
"""
frame = bytes([8, height >> 8, height & 0xFF, width >> 8, width & 0xFF, 1, 1, 0x11, 0])
return (
b"\xff\xd8"
b"\xff\xe0\x00\x10JFIF\x00\x01\x01\x00\x00\x01\x00\x01\x00\x00"
+ b"\xff\xc0"
+ (len(frame) + 2).to_bytes(2, "big")
+ frame
+ b"\xff\xd9"
)
# --- pdf ---------------------------------------------------------------------
#
# TWO images on one page, and they are deliberately of the two kinds the
# measurement on a reference PDF found: 29 `DCTDecode` objects, which arrive as a finished
# JPEG file, and 21 `FlateDecode` ones, which arrive as raw samples with the
# colour model in the dictionary beside them and have to be encoded to be
# carried at all. A fixture with only one kind would leave half the reader
# unexercised, and it is the encoded half that can be silently wrong.
PDF_GRAY_WIDTH, PDF_GRAY_HEIGHT = 4, 3
PDF_GRAY_SAMPLES = bytes([0, 60, 120, 180, 20, 80, 140, 200, 40, 100, 160, 255])
PDF_JPEG_WIDTH, PDF_JPEG_HEIGHT = 360, 269
PDF_CONTENT = (
b"BT /F1 12 Tf 20 170 Td (Hevetidsklasse er gitt i tabell 84-2) Tj ET\n"
b"q 80 0 0 60 20 90 cm /ImFlate Do Q\n"
b"q 80 0 0 60 20 20 cm /ImJpeg Do Q\n"
)
def build_image_pdf() -> bytes:
"""A one-page PDF with a Flate image and a DCT image in its resources."""
flate = zlib.compress(PDF_GRAY_SAMPLES, 9)
jpeg = jpeg_header(PDF_JPEG_WIDTH, PDF_JPEG_HEIGHT)
objects = [
b"<< /Type /Catalog /Pages 2 0 R >>",
b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>",
b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 200 200] /Contents 4 0 R "
b"/Resources << /Font << /F1 5 0 R >> "
b"/XObject << /ImFlate 6 0 R /ImJpeg 7 0 R >> >> >>",
b"<< /Length "
+ str(len(PDF_CONTENT)).encode()
+ b" >>\nstream\n"
+ PDF_CONTENT
+ b"endstream",
b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica /Encoding /WinAnsiEncoding >>",
b"<< /Type /XObject /Subtype /Image /Width "
+ str(PDF_GRAY_WIDTH).encode()
+ b" /Height "
+ str(PDF_GRAY_HEIGHT).encode()
+ b" /ColorSpace /DeviceGray /BitsPerComponent 8 /Filter /FlateDecode /Length "
+ str(len(flate)).encode()
+ b" >>\nstream\n"
+ flate
+ b"\nendstream",
b"<< /Type /XObject /Subtype /Image /Width "
+ str(PDF_JPEG_WIDTH).encode()
+ b" /Height "
+ str(PDF_JPEG_HEIGHT).encode()
+ b" /ColorSpace /DeviceGray /BitsPerComponent 8 /Filter /DCTDecode /Length "
+ str(len(jpeg)).encode()
+ b" >>\nstream\n"
+ jpeg
+ b"\nendstream",
]
out = bytearray(b"%PDF-1.4\n")
offsets = []
for number, body in enumerate(objects, start=1):
offsets.append(len(out))
out += str(number).encode() + b" 0 obj\n" + body + b"\nendobj\n"
xref_at = len(out)
size = str(len(objects) + 1).encode()
out += b"xref\n0 " + size + b"\n0000000000 65535 f \n"
for offset in offsets:
out += ("%010d 00000 n \n" % offset).encode()
out += b"trailer\n<< /Size " + size + b" /Root 1 0 R >>\n"
out += b"startxref\n" + str(xref_at).encode() + b"\n%%EOF\n"
return bytes(out)
# --- html --------------------------------------------------------------------
#
# THREE `<img>` and only two of them can be carried. The remote one is the
# boundary written as a fixture: this package never opens a socket during
# extraction (the network gate is an explicit per-run opt-in and extraction is
# not on that path), so a remote source becomes a pointer WITHOUT a file, and
# the gate counts it as found-and-not-carried rather than as absent.
HTML_DOCUMENT = """<!doctype html>
<html><head><title>Kapittel 84</title></head>
<body>
<h1>84 Brød og boller</h1>
<p>Hevetidsklasse for de enkelte deigtyper er gitt i tabell 84-2.</p>
<img src="graphics/tabell-84-2.png" alt="Tabell 84-2 Hevetidsklasser">
<p>Figuren under viser prinsippet.</p>
<figure>
<img src="graphics/figur-84-1.png" alt="Figur 84-1 Prinsippskisse">
<figcaption>Figur 84-1 Prinsippskisse</figcaption>
</figure>
<p>Og en som ligger et annet sted:</p>
<img src="https://example.invalid/ekstern.png" alt="Ekstern figur">
</body></html>
"""
# --- niso-sts ----------------------------------------------------------------
#
# The shape a real NISO-STS delivery has, measured 2026-09-16: 50 `<graphic>`
# elements, every one a direct child of a `<sec>`, none inside a `<table-wrap>`,
# each carrying a bare file name in `xlink:href` that resolves against a sibling
# `graphics/` directory. No `<caption>` anywhere near them -- the caption a
# human reads is a `<p>` the extractor already emits on its own line.
STS_DOCUMENT = """<?xml version="1.0" encoding="UTF-8"?>
<standard xmlns:xlink="http://www.w3.org/1999/xlink">
<front><std-ident><doc-number>P761</doc-number></std-ident></front>
<body>
<sec>
<label>84</label>
<title>Brød og boller</title>
<sec>
<label>84.1</label>
<title>Hevetider</title>
<p>Hevetidsklasse er gitt i tabell 84-2.</p>
<graphic xlink:href="graphics/tabell-84-2.png"/>
<p>Figur 84-1 viser prinsippet.</p>
<graphic xlink:href="figur-84-1.png"/>
</sec>
</sec>
</body>
</standard>
"""
def build_docx() -> bytes:
"""A `.docx` with one embedded image, its alt text on the drawing."""
image = png(40, 30, value=0x30)
parts: dict[str, str | bytes] = {
"[Content_Types].xml": _XML
+ '<Types xmlns="http://schemas.openxmlformats.org/package/2006/content-types">'
+ '<Default Extension="xml" ContentType="application/xml"/>'
+ '<Default Extension="rels" ContentType="application/vnd.openxmlformats-package'
+ '.relationships+xml"/>'
+ '<Default Extension="png" ContentType="image/png"/>'
+ '<Override PartName="/word/document.xml" ContentType="application/vnd'
+ '.openxmlformats-officedocument.wordprocessingml.document.main+xml"/>'
+ "</Types>",
"_rels/.rels": _XML
+ '<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships">'
+ '<Relationship Id="rId1" Type="http://schemas.openxmlformats.org/officeDocument/2006'
+ '/relationships/officeDocument" Target="word/document.xml"/></Relationships>',
"word/_rels/document.xml.rels": _XML
+ '<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships">'
+ '<Relationship Id="rIdImg" Type="http://schemas.openxmlformats.org/officeDocument'
+ '/2006/relationships/image" Target="media/tabell-84-2.png"/></Relationships>',
"word/media/tabell-84-2.png": image,
"word/document.xml": _XML
+ '<w:document xmlns:w="http://schemas.openxmlformats.org/wordprocessingml/2006/main"'
+ ' xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships"'
+ ' xmlns:wp="http://schemas.openxmlformats.org/drawingml/2006/wordprocessingDrawing"'
+ ' xmlns:a="http://schemas.openxmlformats.org/drawingml/2006/main"'
+ ' xmlns:pic="http://schemas.openxmlformats.org/drawingml/2006/picture"><w:body>'
+ "<w:p><w:r><w:t>Hevetidsklasse er gitt i tabell 84-2.</w:t></w:r></w:p>"
+ '<w:p><w:r><w:drawing><wp:inline><wp:extent cx="381000" cy="285750"/>'
+ '<wp:docPr id="1" name="Bilde 1" descr="Tabell 84-2 Hevetidsklasser"/>'
+ '<a:graphic><a:graphicData uri="http://schemas.openxmlformats.org/drawingml/2006'
+ '/picture"><pic:pic><pic:nvPicPr><pic:cNvPr id="1" name="tabell-84-2.png"'
+ ' descr="Tabell 84-2 Hevetidsklasser"/><pic:cNvPicPr/></pic:nvPicPr>'
+ '<pic:blipFill><a:blip r:embed="rIdImg"/><a:stretch><a:fillRect/></a:stretch>'
+ '</pic:blipFill><pic:spPr><a:xfrm><a:off x="0" y="0"/>'
+ '<a:ext cx="381000" cy="285750"/></a:xfrm>'
+ '<a:prstGeom prst="rect"><a:avLst/></a:prstGeom></pic:spPr></pic:pic>'
+ "</a:graphicData></a:graphic></wp:inline></w:drawing></w:r></w:p>"
+ "<w:p><w:r><w:t>Etter tabellen kommer steketidene.</w:t></w:r></w:p>"
+ "</w:body></w:document>",
}
return build_container(parts)
def build_pptx() -> bytes:
"""A `.pptx` with one titled slide and one embedded image on it."""
image = png(48, 36, value=0x70)
parts: dict[str, str | bytes] = {
"[Content_Types].xml": _XML
+ '<Types xmlns="http://schemas.openxmlformats.org/package/2006/content-types">'
+ '<Default Extension="xml" ContentType="application/xml"/>'
+ '<Default Extension="rels" ContentType="application/vnd.openxmlformats-package'
+ '.relationships+xml"/>'
+ '<Default Extension="png" ContentType="image/png"/>'
+ '<Override PartName="/ppt/presentation.xml" ContentType="application/vnd'
+ '.openxmlformats-officedocument.presentationml.presentation.main+xml"/>'
+ '<Override PartName="/ppt/slides/slide1.xml" ContentType="application/vnd'
+ '.openxmlformats-officedocument.presentationml.slide+xml"/>'
+ "</Types>",
"_rels/.rels": _XML
+ '<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships">'
+ '<Relationship Id="rId1" Type="http://schemas.openxmlformats.org/officeDocument/2006'
+ '/relationships/officeDocument" Target="ppt/presentation.xml"/></Relationships>',
"ppt/_rels/presentation.xml.rels": _XML
+ '<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships">'
+ '<Relationship Id="rId2" Type="http://schemas.openxmlformats.org/officeDocument/2006'
+ '/relationships/slide" Target="slides/slide1.xml"/></Relationships>',
"ppt/presentation.xml": _XML
+ '<p:presentation xmlns:p="http://schemas.openxmlformats.org/presentationml/2006/main"'
+ ' xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships">'
+ '<p:sldIdLst><p:sldId id="256" r:id="rId2"/></p:sldIdLst></p:presentation>',
"ppt/slides/_rels/slide1.xml.rels": _XML
+ '<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships">'
+ '<Relationship Id="rIdImg" Type="http://schemas.openxmlformats.org/officeDocument'
+ '/2006/relationships/image" Target="../media/skisse.png"/></Relationships>',
"ppt/media/skisse.png": image,
"ppt/slides/slide1.xml": _XML
+ '<p:sld xmlns:p="http://schemas.openxmlformats.org/presentationml/2006/main"'
+ ' xmlns:a="http://schemas.openxmlformats.org/drawingml/2006/main"'
+ ' xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships">'
+ "<p:cSld><p:spTree>"
+ '<p:nvGrpSpPr><p:cNvPr id="1" name=""/><p:cNvGrpSpPr/><p:nvPr/></p:nvGrpSpPr>'
+ "<p:grpSpPr/>"
+ '<p:sp><p:nvSpPr><p:cNvPr id="2" name="Tittel 1"/><p:cNvSpPr/>'
+ '<p:nvPr><p:ph type="title"/></p:nvPr></p:nvSpPr><p:spPr/>'
+ "<p:txBody><a:bodyPr/><a:p><a:r><a:t>Hevetider</a:t></a:r></a:p></p:txBody></p:sp>"
+ '<p:pic><p:nvPicPr><p:cNvPr id="3" name="skisse.png" descr="Prinsippskisse"/>'
+ "<p:cNvPicPr/><p:nvPr/></p:nvPicPr>"
+ '<p:blipFill><a:blip r:embed="rIdImg"/><a:stretch><a:fillRect/></a:stretch>'
+ '</p:blipFill><p:spPr><a:xfrm><a:off x="0" y="0"/>'
+ '<a:ext cx="457200" cy="342900"/></a:xfrm>'
+ '<a:prstGeom prst="rect"><a:avLst/></a:prstGeom></p:spPr></p:pic>'
+ "</p:spTree></p:cSld></p:sld>",
}
return build_container(parts)
def build_container(parts: dict[str, str | bytes]) -> bytes:
out = io.BytesIO()
with zipfile.ZipFile(out, "w", compression=zipfile.ZIP_DEFLATED) as archive:
for name, payload in parts.items():
info = zipfile.ZipInfo(name, date_time=_ZIP_DATE)
info.compress_type = zipfile.ZIP_DEFLATED
archive.writestr(info, payload)
return out.getvalue()
if __name__ == "__main__":
(IMAGES / "graphics").mkdir(parents=True, exist_ok=True)
# FIVE DISTINCT STEMS. One stem across five extensions is refused by the
# door's own SS 3 collision rule -- measured while building this fixture:
# `kapittel-84.{pdf,docx,pptx}` gave `inbox_slug_collision: 2/7` and two of
# the five readers were never exercised at all, with the gate reporting a
# carrying defect that was really a fixture defect.
written: list[tuple[str, bytes]] = [
("graphics/tabell-84-2.png", png(120, 90, value=0x20)),
("graphics/figur-84-1.png", png(64, 48, value=0x80)),
("kapittel-84-web.html", HTML_DOCUMENT.encode("utf-8")),
("kapittel-84-sts.xml", STS_DOCUMENT.encode("utf-8")),
("kapittel-84-tabell.pdf", build_image_pdf()),
("kapittel-84-notat.docx", build_docx()),
("kapittel-84-presentasjon.pptx", build_pptx()),
]
for name, payload in written:
(IMAGES / name).write_bytes(payload)
print(f"wrote image-inbox/{name}")
Reference in a new issue View git blame Copy permalink