llm-ingestion-okf/tests/fixtures/make_k2_office.py

"""Regenerate the K2 office fixture set: one pptx, one odt, one rtf.

Three containers carrying the SAME document, so the only variable between the
three measurements is the container and the reader that opens it. Hand-laid,
part by part, with no generator library anywhere -- for the reason
`make_fixtures.py` states and this set inherits: a file written by the
converter and then read by the converter proves only that the converter agrees
with itself, and would stay green through any conversion defect that is
symmetric, which is most of them. The order that commissioned this set offered
pandoc as one generator option; this repository's committed fixture policy
forbids it, and the policy wins.

The document is invented, and that is a constraint rather than a detail: a
plausible Norwegian requirements sheet carrying the diacritics the pipeline has
to survive, and nobody's real document. No private file, nothing from
`~/Documents`, nothing from a customer.

Run from the repository root:  python3 tests/fixtures/make_k2_office.py
"""

from __future__ import annotations

import io
import zipfile
from pathlib import Path

HERE = Path(__file__).parent
OUT = HERE / "k2-office"
#: Round 10's rtf variants live in their OWN directory, and that is not tidiness.
#: `test_k2_office_fixtures.py` reads N off `k2-office/` by listing it, because
#: Door B walks a drop directory RECURSIVELY -- anything parked beside the three
#: containers would enter that run and the denominator would stop being three.
RTF_OUT = HERE / "k2-rtf-variants"

TITLE = "Kravspesifikasjon for tunnelbelysning"
INTRO = "Dokumentet samler kravene til belysning i vegtunneler over 500 meter."

# The pairing table. Every label ends in a colon, which is the rule
# `okf_fidelity.label_value_rows` selects on -- so these twenty rows ARE the
# pairable denominator, and a converter that drops a value or breaks a row over
# two output lines shows up as a fall from 20.
PAIRS: tuple[tuple[str, str], ...] = (
    ("Dokumentnummer:", "SVV-2026-0417"),
    ("Tittel:", "Tunnelbelysning i hovedløpet"),
    ("Ansvarlig avdeling:", "Utbyggingsdivisjonen"),
    ("Fagområde:", "Elektro og belysning"),
    ("Versjon:", "2.3 godkjent"),
    ("Gyldig fra:", "01.03.2026"),
    ("Erstatter:", "SVV-2024-0188"),
    ("Tunnellengde:", "1240 meter"),
    ("Dimensjonerende fart:", "80 km/t"),
    ("Årsdøgntrafikk:", "12400 kjøretøy"),
    ("Terskelluminans:", "145 candela"),
    ("Overgangssone:", "Tre trinn nedtrapping"),
    ("Innerstrekning:", "3,0 candela"),
    ("Utkjøringssone:", "Ingen forsterkning"),
    ("Fargetemperatur:", "4000 kelvin"),
    ("Fargegjengivelse:", "Ra større enn 70"),
    ("Nødbelysning:", "60 minutter drift"),
    ("Vedlikeholdsfaktor:", "0,80 beregnet"),
    ("Målemetode:", "Måling med luminanskamera"),
    ("Avvikshåndtering:", "Søknad om fravik"),
)

# The 4x4 the order asks for, and a second question. No cell here ends in a
# colon, so the grid contributes to COVERAGE and to nothing else: a converter
# that flattens a grid into prose keeps its coverage and loses its shape, and
# the two numbers are meant to be able to move independently.
# The caption between the two tables. A PARAGRAPH, not an empty one: an
# empty `\\pard\\par` left the converter joining the 2-column rows and the
# 4-column rows into a single four-column table, so the grid stopped being a
# second table at all.
GRID_CAPTION = "Luminansmatrise per trafikklasse"

GRID: tuple[tuple[str, ...], ...] = (
    ("Sone", "Klasse A", "Klasse B", "Klasse C"),
    ("Terskel", "150 cd", "120 cd", "95 cd"),
    ("Overgang", "45 cd", "36 cd", "28 cd"),
    ("Indre", "3,5 cd", "3,0 cd", "2,5 cd"),
)


def _escape(text: str) -> str:
    return text.replace("&", "&amp;").replace("<", "&lt;").replace(">", "&gt;")


_ZIP_DATE = (2020, 1, 1, 0, 0, 0)
_XML = '<?xml version="1.0" encoding="UTF-8" standalone="yes"?>'


def build_container(parts: dict[str, str], *, stored_first: str | None = None) -> bytes:
    """Zip the parts with a fixed timestamp, so the fixture is byte-reproducible.

    A zip records mtime, so without a constant `date_time` the fixture would
    differ on every regeneration and `git diff --quiet` could not be the check.
    `stored_first` exists for ODF, whose specification requires `mimetype` to be
    the first member and stored uncompressed.
    """
    out = io.BytesIO()
    with zipfile.ZipFile(out, "w", compression=zipfile.ZIP_DEFLATED) as archive:
        names = list(parts)
        if stored_first is not None:
            names.remove(stored_first)
            info = zipfile.ZipInfo(stored_first, date_time=_ZIP_DATE)
            info.compress_type = zipfile.ZIP_STORED
            archive.writestr(info, parts[stored_first])
        for name in names:
            info = zipfile.ZipInfo(name, date_time=_ZIP_DATE)
            info.compress_type = zipfile.ZIP_DEFLATED
            archive.writestr(info, parts[name])
    return out.getvalue()


# --- odt ---------------------------------------------------------------------

_ODT_NS = (
    ' xmlns:office="urn:oasis:names:tc:opendocument:xmlns:office:1.0"'
    ' xmlns:text="urn:oasis:names:tc:opendocument:xmlns:text:1.0"'
    ' xmlns:table="urn:oasis:names:tc:opendocument:xmlns:table:1.0"'
    ' xmlns:style="urn:oasis:names:tc:opendocument:xmlns:style:1.0"'
    ' xmlns:fo="urn:oasis:names:tc:opendocument:xmlns:xsl-fo-compatible:1.0"'
    ' office:version="1.3"'
)


def _odt_table(name: str, rows: tuple[tuple[str, ...], ...]) -> str:
    columns = len(rows[0])
    body = "".join(
        "<table:table-row>"
        + "".join(
            f'<table:table-cell office:value-type="string">'
            f"<text:p>{_escape(cell)}</text:p></table:table-cell>"
            for cell in row
        )
        + "</table:table-row>"
        for row in rows
    )
    return (
        f'<table:table table:name="{name}">'
        f'<table:table-column table:number-columns-repeated="{columns}"/>'
        f"{body}</table:table>"
    )


def odt_parts() -> dict[str, str]:
    content = (
        f"{_XML}<office:document-content{_ODT_NS}><office:body><office:text>"
        f'<text:h text:outline-level="1">{_escape(TITLE)}</text:h>'
        f"<text:p>{_escape(INTRO)}</text:p>"
        + _odt_table("Krav", tuple(PAIRS))
        + f"<text:p>{_escape(GRID_CAPTION)}</text:p>"
        + _odt_table("Luminansmatrise", GRID)
        + "</office:text></office:body></office:document-content>"
    )
    return {
        "mimetype": "application/vnd.oasis.opendocument.text",
        "META-INF/manifest.xml": _XML
        + '<manifest:manifest xmlns:manifest="urn:oasis:names:tc:opendocument:xmlns:manifest:1.0"'
        + ' manifest:version="1.3">'
        + '<manifest:file-entry manifest:full-path="/"'
        + ' manifest:media-type="application/vnd.oasis.opendocument.text"/>'
        + '<manifest:file-entry manifest:full-path="content.xml"'
        + ' manifest:media-type="text/xml"/>'
        + '<manifest:file-entry manifest:full-path="styles.xml"'
        + ' manifest:media-type="text/xml"/>'
        + "</manifest:manifest>",
        "styles.xml": _XML
        + f"<office:document-styles{_ODT_NS}><office:styles/></office:document-styles>",
        "content.xml": content,
    }


# --- pptx --------------------------------------------------------------------

_A = "http://schemas.openxmlformats.org/drawingml/2006/main"
_P = "http://schemas.openxmlformats.org/presentationml/2006/main"
_R = "http://schemas.openxmlformats.org/officeDocument/2006/relationships"


def _pptx_text_body(text: str) -> str:
    return "<p:txBody><a:bodyPr/><a:p><a:r><a:t>" + _escape(text) + "</a:t></a:r></a:p></p:txBody>"


def _pptx_shape(shape_id: int, name: str, text: str) -> str:
    return (
        "<p:sp><p:nvSpPr>"
        f'<p:cNvPr id="{shape_id}" name="{_escape(name)}"/><p:cNvSpPr/><p:nvPr/>'
        "</p:nvSpPr><p:spPr/>" + _pptx_text_body(text) + "</p:sp>"
    )


def _pptx_table(shape_id: int, name: str, rows: tuple[tuple[str, ...], ...]) -> str:
    columns = len(rows[0])
    grid = "".join('<a:gridCol w="2000000"/>' for _ in range(columns))
    body = "".join(
        '<a:tr h="370840">'
        + "".join(
            "<a:tc><a:txBody><a:bodyPr/><a:p><a:r><a:t>"
            + _escape(cell)
            + "</a:t></a:r></a:p></a:txBody><a:tcPr/></a:tc>"
            for cell in row
        )
        + "</a:tr>"
        for row in rows
    )
    return (
        "<p:graphicFrame><p:nvGraphicFramePr>"
        f'<p:cNvPr id="{shape_id}" name="{_escape(name)}"/>'
        "<p:cNvGraphicFramePr/><p:nvPr/></p:nvGraphicFramePr>"
        '<p:xfrm><a:off x="0" y="0"/><a:ext cx="8000000" cy="4000000"/></p:xfrm>'
        f'<a:graphic><a:graphicData uri="{_A}/table">'
        f"<a:tbl><a:tblPr/><a:tblGrid>{grid}</a:tblGrid>{body}</a:tbl>"
        "</a:graphicData></a:graphic></p:graphicFrame>"
    )


def _pptx_slide(shapes: str) -> str:
    return (
        f'{_XML}<p:sld xmlns:a="{_A}" xmlns:r="{_R}" xmlns:p="{_P}">'
        "<p:cSld><p:spTree>"
        '<p:nvGrpSpPr><p:cNvPr id="1" name=""/><p:cNvGrpSpPr/><p:nvPr/></p:nvGrpSpPr>'
        "<p:grpSpPr/>" + shapes + "</p:spTree></p:cSld></p:sld>"
    )


def pptx_parts() -> dict[str, str]:
    slide_one = _pptx_slide(
        _pptx_shape(2, "Tittel", TITLE)
        + _pptx_shape(3, "Ingress", INTRO)
        + _pptx_table(4, "Kravtabell", tuple(PAIRS))
    )
    slide_two = _pptx_slide(
        _pptx_shape(2, "Undertittel", GRID_CAPTION) + _pptx_table(3, "Luminansmatrise", GRID)
    )
    rels = (
        f'{_XML}<Relationships xmlns="{_R}">'
        '<Relationship Id="rId1" Type="{t}/slide" Target="slides/slide1.xml"/>'
        '<Relationship Id="rId2" Type="{t}/slide" Target="slides/slide2.xml"/>'
        "</Relationships>"
    ).format(t=_R)
    return {
        "[Content_Types].xml": _XML
        + '<Types xmlns="http://schemas.openxmlformats.org/package/2006/content-types">'
        + '<Default Extension="xml" ContentType="application/xml"/>'
        + '<Default Extension="rels" ContentType="application/vnd.openxmlformats-package.relationships+xml"/>'
        + '<Override PartName="/ppt/presentation.xml" ContentType="application/vnd.openxmlformats-officedocument.presentationml.presentation.main+xml"/>'
        + '<Override PartName="/ppt/slides/slide1.xml" ContentType="application/vnd.openxmlformats-officedocument.presentationml.slide+xml"/>'
        + '<Override PartName="/ppt/slides/slide2.xml" ContentType="application/vnd.openxmlformats-officedocument.presentationml.slide+xml"/>'
        + "</Types>",
        "_rels/.rels": _XML
        + f'<Relationships xmlns="{_R}">'
        + f'<Relationship Id="rId1" Type="{_R}/officeDocument" Target="ppt/presentation.xml"/>'
        + "</Relationships>",
        "ppt/_rels/presentation.xml.rels": rels,
        "ppt/presentation.xml": _XML
        + f'<p:presentation xmlns:a="{_A}" xmlns:r="{_R}" xmlns:p="{_P}">'
        + '<p:sldIdLst><p:sldId id="256" r:id="rId1"/><p:sldId id="257" r:id="rId2"/></p:sldIdLst>'
        + "</p:presentation>",
        "ppt/slides/slide1.xml": slide_one,
        "ppt/slides/slide2.xml": slide_two,
        "ppt/slides/_rels/slide1.xml.rels": _XML + f'<Relationships xmlns="{_R}"/>',
        "ppt/slides/_rels/slide2.xml.rels": _XML + f'<Relationships xmlns="{_R}"/>',
    }


# --- rtf ---------------------------------------------------------------------
#
# RTF carries no unicode of its own: a character above the code page is written
# as an escape with an ASCII replacement beside it, for a reader too old to
# understand the escape. Emitting the letters raw would make the fixture's bytes
# depend on a code page nobody declared.
#
# THE SPACE IN `\\uN ?` IS LOAD-BEARING AND IT IS NOT COSMETIC. The form Word
# emits is `\\uN?` with no delimiter, and the vendored converter reads that as
# the control word delimited BY the `?`, then applies `\\uc1` to the character
# AFTER it. Measured directly: `A\\u248?BC` comes back as `AoC` with the ring
# letter in place and the `B` GONE; `A\\u248?xBC` comes back as `AoBC`, which
# is the same rule seen from the other side. An explicit space delimits the
# control word, so the `?` is what gets skipped and the text survives. The
# fixture is therefore written in the form that round-trips, and the form that
# does not is recorded in `docs/2026-09-07-k2-pptx-odt-rtf-fixtures.md` as a
# converter finding rather than worked around in silence.

_RTF_CELL_WIDTH = 3000


def _rtf_escape(text: str) -> str:
    out = []
    for char in text:
        if char in "\\{}":
            out.append("\\" + char)
        elif ord(char) < 128:
            out.append(char)
        else:
            out.append(f"\\u{ord(char)} ?")
    return "".join(out)


def _rtf_row(row: tuple[str, ...]) -> str:
    """One table row. `\\pard\\intbl` PER CELL, and it is load-bearing.

    Measured against the vendored converter while building this fixture: the
    same rows WITHOUT `\\intbl` are read as each row nested inside the previous
    one -- five label/value rows came back as five levels of nested table and
    2076 characters where 117 were expected, exit code 0 and no warning. That
    is the same shape as the missing `word/styles.xml` and the `inlineStr`
    xlsx recorded in `make_fixtures.py`: structurally plausible input, silently
    wrong output, nothing anywhere saying so.
    """
    borders = "".join(f"\\cellx{_RTF_CELL_WIDTH * (index + 1)}" for index in range(len(row)))
    cells = "".join(f"\\pard\\intbl {_rtf_escape(cell)}\\cell " for cell in row)
    return "\\trowd\\trgaph108" + borders + "\n" + cells + "\\row\n"


# --- the three rtf VARIANTS round 10 added ------------------------------------
#
# WHY THEY EXIST. Round 9 measured the `rtf` row on ONE document and it came
# back at 0 of 0 declared headings, 0 concepts, 1368 of 1368 characters in no
# segment. A repair proposed on N = 1 is a repair fitted to one file, so round
# 10's order required N >= 3 before any rule was written. Three lawful sources
# were offered; this is the first of them, and it is the one this repository's
# committed-fixture policy allows: hand-laid here, in the same file, with the
# fasit written BEFORE the measurement.
#
# WHAT EACH ONE VARIES, and it is one axis each:
#
#   -fet-alene        the title is bold at BODY size, with no point-size change
#                     at all. Variant A (`krav-rikt-tekstformat.rtf`) sets its
#                     title bold AND at 16pt, so without this document a rule
#                     reading "bold" could be passing on the size instead.
#   -fet-i-avsnitt    a bold PHRASE inside a running sentence, which must NOT
#                     become a boundary. The document's own known-negative.
#   -stil             the container DECLARES heading styles (`\\s1`, `\\s2`) in a
#                     stylesheet. This is the control for the whole repair, and
#                     it is a control that FAILED in an informative direction:
#                     measured, the vendored converter discards the style and
#                     emits the same bold line, so "read the declared style"
#                     is not a route that exists for `rtf`. The fixture is kept
#                     precisely because it pins that.
#
# The bodies are deliberately short. These documents answer where a boundary
# is, not how much text survives -- variant A already carries the coverage and
# fidelity counts for the container.

RTF_VARIANT_TITLE = "Kravspesifikasjon for tunnelbelysning"
RTF_VARIANT_SECTION = "Luminanskrav per trafikklasse"
RTF_VARIANT_BODY_ONE = "Dokumentet samler kravene til belysning i vegtunneler over 500 meter."
RTF_VARIANT_BODY_TWO = "Terskelluminansen er 145 candela og gjelder i hele overgangssonen."
#: The bold RUN in `-fet-i-avsnitt`, set inside a sentence that continues past
#: it. Authored as emphasis, never as a title.
RTF_VARIANT_EMPHASIS = "fravik"


def _rtf_document(paragraphs: list[str]) -> bytes:
    """Wrap hand-written paragraph bodies in the minimal rtf container."""
    header = "{\\rtf1\\ansi\\ansicpg1252\\deff0{\\fonttbl{\\f0\\froman Times New Roman;}}"
    return (header + "".join(paragraphs) + "}").encode("ascii")


def rtf_bold_only_bytes() -> bytes:
    """Title and section name bold at BODY size -- no point-size change."""
    return _rtf_document(
        [
            "\\pard\\sa180\\b " + _rtf_escape(RTF_VARIANT_TITLE) + "\\b0\\par\n",
            "\\pard\\sa180 " + _rtf_escape(RTF_VARIANT_BODY_ONE) + "\\par\n",
            "\\pard\\sa180\\b " + _rtf_escape(RTF_VARIANT_SECTION) + "\\b0\\par\n",
            "\\pard\\sa180 " + _rtf_escape(RTF_VARIANT_BODY_TWO) + "\\par\n",
        ]
    )


def rtf_bold_inside_paragraph_bytes() -> bytes:
    """One bold title, and one bold phrase mid-sentence that is not a title."""
    sentence = (
        "Avvik fra kravene krever soknad om "
        + "\\b "
        + _rtf_escape(RTF_VARIANT_EMPHASIS)
        + "\\b0 "
        + " fra byggherren for arbeidet starter."
    )
    return _rtf_document(
        [
            "\\pard\\sa180\\b " + _rtf_escape(RTF_VARIANT_TITLE) + "\\b0\\par\n",
            "\\pard\\sa180 " + _rtf_escape(RTF_VARIANT_BODY_ONE) + "\\par\n",
            "\\pard\\sa180 " + sentence + "\\par\n",
        ]
    )


def rtf_styled_bytes() -> bytes:
    """The container DECLARES heading styles. The converter discards them."""
    stylesheet = "{\\stylesheet{\\s0 Normal;}{\\s1\\b\\fs32 heading 1;}{\\s2\\b\\fs28 heading 2;}}"
    return _rtf_document(
        [
            stylesheet,
            "\\pard\\s1\\b\\fs32 " + _rtf_escape(RTF_VARIANT_TITLE) + "\\b0\\fs24\\par\n",
            "\\pard\\s0\\sa180 " + _rtf_escape(RTF_VARIANT_BODY_ONE) + "\\par\n",
            "\\pard\\s2\\b\\fs28 " + _rtf_escape(RTF_VARIANT_SECTION) + "\\b0\\fs24\\par\n",
            "\\pard\\s0\\sa180 " + _rtf_escape(RTF_VARIANT_BODY_TWO) + "\\par\n",
        ]
    )


def rtf_bytes() -> bytes:
    body = [
        "{\\rtf1\\ansi\\ansicpg1252\\deff0",
        "{\\fonttbl{\\f0\\froman Times New Roman;}}",
        "\\pard\\sa180\\b\\fs32 " + _rtf_escape(TITLE) + "\\b0\\fs24\\par\n",
        "\\pard\\sa180 " + _rtf_escape(INTRO) + "\\par\n",
    ]
    body.extend(_rtf_row(row) for row in PAIRS)
    body.append("\\pard\\sa180 " + _rtf_escape(GRID_CAPTION) + "\\par\n")
    body.extend(_rtf_row(row) for row in GRID)
    body.append("}")
    return "".join(body).encode("ascii")


if __name__ == "__main__":
    OUT.mkdir(parents=True, exist_ok=True)
    RTF_OUT.mkdir(parents=True, exist_ok=True)
    written = {
        OUT / "krav-presentasjon.pptx": build_container(pptx_parts()),
        OUT / "krav-tekstdokument.odt": build_container(odt_parts(), stored_first="mimetype"),
        OUT / "krav-rikt-tekstformat.rtf": rtf_bytes(),
        RTF_OUT / "krav-rikt-tekstformat-fet-alene.rtf": rtf_bold_only_bytes(),
        RTF_OUT / "krav-rikt-tekstformat-fet-i-avsnitt.rtf": rtf_bold_inside_paragraph_bytes(),
        RTF_OUT / "krav-rikt-tekstformat-stil.rtf": rtf_styled_bytes(),
    }
    for path, payload in sorted(written.items()):
        path.write_bytes(payload)
        print(f"wrote {path.parent.name}/{path.name} ({len(payload)} bytes)")