llm-ingestion-okf/tests/test_fidelity.py

"""The fidelity instrument: it must be able to see a loss before a 100 % means anything.

This module exists because the arm A report of 2026-08-29 published `docx`
193/196 and `xlsx` 193/193 without shipping the command that produced them.
Those figures could not be re-measured against a new converter version, against
the product path, or at all -- so a later run reporting the same numbers would
have been agreement with a memory rather than with a measurement.

The load-bearing test here is the NEGATIVE CONTROL: an instrument that returns
100 % on text with strings deliberately removed is an instrument that returns
100 % on everything, and every green figure it ever produced would be
worthless. `test_a_dropped_string_is_counted_as_lost` is what makes the rest
of the numbers mean something.
"""

from __future__ import annotations

import sys
import zipfile
from collections.abc import Callable
from pathlib import Path

import pytest

sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tools"))

import okf_fidelity

DOCUMENT_XML = """<?xml version="1.0"?>
<w:document xmlns:w="x"><w:body>
<w:p><w:r><w:t>Krav til seksjonering av bygget</w:t></w:r></w:p>
<w:p><w:r><w:t>Navn tilbyder:</w:t></w:r></w:p>
<w:p><w:r><w:t>Roemningsveier skal vaere uavhengige</w:t></w:r></w:p>
<w:p><w:r><w:t>ok</w:t></w:r></w:p>
</w:body></w:document>
"""

SHARED_XML = """<?xml version="1.0"?>
<sst xmlns="x"><si><t>Prisskjema for entreprisen</t></si><si><t>Sum eks mva</t></si></sst>
"""


def docx(tmp_path: Path) -> Path:
    path = tmp_path / "d.docx"
    with zipfile.ZipFile(path, "w") as archive:
        archive.writestr("word/document.xml", DOCUMENT_XML)
    return path


def xlsx(tmp_path: Path) -> Path:
    path = tmp_path / "s.xlsx"
    with zipfile.ZipFile(path, "w") as archive:
        archive.writestr("xl/sharedStrings.xml", SHARED_XML)
    return path


def test_the_fasit_comes_from_the_file_not_from_a_converter(tmp_path: Path) -> None:
    """A fasit derived from one converter would score it on its own homework."""
    strings = okf_fidelity.source_strings(docx(tmp_path))
    assert "krav til seksjonering av bygget" in strings
    # Below the floor: a two-character string appears in almost any output by
    # accident and would inflate every converter's score equally.
    assert "ok" not in strings


def test_full_coverage_is_reported_when_every_string_survives(tmp_path: Path) -> None:
    path = docx(tmp_path)
    text = "\n".join(okf_fidelity.source_strings(path))
    result = okf_fidelity.score(path, text)
    assert (result.covered, result.total) == (3, 3)


def test_a_dropped_string_is_counted_as_lost(tmp_path: Path) -> None:
    """THE negative control. An instrument that cannot see a loss measures nothing.

    Without this test every 100 % above is compatible with a matcher that
    always says yes, and the whole K2 figure would be decoration.
    """
    path = docx(tmp_path)
    kept = list(okf_fidelity.source_strings(path))[:-1]
    result = okf_fidelity.score(path, "\n".join(kept))
    assert result.covered == 2
    assert result.total == 3
    assert result.covered < result.total


def test_the_converters_own_markup_is_not_counted_as_a_loss(tmp_path: Path) -> None:
    """Measured on the corpus: markdown escapes read as 47 missing strings.

    The question is whether the STRING survived, not whether the markup
    matches -- a converter is entitled to escape a bracket and to wrap a bold
    run, and scoring that as data loss would attribute a formatting choice to
    the pipeline.
    """
    path = docx(tmp_path)
    escaped = "\\[Krav til seksjonering av bygget\\] **Navn tilbyder:** value\nRoemningsveier skal vaere uavhengige"
    result = okf_fidelity.score(path, escaped)
    assert result.covered == 3


def test_pairing_needs_a_value_beside_the_label_not_just_the_label(tmp_path: Path) -> None:
    """A label alone on a line is the failure the criterion exists to catch."""
    path = docx(tmp_path)
    alone = "Navn tilbyder:\nKrav til seksjonering av bygget"
    beside = "Navn tilbyder: Entreprenoer AS\nKrav til seksjonering av bygget"
    assert okf_fidelity.score(path, alone).paired == 0
    assert okf_fidelity.score(path, beside).paired == 1
    assert okf_fidelity.score(path, beside).pairable == 1


def test_a_workbooks_shared_strings_are_read(tmp_path: Path) -> None:
    strings = okf_fidelity.source_strings(xlsx(tmp_path))
    assert set(strings) == {"prisskjema for entreprisen", "sum eks mva"}


def test_a_type_with_no_reader_is_refused_rather_than_scored_zero(tmp_path: Path) -> None:
    """Zero coverage and "no instrument" are different facts.

    The type here USED to be `.pptx`, which now has a reader. Replaced in place
    rather than deleted: the property is about the refusal, not about which
    suffix happens to lack a reader this week, and dropping the test with the
    reader would have removed the guarantee along with its example.
    """
    path = tmp_path / "x.epub"
    with zipfile.ZipFile(path, "w") as archive:
        archive.writestr("a", "b")
    with pytest.raises(ValueError):
        okf_fidelity.source_strings(path)


# --- the three office rows the corpus has none of --------------------------
#
# `pptx`, `odt` and `rtf` had no source-string reader here, so the instrument
# refused them and K2 for those rows could not be run at all -- see
# `docs/2026-09-07-k2-pptx-odt-rtf-fixtures.md`. Each reader is checked the
# same way as the two above: it finds the document's own strings, and it is
# able to see a loss.

SLIDE_XML = """<?xml version="1.0"?>
<p:sld xmlns:a="a" xmlns:p="p"><p:cSld><p:spTree>
<p:sp><p:txBody><a:p><a:r><a:t>Kravspesifikasjon for kjeller</a:t></a:r></a:p></p:txBody></p:sp>
<p:graphicFrame><a:tbl><a:tr>
<a:tc><a:txBody><a:p><a:r><a:t>Navn tilbyder:</a:t></a:r></a:p></a:txBody></a:tc>
<a:tc><a:txBody><a:p><a:r><a:t>Entreprenoer AS</a:t></a:r></a:p></a:txBody></a:tc>
</a:tr></a:tbl></p:graphicFrame>
</p:spTree></p:cSld></p:sld>
"""

CONTENT_XML = """<?xml version="1.0"?>
<office:document-content xmlns:office="o" xmlns:text="t" xmlns:table="tb">
<office:body><office:text>
<text:h text:outline-level="1">Kravspesifikasjon for kjeller</text:h>
<table:table><table:table-row>
<table:table-cell><text:p>Navn <text:span>tilbyder</text:span>:</text:p></table:table-cell>
<table:table-cell><text:p>Entreprenoer AS</text:p></table:table-cell>
</table:table-row></table:table>
</office:text></office:body></office:document-content>
"""

RTF_STREAM = (
    "{\\rtf1\\ansi\\deff0{\\fonttbl{\\f0\\froman Times New Roman;}}\n"
    "\\pard Kravspesifikasjon for kjeller\\par\n"
    "\\trowd\\cellx3000\\cellx6000\n"
    "\\pard\\intbl Navn tilbyder:\\cell \\pard\\intbl Entrepren\\u248 ?r AS\\cell \\row\n"
    "}"
)


def pptx(tmp_path: Path) -> Path:
    path = tmp_path / "p.pptx"
    with zipfile.ZipFile(path, "w") as archive:
        archive.writestr("ppt/slides/slide1.xml", SLIDE_XML)
    return path


def odt(tmp_path: Path) -> Path:
    path = tmp_path / "o.odt"
    with zipfile.ZipFile(path, "w") as archive:
        archive.writestr("content.xml", CONTENT_XML)
    return path


def rtf(tmp_path: Path) -> Path:
    path = tmp_path / "r.rtf"
    path.write_text(RTF_STREAM, encoding="ascii")
    return path


def test_a_presentations_slide_text_is_read(tmp_path: Path) -> None:
    """One string per `a:p`, table cells included -- a cell is a paragraph."""
    assert set(okf_fidelity.source_strings(pptx(tmp_path))) == {
        "kravspesifikasjon for kjeller",
        "navn tilbyder:",
        "entreprenoer as",
    }


def test_an_odf_bodys_paragraphs_are_read(tmp_path: Path) -> None:
    """Headings count, and a run split across a `text:span` is still one string."""
    assert set(okf_fidelity.source_strings(odt(tmp_path))) == {
        "kravspesifikasjon for kjeller",
        "navn tilbyder:",
        "entreprenoer as",
    }


def test_an_rtf_streams_paragraphs_and_cells_are_read(tmp_path: Path) -> None:
    """The control words are markup; the text between them is the document."""
    assert set(okf_fidelity.source_strings(rtf(tmp_path))) == {
        "kravspesifikasjon for kjeller",
        "navn tilbyder:",
        "entreprenør as",
    }


@pytest.mark.parametrize("build", [pptx, odt, rtf])
def test_each_new_reader_can_see_a_loss(build: Callable[[Path], Path], tmp_path: Path) -> None:
    """The negative control, once per reader.

    A reader that returns its strings and an instrument that always says yes
    are indistinguishable from a green number, so every reader added here owes
    the same proof as the first two.
    """
    path = build(tmp_path)
    strings = list(okf_fidelity.source_strings(path))
    result = okf_fidelity.score(path, "\n".join(strings[:-1]))
    assert result.covered == len(strings) - 1 < result.total


@pytest.mark.parametrize("build", [pptx, odt, rtf])
def test_each_new_reader_pairs_a_label_only_when_a_value_sits_beside_it(
    build: Callable[[Path], Path], tmp_path: Path
) -> None:
    path = build(tmp_path)
    label = next(item for item in okf_fidelity.source_strings(path) if item.endswith(":"))
    assert okf_fidelity.score(path, label).paired == 0
    assert okf_fidelity.score(path, f"{label} en verdi").paired == 1