llm-ingestion-okf/tools/okf_fidelity.py

"""Measure what survives conversion: the source's own strings, and their pairing.

The instrument behind K2. It exists because the arm A report of 2026-08-29
published `docx` 193/196 and `xlsx` 193/193 without shipping the command that
produced them, so those figures could not be re-measured -- against a new
converter version, against the product path, or at all.

Two questions, both of them the source document's own:

- **coverage** -- of the strings the file itself stores (a `docx` paragraph, an
  `xlsx` cell), how many appear in the converted text. Not character count: a
  converter emitting 2.7x more characters can carry FEWER source strings, which
  arm A measured directly.
- **pairing** -- of the source rows carrying a label and a value, how many keep
  both on one output line. That is the criterion a requirement table is read
  by, and the one that separates a usable conversion from a pile of words.

Comparison is normalised (whitespace collapsed, case folded) because a
converter is entitled to re-wrap; it is not entitled to lose a string.

Run standalone and product side by side. A fall from one to the other is a
question this repository has to answer, not a number to publish: a product
figure alone cannot tell a converter limit from a defect we introduced.
"""

from __future__ import annotations

import argparse
import re
import sys
import zipfile
from dataclasses import dataclass
from pathlib import Path

sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))

from llm_ingestion_okf.extract import extract_text  # noqa: E402

#: A source string shorter than this is not evidence either way -- a lone digit
#: or a stray bullet appears in almost any output by accident, so counting it
#: would inflate coverage for every converter equally and measure nothing.
MIN_STRING = 3

_WS = re.compile(r"\s+")
_XML_TEXT = re.compile(r"<(?:w:t|t)(?:\s[^>]*)?>(.*?)</(?:w:t|t)>", re.DOTALL)
_XML_PARA = re.compile(r"<w:p[\s>].*?</w:p>", re.DOTALL)
_XML_SI = re.compile(r"<si>(.*?)</si>", re.DOTALL)
_ENTITIES = (("&amp;", "&"), ("&lt;", "<"), ("&gt;", ">"), ("&quot;", '"'), ("&apos;", "'"))


#: Stripped from both sides before comparison. These are the converter's own
#: MARKUP, not the document's content: markdown escapes a `[`, wraps a bold run
#: in `**`, and writes an en dash as `--`. Measured on the corpus -- without
#: this, `docx` coverage reads 149/196 while every one of the 47 "missing"
#: strings is present in the output verbatim, wearing an escape. The question
#: is whether the STRING survived, not whether the markup matches.
_MARKUP = str.maketrans({char: None for char in "\\*_`"})
_DASHES = str.maketrans({"\u2013": "-", "\u2014": "-", "\u00a0": " ", "\u2019": "'"})


def normalise(text: str) -> str:
    for entity, char in _ENTITIES:
        text = text.replace(entity, char)
    text = text.translate(_MARKUP).translate(_DASHES)
    text = text.replace("--", "-")
    return _WS.sub(" ", text).strip().casefold()


def _runs(fragment: str) -> str:
    return "".join(match.group(1) for match in _XML_TEXT.finditer(fragment))


def source_strings(path: Path) -> tuple[str, ...]:
    """The strings the FILE itself stores, deduplicated and in document order.

    Read from the package's own XML rather than from any converter's output:
    a fasit derived from one converter would score that converter on its own
    homework.
    """
    suffix = path.suffix.lower()
    with zipfile.ZipFile(path) as archive:
        names = set(archive.namelist())
        if suffix == ".docx":
            body = archive.read("word/document.xml").decode("utf-8", "replace")
            found = [_runs(match.group(0)) for match in _XML_PARA.finditer(body)]
        elif suffix == ".xlsx":
            found = []
            if "xl/sharedStrings.xml" in names:
                shared = archive.read("xl/sharedStrings.xml").decode("utf-8", "replace")
                found = [_runs(match.group(1)) for match in _XML_SI.finditer(shared)]
        else:
            raise ValueError(f"no source-string reader for {suffix!r}")

    seen: dict[str, None] = {}
    for item in found:
        value = normalise(item)
        if len(value) >= MIN_STRING:
            seen.setdefault(value, None)
    return tuple(seen)


def label_value_rows(strings: tuple[str, ...]) -> tuple[str, ...]:
    """Source strings that read as a label expecting a value beside it.

    A row is usable for pairing when its own text ends in a colon or is a short
    noun phrase followed by a value elsewhere on the row. Approximated by the
    colon rule alone, which is stated rather than tuned: a threshold fitted to
    make a number look good is not a measurement.
    """
    return tuple(item for item in strings if item.endswith(":"))


@dataclass(frozen=True)
class Fidelity:
    path: str
    covered: int
    total: int
    paired: int
    pairable: int

    def line(self) -> str:
        share = 100.0 * self.covered / self.total if self.total else 0.0
        pair = 100.0 * self.paired / self.pairable if self.pairable else 0.0
        return (
            f"{self.path}: coverage {self.covered}/{self.total} ({share:.1f} %), "
            f"paired {self.paired}/{self.pairable} ({pair:.1f} %)"
        )


def score(path: Path, text: str) -> Fidelity:
    strings = source_strings(path)
    haystack = normalise(text)
    lines = [normalise(line) for line in text.splitlines()]
    covered = sum(1 for item in strings if item in haystack)
    pairable = label_value_rows(strings)
    paired = sum(
        1 for label in pairable if any(label in line and len(line) > len(label) for line in lines)
    )
    return Fidelity(
        path=path.name, covered=covered, total=len(strings), paired=paired, pairable=len(pairable)
    )


def product_text(path: Path) -> str:
    """The text the LIBRARY produces -- the same call the door makes."""
    return extract_text(path.name, path.read_bytes())


def standalone_text(path: Path) -> str:
    """The text the converter produces on its own, at the pinned version."""
    import pypandoc

    from llm_ingestion_okf._pandoc import converter_path

    formats = {".docx": "docx", ".xlsx": "xlsx", ".pptx": "pptx", ".odt": "odt", ".rtf": "rtf"}
    with converter_path():
        return str(
            pypandoc.convert_file(
                str(path),
                "markdown",
                format=formats[path.suffix.lower()],
                extra_args=["--eol=lf", "--wrap=none"],
            )
        )


def main(argv: list[str] | None = None) -> int:
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("files", nargs="+", type=Path)
    args = parser.parse_args(argv)
    for path in args.files:
        print(f"--- {path.name}")
        print(f"  standalone  {score(path, standalone_text(path)).line()}")
        print(f"  product     {score(path, product_text(path)).line()}")
    return 0


if __name__ == "__main__":
    raise SystemExit(main())