feat(tools): re-measurable K2 fidelity instrument with a negative control

tools/okf_fidelity.py

@@ -0,0 +1,179 @@
+"""Measure what survives conversion: the source's own strings, and their pairing.
+
+The instrument behind K2. It exists because the arm A report of 2026-08-29
+published `docx` 193/196 and `xlsx` 193/193 without shipping the command that
+produced them, so those figures could not be re-measured -- against a new
+converter version, against the product path, or at all.
+
+Two questions, both of them the source document's own:
+
+- **coverage** -- of the strings the file itself stores (a `docx` paragraph, an
+  `xlsx` cell), how many appear in the converted text. Not character count: a
+  converter emitting 2.7x more characters can carry FEWER source strings, which
+  arm A measured directly.
+- **pairing** -- of the source rows carrying a label and a value, how many keep
+  both on one output line. That is the criterion a requirement table is read
+  by, and the one that separates a usable conversion from a pile of words.
+
+Comparison is normalised (whitespace collapsed, case folded) because a
+converter is entitled to re-wrap; it is not entitled to lose a string.
+
+Run standalone and product side by side. A fall from one to the other is a
+question this repository has to answer, not a number to publish: a product
+figure alone cannot tell a converter limit from a defect we introduced.
+"""
+
+from __future__ import annotations
+
+import argparse
+import re
+import sys
+import zipfile
+from dataclasses import dataclass
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
+
+from llm_ingestion_okf.extract import extract_text  # noqa: E402
+
+#: A source string shorter than this is not evidence either way -- a lone digit
+#: or a stray bullet appears in almost any output by accident, so counting it
+#: would inflate coverage for every converter equally and measure nothing.
+MIN_STRING = 3
+
+_WS = re.compile(r"\s+")
+_XML_TEXT = re.compile(r"<(?:w:t|t)(?:\s[^>]*)?>(.*?)</(?:w:t|t)>", re.DOTALL)
+_XML_PARA = re.compile(r"<w:p[\s>].*?</w:p>", re.DOTALL)
+_XML_SI = re.compile(r"<si>(.*?)</si>", re.DOTALL)
+_ENTITIES = (("&amp;", "&"), ("&lt;", "<"), ("&gt;", ">"), ("&quot;", '"'), ("&apos;", "'"))
+
+
+#: Stripped from both sides before comparison. These are the converter's own
+#: MARKUP, not the document's content: markdown escapes a `[`, wraps a bold run
+#: in `**`, and writes an en dash as `--`. Measured on the corpus -- without
+#: this, `docx` coverage reads 149/196 while every one of the 47 "missing"
+#: strings is present in the output verbatim, wearing an escape. The question
+#: is whether the STRING survived, not whether the markup matches.
+_MARKUP = str.maketrans({char: None for char in "\\*_`"})
+_DASHES = str.maketrans({"\u2013": "-", "\u2014": "-", "\u00a0": " ", "\u2019": "'"})
+
+
+def normalise(text: str) -> str:
+    for entity, char in _ENTITIES:
+        text = text.replace(entity, char)
+    text = text.translate(_MARKUP).translate(_DASHES)
+    text = text.replace("--", "-")
+    return _WS.sub(" ", text).strip().casefold()
+
+
+def _runs(fragment: str) -> str:
+    return "".join(match.group(1) for match in _XML_TEXT.finditer(fragment))
+
+
+def source_strings(path: Path) -> tuple[str, ...]:
+    """The strings the FILE itself stores, deduplicated and in document order.
+
+    Read from the package's own XML rather than from any converter's output:
+    a fasit derived from one converter would score that converter on its own
+    homework.
+    """
+    suffix = path.suffix.lower()
+    with zipfile.ZipFile(path) as archive:
+        names = set(archive.namelist())
+        if suffix == ".docx":
+            body = archive.read("word/document.xml").decode("utf-8", "replace")
+            found = [_runs(match.group(0)) for match in _XML_PARA.finditer(body)]
+        elif suffix == ".xlsx":
+            found = []
+            if "xl/sharedStrings.xml" in names:
+                shared = archive.read("xl/sharedStrings.xml").decode("utf-8", "replace")
+                found = [_runs(match.group(1)) for match in _XML_SI.finditer(shared)]
+        else:
+            raise ValueError(f"no source-string reader for {suffix!r}")
+
+    seen: dict[str, None] = {}
+    for item in found:
+        value = normalise(item)
+        if len(value) >= MIN_STRING:
+            seen.setdefault(value, None)
+    return tuple(seen)
+
+
+def label_value_rows(strings: tuple[str, ...]) -> tuple[str, ...]:
+    """Source strings that read as a label expecting a value beside it.
+
+    A row is usable for pairing when its own text ends in a colon or is a short
+    noun phrase followed by a value elsewhere on the row. Approximated by the
+    colon rule alone, which is stated rather than tuned: a threshold fitted to
+    make a number look good is not a measurement.
+    """
+    return tuple(item for item in strings if item.endswith(":"))
+
+
+@dataclass(frozen=True)
+class Fidelity:
+    path: str
+    covered: int
+    total: int
+    paired: int
+    pairable: int
+
+    def line(self) -> str:
+        share = 100.0 * self.covered / self.total if self.total else 0.0
+        pair = 100.0 * self.paired / self.pairable if self.pairable else 0.0
+        return (
+            f"{self.path}: coverage {self.covered}/{self.total} ({share:.1f} %), "
+            f"paired {self.paired}/{self.pairable} ({pair:.1f} %)"
+        )
+
+
+def score(path: Path, text: str) -> Fidelity:
+    strings = source_strings(path)
+    haystack = normalise(text)
+    lines = [normalise(line) for line in text.splitlines()]
+    covered = sum(1 for item in strings if item in haystack)
+    pairable = label_value_rows(strings)
+    paired = sum(
+        1 for label in pairable if any(label in line and len(line) > len(label) for line in lines)
+    )
+    return Fidelity(
+        path=path.name, covered=covered, total=len(strings), paired=paired, pairable=len(pairable)
+    )
+
+
+def product_text(path: Path) -> str:
+    """The text the LIBRARY produces -- the same call the door makes."""
+    return extract_text(path.name, path.read_bytes())
+
+
+def standalone_text(path: Path) -> str:
+    """The text the converter produces on its own, at the pinned version."""
+    import pypandoc
+
+    from llm_ingestion_okf._pandoc import converter_path
+
+    formats = {".docx": "docx", ".xlsx": "xlsx", ".pptx": "pptx", ".odt": "odt", ".rtf": "rtf"}
+    with converter_path():
+        return str(
+            pypandoc.convert_file(
+                str(path),
+                "markdown",
+                format=formats[path.suffix.lower()],
+                extra_args=["--eol=lf", "--wrap=none"],
+            )
+        )
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("files", nargs="+", type=Path)
+    args = parser.parse_args(argv)
+    for path in args.files:
+        print(f"--- {path.name}")
+        print(f"  standalone  {score(path, standalone_text(path)).line()}")
+        print(f"  product     {score(path, product_text(path)).line()}")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())