feat(tools): re-measurable K2 fidelity instrument with a negative control

tests/test_fidelity.py

@@ -0,0 +1,121 @@
+"""The fidelity instrument: it must be able to see a loss before a 100 % means anything.
+
+This module exists because the arm A report of 2026-08-29 published `docx`
+193/196 and `xlsx` 193/193 without shipping the command that produced them.
+Those figures could not be re-measured against a new converter version, against
+the product path, or at all -- so a later run reporting the same numbers would
+have been agreement with a memory rather than with a measurement.
+
+The load-bearing test here is the NEGATIVE CONTROL: an instrument that returns
+100 % on text with strings deliberately removed is an instrument that returns
+100 % on everything, and every green figure it ever produced would be
+worthless. `test_a_dropped_string_is_counted_as_lost` is what makes the rest
+of the numbers mean something.
+"""
+
+from __future__ import annotations
+
+import sys
+import zipfile
+from pathlib import Path
+
+import pytest
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tools"))
+
+import okf_fidelity  # noqa: E402
+
+DOCUMENT_XML = """<?xml version="1.0"?>
+<w:document xmlns:w="x"><w:body>
+<w:p><w:r><w:t>Krav til seksjonering av bygget</w:t></w:r></w:p>
+<w:p><w:r><w:t>Navn tilbyder:</w:t></w:r></w:p>
+<w:p><w:r><w:t>Roemningsveier skal vaere uavhengige</w:t></w:r></w:p>
+<w:p><w:r><w:t>ok</w:t></w:r></w:p>
+</w:body></w:document>
+"""
+
+SHARED_XML = """<?xml version="1.0"?>
+<sst xmlns="x"><si><t>Prisskjema for entreprisen</t></si><si><t>Sum eks mva</t></si></sst>
+"""
+
+
+def docx(tmp_path: Path) -> Path:
+    path = tmp_path / "d.docx"
+    with zipfile.ZipFile(path, "w") as archive:
+        archive.writestr("word/document.xml", DOCUMENT_XML)
+    return path
+
+
+def xlsx(tmp_path: Path) -> Path:
+    path = tmp_path / "s.xlsx"
+    with zipfile.ZipFile(path, "w") as archive:
+        archive.writestr("xl/sharedStrings.xml", SHARED_XML)
+    return path
+
+
+def test_the_fasit_comes_from_the_file_not_from_a_converter(tmp_path: Path) -> None:
+    """A fasit derived from one converter would score it on its own homework."""
+    strings = okf_fidelity.source_strings(docx(tmp_path))
+    assert "krav til seksjonering av bygget" in strings
+    # Below the floor: a two-character string appears in almost any output by
+    # accident and would inflate every converter's score equally.
+    assert "ok" not in strings
+
+
+def test_full_coverage_is_reported_when_every_string_survives(tmp_path: Path) -> None:
+    path = docx(tmp_path)
+    text = "\n".join(okf_fidelity.source_strings(path))
+    result = okf_fidelity.score(path, text)
+    assert (result.covered, result.total) == (3, 3)
+
+
+def test_a_dropped_string_is_counted_as_lost(tmp_path: Path) -> None:
+    """THE negative control. An instrument that cannot see a loss measures nothing.
+
+    Without this test every 100 % above is compatible with a matcher that
+    always says yes, and the whole K2 figure would be decoration.
+    """
+    path = docx(tmp_path)
+    kept = list(okf_fidelity.source_strings(path))[:-1]
+    result = okf_fidelity.score(path, "\n".join(kept))
+    assert result.covered == 2
+    assert result.total == 3
+    assert result.covered < result.total
+
+
+def test_the_converters_own_markup_is_not_counted_as_a_loss(tmp_path: Path) -> None:
+    """Measured on the corpus: markdown escapes read as 47 missing strings.
+
+    The question is whether the STRING survived, not whether the markup
+    matches -- a converter is entitled to escape a bracket and to wrap a bold
+    run, and scoring that as data loss would attribute a formatting choice to
+    the pipeline.
+    """
+    path = docx(tmp_path)
+    escaped = "\\[Krav til seksjonering av bygget\\] **Navn tilbyder:** value\nRoemningsveier skal vaere uavhengige"
+    result = okf_fidelity.score(path, escaped)
+    assert result.covered == 3
+
+
+def test_pairing_needs_a_value_beside_the_label_not_just_the_label(tmp_path: Path) -> None:
+    """A label alone on a line is the failure the criterion exists to catch."""
+    path = docx(tmp_path)
+    alone = "Navn tilbyder:\nKrav til seksjonering av bygget"
+    beside = "Navn tilbyder: Entreprenoer AS\nKrav til seksjonering av bygget"
+    assert okf_fidelity.score(path, alone).paired == 0
+    assert okf_fidelity.score(path, beside).paired == 1
+    assert okf_fidelity.score(path, beside).pairable == 1
+
+
+def test_a_workbooks_shared_strings_are_read(tmp_path: Path) -> None:
+    strings = okf_fidelity.source_strings(xlsx(tmp_path))
+    assert set(strings) == {"prisskjema for entreprisen", "sum eks mva"}
+
+
+def test_a_type_with_no_reader_is_refused_rather_than_scored_zero(tmp_path: Path) -> None:
+    """Zero coverage and "no instrument" are different facts."""
+    path = tmp_path / "x.pptx"
+    with zipfile.ZipFile(path, "w") as archive:
+        archive.writestr("a", "b")
+    with pytest.raises(ValueError):
+        okf_fidelity.source_strings(path)

tools/okf_fidelity.py

@@ -0,0 +1,179 @@
+"""Measure what survives conversion: the source's own strings, and their pairing.
+
+The instrument behind K2. It exists because the arm A report of 2026-08-29
+published `docx` 193/196 and `xlsx` 193/193 without shipping the command that
+produced them, so those figures could not be re-measured -- against a new
+converter version, against the product path, or at all.
+
+Two questions, both of them the source document's own:
+
+- **coverage** -- of the strings the file itself stores (a `docx` paragraph, an
+  `xlsx` cell), how many appear in the converted text. Not character count: a
+  converter emitting 2.7x more characters can carry FEWER source strings, which
+  arm A measured directly.
+- **pairing** -- of the source rows carrying a label and a value, how many keep
+  both on one output line. That is the criterion a requirement table is read
+  by, and the one that separates a usable conversion from a pile of words.
+
+Comparison is normalised (whitespace collapsed, case folded) because a
+converter is entitled to re-wrap; it is not entitled to lose a string.
+
+Run standalone and product side by side. A fall from one to the other is a
+question this repository has to answer, not a number to publish: a product
+figure alone cannot tell a converter limit from a defect we introduced.
+"""
+
+from __future__ import annotations
+
+import argparse
+import re
+import sys
+import zipfile
+from dataclasses import dataclass
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
+
+from llm_ingestion_okf.extract import extract_text  # noqa: E402
+
+#: A source string shorter than this is not evidence either way -- a lone digit
+#: or a stray bullet appears in almost any output by accident, so counting it
+#: would inflate coverage for every converter equally and measure nothing.
+MIN_STRING = 3
+
+_WS = re.compile(r"\s+")
+_XML_TEXT = re.compile(r"<(?:w:t|t)(?:\s[^>]*)?>(.*?)</(?:w:t|t)>", re.DOTALL)
+_XML_PARA = re.compile(r"<w:p[\s>].*?</w:p>", re.DOTALL)
+_XML_SI = re.compile(r"<si>(.*?)</si>", re.DOTALL)
+_ENTITIES = (("&amp;", "&"), ("&lt;", "<"), ("&gt;", ">"), ("&quot;", '"'), ("&apos;", "'"))
+
+
+#: Stripped from both sides before comparison. These are the converter's own
+#: MARKUP, not the document's content: markdown escapes a `[`, wraps a bold run
+#: in `**`, and writes an en dash as `--`. Measured on the corpus -- without
+#: this, `docx` coverage reads 149/196 while every one of the 47 "missing"
+#: strings is present in the output verbatim, wearing an escape. The question
+#: is whether the STRING survived, not whether the markup matches.
+_MARKUP = str.maketrans({char: None for char in "\\*_`"})
+_DASHES = str.maketrans({"\u2013": "-", "\u2014": "-", "\u00a0": " ", "\u2019": "'"})
+
+
+def normalise(text: str) -> str:
+    for entity, char in _ENTITIES:
+        text = text.replace(entity, char)
+    text = text.translate(_MARKUP).translate(_DASHES)
+    text = text.replace("--", "-")
+    return _WS.sub(" ", text).strip().casefold()
+
+
+def _runs(fragment: str) -> str:
+    return "".join(match.group(1) for match in _XML_TEXT.finditer(fragment))
+
+
+def source_strings(path: Path) -> tuple[str, ...]:
+    """The strings the FILE itself stores, deduplicated and in document order.
+
+    Read from the package's own XML rather than from any converter's output:
+    a fasit derived from one converter would score that converter on its own
+    homework.
+    """
+    suffix = path.suffix.lower()
+    with zipfile.ZipFile(path) as archive:
+        names = set(archive.namelist())
+        if suffix == ".docx":
+            body = archive.read("word/document.xml").decode("utf-8", "replace")
+            found = [_runs(match.group(0)) for match in _XML_PARA.finditer(body)]
+        elif suffix == ".xlsx":
+            found = []
+            if "xl/sharedStrings.xml" in names:
+                shared = archive.read("xl/sharedStrings.xml").decode("utf-8", "replace")
+                found = [_runs(match.group(1)) for match in _XML_SI.finditer(shared)]
+        else:
+            raise ValueError(f"no source-string reader for {suffix!r}")
+
+    seen: dict[str, None] = {}
+    for item in found:
+        value = normalise(item)
+        if len(value) >= MIN_STRING:
+            seen.setdefault(value, None)
+    return tuple(seen)
+
+
+def label_value_rows(strings: tuple[str, ...]) -> tuple[str, ...]:
+    """Source strings that read as a label expecting a value beside it.
+
+    A row is usable for pairing when its own text ends in a colon or is a short
+    noun phrase followed by a value elsewhere on the row. Approximated by the
+    colon rule alone, which is stated rather than tuned: a threshold fitted to
+    make a number look good is not a measurement.
+    """
+    return tuple(item for item in strings if item.endswith(":"))
+
+
+@dataclass(frozen=True)
+class Fidelity:
+    path: str
+    covered: int
+    total: int
+    paired: int
+    pairable: int
+
+    def line(self) -> str:
+        share = 100.0 * self.covered / self.total if self.total else 0.0
+        pair = 100.0 * self.paired / self.pairable if self.pairable else 0.0
+        return (
+            f"{self.path}: coverage {self.covered}/{self.total} ({share:.1f} %), "
+            f"paired {self.paired}/{self.pairable} ({pair:.1f} %)"
+        )
+
+
+def score(path: Path, text: str) -> Fidelity:
+    strings = source_strings(path)
+    haystack = normalise(text)
+    lines = [normalise(line) for line in text.splitlines()]
+    covered = sum(1 for item in strings if item in haystack)
+    pairable = label_value_rows(strings)
+    paired = sum(
+        1 for label in pairable if any(label in line and len(line) > len(label) for line in lines)
+    )
+    return Fidelity(
+        path=path.name, covered=covered, total=len(strings), paired=paired, pairable=len(pairable)
+    )
+
+
+def product_text(path: Path) -> str:
+    """The text the LIBRARY produces -- the same call the door makes."""
+    return extract_text(path.name, path.read_bytes())
+
+
+def standalone_text(path: Path) -> str:
+    """The text the converter produces on its own, at the pinned version."""
+    import pypandoc
+
+    from llm_ingestion_okf._pandoc import converter_path
+
+    formats = {".docx": "docx", ".xlsx": "xlsx", ".pptx": "pptx", ".odt": "odt", ".rtf": "rtf"}
+    with converter_path():
+        return str(
+            pypandoc.convert_file(
+                str(path),
+                "markdown",
+                format=formats[path.suffix.lower()],
+                extra_args=["--eol=lf", "--wrap=none"],
+            )
+        )
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("files", nargs="+", type=Path)
+    args = parser.parse_args(argv)
+    for path in args.files:
+        print(f"--- {path.name}")
+        print(f"  standalone  {score(path, standalone_text(path)).line()}")
+        print(f"  product     {score(path, product_text(path)).line()}")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())