llm-ingestion-okf/tools/okf_gate_mutants.py

"""Mutation run over the content-accounting gate and its witness.

The gate judges `okf build`; this asks what judges the gate. Each mutant is ONE
textual edit that makes the instrument weaker in a way a reader would call a
defect, and the question is whether the suite goes red. A mutant that survives
names a check nothing holds.

Twenty-six of them are an independent review's, ported to the code as it
stands rather than re-invented: at `0b00de4` twelve of them survived the
forty-two tests, among them `main` always returning 0 and six of the seven
witness mutants -- which had no fixture that could exercise the element they
removed.

A MUTANT MAY EDIT `src/`, and until 2026-09-19 one could not. The copy is run
with the venv's interpreter, which carries an EDITABLE install pointing at the
original tree, so `import llm_ingestion_okf` in the copy resolved to the
working tree and a `src/` mutant was reported as a survivor without ever
having been applied -- measured on X5, which survived 112 green tests and then
died on the first run with `PYTHONPATH` set. The subprocess now gets the
copy's own `src/` on `PYTHONPATH`, which wins over the editable finder. The
gate and the witness were never affected: the suite reaches those through
`sys.path.insert(0, TOOLS)` off its own location, which is already the copy.

    python3 tools/okf_gate_mutants.py

Runs on a COPY of the tree in a temporary directory: the working tree is never
edited, so an interrupted run cannot leave a mutant behind. A pattern that does
not match the expected number of times is reported as an ERROR and the run
exits 2 -- a mutant that could not be applied was never measured, and counting
it as killed is the same mistake as reading an empty search as an absence.

A SURVIVOR EXITS 1. Until 2026-09-19 the run ended `2 if errors else 0`, so
`killed 0 of 1` with the survivor printed beside it was an exit 0 and nothing
could fail on the finding this harness exists to produce (H4).
"""

from __future__ import annotations

import os
import shutil
import subprocess
import sys
import tempfile
from collections.abc import Sequence
from dataclasses import dataclass
from pathlib import Path

REPO = Path(__file__).resolve().parent.parent
GATE = "tools/okf_accounting_gate.py"
WITNESS = "tools/okf_witness.py"
SUITE = "tests/test_accounting_gate.py"


@dataclass(frozen=True)
class Mutant:
    label: str
    file: str
    old: str
    new: str
    #: How many times `old` must occur; `first_only` edits the first of them
    #: when a line is written more than once on purpose.
    occurrences: int = 1
    first_only: bool = False
    #: The test file this mutant is judged by. Defaults to the gate's own
    #: suite, which every mutant used until 2026-09-19 -- which is why the
    #: three PM found in `43331fc` could not be added here: they are held by
    #: the SHY door's suite and by the gate's row 3, and a runner that can
    #: only run one file cannot ask about them.
    suite: str = SUITE


MUTANTS: tuple[Mutant, ...] = (
    Mutant(
        "M01 m=0 counts as green ('not measured' -> GREEN)",
        GATE,
        "GREEN if m > 0 and k == m else RED",
        "GREEN if k == m else RED",
    ),
    Mutant(
        "M02 row 6 never fails the gate",
        GATE,
        "self.status == RED and self.number <= 6",
        "self.status == RED and self.number <= 5",
    ),
    Mutant(
        "M03 a missing corpus is SKIPPED locally too",
        GATE,
        "status = SKIPPED if ci else RED",
        "status = SKIPPED",
    ),
    Mutant(
        "M04 double booking of elements never counted",
        GATE,
        "double += booked - have",
        "double += 0",
    ),
    Mutant(
        "M05 'declared carried, bytes absent' never a false claim",
        GATE,
        'false_claim = fate == "carried" and not carried',
        "false_claim = False",
    ),
    Mutant(
        "M06 an unpointed duplicate counts as carried",
        GATE,
        "carried = bool(pointed_by) and asset_holds(build, corpus / name)",
        "carried = asset_holds(build, corpus / name)",
    ),
    Mutant(
        "M07 row 2 accepts ANY declared inventory",
        GATE,
        "if got != want:",
        "if got is None:",
    ),
    Mutant(
        "M08 row 1 ignores a stale committed fasit",
        GATE,
        'if fresh["documents"].get(name) == entry:',
        "if True:",
    ),
    Mutant(
        "M09 row 4 ignores the Images 'found' count",
        GATE,
        "elif int(found.group(2)) != declared_images:",
        "elif False:",
    ),
    Mutant(
        "M10 row 4 accepts any element total in the log line",
        GATE,
        'rf"{re.escape(doc)}: {total} elements found in the source, 0 carried: "',
        'rf"{re.escape(doc)}: \\d+ elements found in the source, 0 carried: "',
    ),
    Mutant(
        "M11 an unavailable witness is agreement",
        GATE,
        'return ["a witness is unavailable"]',
        "return []",
    ),
    Mutant(
        "M12 row 5 counts every pair as agreeing",
        GATE,
        "good = sum(1 for _, problems in pairs if not problems)",
        "good = len(pairs)",
    ),
    Mutant(
        "M13 row 6: clean in ANY of the two builds is enough",
        GATE,
        "if all(unit.clean for unit in parts)",
        "if any(unit.clean for unit in parts)",
    ),
    Mutant(
        "M14 main always exits 0",
        GATE,
        "    if any(r.fails for r in rows):\n        return 1",
        "    if False:\n        return 1",
    ),
    Mutant(
        "M15 T silently drops .rtf",
        GATE,
        'return sorted(set(re.findall(r"^\\| `(\\.[a-z0-9]+)` \\|", section, re.MULTILINE)))',
        'return sorted(\n        set(re.findall(r"^\\| `(\\.[a-z0-9]+)` \\|", section, re.MULTILINE)) - {".rtf"}\n    )',
    ),
    Mutant(
        "M16 an element with no declared fate is ignored",
        GATE,
        "for element in sorted(set(elements) | set(fates)):",
        "for element in sorted(set(fates)):",
    ),
    Mutant(
        "M17 conservation always 'held'",
        GATE,
        'return build.exit_code == 0 and "K1b FAILED" not in build.log',
        "return True",
    ),
    Mutant(
        "M18 row 3 counts every unit as clean",
        GATE,
        "clean = sum(1 for u in units if u.clean)",
        "clean = len(units)",
    ),
    Mutant(
        "M19 a negative rejection is absorbed into the total",
        GATE,
        "booked = carried + pointer + sum(rejected.values())",
        "booked = carried + pointer + abs(sum(rejected.values()))",
    ),
    # --- the judge's own reading of the bundle, added 2026-09-18 -------------
    Mutant(
        "M20 a booked carry is never looked for in the bundle",
        GATE,
        "found = sum(1 for pieces in sayable if all(finder(piece) for piece in pieces))",
        "found = len(elements_pieces)",
    ),
    Mutant(
        "M21 an unverified booking is clean",
        GATE,
        "            self.unaccounted or self.double or self.unverified "
        "or self.invalid or self.refused",
        "            self.unaccounted or self.double",
    ),
    # H1's own column, mutated on its own: M21 above removes `unverified`,
    # `invalid` AND `refused` at once, so it would be killed by any one of the
    # three. This one takes only the fifth.
    Mutant(
        "X3 a document refused whole is clean again",
        GATE,
        '    if status == "rejected" and not persisted and total > 0:',
        "    if False:",
    ),
    Mutant(
        "M22 any asset file at all proves a carry",
        GATE,
        "        found == digest and name.startswith(digest[:12])"
        " for name, found in build.assets.items()",
        "        True for name, found in build.assets.items()",
    ),
    Mutant(
        "M23 a corpus refused whole is not reported",
        GATE,
        "    if persisted:\n        return None",
        "    if True:\n        return None",
    ),
    Mutant(
        "M24 a rejection code outside the closed list is accepted",
        GATE,
        "        unknown = sorted(c for c in rejected if c not in REJECTION_CODES)",
        "        unknown = []",
    ),
    Mutant(
        "M25 a skipped row leaves the verdict unqualified",
        GATE,
        '        verdict += f" (row {row.number} not run: {row.reason})"',
        '        verdict += ""',
    ),
    Mutant(
        "W01 docx: separator footnotes counted",
        WITNESS,
        'if role != "comment" and int(note.get(f"{_W}id", "0")) <= 0:',
        'if role != "comment" and int(note.get(f"{_W}id", "0")) <= -1:',
    ),
    Mutant(
        "W02 pptx: pictures never counted",
        WITNESS,
        '            for _ in root.iter(f"{_P}pic"):\n                count.add("image")',
        '            for _ in ():\n                count.add("image")',
    ),
    Mutant(
        "W03 sts (xml witness): footnotes have no role",
        WITNESS,
        '    if tag == "fn":\n        return "footnote"',
        '    if tag == "fn":\n        return None',
    ),
    Mutant(
        "W04 xlsx: images never counted",
        WITNESS,
        '                for _ in root.iter(f"{_XDR}pic"):\n                    count.add("image")',
        '                for _ in ():\n                    count.add("image")',
    ),
    Mutant(
        "W05 odt: list items never counted",
        WITNESS,
        '    for item in root.iter(f"{_TEXT}list-item"):',
        "    for item in ():",
    ),
    Mutant(
        "W06 docx: tables never counted",
        WITNESS,
        '    for table in root.iter(f"{_W}tbl"):',
        "    for table in ():",
    ),
    Mutant(
        "W07 sts (xml witness): table cells have no role",
        WITNESS,
        '    if tag in ("td", "th"):\n        return "cell"',
        '    if tag in ("td", "th"):\n        return None',
    ),
    Mutant(
        "W08 docx: a header or footer is never counted",
        WITNESS,
        '                    count.add("header_footer", *lines)',
        "                    pass",
    ),
    # PM's own mutant, written outside this file 2026-09-18 and SURVIVING 155
    # green tests (H2). B-1 gave the judge six refusals; this was the one
    # nothing held from either side.
    Mutant(
        "X2 a report may declare a document rejected while the bundle holds it",
        GATE,
        '    if status == "rejected" and persisted:\n        invalid += 1',
        "    if False:\n        invalid += 1",
    ),
    # PM's sjekkpunkt 2026-09-19: the conversion route the viewable-asset round
    # added reads two digests out of the bundle, which put an untrusted document
    # inside the judge's own input. Three edits, one per check that closes it.
    # X5 edits `src/`, which no other mutant here does: the door that keeps a
    # LABEL from emitting the judge's grammar lives in the build, and the test
    # that fells it lives in this gate's suite, so it is measured from here.
    Mutant(
        "X3 the conversion claim is read from anywhere in the bundle text",
        GATE,
        """    declared = _declared_conversions(build)
    if not declared:
        return {}
    found: dict[str, str] = {}""",
        """    declared = _declared_conversions(build)
    del declared
    return {
        m.group("before"): m.group("after") for m in _CONVERSION.finditer(build.bundle_text)
    }
    found: dict[str, str] = {}""",
    ),
    Mutant(
        "X4 the claim need not be about the asset its block points at",
        GATE,
        'if (before, after) in declared and pointer.group("asset").startswith(after[:12]):',
        "if (before, after) in declared:",
    ),
    # X5 moved suites 2026-09-19. It was felled by the gate's own suite while
    # the gate read its claim out of the bundle; once the claim came from the
    # run's ledger, a document-supplied field could not reach the gate at all
    # and the mutant survived the WHOLE suite -- measured, 2134 passed. The
    # property did not stop mattering: the line is in every concept body, and
    # a bundle must not state a conversion the run never performed. It is
    # measured where it lives now.
    Mutant(
        "X5 a document-supplied label may emit a checksum field",
        "src/llm_ingestion_okf/assets.py",
        'return _CHECKSUM_FIELD.sub("sha256 ", collapsed.replace("[", "(").replace("]", ")"))',
        'return collapsed.replace("[", "(").replace("]", ")")',
        suite="tests/test_assets.py",
    ),
    # PM's sjekkpunkt 2026-09-19 on `ae441ab`: the round above bound the claim
    # to a pointer BLOCK, and one HTML file with two `<p>` elements writes one.
    # X6 is that state exactly -- the shape believed without the run's ledger.
    Mutant(
        "X6 a pointer block is believed without the run having booked it",
        GATE,
        'if (before, after) in declared and pointer.group("asset").startswith(after[:12]):',
        'if pointer.group("asset").startswith(after[:12]):',
    ),
    Mutant(
        "X7 the run's ledger is read from the bundle instead of the accounting",
        GATE,
        "    accounting = build.accounting\n    if not isinstance(accounting, dict):",
        "    accounting = None\n    if not isinstance(accounting, dict):",
    ),
    # The RLE8 cursor rule, judged by the file that measures it against an
    # independent decoder. Without a per-mutant suite this could not be asked
    # from here at all.
    Mutant(
        "X8 a stream may end before the cursor reaches the frame",
        "src/llm_ingestion_okf/assets.py",
        "    if y < height - 1 or (y == height - 1 and x < width):",
        "    if False:",
        suite="tests/test_asset_viewable.py",
    ),
    # PM's P8 from `44ad845`: the cursor rule was held by no arm that could
    # see its ROW clause, so a rule one row too lenient survived 51 tests.
    Mutant(
        "P8 the cursor rule is one row too lenient (height - 1 -> height - 2)",
        "src/llm_ingestion_okf/assets.py",
        "    if y < height - 1 or (y == height - 1 and x < width):",
        "    if y < height - 2 or (y == height - 1 and x < width):",
        suite="tests/test_asset_viewable.py",
    ),
    # And the clause added beside it, so the round that wrote it leaves a
    # mutant behind rather than only a test.
    Mutant(
        "P13 an end-of-line escape may claim a row it never started",
        "src/llm_ingestion_okf/assets.py",
        "        if value == 0:\n            if x == 0:",
        "        if value == 0:\n            if False:",
        suite="tests/test_asset_viewable.py",
    ),
    # PM's three survivors from `43331fc`, held until now by ordinary tests
    # and not by what runs AS the gate.
    Mutant(
        "P6 the normalisation door removes U+00A0 as well",
        "src/llm_ingestion_okf/extract.py",
        '    return text.replace(SOFT_HYPHEN, ""), removed',
        '    return text.replace(SOFT_HYPHEN, "").replace("\\u00a0", ""), removed',
        suite="tests/test_soft_hyphen_door.py",
    ),
    Mutant(
        "P11 row 3's detail line drops the refused count",
        GATE,
        'f"unverified={u.unverified} invalid={u.invalid} refused={u.refused}"',
        'f"unverified={u.unverified} invalid={u.invalid}"',
    ),
    Mutant(
        "P12 row 3's reason drops the refused-whole clause",
        GATE,
        'f"{refused} element(s) lost with {refused_docs} of {documents} document(s) refused whole"',
        'f"{refused} element(s) lost"',
    ),
)


def _apply(text: str, mutant: Mutant) -> str:
    return text.replace(mutant.old, mutant.new, 1 if mutant.first_only else -1)


def verdict(survived: Sequence[str], errors: Sequence[str]) -> int:
    """The run's exit code: 0 clean, 1 a mutant survived, 2 one was not measured.

    Until 2026-09-19 this was `2 if errors else 0`, so a run that printed
    `killed 0 of 1` and named its survivor exited 0 and no caller could fail
    on it (H4). A mutant that survived names a check nothing holds, which is
    the finding this harness exists to produce; a mutant that could not be
    applied was never measured at all, and that outranks it.
    """
    if errors:
        return 2
    return 1 if survived else 0


def main(argv: list[str] | None = None) -> int:
    del argv
    survived: list[str] = []
    killed = 0
    errors: list[str] = []
    with tempfile.TemporaryDirectory() as tmp:
        root = Path(tmp) / "tree"
        shutil.copytree(
            REPO,
            root,
            ignore=shutil.ignore_patterns(".git", ".venv", "__pycache__", "*.egg-info"),
        )
        for mutant in MUTANTS:
            path = root / mutant.file
            original = path.read_text(encoding="utf-8")
            seen = original.count(mutant.old)
            if seen != mutant.occurrences:
                errors.append(
                    f"{mutant.label}: pattern occurs {seen} times, expected "
                    f"{mutant.occurrences} -- NOT MEASURED"
                )
                continue
            path.write_text(_apply(original, mutant), encoding="utf-8")
            environment = dict(os.environ)
            environment["PYTHONPATH"] = str(root / "src")
            try:
                run = subprocess.run(
                    [
                        sys.executable,
                        "-m",
                        "pytest",
                        mutant.suite,
                        "-q",
                        "-x",
                        "-p",
                        "no:cacheprovider",
                    ],
                    cwd=root,
                    capture_output=True,
                    text=True,
                    env=environment,
                )
            finally:
                path.write_text(original, encoding="utf-8")
            tail = (run.stdout.strip().splitlines() or [run.stderr.strip()[-160:]])[-1]
            if run.returncode != 0:
                killed += 1
                print(f"killed   {mutant.label}  [{tail}]")
            else:
                survived.append(mutant.label)
                print(f"SURVIVED {mutant.label}  [{tail}]")
    print(f"\nkilled {killed} of {len(MUTANTS)}")
    for label in survived:
        print(f"  survived: {label}")
    for problem in errors:
        print(f"  ERROR: {problem}")
    return verdict(survived, errors)


if __name__ == "__main__":
    raise SystemExit(main())