test(soek): a red search gate measuring what the asker RECEIVES, one exit code

`tools/okf_soek_gate.py` runs the question sets through the same
`consume.build_payload` path `okf consume` and MCP's `okf_ask` use, at the
shipped k and limit, and counts on the DELIVERED excerpts rather than on an
internal ranking -- a rank nobody receives is not an answer.

Written RED: it is the finish line a later round has to make green, and a
gate that is green on the day it is written has measured nothing. Thresholds
are named constants carrying their reason in a comment. Exit 0 only when every
row holds, 1 with the full table otherwise, 2 when the collection cannot be
found -- never 0 hits against nothing. Two runs print identical bytes.

The sets and the collection are both INPUTS: nothing about anyone's corpus is
committed, and no row names a document, a question or a quote.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

tests/test_soek_gate.py

@@ -0,0 +1,284 @@
+"""The measuring instrument of `tools/okf_soek_gate.py`, over a synthetic corpus.
+
+The gate itself is NOT in this suite: it is red by construction against a real
+collection, and a red gate in a green suite is a suite nobody trusts. What is
+here is the part that can lie quietly -- the hit rule, the counting, the
+missing-fixture state and the merge -- measured against a corpus this file
+builds, where every answer is forced by the fixture rather than by a ranking.
+
+The synthetic sets are written in the SAME shapes the real ones carry, so the
+code path is measured even on a machine where no real set exists.
+"""
+
+from __future__ import annotations
+
+import json
+import sys
+from pathlib import Path
+
+import pytest
+
+TOOLS = Path(__file__).resolve().parent.parent / "tools"
+if str(TOOLS) not in sys.path:
+    sys.path.insert(0, str(TOOLS))
+
+import okf_retrieval_gate as retrieval  # noqa: E402
+import okf_soek_gate as gate  # noqa: E402
+
+from llm_ingestion_okf import consume  # noqa: E402
+
+# --- the synthetic corpus -----------------------------------------------------
+#
+# Reuses the retrieval gate's bundle writer rather than a second one: two
+# writers would let the two gates' fixtures drift apart in shape, and the shape
+# is the thing both of them depend on.
+
+QUOTE_DELIVERED = "the roof beam is replaced every twelve years"
+QUOTE_PRESENT_NOT_DELIVERED = "the cellar key is kept by the treasurer"
+QUOTE_ABSENT = "the gondola runs on alternating tuesdays"
+
+SPEC = retrieval.BundleSpec(
+    "soek-synthetic",
+    (
+        retrieval.DocumentSpec(
+            "handbook",
+            "handbook.md",
+            (
+                retrieval.ConceptSpec(
+                    slug="roof",
+                    title="Roof maintenance",
+                    body=f"The club inspects the cabin roof. {QUOTE_DELIVERED}.",
+                ),
+            ),
+        ),
+        retrieval.DocumentSpec(
+            "cellar",
+            "cellar.md",
+            (
+                retrieval.ConceptSpec(
+                    slug="keys",
+                    title="Key holding",
+                    # Carries none of the roof question's words, so it is in the
+                    # collection and unreachable from that question: class
+                    # `soekefeil` is forced, not hoped for.
+                    body=f"Storage arrangements. {QUOTE_PRESENT_NOT_DELIVERED}.",
+                ),
+            ),
+        ),
+    ),
+)
+
+
+@pytest.fixture(scope="module")
+def bundle(tmp_path_factory: pytest.TempPathFactory) -> Path:
+    root = tmp_path_factory.mktemp("soek") / "bundle"
+    return retrieval.build_bundle(root, SPEC)
+
+
+def write_sets(directory: Path, **files: object) -> Path:
+    directory.mkdir(parents=True, exist_ok=True)
+    for name, payload in files.items():
+        (directory / f"{name}.json").write_text(
+            json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8"
+        )
+    return directory
+
+
+FASE = {
+    "schema": "fase-sporsmaal/1",
+    "frozen": "synthetic",
+    "hit_rule": "source_file == <doc>.md AND the excerpt contains the quote.",
+    "questions": [
+        {
+            "id": "S1",
+            "class": "docs",
+            "question": "roof beam replacement interval",
+            "fasit": [{"doc": "handbook", "quote": QUOTE_DELIVERED}],
+        },
+        {
+            "id": "S2",
+            "class": "release_only",
+            "question": "roof beam replacement interval",
+            # In the collection, unreachable from this question's words.
+            "fasit": [{"doc": "cellar", "quote": QUOTE_PRESENT_NOT_DELIVERED}],
+        },
+        {
+            "id": "S3",
+            "class": "release_only",
+            "question": "roof beam replacement interval",
+            "fasit": [{"doc": "handbook", "quote": QUOTE_ABSENT}],
+        },
+    ],
+}
+
+
+# --- the hit rule -------------------------------------------------------------
+
+
+def test_the_hit_rule_needs_the_source_AND_the_quote() -> None:
+    """Either half alone is not a hit -- the sets' own rule, verbatim."""
+    right = {"source_file": "handbook.md", "text": f"x {QUOTE_DELIVERED} y"}
+    wrong_source = {"source_file": "cellar.md", "text": f"x {QUOTE_DELIVERED} y"}
+    wrong_quote = {"source_file": "handbook.md", "text": "x nothing of the sort y"}
+    assert gate.excerpt_carries(right, "handbook", QUOTE_DELIVERED)
+    assert not gate.excerpt_carries(wrong_source, "handbook", QUOTE_DELIVERED)
+    assert not gate.excerpt_carries(wrong_quote, "handbook", QUOTE_DELIVERED)
+
+
+def test_the_hit_rule_folds_case_and_collapses_whitespace() -> None:
+    excerpt = {"source_file": "handbook.md", "text": "The  ROOF\n  beam\tis replaced"}
+    assert gate.excerpt_carries(excerpt, "handbook", "the roof beam is   replaced")
+
+
+def test_any_fasit_entry_suffices() -> None:
+    """Three entries, one match: a hit. The rule says so in both real sets."""
+    excerpts = [{"source_file": "handbook.md", "text": QUOTE_DELIVERED}]
+    fasit = [
+        {"doc": "cellar", "quote": QUOTE_ABSENT},
+        {"doc": "handbook", "quote": QUOTE_DELIVERED},
+        {"doc": "handbook", "quote": QUOTE_ABSENT},
+    ]
+    assert gate.question_hit(excerpts, fasit)
+    assert not gate.question_hit(excerpts, fasit[:1])
+
+
+# --- counting and miss classification ----------------------------------------
+
+
+def test_counting_and_miss_classes_over_the_synthetic_corpus(bundle: Path, tmp_path: Path) -> None:
+    """One hit, one search failure, one build failure -- each forced by the fixture."""
+    sets = gate.load_sets(write_sets(tmp_path / "sets", **{"fase-sporsmaal": FASE}))
+    report = gate.run(bundle, sets)
+    row = report.row("a")
+    assert (row.measured, row.denominator) == (1, 3)
+    classes = {miss.question_id: miss.klass for miss in row.misses}
+    assert classes == {"S2": gate.SEARCH_FAILURE, "S3": gate.BUILD_FAILURE}
+
+
+def test_a_miss_is_a_build_failure_only_when_no_fasit_quote_is_in_the_collection(
+    bundle: Path,
+) -> None:
+    """The denominator of 'is it even there' is the collection, not the payload."""
+    text = gate.collection_text(bundle)
+    assert gate.collapse(QUOTE_PRESENT_NOT_DELIVERED) in text["cellar.md"]
+    assert gate.collapse(QUOTE_ABSENT) not in text.get("handbook.md", "")
+
+
+def test_release_only_counts_within_the_phase_set(bundle: Path, tmp_path: Path) -> None:
+    """Series (b) is a subset of (a), never a second set and never a second ask."""
+    sets = gate.load_sets(write_sets(tmp_path / "sets", **{"fase-sporsmaal": FASE}))
+    report = gate.run(bundle, sets)
+    assert report.row("b").denominator == 2
+    assert report.row("b").measured == 0
+
+
+# --- the missing fixture ------------------------------------------------------
+
+
+def test_a_missing_fixture_is_red_and_never_zero_hits(bundle: Path, tmp_path: Path) -> None:
+    """The distinction the order names: not run is not the same fact as no hits."""
+    sets = gate.load_sets(write_sets(tmp_path / "sets", **{"fase-sporsmaal": FASE}))
+    report = gate.run(bundle, sets)
+    for key in ("c", "d", "e", "f", "g1"):
+        row = report.row(key)
+        assert row.measured is None, key
+        assert row.holds() is None, key
+        assert gate.MISSING_FIXTURE in row.render(), key
+    assert report.exit_code() == 1
+
+
+def test_an_unreadable_fixture_is_wrong_input_not_a_red_row(tmp_path: Path) -> None:
+    directory = tmp_path / "sets"
+    directory.mkdir()
+    (directory / "fase-sporsmaal.json").write_text("{not json", encoding="utf-8")
+    with pytest.raises(gate.GateUsage):
+        gate.load_sets(directory)
+
+
+def test_a_missing_collection_exits_two(tmp_path: Path) -> None:
+    code = gate.main(["--bundle", str(tmp_path / "nowhere"), "--sets", str(tmp_path)])
+    assert code == 2
+
+
+# --- the merge ----------------------------------------------------------------
+
+
+def test_the_gate_has_no_merge_of_its_own() -> None:
+    """Since v1.1 C2 the sub-questions of series (e) and (f) go to the product
+    in ONE call, so the merge measured is the one a reader receives."""
+    assert not hasattr(gate, "merge_round_robin")
+
+
+def test_the_gate_asks_every_subquestion_in_one_call(bundle: Path) -> None:
+    parts = ["roof beam replacement interval", "storage arrangements"]
+    assert gate.Asker(bundle).many(parts) == consume.build_multi_payload(bundle, questions=parts)
+
+
+# --- the thresholds and the verdict ------------------------------------------
+
+
+def test_every_threshold_is_a_named_constant() -> None:
+    """A bar read off a literal in a row is a bar nobody can find again."""
+    for name in (
+        "THRESHOLD_PHASE",
+        "THRESHOLD_RELEASE_ONLY",
+        "THRESHOLD_HOLDOUT",
+        "THRESHOLD_NORWEGIAN_DIRECT",
+        "THRESHOLD_NORWEGIAN_SUBQUESTIONS",
+        "THRESHOLD_OPERATOR",
+        "THRESHOLD_NEGATIVE_FLAGGED",
+        "THRESHOLD_POSITIVE_MISFLAGGED",
+        "THRESHOLD_LARGEST_EXCERPT",
+    ):
+        assert isinstance(getattr(gate, name), int)
+
+
+def test_a_row_at_the_bar_holds_and_one_under_it_does_not() -> None:
+    at = gate.Row("x", "x", measured=3, denominator=5, threshold=3)
+    under = gate.Row("x", "x", measured=2, denominator=5, threshold=3)
+    assert at.holds() and not under.holds()
+
+
+def test_an_at_most_row_reads_the_other_way() -> None:
+    at = gate.Row("x", "x", measured=2, denominator=7, threshold=2, at_most=True)
+    over = gate.Row("x", "x", measured=3, denominator=7, threshold=2, at_most=True)
+    assert at.holds() and not over.holds()
+
+
+def test_exit_zero_needs_every_row(bundle: Path, tmp_path: Path) -> None:
+    """One row short of its bar is exit 1, whichever row it is."""
+    sets = gate.load_sets(write_sets(tmp_path / "sets", **{"fase-sporsmaal": FASE}))
+    report = gate.run(bundle, sets)
+    assert report.exit_code() == 1
+    green = gate.Report(
+        collection="x",
+        rows=[gate.Row("a", "a", measured=1, denominator=1, threshold=1)],
+        notes=(),
+    )
+    assert green.exit_code() == 0
+
+
+# --- determinism --------------------------------------------------------------
+
+
+def test_two_renderings_are_byte_identical(bundle: Path, tmp_path: Path) -> None:
+    sets = gate.load_sets(write_sets(tmp_path / "sets", **{"fase-sporsmaal": FASE}))
+    first = gate.run(bundle, sets).render()
+    second = gate.run(bundle, sets).render()
+    assert first.encode("utf-8") == second.encode("utf-8")
+
+
+def test_the_rendering_carries_no_absolute_path(bundle: Path, tmp_path: Path) -> None:
+    """The table is pasted into STATE and a commit; a scratch path in it is noise."""
+    sets = gate.load_sets(write_sets(tmp_path / "sets", **{"fase-sporsmaal": FASE}))
+    rendered = gate.run(bundle, sets).render()
+    assert str(bundle) not in rendered
+    assert not any(line.strip().startswith("/") for line in rendered.splitlines())
+
+
+# --- the negative signal ------------------------------------------------------
+
+
+def test_the_negative_signal_is_the_retrieval_gates_reading() -> None:
+    """One repository, one definition of 'the payload says it does not know'."""
+    assert gate.uncovered_signal is retrieval.marked