test(retrieval-gate): row 5 must read a hold-out set in the schema it declares — red

The registered hold-out set is written in a consumer's own question-set
schema, the one row 8 already reads, and row 5 reads every set with the
gate's synthetic reader: "not a question set this gate can read:
'set_id'", a NO no set in that schema can turn.

Three tests over an invented set in that schema and the synthetic
bundle: 4 of 5 clears 0.8 and falls under 0.9 through the same code
path; an unknown schema is a NO that names the schema; and the row
prints the share and never which hold-out question missed, with the
share itself as the known-positive in the same run. All three fail on
cb0c10b with the GateUsage above.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/test_retrieval_gate.py

@@ -1510,3 +1510,130 @@ def test_a_hold_out_set_with_no_question_clears_no_threshold(tmp_path: Path) ->
         "clears the threshold: NO (the hold-out set carries no question" in detail
         for detail in row.details
     ), row.details
+
+
+# --- row 5 reads a hold-out set in the schema it declares --------------------
+#
+# The registered hold-out set is written in a consumer's own question-set
+# schema, the one row 8 already reads, and row 5 used to read every set with
+# the gate's synthetic reader: `not a question set this gate can read:
+# 'set_id'`, a NO no set in that schema could ever turn. Every file below is
+# written in that shape with INVENTED content over the synthetic bundle; the
+# real hold-out set is never read by a test.
+
+#: Distinctive on purpose: a row that printed a question's id or text would
+#: carry one of these, and nothing else in the gate's output can.
+_HOLD_OUT_MISS_ID = "HOLDOUT-MISS-7Q"
+_HOLD_OUT_MISS_TEXT = "Hvilken farge har den fraflyttede fyrlykta paa Utvaer?"
+_HOLD_OUT_HIT_IDS = ("HOLDOUT-H1", "HOLDOUT-H2", "HOLDOUT-H3", "HOLDOUT-H4")
+
+
+def _fase_set(path: Path, *, schema: str = "fase-sporsmaal/1") -> Path:
+    """Four questions the positive bundle answers and one it cannot: a
+    share of 4 of 5, counted here off the file and not off the row."""
+    hits = [
+        ("Naar kontrolleres vinterberedskapen?", "haandbok", "innen 1. november"),
+        ("Hvordan kvitteres noekkelen ut?", "haandbok", "signatur i noekkelboka"),
+        ("Hvor ofte byttes batteriet i roekvarsleren?", "haandbok", "en gang i aaret"),
+        ("Hva vedtok aarsmoetet om kontingenten?", "referat", "480 kroner"),
+    ]
+    questions = [
+        {"id": qid, "question": text, "fasit": [{"doc": doc, "quote": quote}]}
+        for qid, (text, doc, quote) in zip(_HOLD_OUT_HIT_IDS, hits)
+    ]
+    questions.append(
+        {
+            "id": _HOLD_OUT_MISS_ID,
+            "question": _HOLD_OUT_MISS_TEXT,
+            "fasit": [{"doc": "haandbok", "quote": "fyrlykta er malt i gult og fiolett"}],
+        }
+    )
+    path.write_text(
+        json.dumps(
+            {
+                "schema": schema,
+                "hit_rule": "an excerpt whose source_file is <doc>.md AND carries the quote",
+                "questions": questions,
+            },
+            ensure_ascii=False,
+        ),
+        encoding="utf-8",
+    )
+    return path
+
+
+def test_row_five_measures_a_hold_out_set_in_the_wiki_schema(tmp_path: Path) -> None:
+    """Both directions through the same set and the same bundle: 4 of 5
+    clears 0.8 and falls under 0.9. The share is counted off the file first,
+    so the row is not the only thing that knows it."""
+    held_out = _fase_set(tmp_path / "held-out.json")
+    written = json.loads(held_out.read_text(encoding="utf-8"))
+    assert len(written["questions"]) == 5
+    bundle = _bundles(tmp_path)["positive"]
+
+    clears = gate.row_five(
+        _registration(tmp_path, set_path=held_out, bundle=bundle, threshold=0.8, name="c.json"),
+        provenance=lambda _: _carried_by_git(),
+    )
+    assert any(
+        "the measured hold-out clears the threshold: yes (4 of 5 = 0.8000 against 0.8000)" in detail
+        for detail in clears.details
+    ), clears.details
+    assert (clears.k, clears.m, clears.status) == (11, 11, gate.GREEN)
+
+    falls = gate.row_five(
+        _registration(tmp_path, set_path=held_out, bundle=bundle, threshold=0.9, name="f.json"),
+        provenance=lambda _: _carried_by_git(),
+    )
+    assert any(
+        "the measured hold-out clears the threshold: NO (4 of 5 = 0.8000 against 0.9000)" in detail
+        for detail in falls.details
+    ), falls.details
+    assert (falls.k, falls.m, falls.status) == (10, 11, gate.RED)
+
+
+def test_a_hold_out_set_in_an_unknown_schema_is_a_named_no(tmp_path: Path) -> None:
+    """Fail closed: a set read in a shape it was not written in measures
+    nothing, so an unknown schema is a NO that NAMES the schema -- never a
+    guess at a reader and never an exception."""
+    held_out = _fase_set(tmp_path / "held-out.json", schema="noe-annet/9")
+    row = gate.row_five(
+        _registration(
+            tmp_path, set_path=held_out, bundle=_bundles(tmp_path)["positive"], threshold=0.8
+        ),
+        provenance=lambda _: _carried_by_git(),
+    )
+    verdict = [d for d in row.details if "the measured hold-out clears the threshold" in d]
+    assert len(verdict) == 1, row.details
+    assert ": NO (" in verdict[0]
+    assert "schema `noe-annet/9`" in verdict[0], verdict[0]
+    assert "fase-sporsmaal/1" in verdict[0], verdict[0]
+    assert row.fails
+
+
+def test_row_five_prints_the_share_and_never_which_hold_out_question_missed(
+    tmp_path: Path,
+) -> None:
+    """A hold-out set whose misses are printed is a tuning set by the next
+    session. The known-positive in the same run: the row DID measure, one
+    question missed, and the share is printed -- so the absence of the ids
+    and the text below is the row's silence and not a row that never ran."""
+    held_out = _fase_set(tmp_path / "held-out.json")
+    for threshold in (0.8, 0.9):
+        row = gate.row_five(
+            _registration(
+                tmp_path,
+                set_path=held_out,
+                bundle=_bundles(tmp_path)["positive"],
+                threshold=threshold,
+                name=f"r{threshold}.json",
+            ),
+            provenance=lambda _: _carried_by_git(),
+        )
+        text = gate.render([row]) + json.dumps(row.to_json(), ensure_ascii=False)
+        assert "4 of 5 = 0.8000" in text, text
+        assert _HOLD_OUT_MISS_ID not in text
+        assert _HOLD_OUT_MISS_TEXT not in text
+        assert "fyrlykta" not in text
+        for hit in _HOLD_OUT_HIT_IDS:
+            assert hit not in text