open/llm-ingestion-okf
1
0
Fork 0
Code Issues Releases 4 Activity

test(gates): retire the K2 track; re-measure the retrieval gate's premises for BM25

Browse source 
Operator decision 2026-09-21: the test track built on material tied to the
operator's employer (K2) is retired -- not re-measured, not frozen. Public
tests and gates run on invented material.

Retrieval gate:
- The four FUSION_PREMISE xfails are gone and pass through their INPUTS: the
  synthetic MISS, LOOKUP and QUOTA bundles were re-measured for BM25 (the
  miss fasit no longer shares the rare word `maa`; lookup and quota decoys
  carry the question's words so each partition and the quota decide their
  own fixture). SPECS_SHA256 moved with them. Rows 2 and 3 green again.
- Row 7's mutants M04, M06, M07, M08, M10 now patch `bm25`, the code the
  default runs. Three survive with 0 ranks moved (passage body, title
  weight, bm25.RRF_K), each with its mechanism printed. M07 was not forced:
  every synthetic body carries its title as a heading.
- Row 9 (K2) removed; row 8 requires `wiki-20` alone, the `r761` and
  `vegnormal` adapters are gone. Chose the broad reading of "K2" because the
  operator decision defines it as employer-tied material and the order's
  grep includes `vegnormal`.

Also removed: tests/test_default_bundle_pin.py, the K2 arms of
test_okf_consume, the four real-arm tests of test_quality, the R761 soft
hyphen test, the N101/N200 delivery tests and okf_accounting_gate's default
real corpus (and H5's guard, which only existed for those defaults). Two
fixtures carrying road-standard identifiers are rewritten with invented ones.

Gate after: 1 10/10, 2 7/7, 3 5/5, 4 6/6, 5 0/1, 6 10/10, 7 11/14,
8 NOT RUN -> GATE RED: rows 5, 7, 8. Suite 2423 passed, 1 skipped,
0 xfailed.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Kjell Tore Guttormsen 2026-09-21 10:05:43 +02:00
commit 3d149f955a
12 changed files with 288 additions and 1096 deletions
Download patch file Download diff file Expand all files Collapse all files

221
tests/test_retrieval_gate.py
View file

@ -10,10 +10,12 @@ Three rules this suite is written under, all of them the house pattern:
- **Every zero carries a control.** The socket guard is fired before its
silence during a run counts as evidence; the sha pin is shown to refuse a
tampered set before its silence on the committed ones means anything.
- **Nothing here touches a private corpus or the network.** The real sets in
`claude-code-llm-wiki` and `vegnormal-okf` are never read by a test: the
adapters are exercised against files written in their shapes, with invented
content, inside `tmp_path`.
- **Nothing here touches a private corpus or the network.** The real set in
`claude-code-llm-wiki` is never read by a test: its adapter is exercised
against files written in its shape, with invented content, inside
`tmp_path`. The test track built on material tied to the operator's
employer was retired 2026-09-21, and with it the gate's row 9 and two
adapters; nothing below reads, needs or names that material.
"""
from __future__ import annotations
@ -130,19 +132,6 @@ def test_row_one_is_green_when_the_ranker_delivers_every_fasit(tmp_path: Path) -
assert (row.k, row.m, row.status) == (9, 9, gate.GREEN)
#: v1.1 C1 moved the default ranking to BM25, and four of this gate's premises
#: were built against the fusion it replaced: `set-miss` and B1/B2 force a
#: concept BELOW k that BM25 now delivers, `set-quota` needs the source quota to
#: decide a cut it no longer decides, and row 7's mutants patch fusion code the
#: default no longer runs. The gate says so itself (`premise broken`, `NOT
#: RUN`, `8 of 14`). STRICT, so the day the fixtures are re-measured for BM25
#: these flip loudly instead of staying skipped.
FUSION_PREMISE = pytest.mark.xfail(
strict=True,
reason="fixture premise built against the fusion ranking; re-measure for BM25",
)
def test_row_one_is_red_when_a_fasit_is_not_delivered(tmp_path: Path) -> None:
path, sha = _set_file(
tmp_path / "set.json",
@ -180,7 +169,6 @@ def test_row_one_never_counts_a_question_that_declares_the_class_it_forces(
# --- row 2 --------------------------------------------------------------------
@FUSION_PREMISE
def test_row_two_is_green_when_every_class_is_the_one_its_fixture_forces(
tmp_path: Path,
) -> None:
@ -276,7 +264,6 @@ class _AlwaysTheQuota(dict[str, str]):
return "source_quota_exceeded"
@FUSION_PREMISE
def test_row_three_is_green_on_the_shipped_code(tmp_path: Path) -> None:
"""And the same three fixtures, unmutated, are the green direction.
@ -494,7 +481,6 @@ def _noop_mutant() -> gate.Mutant:
return gate.Mutant("N01 nothing is changed", 1, lambda: gate._patched())
@FUSION_PREMISE
def test_row_seven_is_green_when_every_mutant_is_felled(tmp_path: Path) -> None:
cases, _ = gate.synthetic_cases(tmp_path / "bundles", FIXTURES)
baseline = gate.deterministic_rows(cases)
@ -538,7 +524,7 @@ def test_a_mutant_is_felled_by_the_row_that_got_worse_and_never_by_one_that_did_
assert after[3] >= before[3]
# --- rows 8 and 9 -------------------------------------------------------------
# --- row 8 --------------------------------------------------------------------
def test_row_eight_is_never_green_when_it_did_not_run() -> None:
@ -574,72 +560,21 @@ def test_the_wiki_adapter_reads_its_own_shape_and_hits_by_source_file(tmp_path:
assert question_set.questions[0].fasit[0].by == "source_file"
assert question_set.questions[0].fasit[0].value == "haandbok.md"
row = gate.row_eight([(question_set, {"wiki": _bundles(tmp_path)["positive"]})])
# J2: one set of three is NOT a measurement of the three -- this asserted
# GREEN until 2026-09-19, which is the breakthrough PM measured.
assert row.status == gate.NOT_RUN
# The one required set, answered: row 8's own green direction. What
# refuses a self-written file of this shape is the pin on the command-line
# path (`test_a_wiki_set_of_one_question_is_refused_on_the_command_line`).
assert (row.k, row.m, row.status) == (1, 1, gate.GREEN)
assert any("citation granularity" in detail for detail in row.details)
def test_the_r761_adapter_splits_the_known_negative_out_of_the_questions(
tmp_path: Path,
) -> None:
path = tmp_path / "r761-shaped.json"
path.write_text(
json.dumps(
{
"fasit_form": "the normalised STS title",
"sporsmal": [
{"id": "S1", "sporsmal": "Hva sier punktet?", "fasit": "4.2 Vakthold"},
{"id": "KN", "sporsmal": "Hvilken safran passer til fiskesuppe?", "fasit": ""},
],
}
),
encoding="utf-8",
)
question_set = gate.read_real_set("r761", path, gate.sha256_of(path))
assert [q.id for q in question_set.questions] == ["S1"]
assert [c.id for c in question_set.controls] == ["KN"]
assert question_set.questions[0].fasit[0].by == "title"
# No quote in this set's fasit, so it is concept granularity and says so.
assert not question_set.quoted
def test_the_vegnormal_adapter_makes_one_question_per_standard(tmp_path: Path) -> None:
path = tmp_path / "vegnormal-shaped.json"
path.write_text(
json.dumps(
{
"sporsmal": [
{
"id": "T1-1",
"sporsmal": "Hva viser kravet til?",
"must_cite": [
{"normal": "N100:2023", "req_number": "2.3.2-3", "fil": "a"},
{"normal": "N200:2024", "req_number": "1.6.3-3", "fil": "b"},
],
}
]
}
),
encoding="utf-8",
)
question_set = gate.read_real_set("vegnormal", path, gate.sha256_of(path))
assert sorted(q.id for q in question_set.questions) == ["T1-1/N100:2023", "T1-1/N200:2024"]
assert {q.bundle for q in question_set.questions} == {"N100:2023", "N200:2024"}
assert question_set.units == 2
def test_an_unknown_real_set_name_is_refused(tmp_path: Path) -> None:
path = tmp_path / "x.json"
path.write_text("{}", encoding="utf-8")
with pytest.raises(gate.GateUsage):
gate.read_real_set("something-else", path, gate.sha256_of(path))
def test_row_nine_states_k2_s_denominator_and_never_passes() -> None:
row = gate.row_nine()
assert (row.k, row.m, row.status) == (0, 6, gate.RED)
assert any("the answer key does not" in detail for detail in row.details)
# The two retired adapters are unknown names now, like any other.
for name in ("something-else", "r761", "vegnormal"):
with pytest.raises(gate.GateUsage) as refusal:
gate.read_real_set(name, path, gate.sha256_of(path))
assert "unknown real set" in str(refusal.value)
# --- the verdict --------------------------------------------------------------
@ -667,14 +602,16 @@ def test_the_verdict_names_every_failing_row() -> None:
def test_the_gate_is_red_today_and_says_which_rows(tmp_path: Path) -> None:
rows = gate.evaluate(tmp_path / "bundles")
by_number = {row.number: row for row in rows}
assert sorted(by_number) == [1, 2, 3, 4, 5, 6, 7, 8, 9]
# Rows 2, 3 and 7 went red with v1.1 C1's BM25 default: see
# `FUSION_PREMISE`. Row 1 holds every fasit it held.
assert [row.number for row in rows if row.fails] == [2, 3, 5, 7, 8, 9]
assert sorted(by_number) == [1, 2, 3, 4, 5, 6, 7, 8]
# Rows 2 and 3 are green again since the synthetic corpus was re-measured
# for BM25 (2026-09-21). Row 7 is red on three survivors, each printed with
# what it moved.
assert [row.number for row in rows if row.fails] == [5, 7, 8]
assert (by_number[1].k, by_number[1].m) == (10, 10)
assert (by_number[2].k, by_number[2].m) == (5, 7)
assert (by_number[3].k, by_number[3].m) == (3, 5)
assert (by_number[6].k, by_number[6].m) == (12, 12)
assert (by_number[2].k, by_number[2].m) == (7, 7)
assert (by_number[3].k, by_number[3].m) == (5, 5)
assert (by_number[6].k, by_number[6].m) == (10, 10)
assert (by_number[7].k, by_number[7].m) == (11, 14)
def test_the_same_tree_measures_the_same_twice(tmp_path: Path) -> None:
@ -688,15 +625,15 @@ def test_the_command_exits_one_and_prints_every_row(
) -> None:
assert gate.main([]) == 1
printed = capsys.readouterr().out
for number in range(1, 10):
for number in range(1, 9):
assert f"\n{number} " in f"\n{printed}"
assert "GATE RED: rows 2, 3, 5, 7, 8, 9" in printed
assert "GATE RED: rows 5, 7, 8" in printed
def test_the_json_form_carries_the_same_rows(capsys: pytest.CaptureFixture[str]) -> None:
assert gate.main(["--json"]) == 1
payload = json.loads(capsys.readouterr().out)
assert [row["row"] for row in payload["rows"]] == list(range(1, 10))
assert [row["row"] for row in payload["rows"]] == list(range(1, 9))
assert payload["gate"] == gate.RED
assert set(payload["classes"]) == {"a", "b", "c", "d", "e"}
assert payload["limits"]
@ -879,26 +816,25 @@ def _hitting_set(set_id: str, *, quote: str, entries: int = 1) -> gate.QuestionS
def test_j2_row_eight_is_not_run_when_a_required_set_is_left_out(tmp_path: Path) -> None:
"""PM's J2: one set of three came back `6 of 6 GREEN`."""
"""PM's J2: one set of three came back `6 of 6 GREEN`. The rule outlived
the two retired sets: a set that is not the required one measures
something, and the row is still not a measurement of the required set."""
bundles = {"positive": _bundles(tmp_path)["positive"]}
row = gate.row_eight([(_hitting_set("wiki-20", quote="innen 1. november"), bundles)])
row = gate.row_eight([(_hitting_set("some-other-set", quote="innen 1. november"), bundles)])
assert row.status == gate.NOT_RUN
assert row.fails
assert "r761-sk2" in row.reason and "vegnormal-32" in row.reason
assert "wiki-20" in row.reason
# The numbers it DID measure are still carried: a missing set must not cost
# the reader the set that ran.
assert any("wiki-20: " in detail for detail in row.details)
assert any("some-other-set: " in detail for detail in row.details)
def test_row_eight_is_green_only_with_all_three_named_sets(tmp_path: Path) -> None:
def test_row_eight_is_green_only_with_every_named_set(tmp_path: Path) -> None:
bundles = {"positive": _bundles(tmp_path)["positive"]}
real = [
(_hitting_set(name, quote="innen 1. november"), bundles)
for name in ("wiki-20", "r761-sk2", "vegnormal-32")
]
real = [(_hitting_set(name, quote="innen 1. november"), bundles) for name in ("wiki-20",)]
row = gate.row_eight(real)
assert row.status == gate.GREEN
assert (row.k, row.m) == (3, 3)
assert (row.k, row.m) == (1, 1)
def test_j2b_row_eight_never_sums_the_two_granularities_into_its_headline(
@ -909,17 +845,16 @@ def test_j2b_row_eight_never_sums_the_two_granularities_into_its_headline(
bundles = {"positive": _bundles(tmp_path)["positive"]}
real = [
(_hitting_set("wiki-20", quote="innen 1. november", entries=2), bundles),
(_hitting_set("r761-sk2", quote=""), bundles),
(_hitting_set("vegnormal-32", quote=""), bundles),
(_hitting_set("an-invented-concept-set", quote=""), bundles),
]
row = gate.row_eight(real)
# Three questions, one per set: the headline is at QUESTION granularity.
# The sum it must NOT be is 4 of 4 -- two citation units plus two concept
# units -- which is why the first set carries two fasit entries.
assert (row.k, row.m) == (3, 3)
# Two questions, one per set: the headline is at QUESTION granularity.
# The sum it must NOT be is 3 of 3 -- two citation units plus one concept
# unit -- which is why the first set carries two fasit entries.
assert (row.k, row.m) == (2, 2)
assert "question" in row.reason
assert any(
"2 of 2 at citation granularity, 2 of 2 at concept granularity" in detail
"2 of 2 at citation granularity, 1 of 1 at concept granularity" in detail
for detail in row.details
)
@ -1081,40 +1016,6 @@ def test_a_corpus_that_is_not_the_pinned_corpus_is_refused(
assert "not the corpus that was pinned" in str(refusal.value)
def _k2_shaped(tmp_path: Path, questions: int) -> tuple[Path, str]:
"""A K2 gold set in this gate's own shape. `set-positive.json` carries
exactly six questions the bundle answers, which is K2's denominator."""
spec = json.loads((FIXTURES / "set-positive.json").read_text(encoding="utf-8"))
spec["set_id"] = "k2-gold"
spec["questions"] = spec["questions"][:questions]
path = tmp_path / "k2.json"
path.write_text(json.dumps(spec, ensure_ascii=False), encoding="utf-8")
return path, gate.sha256_of(path)
def test_row_nine_is_green_when_a_k2_gold_set_arrives(tmp_path: Path) -> None:
path, sha = _k2_shaped(tmp_path, gate.K2_QUESTIONS)
question_set = gate.load_set(path, sha)
row = gate.row_nine((question_set, _bundles(tmp_path)))
assert (row.k, row.m, row.status) == (6, 6, gate.GREEN)
def test_row_nine_is_red_when_the_gold_set_is_not_answered(tmp_path: Path) -> None:
path, sha = _k2_shaped(tmp_path, gate.K2_QUESTIONS)
question_set = gate.load_set(path, sha)
bundles = dict(_bundles(tmp_path))
bundles["positive"] = bundles["miss"] # the same six questions, the wrong bundle
row = gate.row_nine((question_set, bundles))
assert (row.k, row.m, row.status) == (0, 6, gate.RED)
def test_a_k2_set_of_another_size_is_another_set_and_is_refused(tmp_path: Path) -> None:
path, sha = _k2_shaped(tmp_path, gate.K2_QUESTIONS - 1)
with pytest.raises(gate.GateUsage) as refusal:
gate._k2_set([str(path), sha, str(tmp_path)])
assert "K2's denominator" in str(refusal.value)
def test_j7_a_cut_that_delivers_nothing_is_exit_two_and_never_a_quiet_green_row(
tmp_path: Path,
) -> None:
@ -1135,7 +1036,7 @@ def test_row_eight_names_the_bundle_identity_of_every_set_it_measured(
) -> None:
"""The row printed the SET's sha256 and never the bundle's identity.
Measured 2026-09-19 by PM with one pinned r761 set: three bundles gave
Measured 2026-09-19 by PM with one pinned real set: three bundles gave
4 of 7 and a fourth gave 0 of 7, while the session recorded 7 of 7 -- from
a bundle no line of the output named. A number nobody can attach to a
bundle cannot be reproduced and cannot be felled.
@ -1158,15 +1059,15 @@ def test_row_eight_names_the_bundle_identity_of_every_set_it_measured(
assert ref[:24] in block, f"{set_id}: the bundle ref is nowhere in the output"
def test_all_three_real_sets_are_pinned_and_not_only_the_first(tmp_path: Path) -> None:
"""Every one of the three, driven through the command-line path.
def test_every_required_real_set_is_pinned(tmp_path: Path) -> None:
"""Every required set, driven through the command-line path.
The names and the count are written out here rather than read from
`REQUIRED_REAL_SETS`: a test taking its denominator from the tuple it is
checking would stay green if a name were dropped from both.
"""
names = [("wiki", "wiki-20"), ("r761", "r761-sk2"), ("vegnormal", "vegnormal-32")]
assert len(names) == 3
names = [("wiki", "wiki-20")]
assert len(names) == len(gate.REQUIRED_REAL_SETS) == 1
bundle = _bundles(tmp_path)["positive"]
for adapter, set_id in names:
path = tmp_path / f"{adapter}-invented.json"
@ -1178,29 +1079,14 @@ def test_all_three_real_sets_are_pinned_and_not_only_the_first(tmp_path: Path) -
def _invented_real_set(adapter: str) -> str:
"""One question, in the adapter's own shape, with invented content."""
if adapter == "wiki":
return json.dumps(
{
"questions": [
{
"id": "W1",
"question": "Naar kontrolleres vinterberedskapen?",
"fasit": [{"doc": "haandbok", "quote": "innen 1. november"}],
}
]
}
)
if adapter == "r761":
return json.dumps(
{"sporsmal": [{"id": "S1", "sporsmal": "Hva sier punktet?", "fasit": "4.2 Vakthold"}]}
)
assert adapter == "wiki", adapter
return json.dumps(
{
"sporsmal": [
"questions": [
{
"id": "T1-1",
"sporsmal": "Hva viser kravet til?",
"must_cite": [{"normal": "N100:2023", "req_number": "2.3.2-3"}],
"id": "W1",
"question": "Naar kontrolleres vinterberedskapen?",
"fasit": [{"doc": "haandbok", "quote": "innen 1. november"}],
}
]
}
@ -1282,7 +1168,6 @@ def test_a_threshold_that_is_not_a_number_is_refused(tmp_path: Path) -> None:
assert row.fails
@FUSION_PREMISE
def test_the_threshold_is_compared_with_the_measured_hold_out(tmp_path: Path) -> None:
"""Both directions, from the same code path: a set the bundle answers
clears a threshold under it, and a set it does not answer falls under one