open/llm-ingestion-okf
1
0
Fork 0
Code Issues Releases 4 Activity

fix(retrieval-gate): the seven small findings -- a corpus pin, a K2 input, an observed term

Browse source 
PM's checkpoint left seven small findings beside the four bearing ones. Six
are closed here (the seventh, running row 8 against the real sets, follows).

G9 -- THE CONFIRMATION TERM IS OBSERVED NOW. `hit = bool(hit_ids) and
bool(confirmed)` survived all 46 tests, because every mutation of the text
empties `hit_ids` one step earlier. The shape that reaches it is a delivery
that still CARRIES the citation and is no longer the concept file's bytes:
`M14` is that mutation and it is FELLED (row 7 goes 11 of 13 to 12 of 14, bar
12 of 13 to 13 of 14, still RED, the same two survivors), and a test drives it
with its known-positive in the same test. No production line changed: the term
was always observable, it was unobserved.

AND THAT MEASURES THE JUDGE'S INDEPENDENCE RATHER THAN ASSERTING IT. The judge
does read the bundle through `consume.read_concept` and `delivered_text` --
PM's finding -- but the index is warmed BEFORE the first mutation, so the two
sides do not move together. Measured both ways: index warmed first, every unit
is a miss with `confirmed False`; index built UNDER the same patch, every unit
is a hit. The gate never builds one under a mutation. Stated in `LIMITS` with
that measurement, rather than closed by re-implementing a normalisation rule
this repository already owns once.

SPECS -- the synthetic corpus is pinned like the sets (`SPECS_SHA256` over
`specs_digest`). PM's corpus tuning was caught by row 2's forced classes and
not by a pin, and a more careful tuning was left standing.

ROW 9 TAKES AN INPUT. `--k2 SET SHA256 BUNDLE` reads a gold set in this gate's
own set shape; `K2_QUESTIONS` stays the denominator whatever the file carries,
and a set of another size is refused (exit 2) as another set wearing this
one's name. Without a set the row stays RED and not NOT RUN -- ITS denominator
is known, six recorded questions, so the absence is measured; row 8's is not
known until the sets arrive. Both fail the gate identically. This is a
deliberate divergence from the order's parenthetical, stated here and in the
row.

MYPY. `mypy --strict` on this file goes 8 errors to 0, the four in
`read_real_set` among them (`questions = []` against a name inferred
`tuple[Question, ...]`) -- the adapters that meet the real sets.

63 passed. The verdict is unchanged: GATE RED: rows 3, 4, 5, 7, 8, 9.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Kjell Tore Guttormsen 2026-09-19 21:24:40 +02:00
commit 714aafbff2
Signed by: ktg
SSH key fingerprint: SHA256:JakMjO6FTBBzN0Bhfj9saOoEjaFxlSdYuZQQpM/lF9Q
2 changed files with 257 additions and 31 deletions
Download patch file Download diff file Expand all files Collapse all files

77
tests/test_retrieval_gate.py
View file

@ -979,7 +979,84 @@ def test_j3_row_seven_refuses_a_mutant_list_that_is_not_the_pinned_roster(
assert "roster" in row.reason or "duplicate" in row.reason
def test_g9_a_delivery_that_carries_the_citation_and_not_the_bundles_bytes_is_not_a_hit(
tmp_path: Path,
) -> None:
"""PM's G9: `hit = bool(hit_ids) and bool(confirmed)` -- removing the
second term left all 46 tests green, because every mutation that touched
the text emptied `hit_ids` one step earlier. The shape that reaches the
term is a delivery that still CARRIES the quote and is no longer the
concept file's bytes, and the judge must have read the bundle first."""
bundles = _bundles(tmp_path)
gate.bundle_index(bundles["positive"]) # the judge reads the bundle BEFORE the payload
question_set = gate.load_set(
FIXTURES / "set-positive.json", gate.SYNTHETIC_SETS["set-positive.json"]
)
with gate._extend_delivered():
units = gate.measure_case(question_set, bundles).units
assert units, "the known-positive: this set delivers on the unmutated run"
assert [unit.hit for unit in units] == [False] * len(units)
assert all(unit.confirmed is False for unit in units)
assert all(unit.detail == "the delivered text is not the bundle's bytes" for unit in units)
def test_the_unmutated_run_of_that_same_set_is_every_hit(tmp_path: Path) -> None:
units = _case(tmp_path, "set-positive.json").units
assert [unit.hit for unit in units] == [True] * len(units)
def test_the_mutant_roster_is_pinned_apart_from_the_list_it_names() -> None:
assert tuple(mutant.label for mutant in gate.MUTANTS) == gate.MUTANT_ROSTER
assert len(gate.MUTANT_ROSTER) == gate.MUTANT_COUNT
assert len(set(gate.MUTANT_ROSTER)) == len(gate.MUTANT_ROSTER)
def test_the_synthetic_corpus_is_pinned_like_the_sets(tmp_path: Path) -> None:
assert gate.specs_digest() == gate.SPECS_SHA256
tuned = dict(gate.SPECS)
first = tuned["positive"]
tuned["positive"] = gate.BundleSpec(first.bundle_id, first.documents[:1])
assert gate.specs_digest(tuned) != gate.SPECS_SHA256
def test_a_corpus_that_is_not_the_pinned_corpus_is_refused(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setattr(gate, "SPECS_SHA256", "0" * 64)
with pytest.raises(gate.GateUsage) as refusal:
gate.synthetic_bundles(tmp_path / "bundles")
assert "not the corpus that was pinned" in str(refusal.value)
def _k2_shaped(tmp_path: Path, questions: int) -> tuple[Path, str]:
"""A K2 gold set in this gate's own shape. `set-positive.json` carries
exactly six questions the bundle answers, which is K2's denominator."""
spec = json.loads((FIXTURES / "set-positive.json").read_text(encoding="utf-8"))
spec["set_id"] = "k2-gold"
spec["questions"] = spec["questions"][:questions]
path = tmp_path / "k2.json"
path.write_text(json.dumps(spec, ensure_ascii=False), encoding="utf-8")
return path, gate.sha256_of(path)
def test_row_nine_is_green_when_a_k2_gold_set_arrives(tmp_path: Path) -> None:
path, sha = _k2_shaped(tmp_path, gate.K2_QUESTIONS)
question_set = gate.load_set(path, sha)
row = gate.row_nine((question_set, _bundles(tmp_path)))
assert (row.k, row.m, row.status) == (6, 6, gate.GREEN)
def test_row_nine_is_red_when_the_gold_set_is_not_answered(tmp_path: Path) -> None:
path, sha = _k2_shaped(tmp_path, gate.K2_QUESTIONS)
question_set = gate.load_set(path, sha)
bundles = dict(_bundles(tmp_path))
bundles["positive"] = bundles["miss"] # the same six questions, the wrong bundle
row = gate.row_nine((question_set, bundles))
assert (row.k, row.m, row.status) == (0, 6, gate.RED)
def test_a_k2_set_of_another_size_is_another_set_and_is_refused(tmp_path: Path) -> None:
path, sha = _k2_shaped(tmp_path, gate.K2_QUESTIONS - 1)
with pytest.raises(gate.GateUsage) as refusal:
gate._k2_set([str(path), sha, str(tmp_path)])
assert "K2's denominator" in str(refusal.value)

211
tools/okf_retrieval_gate.py
View file

@ -440,9 +440,58 @@ SPECS: Mapping[str, BundleSpec] = {
}
def synthetic_bundles(root: Path) -> dict[str, Path]:
def specs_digest(specs: Mapping[str, BundleSpec] = SPECS) -> str:
"""The synthetic corpus as one sha256 over its own fields.
The SETS are pinned and the CORPUS was not, which is a hole one size
smaller than the one it guards: PM tuned the corpus 2026-09-19 and took
row 3 green, and what caught it was row 2's forced classes rather than a
pin. A more careful tuning that preserved those classes was left standing.
A digest over the specs is not a digest over the bundle's bytes -- that is
`build_bundle`'s job and it is deterministic -- but it is the same
guarantee the sets have: these bytes, or exit 2.
"""
canonical = json.dumps(
{
name: [
[
document.directory,
document.source_file,
[
[
concept.slug,
concept.title,
concept.body,
concept.description,
concept.repeat,
]
for concept in document.concepts
],
]
for document in spec.documents
]
for name, spec in sorted(specs.items())
},
ensure_ascii=False,
sort_keys=True,
)
return hashlib.sha256(canonical.encode("utf-8")).hexdigest()
#: The synthetic corpus, pinned the way the sets are.
SPECS_SHA256 = "8d999838f72a4c151e12ff6ac511b253c3437dba6290d2a7ea97dc546747242d"
def synthetic_bundles(root: Path, specs: Mapping[str, BundleSpec] = SPECS) -> dict[str, Path]:
"""Every synthetic bundle, written once and reused by every row."""
return {name: build_bundle(root / name, spec) for name, spec in SPECS.items()}
measured = specs_digest(specs)
if specs is SPECS and measured != SPECS_SHA256:
raise GateUsage(
f"the synthetic corpus is not the corpus that was pinned: expected "
f"{SPECS_SHA256}, measured {measured}. Every row below counts against "
"these documents; move the pin in the same commit that moves them"
)
return {name: build_bundle(root / name, spec) for name, spec in specs.items()}
# --- the sets -----------------------------------------------------------------
@ -805,8 +854,9 @@ def measure_units(bundle: Path, question: Question) -> list[Unit]:
target = (in_body or holding or tuple(sorted(candidates)) or ("",))[0]
rank = None
excerpt = delivered.get(target)
if excerpt is not None and isinstance(excerpt.get("rank"), int):
rank = int(excerpt["rank"])
if excerpt is not None:
declared = excerpt.get("rank")
rank = declared if isinstance(declared, int) else None
label = withheld.get(target) if target else None
truth = (
"delivered"
@ -1052,8 +1102,8 @@ def row_three(cases: Sequence[Case]) -> Row:
declared = unit.expect_class in WITHHELD_CLASSES
if declared or unit.label_default is not None:
judged.append(unit)
agreeing = []
lying = []
agreeing: list[Unit] = []
lying: list[Unit] = []
for unit in judged:
if unit.label_default is None:
lying.append(unit)
@ -1119,7 +1169,9 @@ def marked(payload: Mapping[str, object]) -> bool:
"""
counts = payload.get("denominators")
assert isinstance(counts, dict)
return int(counts["delivered"]) == 0
delivered = counts["delivered"]
assert isinstance(delivered, int)
return delivered == 0
def row_four(cases: Sequence[Case]) -> Row:
@ -1442,6 +1494,18 @@ def _truncate_delivered() -> contextlib.AbstractContextManager[None]:
return _patched(delivered_text=mutant)
def _extend_delivered() -> contextlib.AbstractContextManager[None]:
"""The delivered text still CARRIES the citation and is no longer the
concept file's bytes: the one shape that reaches `confirmed` at all, since
every other mutation of the text empties `hit_ids` one step earlier."""
original = consume.delivered_text
def mutant(body: str) -> str:
return original(body) + "\n\nEn setning som ikke staar i konseptfila."
return _patched(delivered_text=mutant)
def _drop_text_key() -> contextlib.AbstractContextManager[None]:
original = consume.excerpt_for
@ -1500,6 +1564,9 @@ MUTANTS: tuple[Mutant, ...] = (
Mutant("M11 the cut takes the LAST k", 1, _last_k),
Mutant("M12 the delivered text is truncated to 40 characters", 6, _truncate_delivered),
Mutant("M13 the excerpt carries no text", 6, _drop_text_key),
Mutant(
"M14 the delivered text carries a sentence the concept file does not", 6, _extend_delivered
),
)
@ -1525,10 +1592,11 @@ MUTANT_ROSTER: tuple[str, ...] = (
"M11 the cut takes the LAST k",
"M12 the delivered text is truncated to 40 characters",
"M13 the excerpt carries no text",
"M14 the delivered text carries a sentence the concept file does not",
)
#: The roster's length, written as a number so appending is not one edit.
MUTANT_COUNT = 13
MUTANT_COUNT = 14
def _score(rows: Sequence[Row]) -> dict[int, int]:
@ -1696,8 +1764,8 @@ def read_real_set(name: str, path: Path, expected_sha256: str) -> QuestionSet:
)
return QuestionSet("wiki-20", "wiki", path, measured, questions, ())
if name == "r761":
questions = []
controls = []
r761_questions: list[Question] = []
controls: list[Control] = []
for entry in spec["sporsmal"]:
if str(entry["id"]).startswith("KN"):
controls.append(
@ -1706,7 +1774,7 @@ def read_real_set(name: str, path: Path, expected_sha256: str) -> QuestionSet:
)
)
continue
questions.append(
r761_questions.append(
Question(
id=str(entry["id"]),
question=str(entry["sporsmal"]),
@ -1715,9 +1783,11 @@ def read_real_set(name: str, path: Path, expected_sha256: str) -> QuestionSet:
fasit=(Fasit(by="title", value=str(entry["fasit"])),),
)
)
return QuestionSet("r761-sk2", "r761", path, measured, tuple(questions), tuple(controls))
return QuestionSet(
"r761-sk2", "r761", path, measured, tuple(r761_questions), tuple(controls)
)
if name == "vegnormal":
questions = []
vegnormal_questions: list[Question] = []
for entry in spec["sporsmal"]:
by_normal: dict[str, list[Fasit]] = {}
for item in entry["must_cite"]:
@ -1729,7 +1799,7 @@ def read_real_set(name: str, path: Path, expected_sha256: str) -> QuestionSet:
# count is unchanged, which is what the denominator counts.
for normal, fasit in sorted(by_normal.items()):
suffix = f"/{normal}" if len(by_normal) > 1 else ""
questions.append(
vegnormal_questions.append(
Question(
id=f"{entry['id']}{suffix}",
question=str(entry["sporsmal"]),
@ -1737,7 +1807,7 @@ def read_real_set(name: str, path: Path, expected_sha256: str) -> QuestionSet:
bundle=normal,
)
)
return QuestionSet("vegnormal-32", "", path, measured, tuple(questions), ())
return QuestionSet("vegnormal-32", "", path, measured, tuple(vegnormal_questions), ())
raise GateUsage(f"unknown real set `{name}`; one of wiki, r761, vegnormal")
@ -1846,22 +1916,65 @@ def row_eight(real: Sequence[tuple[QuestionSet, Mapping[str, Path]]]) -> Row:
)
def row_nine() -> Row:
"""K2: the bundles are on this machine and the gold set is nowhere."""
return Row(
def row_nine(k2: tuple[QuestionSet, Mapping[str, Path]] | None = None) -> Row:
"""K2: the bundles are on this machine and the gold set is nowhere.
IT TAKES AN INPUT, so it is a measurement and not a placeholder. Until
2026-09-19 this row was a hard-coded RED that could not have gone green on
the day somebody wrote the set; it now reads one through `--k2`, in this
gate's own set shape, and `K2_QUESTIONS` is the denominator whatever the
file carries -- a set of five would be a different set with this one's
name.
WITHOUT A SET IT STAYS RED rather than NOT RUN, and that is this row's own
published rule ("a set that cannot be measured is a red number, never an
absent row"): the denominator is KNOWN -- six questions, recorded -- so
the absence is measured. Row 8 says NOT RUN because ITS denominator is not
known until the sets arrive. Both fail the gate identically.
"""
if k2 is None:
return Row(
9,
"K2, the sixth set",
0,
K2_QUESTIONS,
RED,
f"not measured: 0 of {K2_QUESTIONS} questions have a gold set anywhere",
[
" the bundles exist (~/corpora/okf-telling-20260829/K2-bundle-*), the "
"answer key does not",
" a set that cannot be measured is a red number, never an absent row",
" who can write it: whoever holds the K2 corpus -- it names documents "
"that may not be committed here, so it arrives as a path plus a sha256",
" the shape to write it in is this gate's own set shape, the one "
"`tests/fixtures/retrieval/set-*.json` is written in",
],
)
question_set, bundles = k2
units = [
unit
for question in question_set.questions
for unit in measure_units(bundles[question.bundle or question_set.bundle], question)
]
answered = len({unit.question_id for unit in units if unit.hit})
details = [
f" {question_set.set_id}: {answered} of {K2_QUESTIONS} questions | "
f"{sum(1 for unit in units if unit.hit)} of {len(units)} fasit entries "
f"({'citation' if question_set.quoted else 'concept'} granularity) | "
f"sha256 {question_set.sha256[:12]}"
]
details += [
f" miss {unit.question_id} {unit.named}: class {unit.klass or '-'} ({unit.detail})"
for unit in units
if not unit.hit
]
return _row(
9,
"K2, the sixth set",
0,
answered,
K2_QUESTIONS,
RED,
f"not measured: 0 of {K2_QUESTIONS} questions have a gold set anywhere",
[
" the bundles exist (~/corpora/okf-telling-20260829/K2-bundle-*), the "
"answer key does not",
" a set that cannot be measured is a red number, never an absent row",
" who can write it: whoever holds the K2 corpus -- it names documents "
"that may not be committed here, so it arrives as a path plus a sha256",
],
f"the recorded denominator is {K2_QUESTIONS} questions, whatever the file carries",
details,
)
@ -1879,6 +1992,12 @@ LIMITS: tuple[str, ...] = (
"payload that grows a confidence field moves this row and not the other way.",
"Row 6 proves the delivered bytes are the bundle's bytes. It cannot prove "
"they answer the question.",
"The judge reads a concept through `consume.read_concept` and "
"`consume.delivered_text` -- the same parser it judges, not the same "
"NUMBER. Measured 2026-09-19: the index is warmed BEFORE the first "
"mutation, so a mutation of `delivered_text` moves the payload and not the "
"judge (M14 is felled); an index built UNDER such a mutation would move "
"both sides equally, and the gate never builds one.",
)
#: Exceptions to 100 %. The first one that arises is the OPERATOR's, so this
@ -1920,6 +2039,7 @@ def evaluate(
real: Sequence[tuple[QuestionSet, Mapping[str, Path]]] = (),
mutants: Sequence[Mutant] = MUTANTS,
sets: Mapping[str, str] = SYNTHETIC_SETS,
k2: tuple[QuestionSet, Mapping[str, Path]] | None = None,
) -> list[Row]:
cases, _ = synthetic_cases(root, fixtures, sets)
rows = deterministic_rows(cases)
@ -1928,7 +2048,7 @@ def evaluate(
row_five(registration),
row_seven(cases, rows, mutants=mutants),
row_eight(real),
row_nine(),
row_nine(k2),
]
@ -1966,7 +2086,7 @@ def _bundle_map(value: str) -> dict[str, Path]:
def _real_sets(
arguments: Sequence[Sequence[str]],
) -> list[tuple[QuestionSet, Mapping[str, Path]]]:
real = []
real: list[tuple[QuestionSet, Mapping[str, Path]]] = []
for name, path, sha, bundle in arguments:
question_set = read_real_set(name, Path(path).expanduser(), sha)
bundles = _bundle_map(bundle)
@ -1976,6 +2096,25 @@ def _real_sets(
return real
def _k2_set(
argument: Sequence[str] | None,
) -> tuple[QuestionSet, Mapping[str, Path]] | None:
if not argument:
return None
path, sha, bundle = argument
question_set = load_set(Path(path).expanduser(), sha)
if len(question_set.questions) != K2_QUESTIONS:
raise GateUsage(
f"{path}: K2's denominator is {K2_QUESTIONS} questions and this set "
f"carries {len(question_set.questions)}; a set of another size is "
"another set wearing this one's name"
)
bundles = _bundle_map(bundle)
if list(bundles) == [""]:
bundles = {question_set.bundle: bundles[""]}
return question_set, bundles
def main(argv: list[str] | None = None) -> int:
parser = argparse.ArgumentParser(description=__doc__.split("\n", 1)[0])
parser.add_argument("--json", action="store_true", help="emit the rows as JSON")
@ -1990,6 +2129,15 @@ def main(argv: list[str] | None = None) -> int:
"BUNDLE is a path, or `key=path,key=path` for a set spanning bundles"
),
)
parser.add_argument(
"--k2",
nargs=3,
metavar=("SET", "SHA256", "BUNDLE"),
help=(
"run row 9 against a K2 gold set, written in this gate's own set "
"shape; the denominator stays the recorded six questions"
),
)
parser.add_argument(
"--holdout",
type=Path,
@ -1999,8 +2147,9 @@ def main(argv: list[str] | None = None) -> int:
args = parser.parse_args(argv)
try:
real = _real_sets(args.real)
k2 = _k2_set(args.k2)
with tempfile.TemporaryDirectory(prefix="okf-retrieval-gate-") as scratch:
rows = evaluate(Path(scratch), registration=args.holdout, real=real)
rows = evaluate(Path(scratch), registration=args.holdout, real=real, k2=k2)
except GateUsage as error:
print(f"okf-retrieval-gate: {error}", file=sys.stderr)
return 2