open/llm-ingestion-okf
1
0
Fork 0
Code Issues Releases 4 Activity

fix(retrieval-gate): row 5 reads git for the half a registration cannot assert

Browse source 
PM's J1. Three of row 5's ten checks now read the repository's history instead
of the registration's own fields:

- the registration is COMMITTED and unmodified against HEAD -- a file written
  in a working tree is not a pre-registration;
- the commit that ADDED it is not itself a change to the ranking -- a
  threshold and the change it is supposed to bind, in one commit, is neither;
- a change to the ranking landed AFTER it. This is the one that cannot be
  self-attested: it is satisfied only in the order a pre-registration actually
  happens, and it is red today because neither half has happened.

The three come from `git_provenance`, a `Provenance` the row receives the way
it already receives a path, so both directions are driven from the test and
the default is the real git. Outside a git tree, or with no git on PATH, the
fields come back in their REFUSING form with the reason printed -- unknown is
not unknown-and-therefore-fine.

WHAT GIT CANNOT PROVE IS STATED IN THE ROW, not implied: history shows ORDER,
never that nobody read the number before writing the threshold -- a number can
be read from an uncommitted tree. Order is what these three checks are.

Row 5 is unchanged where it matters today: `0 of 1 RED`, no registration. The
two existing direction tests move 7 -> 10 checks and a third arm is added that
drives each git check red on its own. 54 passed.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Kjell Tore Guttormsen 2026-09-19 21:08:52 +02:00
commit 30cbb69ac0
Signed by: ktg
SSH key fingerprint: SHA256:JakMjO6FTBBzN0Bhfj9saOoEjaFxlSdYuZQQpM/lF9Q
2 changed files with 169 additions and 6 deletions
Download patch file Download diff file Expand all files Collapse all files

56
tests/test_retrieval_gate.py
View file

@ -329,6 +329,21 @@ def test_a_covered_question_that_came_back_marked_would_fail_the_same_row(
# --- row 5 --------------------------------------------------------------------
def _carried_by_git(
*, tracked: bool = True, same_commit: bool = False, after: int = 1
) -> gate.Provenance:
"""A history row 5 reads instead of believing the registration. The three
git checks are driven from HERE, never from a file the test writes, which
is the whole point of the row."""
return gate.Provenance(
tracked=tracked,
unmodified=tracked,
commit="0" * 40 if tracked else "",
commit_touches_ranking=same_commit,
ranking_commits_after=after,
)
def test_row_five_is_red_while_no_hold_out_is_registered(tmp_path: Path) -> None:
row = gate.row_five(tmp_path / "absent.json")
assert (row.k, row.m, row.status) == (0, 1, gate.RED)
@ -352,8 +367,8 @@ def test_row_five_is_green_for_a_registration_that_carries_all_seven(tmp_path: P
),
encoding="utf-8",
)
row = gate.row_five(registration)
assert (row.k, row.m, row.status) == (7, 7, gate.GREEN)
row = gate.row_five(registration, provenance=lambda _: _carried_by_git())
assert (row.k, row.m, row.status) == (10, 10, gate.GREEN)
def test_row_five_falls_on_a_number_read_before_its_threshold_was_written(
@ -375,11 +390,44 @@ def test_row_five_falls_on_a_number_read_before_its_threshold_was_written(
),
encoding="utf-8",
)
row = gate.row_five(registration)
assert (row.k, row.m, row.status) == (6, 7, gate.RED)
row = gate.row_five(registration, provenance=lambda _: _carried_by_git())
assert (row.k, row.m, row.status) == (9, 10, gate.RED)
assert any("no reading predates the threshold: NO" in detail for detail in row.details)
def test_row_five_falls_when_the_registration_rides_in_on_the_ranking_change(
tmp_path: Path,
) -> None:
"""The three git checks, each driven red on its own: a file nobody
committed, a threshold committed together with the ranking change, and a
registration no ranking change has come after."""
held = tmp_path / "held.json"
held.write_text('{"questions": []}', encoding="utf-8")
registration = tmp_path / "registration.json"
registration.write_text(
json.dumps(
{
"set": str(held),
"sha256": gate.sha256_of(held),
"threshold": "hit@payload at unit granularity >= 0.8",
"threshold_written_at": "2026-09-19T10:00:00Z",
"written_by": "somebody",
"readings": [],
}
),
encoding="utf-8",
)
arms = {
"uncommitted": (_carried_by_git(tracked=False), "committed"),
"same commit as the ranking": (_carried_by_git(same_commit=True), "same commit"),
"nothing changed since": (_carried_by_git(after=0), "0 commit(s) touching"),
}
for label, (history, expected) in arms.items():
row = gate.row_five(registration, provenance=lambda _, h=history: h)
assert row.status == gate.RED, label
assert any(expected in detail for detail in row.details), (label, row.details)
# --- row 6 --------------------------------------------------------------------

119
tools/okf_retrieval_gate.py
View file

@ -45,10 +45,11 @@ import argparse
import contextlib
import hashlib
import json
import subprocess
import sys
import tempfile
from collections.abc import Callable, Iterator, Mapping, Sequence
from dataclasses import dataclass, field
from dataclasses import dataclass, field, replace
from pathlib import Path
from typing import Any
@ -1155,6 +1156,74 @@ def row_four(cases: Sequence[Case]) -> Row:
)
#: The path a capability session changes. Row 5 reads git for the ONE thing a
#: registration cannot assert about itself: that it was already committed when
#: that path moved.
RANKING_PATH = "src/llm_ingestion_okf/consume.py"
@dataclass(frozen=True)
class Provenance:
"""What GIT says about a file. Every field here is a fact about the
repository's history, which is the one thing the file cannot also write.
Unknown is NOT unknown-and-therefore-fine: outside a git tree, or with no
git on PATH, the fields come back in their refusing form and the note says
why.
"""
tracked: bool
unmodified: bool
commit: str
commit_touches_ranking: bool
ranking_commits_after: int
note: str = ""
def _git(repo: Path, *arguments: str) -> tuple[int, str]:
try:
finished = subprocess.run(
["git", "-C", str(repo), *arguments],
capture_output=True,
text=True,
check=False,
)
except OSError as error:
return 127, str(error)
return finished.returncode, finished.stdout
def git_provenance(path: Path, *, repo: Path = REPO, ranking: str = RANKING_PATH) -> Provenance:
"""`path`'s history, read from git and never from `path`."""
refusing = Provenance(False, False, "", True, 0)
code, _ = _git(repo, "rev-parse", "--git-dir")
if code != 0:
return replace(refusing, note="not a git tree, or git is not on PATH")
tracked = _git(repo, "ls-files", "--error-unmatch", "--", str(path))[0] == 0
if not tracked:
return replace(refusing, note=f"{_display(path)} is not tracked in this repository")
unmodified = _git(repo, "diff", "--quiet", "HEAD", "--", str(path))[0] == 0
_, log = _git(repo, "log", "--diff-filter=A", "--format=%H", "--", str(path))
commits = [line.strip() for line in log.splitlines() if line.strip()]
if not commits:
return replace(
refusing,
tracked=True,
unmodified=unmodified,
note="no commit adds this file; it is staged and not committed",
)
commit = commits[-1]
_, touched = _git(repo, "show", "--pretty=", "--name-only", commit)
_, after = _git(repo, "log", "--format=%H", f"{commit}..HEAD", "--", ranking)
return Provenance(
tracked=True,
unmodified=unmodified,
commit=commit,
commit_touches_ranking=ranking in touched.split(),
ranking_commits_after=len([line for line in after.splitlines() if line.strip()]),
)
def _display(path: Path) -> str:
"""Repo-relative where it is inside the repo, absolute otherwise: a test
drives this row from `tmp_path`, and `relative_to` raises there."""
@ -1164,12 +1233,31 @@ def _display(path: Path) -> str:
return str(path)
def row_five(registration: Path = HOLDOUT_REGISTRATION) -> Row:
def row_five(
registration: Path = HOLDOUT_REGISTRATION,
*,
provenance: Callable[[Path], Provenance] = git_provenance,
) -> Row:
"""The hold-out set: written blind, frozen before the first capability
line, its threshold written before anyone saw the number.
Report-only without a written threshold is not a protection, so the
absence of a threshold is red rather than absent.
AND NEITHER IS A PROTECTION THE FILE WRITES ABOUT ITSELF. Until
2026-09-19 every check here read a field the registration owned, and two
files PM wrote in the moment came back `7 of 7 GREEN`. Three checks now
read GIT instead: the registration is committed and unmodified, the commit
that ADDED it is not itself a change to the ranking, and a change to the
ranking landed AFTER it. The third is the one that cannot be self-attested
-- it is green only in the order a pre-registration actually happens, the
registration first and the ranking change second, and it is red today
because neither has happened.
WHAT GIT CANNOT PROVE, stated rather than implied: that nobody read the
number before writing the threshold. A number can be read from an
uncommitted working tree, and no history shows that. What history does
show is ORDER, and order is what these three checks are.
"""
if not registration.is_file():
return Row(
@ -1183,6 +1271,9 @@ def row_five(registration: Path = HOLDOUT_REGISTRATION) -> Row:
" a set written by a session other than the one that changes the "
"ranking, frozen with sha256 before the first capability line",
" its threshold written, with a date, before its number is read",
" and COMMITTED before the ranking moves: git must show the "
f"registration in a commit of its own, with a later commit to "
f"{RANKING_PATH}. That is the half a session cannot write about itself",
],
)
try:
@ -1225,6 +1316,30 @@ def row_five(registration: Path = HOLDOUT_REGISTRATION) -> Row:
f"{len(early)} reading(s) before {written_at}" if early else "0 early readings",
)
)
history = provenance(registration)
checks.append(
(
"git: the registration is committed, unmodified",
history.tracked and history.unmodified and bool(history.commit),
history.note or (f"{history.commit[:12]} clean" if history.unmodified else "modified"),
)
)
checks.append(
(
"git: its commit is not itself a ranking change",
bool(history.commit) and not history.commit_touches_ranking,
f"{RANKING_PATH} in the same commit"
if history.commit_touches_ranking
else "separate commit",
)
)
checks.append(
(
"git: a ranking change landed after it",
history.ranking_commits_after > 0,
f"{history.ranking_commits_after} commit(s) touching {RANKING_PATH} since",
)
)
passed = sum(1 for _, ok, _ in checks if ok)
return _row(
5,