test(retrieval-gate): row 8 must name the bundle it measured, and the three real sets must be pinned
Red first, on behaviour: the identity assertion fails on the rendered
details, and the two pin assertions fail with DID NOT RAISE. Neither dies
on an import or an attribute.
PM measured both on 23588e5: one pinned r761 set gives 4 of 7 on three
bundles and 0 of 7 on a fourth while the session recorded 7 of 7, from a
bundle the output never names; and three self-written one-question files
plus a self-written bundle read `3 of 3 GREEN`.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
parent
c5bfb32ce6
commit
3dc2491083
1 changed files with 110 additions and 0 deletions
|
|
@ -1090,3 +1090,113 @@ def test_j7_a_cut_that_delivers_nothing_is_exit_two_and_never_a_quiet_green_row(
|
|||
with gate._wrap_cut(k=0), pytest.raises(Exception) as refusal:
|
||||
[gate.measure_case(case.question_set, case.bundles) for case in cases]
|
||||
assert not isinstance(refusal.value, AssertionError), refusal.value
|
||||
|
||||
|
||||
# --- step 0: the real sets are pinned, and row 8 names what it measured -------
|
||||
|
||||
|
||||
def test_row_eight_names_the_bundle_identity_of_every_set_it_measured(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
"""The row printed the SET's sha256 and never the bundle's identity.
|
||||
|
||||
Measured 2026-09-19 by PM with one pinned r761 set: three bundles gave
|
||||
4 of 7 and a fourth gave 0 of 7, while the session recorded 7 of 7 -- from
|
||||
a bundle no line of the output named. A number nobody can attach to a
|
||||
bundle cannot be reproduced and cannot be felled.
|
||||
"""
|
||||
bundles = {"positive": _bundles(tmp_path)["positive"]}
|
||||
real = [
|
||||
(_hitting_set(name, quote="innen 1. november"), bundles)
|
||||
for name in gate.REQUIRED_REAL_SETS
|
||||
]
|
||||
row = gate.row_eight(real)
|
||||
# The identity is read here, independently of the gate, from the same
|
||||
# bundle the row was handed.
|
||||
bundle_id = consume.root_bundle_id_of(bundles["positive"])
|
||||
ref = consume.bundle_ref(bundles["positive"])
|
||||
assert bundle_id and ref.startswith("sha256-tree:")
|
||||
for set_id in gate.REQUIRED_REAL_SETS:
|
||||
named = [detail for detail in row.details if detail.strip().startswith(f"{set_id}:")]
|
||||
assert named, f"{set_id} has no line of its own"
|
||||
block = "\n".join(row.details)
|
||||
assert bundle_id in block, f"{set_id}: the bundle_id is nowhere in the output"
|
||||
assert ref[:24] in block, f"{set_id}: the bundle ref is nowhere in the output"
|
||||
|
||||
|
||||
def test_all_three_real_sets_are_pinned_and_not_only_the_first(tmp_path: Path) -> None:
|
||||
"""Every one of the three, driven through the command-line path.
|
||||
|
||||
The names and the count are written out here rather than read from
|
||||
`REQUIRED_REAL_SETS`: a test taking its denominator from the tuple it is
|
||||
checking would stay green if a name were dropped from both.
|
||||
"""
|
||||
names = [("wiki", "wiki-20"), ("r761", "r761-sk2"), ("vegnormal", "vegnormal-32")]
|
||||
assert len(names) == 3
|
||||
bundle = _bundles(tmp_path)["positive"]
|
||||
for adapter, set_id in names:
|
||||
path = tmp_path / f"{adapter}-invented.json"
|
||||
path.write_text(_invented_real_set(adapter), encoding="utf-8")
|
||||
with pytest.raises(gate.GateUsage) as refusal:
|
||||
gate._real_sets([(adapter, str(path), gate.sha256_of(path), str(bundle))])
|
||||
assert set_id in str(refusal.value)
|
||||
|
||||
|
||||
def _invented_real_set(adapter: str) -> str:
|
||||
"""One question, in the adapter's own shape, with invented content."""
|
||||
if adapter == "wiki":
|
||||
return json.dumps(
|
||||
{
|
||||
"questions": [
|
||||
{
|
||||
"id": "W1",
|
||||
"question": "Naar kontrolleres vinterberedskapen?",
|
||||
"fasit": [{"doc": "haandbok", "quote": "innen 1. november"}],
|
||||
}
|
||||
]
|
||||
}
|
||||
)
|
||||
if adapter == "r761":
|
||||
return json.dumps(
|
||||
{"sporsmal": [{"id": "S1", "sporsmal": "Hva sier punktet?", "fasit": "4.2 Vakthold"}]}
|
||||
)
|
||||
return json.dumps(
|
||||
{
|
||||
"sporsmal": [
|
||||
{
|
||||
"id": "T1-1",
|
||||
"sporsmal": "Hva viser kravet til?",
|
||||
"must_cite": [{"normal": "N100:2023", "req_number": "2.3.2-3"}],
|
||||
}
|
||||
]
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def test_a_wiki_set_of_one_question_is_refused_on_the_command_line(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
"""PM's measured attack: three self-written one-question files and a
|
||||
self-written bundle read `3 of 3 GREEN`, because both the set AND its
|
||||
sha256 came from the command line and nothing said how big `wiki-20` is.
|
||||
"""
|
||||
path = tmp_path / "wiki-shaped.json"
|
||||
path.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"schema": "1",
|
||||
"questions": [
|
||||
{
|
||||
"id": "W1",
|
||||
"question": "Naar kontrolleres vinterberedskapen?",
|
||||
"fasit": [{"doc": "haandbok", "quote": "innen 1. november"}],
|
||||
}
|
||||
],
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
bundle = _bundles(tmp_path)["positive"]
|
||||
with pytest.raises(gate.GateUsage) as refusal:
|
||||
gate._real_sets([("wiki", str(path), gate.sha256_of(path), str(bundle))])
|
||||
assert "wiki-20" in str(refusal.value)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue