test(retrieval-gate): row 8's judge must read a passage as a passage — red

Since v1.1 C1 a concept over PASSAGE_CHARS is delivered as its answering
passage (heading, [...], span, [...]), and the judge still demands the whole
body (the row-6 identity), so every such delivery is class e even when the span
carries the citation. PM's re-measurement of e503f6a found this to be the
dominant miss class of row 8 on the real set.

Pinned before the judge moves, on an invented one-concept bundle:
- a passage whose span carries the quote is a HIT; the offsets count in the
  DELIVERED body, and a test asserts that reading them in the concept file
  misses (trap 1);
- the same passage with one invented sentence, in the span or between the
  marker line and the span, is class e, named (trap 2: "the span occurs
  somewhere in the text" would count it);
- a quote the passage carries only in its heading or its markers is class d,
  beside the known-positive in the same run;
- row 7 carries M15, passage cheating, as its fifteenth mutant.

Fails on e503f6a: five assertion failures and one ValueError (M15 does not
exist yet); no collection error.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Kjell Tore Guttormsen 2026-09-22 23:53:05 +02:00
commit 70c388d419

View file

@ -532,6 +532,9 @@ def test_a_mutant_is_felled_by_the_row_that_got_worse_and_never_by_one_that_did_
"M06 ", # the passage signal reads no body
"M07 ", # the field signal weighs no title and no path
"M10 ", # the fusion is flattened
# A passage carrying one sentence the file does not: M14's shape for
# the delivery form v1.1 C1 added.
"M15 ",
],
)
def test_row_seven_fells_every_mutant_of_a_mechanism_the_ranking_runs(
@ -1028,6 +1031,140 @@ def test_the_mutant_roster_is_pinned_apart_from_the_list_it_names() -> None:
assert len(set(gate.MUTANT_ROSTER)) == len(gate.MUTANT_ROSTER)
# --- a passage is judged as a passage ------------------------------------------
#
# Since v1.1 C1 a concept over `consume.PASSAGE_CHARS` is delivered as its
# answering passage: the nearest heading above the span, `[...]`, the span,
# `[...]`. The row-6 identity (delivered text == the whole body) is older than
# that and read every such delivery as class e. The judge must now accept a
# passage that IS its exact reconstruction from the bundle's bytes, and nothing
# looser: "the span occurs somewhere in the text" accepts an invented sentence
# beside it.
_HOUSE_NOTE = "Styret gjennomgaar notatet og foerer det inn i arkivet. Sekretaeren sender kopi."
_INVENTED = "En setning som ikke staar i konseptfila."
def _house_bundle(tmp_path: Path) -> Path:
"""One concept of about 6 600 characters: a heading ten lines in, the
answer thirty lines below it, and a literal `[...]` on the LAST line, far
outside any span centred on the answer."""
body = "\n".join(
[_HOUSE_NOTE] * 10
+ ["## Teknisk rom"]
+ [_HOUSE_NOTE] * 30
+ ["Varmepumpa i kjelleren faar service av roerleggeren hvert aar."]
+ [_HOUSE_NOTE] * 40
+ ["Se vedlegg [...] i arkivet."]
)
spec = gate.BundleSpec(
"retrieval-house",
(
gate.DocumentSpec(
"husbok", "husbok.md", (gate.ConceptSpec("drift", "Drift av huset", body),)
),
*(
gate.DocumentSpec(
f"skriv-{number}",
f"skriv-{number}.md",
(gate.ConceptSpec("skriv", f"Skriv {number}", "Skrivet er arkivert."),),
)
for number in range(3)
),
),
)
return gate.build_bundle(tmp_path / "house", spec)
def _house_question(*quotes: str) -> gate.Question:
return gate.Question(
id="H1",
question="Hvem gir varmepumpa i kjelleren service?",
fasit=tuple(gate.Fasit("concept", "husbok/drift", quote) for quote in quotes),
)
def _inject(where: str) -> pytest.MonkeyPatch:
"""A passage carrying one sentence the concept file does not: at the end of
the span, or between the heading's marker line and the span."""
original = consume.as_passage
def mutant(excerpt: dict[str, object], window: int) -> dict[str, object]:
out = original(excerpt, window)
if "passage" in out:
text = str(out["text"])
if where == "span":
cut = text.rindex(f"\n{consume.PASSAGE_ELISION}")
out["text"] = f"{text[:cut]} {_INVENTED}{text[cut:]}"
else:
marker = f"{consume.PASSAGE_ELISION}\n"
cut = text.index(marker) + len(marker)
out["text"] = f"{text[:cut]}{_INVENTED}\n{text[cut:]}"
return out
patch = pytest.MonkeyPatch()
patch.setattr(consume, "as_passage", mutant)
return patch
def test_a_passage_whose_span_carries_the_quote_is_a_hit(tmp_path: Path) -> None:
bundle = _house_bundle(tmp_path)
gate.bundle_index(bundle)
question = _house_question("service av roerleggeren")
payload = consume.build_payload(bundle, question=question.question, withheld_full=True)
(excerpt,) = [e for e in payload["excerpts"] if e["concept_id"] == "husbok/drift"]
passage = excerpt["passage"]
assert isinstance(passage, dict), "the premise: this concept is delivered as a passage"
start, end = passage["start"], passage["end"]
assert 0 < start and end < passage["of"], "the premise: `[...]` on both sides"
# FELLE 1, pinned: the offsets count in the DELIVERED body. Read against
# the concept file they land a frontmatter's length off.
view = gate.bundle_index(bundle).concepts["husbok/drift"]
assert view.body[start:end] in str(excerpt["text"])
assert view.whole[start:end] not in str(excerpt["text"])
(unit,) = gate.measure_units(bundle, question)
assert (unit.hit, unit.confirmed, unit.klass) == (True, True, None), unit
@pytest.mark.parametrize("where", ["span", "between heading and span"])
def test_a_passage_carrying_an_invented_sentence_is_not_a_hit(tmp_path: Path, where: str) -> None:
"""FELLE 2, pinned: the quote is still in the span and the span is still in
the text, so "the span occurs somewhere" would count this. The delivered
text must BE the reconstruction."""
bundle = _house_bundle(tmp_path)
gate.bundle_index(bundle) # the judge reads the bundle BEFORE the payload
question = _house_question("service av roerleggeren")
patch = _inject(where)
try:
(unit,) = gate.measure_units(bundle, question)
finally:
patch.undo()
assert unit.hit is False
assert unit.confirmed is False
assert unit.klass == "e"
assert unit.detail == "the delivered passage is not its reconstruction from the bundle's bytes"
def test_a_quote_the_passage_carries_only_outside_its_span_is_not_a_hit(tmp_path: Path) -> None:
"""The heading and the markers are delivered and are not the span: a quote
found only there was delivered without the citation (class d). The first
unit is the known-positive of the same run."""
bundle = _house_bundle(tmp_path)
units = gate.measure_units(
bundle, _house_question("service av roerleggeren", "Teknisk rom", "[...]")
)
assert [(unit.hit, unit.klass) for unit in units] == [
(True, None),
(False, "d"),
(False, "d"),
], units
def test_row_seven_names_passage_cheating() -> None:
assert gate.MUTANT_COUNT == 15
assert gate.MUTANT_ROSTER[-1] == "M15 the passage carries a sentence the concept file does not"
def test_the_synthetic_corpus_is_pinned_like_the_sets(tmp_path: Path) -> None:
assert gate.specs_digest() == gate.SPECS_SHA256
tuned = dict(gate.SPECS)