test(retrieval-gate): row 8's judge must read a passage as a passage — red
Since v1.1 C1 a concept over PASSAGE_CHARS is delivered as its answering passage (heading, [...], span, [...]), and the judge still demands the whole body (the row-6 identity), so every such delivery is class e even when the span carries the citation. PM's re-measurement ofe503f6afound this to be the dominant miss class of row 8 on the real set. Pinned before the judge moves, on an invented one-concept bundle: - a passage whose span carries the quote is a HIT; the offsets count in the DELIVERED body, and a test asserts that reading them in the concept file misses (trap 1); - the same passage with one invented sentence, in the span or between the marker line and the span, is class e, named (trap 2: "the span occurs somewhere in the text" would count it); - a quote the passage carries only in its heading or its markers is class d, beside the known-positive in the same run; - row 7 carries M15, passage cheating, as its fifteenth mutant. Fails one503f6a: five assertion failures and one ValueError (M15 does not exist yet); no collection error. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
e503f6abd7
commit
70c388d419
1 changed files with 137 additions and 0 deletions
|
|
@ -532,6 +532,9 @@ def test_a_mutant_is_felled_by_the_row_that_got_worse_and_never_by_one_that_did_
|
|||
"M06 ", # the passage signal reads no body
|
||||
"M07 ", # the field signal weighs no title and no path
|
||||
"M10 ", # the fusion is flattened
|
||||
# A passage carrying one sentence the file does not: M14's shape for
|
||||
# the delivery form v1.1 C1 added.
|
||||
"M15 ",
|
||||
],
|
||||
)
|
||||
def test_row_seven_fells_every_mutant_of_a_mechanism_the_ranking_runs(
|
||||
|
|
@ -1028,6 +1031,140 @@ def test_the_mutant_roster_is_pinned_apart_from_the_list_it_names() -> None:
|
|||
assert len(set(gate.MUTANT_ROSTER)) == len(gate.MUTANT_ROSTER)
|
||||
|
||||
|
||||
# --- a passage is judged as a passage ------------------------------------------
|
||||
#
|
||||
# Since v1.1 C1 a concept over `consume.PASSAGE_CHARS` is delivered as its
|
||||
# answering passage: the nearest heading above the span, `[...]`, the span,
|
||||
# `[...]`. The row-6 identity (delivered text == the whole body) is older than
|
||||
# that and read every such delivery as class e. The judge must now accept a
|
||||
# passage that IS its exact reconstruction from the bundle's bytes, and nothing
|
||||
# looser: "the span occurs somewhere in the text" accepts an invented sentence
|
||||
# beside it.
|
||||
|
||||
_HOUSE_NOTE = "Styret gjennomgaar notatet og foerer det inn i arkivet. Sekretaeren sender kopi."
|
||||
_INVENTED = "En setning som ikke staar i konseptfila."
|
||||
|
||||
|
||||
def _house_bundle(tmp_path: Path) -> Path:
|
||||
"""One concept of about 6 600 characters: a heading ten lines in, the
|
||||
answer thirty lines below it, and a literal `[...]` on the LAST line, far
|
||||
outside any span centred on the answer."""
|
||||
body = "\n".join(
|
||||
[_HOUSE_NOTE] * 10
|
||||
+ ["## Teknisk rom"]
|
||||
+ [_HOUSE_NOTE] * 30
|
||||
+ ["Varmepumpa i kjelleren faar service av roerleggeren hvert aar."]
|
||||
+ [_HOUSE_NOTE] * 40
|
||||
+ ["Se vedlegg [...] i arkivet."]
|
||||
)
|
||||
spec = gate.BundleSpec(
|
||||
"retrieval-house",
|
||||
(
|
||||
gate.DocumentSpec(
|
||||
"husbok", "husbok.md", (gate.ConceptSpec("drift", "Drift av huset", body),)
|
||||
),
|
||||
*(
|
||||
gate.DocumentSpec(
|
||||
f"skriv-{number}",
|
||||
f"skriv-{number}.md",
|
||||
(gate.ConceptSpec("skriv", f"Skriv {number}", "Skrivet er arkivert."),),
|
||||
)
|
||||
for number in range(3)
|
||||
),
|
||||
),
|
||||
)
|
||||
return gate.build_bundle(tmp_path / "house", spec)
|
||||
|
||||
|
||||
def _house_question(*quotes: str) -> gate.Question:
|
||||
return gate.Question(
|
||||
id="H1",
|
||||
question="Hvem gir varmepumpa i kjelleren service?",
|
||||
fasit=tuple(gate.Fasit("concept", "husbok/drift", quote) for quote in quotes),
|
||||
)
|
||||
|
||||
|
||||
def _inject(where: str) -> pytest.MonkeyPatch:
|
||||
"""A passage carrying one sentence the concept file does not: at the end of
|
||||
the span, or between the heading's marker line and the span."""
|
||||
original = consume.as_passage
|
||||
|
||||
def mutant(excerpt: dict[str, object], window: int) -> dict[str, object]:
|
||||
out = original(excerpt, window)
|
||||
if "passage" in out:
|
||||
text = str(out["text"])
|
||||
if where == "span":
|
||||
cut = text.rindex(f"\n{consume.PASSAGE_ELISION}")
|
||||
out["text"] = f"{text[:cut]} {_INVENTED}{text[cut:]}"
|
||||
else:
|
||||
marker = f"{consume.PASSAGE_ELISION}\n"
|
||||
cut = text.index(marker) + len(marker)
|
||||
out["text"] = f"{text[:cut]}{_INVENTED}\n{text[cut:]}"
|
||||
return out
|
||||
|
||||
patch = pytest.MonkeyPatch()
|
||||
patch.setattr(consume, "as_passage", mutant)
|
||||
return patch
|
||||
|
||||
|
||||
def test_a_passage_whose_span_carries_the_quote_is_a_hit(tmp_path: Path) -> None:
|
||||
bundle = _house_bundle(tmp_path)
|
||||
gate.bundle_index(bundle)
|
||||
question = _house_question("service av roerleggeren")
|
||||
payload = consume.build_payload(bundle, question=question.question, withheld_full=True)
|
||||
(excerpt,) = [e for e in payload["excerpts"] if e["concept_id"] == "husbok/drift"]
|
||||
passage = excerpt["passage"]
|
||||
assert isinstance(passage, dict), "the premise: this concept is delivered as a passage"
|
||||
start, end = passage["start"], passage["end"]
|
||||
assert 0 < start and end < passage["of"], "the premise: `[...]` on both sides"
|
||||
# FELLE 1, pinned: the offsets count in the DELIVERED body. Read against
|
||||
# the concept file they land a frontmatter's length off.
|
||||
view = gate.bundle_index(bundle).concepts["husbok/drift"]
|
||||
assert view.body[start:end] in str(excerpt["text"])
|
||||
assert view.whole[start:end] not in str(excerpt["text"])
|
||||
(unit,) = gate.measure_units(bundle, question)
|
||||
assert (unit.hit, unit.confirmed, unit.klass) == (True, True, None), unit
|
||||
|
||||
|
||||
@pytest.mark.parametrize("where", ["span", "between heading and span"])
|
||||
def test_a_passage_carrying_an_invented_sentence_is_not_a_hit(tmp_path: Path, where: str) -> None:
|
||||
"""FELLE 2, pinned: the quote is still in the span and the span is still in
|
||||
the text, so "the span occurs somewhere" would count this. The delivered
|
||||
text must BE the reconstruction."""
|
||||
bundle = _house_bundle(tmp_path)
|
||||
gate.bundle_index(bundle) # the judge reads the bundle BEFORE the payload
|
||||
question = _house_question("service av roerleggeren")
|
||||
patch = _inject(where)
|
||||
try:
|
||||
(unit,) = gate.measure_units(bundle, question)
|
||||
finally:
|
||||
patch.undo()
|
||||
assert unit.hit is False
|
||||
assert unit.confirmed is False
|
||||
assert unit.klass == "e"
|
||||
assert unit.detail == "the delivered passage is not its reconstruction from the bundle's bytes"
|
||||
|
||||
|
||||
def test_a_quote_the_passage_carries_only_outside_its_span_is_not_a_hit(tmp_path: Path) -> None:
|
||||
"""The heading and the markers are delivered and are not the span: a quote
|
||||
found only there was delivered without the citation (class d). The first
|
||||
unit is the known-positive of the same run."""
|
||||
bundle = _house_bundle(tmp_path)
|
||||
units = gate.measure_units(
|
||||
bundle, _house_question("service av roerleggeren", "Teknisk rom", "[...]")
|
||||
)
|
||||
assert [(unit.hit, unit.klass) for unit in units] == [
|
||||
(True, None),
|
||||
(False, "d"),
|
||||
(False, "d"),
|
||||
], units
|
||||
|
||||
|
||||
def test_row_seven_names_passage_cheating() -> None:
|
||||
assert gate.MUTANT_COUNT == 15
|
||||
assert gate.MUTANT_ROSTER[-1] == "M15 the passage carries a sentence the concept file does not"
|
||||
|
||||
|
||||
def test_the_synthetic_corpus_is_pinned_like_the_sets(tmp_path: Path) -> None:
|
||||
assert gate.specs_digest() == gate.SPECS_SHA256
|
||||
tuned = dict(gate.SPECS)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue