test(consume): a section whose whole title the question answers loses to its narrower neighbour
RED. The fixture is handwritten and carries no sentence from any corpus: one
source document, one broad section titled with the question's subject alone,
one narrower section adding a qualifier the question never uses, one
known-negative whose title shares four leading characters with the subject,
and twelve fillers.
Measured on the fixture at HEAD (02f9876):
1 bb-narrow 0.04918 lex 4 'Temporary anchoring'
2 cc-prefix 0.04918 lex 4 'Anchorage'
3 aa-broad 0.04866 lex 3 'Anchoring'
which is the same shape as the three R761 misses. pytest -q on this file:
4 failed, 1 passed. The one that passes is the characterisation the rule
stands on -- with one source document the third signal takes ONE distinct
value over the whole bundle, so a third of the fusion carries no information.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
parent
02f9876e16
commit
14ae36ad2e
1 changed files with 175 additions and 0 deletions
175
tests/test_title_covered.py
Normal file
175
tests/test_title_covered.py
Normal file
|
|
@ -0,0 +1,175 @@
|
|||
"""A section whose WHOLE title the question accounts for, read before its
|
||||
narrower neighbour.
|
||||
|
||||
THE DEFECT, in the mechanism rather than in a corpus. Both lexical signals in
|
||||
`concept_scores` are unnormalised COVERAGE COUNTS -- one per question token the
|
||||
candidate answers to. Nothing in the fusion measures how much of the CANDIDATE
|
||||
the question accounts for, so a section titled with the question's subject and
|
||||
nothing else scores exactly what a narrower section titled with that subject
|
||||
PLUS a qualifier scores, and then loses to it on the body count, because a
|
||||
longer or differently-worded body reaches one more of the question's tokens.
|
||||
|
||||
The fixture below is that shape and nothing else: one source document, one
|
||||
broad section whose title is the question's subject, one narrower section whose
|
||||
title is that subject plus a qualifier the question never uses, and fillers so
|
||||
no signal is degenerate for want of members. It carries no sentence from any
|
||||
corpus and names no real document.
|
||||
|
||||
WHY A PARTITION AND NOT A FOURTH SIGNAL. RRF consumes RANKS ONLY, so any one
|
||||
signal contributes at most `1/(RRF_K + 1)`, and with shared ranks a signal
|
||||
whose positive group is SMALL separates least of all: the group's members take
|
||||
position 1 and everyone else position `len(group) + 1`, so a rule that fires on
|
||||
two concepts of 2 761 is worth `1/61 - 1/63` to them. Precision is exactly such
|
||||
a rule. `lookup_hits` above it is the same shape and was made a partition for
|
||||
the same measured reason.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
PROJECT_ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(PROJECT_ROOT / "tools"))
|
||||
|
||||
import okf_consume # noqa: E402
|
||||
|
||||
QUESTION = "Which requirements apply to anchoring on a bridge?"
|
||||
|
||||
_FRONTMATTER = (
|
||||
"---\ntype: reference\ntitle: {title}\nsource_file: {slug}.md\n"
|
||||
"source_sha256: {digest}\ningested_at: 2026-09-01T00:00:00Z\n"
|
||||
"adjudication: proposed\nbundle_id: covered-fixture\n"
|
||||
"verified: [{{ by: process:okf-check, at: 2026-09-01T00:00:00Z }}]\n---\n\n"
|
||||
)
|
||||
|
||||
|
||||
def _covered_bundle(root: Path, *, fillers: int = 12) -> Path:
|
||||
"""One document, so the document prior separates nothing at all.
|
||||
|
||||
`aa-broad` is the section the question is about and its title is the
|
||||
question's subject alone. `bb-narrow` is a narrower section whose title
|
||||
adds a qualifier the question never uses, and whose body reaches ONE more
|
||||
question token, which is the whole of its present advantage. `cc-prefix` is
|
||||
the known-negative for the matcher: its title shares four leading
|
||||
characters with the subject and is not that subject.
|
||||
"""
|
||||
(root / "part").mkdir(parents=True)
|
||||
(root / "index.md").write_text(
|
||||
"---\nokf_version: 0.2\nbundle_id: covered-fixture\n---\n\n"
|
||||
"- [part (index)](part/index.md)\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
entries: list[str] = []
|
||||
|
||||
def add(slug: str, title: str, body: str) -> None:
|
||||
entries.append(f"- [{title}]({slug}.md) \u2014 adjudication: proposed\n")
|
||||
(root / "part" / f"{slug}.md").write_text(
|
||||
_FRONTMATTER.format(title=title, slug=slug, digest="1" * 64) + f"## {title}\n\n" + body,
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
add(
|
||||
"aa-broad",
|
||||
"Anchoring",
|
||||
"Requirements for anchoring are stated in this section.\n" * 3,
|
||||
)
|
||||
add(
|
||||
"bb-narrow",
|
||||
"Temporary anchoring",
|
||||
"Requirements for anchoring of a bridge are stated in this section.\n" * 3,
|
||||
)
|
||||
add(
|
||||
"cc-prefix",
|
||||
"Anchorage",
|
||||
"Requirements for anchorage of a bridge are stated in this section.\n" * 3,
|
||||
)
|
||||
for number in range(1, fillers + 1):
|
||||
add(
|
||||
f"dd-{number:02d}",
|
||||
f"Bridge works {number:02d}",
|
||||
"Requirements for the works on a bridge are stated in this section.\n" * 3,
|
||||
)
|
||||
(root / "part" / "index.md").write_text("".join(entries), encoding="utf-8")
|
||||
return root
|
||||
|
||||
|
||||
def _ranking(root: Path, **kwargs: object) -> list[str]:
|
||||
concepts = [
|
||||
okf_consume.read_concept(
|
||||
root / f"{concept_id}.md", bundle_root=root, root_bundle_id="covered-fixture"
|
||||
)
|
||||
for concept_id in okf_consume.enumerate_concepts(root)
|
||||
]
|
||||
ranked = okf_consume.concept_scores(
|
||||
concepts,
|
||||
QUESTION,
|
||||
okf_consume.document_scores(root, QUESTION),
|
||||
**kwargs, # type: ignore[arg-type]
|
||||
)
|
||||
return [concept.concept_id for concept, _, _ in ranked]
|
||||
|
||||
|
||||
def _position(order: list[str], slug: str) -> int:
|
||||
for index, concept_id in enumerate(order, start=1):
|
||||
if concept_id.endswith(slug):
|
||||
return index
|
||||
raise AssertionError(f"{slug} is not in the ranking at all")
|
||||
|
||||
|
||||
def test_the_document_prior_separates_nothing_on_a_one_document_bundle(tmp_path: Path) -> None:
|
||||
# CHARACTERISATION of the ground the rule stands on: with one source
|
||||
# document every concept takes the same value from the third signal, so
|
||||
# a third of the fusion carries no information here.
|
||||
root = _covered_bundle(tmp_path / "bundle")
|
||||
prior = okf_consume.document_scores(root, QUESTION)
|
||||
assert len(set(prior.values())) == 1
|
||||
|
||||
|
||||
def test_the_narrow_section_outranks_the_broad_one_without_the_rule(tmp_path: Path) -> None:
|
||||
# The defect itself, stated as a green characterisation so the fix below
|
||||
# has a measured before.
|
||||
root = _covered_bundle(tmp_path / "bundle")
|
||||
order = _ranking(root, title_covered=False)
|
||||
assert _position(order, "bb-narrow") < _position(order, "aa-broad")
|
||||
|
||||
|
||||
def test_the_question_answering_the_whole_title_takes_the_broad_section_first(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
root = _covered_bundle(tmp_path / "bundle")
|
||||
order = _ranking(root)
|
||||
assert _position(order, "aa-broad") == 1
|
||||
assert _position(order, "aa-broad") < _position(order, "bb-narrow")
|
||||
|
||||
|
||||
def test_the_rule_reads_the_title_by_EQUALITY_and_not_by_shared_prefix(tmp_path: Path) -> None:
|
||||
"""The known-negative for the matcher, measured before the rule was written.
|
||||
|
||||
`anchorage` and `anchoring` share four leading characters, which is what
|
||||
`tokens_match` accepts. Under a prefix-matching form of this rule the
|
||||
fixture's `cc-prefix` would be floated beside the section the question
|
||||
actually names.
|
||||
"""
|
||||
root = _covered_bundle(tmp_path / "bundle")
|
||||
hits = okf_consume.title_covered_hits(
|
||||
[
|
||||
okf_consume.read_concept(
|
||||
root / f"{concept_id}.md", bundle_root=root, root_bundle_id="covered-fixture"
|
||||
)
|
||||
for concept_id in okf_consume.enumerate_concepts(root)
|
||||
],
|
||||
QUESTION,
|
||||
)
|
||||
assert [concept_id.rsplit("/", 1)[-1] for concept_id in hits] == ["aa-broad"]
|
||||
|
||||
|
||||
def test_the_rule_has_an_explicit_opt_out_the_cli_exposes(tmp_path: Path) -> None:
|
||||
""" "A default a caller cannot turn off is not a default" -- this repo's rule."""
|
||||
root = _covered_bundle(tmp_path / "bundle")
|
||||
with_rule = okf_consume.build_payload(root, question=QUESTION)
|
||||
without = okf_consume.build_payload(root, question=QUESTION, title_covered=False)
|
||||
assert with_rule != without
|
||||
arguments = okf_consume.parse_args([str(root), "--question", QUESTION, "--no-title-covered"])
|
||||
assert arguments.title_covered is False
|
||||
assert okf_consume.parse_args([str(root), "--question", QUESTION]).title_covered is True
|
||||
Loading…
Add table
Add a link
Reference in a new issue