llm-ingestion-okf/tests/test_title_covered.py

"""A section whose WHOLE title the question accounts for, read before its
narrower neighbour.

THE DEFECT, in the mechanism rather than in a corpus. Both lexical signals in
`concept_scores` are unnormalised COVERAGE COUNTS -- one per question token the
candidate answers to. Nothing in the fusion measures how much of the CANDIDATE
the question accounts for, so a section titled with the question's subject and
nothing else scores exactly what a narrower section titled with that subject
PLUS a qualifier scores, and then loses to it on the body count, because a
longer or differently-worded body reaches one more of the question's tokens.

The fixture below is that shape and nothing else: one source document, one
broad section whose title is the question's subject, one narrower section whose
title is that subject plus a qualifier the question never uses, and fillers so
no signal is degenerate for want of members. It carries no sentence from any
corpus and names no real document.

WHY A PARTITION AND NOT A FOURTH SIGNAL. RRF consumes RANKS ONLY, so any one
signal contributes at most `1/(RRF_K + 1)`, and with shared ranks a signal
whose positive group is SMALL separates least of all: the group's members take
position 1 and everyone else position `len(group) + 1`, so a rule that fires on
two concepts of 2 761 is worth `1/61 - 1/63` to them. Precision is exactly such
a rule. `lookup_hits` above it is the same shape and was made a partition for
the same measured reason.
"""

from __future__ import annotations

import sys
from pathlib import Path

PROJECT_ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(PROJECT_ROOT / "tools"))

import okf_consume  # noqa: E402

QUESTION = "Which requirements apply to anchoring on a bridge?"

_FRONTMATTER = (
    "---\ntype: reference\ntitle: {title}\nsource_file: {slug}.md\n"
    "source_sha256: {digest}\ningested_at: 2026-09-01T00:00:00Z\n"
    "adjudication: proposed\nbundle_id: covered-fixture\n"
    "verified: [{{ by: process:okf-check, at: 2026-09-01T00:00:00Z }}]\n---\n\n"
)


def _covered_bundle(root: Path, *, fillers: int = 12) -> Path:
    """One document, so the document prior separates nothing at all.

    `aa-broad` is the section the question is about and its title is the
    question's subject alone. `bb-narrow` is a narrower section whose title
    adds a qualifier the question never uses, and whose body reaches ONE more
    question token, which is the whole of its present advantage. `cc-prefix` is
    the known-negative for the matcher: its title shares four leading
    characters with the subject and is not that subject.
    """
    (root / "part").mkdir(parents=True)
    (root / "index.md").write_text(
        "---\nokf_version: 0.2\nbundle_id: covered-fixture\n---\n\n"
        "- [part (index)](part/index.md)\n",
        encoding="utf-8",
    )
    entries: list[str] = []

    def add(slug: str, title: str, body: str) -> None:
        entries.append(f"- [{title}]({slug}.md) \u2014 adjudication: proposed\n")
        (root / "part" / f"{slug}.md").write_text(
            _FRONTMATTER.format(title=title, slug=slug, digest="1" * 64) + f"## {title}\n\n" + body,
            encoding="utf-8",
        )

    add(
        "aa-broad",
        "Anchoring",
        "Requirements for anchoring are stated in this section.\n" * 3,
    )
    add(
        "bb-narrow",
        "Temporary anchoring",
        "Requirements for anchoring of a bridge are stated in this section.\n" * 3,
    )
    add(
        "cc-prefix",
        "Anchorage",
        "Requirements for anchorage of a bridge are stated in this section.\n" * 3,
    )
    for number in range(1, fillers + 1):
        add(
            f"dd-{number:02d}",
            f"Bridge works {number:02d}",
            "Requirements for the works on a bridge are stated in this section.\n" * 3,
        )
    (root / "part" / "index.md").write_text("".join(entries), encoding="utf-8")
    return root


def _ranking(root: Path, **kwargs: object) -> list[str]:
    concepts = [
        okf_consume.read_concept(
            root / f"{concept_id}.md", bundle_root=root, root_bundle_id="covered-fixture"
        )
        for concept_id in okf_consume.enumerate_concepts(root)
    ]
    ranked = okf_consume.concept_scores(
        concepts,
        QUESTION,
        okf_consume.document_scores(root, QUESTION),
        **kwargs,  # type: ignore[arg-type]
    )
    return [concept.concept_id for concept, _, _ in ranked]


def _position(order: list[str], slug: str) -> int:
    for index, concept_id in enumerate(order, start=1):
        if concept_id.endswith(slug):
            return index
    raise AssertionError(f"{slug} is not in the ranking at all")


def test_the_document_prior_separates_nothing_on_a_one_document_bundle(tmp_path: Path) -> None:
    # CHARACTERISATION of the ground the rule stands on: with one source
    # document every concept takes the same value from the third signal, so
    # a third of the fusion carries no information here.
    root = _covered_bundle(tmp_path / "bundle")
    prior = okf_consume.document_scores(root, QUESTION)
    assert len(set(prior.values())) == 1


def test_the_narrow_section_outranks_the_broad_one_without_the_rule(tmp_path: Path) -> None:
    # The defect itself, stated as a green characterisation so the fix below
    # has a measured before.
    root = _covered_bundle(tmp_path / "bundle")
    order = _ranking(root, title_covered=False)
    assert _position(order, "bb-narrow") < _position(order, "aa-broad")


def test_the_question_answering_the_whole_title_takes_the_broad_section_first(
    tmp_path: Path,
) -> None:
    root = _covered_bundle(tmp_path / "bundle")
    order = _ranking(root)
    assert _position(order, "aa-broad") == 1
    assert _position(order, "aa-broad") < _position(order, "bb-narrow")


def test_the_rule_reads_the_title_by_EQUALITY_and_not_by_shared_prefix(tmp_path: Path) -> None:
    """The known-negative for the matcher, measured before the rule was written.

    `anchorage` and `anchoring` share four leading characters, which is what
    `tokens_match` accepts. Under a prefix-matching form of this rule the
    fixture's `cc-prefix` would be floated beside the section the question
    actually names.
    """
    root = _covered_bundle(tmp_path / "bundle")
    hits = okf_consume.title_covered_hits(
        [
            okf_consume.read_concept(
                root / f"{concept_id}.md", bundle_root=root, root_bundle_id="covered-fixture"
            )
            for concept_id in okf_consume.enumerate_concepts(root)
        ],
        QUESTION,
    )
    assert [concept_id.rsplit("/", 1)[-1] for concept_id in hits] == ["aa-broad"]


def test_the_rule_has_an_explicit_opt_out_the_cli_exposes(tmp_path: Path) -> None:
    """ "A default a caller cannot turn off is not a default" -- this repo's rule."""
    root = _covered_bundle(tmp_path / "bundle")
    with_rule = okf_consume.build_payload(root, question=QUESTION)
    without = okf_consume.build_payload(root, question=QUESTION, title_covered=False)
    assert with_rule != without
    arguments = okf_consume.parse_args([str(root), "--question", QUESTION, "--no-title-covered"])
    assert arguments.title_covered is False
    assert okf_consume.parse_args([str(root), "--question", QUESTION]).title_covered is True


def test_a_question_that_NAMES_a_concept_still_reads_that_one_first(tmp_path: Path) -> None:
    """The partition lands BELOW `lookup_hits`, and that is asserted rather than
    remembered.

    A question carrying an identifier a concept holds verbatim is a lookup, not
    a search, and this rule must not out-rank it. Without the assertion the
    ordering of the two partitions is a property of the source's line order.
    """
    root = _covered_bundle(tmp_path / "bundle")
    concepts = [
        okf_consume.read_concept(
            root / f"{concept_id}.md", bundle_root=root, root_bundle_id="covered-fixture"
        )
        for concept_id in okf_consume.enumerate_concepts(root)
    ]
    question = "Which requirements apply to anchoring on a bridge under 4.2-1?"
    (root / "part" / "ee-named.md").write_text(
        _FRONTMATTER.format(
            title="Clause 4.2-1 on temporary works", slug="ee-named", digest="2" * 64
        )
        + "## Clause 4.2-1 on temporary works\n\nA clause about works.\n",
        encoding="utf-8",
    )
    named = okf_consume.read_concept(
        root / "part" / "ee-named.md", bundle_root=root, root_bundle_id="covered-fixture"
    )
    concepts.append(named)
    assert okf_consume.lookup_hits(concepts, question) == (named.concept_id,)
    assert okf_consume.title_covered_hits(concepts, question)
    # THE CONTROL, so this is not green for want of a competitor: with the
    # lookup partition off, the covered concept IS first, so the assertion
    # below measures which partition wins rather than that only one fires.
    without_lookup = okf_consume.concept_scores(
        concepts, question, okf_consume.document_scores(root, question), lookup=False
    )
    assert without_lookup[0][0].concept_id.endswith("aa-broad")
    ranked = okf_consume.concept_scores(
        concepts, question, okf_consume.document_scores(root, question)
    )
    assert ranked[0][0].concept_id == named.concept_id


def test_the_payload_signature_defaults_are_the_consume_command_defaults() -> None:
    """O6's defect, in the other command.

    `cli.build`'s signature defaulted two flags to `False` while argparse
    defaulted both to `True`, and a caller reaching `build()` as a function read
    the signature -- so two build paths emitted different bundles and the test
    that held them equal compared the wrong side against itself. `build_payload`
    has the same two entry points, so it gets the same check: every
    same-named parameter must carry the same default on both sides.
    """
    import inspect

    signature = inspect.signature(okf_consume.build_payload).parameters
    parsed = okf_consume.parse_args(["bundle", "--question", "q"])
    shared = {name for name in signature if hasattr(parsed, name)}
    assert "title_covered" in shared, "the new parameter is not reachable from the CLI"
    mismatched = {
        name: (signature[name].default, getattr(parsed, name))
        for name in sorted(shared)
        if signature[name].default is not inspect.Parameter.empty
        and signature[name].default != getattr(parsed, name)
    }
    assert mismatched == {}, f"signature and argparse disagree: {mismatched}"