feat(consume): weight a lexical hit by its rarity, off by default and measured

O2b asked whether a requirement number can be made worth more than a common
word by weighting each hit with the token's rarity in the bundle, with no
hand-set constant and no declared token class. It can, on one of the three
bundles, and the two it cannot are decomposed rather than guessed.

The rule is log(N/df) over the concepts' own tokens, counted with the same
four-character prefix rule a hit is scored with. It enters the RANKING and
never the GATE: `lexical` stays a count, because `krav` weighs exactly 0 on
all three bundles and a weighted gate would drop every concept matching only
that word -- which is the gate 54a0bc2 falsified for other reasons. One df
table per bundle reaches every stage that scores the question against text,
including the document prior. One pass, 0.241 s over 1 133 concepts.

Measured on four corpora, before and after, with every published figure
reproduced first: gold fused rank 96 -> 103, 9 -> 8 (withheld -> DELIVERED at
rank 8) and 35 -> 35; K2's priced sheet candidate rank 10 -> 2 with the cost
vocabulary and 251 -> 78 without; Q-good unmoved at rank 1; hit@8 5 of 6 with
every rank identical; the S7 control payload byte-identical on the default
command.

DEFAULT OFF, decided by the number and not by taste: it does not win on all
four, because N100's gold loses seven rank positions. Off means the bytes that
were already published, and that is measured -- 8 of 8 payload digests
identical against a frozen copy of 56c1205 built with git archive.

Two limits, both someone else's mechanism and both named: MIN_SHARED_PREFIX=4
makes a unique identifier read as 135-of-446 common on N100, so the weight
correctly ranks a common adjective above the exact requirement number; and RRF
consumes RANKS only, so on N500 -- where the gold already leads the one signal
that can see the identifier, and the other two cannot see it at all -- no
weighting inside a signal can move anything.

Consumption-side only, so no rebuild: the K2 bundle ref 2f82fcfe... stands.

Report: docs/2026-09-08-sjeldenhetsvekt.md. 13 new tests, red first; 8
mutations, 8 red, two of them only after the survivors were read as code -- one
exposed a fixture that put the identifier where the real corpus does not, and
the corrected fixture is what found the RRF limit. Suite 1295 -> 1308.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

tests/test_okf_consume.py

@@ -1116,6 +1116,7 @@ def test_no_corpus_document_name_reaches_any_file_this_work_tracks() -> None:
         PROJECT_ROOT / "docs" / "2026-09-08-blindsone-laas2-budsjett-k2.md",
         PROJECT_ROOT / "docs" / "2026-09-08-prisform-og-loggen-k2.md",
         PROJECT_ROOT / "docs" / "2026-09-08-kravnummer-tokenisering.md",
+        PROJECT_ROOT / "docs" / "2026-09-08-sjeldenhetsvekt.md",
         PROJECT_ROOT / "README.md",
         PROJECT_ROOT / "CLAUDE.md",
     ]
@@ -1589,3 +1590,306 @@ def test_the_cli_exposes_the_reservation_and_omitting_it_reproduces_the_default_
     assert plain.stdout == okf_consume.serialise(_eviction_payload(root))
     assert reserved.stdout == okf_consume.serialise(_eviction_payload(root, reserve_top_rank=True))
     assert plain.stdout != reserved.stdout
+
+
+# --- Step 12: the rarity weight (O2b) -----------------------------------------
+
+#: The situation the previous session measured and could not close, recreated
+#: small: every concept carries the common word, exactly one carries the
+#: identifier, and the identifier is worth the same as the common word because
+#: `_overlap` counts. Synthetic rather than borrowed from a bundle, because a
+#: fixture that is a corpus measures that corpus.
+RARITY_QUESTION = "Hva krever Krav 10.2-2 i N500? Gjengi det sentrale vilkåret."
+RARITY_GOLD = "krav/c-29"
+
+
+def _synthetic(concept_id: str, title: str, body: str) -> okf_consume.Concept:
+    return okf_consume.Concept(
+        path=Path(concept_id),
+        concept_id=concept_id,
+        bundle_id="rarity-fixture",
+        bundle_id_inherited=False,
+        sha256="0" * 64,
+        okf_type="Krav",
+        title=title,
+        source_file="synthetic.md",
+        adjudication="unknown",
+        adjudication_present=False,
+        frontmatter={},
+        body=body,
+    )
+
+
+def _rarity_corpus(*, identifier_in_body: bool) -> list[okf_consume.Concept]:
+    """29 concepts bearing the common words, one bearing the identifier.
+
+    The gold's `concept_id` sorts LAST, so nothing but the score can lift it:
+    on a tie the declared tie-break puts it at the bottom.
+
+    `identifier_in_body` is the whole difference between the two shapes the
+    real corpora turned out to have, and it decides whether the weight can do
+    anything at all -- see the two tests below.
+    """
+    common = "Kravet i N500 gjengir det sentrale vilkåret for anlegget."
+    concepts = [
+        _synthetic(f"krav/c-{index:02d}", f"Krav 3.{index}-1 Alminnelig bestemmelse", common)
+        for index in range(29)
+    ]
+    body = (
+        "Krav 10.2-2 stiller vilkår om anlegget."
+        if identifier_in_body
+        else "Tekniske bygg stiller vilkår om anlegget."
+    )
+    concepts.append(_synthetic(RARITY_GOLD, "Krav 10.2-2 Tekniske bygg", body))
+    return concepts
+
+
+def _rank_of(ranked: list[tuple[okf_consume.Concept, float, int]], concept_id: str) -> int:
+    return [concept.concept_id for concept, _, _ in ranked].index(concept_id) + 1
+
+
+def _signal_rank(
+    concepts: list[okf_consume.Concept], question: str, gold: str, weights: object
+) -> int:
+    """Where the gold sits on the title-and-id signal alone, by the ranker's own rule."""
+    tokens = okf_consume.normalise(question)
+    scores = {
+        concept.concept_id: okf_consume._overlap(
+            tokens,
+            f"{concept.title} {concept.concept_id.replace('/', ' ')}",
+            weights=weights,  # type: ignore[arg-type]
+        )
+        for concept in concepts
+    }
+    order = sorted(scores, key=lambda key: (-scores[key], key))
+    return order.index(gold) + 1
+
+
+def test_counting_leaves_the_identifier_worth_no_more_than_the_common_word() -> None:
+    # The known-positive for the weight: without it, a rank of 1 with the
+    # weight on would prove nothing, because nothing would have been shown to
+    # hold the gold down in the first place.
+    for identifier_in_body in (False, True):
+        concepts = _rarity_corpus(identifier_in_body=identifier_in_body)
+        ranked = okf_consume.concept_scores(concepts, RARITY_QUESTION, {})
+        assert _rank_of(ranked, RARITY_GOLD) == 18
+        lexical = {concept.concept_id: value for concept, _, value in ranked}
+        assert lexical["krav/c-00"] == 6
+
+
+def test_weighting_a_hit_by_its_rarity_lifts_the_identifier_into_the_cut() -> None:
+    # The arm where the weight can act: the identifier is in the body too, so
+    # weighting REORDERS the body signal. This is the shape N200 and the K2
+    # price sheet have, and both moved (withheld -> delivered rank 8; candidate
+    # rank 10 -> 2).
+    concepts = _rarity_corpus(identifier_in_body=True)
+    weights = okf_consume.rarity_weights(
+        okf_consume.normalise(RARITY_QUESTION), okf_consume.searchable_text(concepts)
+    )
+    ranked = okf_consume.concept_scores(concepts, RARITY_QUESTION, {}, weights=weights)
+    assert _rank_of(ranked, RARITY_GOLD) <= 8
+
+
+def test_a_rarity_weight_cannot_move_a_signal_the_gold_already_leads() -> None:
+    """The falsification, kept as a test so it cannot quietly stop being true.
+
+    RRF consumes RANKS ONLY -- that is the documented reason it was chosen. So
+    a weight changes the fused order only where it changes some signal's ORDER.
+    When the identifier is in the title alone, the gold already leads the title
+    signal by counting, the weight makes its lead larger and its RANK identical,
+    and the two signals that hold it down cannot see the identifier at all.
+    Measured on the real bundles: N500 35 -> 35, N100 96 -> 103.
+    """
+    concepts = _rarity_corpus(identifier_in_body=False)
+    weights = okf_consume.rarity_weights(
+        okf_consume.normalise(RARITY_QUESTION), okf_consume.searchable_text(concepts)
+    )
+    # The weight DOES do its job on the score: the gold's title is worth more
+    # than every other title, by more than counting made it worth.
+    plain = okf_consume._overlap(
+        okf_consume.normalise(RARITY_QUESTION), f"{concepts[-1].title} krav c-29"
+    )
+    lifted = okf_consume._overlap(
+        okf_consume.normalise(RARITY_QUESTION), f"{concepts[-1].title} krav c-29", weights=weights
+    )
+    assert lifted > plain
+    # And the rank it buys is the rank it already had, so the fusion sees none of it.
+    assert _signal_rank(concepts, RARITY_QUESTION, RARITY_GOLD, None) == 1
+    assert _signal_rank(concepts, RARITY_QUESTION, RARITY_GOLD, weights) == 1
+    ranked = okf_consume.concept_scores(concepts, RARITY_QUESTION, {}, weights=weights)
+    assert _rank_of(ranked, RARITY_GOLD) == 18
+
+
+def test_the_weight_leaves_the_lexical_count_a_count_so_the_cut_is_untouched() -> None:
+    # SS: the gate is a different axis from the ranking, and a token every
+    # concept carries weighs exactly zero. Were `lexical` the weighted sum, a
+    # concept matching only that token would become `no_lexical_match` -- the
+    # gate `54a0bc2` falsified rarity weighting FOR.
+    concepts = _rarity_corpus(identifier_in_body=True)
+    weights = okf_consume.rarity_weights(
+        okf_consume.normalise(RARITY_QUESTION), okf_consume.searchable_text(concepts)
+    )
+    plain = {
+        c.concept_id: v for c, _, v in okf_consume.concept_scores(concepts, RARITY_QUESTION, {})
+    }
+    weighted = {
+        c.concept_id: v
+        for c, _, v in okf_consume.concept_scores(concepts, RARITY_QUESTION, {}, weights=weights)
+    }
+    assert plain == weighted
+
+
+def _equal_frequency_corpus() -> list[okf_consume.Concept]:
+    """16 concepts over three words, each word borne by exactly 8 of them.
+
+    Equal `df` is the condition under which the weight provably cannot reorder
+    anything: every hit is multiplied by the same positive number, and RRF
+    consumes ranks. Concepts carry one word or all three, so the order being
+    preserved is a real order and not a single tie.
+    """
+    words = ("vilkåret", "anlegget", "kravet")
+    bodies = [" ".join(words)] * 4 + [words[0]] * 4 + [words[1]] * 4 + [words[2]] * 4
+    return [
+        _synthetic(f"felles/e-{index:02d}", f"Bestemmelse {index}", f"Denne teksten nevner {body}.")
+        for index, body in enumerate(bodies)
+    ]
+
+
+def test_a_question_of_equally_common_words_comes_back_in_the_order_it_had() -> None:
+    concepts = _equal_frequency_corpus()
+    question = "Hva sier vilkåret om anlegget og kravet?"
+    texts = okf_consume.searchable_text(concepts)
+    weights = okf_consume.rarity_weights(okf_consume.normalise(question), texts)
+    # The three borne words weigh the same; `hva` and `sier` are borne by no
+    # concept at all, and their weight is never consumed because they never hit.
+    assert len({round(weights[word], 12) for word in ("vilkåret", "anlegget", "kravet")}) == 1
+    plain = [c.concept_id for c, _, _ in okf_consume.concept_scores(concepts, question, {})]
+    weighted = [
+        c.concept_id
+        for c, _, _ in okf_consume.concept_scores(concepts, question, {}, weights=weights)
+    ]
+    assert weighted == plain
+    assert len(set(plain)) == 16
+
+
+def test_a_word_every_concept_carries_weighs_exactly_nothing() -> None:
+    concepts = _rarity_corpus(identifier_in_body=True)
+    weights = okf_consume.rarity_weights(
+        okf_consume.normalise(RARITY_QUESTION), okf_consume.searchable_text(concepts)
+    )
+    assert weights["krav"] == 0.0
+    assert weights["10.2-2"] > weights["n500"] > 0.0
+
+
+def test_a_question_token_no_concept_carries_is_weighted_but_never_consumed() -> None:
+    concepts = _rarity_corpus(identifier_in_body=True)
+    weights = okf_consume.rarity_weights(("kabelbroer",), okf_consume.searchable_text(concepts))
+    assert weights["kabelbroer"] > 0.0
+    assert okf_consume._overlap(("kabelbroer",), concepts[0].body, weights=weights) == 0.0
+
+
+def test_the_weight_is_computed_from_the_bundle_and_not_from_a_constant() -> None:
+    # Same question, two corpora, two different weights for the same token:
+    # the number comes from the bundle in hand or it comes from nowhere.
+    small = okf_consume.rarity_weights(("n500",), ["N500 gjelder", "noe helt annet"])
+    large = okf_consume.rarity_weights(("n500",), ["N500 gjelder"] + ["noe helt annet"] * 9)
+    assert large["n500"] > small["n500"]
+
+
+def test_the_weight_is_off_by_default_and_the_default_payload_is_unmoved() -> None:
+    # The decision recorded as a test rather than as a sentence: measured on
+    # four corpora the weight delivers one gold and costs another seven rank
+    # positions, so it ships OFF, and OFF has to mean the bytes that were
+    # already published.
+    plain = okf_consume.build_payload(FIXTURE, question="Hvordan skal prisene fylles ut?")
+    explicit = okf_consume.build_payload(
+        FIXTURE, question="Hvordan skal prisene fylles ut?", rarity_weight=False
+    )
+    assert okf_consume.serialise(plain) == okf_consume.serialise(explicit)
+
+
+def test_the_cli_exposes_the_weight_and_omitting_it_reproduces_the_default_bytes() -> None:
+    question = "Hva krever Krav 10.2-2 om sentrale vilkår?"
+    plain = _run(str(FIXTURE), "--question", question)
+    weighted = _run(str(FIXTURE), "--question", question, "--rarity-weight")
+    assert plain.returncode == 0, plain.stderr
+    assert weighted.returncode == 0, weighted.stderr
+    assert plain.stdout == okf_consume.serialise(
+        okf_consume.build_payload(FIXTURE, question=question)
+    )
+    assert weighted.stdout == okf_consume.serialise(
+        okf_consume.build_payload(FIXTURE, question=question, rarity_weight=True)
+    )
+
+
+def test_the_weight_reaches_the_document_prior_and_not_only_the_concept_signals() -> None:
+    # One df table for the bundle, used wherever the question is scored against
+    # its text. A weight applied to two of three fused signals would be a
+    # second ranker rather than one statistic.
+    question = "Hvordan skal prisene fylles ut?"
+    weights = okf_consume.rarity_weights(
+        okf_consume.normalise(question), okf_consume.searchable_text(_fixture_concepts())
+    )
+    plain = okf_consume.document_scores(FIXTURE, question)
+    weighted = okf_consume.document_scores(FIXTURE, question, weights=weights)
+    assert plain.keys() == weighted.keys()
+    assert plain != weighted
+
+
+def test_build_payload_hands_the_same_weights_to_the_document_prior(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    """ONE df table for the bundle, reaching every stage that scores text.
+
+    Asserted on the call rather than on an output, and that is the point: on
+    all three real bundles the document prior is a two-document density whose
+    ORDER the weight does not change, so a version passing the weights to the
+    concept signals alone ranks identically there -- measured, 103/8/35 either
+    way. The commitment is still that one statistic reaches every stage, and a
+    commitment no output can distinguish has to be checked where it is made.
+    """
+    seen: list[object] = []
+    original = okf_consume.document_scores
+
+    def spy(*args: object, **kwargs: object) -> dict[str, float]:
+        seen.append(kwargs.get("weights"))
+        return original(*args, **kwargs)  # type: ignore[arg-type]
+
+    monkeypatch.setattr(okf_consume, "document_scores", spy)
+    question = "Hvordan skal prisene fylles ut?"
+    okf_consume.build_payload(FIXTURE, question=question)
+    okf_consume.build_payload(FIXTURE, question=question, rarity_weight=True)
+    assert seen[0] is None
+    expected = okf_consume.rarity_weights(
+        okf_consume.normalise(question), okf_consume.searchable_text(_fixture_concepts())
+    )
+    assert seen[1] == expected
+
+
+def test_the_weight_reaches_the_title_signal_and_reorders_it() -> None:
+    # The gold's title answers TWO question tokens and every other title
+    # answers THREE, so counting puts the gold behind all of them; only the
+    # rarity of the identifier can turn that around, and only if the weight
+    # reaches the title-and-id signal. Bodies are identical, so the body signal
+    # decides nothing. This is the N200 shape, where the gold's title-signal
+    # rank moved 8 -> 4 and the gold went from withheld to delivered.
+    body = "Denne bestemmelsen gjelder for anlegget."
+    concepts = [
+        _synthetic(f"krav/t-{index:02d}", f"Krav 3.{index}-1 N500 gjengir bestemmelsen", body)
+        for index in range(29)
+    ]
+    concepts.append(_synthetic("krav/t-29", "Krav 10.2-2 Tekniske bygg", body))
+    question = "Hva krever Krav 10.2-2 i N500? Gjengi det sentrale vilkåret."
+    weights = okf_consume.rarity_weights(
+        okf_consume.normalise(question), okf_consume.searchable_text(concepts)
+    )
+    assert _signal_rank(concepts, question, "krav/t-29", None) == 30
+    assert _signal_rank(concepts, question, "krav/t-29", weights) == 1
+    plain = okf_consume.concept_scores(concepts, question, {})
+    weighted = okf_consume.concept_scores(concepts, question, {}, weights=weights)
+    # Fused: 30 -> 18. The title signal is reordered from last to first and the
+    # fused rank moves by twelve, not to one -- the other two signals still
+    # cannot see the identifier. That gap IS the finding of this session, and
+    # the numbers are here so a change to either half shows up as a diff.
+    assert _rank_of(plain, "krav/t-29") == 30
+    assert _rank_of(weighted, "krav/t-29") == 18