feat(consume): rank documents from the indexes with stem matching

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

tests/test_okf_consume.py

@@ -25,6 +25,7 @@ import hashlib
 import json
 import os
 import sys
+import unicodedata
 from pathlib import Path
 
 import pytest
@@ -309,3 +310,73 @@ def test_the_default_limit_admits_a_concept_the_size_of_the_price_form() -> None
 def test_the_budget_unit_and_instrument_are_named_rather_than_implied() -> None:
     assert "byte" in okf_consume.BUDGET_UNIT
     assert "ensure_ascii=False" in okf_consume.BUDGET_INSTRUMENT
+
+
+# --- Step 5: stage-one document ranking --------------------------------------
+
+
+def test_normalise_is_nfc_stable_on_the_one_letter_that_decomposes() -> None:
+    # `NFD("å")` is `a` + U+030A, and the combining ring is not `\w`, so an
+    # un-normalised split returns `["a", "rlig"]`. `æ` and `ø` have NO canonical
+    # decomposition, so a test built on `miljø` passes while the bug is live --
+    # the known-positive here MUST use `å`.
+    composed = unicodedata.normalize("NFC", "årlig kontroll")
+    decomposed = unicodedata.normalize("NFD", "årlig kontroll")
+    assert composed != decomposed, "the control is broken: the two forms are identical"
+    assert okf_consume.normalise(decomposed) == okf_consume.normalise(composed)
+    assert "årlig" in okf_consume.normalise(decomposed)
+
+
+def test_normalise_drops_tokens_under_three_characters() -> None:
+    assert okf_consume.normalise("er en pris i et skjema") == ("pris", "skjema")
+
+
+def test_two_tokens_match_on_a_shared_prefix_of_four_and_not_of_three() -> None:
+    # "Stem-substring" is not an implementable rule: neither `prisene` nor
+    # `prissammenstilling` contains the other. Shared prefix does the work --
+    # `pris|ene` and `pris|sammenstilling` share 4. A 3-character floor
+    # over-matches Norwegian function words.
+    assert okf_consume.tokens_match("prisene", "prissammenstilling")
+    assert okf_consume.tokens_match("kontrollen", "kontroll")
+    assert not okf_consume.tokens_match("pris", "pri")
+    assert not okf_consume.tokens_match("krav", "kraft")
+
+
+def test_a_question_naming_a_directorys_subject_ranks_that_directory_first() -> None:
+    scores = okf_consume.document_scores(FIXTURE, "Hvordan skal prisene fylles ut?")
+    assert scores, "no document scored, so 'ranks first' would measure nothing"
+    assert max(scores, key=lambda key: (scores[key], key)) == "krav"
+
+
+def test_a_question_about_a_different_subject_ranks_a_different_directory() -> None:
+    # The control on the test above: without it, a scorer returning "krav"
+    # unconditionally would pass.
+    scores = okf_consume.document_scores(FIXTURE, "Hva er omfanget og formaalet?")
+    assert max(scores, key=lambda key: (scores[key], key)) == "scope"
+
+
+def test_curated_prose_in_an_index_is_ignored_rather_than_scored(tmp_path: Path) -> None:
+    root = tmp_path / "bundle"
+    _copy_bundle(FIXTURE, root)
+    index = root / "krav" / "index.md"
+    index.write_text(
+        "Denne mappen handler om priser og prissammenstilling.\n\n"
+        + index.read_text(encoding="utf-8"),
+        encoding="utf-8",
+    )
+    assert (
+        okf_consume.DEFAULT_PROFILE.index.parse_entry(
+            "Denne mappen handler om priser og prissammenstilling."
+        )
+        is None
+    )
+    assert okf_consume.document_scores(root, "Hvordan skal prisene fylles ut?") == (
+        okf_consume.document_scores(FIXTURE, "Hvordan skal prisene fylles ut?")
+    )
+
+
+def test_document_scores_are_identical_across_two_calls() -> None:
+    question = "Hvordan skal prisene fylles ut?"
+    assert okf_consume.document_scores(FIXTURE, question) == okf_consume.document_scores(
+        FIXTURE, question
+    )

tools/okf_consume.py

@@ -33,8 +33,10 @@ from __future__ import annotations
 
 import hashlib
 import json
+import re
 import sys
-from collections.abc import Mapping
+import unicodedata
+from collections.abc import Mapping, Sequence
 from dataclasses import dataclass
 from pathlib import Path
 from typing import Literal
@@ -472,3 +474,105 @@ def known_positive() -> tuple[str, int, int]:
     """
     measured = measure(_KNOWN_POSITIVE_PATH.read_text(encoding="utf-8"))
     return KNOWN_POSITIVE_CASE, KNOWN_POSITIVE_EXPECTED, measured
+
+
+# --- Stage one: which documents are worth opening -----------------------------
+
+#: The shortest token this instrument scores. Two characters in Norwegian are
+#: almost always a function word (`og`, `er`, `en`, `av`, `de`), and a matcher
+#: that scores them ranks every document equally.
+MIN_TOKEN_LENGTH = 3
+
+#: How many leading characters two tokens must share to count as a match.
+#:
+#: THIS INSTRUMENT'S OWN CONSTANT, and a measurement rather than a preference.
+#: Token equality fails on Norwegian compounds: the question word `prisene`
+#: equals none of a price-form concept's `title`, `source_file` or path tokens.
+#: Plain substring containment does not save it either -- neither `prisene` nor
+#: `prissammenstilling` contains the other. A shared prefix does: `pris|ene` and
+#: `pris|sammenstilling` share 4.
+#:
+#: MEASURED HERE, 2026-09-07, over the 629-concept K2 corpus for the question
+#: token `prisene`: a 4-character floor matches **3** concepts -- over the
+#: concept id alone AND over title + `source_file` + id together, the same 3 --
+#: and the price-form gold is among them. The plan this implements recorded 6
+#: for the same measurement; 6 is not reproducible with this rule, and the
+#: number that is reproducible is the one carried here. A 3-character floor
+#: over-matches Norwegian function words.
+MIN_SHARED_PREFIX = 4
+
+_TOKEN_SPLIT_RE = re.compile(r"[^0-9a-zà-öø-ÿ]+")
+
+
+def normalise(text: str) -> tuple[str, ...]:
+    """Text as comparable tokens: NFC first, then casefold, then split.
+
+    NFC FIRST is load-bearing and not tidiness. macOS hands filenames over
+    decomposed, so `å` arrives as `a` + U+030A; the combining ring is not a word
+    character, so an un-normalised split turns `årlig` into `a` and `rlig` and
+    the term is silently lost. `æ` and `ø` have no canonical decomposition, so a
+    test built on either passes while the bug is live -- which is why the
+    known-positive for this function uses `å`.
+    """
+    folded = unicodedata.normalize("NFC", text).casefold()
+    return tuple(token for token in _TOKEN_SPLIT_RE.split(folded) if len(token) >= MIN_TOKEN_LENGTH)
+
+
+def tokens_match(left: str, right: str) -> bool:
+    """Whether two tokens share a leading prefix of at least `MIN_SHARED_PREFIX`.
+
+    Symmetric, and it degrades to equality for short tokens: two 4-character
+    tokens match only if they are the same word.
+    """
+    limit = min(len(left), len(right))
+    if limit < MIN_SHARED_PREFIX:
+        return False
+    shared = 0
+    while shared < limit and left[shared] == right[shared]:
+        shared += 1
+    return shared >= MIN_SHARED_PREFIX
+
+
+def _overlap(question_tokens: Sequence[str], candidate: str) -> int:
+    """How many of the question's tokens the candidate text answers to."""
+    candidate_tokens = normalise(candidate)
+    return sum(
+        1
+        for token in question_tokens
+        if any(tokens_match(token, other) for other in candidate_tokens)
+    )
+
+
+def document_scores(
+    bundle_root: Path, question: str, *, profile: BundleProfile = DEFAULT_PROFILE
+) -> dict[str, float]:
+    """One score per top-level document, from the indexes and the path alone.
+
+    A "document" is a top-level entry: a directory, or -- per the corpus's own
+    shape -- a concept sitting at the root, which has no directory to inherit
+    from and is therefore its own document. Measured on K2, 11 of 629 concepts
+    are root-level, and they are exactly the 11 carrying neither `adjudication`
+    nor `bundle_id`; scoring them as members of some parent would put one bug in
+    three places.
+
+    Reads the INDEX TREE only. No directory is enumerated here or anywhere else
+    in this command (SS 9.2).
+    """
+    question_tokens = normalise(question)
+    indexes, concepts = _walk_index_tree(bundle_root, profile=profile)
+    scores: dict[str, float] = {}
+    for concept_id in concepts:
+        document = concept_id.split("/", 1)[0]
+        scores.setdefault(document, 0.0)
+        scores[document] += float(_overlap(question_tokens, concept_id.replace("/", " ")))
+    for relative in indexes:
+        document = relative.split("/", 1)[0]
+        if document == profile.index.name:
+            continue
+        scores.setdefault(document, 0.0)
+        for line in (bundle_root / relative).read_text(encoding="utf-8").splitlines():
+            entry = profile.index.parse_entry(line)
+            if entry is None:
+                continue
+            scores[document] += float(_overlap(question_tokens, entry.label))
+    return scores