feat(consume): rank documents from the indexes with stem matching

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

tests/test_okf_consume.py

@@ -25,6 +25,7 @@ import hashlib
 import json
 import os
 import sys
+import unicodedata
 from pathlib import Path
 
 import pytest
@@ -309,3 +310,73 @@ def test_the_default_limit_admits_a_concept_the_size_of_the_price_form() -> None
 def test_the_budget_unit_and_instrument_are_named_rather_than_implied() -> None:
     assert "byte" in okf_consume.BUDGET_UNIT
     assert "ensure_ascii=False" in okf_consume.BUDGET_INSTRUMENT
+
+
+# --- Step 5: stage-one document ranking --------------------------------------
+
+
+def test_normalise_is_nfc_stable_on_the_one_letter_that_decomposes() -> None:
+    # `NFD("å")` is `a` + U+030A, and the combining ring is not `\w`, so an
+    # un-normalised split returns `["a", "rlig"]`. `æ` and `ø` have NO canonical
+    # decomposition, so a test built on `miljø` passes while the bug is live --
+    # the known-positive here MUST use `å`.
+    composed = unicodedata.normalize("NFC", "årlig kontroll")
+    decomposed = unicodedata.normalize("NFD", "årlig kontroll")
+    assert composed != decomposed, "the control is broken: the two forms are identical"
+    assert okf_consume.normalise(decomposed) == okf_consume.normalise(composed)
+    assert "årlig" in okf_consume.normalise(decomposed)
+
+
+def test_normalise_drops_tokens_under_three_characters() -> None:
+    assert okf_consume.normalise("er en pris i et skjema") == ("pris", "skjema")
+
+
+def test_two_tokens_match_on_a_shared_prefix_of_four_and_not_of_three() -> None:
+    # "Stem-substring" is not an implementable rule: neither `prisene` nor
+    # `prissammenstilling` contains the other. Shared prefix does the work --
+    # `pris|ene` and `pris|sammenstilling` share 4. A 3-character floor
+    # over-matches Norwegian function words.
+    assert okf_consume.tokens_match("prisene", "prissammenstilling")
+    assert okf_consume.tokens_match("kontrollen", "kontroll")
+    assert not okf_consume.tokens_match("pris", "pri")
+    assert not okf_consume.tokens_match("krav", "kraft")
+
+
+def test_a_question_naming_a_directorys_subject_ranks_that_directory_first() -> None:
+    scores = okf_consume.document_scores(FIXTURE, "Hvordan skal prisene fylles ut?")
+    assert scores, "no document scored, so 'ranks first' would measure nothing"
+    assert max(scores, key=lambda key: (scores[key], key)) == "krav"
+
+
+def test_a_question_about_a_different_subject_ranks_a_different_directory() -> None:
+    # The control on the test above: without it, a scorer returning "krav"
+    # unconditionally would pass.
+    scores = okf_consume.document_scores(FIXTURE, "Hva er omfanget og formaalet?")
+    assert max(scores, key=lambda key: (scores[key], key)) == "scope"
+
+
+def test_curated_prose_in_an_index_is_ignored_rather_than_scored(tmp_path: Path) -> None:
+    root = tmp_path / "bundle"
+    _copy_bundle(FIXTURE, root)
+    index = root / "krav" / "index.md"
+    index.write_text(
+        "Denne mappen handler om priser og prissammenstilling.\n\n"
+        + index.read_text(encoding="utf-8"),
+        encoding="utf-8",
+    )
+    assert (
+        okf_consume.DEFAULT_PROFILE.index.parse_entry(
+            "Denne mappen handler om priser og prissammenstilling."
+        )
+        is None
+    )
+    assert okf_consume.document_scores(root, "Hvordan skal prisene fylles ut?") == (
+        okf_consume.document_scores(FIXTURE, "Hvordan skal prisene fylles ut?")
+    )
+
+
+def test_document_scores_are_identical_across_two_calls() -> None:
+    question = "Hvordan skal prisene fylles ut?"
+    assert okf_consume.document_scores(FIXTURE, question) == okf_consume.document_scores(
+        FIXTURE, question
+    )