feat(consume): rank documents from the indexes with stem matching

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

tools/okf_consume.py

@@ -33,8 +33,10 @@ from __future__ import annotations
 
 import hashlib
 import json
+import re
 import sys
-from collections.abc import Mapping
+import unicodedata
+from collections.abc import Mapping, Sequence
 from dataclasses import dataclass
 from pathlib import Path
 from typing import Literal
@@ -472,3 +474,105 @@ def known_positive() -> tuple[str, int, int]:
     """
     measured = measure(_KNOWN_POSITIVE_PATH.read_text(encoding="utf-8"))
     return KNOWN_POSITIVE_CASE, KNOWN_POSITIVE_EXPECTED, measured
+
+
+# --- Stage one: which documents are worth opening -----------------------------
+
+#: The shortest token this instrument scores. Two characters in Norwegian are
+#: almost always a function word (`og`, `er`, `en`, `av`, `de`), and a matcher
+#: that scores them ranks every document equally.
+MIN_TOKEN_LENGTH = 3
+
+#: How many leading characters two tokens must share to count as a match.
+#:
+#: THIS INSTRUMENT'S OWN CONSTANT, and a measurement rather than a preference.
+#: Token equality fails on Norwegian compounds: the question word `prisene`
+#: equals none of a price-form concept's `title`, `source_file` or path tokens.
+#: Plain substring containment does not save it either -- neither `prisene` nor
+#: `prissammenstilling` contains the other. A shared prefix does: `pris|ene` and
+#: `pris|sammenstilling` share 4.
+#:
+#: MEASURED HERE, 2026-09-07, over the 629-concept K2 corpus for the question
+#: token `prisene`: a 4-character floor matches **3** concepts -- over the
+#: concept id alone AND over title + `source_file` + id together, the same 3 --
+#: and the price-form gold is among them. The plan this implements recorded 6
+#: for the same measurement; 6 is not reproducible with this rule, and the
+#: number that is reproducible is the one carried here. A 3-character floor
+#: over-matches Norwegian function words.
+MIN_SHARED_PREFIX = 4
+
+_TOKEN_SPLIT_RE = re.compile(r"[^0-9a-zà-öø-ÿ]+")
+
+
+def normalise(text: str) -> tuple[str, ...]:
+    """Text as comparable tokens: NFC first, then casefold, then split.
+
+    NFC FIRST is load-bearing and not tidiness. macOS hands filenames over
+    decomposed, so `å` arrives as `a` + U+030A; the combining ring is not a word
+    character, so an un-normalised split turns `årlig` into `a` and `rlig` and
+    the term is silently lost. `æ` and `ø` have no canonical decomposition, so a
+    test built on either passes while the bug is live -- which is why the
+    known-positive for this function uses `å`.
+    """
+    folded = unicodedata.normalize("NFC", text).casefold()
+    return tuple(token for token in _TOKEN_SPLIT_RE.split(folded) if len(token) >= MIN_TOKEN_LENGTH)
+
+
+def tokens_match(left: str, right: str) -> bool:
+    """Whether two tokens share a leading prefix of at least `MIN_SHARED_PREFIX`.
+
+    Symmetric, and it degrades to equality for short tokens: two 4-character
+    tokens match only if they are the same word.
+    """
+    limit = min(len(left), len(right))
+    if limit < MIN_SHARED_PREFIX:
+        return False
+    shared = 0
+    while shared < limit and left[shared] == right[shared]:
+        shared += 1
+    return shared >= MIN_SHARED_PREFIX
+
+
+def _overlap(question_tokens: Sequence[str], candidate: str) -> int:
+    """How many of the question's tokens the candidate text answers to."""
+    candidate_tokens = normalise(candidate)
+    return sum(
+        1
+        for token in question_tokens
+        if any(tokens_match(token, other) for other in candidate_tokens)
+    )
+
+
+def document_scores(
+    bundle_root: Path, question: str, *, profile: BundleProfile = DEFAULT_PROFILE
+) -> dict[str, float]:
+    """One score per top-level document, from the indexes and the path alone.
+
+    A "document" is a top-level entry: a directory, or -- per the corpus's own
+    shape -- a concept sitting at the root, which has no directory to inherit
+    from and is therefore its own document. Measured on K2, 11 of 629 concepts
+    are root-level, and they are exactly the 11 carrying neither `adjudication`
+    nor `bundle_id`; scoring them as members of some parent would put one bug in
+    three places.
+
+    Reads the INDEX TREE only. No directory is enumerated here or anywhere else
+    in this command (SS 9.2).
+    """
+    question_tokens = normalise(question)
+    indexes, concepts = _walk_index_tree(bundle_root, profile=profile)
+    scores: dict[str, float] = {}
+    for concept_id in concepts:
+        document = concept_id.split("/", 1)[0]
+        scores.setdefault(document, 0.0)
+        scores[document] += float(_overlap(question_tokens, concept_id.replace("/", " ")))
+    for relative in indexes:
+        document = relative.split("/", 1)[0]
+        if document == profile.index.name:
+            continue
+        scores.setdefault(document, 0.0)
+        for line in (bundle_root / relative).read_text(encoding="utf-8").splitlines():
+            entry = profile.index.parse_entry(line)
+            if entry is None:
+                continue
+            scores[document] += float(_overlap(question_tokens, entry.label))
+    return scores