feat(s31): Retriever seam — StructuralRetriever default + HybridRanker with injected similarity

src/portfolio_optimiser/semretrieval.py

@@ -26,6 +26,7 @@ from __future__ import annotations
 
 import hashlib
 import os
+from collections.abc import Callable, Sequence
 from typing import TYPE_CHECKING, Protocol, runtime_checkable
 
 # Pin BLAS threads BEFORE numpy is imported — OpenBLAS reads these at import time, and a
@@ -39,7 +40,10 @@ os.environ.setdefault("MKL_NUM_THREADS", "1")
 import numpy as np  # noqa: E402  — must follow the thread pins above; see module docstring
 
 if TYPE_CHECKING:  # verdicts imports agent_framework — keep it out of the runtime import graph
-    from portfolio_optimiser.verdicts import ProposalFeatures
+    from portfolio_optimiser.verdicts import ProposalFeatures, Verdict
+
+    # The structural score, passed in rather than imported — see module docstring.
+    SimilarityFn = Callable[[ProposalFeatures, ProposalFeatures], float]
 
 # Dimension of the fake embedding. Small enough that a 500+ verdict base is a trivial matmul,
 # large enough that distinct features do not collide.
@@ -124,3 +128,62 @@ def cosine(a: np.ndarray, b: np.ndarray) -> float:
     if norm_a == 0.0 or norm_b == 0.0:
         return 0.0
     return float(np.dot(a, b) / (norm_a * norm_b))
+
+
+@runtime_checkable
+class Retriever(Protocol):
+    """Ranks candidate verdicts against a query and returns the top ``k``.
+
+    ``VerdictStore.retrieve`` delegates to one of these. The store's default is
+    ``StructuralRetriever``, which reproduces the pre-seam ordering exactly, so installing a
+    retriever is the ONLY way retrieval behaviour changes."""
+
+    def rank(
+        self, query: ProposalFeatures, candidates: Sequence[Verdict], k: int
+    ) -> list[Verdict]: ...
+
+
+class StructuralRetriever:
+    """Today's ranking, behind the seam: weighted structural similarity, ties broken by ``id``.
+
+    ``similarity`` is INJECTED rather than imported — importing it would pull ``verdicts`` (and
+    thus ``agent_framework``) into this module's runtime graph. This class is the store's wired
+    default, not a parallel implementation kept for reference."""
+
+    def __init__(self, similarity: SimilarityFn) -> None:
+        self._similarity = similarity
+
+    def rank(self, query: ProposalFeatures, candidates: Sequence[Verdict], k: int) -> list[Verdict]:
+        return sorted(
+            candidates,
+            key=lambda v: (-self._similarity(query, v.proposal_features), v.id),
+        )[:k]
+
+
+class HybridRanker:
+    """Blends semantic cosine with the structural score: ``w * cosine + (1 - w) * structural``.
+
+    Both terms live in ``[0, 1]`` (the embedder yields non-negative unit vectors), so ``weight``
+    means what it reads as. Ordering uses the TOTAL order ``(-round(score, 9), id)``: rounding
+    absorbs any last-bit float noise, and ``id`` makes the result independent of input order even
+    when two candidates score identically."""
+
+    def __init__(
+        self,
+        embedder: Embedder,
+        similarity: SimilarityFn,
+        weight: float = SEMANTIC_WEIGHT_DEFAULT,
+    ) -> None:
+        self._embedder = embedder
+        self._similarity = similarity
+        self._weight = weight
+
+    def rank(self, query: ProposalFeatures, candidates: Sequence[Verdict], k: int) -> list[Verdict]:
+        query_vector = self._embedder(query)
+
+        def score(verdict: Verdict) -> float:
+            semantic = cosine(query_vector, self._embedder(verdict.proposal_features))
+            structural = self._similarity(query, verdict.proposal_features)
+            return self._weight * semantic + (1.0 - self._weight) * structural
+
+        return sorted(candidates, key=lambda v: (-round(score(v), 9), v.id))[:k]

src/portfolio_optimiser/verdicts.py

@@ -28,7 +28,7 @@ from typing import Any
 
 from agent_framework import ContextProvider, SessionContext
 
-from portfolio_optimiser import okf
+from portfolio_optimiser import okf, semretrieval
 
 _log = logging.getLogger(__name__)
 
@@ -251,6 +251,11 @@ class VerdictStore:
 
     verdicts: list[Verdict]
 
+    # S3.1 opt-in seam: ``None`` means the structural default, which is byte-identical to the
+    # pre-seam inline sort. Only an explicit caller (``run.py --semantic-retrieval``) installs a
+    # different ranker, so default retrieval behaviour is unchanged.
+    retriever: semretrieval.Retriever | None = None
+
     @classmethod
     def from_dir(cls, directory: str) -> VerdictStore:
         """Build a store from an async inbox folder (convenience constructor). ``run_project`` uses
@@ -260,15 +265,20 @@ class VerdictStore:
         return cls(verdicts=load_verdicts_from_dir(directory))
 
     def retrieve(self, query: ProposalFeatures, k: int) -> list[Verdict]:
-        """Return the top-``k`` verdicts by structural similarity. Deterministic: ties break
+        """Return the top-``k`` verdicts. Deterministic: ties break by verdict id, so ordering
-        by verdict id, so ordering is stable across runs."""
+        is stable across runs.
+
+        Ranking is delegated to ``self.retriever``, defaulting to ``StructuralRetriever`` — the
+        same weighted structural score and ``(-similarity, id)`` key as before the seam existed.
+        A caller that installs a ``HybridRanker`` opts into an additional semantic term."""
         if k <= 0:
             raise ValueError(f"k must be positive, got {k}")
-        ranked = sorted(
+        ranker = (
-            self.verdicts,
+            self.retriever
-            key=lambda v: (-similarity(query, v.proposal_features), v.id),
+            if self.retriever is not None
+            else semretrieval.StructuralRetriever(similarity)
         )
-        return ranked[:k]
+        return ranker.rank(query, self.verdicts, k)
 
     def add(self, verdict: Verdict) -> None:
         """Persist a captured verdict in-memory.

tests/test_semretrieval.py

@@ -21,7 +21,19 @@ from pathlib import Path
 import numpy as np
 import pytest
 
-from portfolio_optimiser.semretrieval import EMBED_DIM, FakeEmbedder, cosine
+from portfolio_optimiser.semretrieval import (
+    EMBED_DIM,
+    FakeEmbedder,
+    HybridRanker,
+    StructuralRetriever,
+    cosine,
+)
+from portfolio_optimiser.verdicts import (
+    ProposalFeatures,
+    Verdict,
+    VerdictStore,
+    similarity,
+)
 
 _SRC = Path(__file__).resolve().parents[1] / "src"
 
@@ -131,3 +143,127 @@ def test_cosine_of_zero_norm_is_zero() -> None:
     assert cosine(zeros, vec) == 0.0
     assert cosine(vec, zeros) == 0.0
     assert cosine(zeros, zeros) == 0.0
+
+
+# --- SC1: the Retriever seam is additive — the DEFAULT ranking is byte-identical to today ---
+
+_QUERY = ProposalFeatures(
+    affected_codes=frozenset({"05.2", "03.1"}),
+    measure_type="scope_reduction",
+    claimed_saving_nok=220_000,  # bucket [100k, 500k)
+    description="asphalt base course reduction near school",
+)
+
+
+def _store_with_true_match_and_decoys() -> tuple[VerdictStore, str]:
+    """Mirrors ``tests/test_verdicts.py::_store_with_true_match_and_decoys`` — the true match
+    shares the STRUCTURED fields with the query but uses different wording; the decoys share
+    surface text but differ structurally."""
+    true_match = Verdict(
+        id="TRUE",
+        proposal_features=ProposalFeatures(
+            affected_codes=frozenset({"05.2", "03.1"}),
+            measure_type="scope_reduction",
+            claimed_saving_nok=200_000,
+            description="zzz totally unrelated wording alpha beta",
+        ),
+        decision="approved",
+        rationale="prior scope reduction on the same codes was approved",
+    )
+    decoy_low = Verdict(
+        id="DECOY-LOW",
+        proposal_features=ProposalFeatures(
+            affected_codes=frozenset({"09.1"}),
+            measure_type="rate_renegotiation",
+            claimed_saving_nok=50_000,
+            description="asphalt base course reduction near school",
+        ),
+        decision="rejected",
+        rationale="surface-text decoy",
+    )
+    decoy_high = Verdict(
+        id="DECOY-HIGH",
+        proposal_features=ProposalFeatures(
+            affected_codes=frozenset({"21.2"}),
+            measure_type="material_substitution",
+            claimed_saving_nok=700_000,
+            description="asphalt base course reduction extra words",
+        ),
+        decision="rejected",
+        rationale="surface-text decoy",
+    )
+    return VerdictStore(verdicts=[decoy_low, true_match, decoy_high]), "TRUE"
+
+
+def test_default_retriever_is_none() -> None:
+    """Opt-in seam: a store built the ordinary way carries no retriever, so nothing about
+    default retrieval changes unless a caller deliberately installs one."""
+    store, _ = _store_with_true_match_and_decoys()
+    assert store.retriever is None
+
+
+def test_default_retrieve_matches_the_pre_seam_structural_sort() -> None:
+    """SC1 — with no retriever installed, ``retrieve`` must reproduce the exact ordering the
+    pre-seam inline sort produced: key ``(-similarity, id)``, sliced to k."""
+    store, _ = _store_with_true_match_and_decoys()
+    expected = [
+        v.id
+        for v in sorted(
+            store.verdicts,
+            key=lambda v: (-similarity(_QUERY, v.proposal_features), v.id),
+        )[:3]
+    ]
+    assert [h.id for h in store.retrieve(_QUERY, k=3)] == expected
+
+
+def test_structural_retriever_is_the_wired_default_not_dead_code() -> None:
+    """SC1 — the default path goes through ``StructuralRetriever``, so it is exercised code,
+    not an unused parallel implementation that can silently rot."""
+    store, _ = _store_with_true_match_and_decoys()
+    direct = StructuralRetriever(similarity).rank(_QUERY, store.verdicts, 3)
+    assert [h.id for h in store.retrieve(_QUERY, k=3)] == [v.id for v in direct]
+
+
+def test_retrieve_still_rejects_non_positive_k() -> None:
+    """The pre-seam ``k <= 0`` guard must survive the delegation rewrite
+    (asserted independently by tests/test_verdicts.py::test_retrieve_rejects_non_positive_k)."""
+    store, _ = _store_with_true_match_and_decoys()
+    with pytest.raises(ValueError):
+        store.retrieve(_QUERY, k=0)
+
+
+def test_installed_retriever_actually_routes_retrieval() -> None:
+    """Delegation proof: a sentinel retriever must receive the call and its result must be
+    what ``retrieve`` returns — otherwise the seam is decorative."""
+    store, _ = _store_with_true_match_and_decoys()
+    calls: list[tuple[ProposalFeatures, int, int]] = []
+
+    class _Sentinel:
+        def rank(self, query: ProposalFeatures, candidates: list[Verdict], k: int) -> list[Verdict]:
+            calls.append((query, len(candidates), k))
+            return list(reversed(candidates))[:k]
+
+    store.retriever = _Sentinel()
+    hits = store.retrieve(_QUERY, k=2)
+    assert calls == [(_QUERY, 3, 2)]
+    assert [h.id for h in hits] == ["DECOY-HIGH", "TRUE"]
+
+
+def test_hybrid_ranker_is_injectable_and_routes_retrieval() -> None:
+    """A HybridRanker installed on the store must be the ranker ``retrieve`` delegates to."""
+    store, _ = _store_with_true_match_and_decoys()
+    ranker = HybridRanker(FakeEmbedder(), similarity)
+    expected = [v.id for v in ranker.rank(_QUERY, store.verdicts, 3)]
+    store.retriever = ranker
+    assert [h.id for h in store.retrieve(_QUERY, k=3)] == expected
+
+
+def test_hybrid_ranker_at_weight_zero_equals_the_structural_ranking() -> None:
+    """Weight 0 removes the semantic term entirely — the blend must then collapse onto the
+    structural ordering. This is the control the Step-5 detach test relies on."""
+    store, _ = _store_with_true_match_and_decoys()
+    hybrid = HybridRanker(FakeEmbedder(), similarity, weight=0.0)
+    structural = StructuralRetriever(similarity)
+    assert [v.id for v in hybrid.rank(_QUERY, store.verdicts, 3)] == [
+        v.id for v in structural.rank(_QUERY, store.verdicts, 3)
+    ]