feat(s31): semretrieval thread-pinned Embedder seam + fake embedder + cosine

src/portfolio_optimiser/semretrieval.py

@@ -0,0 +1,126 @@
+"""S3.1 — semantic retrieval: brute-force cosine over embedded proposal features (D-C = numpy).
+
+The verdict store ranks structurally by design (``verdicts.similarity``: Jaccard over affected
+cost codes + measure type + magnitude bucket), and surface ``description`` text is deliberately
+excluded so a true structural match beats text decoys. That ranking cannot see a semantically
+related prior verdict carrying a *different* code set. This module adds the missing signal as a
+**strictly opt-in** seam: an ``Embedder`` producing vectors, and a ``Retriever`` that blends
+cosine with the structural score. Default retrieval is unchanged — the hybrid only ranks when a
+caller explicitly asks for it (``run.py --semantic-retrieval``).
+
+**MAF-free** (D7-portable): stdlib + numpy only. ``ProposalFeatures``/``Verdict`` are imported
+ONLY under ``TYPE_CHECKING`` and every runtime access is duck-typed, and the structural score is
+**injected as a callable** rather than imported — so importing (or running) this module never
+pulls in ``portfolio_optimiser.verdicts`` and therefore never pulls in ``agent_framework``.
+Registered in ``tests/test_okf.py``'s ``_MAF_FREE_MODULES`` (direct-import AST guard
+``test_okf_is_maf_free``) and probed transitively — including *after* a ``rank()`` call, which
+catches a lazy runtime import an import-time guard would miss — by
+``tests/test_semretrieval_loadbearing.py``.
+
+Determinism: BLAS reduction order depends on the thread count, so the thread pins below are set
+before numpy is imported (this module is the only numpy importer in the package). Vectors are
+float64 and C-contiguous; ranking uses the total-order key ``(-round(score, 9), id)``.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import os
+from typing import TYPE_CHECKING, Protocol, runtime_checkable
+
+# Pin BLAS threads BEFORE numpy is imported — OpenBLAS reads these at import time, and a
+# thread-count-dependent reduction order is the one thing that makes float64 dot products drift
+# between environments. Pinned to 1 => bit-exact run-to-run. ``setdefault`` so an operator can
+# still override deliberately.
+os.environ.setdefault("OPENBLAS_NUM_THREADS", "1")
+os.environ.setdefault("OMP_NUM_THREADS", "1")
+os.environ.setdefault("MKL_NUM_THREADS", "1")
+
+import numpy as np  # noqa: E402  — must follow the thread pins above; see module docstring
+
+if TYPE_CHECKING:  # verdicts imports agent_framework — keep it out of the runtime import graph
+    from portfolio_optimiser.verdicts import ProposalFeatures
+
+# Dimension of the fake embedding. Small enough that a 500+ verdict base is a trivial matmul,
+# large enough that distinct features do not collide.
+EMBED_DIM = 64
+
+# Blend weight for HybridRanker: score = weight * cosine + (1 - weight) * structural.
+# 0.5 gives the two falsifiable signals equal say; the caller can override per construction.
+SEMANTIC_WEIGHT_DEFAULT = 0.5
+
+# Mirrors ``verdicts._MAGNITUDE_BUCKETS``. DUPLICATED, not imported, on purpose: importing it
+# would pull ``verdicts`` (and therefore ``agent_framework``) into this module's runtime graph
+# and break the MAF-free guard. Drift is tolerable here — these edges only shape the fake
+# embedder's projection, never the structural score, which stays the single source of truth.
+_MAGNITUDE_BUCKETS = [(0.0, 1e5), (1e5, 5e5), (5e5, 1e6), (1e6, float("inf"))]
+
+
+@runtime_checkable
+class Embedder(Protocol):
+    """Maps structured proposal features to a fixed-length float64 vector.
+
+    The real implementation (an embeddings client) is a config-only extension point and is NOT
+    built here — tests and the offline path use ``FakeEmbedder``."""
+
+    def __call__(self, features: ProposalFeatures) -> np.ndarray: ...
+
+
+def _magnitude_bucket(value: float) -> int:
+    for i, (low, high) in enumerate(_MAGNITUDE_BUCKETS):
+        if low <= value < high:
+            return i
+    return len(_MAGNITUDE_BUCKETS) - 1
+
+
+def _canonical_feature_string(features: ProposalFeatures) -> str:
+    """Order-independent textual projection of the features. ``affected_codes`` is a set, so it
+    is sorted; the raw saving is bucketed so near-identical amounts land on the same point."""
+    return "|".join(
+        (
+            ",".join(sorted(features.affected_codes)),
+            features.measure_type,
+            str(_magnitude_bucket(features.claimed_saving_nok)),
+            features.description,
+        )
+    )
+
+
+def _expand(canonical: str, n_bytes: int) -> bytes:
+    """Deterministic byte expansion via counter-prefixed sha256. Never ``hash()`` — that is
+    salted per process (PYTHONHASHSEED) and would make vectors differ between runs."""
+    out = bytearray()
+    counter = 0
+    while len(out) < n_bytes:
+        out += hashlib.sha256(f"{counter}:{canonical}".encode()).digest()
+        counter += 1
+    return bytes(out[:n_bytes])
+
+
+class FakeEmbedder:
+    """Offline stand-in embedder: a pure, non-negative, L2-normalized sha256 projection.
+
+    Non-negative components keep ``cosine`` in ``[0, 1]``, matching the structural score's range
+    so the two blend without a scale mismatch. It carries no real semantics — it exists so the
+    seam is exercisable offline at zero cost; the load-bearing proof of the seam is that removing
+    the cosine term flips the ranking, not that this projection is meaningful."""
+
+    def __call__(self, features: ProposalFeatures) -> np.ndarray:
+        raw = _expand(_canonical_feature_string(features), EMBED_DIM)
+        vec = np.ascontiguousarray([b / 255.0 for b in raw], dtype="<f8")
+        norm = float(np.linalg.norm(vec))
+        if norm == 0.0:
+            return np.zeros(EMBED_DIM, dtype="<f8")
+        return np.ascontiguousarray(vec / norm, dtype="<f8")
+
+
+def cosine(a: np.ndarray, b: np.ndarray) -> float:
+    """Cosine similarity, with a zero-norm guard returning ``0.0``.
+
+    The guard is load-bearing: a NaN reaching the ranking sort key would corrupt ordering
+    silently rather than failing loudly."""
+    norm_a = float(np.linalg.norm(a))
+    norm_b = float(np.linalg.norm(b))
+    if norm_a == 0.0 or norm_b == 0.0:
+        return 0.0
+    return float(np.dot(a, b) / (norm_a * norm_b))