feat(s31): semretrieval thread-pinned Embedder seam + fake embedder + cosine

src/portfolio_optimiser/semretrieval.py

@@ -0,0 +1,126 @@
+"""S3.1 — semantic retrieval: brute-force cosine over embedded proposal features (D-C = numpy).
+
+The verdict store ranks structurally by design (``verdicts.similarity``: Jaccard over affected
+cost codes + measure type + magnitude bucket), and surface ``description`` text is deliberately
+excluded so a true structural match beats text decoys. That ranking cannot see a semantically
+related prior verdict carrying a *different* code set. This module adds the missing signal as a
+**strictly opt-in** seam: an ``Embedder`` producing vectors, and a ``Retriever`` that blends
+cosine with the structural score. Default retrieval is unchanged — the hybrid only ranks when a
+caller explicitly asks for it (``run.py --semantic-retrieval``).
+
+**MAF-free** (D7-portable): stdlib + numpy only. ``ProposalFeatures``/``Verdict`` are imported
+ONLY under ``TYPE_CHECKING`` and every runtime access is duck-typed, and the structural score is
+**injected as a callable** rather than imported — so importing (or running) this module never
+pulls in ``portfolio_optimiser.verdicts`` and therefore never pulls in ``agent_framework``.
+Registered in ``tests/test_okf.py``'s ``_MAF_FREE_MODULES`` (direct-import AST guard
+``test_okf_is_maf_free``) and probed transitively — including *after* a ``rank()`` call, which
+catches a lazy runtime import an import-time guard would miss — by
+``tests/test_semretrieval_loadbearing.py``.
+
+Determinism: BLAS reduction order depends on the thread count, so the thread pins below are set
+before numpy is imported (this module is the only numpy importer in the package). Vectors are
+float64 and C-contiguous; ranking uses the total-order key ``(-round(score, 9), id)``.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import os
+from typing import TYPE_CHECKING, Protocol, runtime_checkable
+
+# Pin BLAS threads BEFORE numpy is imported — OpenBLAS reads these at import time, and a
+# thread-count-dependent reduction order is the one thing that makes float64 dot products drift
+# between environments. Pinned to 1 => bit-exact run-to-run. ``setdefault`` so an operator can
+# still override deliberately.
+os.environ.setdefault("OPENBLAS_NUM_THREADS", "1")
+os.environ.setdefault("OMP_NUM_THREADS", "1")
+os.environ.setdefault("MKL_NUM_THREADS", "1")
+
+import numpy as np  # noqa: E402  — must follow the thread pins above; see module docstring
+
+if TYPE_CHECKING:  # verdicts imports agent_framework — keep it out of the runtime import graph
+    from portfolio_optimiser.verdicts import ProposalFeatures
+
+# Dimension of the fake embedding. Small enough that a 500+ verdict base is a trivial matmul,
+# large enough that distinct features do not collide.
+EMBED_DIM = 64
+
+# Blend weight for HybridRanker: score = weight * cosine + (1 - weight) * structural.
+# 0.5 gives the two falsifiable signals equal say; the caller can override per construction.
+SEMANTIC_WEIGHT_DEFAULT = 0.5
+
+# Mirrors ``verdicts._MAGNITUDE_BUCKETS``. DUPLICATED, not imported, on purpose: importing it
+# would pull ``verdicts`` (and therefore ``agent_framework``) into this module's runtime graph
+# and break the MAF-free guard. Drift is tolerable here — these edges only shape the fake
+# embedder's projection, never the structural score, which stays the single source of truth.
+_MAGNITUDE_BUCKETS = [(0.0, 1e5), (1e5, 5e5), (5e5, 1e6), (1e6, float("inf"))]
+
+
+@runtime_checkable
+class Embedder(Protocol):
+    """Maps structured proposal features to a fixed-length float64 vector.
+
+    The real implementation (an embeddings client) is a config-only extension point and is NOT
+    built here — tests and the offline path use ``FakeEmbedder``."""
+
+    def __call__(self, features: ProposalFeatures) -> np.ndarray: ...
+
+
+def _magnitude_bucket(value: float) -> int:
+    for i, (low, high) in enumerate(_MAGNITUDE_BUCKETS):
+        if low <= value < high:
+            return i
+    return len(_MAGNITUDE_BUCKETS) - 1
+
+
+def _canonical_feature_string(features: ProposalFeatures) -> str:
+    """Order-independent textual projection of the features. ``affected_codes`` is a set, so it
+    is sorted; the raw saving is bucketed so near-identical amounts land on the same point."""
+    return "|".join(
+        (
+            ",".join(sorted(features.affected_codes)),
+            features.measure_type,
+            str(_magnitude_bucket(features.claimed_saving_nok)),
+            features.description,
+        )
+    )
+
+
+def _expand(canonical: str, n_bytes: int) -> bytes:
+    """Deterministic byte expansion via counter-prefixed sha256. Never ``hash()`` — that is
+    salted per process (PYTHONHASHSEED) and would make vectors differ between runs."""
+    out = bytearray()
+    counter = 0
+    while len(out) < n_bytes:
+        out += hashlib.sha256(f"{counter}:{canonical}".encode()).digest()
+        counter += 1
+    return bytes(out[:n_bytes])
+
+
+class FakeEmbedder:
+    """Offline stand-in embedder: a pure, non-negative, L2-normalized sha256 projection.
+
+    Non-negative components keep ``cosine`` in ``[0, 1]``, matching the structural score's range
+    so the two blend without a scale mismatch. It carries no real semantics — it exists so the
+    seam is exercisable offline at zero cost; the load-bearing proof of the seam is that removing
+    the cosine term flips the ranking, not that this projection is meaningful."""
+
+    def __call__(self, features: ProposalFeatures) -> np.ndarray:
+        raw = _expand(_canonical_feature_string(features), EMBED_DIM)
+        vec = np.ascontiguousarray([b / 255.0 for b in raw], dtype="<f8")
+        norm = float(np.linalg.norm(vec))
+        if norm == 0.0:
+            return np.zeros(EMBED_DIM, dtype="<f8")
+        return np.ascontiguousarray(vec / norm, dtype="<f8")
+
+
+def cosine(a: np.ndarray, b: np.ndarray) -> float:
+    """Cosine similarity, with a zero-norm guard returning ``0.0``.
+
+    The guard is load-bearing: a NaN reaching the ranking sort key would corrupt ordering
+    silently rather than failing loudly."""
+    norm_a = float(np.linalg.norm(a))
+    norm_b = float(np.linalg.norm(b))
+    if norm_a == 0.0 or norm_b == 0.0:
+        return 0.0
+    return float(np.dot(a, b) / (norm_a * norm_b))

tests/test_semretrieval.py

@@ -0,0 +1,133 @@
+"""S3.1 — semantic retrieval unit tests (SC4 here; SC1/SC3/SC7 added by later steps).
+
+The embedder must be a PURE function of the structured features: identical features yield a
+byte-identical float64 vector in this process AND in a fresh process started with a different
+``PYTHONHASHSEED`` — the cross-process leg is what rules out Python's salted builtin ``hash()``
+(the exact failure mode that would make retrieval order drift between runs).
+
+Components are non-negative, so ``cosine`` stays in ``[0, 1]`` and blends with the structural
+score (also ``[0, 1]``) without a scale mismatch.
+
+Pattern: tests/test_verdicts.py (determinism) + tests/test_retrieval.py (fixtures).
+"""
+
+from __future__ import annotations
+
+import subprocess
+import sys
+import types
+from pathlib import Path
+
+import numpy as np
+import pytest
+
+from portfolio_optimiser.semretrieval import EMBED_DIM, FakeEmbedder, cosine
+
+_SRC = Path(__file__).resolve().parents[1] / "src"
+
+
+def _features(
+    codes: frozenset[str] = frozenset({"05.2", "03.1"}),
+    measure_type: str = "scope_reduction",
+    saving: float = 220_000.0,
+    description: str = "asphalt base course reduction near school",
+) -> types.SimpleNamespace:
+    """Duck-typed stand-in for ``ProposalFeatures`` — semretrieval never imports ``verdicts``
+    at runtime, so the unit tests do not need the real (MAF-bound) type either."""
+    return types.SimpleNamespace(
+        affected_codes=codes,
+        measure_type=measure_type,
+        claimed_saving_nok=saving,
+        description=description,
+    )
+
+
+def test_fake_embedder_is_pure() -> None:
+    embedder = FakeEmbedder()
+    assert np.array_equal(embedder(_features()), embedder(_features()))
+
+
+def test_fake_embedder_has_fixed_float64_dimension() -> None:
+    vec = FakeEmbedder()(_features())
+    assert vec.shape == (EMBED_DIM,)
+    assert vec.dtype == np.dtype("<f8")
+
+
+def test_fake_embedder_is_insensitive_to_code_set_ordering() -> None:
+    """``affected_codes`` is a set — the canonical string sorts it, so insertion order
+    cannot leak into the vector."""
+    a = FakeEmbedder()(_features(codes=frozenset({"05.2", "03.1"})))
+    b = FakeEmbedder()(_features(codes=frozenset({"03.1", "05.2"})))
+    assert np.array_equal(a, b)
+
+
+def test_distinct_features_give_distinct_vectors() -> None:
+    embedder = FakeEmbedder()
+    baseline = embedder(_features())
+    for variant in (
+        _features(codes=frozenset({"09.1"})),
+        _features(measure_type="rate_renegotiation"),
+        _features(saving=900_000.0),  # different magnitude bucket
+        _features(description="totally different wording"),
+    ):
+        assert not np.array_equal(baseline, embedder(variant))
+
+
+def test_fake_embedder_components_are_non_negative() -> None:
+    """Non-negative components keep ``cosine`` in [0, 1], matching the structural score's
+    range — no scale mismatch when the two are blended by HybridRanker."""
+    vec = FakeEmbedder()(_features())
+    assert bool((vec >= 0.0).all())
+
+
+def test_fake_embedder_is_bit_stable_across_processes(tmp_path: Path) -> None:
+    """RED for any ``hash()``-based implementation: the child runs with a different
+    ``PYTHONHASHSEED``, so a salted hash would produce different bytes."""
+    embedder = FakeEmbedder()
+    here = tmp_path / "here.npy"
+    np.save(here, embedder(_features()))
+
+    there = tmp_path / "there.npy"
+    script = (
+        "import types, numpy as np\n"
+        "from portfolio_optimiser.semretrieval import FakeEmbedder\n"
+        "f = types.SimpleNamespace(\n"
+        "    affected_codes=frozenset({'05.2', '03.1'}),\n"
+        "    measure_type='scope_reduction',\n"
+        "    claimed_saving_nok=220000.0,\n"
+        "    description='asphalt base course reduction near school',\n"
+        ")\n"
+        f"np.save({str(there)!r}, FakeEmbedder()(f))\n"
+    )
+    result = subprocess.run(
+        [sys.executable, "-c", script],
+        env={"PATH": "/usr/bin:/bin", "PYTHONPATH": str(_SRC), "PYTHONHASHSEED": "12345"},
+        capture_output=True,
+        text=True,
+        check=False,
+    )
+    assert result.returncode == 0, result.stderr
+    assert here.read_bytes() == there.read_bytes()
+
+
+def test_cosine_of_identical_unit_vectors_is_one() -> None:
+    """Approximate by necessity: a float64 dot of a normalized vector with itself lands within
+    an ulp or two of 1.0, never exactly on it. Ranking never depends on the exact value —
+    ``HybridRanker`` rounds to 9 decimals and breaks remaining ties on ``id``."""
+    vec = FakeEmbedder()(_features())
+    assert cosine(vec, vec) == pytest.approx(1.0)
+
+
+def test_cosine_is_bounded_to_unit_interval() -> None:
+    embedder = FakeEmbedder()
+    score = cosine(embedder(_features()), embedder(_features(codes=frozenset({"09.1"}))))
+    assert 0.0 <= score <= 1.0
+
+
+def test_cosine_of_zero_norm_is_zero() -> None:
+    """A zero vector must never produce NaN — NaN in a sort key silently corrupts ranking."""
+    zeros = np.zeros(EMBED_DIM, dtype="<f8")
+    vec = FakeEmbedder()(_features())
+    assert cosine(zeros, vec) == 0.0
+    assert cosine(vec, zeros) == 0.0
+    assert cosine(zeros, zeros) == 0.0