feat(s31): deterministic .npy+jsonl vector store with fail-fast shape check

src/portfolio_optimiser/semretrieval.py

@@ -25,8 +25,10 @@ float64 and C-contiguous; ranking uses the total-order key ``(-round(score, 9),
 from __future__ import annotations
 
 import hashlib
+import json
 import os
 from collections.abc import Callable, Sequence
+from pathlib import Path
 from typing import TYPE_CHECKING, Protocol, runtime_checkable
 
 # Pin BLAS threads BEFORE numpy is imported — OpenBLAS reads these at import time, and a
@@ -187,3 +189,79 @@ class HybridRanker:
             return self._weight * semantic + (1.0 - self._weight) * structural
 
         return sorted(candidates, key=lambda v: (-round(score(v), 9), v.id))[:k]
+
+
+# --- Vector store: a rebuildable cache, never authoritative -----------------------------------
+# Exactly two files. The `.npy` holds the matrix; the `.jsonl` sidecar holds the row->verdict-id
+# mapping, one object per line in the repo's deterministic on-disk idiom (`outbox._dump`). There
+# is deliberately no third `meta.json`: the numpy version that fixes the `.npy` header is already
+# pinned in `uv.lock`, so a provenance file would only be a second place to drift.
+
+_VECTORS_NPY = "vectors.npy"
+_VECTORS_JSONL = "vectors.jsonl"
+
+
+def save_vector_store(
+    directory: str | Path,
+    verdicts: Sequence[Verdict],
+    embedder: Embedder,
+) -> None:
+    """Embed ``verdicts`` and write the two store artifacts into ``directory``.
+
+    Byte-deterministic: verdicts are sorted by ``id`` before embedding, the matrix is C-contiguous
+    float64, and the sidecar uses sorted keys with LF endings. The same verdicts in any insertion
+    order therefore produce identical bytes. Both files are written atomically (temp + replace) so
+    an interrupted run leaves the previous store intact rather than a half-written one."""
+    path = Path(directory)
+    path.mkdir(parents=True, exist_ok=True)
+    ordered = sorted(verdicts, key=lambda v: v.id)
+
+    if ordered:
+        matrix = np.ascontiguousarray(
+            np.vstack([embedder(v.proposal_features) for v in ordered]), dtype="<f8"
+        )
+    else:
+        matrix = np.empty((0, EMBED_DIM), dtype="<f8")
+
+    npy_tmp = path / f"{_VECTORS_NPY}.tmp"
+    # Written through a file handle, not a path: np.save appends ".npy" to a path that lacks it,
+    # which would turn the temp file into `vectors.npy.tmp.npy` and defeat the atomic replace.
+    with npy_tmp.open("wb") as handle:
+        np.save(handle, matrix)
+    os.replace(npy_tmp, path / _VECTORS_NPY)
+
+    jsonl_tmp = path / f"{_VECTORS_JSONL}.tmp"
+    jsonl_tmp.write_text(
+        "".join(json.dumps({"id": v.id}, sort_keys=True) + "\n" for v in ordered),
+        encoding="utf-8",
+        newline="\n",
+    )
+    os.replace(jsonl_tmp, path / _VECTORS_JSONL)
+
+
+def load_vector_store(directory: str | Path) -> tuple[list[str], np.ndarray] | None:
+    """Load the store as ``(ids, matrix)``, or ``None`` when it has not been built.
+
+    Absence is tolerated because the store is a rebuildable cache — a caller without one simply
+    degrades to structural ranking. A row/line MISMATCH is not tolerated: it would silently map
+    every vector to the wrong verdict id and mis-rank without any error, so it raises
+    ``ValueError`` (the fail-fast posture of ``ledger.SavingsLedger.load``)."""
+    path = Path(directory)
+    npy_path, jsonl_path = path / _VECTORS_NPY, path / _VECTORS_JSONL
+    if not npy_path.is_file() or not jsonl_path.is_file():
+        return None
+
+    with npy_path.open("rb") as handle:
+        matrix = np.load(handle, allow_pickle=False)
+    ids = [
+        json.loads(line)["id"]
+        for line in jsonl_path.read_text(encoding="utf-8").splitlines()
+        if line.strip()
+    ]
+
+    if matrix.shape[0] != len(ids):
+        raise ValueError(
+            f"vector store is inconsistent: {matrix.shape[0]} matrix row(s) but {len(ids)} "
+            f"id line(s) in {str(path)!r} — rebuild the store"
+        )
+    return ids, matrix