feat(s31): deterministic .npy+jsonl vector store with fail-fast shape check

src/portfolio_optimiser/semretrieval.py

@@ -25,8 +25,10 @@ float64 and C-contiguous; ranking uses the total-order key ``(-round(score, 9),
 from __future__ import annotations
 
 import hashlib
+import json
 import os
 from collections.abc import Callable, Sequence
+from pathlib import Path
 from typing import TYPE_CHECKING, Protocol, runtime_checkable
 
 # Pin BLAS threads BEFORE numpy is imported — OpenBLAS reads these at import time, and a
@@ -187,3 +189,79 @@ class HybridRanker:
             return self._weight * semantic + (1.0 - self._weight) * structural
 
         return sorted(candidates, key=lambda v: (-round(score(v), 9), v.id))[:k]
+
+
+# --- Vector store: a rebuildable cache, never authoritative -----------------------------------
+# Exactly two files. The `.npy` holds the matrix; the `.jsonl` sidecar holds the row->verdict-id
+# mapping, one object per line in the repo's deterministic on-disk idiom (`outbox._dump`). There
+# is deliberately no third `meta.json`: the numpy version that fixes the `.npy` header is already
+# pinned in `uv.lock`, so a provenance file would only be a second place to drift.
+
+_VECTORS_NPY = "vectors.npy"
+_VECTORS_JSONL = "vectors.jsonl"
+
+
+def save_vector_store(
+    directory: str | Path,
+    verdicts: Sequence[Verdict],
+    embedder: Embedder,
+) -> None:
+    """Embed ``verdicts`` and write the two store artifacts into ``directory``.
+
+    Byte-deterministic: verdicts are sorted by ``id`` before embedding, the matrix is C-contiguous
+    float64, and the sidecar uses sorted keys with LF endings. The same verdicts in any insertion
+    order therefore produce identical bytes. Both files are written atomically (temp + replace) so
+    an interrupted run leaves the previous store intact rather than a half-written one."""
+    path = Path(directory)
+    path.mkdir(parents=True, exist_ok=True)
+    ordered = sorted(verdicts, key=lambda v: v.id)
+
+    if ordered:
+        matrix = np.ascontiguousarray(
+            np.vstack([embedder(v.proposal_features) for v in ordered]), dtype="<f8"
+        )
+    else:
+        matrix = np.empty((0, EMBED_DIM), dtype="<f8")
+
+    npy_tmp = path / f"{_VECTORS_NPY}.tmp"
+    # Written through a file handle, not a path: np.save appends ".npy" to a path that lacks it,
+    # which would turn the temp file into `vectors.npy.tmp.npy` and defeat the atomic replace.
+    with npy_tmp.open("wb") as handle:
+        np.save(handle, matrix)
+    os.replace(npy_tmp, path / _VECTORS_NPY)
+
+    jsonl_tmp = path / f"{_VECTORS_JSONL}.tmp"
+    jsonl_tmp.write_text(
+        "".join(json.dumps({"id": v.id}, sort_keys=True) + "\n" for v in ordered),
+        encoding="utf-8",
+        newline="\n",
+    )
+    os.replace(jsonl_tmp, path / _VECTORS_JSONL)
+
+
+def load_vector_store(directory: str | Path) -> tuple[list[str], np.ndarray] | None:
+    """Load the store as ``(ids, matrix)``, or ``None`` when it has not been built.
+
+    Absence is tolerated because the store is a rebuildable cache — a caller without one simply
+    degrades to structural ranking. A row/line MISMATCH is not tolerated: it would silently map
+    every vector to the wrong verdict id and mis-rank without any error, so it raises
+    ``ValueError`` (the fail-fast posture of ``ledger.SavingsLedger.load``)."""
+    path = Path(directory)
+    npy_path, jsonl_path = path / _VECTORS_NPY, path / _VECTORS_JSONL
+    if not npy_path.is_file() or not jsonl_path.is_file():
+        return None
+
+    with npy_path.open("rb") as handle:
+        matrix = np.load(handle, allow_pickle=False)
+    ids = [
+        json.loads(line)["id"]
+        for line in jsonl_path.read_text(encoding="utf-8").splitlines()
+        if line.strip()
+    ]
+
+    if matrix.shape[0] != len(ids):
+        raise ValueError(
+            f"vector store is inconsistent: {matrix.shape[0]} matrix row(s) but {len(ids)} "
+            f"id line(s) in {str(path)!r} — rebuild the store"
+        )
+    return ids, matrix

tests/test_semretrieval.py

@@ -13,6 +13,7 @@ Pattern: tests/test_verdicts.py (determinism) + tests/test_retrieval.py (fixture
 
 from __future__ import annotations
 
+import json
 import subprocess
 import sys
 import types
@@ -27,6 +28,8 @@ from portfolio_optimiser.semretrieval import (
     HybridRanker,
     StructuralRetriever,
     cosine,
+    load_vector_store,
+    save_vector_store,
 )
 from portfolio_optimiser.verdicts import (
     ProposalFeatures,
@@ -267,3 +270,82 @@ def test_hybrid_ranker_at_weight_zero_equals_the_structural_ranking() -> None:
     assert [v.id for v in hybrid.rank(_QUERY, store.verdicts, 3)] == [
         v.id for v in structural.rank(_QUERY, store.verdicts, 3)
     ]
+
+
+# --- SC3 / SC7: the vector store is byte-deterministic and fails fast on a row/line mismatch ---
+
+
+def test_vector_store_is_byte_identical_regardless_of_insertion_order(tmp_path: Path) -> None:
+    """SC3 — the store sorts by id before embedding, so the same verdicts written in a different
+    order must produce byte-identical artifacts. Pattern:
+    tests/test_ledger.py::test_save_is_byte_identical_regardless_of_order."""
+    store, _ = _store_with_true_match_and_decoys()
+    forward, reverse = tmp_path / "forward", tmp_path / "reverse"
+
+    save_vector_store(forward, store.verdicts, FakeEmbedder())
+    save_vector_store(reverse, list(reversed(store.verdicts)), FakeEmbedder())
+
+    assert (forward / "vectors.npy").read_bytes() == (reverse / "vectors.npy").read_bytes()
+    assert (forward / "vectors.jsonl").read_bytes() == (reverse / "vectors.jsonl").read_bytes()
+
+
+def test_vector_store_round_trips_ids_sorted(tmp_path: Path) -> None:
+    store, _ = _store_with_true_match_and_decoys()
+    save_vector_store(tmp_path, store.verdicts, FakeEmbedder())
+
+    loaded = load_vector_store(tmp_path)
+    assert loaded is not None
+    ids, matrix = loaded
+    assert ids == sorted(v.id for v in store.verdicts)
+    assert matrix.shape == (len(ids), EMBED_DIM)
+    assert matrix.dtype == np.dtype("<f8")
+
+    embedder = FakeEmbedder()
+    by_id = {v.id: v for v in store.verdicts}
+    for row, verdict_id in enumerate(ids):
+        assert np.array_equal(matrix[row], embedder(by_id[verdict_id].proposal_features))
+
+
+def test_vector_store_jsonl_is_one_sorted_object_per_line(tmp_path: Path) -> None:
+    """The id sidecar follows the repo's deterministic on-disk idiom: sorted keys, LF endings,
+    trailing newline — so a diff of two runs is empty rather than noisy."""
+    store, _ = _store_with_true_match_and_decoys()
+    save_vector_store(tmp_path, store.verdicts, FakeEmbedder())
+
+    raw = (tmp_path / "vectors.jsonl").read_bytes().decode("utf-8")
+    assert raw.endswith("\n")
+    assert "\r" not in raw
+    assert [json.loads(line) for line in raw.splitlines()] == [
+        {"id": v.id} for v in sorted(store.verdicts, key=lambda v: v.id)
+    ]
+
+
+def test_ranking_is_stable_over_twenty_repetitions() -> None:
+    """The NFR asks for >=20 repetitions, not a single re-run: BLAS reduction order is the thing
+    that would make this drift, and it is pinned at module import."""
+    base, _ = _store_with_true_match_and_decoys()
+    ranker = HybridRanker(FakeEmbedder(), similarity)
+    first = [v.id for v in ranker.rank(_QUERY, base.verdicts, 3)]
+    for _ in range(20):
+        assert [v.id for v in ranker.rank(_QUERY, base.verdicts, 3)] == first
+
+
+def test_missing_vector_store_loads_as_none(tmp_path: Path) -> None:
+    """Tolerant on absence — the store is a rebuildable cache, so a caller with no store simply
+    degrades to structural ranking rather than crashing."""
+    assert load_vector_store(tmp_path / "never-written") is None
+    assert load_vector_store(tmp_path) is None  # directory exists but holds no artifacts
+
+
+def test_vector_store_row_line_mismatch_fails_fast(tmp_path: Path) -> None:
+    """SC7 — a truncated sidecar would silently mis-map every vector to the wrong verdict id.
+    That must raise, not rank garbage (mirrors SavingsLedger.load's fail-fast posture)."""
+    store, _ = _store_with_true_match_and_decoys()
+    save_vector_store(tmp_path, store.verdicts, FakeEmbedder())
+
+    sidecar = tmp_path / "vectors.jsonl"
+    kept = sidecar.read_text(encoding="utf-8").splitlines()[:-1]
+    sidecar.write_text("".join(line + "\n" for line in kept), encoding="utf-8")
+
+    with pytest.raises(ValueError, match="row"):
+        load_vector_store(tmp_path)