feat(s31): --semantic-retrieval opt-in threaded through run_project/run_portfolio

src/portfolio_optimiser/run.py

@@ -50,6 +50,11 @@ from portfolio_optimiser.provenance import ProvenanceStamp
 from portfolio_optimiser.reference_domain import Project, load_reference_projects
 from portfolio_optimiser.validator import Rejection, ValidatedProposal
 from portfolio_optimiser import okf, outbox
+from portfolio_optimiser.semretrieval import (
+    SEMANTIC_WEIGHT_DEFAULT,
+    FakeEmbedder,
+    HybridRanker,
+)
 from portfolio_optimiser.verdicts import (
     ExpeLContextProvider,
     ProposalFeatures,
@@ -58,6 +63,7 @@ from portfolio_optimiser.verdicts import (
     bundle_candidate_features,
     capture_verdict,
     load_verdicts_from_dir,
+    similarity,
 )
 from portfolio_optimiser.value_report import (
     build_value_report,
@@ -251,6 +257,7 @@ async def run_project(
     notify: Callable[[Verdict], None] | None = None,
     meter: TokenMeter | None = None,
     live_dry_run: bool = False,
+    semantic_retrieval: bool = False,
 ) -> RunResult | DryRunReport:
     """Run the vertical slice for ONE project. ``client_factory`` is the test-injection seam
     (defaults to the real backend). ``verdict_input`` carries the expert decision/rationale
@@ -271,7 +278,12 @@ async def run_project(
     crossed, and ``ValueError`` when ``outbox_dir`` is set without a ``run_id``. ``live_dry_run``
     (S4.2, comparison protocol §4 pkt 2/3) is the offline drill: it walks the whole path up to the
     EAGER client build, writes the run-config artefact (when ``outbox_dir`` is set), and returns a
-    ``DryRunReport`` BEFORE the first model call (``debate.run``) — zero chat calls."""
+    ``DryRunReport`` BEFORE the first model call (``debate.run``) — zero chat calls.
+    ``semantic_retrieval`` (S3.1) is the opt-in scaling seam: when true, the store's ranker is
+    swapped for a ``HybridRanker`` that blends brute-force cosine over embedded features with the
+    structural score, so a semantically related prior verdict carrying a DIFFERENT cost-code set
+    can reach the Step-1 fold. Default false keeps the structural, text-excluded ranking exactly
+    as before."""
     # 0. Fail-fast: an outbox write is byte-deterministic and keyed on run_id — no wall-clock default.
     if outbox_dir is not None and run_id is None:
         raise ValueError(
@@ -376,6 +388,13 @@ async def run_project(
     # post-generation into a discarded SessionContext (step 7 below), so a prior verdict could not
     # reach the next hypothesis. Bundle-driven path with a populated store only; the road path is
     # untouched (its post-hoc, proposal-keyed retrieval below is unchanged).
+    # S3.1 opt-in: install the hybrid ranker on the RESOLVED store, immediately before the fold it
+    # is meant to affect. Anchoring matters — set at the verdict_dir-conditional resolution above
+    # it would miss every run without an inbox, and set after the fold it would rank nothing that
+    # reaches this prompt. Flag off => store.retriever stays None => StructuralRetriever default.
+    if semantic_retrieval and store is not None:
+        store.retriever = HybridRanker(FakeEmbedder(), similarity, SEMANTIC_WEIGHT_DEFAULT)
+
     if bundle_dir is not None and store is not None and store.verdicts:
         expel_query = bundle_candidate_features(bundle_dir)
         fewshot = ExpeLContextProvider(store, expel_query, k=top_k).format_fewshot()
@@ -526,6 +545,7 @@ async def run_portfolio(
     max_tokens: int = 100_000,
     top_k: int = 3,
     meter_factory: Callable[[], TokenMeter] | None = None,
+    semantic_retrieval: bool = False,
 ) -> PortfolioResult:
     """Fan out over a portfolio of independent projects SEQUENTIALLY, composing ``run_project``
     as-is (every project's execution state — meter, debate, retrieval context — is built fresh
@@ -551,6 +571,10 @@ async def run_portfolio(
     projects = {p.id: p for p in load_reference_projects()}
     ids = list(project_ids) if project_ids is not None else list(projects)
     store = store if store is not None else VerdictStore(verdicts=[])
+    # S3.1: one shared store is threaded across every project, so installing the hybrid here —
+    # before the loop — covers every project's Step-1 fold in this pass.
+    if semantic_retrieval:
+        store.retriever = HybridRanker(FakeEmbedder(), similarity, SEMANTIC_WEIGHT_DEFAULT)
     ledger = ledger if ledger is not None else SavingsLedger(entries=[])
     goals = goals if goals is not None else GoalConfig()
     portfolio_baseline_ore = _to_ore(sum(projects[p].total_cost for p in ids if p in projects))
@@ -673,6 +697,14 @@ def main(argv: list[str] | None = None) -> int:
         help="portfolio mode: accumulated savings ledger JSON (fail-fast) read for the goal-stop "
         "(earlier out-of-band HITL realizations — never built during the pass)",
     )
+    parser.add_argument(
+        "--semantic-retrieval",
+        action="store_true",
+        help="S3.1 opt-in scaling seam: rank prior verdicts with a hybrid of brute-force cosine "
+        "over embedded features and the structural score, so a semantically related verdict with a "
+        "DIFFERENT cost-code set can reach the hypothesis prompt. Valid in both modes; OFF by "
+        "default, and off means the structural, text-excluded ranking is unchanged",
+    )
     parser.add_argument("--decision", default="approved", choices=["approved", "rejected"])
     parser.add_argument("--rationale", default="reviewed by expert")
     parser.add_argument(
@@ -721,6 +753,7 @@ def main(argv: list[str] | None = None) -> int:
             "--outbox-dir": args.outbox_dir is not None,
             "--run-id": args.run_id is not None,
             "--dimension-config": args.dimension_config is not None,
+            "--semantic-retrieval": args.semantic_retrieval,
         }
         if any(report_forbidden.values()):
             print(
@@ -792,6 +825,7 @@ def main(argv: list[str] | None = None) -> int:
                     dimension=dimension,
                     ledger=ledger,
                     goals=goals,
+                    semantic_retrieval=args.semantic_retrieval,
                 )
             )
         except (ValueError, FileNotFoundError, ValidationError) as exc:
@@ -878,6 +912,7 @@ def main(argv: list[str] | None = None) -> int:
                     outbox_dir=args.outbox_dir,
                     run_id=args.run_id,
                     verdict_input={"decision": args.decision, "rationale": args.rationale},
+                    semantic_retrieval=args.semantic_retrieval,
                 )
             ),
         )