open/portfolio-optimiser
1
0
Fork 0
Code Issues Releases 1 Activity

fix(s31): close 1 review BLOCKER — per-call retriever + run_portfolio forwards semantic_retrieval

Browse source
This commit is contained in:
Kjell Tore Guttormsen 2026-07-25 12:37:15 +02:00
commit fc69285f2c
3 changed files with 231 additions and 57 deletions
Download patch file Download diff file Expand all files Collapse all files

33
src/portfolio_optimiser/run.py
View file

@ -388,16 +388,23 @@ async def run_project(
# post-generation into a discarded SessionContext (step 7 below), so a prior verdict could not
# reach the next hypothesis. Bundle-driven path with a populated store only; the road path is
# untouched (its post-hoc, proposal-keyed retrieval below is unchanged).
# S3.1 opt-in: install the hybrid ranker on the RESOLVED store, immediately before the fold it
# is meant to affect. Anchoring matters — set at the verdict_dir-conditional resolution above
# it would miss every run without an inbox, and set after the fold it would rank nothing that
# reaches this prompt. Flag off => store.retriever stays None => StructuralRetriever default.
if semantic_retrieval and store is not None:
store.retriever = HybridRanker(FakeEmbedder(), similarity, SEMANTIC_WEIGHT_DEFAULT)
# S3.1 opt-in: build the hybrid ranker as a LOCAL, then pass it explicitly at each retrieval
# this run performs. It is deliberately not assigned to ``store.retriever``: the store is
# caller-owned (``run_portfolio`` threads one store across every project, and a library caller
# may reuse theirs), so a store-global assignment leaked this run's opt-in into every later use
# of that object — including a subsequent run with the flag OFF. Flag off => ranker stays None
# => ``retrieve`` falls through to the StructuralRetriever default.
ranker = (
HybridRanker(FakeEmbedder(), similarity, SEMANTIC_WEIGHT_DEFAULT)
if semantic_retrieval
else None
)
if bundle_dir is not None and store is not None and store.verdicts:
expel_query = bundle_candidate_features(bundle_dir)
fewshot = ExpeLContextProvider(store, expel_query, k=top_k).format_fewshot()
fewshot = ExpeLContextProvider(
store, expel_query, k=top_k, retriever=ranker
).format_fewshot()
gen_context = f"{fewshot}\n\n{gen_context}"
# 5. Structured candidate -> blocking validation on the NUMBERS; token bound = the meter.
@ -457,10 +464,10 @@ async def run_project(
# is NOT what reaches the prompt.
store = store if store is not None else VerdictStore(verdicts=[])
features = _features_of(proposal)
provider = ExpeLContextProvider(store, features, k=top_k)
provider = ExpeLContextProvider(store, features, k=top_k, retriever=ranker)
sctx = SessionContext(input_messages=[], instructions=[])
await provider.before_run(agent=None, session=None, context=sctx, state={})
retrieved = store.retrieve(features, k=top_k) if store.verdicts else []
retrieved = store.retrieve(features, k=top_k, retriever=ranker) if store.verdicts else []
# 8. Layer-2 (out-of-band): capture the durable verdict + persist; B11 notify is a stub.
verdict = capture_verdict(features, verdict_input["decision"], verdict_input["rationale"])
@ -571,10 +578,9 @@ async def run_portfolio(
projects = {p.id: p for p in load_reference_projects()}
ids = list(project_ids) if project_ids is not None else list(projects)
store = store if store is not None else VerdictStore(verdicts=[])
# S3.1: one shared store is threaded across every project, so installing the hybrid here —
# before the loop — covers every project's Step-1 fold in this pass.
if semantic_retrieval:
store.retriever = HybridRanker(FakeEmbedder(), similarity, SEMANTIC_WEIGHT_DEFAULT)
# S3.1: the flag is FORWARDED per project rather than installed on the shared store here. The
# store is threaded across every project in the pass (and may be caller-owned), so a pre-loop
# assignment outlived the pass; forwarding keeps the opt-in scoped to each run's own retrievals.
ledger = ledger if ledger is not None else SavingsLedger(entries=[])
goals = goals if goals is not None else GoalConfig()
portfolio_baseline_ore = _to_ore(sum(projects[p].total_cost for p in ids if p in projects))
@ -625,6 +631,7 @@ async def run_portfolio(
max_rounds=max_rounds,
max_tokens=max_tokens,
top_k=top_k,
semantic_retrieval=semantic_retrieval,
meter=meter_factory() if meter_factory is not None else None,
),
)

42
src/portfolio_optimiser/verdicts.py
View file

@ -264,20 +264,30 @@ class VerdictStore:
that want a fresh store straight from a folder."""
return cls(verdicts=load_verdicts_from_dir(directory))
def retrieve(self, query: ProposalFeatures, k: int) -> list[Verdict]:
def retrieve(
self,
query: ProposalFeatures,
k: int,
*,
retriever: semretrieval.Retriever | None = None,
) -> list[Verdict]:
"""Return the top-``k`` verdicts. Deterministic: ties break by verdict id, so ordering
is stable across runs.
Ranking is delegated to ``self.retriever``, defaulting to ``StructuralRetriever`` — the
same weighted structural score and ``(-similarity, id)`` key as before the seam existed.
A caller that installs a ``HybridRanker`` opts into an additional semantic term."""
Ranking resolves in three steps: the explicit ``retriever`` argument, then
``self.retriever``, then ``StructuralRetriever`` — the same weighted structural score and
``(-similarity, id)`` key as before the seam existed.
``retriever`` is PER CALL and keyword-only. It exists because the alternative — assigning
``store.retriever`` — mutates an object the caller owns, so an opt-in made for one
retrieval silently governed every later use of that store (including a subsequent run with
the flag OFF). ``self.retriever`` survives as the store-level default for callers that
genuinely want a store to rank one way for its whole lifetime; see ``docs/extending.md``."""
if k <= 0:
raise ValueError(f"k must be positive, got {k}")
ranker = (
self.retriever
if self.retriever is not None
else semretrieval.StructuralRetriever(similarity)
)
ranker = retriever if retriever is not None else self.retriever
if ranker is None:
ranker = semretrieval.StructuralRetriever(similarity)
return ranker.rank(query, self.verdicts, k)
def add(self, verdict: Verdict) -> None:
@ -298,14 +308,24 @@ class ExpeLContextProvider(ContextProvider):
"""Wraps ``VerdictStore.retrieve`` for ExpeL few-shot injection via the GA
``ContextProvider`` hook."""
def __init__(self, store: VerdictStore, query: ProposalFeatures, *, k: int = 3) -> None:
def __init__(
self,
store: VerdictStore,
query: ProposalFeatures,
*,
k: int = 3,
retriever: semretrieval.Retriever | None = None,
) -> None:
super().__init__(source_id="expel-verdictstore")
self._store = store
self._query = query
self._k = k
# Per-call ranker, threaded straight through to ``retrieve`` — see its docstring for why
# this is a parameter rather than an assignment on the store.
self._retriever = retriever
def format_fewshot(self) -> str:
hits = self._store.retrieve(self._query, self._k)
hits = self._store.retrieve(self._query, self._k, retriever=self._retriever)
body = "\n".join(f"- [{v.id}] {v.decision}: {v.rationale}" for v in hits)
return f"Relevant prior verdicts (ExpeL few-shot):\n{body}"

209
tests/test_run_cli.py
View file

@ -431,48 +431,86 @@ _VERDICT_INPUT = {"decision": "approved", "rationale": "expert reviewed (sim)"}
# "0.37" appears nowhere in the bundle (grep-verified), so its presence in a prompt can only have
# come through the ExpeL fold — it cannot be leaked by bundle context.
_TIE_MARKER = "realiseringsgrad=0.37"
_MARKER_ID = "zz-s31-run-marker"
_DISTRACTOR_ID = "aa-s31-run-distractor"
# The pair ties on the STRUCTURAL score while carrying different code sets: each keeps the query's
# own code and adds one foreign code, so Jaccard is equal (0.700 each) but the canonical embedding
# string differs and cosine has something to separate. The previous fixture tied by being
# structurally IDENTICAL and differing only in prose — a shape the framework cannot mint, since
# ``_mint_id`` ignores description and both candidates would collapse onto one id.
_MARKER_EXTRA_CODE = "01.1"
_DISTRACTOR_EXTRA_CODE = "01.4"
def _tied_pair_store():
"""Two verdicts that are structurally IDENTICAL to the bundle's candidate (similarity 1.0
each), so the structural ranking can only separate them on ``id`` — and ``_MARKER_ID`` loses
that tie. Only the cosine term can promote it.
def _tied_pair_verdicts():
"""The tied pair, minted through ``capture_verdict`` — the real minting path.
The bundle's own seed verdict is deliberately NOT included: its features are byte-identical to
the query, making it a perfect cosine match that would win every hybrid ranking and so could
never demonstrate a tie-break."""
from portfolio_optimiser.verdicts import Verdict, VerdictStore, bundle_candidate_features
The bundle's own seed verdict is deliberately NOT included: its features are identical to the
query, making it a perfect cosine match that would win every hybrid ranking and so could never
demonstrate a tie-break."""
from portfolio_optimiser.verdicts import (
VerdictStore,
bundle_candidate_features,
capture_verdict,
)
query = bundle_candidate_features(str(BUNDLE_DIR))
def tied(verdict_id: str, description: str, rationale: str) -> Verdict:
return Verdict(
id=verdict_id,
proposal_features=ProposalFeatures(
affected_codes=query.affected_codes,
def tied(extra_code: str, decision: str, rationale: str):
return capture_verdict(
ProposalFeatures(
affected_codes=query.affected_codes | {extra_code},
measure_type=query.measure_type,
# ``description == measure`` is what both live minting paths emit
# (``run._features_of`` / ``verdicts._features_from_ir``).
claimed_saving_nok=query.claimed_saving_nok,
description=description,
description=query.measure_type,
),
decision="approved",
rationale=rationale,
decision,
rationale,
)
return VerdictStore(
verdicts=[
tied(
_DISTRACTOR_ID,
"avvist forslag om reforhandling av renholdskontrakt i administrasjonsbygget",
"ingen realiseringsdata for dette tiltaket",
),
tied(
_MARKER_ID,
"LED-retrofit i kontorlokaler: 90 W armaturer erstattet med 40 W",
f"tidligere LED-dom [{_TIE_MARKER}]",
),
]
distractor = tied(
_DISTRACTOR_EXTRA_CODE,
"rejected",
"ingen realiseringsdata for dette tiltaket",
)
marker = tied(_MARKER_EXTRA_CODE, "approved", f"tidligere LED-dom [{_TIE_MARKER}]")
return VerdictStore(verdicts=[distractor, marker]), marker.id, distractor.id
# Minted once at module scope so the tests can name the ids. ``_MARKER_ID`` sorts AFTER
# ``_DISTRACTOR_ID``, so the structural ``(-similarity, id)`` key puts the DISTRACTOR first and
# only the cosine term can overturn it.
_MARKER_ID = _tied_pair_verdicts()[1]
_DISTRACTOR_ID = _tied_pair_verdicts()[2]
def _tied_pair_store():
return _tied_pair_verdicts()[0]
def _bundle_reference_project(tmp_path: Path):
"""A single bundle-backed project for the portfolio arm, pointed at the SAME bundle the tied
pair was derived from — a different bundle would yield a different query and untie the pair.
Topology mirrors ``tests/test_portfolio_learning_loadbearing.py::_bundle_kplus1``."""
from portfolio_optimiser.reference_domain import Project
docs = tmp_path / "portfolio-docs"
docs.mkdir()
(docs / "cost.txt").write_text(
"LED-retrofit av lysrorarmaturer i kontorlokaler reduserte energikostnaden.",
encoding="utf-8",
)
return Project(
id=_PID,
name="Bundle-backed portfolio project",
description="bundle-backed project for the semantic-retrieval forwarding proof",
currency="NOK",
cost_items=(),
docs_dir=str(docs),
verdict_input=_VERDICT_INPUT,
bundle_dir=str(BUNDLE_DIR),
verdict_dir=None,
)
@ -575,3 +613,112 @@ async def test_semantic_retrieval_off_leaves_the_structural_pick_in_the_prompt(
assert any(_DISTRACTOR_ID in p for p in gen_prompts), (
"the structural winner did not reach the prompt — the default fold is broken"
)
async def test_semantic_retrieval_does_not_leak_into_a_reused_store(
make_recording_client_factory,
) -> None:
"""LEAK CONTROL — the opt-in must not outlive the run that asked for it.
The store is caller-owned. Installing the ranker on it (``store.retriever = ...``) meant a
flag-ON run silently governed every LATER retrieval on that same object, so a subsequent run
with the flag OFF still ranked semantically. Here the SAME store instance is driven twice:
once with the flag on, once without. The second run must give the STRUCTURAL pick, and the
store must be left exactly as the caller handed it over.
Detach point: assign ``store.retriever`` in ``run_project`` instead of passing the ranker per
call → RED."""
store = _tied_pair_store()
assert store.retriever is None # precondition: the caller handed over a clean store
factory_on, _ = make_recording_client_factory(_ENERGY_REPLY)
await run_project(
_PID,
"local",
docs_dir=str(BUNDLE_DIR),
bundle_dir=str(BUNDLE_DIR),
verdict_input=_VERDICT_INPUT,
store=store,
client_factory=factory_on,
top_k=1,
semantic_retrieval=True,
)
factory_off, recorded_off = make_recording_client_factory(_ENERGY_REPLY)
await run_project(
_PID,
"local",
docs_dir=str(BUNDLE_DIR),
bundle_dir=str(BUNDLE_DIR),
verdict_input=_VERDICT_INPUT,
store=store,
client_factory=factory_off,
top_k=1,
)
assert store.retriever is None, (
"run_project mutated the caller's store — the opt-in leaked out of the run that asked "
"for it"
)
assert all(_TIE_MARKER not in p for p in _generation_prompts(recorded_off)), (
"the flag-OFF run still ranked semantically — the previous run's opt-in leaked through "
"the shared store"
)
async def test_run_portfolio_forwards_semantic_retrieval_to_each_project(
make_recording_client_factory, monkeypatch, tmp_path
) -> None:
"""WIRING — ``run_portfolio(semantic_retrieval=True)`` must reach each project's Step-1 fold.
No test in the repo drove the portfolio path with this flag before: the pre-loop
``store.retriever`` install was covered only by the single-project tests, so deleting the
forwarding would have gone unnoticed.
Detach point: drop ``semantic_retrieval=semantic_retrieval`` from the ``run_project`` call
inside ``run_portfolio`` → RED."""
factory, recorded = make_recording_client_factory(_ENERGY_REPLY)
store = _tied_pair_store()
project = _bundle_reference_project(tmp_path)
monkeypatch.setattr("portfolio_optimiser.run.load_reference_projects", lambda: [project])
await run.run_portfolio(
[project.id],
profile="local",
store=store,
client_factory=factory,
top_k=1,
semantic_retrieval=True,
)
gen_prompts = _generation_prompts(recorded)
assert gen_prompts, "the generation call must have happened"
assert any(_TIE_MARKER in p for p in gen_prompts), (
"the cosine-surfaced verdict did not reach the hypothesis prompt — run_portfolio is not "
"forwarding semantic_retrieval to run_project"
)
async def test_run_portfolio_without_the_flag_keeps_the_structural_pick(
make_recording_client_factory, monkeypatch, tmp_path
) -> None:
"""CAUSALITY CONTROL for the portfolio arm — the identical pass with the flag absent must
carry the structural winner and no marker."""
factory, recorded = make_recording_client_factory(_ENERGY_REPLY)
store = _tied_pair_store()
project = _bundle_reference_project(tmp_path)
monkeypatch.setattr("portfolio_optimiser.run.load_reference_projects", lambda: [project])
await run.run_portfolio(
[project.id],
profile="local",
store=store,
client_factory=factory,
top_k=1,
)
assert all(_TIE_MARKER not in p for p in recorded), (
"the marker reached a prompt without the flag — the portfolio default is not structural"
)