fix(s31): close 1 review BLOCKER — per-call retriever + run_portfolio forwards semantic_retrieval

This commit is contained in:
Kjell Tore Guttormsen 2026-07-25 12:37:15 +02:00
commit fc69285f2c
3 changed files with 231 additions and 57 deletions

View file

@ -431,48 +431,86 @@ _VERDICT_INPUT = {"decision": "approved", "rationale": "expert reviewed (sim)"}
# "0.37" appears nowhere in the bundle (grep-verified), so its presence in a prompt can only have
# come through the ExpeL fold — it cannot be leaked by bundle context.
_TIE_MARKER = "realiseringsgrad=0.37"
_MARKER_ID = "zz-s31-run-marker"
_DISTRACTOR_ID = "aa-s31-run-distractor"
# The pair ties on the STRUCTURAL score while carrying different code sets: each keeps the query's
# own code and adds one foreign code, so Jaccard is equal (0.700 each) but the canonical embedding
# string differs and cosine has something to separate. The previous fixture tied by being
# structurally IDENTICAL and differing only in prose — a shape the framework cannot mint, since
# ``_mint_id`` ignores description and both candidates would collapse onto one id.
_MARKER_EXTRA_CODE = "01.1"
_DISTRACTOR_EXTRA_CODE = "01.4"
def _tied_pair_store():
"""Two verdicts that are structurally IDENTICAL to the bundle's candidate (similarity 1.0
each), so the structural ranking can only separate them on ``id`` and ``_MARKER_ID`` loses
that tie. Only the cosine term can promote it.
def _tied_pair_verdicts():
"""The tied pair, minted through ``capture_verdict`` — the real minting path.
The bundle's own seed verdict is deliberately NOT included: its features are byte-identical to
the query, making it a perfect cosine match that would win every hybrid ranking and so could
never demonstrate a tie-break."""
from portfolio_optimiser.verdicts import Verdict, VerdictStore, bundle_candidate_features
The bundle's own seed verdict is deliberately NOT included: its features are identical to the
query, making it a perfect cosine match that would win every hybrid ranking and so could never
demonstrate a tie-break."""
from portfolio_optimiser.verdicts import (
VerdictStore,
bundle_candidate_features,
capture_verdict,
)
query = bundle_candidate_features(str(BUNDLE_DIR))
def tied(verdict_id: str, description: str, rationale: str) -> Verdict:
return Verdict(
id=verdict_id,
proposal_features=ProposalFeatures(
affected_codes=query.affected_codes,
def tied(extra_code: str, decision: str, rationale: str):
return capture_verdict(
ProposalFeatures(
affected_codes=query.affected_codes | {extra_code},
measure_type=query.measure_type,
# ``description == measure`` is what both live minting paths emit
# (``run._features_of`` / ``verdicts._features_from_ir``).
claimed_saving_nok=query.claimed_saving_nok,
description=description,
description=query.measure_type,
),
decision="approved",
rationale=rationale,
decision,
rationale,
)
return VerdictStore(
verdicts=[
tied(
_DISTRACTOR_ID,
"avvist forslag om reforhandling av renholdskontrakt i administrasjonsbygget",
"ingen realiseringsdata for dette tiltaket",
),
tied(
_MARKER_ID,
"LED-retrofit i kontorlokaler: 90 W armaturer erstattet med 40 W",
f"tidligere LED-dom [{_TIE_MARKER}]",
),
]
distractor = tied(
_DISTRACTOR_EXTRA_CODE,
"rejected",
"ingen realiseringsdata for dette tiltaket",
)
marker = tied(_MARKER_EXTRA_CODE, "approved", f"tidligere LED-dom [{_TIE_MARKER}]")
return VerdictStore(verdicts=[distractor, marker]), marker.id, distractor.id
# Minted once at module scope so the tests can name the ids. ``_MARKER_ID`` sorts AFTER
# ``_DISTRACTOR_ID``, so the structural ``(-similarity, id)`` key puts the DISTRACTOR first and
# only the cosine term can overturn it.
_MARKER_ID = _tied_pair_verdicts()[1]
_DISTRACTOR_ID = _tied_pair_verdicts()[2]
def _tied_pair_store():
return _tied_pair_verdicts()[0]
def _bundle_reference_project(tmp_path: Path):
"""A single bundle-backed project for the portfolio arm, pointed at the SAME bundle the tied
pair was derived from a different bundle would yield a different query and untie the pair.
Topology mirrors ``tests/test_portfolio_learning_loadbearing.py::_bundle_kplus1``."""
from portfolio_optimiser.reference_domain import Project
docs = tmp_path / "portfolio-docs"
docs.mkdir()
(docs / "cost.txt").write_text(
"LED-retrofit av lysrorarmaturer i kontorlokaler reduserte energikostnaden.",
encoding="utf-8",
)
return Project(
id=_PID,
name="Bundle-backed portfolio project",
description="bundle-backed project for the semantic-retrieval forwarding proof",
currency="NOK",
cost_items=(),
docs_dir=str(docs),
verdict_input=_VERDICT_INPUT,
bundle_dir=str(BUNDLE_DIR),
verdict_dir=None,
)
@ -575,3 +613,112 @@ async def test_semantic_retrieval_off_leaves_the_structural_pick_in_the_prompt(
assert any(_DISTRACTOR_ID in p for p in gen_prompts), (
"the structural winner did not reach the prompt — the default fold is broken"
)
async def test_semantic_retrieval_does_not_leak_into_a_reused_store(
make_recording_client_factory,
) -> None:
"""LEAK CONTROL — the opt-in must not outlive the run that asked for it.
The store is caller-owned. Installing the ranker on it (``store.retriever = ...``) meant a
flag-ON run silently governed every LATER retrieval on that same object, so a subsequent run
with the flag OFF still ranked semantically. Here the SAME store instance is driven twice:
once with the flag on, once without. The second run must give the STRUCTURAL pick, and the
store must be left exactly as the caller handed it over.
Detach point: assign ``store.retriever`` in ``run_project`` instead of passing the ranker per
call RED."""
store = _tied_pair_store()
assert store.retriever is None # precondition: the caller handed over a clean store
factory_on, _ = make_recording_client_factory(_ENERGY_REPLY)
await run_project(
_PID,
"local",
docs_dir=str(BUNDLE_DIR),
bundle_dir=str(BUNDLE_DIR),
verdict_input=_VERDICT_INPUT,
store=store,
client_factory=factory_on,
top_k=1,
semantic_retrieval=True,
)
factory_off, recorded_off = make_recording_client_factory(_ENERGY_REPLY)
await run_project(
_PID,
"local",
docs_dir=str(BUNDLE_DIR),
bundle_dir=str(BUNDLE_DIR),
verdict_input=_VERDICT_INPUT,
store=store,
client_factory=factory_off,
top_k=1,
)
assert store.retriever is None, (
"run_project mutated the caller's store — the opt-in leaked out of the run that asked "
"for it"
)
assert all(_TIE_MARKER not in p for p in _generation_prompts(recorded_off)), (
"the flag-OFF run still ranked semantically — the previous run's opt-in leaked through "
"the shared store"
)
async def test_run_portfolio_forwards_semantic_retrieval_to_each_project(
make_recording_client_factory, monkeypatch, tmp_path
) -> None:
"""WIRING — ``run_portfolio(semantic_retrieval=True)`` must reach each project's Step-1 fold.
No test in the repo drove the portfolio path with this flag before: the pre-loop
``store.retriever`` install was covered only by the single-project tests, so deleting the
forwarding would have gone unnoticed.
Detach point: drop ``semantic_retrieval=semantic_retrieval`` from the ``run_project`` call
inside ``run_portfolio`` RED."""
factory, recorded = make_recording_client_factory(_ENERGY_REPLY)
store = _tied_pair_store()
project = _bundle_reference_project(tmp_path)
monkeypatch.setattr("portfolio_optimiser.run.load_reference_projects", lambda: [project])
await run.run_portfolio(
[project.id],
profile="local",
store=store,
client_factory=factory,
top_k=1,
semantic_retrieval=True,
)
gen_prompts = _generation_prompts(recorded)
assert gen_prompts, "the generation call must have happened"
assert any(_TIE_MARKER in p for p in gen_prompts), (
"the cosine-surfaced verdict did not reach the hypothesis prompt — run_portfolio is not "
"forwarding semantic_retrieval to run_project"
)
async def test_run_portfolio_without_the_flag_keeps_the_structural_pick(
make_recording_client_factory, monkeypatch, tmp_path
) -> None:
"""CAUSALITY CONTROL for the portfolio arm — the identical pass with the flag absent must
carry the structural winner and no marker."""
factory, recorded = make_recording_client_factory(_ENERGY_REPLY)
store = _tied_pair_store()
project = _bundle_reference_project(tmp_path)
monkeypatch.setattr("portfolio_optimiser.run.load_reference_projects", lambda: [project])
await run.run_portfolio(
[project.id],
profile="local",
store=store,
client_factory=factory,
top_k=1,
)
assert all(_TIE_MARKER not in p for p in recorded), (
"the marker reached a prompt without the flag — the portfolio default is not structural"
)