fix(s31): close 1 review BLOCKER — per-call retriever + run_portfolio forwards semantic_retrieval
This commit is contained in:
parent
969d450b31
commit
fc69285f2c
3 changed files with 231 additions and 57 deletions
|
|
@ -431,48 +431,86 @@ _VERDICT_INPUT = {"decision": "approved", "rationale": "expert reviewed (sim)"}
|
|||
# "0.37" appears nowhere in the bundle (grep-verified), so its presence in a prompt can only have
|
||||
# come through the ExpeL fold — it cannot be leaked by bundle context.
|
||||
_TIE_MARKER = "realiseringsgrad=0.37"
|
||||
_MARKER_ID = "zz-s31-run-marker"
|
||||
_DISTRACTOR_ID = "aa-s31-run-distractor"
|
||||
|
||||
# The pair ties on the STRUCTURAL score while carrying different code sets: each keeps the query's
|
||||
# own code and adds one foreign code, so Jaccard is equal (0.700 each) but the canonical embedding
|
||||
# string differs and cosine has something to separate. The previous fixture tied by being
|
||||
# structurally IDENTICAL and differing only in prose — a shape the framework cannot mint, since
|
||||
# ``_mint_id`` ignores description and both candidates would collapse onto one id.
|
||||
_MARKER_EXTRA_CODE = "01.1"
|
||||
_DISTRACTOR_EXTRA_CODE = "01.4"
|
||||
|
||||
|
||||
def _tied_pair_store():
|
||||
"""Two verdicts that are structurally IDENTICAL to the bundle's candidate (similarity 1.0
|
||||
each), so the structural ranking can only separate them on ``id`` — and ``_MARKER_ID`` loses
|
||||
that tie. Only the cosine term can promote it.
|
||||
def _tied_pair_verdicts():
|
||||
"""The tied pair, minted through ``capture_verdict`` — the real minting path.
|
||||
|
||||
The bundle's own seed verdict is deliberately NOT included: its features are byte-identical to
|
||||
the query, making it a perfect cosine match that would win every hybrid ranking and so could
|
||||
never demonstrate a tie-break."""
|
||||
from portfolio_optimiser.verdicts import Verdict, VerdictStore, bundle_candidate_features
|
||||
The bundle's own seed verdict is deliberately NOT included: its features are identical to the
|
||||
query, making it a perfect cosine match that would win every hybrid ranking and so could never
|
||||
demonstrate a tie-break."""
|
||||
from portfolio_optimiser.verdicts import (
|
||||
VerdictStore,
|
||||
bundle_candidate_features,
|
||||
capture_verdict,
|
||||
)
|
||||
|
||||
query = bundle_candidate_features(str(BUNDLE_DIR))
|
||||
|
||||
def tied(verdict_id: str, description: str, rationale: str) -> Verdict:
|
||||
return Verdict(
|
||||
id=verdict_id,
|
||||
proposal_features=ProposalFeatures(
|
||||
affected_codes=query.affected_codes,
|
||||
def tied(extra_code: str, decision: str, rationale: str):
|
||||
return capture_verdict(
|
||||
ProposalFeatures(
|
||||
affected_codes=query.affected_codes | {extra_code},
|
||||
measure_type=query.measure_type,
|
||||
# ``description == measure`` is what both live minting paths emit
|
||||
# (``run._features_of`` / ``verdicts._features_from_ir``).
|
||||
claimed_saving_nok=query.claimed_saving_nok,
|
||||
description=description,
|
||||
description=query.measure_type,
|
||||
),
|
||||
decision="approved",
|
||||
rationale=rationale,
|
||||
decision,
|
||||
rationale,
|
||||
)
|
||||
|
||||
return VerdictStore(
|
||||
verdicts=[
|
||||
tied(
|
||||
_DISTRACTOR_ID,
|
||||
"avvist forslag om reforhandling av renholdskontrakt i administrasjonsbygget",
|
||||
"ingen realiseringsdata for dette tiltaket",
|
||||
),
|
||||
tied(
|
||||
_MARKER_ID,
|
||||
"LED-retrofit i kontorlokaler: 90 W armaturer erstattet med 40 W",
|
||||
f"tidligere LED-dom [{_TIE_MARKER}]",
|
||||
),
|
||||
]
|
||||
distractor = tied(
|
||||
_DISTRACTOR_EXTRA_CODE,
|
||||
"rejected",
|
||||
"ingen realiseringsdata for dette tiltaket",
|
||||
)
|
||||
marker = tied(_MARKER_EXTRA_CODE, "approved", f"tidligere LED-dom [{_TIE_MARKER}]")
|
||||
return VerdictStore(verdicts=[distractor, marker]), marker.id, distractor.id
|
||||
|
||||
|
||||
# Minted once at module scope so the tests can name the ids. ``_MARKER_ID`` sorts AFTER
|
||||
# ``_DISTRACTOR_ID``, so the structural ``(-similarity, id)`` key puts the DISTRACTOR first and
|
||||
# only the cosine term can overturn it.
|
||||
_MARKER_ID = _tied_pair_verdicts()[1]
|
||||
_DISTRACTOR_ID = _tied_pair_verdicts()[2]
|
||||
|
||||
|
||||
def _tied_pair_store():
|
||||
return _tied_pair_verdicts()[0]
|
||||
|
||||
|
||||
def _bundle_reference_project(tmp_path: Path):
|
||||
"""A single bundle-backed project for the portfolio arm, pointed at the SAME bundle the tied
|
||||
pair was derived from — a different bundle would yield a different query and untie the pair.
|
||||
Topology mirrors ``tests/test_portfolio_learning_loadbearing.py::_bundle_kplus1``."""
|
||||
from portfolio_optimiser.reference_domain import Project
|
||||
|
||||
docs = tmp_path / "portfolio-docs"
|
||||
docs.mkdir()
|
||||
(docs / "cost.txt").write_text(
|
||||
"LED-retrofit av lysrorarmaturer i kontorlokaler reduserte energikostnaden.",
|
||||
encoding="utf-8",
|
||||
)
|
||||
return Project(
|
||||
id=_PID,
|
||||
name="Bundle-backed portfolio project",
|
||||
description="bundle-backed project for the semantic-retrieval forwarding proof",
|
||||
currency="NOK",
|
||||
cost_items=(),
|
||||
docs_dir=str(docs),
|
||||
verdict_input=_VERDICT_INPUT,
|
||||
bundle_dir=str(BUNDLE_DIR),
|
||||
verdict_dir=None,
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -575,3 +613,112 @@ async def test_semantic_retrieval_off_leaves_the_structural_pick_in_the_prompt(
|
|||
assert any(_DISTRACTOR_ID in p for p in gen_prompts), (
|
||||
"the structural winner did not reach the prompt — the default fold is broken"
|
||||
)
|
||||
|
||||
|
||||
async def test_semantic_retrieval_does_not_leak_into_a_reused_store(
|
||||
make_recording_client_factory,
|
||||
) -> None:
|
||||
"""LEAK CONTROL — the opt-in must not outlive the run that asked for it.
|
||||
|
||||
The store is caller-owned. Installing the ranker on it (``store.retriever = ...``) meant a
|
||||
flag-ON run silently governed every LATER retrieval on that same object, so a subsequent run
|
||||
with the flag OFF still ranked semantically. Here the SAME store instance is driven twice:
|
||||
once with the flag on, once without. The second run must give the STRUCTURAL pick, and the
|
||||
store must be left exactly as the caller handed it over.
|
||||
|
||||
Detach point: assign ``store.retriever`` in ``run_project`` instead of passing the ranker per
|
||||
call → RED."""
|
||||
store = _tied_pair_store()
|
||||
assert store.retriever is None # precondition: the caller handed over a clean store
|
||||
|
||||
factory_on, _ = make_recording_client_factory(_ENERGY_REPLY)
|
||||
await run_project(
|
||||
_PID,
|
||||
"local",
|
||||
docs_dir=str(BUNDLE_DIR),
|
||||
bundle_dir=str(BUNDLE_DIR),
|
||||
verdict_input=_VERDICT_INPUT,
|
||||
store=store,
|
||||
client_factory=factory_on,
|
||||
top_k=1,
|
||||
semantic_retrieval=True,
|
||||
)
|
||||
|
||||
factory_off, recorded_off = make_recording_client_factory(_ENERGY_REPLY)
|
||||
await run_project(
|
||||
_PID,
|
||||
"local",
|
||||
docs_dir=str(BUNDLE_DIR),
|
||||
bundle_dir=str(BUNDLE_DIR),
|
||||
verdict_input=_VERDICT_INPUT,
|
||||
store=store,
|
||||
client_factory=factory_off,
|
||||
top_k=1,
|
||||
)
|
||||
|
||||
assert store.retriever is None, (
|
||||
"run_project mutated the caller's store — the opt-in leaked out of the run that asked "
|
||||
"for it"
|
||||
)
|
||||
assert all(_TIE_MARKER not in p for p in _generation_prompts(recorded_off)), (
|
||||
"the flag-OFF run still ranked semantically — the previous run's opt-in leaked through "
|
||||
"the shared store"
|
||||
)
|
||||
|
||||
|
||||
async def test_run_portfolio_forwards_semantic_retrieval_to_each_project(
|
||||
make_recording_client_factory, monkeypatch, tmp_path
|
||||
) -> None:
|
||||
"""WIRING — ``run_portfolio(semantic_retrieval=True)`` must reach each project's Step-1 fold.
|
||||
|
||||
No test in the repo drove the portfolio path with this flag before: the pre-loop
|
||||
``store.retriever`` install was covered only by the single-project tests, so deleting the
|
||||
forwarding would have gone unnoticed.
|
||||
|
||||
Detach point: drop ``semantic_retrieval=semantic_retrieval`` from the ``run_project`` call
|
||||
inside ``run_portfolio`` → RED."""
|
||||
factory, recorded = make_recording_client_factory(_ENERGY_REPLY)
|
||||
store = _tied_pair_store()
|
||||
|
||||
project = _bundle_reference_project(tmp_path)
|
||||
monkeypatch.setattr("portfolio_optimiser.run.load_reference_projects", lambda: [project])
|
||||
|
||||
await run.run_portfolio(
|
||||
[project.id],
|
||||
profile="local",
|
||||
store=store,
|
||||
client_factory=factory,
|
||||
top_k=1,
|
||||
semantic_retrieval=True,
|
||||
)
|
||||
|
||||
gen_prompts = _generation_prompts(recorded)
|
||||
assert gen_prompts, "the generation call must have happened"
|
||||
assert any(_TIE_MARKER in p for p in gen_prompts), (
|
||||
"the cosine-surfaced verdict did not reach the hypothesis prompt — run_portfolio is not "
|
||||
"forwarding semantic_retrieval to run_project"
|
||||
)
|
||||
|
||||
|
||||
async def test_run_portfolio_without_the_flag_keeps_the_structural_pick(
|
||||
make_recording_client_factory, monkeypatch, tmp_path
|
||||
) -> None:
|
||||
"""CAUSALITY CONTROL for the portfolio arm — the identical pass with the flag absent must
|
||||
carry the structural winner and no marker."""
|
||||
factory, recorded = make_recording_client_factory(_ENERGY_REPLY)
|
||||
store = _tied_pair_store()
|
||||
|
||||
project = _bundle_reference_project(tmp_path)
|
||||
monkeypatch.setattr("portfolio_optimiser.run.load_reference_projects", lambda: [project])
|
||||
|
||||
await run.run_portfolio(
|
||||
[project.id],
|
||||
profile="local",
|
||||
store=store,
|
||||
client_factory=factory,
|
||||
top_k=1,
|
||||
)
|
||||
|
||||
assert all(_TIE_MARKER not in p for p in recorded), (
|
||||
"the marker reached a prompt without the flag — the portfolio default is not structural"
|
||||
)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue